commit: ceb2e09126a48fa341d5ff84371db49ae022bc12
parent: bce16f455743c3a6603fb5d2d81c4e8200b38494
Author: feld <feld@feld.me>
Date: Mon, 9 Sep 2019 18:53:09 +0000
Merge branch 'cache-plug' into 'develop'
Add Pleroma.Plugs.Cache
Closes #1174
See merge request pleroma/pleroma!1612
Diffstat:
11 files changed, 457 insertions(+), 7 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -109,6 +109,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Mix Tasks: `mix pleroma.database fix_likes_collections`
- Federation: Remove `likes` from objects.
- Admin API: Added moderation log
+- Web response cache (currently, enabled for ActivityPub)
### Changed
- Configuration: Filter.AnonymizeFilename added ability to retain file extension with custom text
diff --git a/config/config.exs b/config/config.exs
@@ -560,6 +560,10 @@ config :pleroma, :rate_limit, nil
config :pleroma, Pleroma.ActivityExpiration, enabled: true
+config :pleroma, :web_cache_ttl,
+ activity_pub: nil,
+ activity_pub_question: 30_000
+
# Import environment specific config. This must remain at the bottom
# of this file so it overrides the configuration defined above.
import_config "#{Mix.env()}.exs"
diff --git a/docs/config.md b/docs/config.md
@@ -690,3 +690,12 @@ Supported rate limiters:
* `:relation_id_action` for actions on relation with a specific user (follow, unfollow)
* `:statuses_actions` for create / delete / fav / unfav / reblog / unreblog actions on any statuses
* `:status_id_action` for fav / unfav or reblog / unreblog actions on the same status by the same user
+
+## :web_cache_ttl
+
+The expiration time for the web responses cache. Values should be in milliseconds or `nil` to disable expiration.
+
+Available caches:
+
+* `:activity_pub` - activity pub routes (except question activities). Defaults to `nil` (no expiration).
+* `:activity_pub_question` - activity pub routes (question activities). Defaults to `30_000` (30 seconds).
diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
@@ -308,10 +308,19 @@ defmodule Pleroma.Activity do
%{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id
_ -> nil
end)
+ |> purge_web_resp_cache()
end
def delete_by_ap_id(_), do: nil
+ defp purge_web_resp_cache(%Activity{} = activity) do
+ %{path: path} = URI.parse(activity.data["id"])
+ Cachex.del(:web_resp_cache, path)
+ activity
+ end
+
+ defp purge_web_resp_cache(nil), do: nil
+
for {ap_type, type} <- @mastodon_notification_types do
def mastodon_notification_type(%Activity{data: %{"type" => unquote(ap_type)}}),
do: unquote(type)
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
@@ -116,7 +116,8 @@ defmodule Pleroma.Application do
build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000),
build_cachex("scrubber", limit: 2500),
- build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500)
+ build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
+ build_cachex("web_resp", limit: 2500)
]
end
diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
@@ -130,14 +130,16 @@ defmodule Pleroma.Object do
def delete(%Object{data: %{"id" => id}} = object) do
with {:ok, _obj} = swap_object_with_tombstone(object),
deleted_activity = Activity.delete_by_ap_id(id),
- {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
+ {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
+ {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
{:ok, object, deleted_activity}
end
end
def prune(%Object{data: %{"id" => id}} = object) do
with {:ok, object} <- Repo.delete(object),
- {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
+ {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
+ {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
{:ok, object}
end
end
diff --git a/lib/pleroma/plugs/cache.ex b/lib/pleroma/plugs/cache.ex
@@ -0,0 +1,122 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Plugs.Cache do
+ @moduledoc """
+ Caches successful GET responses.
+
+ To enable the cache add the plug to a router pipeline or controller:
+
+ plug(Pleroma.Plugs.Cache)
+
+ ## Configuration
+
+ To configure the plug you need to pass settings as the second argument to the `plug/2` macro:
+
+ plug(Pleroma.Plugs.Cache, [ttl: nil, query_params: true])
+
+ Available options:
+
+ - `ttl`: An expiration time (time-to-live). This value should be in milliseconds or `nil` to disable expiration. Defaults to `nil`.
+ - `query_params`: Take URL query string into account (`true`), ignore it (`false`) or limit to specific params only (list). Defaults to `true`.
+
+ Additionally, you can overwrite the TTL inside a controller action by assigning `cache_ttl` to the connection struct:
+
+ def index(conn, _params) do
+ ttl = 60_000 # one minute
+
+ conn
+ |> assign(:cache_ttl, ttl)
+ |> render("index.html")
+ end
+
+ """
+
+ import Phoenix.Controller, only: [current_path: 1, json: 2]
+ import Plug.Conn
+
+ @behaviour Plug
+
+ @defaults %{ttl: nil, query_params: true}
+
+ @impl true
+ def init([]), do: @defaults
+
+ def init(opts) do
+ opts = Map.new(opts)
+ Map.merge(@defaults, opts)
+ end
+
+ @impl true
+ def call(%{method: "GET"} = conn, opts) do
+ key = cache_key(conn, opts)
+
+ case Cachex.get(:web_resp_cache, key) do
+ {:ok, nil} ->
+ cache_resp(conn, opts)
+
+ {:ok, record} ->
+ send_cached(conn, record)
+
+ {atom, message} when atom in [:ignore, :error] ->
+ render_error(conn, message)
+ end
+ end
+
+ def call(conn, _), do: conn
+
+ # full path including query params
+ defp cache_key(conn, %{query_params: true}), do: current_path(conn)
+
+ # request path without query params
+ defp cache_key(conn, %{query_params: false}), do: conn.request_path
+
+ # request path with specific query params
+ defp cache_key(conn, %{query_params: query_params}) when is_list(query_params) do
+ query_string =
+ conn.params
+ |> Map.take(query_params)
+ |> URI.encode_query()
+
+ conn.request_path <> "?" <> query_string
+ end
+
+ defp cache_resp(conn, opts) do
+ register_before_send(conn, fn
+ %{status: 200, resp_body: body} = conn ->
+ ttl = Map.get(conn.assigns, :cache_ttl, opts.ttl)
+ key = cache_key(conn, opts)
+ content_type = content_type(conn)
+ record = {content_type, body}
+
+ Cachex.put(:web_resp_cache, key, record, ttl: ttl)
+
+ put_resp_header(conn, "x-cache", "MISS from Pleroma")
+
+ conn ->
+ conn
+ end)
+ end
+
+ defp content_type(conn) do
+ conn
+ |> Plug.Conn.get_resp_header("content-type")
+ |> hd()
+ end
+
+ defp send_cached(conn, {content_type, body}) do
+ conn
+ |> put_resp_content_type(content_type, nil)
+ |> put_resp_header("x-cache", "HIT from Pleroma")
+ |> send_resp(:ok, body)
+ |> halt()
+ end
+
+ defp render_error(conn, message) do
+ conn
+ |> put_status(:internal_server_error)
+ |> json(%{error: message})
+ |> halt()
+ end
+end
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -23,6 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
action_fallback(:errors)
+ plug(Pleroma.Plugs.Cache, [query_params: false] when action in [:activity, :object])
plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay])
plug(:set_requester_reachable when action in [:inbox])
plug(:relay_active? when action in [:relay])
@@ -53,8 +54,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
%Object{} = object <- Object.get_cached_by_ap_id(ap_id),
{_, true} <- {:public?, Visibility.is_public?(object)} do
conn
+ |> set_cache_ttl_for(object)
|> put_resp_content_type("application/activity+json")
- |> json(ObjectView.render("object.json", %{object: object}))
+ |> put_view(ObjectView)
+ |> render("object.json", object: object)
else
{:public?, false} ->
{:error, :not_found}
@@ -96,14 +99,36 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
%Activity{} = activity <- Activity.normalize(ap_id),
{_, true} <- {:public?, Visibility.is_public?(activity)} do
conn
+ |> set_cache_ttl_for(activity)
|> put_resp_content_type("application/activity+json")
- |> json(ObjectView.render("object.json", %{object: activity}))
+ |> put_view(ObjectView)
+ |> render("object.json", object: activity)
else
- {:public?, false} ->
- {:error, :not_found}
+ {:public?, false} -> {:error, :not_found}
+ nil -> {:error, :not_found}
end
end
+ defp set_cache_ttl_for(conn, %Activity{object: object}) do
+ set_cache_ttl_for(conn, object)
+ end
+
+ defp set_cache_ttl_for(conn, entity) do
+ ttl =
+ case entity do
+ %Object{data: %{"type" => "Question"}} ->
+ Pleroma.Config.get([:web_cache_ttl, :activity_pub_question])
+
+ %Object{} ->
+ Pleroma.Config.get([:web_cache_ttl, :activity_pub])
+
+ _ ->
+ nil
+ end
+
+ assign(conn, :cache_ttl, ttl)
+ end
+
# GET /relay/following
def following(%{assigns: %{relay: true}} = conn, _params) do
conn
diff --git a/test/object_test.exs b/test/object_test.exs
@@ -53,9 +53,12 @@ defmodule Pleroma.ObjectTest do
assert object == cached_object
+ Cachex.put(:web_resp_cache, URI.parse(object.data["id"]).path, "cofe")
+
Object.delete(cached_object)
{:ok, nil} = Cachex.get(:object_cache, "object:#{object.data["id"]}")
+ {:ok, nil} = Cachex.get(:web_resp_cache, URI.parse(object.data["id"]).path)
cached_object = Object.get_cached_by_ap_id(object.data["id"])
diff --git a/test/plugs/cache_test.exs b/test/plugs/cache_test.exs
@@ -0,0 +1,186 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Plugs.CacheTest do
+ use ExUnit.Case, async: true
+ use Plug.Test
+
+ alias Pleroma.Plugs.Cache
+
+ @miss_resp {200,
+ [
+ {"cache-control", "max-age=0, private, must-revalidate"},
+ {"content-type", "cofe/hot; charset=utf-8"},
+ {"x-cache", "MISS from Pleroma"}
+ ], "cofe"}
+
+ @hit_resp {200,
+ [
+ {"cache-control", "max-age=0, private, must-revalidate"},
+ {"content-type", "cofe/hot; charset=utf-8"},
+ {"x-cache", "HIT from Pleroma"}
+ ], "cofe"}
+
+ @ttl 5
+
+ setup do
+ Cachex.clear(:web_resp_cache)
+ :ok
+ end
+
+ test "caches a response" do
+ assert @miss_resp ==
+ conn(:get, "/")
+ |> Cache.call(%{query_params: false, ttl: nil})
+ |> put_resp_content_type("cofe/hot")
+ |> send_resp(:ok, "cofe")
+ |> sent_resp()
+
+ assert_raise(Plug.Conn.AlreadySentError, fn ->
+ conn(:get, "/")
+ |> Cache.call(%{query_params: false, ttl: nil})
+ |> put_resp_content_type("cofe/hot")
+ |> send_resp(:ok, "cofe")
+ |> sent_resp()
+ end)
+
+ assert @hit_resp ==
+ conn(:get, "/")
+ |> Cache.call(%{query_params: false, ttl: nil})
+ |> sent_resp()
+ end
+
+ test "ttl is set" do
+ assert @miss_resp ==
+ conn(:get, "/")
+ |> Cache.call(%{query_params: false, ttl: @ttl})
+ |> put_resp_content_type("cofe/hot")
+ |> send_resp(:ok, "cofe")
+ |> sent_resp()
+
+ assert @hit_resp ==
+ conn(:get, "/")
+ |> Cache.call(%{query_params: false, ttl: @ttl})
+ |> sent_resp()
+
+ :timer.sleep(@ttl + 1)
+
+ assert @miss_resp ==
+ conn(:get, "/")
+ |> Cache.call(%{query_params: false, ttl: @ttl})
+ |> put_resp_content_type("cofe/hot")
+ |> send_resp(:ok, "cofe")
+ |> sent_resp()
+ end
+
+ test "set ttl via conn.assigns" do
+ assert @miss_resp ==
+ conn(:get, "/")
+ |> Cache.call(%{query_params: false, ttl: nil})
+ |> put_resp_content_type("cofe/hot")
+ |> assign(:cache_ttl, @ttl)
+ |> send_resp(:ok, "cofe")
+ |> sent_resp()
+
+ assert @hit_resp ==
+ conn(:get, "/")
+ |> Cache.call(%{query_params: false, ttl: nil})
+ |> sent_resp()
+
+ :timer.sleep(@ttl + 1)
+
+ assert @miss_resp ==
+ conn(:get, "/")
+ |> Cache.call(%{query_params: false, ttl: nil})
+ |> put_resp_content_type("cofe/hot")
+ |> send_resp(:ok, "cofe")
+ |> sent_resp()
+ end
+
+ test "ignore query string when `query_params` is false" do
+ assert @miss_resp ==
+ conn(:get, "/?cofe")
+ |> Cache.call(%{query_params: false, ttl: nil})
+ |> put_resp_content_type("cofe/hot")
+ |> send_resp(:ok, "cofe")
+ |> sent_resp()
+
+ assert @hit_resp ==
+ conn(:get, "/?cofefe")
+ |> Cache.call(%{query_params: false, ttl: nil})
+ |> sent_resp()
+ end
+
+ test "take query string into account when `query_params` is true" do
+ assert @miss_resp ==
+ conn(:get, "/?cofe")
+ |> Cache.call(%{query_params: true, ttl: nil})
+ |> put_resp_content_type("cofe/hot")
+ |> send_resp(:ok, "cofe")
+ |> sent_resp()
+
+ assert @miss_resp ==
+ conn(:get, "/?cofefe")
+ |> Cache.call(%{query_params: true, ttl: nil})
+ |> put_resp_content_type("cofe/hot")
+ |> send_resp(:ok, "cofe")
+ |> sent_resp()
+ end
+
+ test "take specific query params into account when `query_params` is list" do
+ assert @miss_resp ==
+ conn(:get, "/?a=1&b=2&c=3&foo=bar")
+ |> fetch_query_params()
+ |> Cache.call(%{query_params: ["a", "b", "c"], ttl: nil})
+ |> put_resp_content_type("cofe/hot")
+ |> send_resp(:ok, "cofe")
+ |> sent_resp()
+
+ assert @hit_resp ==
+ conn(:get, "/?bar=foo&c=3&b=2&a=1")
+ |> fetch_query_params()
+ |> Cache.call(%{query_params: ["a", "b", "c"], ttl: nil})
+ |> sent_resp()
+
+ assert @miss_resp ==
+ conn(:get, "/?bar=foo&c=3&b=2&a=2")
+ |> fetch_query_params()
+ |> Cache.call(%{query_params: ["a", "b", "c"], ttl: nil})
+ |> put_resp_content_type("cofe/hot")
+ |> send_resp(:ok, "cofe")
+ |> sent_resp()
+ end
+
+ test "ignore not GET requests" do
+ expected =
+ {200,
+ [
+ {"cache-control", "max-age=0, private, must-revalidate"},
+ {"content-type", "cofe/hot; charset=utf-8"}
+ ], "cofe"}
+
+ assert expected ==
+ conn(:post, "/")
+ |> Cache.call(%{query_params: true, ttl: nil})
+ |> put_resp_content_type("cofe/hot")
+ |> send_resp(:ok, "cofe")
+ |> sent_resp()
+ end
+
+ test "ignore non-successful responses" do
+ expected =
+ {418,
+ [
+ {"cache-control", "max-age=0, private, must-revalidate"},
+ {"content-type", "tea/iced; charset=utf-8"}
+ ], "🥤"}
+
+ assert expected ==
+ conn(:get, "/cofe")
+ |> Cache.call(%{query_params: true, ttl: nil})
+ |> put_resp_content_type("tea/iced")
+ |> send_resp(:im_a_teapot, "🥤")
+ |> sent_resp()
+ end
+end
diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs
@@ -175,6 +175,49 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
assert json_response(conn, 404)
end
+
+ test "it caches a response", %{conn: conn} do
+ note = insert(:note)
+ uuid = String.split(note.data["id"], "/") |> List.last()
+
+ conn1 =
+ conn
+ |> put_req_header("accept", "application/activity+json")
+ |> get("/objects/#{uuid}")
+
+ assert json_response(conn1, :ok)
+ assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"}))
+
+ conn2 =
+ conn
+ |> put_req_header("accept", "application/activity+json")
+ |> get("/objects/#{uuid}")
+
+ assert json_response(conn1, :ok) == json_response(conn2, :ok)
+ assert Enum.any?(conn2.resp_headers, &(&1 == {"x-cache", "HIT from Pleroma"}))
+ end
+
+ test "cached purged after object deletion", %{conn: conn} do
+ note = insert(:note)
+ uuid = String.split(note.data["id"], "/") |> List.last()
+
+ conn1 =
+ conn
+ |> put_req_header("accept", "application/activity+json")
+ |> get("/objects/#{uuid}")
+
+ assert json_response(conn1, :ok)
+ assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"}))
+
+ Object.delete(note)
+
+ conn2 =
+ conn
+ |> put_req_header("accept", "application/activity+json")
+ |> get("/objects/#{uuid}")
+
+ assert "Not found" == json_response(conn2, :not_found)
+ end
end
describe "/object/:uuid/likes" do
@@ -264,6 +307,51 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
assert json_response(conn, 404)
end
+
+ test "it caches a response", %{conn: conn} do
+ activity = insert(:note_activity)
+ uuid = String.split(activity.data["id"], "/") |> List.last()
+
+ conn1 =
+ conn
+ |> put_req_header("accept", "application/activity+json")
+ |> get("/activities/#{uuid}")
+
+ assert json_response(conn1, :ok)
+ assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"}))
+
+ conn2 =
+ conn
+ |> put_req_header("accept", "application/activity+json")
+ |> get("/activities/#{uuid}")
+
+ assert json_response(conn1, :ok) == json_response(conn2, :ok)
+ assert Enum.any?(conn2.resp_headers, &(&1 == {"x-cache", "HIT from Pleroma"}))
+ end
+
+ test "cached purged after activity deletion", %{conn: conn} do
+ user = insert(:user)
+ {:ok, activity} = CommonAPI.post(user, %{"status" => "cofe"})
+
+ uuid = String.split(activity.data["id"], "/") |> List.last()
+
+ conn1 =
+ conn
+ |> put_req_header("accept", "application/activity+json")
+ |> get("/activities/#{uuid}")
+
+ assert json_response(conn1, :ok)
+ assert Enum.any?(conn1.resp_headers, &(&1 == {"x-cache", "MISS from Pleroma"}))
+
+ Activity.delete_by_ap_id(activity.object.data["id"])
+
+ conn2 =
+ conn
+ |> put_req_header("accept", "application/activity+json")
+ |> get("/activities/#{uuid}")
+
+ assert "Not found" == json_response(conn2, :not_found)
+ end
end
describe "/inbox" do
