commit: 83aeb6090072986d39dc52de05f5a23164f8ce04 parent: bc78a875c86db42d67122cfb767f239a55dacbea Author: kaniini <ariadne@dereferenced.org> Date: Sat, 24 Aug 2019 16:01:57 +0000 Merge branch 'activity-expiration' into 'develop' Activity expiration See merge request pleroma/pleroma!1595Diffstat:
19 files changed, 328 insertions(+), 18 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md @@ -50,6 +50,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - ActivityPub: Deactivated user deletion ### Added +- Expiring/ephemeral activites. All activities can have expires_at value set, which controls when they should be deleted automatically. +- Mastodon API: in post_status, the expires_in parameter lets you set the number of seconds until an activity expires. It must be at least one hour. +- Mastodon API: all status JSON responses contain a `pleroma.expires_at` item which states when an activity will expire. The value is only shown to the user who created the activity. To everyone else it's empty. +- Configuration: `ActivityExpiration.enabled` controls whether expired activites will get deleted at the appropriate time. Enabled by default. - Conversations: Add Pleroma-specific conversation endpoints and status posting extensions. Run the `bump_all_conversations` task again to create the necessary data. - **Breaking:** MRF describe API, which adds support for exposing configuration information about MRF policies to NodeInfo. Custom modules will need to be updated by adding, at the very least, `def describe, do: {:ok, %{}}` to the MRF policy modules. diff --git a/config/config.exs b/config/config.exs @@ -456,6 +456,7 @@ config :pleroma, Pleroma.Web.Federator.RetryQueue, max_retries: 5 config :pleroma_job_queue, :queues, + activity_expiration: 10, federator_incoming: 50, federator_outgoing: 50, web_push: 50, @@ -566,6 +567,8 @@ config :pleroma, :rate_limit, account_confirmation_resend: {8_640_000, 5}, ap_routes: {60_000, 15} +config :pleroma, Pleroma.ActivityExpiration, enabled: true + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/test.exs b/config/test.exs @@ -86,11 +86,10 @@ config :joken, default_signer: "yU8uHKq+yyAkZ11Hx//jcdacWc8yQ1bxAAGrplzB0Zwwjkp3 config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock -try do +if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" -rescue - _ -> - IO.puts( - "You may want to create test.secret.exs to declare custom database connection parameters." - ) +else + IO.puts( + "You may want to create test.secret.exs to declare custom database connection parameters." + ) end diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md @@ -25,6 +25,7 @@ Has these additional fields under the `pleroma` object: - `in_reply_to_account_acct`: the `acct` property of User entity for replied user (if any) - `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` - `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` +- `expires_at`: a datetime (iso8601) that states when the post will expire (be deleted automatically), or empty if the post won't expire ## Attachments @@ -86,6 +87,7 @@ Additional parameters can be added to the JSON body/Form data: - `content_type`: string, contain the MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint. - `to`: A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for for post visibility are not affected by this and will still apply. - `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted` or `public`) it can be used to address a List by setting it to `list:LIST_ID`. +- `expires_in`: The number of seconds the posted activity should expire in. When a posted activity expires it will be deleted from the server, and a delete request for it will be federated. This needs to be longer than an hour. - `in_reply_to_conversation_id`: Will reply to a given conversation, addressing only the people who are part of the recipient set of that conversation. Sets the visibility to `direct`. ## PATCH `/api/v1/update_credentials` diff --git a/docs/config.md b/docs/config.md @@ -495,6 +495,10 @@ config :auto_linker, * `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`) * `enabled`: whether scheduled activities are sent to the job queue to be executed +## Pleroma.ActivityExpiration + +# `enabled`: whether expired activities will be sent to the job queue to be deleted + ## Pleroma.Web.Auth.Authenticator * `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Activity do use Ecto.Schema alias Pleroma.Activity + alias Pleroma.ActivityExpiration alias Pleroma.Bookmark alias Pleroma.Notification alias Pleroma.Object @@ -59,6 +60,8 @@ defmodule Pleroma.Activity do # typical case. has_one(:object, Object, on_delete: :nothing, foreign_key: :id) + has_one(:expiration, ActivityExpiration, on_delete: :delete_all) + timestamps() end diff --git a/lib/pleroma/activity_expiration.ex b/lib/pleroma/activity_expiration.ex @@ -0,0 +1,68 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ActivityExpiration do + use Ecto.Schema + + alias Pleroma.Activity + alias Pleroma.ActivityExpiration + alias Pleroma.FlakeId + alias Pleroma.Repo + + import Ecto.Changeset + import Ecto.Query + + @type t :: %__MODULE__{} + @min_activity_lifetime :timer.hours(1) + + schema "activity_expirations" do + belongs_to(:activity, Activity, type: FlakeId) + field(:scheduled_at, :naive_datetime) + end + + def changeset(%ActivityExpiration{} = expiration, attrs) do + expiration + |> cast(attrs, [:scheduled_at]) + |> validate_required([:scheduled_at]) + |> validate_scheduled_at() + end + + def get_by_activity_id(activity_id) do + ActivityExpiration + |> where([exp], exp.activity_id == ^activity_id) + |> Repo.one() + end + + def create(%Activity{} = activity, scheduled_at) do + %ActivityExpiration{activity_id: activity.id} + |> changeset(%{scheduled_at: scheduled_at}) + |> Repo.insert() + end + + def due_expirations(offset \\ 0) do + naive_datetime = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(offset, :millisecond) + + ActivityExpiration + |> where([exp], exp.scheduled_at < ^naive_datetime) + |> Repo.all() + end + + def validate_scheduled_at(changeset) do + validate_change(changeset, :scheduled_at, fn _, scheduled_at -> + if not expires_late_enough?(scheduled_at) do + [scheduled_at: "an ephemeral activity must live for at least one hour"] + else + [] + end + end) + end + + def expires_late_enough?(scheduled_at) do + now = NaiveDateTime.utc_now() + diff = NaiveDateTime.diff(scheduled_at, now, :millisecond) + diff >= @min_activity_lifetime + end +end diff --git a/lib/pleroma/activity_expiration_worker.ex b/lib/pleroma/activity_expiration_worker.ex @@ -0,0 +1,62 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ActivityExpirationWorker do + alias Pleroma.Activity + alias Pleroma.ActivityExpiration + alias Pleroma.Config + alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.Web.CommonAPI + require Logger + use GenServer + import Ecto.Query + + @schedule_interval :timer.minutes(1) + + def start_link(_) do + GenServer.start_link(__MODULE__, nil) + end + + @impl true + def init(_) do + if Config.get([ActivityExpiration, :enabled]) do + schedule_next() + {:ok, nil} + else + :ignore + end + end + + def perform(:execute, expiration_id) do + try do + expiration = + ActivityExpiration + |> where([e], e.id == ^expiration_id) + |> Repo.one!() + + activity = Activity.get_by_id_with_object(expiration.activity_id) + user = User.get_by_ap_id(activity.object.data["actor"]) + CommonAPI.delete(activity.id, user) + rescue + error -> + Logger.error("#{__MODULE__} Couldn't delete expired activity: #{inspect(error)}") + end + end + + @impl true + def handle_info(:perform, state) do + ActivityExpiration.due_expirations(@schedule_interval) + |> Enum.each(fn expiration -> + PleromaJobQueue.enqueue(:activity_expiration, __MODULE__, [:execute, expiration.id]) + end) + + schedule_next() + {:noreply, state} + end + + defp schedule_next do + Process.send_after(self(), :perform, @schedule_interval) + end +end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex @@ -35,7 +35,8 @@ defmodule Pleroma.Application do Pleroma.Emoji, Pleroma.Captcha, Pleroma.FlakeId, - Pleroma.ScheduledActivityWorker + Pleroma.ScheduledActivityWorker, + Pleroma.ActivityExpirationWorker ] ++ cachex_children() ++ hackney_pool_children() ++ diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Activity + alias Pleroma.ActivityExpiration alias Pleroma.Conversation.Participation alias Pleroma.Formatter alias Pleroma.Object @@ -200,6 +201,23 @@ defmodule Pleroma.Web.CommonAPI do end end + defp check_expiry_date({:ok, nil} = res), do: res + + defp check_expiry_date({:ok, in_seconds}) do + expiry = NaiveDateTime.utc_now() |> NaiveDateTime.add(in_seconds) + + if ActivityExpiration.expires_late_enough?(expiry) do + {:ok, expiry} + else + {:error, "Expiry date is too soon"} + end + end + + defp check_expiry_date(expiry_str) do + Ecto.Type.cast(:integer, expiry_str) + |> check_expiry_date() + end + def post(user, %{"status" => status} = data) do limit = Pleroma.Config.get([:instance, :limit]) @@ -226,6 +244,7 @@ defmodule Pleroma.Web.CommonAPI do context <- make_context(in_reply_to, in_reply_to_conversation), cw <- data["spoiler_text"] || "", sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}), + {:ok, expires_at} <- check_expiry_date(data["expires_in"]), full_payload <- String.trim(status <> cw), :ok <- validate_character_limit(full_payload, attachments, limit), object <- @@ -251,15 +270,24 @@ defmodule Pleroma.Web.CommonAPI do preview? = Pleroma.Web.ControllerHelper.truthy_param?(data["preview"]) || false direct? = visibility == "direct" - %{ - to: to, - actor: user, - context: context, - object: object, - additional: %{"cc" => cc, "directMessage" => direct?} - } - |> maybe_add_list_data(user, visibility) - |> ActivityPub.create(preview?) + result = + %{ + to: to, + actor: user, + context: context, + object: object, + additional: %{"cc" => cc, "directMessage" => direct?} + } + |> maybe_add_list_data(user, visibility) + |> ActivityPub.create(preview?) + + if expires_at do + with {:ok, activity} <- result do + {:ok, _} = ActivityExpiration.create(activity, expires_at) + end + end + + result else {:private_to_public, true} -> {:error, dgettext("errors", "The message visibility must be direct")} diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do require Pleroma.Constants alias Pleroma.Activity + alias Pleroma.ActivityExpiration alias Pleroma.Conversation alias Pleroma.Conversation.Participation alias Pleroma.HTML @@ -177,6 +178,15 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do bookmarked = Activity.get_bookmark(activity, opts[:for]) != nil + client_posted_this_activity = opts[:for] && user.id == opts[:for].id + + expires_at = + with true <- client_posted_this_activity, + expiration when not is_nil(expiration) <- + ActivityExpiration.get_by_activity_id(activity.id) do + expiration.scheduled_at + end + thread_muted? = case activity.thread_muted? do thread_muted? when is_boolean(thread_muted?) -> thread_muted? @@ -288,6 +298,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, content: %{"text/plain" => content_plaintext}, spoiler_text: %{"text/plain" => summary_plaintext}, + expires_at: expires_at, direct_conversation_id: direct_conversation_id } } diff --git a/priv/repo/migrations/20190716100804_add_expirations_table.exs b/priv/repo/migrations/20190716100804_add_expirations_table.exs @@ -0,0 +1,10 @@ +defmodule Pleroma.Repo.Migrations.AddExpirationsTable do + use Ecto.Migration + + def change do + create_if_not_exists table(:activity_expirations) do + add(:activity_id, references(:activities, type: :uuid, on_delete: :delete_all)) + add(:scheduled_at, :naive_datetime, null: false) + end + end +end diff --git a/test/activity_expiration_test.exs b/test/activity_expiration_test.exs @@ -0,0 +1,27 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2019 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ActivityExpirationTest do + use Pleroma.DataCase + alias Pleroma.ActivityExpiration + import Pleroma.Factory + + test "finds activities due to be deleted only" do + activity = insert(:note_activity) + expiration_due = insert(:expiration_in_the_past, %{activity_id: activity.id}) + activity2 = insert(:note_activity) + insert(:expiration_in_the_future, %{activity_id: activity2.id}) + + expirations = ActivityExpiration.due_expirations() + + assert length(expirations) == 1 + assert hd(expirations) == expiration_due + end + + test "denies expirations that don't live long enough" do + activity = insert(:note_activity) + now = NaiveDateTime.utc_now() + assert {:error, _} = ActivityExpiration.create(activity, now) + end +end diff --git a/test/activity_expiration_worker_test.exs b/test/activity_expiration_worker_test.exs @@ -0,0 +1,17 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ActivityExpirationWorkerTest do + use Pleroma.DataCase + alias Pleroma.Activity + import Pleroma.Factory + + test "deletes an activity" do + activity = insert(:note_activity) + expiration = insert(:expiration_in_the_past, %{activity_id: activity.id}) + Pleroma.ActivityExpirationWorker.perform(:execute, expiration.id) + + refute Repo.get(Activity, activity.id) + end +end diff --git a/test/activity_test.exs b/test/activity_test.exs @@ -164,4 +164,13 @@ defmodule Pleroma.ActivityTest do Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) end end + + test "add an activity with an expiration" do + activity = insert(:note_activity) + insert(:expiration_in_the_future, %{activity_id: activity.id}) + + Pleroma.ActivityExpiration + |> where([a], a.activity_id == ^activity.id) + |> Repo.one!() + end end diff --git a/test/support/factory.ex b/test/support/factory.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/> +# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Factory do @@ -143,6 +143,25 @@ defmodule Pleroma.Factory do |> Map.merge(attrs) end + defp expiration_offset_by_minutes(attrs, minutes) do + scheduled_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(minutes), :millisecond) + |> NaiveDateTime.truncate(:second) + + %Pleroma.ActivityExpiration{} + |> Map.merge(attrs) + |> Map.put(:scheduled_at, scheduled_at) + end + + def expiration_in_the_past_factory(attrs \\ %{}) do + expiration_offset_by_minutes(attrs, -60) + end + + def expiration_in_the_future_factory(attrs \\ %{}) do + expiration_offset_by_minutes(attrs, 61) + end + def article_activity_factory do article = insert(:article) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs @@ -204,6 +204,21 @@ defmodule Pleroma.Web.CommonAPITest do assert {:error, "The status is over the character limit"} = CommonAPI.post(user, %{"status" => "foobar"}) end + + test "it can handle activities that expire" do + user = insert(:user) + + expires_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.truncate(:second) + |> NaiveDateTime.add(1_000_000, :second) + + assert {:ok, activity} = + CommonAPI.post(user, %{"status" => "chai", "expires_in" => 1_000_000}) + + assert expiration = Pleroma.ActivityExpiration.get_by_activity_id(activity.id) + assert expiration.scheduled_at == expires_at + end end describe "reactions" do diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do alias Ecto.Changeset alias Pleroma.Activity + alias Pleroma.ActivityExpiration alias Pleroma.Config alias Pleroma.Notification alias Pleroma.Object @@ -150,6 +151,32 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert %{"id" => third_id} = json_response(conn_three, 200) refute id == third_id + + # An activity that will expire: + # 2 hours + expires_in = 120 * 60 + + conn_four = + conn + |> post("api/v1/statuses", %{ + "status" => "oolong", + "expires_in" => expires_in + }) + + assert fourth_response = %{"id" => fourth_id} = json_response(conn_four, 200) + assert activity = Activity.get_by_id(fourth_id) + assert expiration = ActivityExpiration.get_by_activity_id(fourth_id) + + estimated_expires_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(expires_in) + |> NaiveDateTime.truncate(:second) + + # This assert will fail if the test takes longer than a minute. I sure hope it never does: + assert abs(NaiveDateTime.diff(expiration.scheduled_at, estimated_expires_at, :second)) < 60 + + assert fourth_response["pleroma"]["expires_at"] == + NaiveDateTime.to_iso8601(expiration.scheduled_at) end test "replying to a status", %{conn: conn} do @@ -403,7 +430,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert %{"visibility" => "direct"} = status assert status["url"] != direct.data["id"] - # User should be able to see his own direct message + # User should be able to see their own direct message res_conn = build_conn() |> assign(:user, user_one) diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs @@ -149,6 +149,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do in_reply_to_account_acct: nil, content: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["content"])}, spoiler_text: %{"text/plain" => HtmlSanitizeEx.strip_tags(object_data["summary"])}, + expires_at: nil, direct_conversation_id: nil } }