commit: 4dabf64af45232afdbbb15d9a697a64e15a749ee
parent: d089ff24600455fefc17e91807c61ddc61ba107a
Author: lambda <lain@soykaf.club>
Date: Mon, 6 May 2019 16:45:22 +0000
Merge branch 'feature/845-improve-status-deletion' into 'develop'
Improve status deletion
Closes #845
See merge request pleroma/pleroma!1104
Diffstat:
9 files changed, 40 insertions(+), 21 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -66,6 +66,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Deactivated users being able to request an access token
- Limit on request body in rich media/relme parsers being ignored resulting in a possible memory leak
- proper Twitter Card generation instead of a dummy
+- Deletions failing for users with a large number of posts
- NodeInfo: Include admins in `staffAccounts`
- ActivityPub: Crashing when requesting empty local user's outbox
- Federation: Handling of objects without `summary` property
diff --git a/config/config.exs b/config/config.exs
@@ -232,7 +232,8 @@ config :pleroma, :instance,
welcome_message: nil,
max_report_comment_size: 1000,
safe_dm_mentions: false,
- healthcheck: false
+ healthcheck: false,
+ repo_batch_size: 500
config :pleroma, :markup,
# XXX - unfortunately, inline images must be enabled by default right now, because
@@ -416,7 +417,8 @@ config :pleroma_job_queue, :queues,
web_push: 50,
mailer: 10,
transmogrifier: 20,
- scheduled_activities: 10
+ scheduled_activities: 10,
+ background: 5
config :pleroma, :fetch_initial_posts,
enabled: false,
diff --git a/docs/config.md b/docs/config.md
@@ -104,6 +104,7 @@ config :pleroma, Pleroma.Emails.Mailer,
* `max_report_comment_size`: The maximum size of the report comment (Default: `1000`)
* `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`)
* `healthcheck`: if set to true, system data will be shown on ``/api/pleroma/healthcheck``.
+* `repo_batch_size`: Repo batch size. The number of loaded rows from the database to the memory for processing chunks. E.g. deleting user statuses.
## :logger
* `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack
diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex
@@ -163,7 +163,7 @@ defmodule Mix.Tasks.Pleroma.User do
Common.start_pleroma()
with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
- User.delete(user)
+ User.perform(:delete, user)
Mix.shell().info("User #{nickname} deleted.")
else
_ ->
@@ -380,7 +380,7 @@ defmodule Mix.Tasks.Pleroma.User do
Common.start_pleroma()
with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
- User.delete_user_activities(user)
+ {:ok, _} = User.delete_user_activities(user)
Mix.shell().info("User #{nickname} statuses deleted.")
else
_ ->
diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
@@ -14,6 +14,8 @@ defmodule Pleroma.Activity do
import Ecto.Query
@type t :: %__MODULE__{}
+ @type actor :: String.t()
+
@primary_key {:id, Pleroma.FlakeId, autogenerate: true}
# https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
@@ -260,4 +262,9 @@ defmodule Pleroma.Activity do
|> where([s], s.actor == ^actor)
|> Repo.all()
end
+
+ @spec query_by_actor(actor()) :: Ecto.Query.t()
+ def query_by_actor(actor) do
+ from(a in Activity, where: a.actor == ^actor)
+ end
end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
@@ -1164,7 +1164,12 @@ defmodule Pleroma.User do
|> update_and_set_cache()
end
- def delete(%User{} = user) do
+ @spec delete(User.t()) :: :ok
+ def delete(%User{} = user),
+ do: PleromaJobQueue.enqueue(:background, __MODULE__, [:delete, user])
+
+ @spec perform(atom(), User.t()) :: {:ok, User.t()}
+ def perform(:delete, %User{} = user) do
{:ok, user} = User.deactivate(user)
# Remove all relationships
@@ -1180,22 +1185,23 @@ defmodule Pleroma.User do
end
def delete_user_activities(%User{ap_id: ap_id} = user) do
- Activity
- |> where(actor: ^ap_id)
- |> Activity.with_preloaded_object()
- |> Repo.all()
- |> Enum.each(fn
- %{data: %{"type" => "Create"}} = activity ->
- activity |> Object.normalize() |> ActivityPub.delete()
+ stream =
+ ap_id
+ |> Activity.query_by_actor()
+ |> Activity.with_preloaded_object()
+ |> Repo.stream()
- # TODO: Do something with likes, follows, repeats.
- _ ->
- "Doing nothing"
- end)
+ Repo.transaction(fn -> Enum.each(stream, &delete_activity(&1)) end, timeout: :infinity)
{:ok, user}
end
+ defp delete_activity(%{data: %{"type" => "Create"}} = activity) do
+ Object.normalize(activity) |> ActivityPub.delete()
+ end
+
+ defp delete_activity(_activity), do: "Doing nothing"
+
def html_filter_policy(%User{info: %{no_rich_text: true}}) do
Pleroma.HTML.Scrubber.TwitterText
end
diff --git a/lib/pleroma/user_invite_token.ex b/lib/pleroma/user_invite_token.ex
@@ -24,7 +24,7 @@ defmodule Pleroma.UserInviteToken do
timestamps()
end
- @spec create_invite(map()) :: UserInviteToken.t()
+ @spec create_invite(map()) :: {:ok, UserInviteToken.t()}
def create_invite(params \\ %{}) do
%UserInviteToken{}
|> cast(params, [:max_use, :expires_at])
diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
@@ -352,7 +352,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
def delete_account(%{assigns: %{user: user}} = conn, params) do
case CommonAPI.Utils.confirm_current_password(user, params["password"]) do
{:ok, user} ->
- Task.start(fn -> User.delete(user) end)
+ User.delete(user)
json(conn, %{status: "success"})
{:error, msg} ->
diff --git a/test/user_test.exs b/test/user_test.exs
@@ -829,10 +829,12 @@ defmodule Pleroma.UserTest do
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{"status" => "2hu"})
- {:ok, _} = User.delete_user_activities(user)
- # TODO: Remove favorites, repeats, delete activities.
- refute Activity.get_by_id(activity.id)
+ Ecto.Adapters.SQL.Sandbox.unboxed_run(Repo, fn ->
+ {:ok, _} = User.delete_user_activities(user)
+ # TODO: Remove favorites, repeats, delete activities.
+ refute Activity.get_by_id(activity.id)
+ end)
end
test ".delete deactivates a user, all follow relationships and all create activities" do
