commit: a431e8c9f7033c739e10b0e8b34c75f2cc1d38d4
parent: 8b4d81609d5627d62b826bcd3e87290cb513495f
Author: feld <feld@feld.me>
Date: Sun, 12 Jan 2020 18:48:59 +0000
Merge branch 'feature/attachments-cleanup' into 'develop'
Delete attachments when status is deleted
See merge request pleroma/pleroma!2036
Diffstat:
8 files changed, 214 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- **Breaking**: MDII uploader
### Changed
+- **Breaking:** attachments are removed along with statuses when there are no other references to it
- **Breaking:** Elixir >=1.8 is now required (was >= 1.7)
- **Breaking:** attachment links (`config :pleroma, :instance, no_attachment_links` and `config :pleroma, Pleroma.Upload, link_name`) disabled by default
- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings)
diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
@@ -17,6 +17,8 @@ defmodule Pleroma.Object do
require Logger
+ @type t() :: %__MODULE__{}
+
schema "objects" do
field(:data, :map)
@@ -79,6 +81,20 @@ defmodule Pleroma.Object do
Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id)))
end
+ @doc """
+ Get a single attachment by it's name and href
+ """
+ @spec get_attachment_by_name_and_href(String.t(), String.t()) :: Object.t() | nil
+ def get_attachment_by_name_and_href(name, href) do
+ query =
+ from(o in Object,
+ where: fragment("(?)->>'name' = ?", o.data, ^name),
+ where: fragment("(?)->>'href' = ?", o.data, ^href)
+ )
+
+ Repo.one(query)
+ end
+
defp warn_on_no_object_preloaded(ap_id) do
"Object.normalize() called without preloaded object (#{inspect(ap_id)}). Consider preloading the object"
|> Logger.debug()
@@ -164,6 +180,7 @@ defmodule Pleroma.Object do
def delete(%Object{data: %{"id" => id}} = object) do
with {:ok, _obj} = swap_object_with_tombstone(object),
+ :ok <- delete_attachments(object),
deleted_activity = Activity.delete_all_by_object_ap_id(id),
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
{:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do
@@ -171,6 +188,77 @@ defmodule Pleroma.Object do
end
end
+ defp delete_attachments(%{data: %{"attachment" => [_ | _] = attachments, "actor" => actor}}) do
+ hrefs =
+ Enum.flat_map(attachments, fn attachment ->
+ Enum.map(attachment["url"], & &1["href"])
+ end)
+
+ names = Enum.map(attachments, & &1["name"])
+
+ uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
+
+ # find all objects for copies of the attachments, name and actor doesn't matter here
+ delete_ids =
+ from(o in Object,
+ where:
+ fragment(
+ "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href'))::jsonb \\?| (?)",
+ o.data,
+ ^hrefs
+ )
+ )
+ |> Repo.all()
+ # we should delete 1 object for any given attachment, but don't delete files if
+ # there are more than 1 object for it
+ |> Enum.reduce(%{}, fn %{
+ id: id,
+ data: %{
+ "url" => [%{"href" => href}],
+ "actor" => obj_actor,
+ "name" => name
+ }
+ },
+ acc ->
+ Map.update(acc, href, %{id: id, count: 1}, fn val ->
+ case obj_actor == actor and name in names do
+ true ->
+ # set id of the actor's object that will be deleted
+ %{val | id: id, count: val.count + 1}
+
+ false ->
+ # another actor's object, just increase count to not delete file
+ %{val | count: val.count + 1}
+ end
+ end)
+ end)
+ |> Enum.map(fn {href, %{id: id, count: count}} ->
+ # only delete files that have single instance
+ with 1 <- count do
+ prefix =
+ case Pleroma.Config.get([Pleroma.Upload, :base_url]) do
+ nil -> "media"
+ _ -> ""
+ end
+
+ base_url = Pleroma.Config.get([__MODULE__, :base_url], Pleroma.Web.base_url())
+
+ file_path = String.trim_leading(href, "#{base_url}/#{prefix}")
+
+ uploader.delete_file(file_path)
+ end
+
+ id
+ end)
+
+ from(o in Object, where: o.id in ^delete_ids)
+ |> Repo.delete_all()
+
+ :ok
+ end
+
+ defp delete_attachments(%{data: _data}), do: :ok
+
def prune(%Object{data: %{"id" => id}} = object) do
with {:ok, object} <- Repo.delete(object),
{:ok, true} <- Cachex.del(:object_cache, "object:#{id}"),
diff --git a/lib/pleroma/uploaders/local.ex b/lib/pleroma/uploaders/local.ex
@@ -5,10 +5,12 @@
defmodule Pleroma.Uploaders.Local do
@behaviour Pleroma.Uploaders.Uploader
+ @impl true
def get_file(_) do
{:ok, {:static_dir, upload_path()}}
end
+ @impl true
def put_file(upload) do
{local_path, file} =
case Enum.reverse(Path.split(upload.path)) do
@@ -33,4 +35,15 @@ defmodule Pleroma.Uploaders.Local do
def upload_path do
Pleroma.Config.get!([__MODULE__, :uploads])
end
+
+ @impl true
+ def delete_file(path) do
+ upload_path()
+ |> Path.join(path)
+ |> File.rm()
+ |> case do
+ :ok -> :ok
+ {:error, posix_error} -> {:error, to_string(posix_error)}
+ end
+ end
end
diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Uploaders.S3 do
# The file name is re-encoded with S3's constraints here to comply with previous
# links with less strict filenames
+ @impl true
def get_file(file) do
config = Config.get([__MODULE__])
bucket = Keyword.fetch!(config, :bucket)
@@ -35,6 +36,7 @@ defmodule Pleroma.Uploaders.S3 do
])}}
end
+ @impl true
def put_file(%Pleroma.Upload{} = upload) do
config = Config.get([__MODULE__])
bucket = Keyword.get(config, :bucket)
@@ -69,6 +71,18 @@ defmodule Pleroma.Uploaders.S3 do
end
end
+ @impl true
+ def delete_file(file) do
+ [__MODULE__, :bucket]
+ |> Config.get()
+ |> ExAws.S3.delete_object(file)
+ |> ExAws.request()
+ |> case do
+ {:ok, %{status_code: 204}} -> :ok
+ error -> {:error, inspect(error)}
+ end
+ end
+
@regex Regex.compile!("[^0-9a-zA-Z!.*/'()_-]")
def strict_encode(name) do
String.replace(name, @regex, "-")
diff --git a/lib/pleroma/uploaders/uploader.ex b/lib/pleroma/uploaders/uploader.ex
@@ -36,6 +36,8 @@ defmodule Pleroma.Uploaders.Uploader do
@callback put_file(Pleroma.Upload.t()) ::
:ok | {:ok, file_spec()} | {:error, String.t()} | :wait_callback
+ @callback delete_file(file :: String.t()) :: :ok | {:error, String.t()}
+
@callback http_callback(Plug.Conn.t(), Map.t()) ::
{:ok, Plug.Conn.t()}
| {:ok, Plug.Conn.t(), file_spec()}
@@ -43,7 +45,6 @@ defmodule Pleroma.Uploaders.Uploader do
@optional_callbacks http_callback: 2
@spec put_file(module(), Pleroma.Upload.t()) :: {:ok, file_spec()} | {:error, String.t()}
-
def put_file(uploader, upload) do
case uploader.put_file(upload) do
:ok -> {:ok, {:file, upload.path}}
diff --git a/test/object_test.exs b/test/object_test.exs
@@ -71,6 +71,74 @@ defmodule Pleroma.ObjectTest do
end
end
+ describe "delete attachments" do
+ clear_config([Pleroma.Upload])
+
+ test "in subdirectories" do
+ Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+
+ file = %Plug.Upload{
+ content_type: "image/jpg",
+ path: Path.absname("test/fixtures/image.jpg"),
+ filename: "an_image.jpg"
+ }
+
+ user = insert(:user)
+
+ {:ok, %Object{} = attachment} =
+ Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)
+
+ %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
+ note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
+
+ uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
+
+ path = href |> Path.dirname() |> Path.basename()
+
+ assert {:ok, ["an_image.jpg"]} == File.ls("#{uploads_dir}/#{path}")
+
+ Object.delete(note)
+
+ assert Object.get_by_id(attachment.id) == nil
+
+ assert {:ok, []} == File.ls("#{uploads_dir}/#{path}")
+ end
+
+ test "with dedupe enabled" do
+ Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+ Pleroma.Config.put([Pleroma.Upload, :filters], [Pleroma.Upload.Filter.Dedupe])
+
+ uploads_dir = Pleroma.Config.get!([Pleroma.Uploaders.Local, :uploads])
+
+ File.mkdir_p!(uploads_dir)
+
+ file = %Plug.Upload{
+ content_type: "image/jpg",
+ path: Path.absname("test/fixtures/image.jpg"),
+ filename: "an_image.jpg"
+ }
+
+ user = insert(:user)
+
+ {:ok, %Object{} = attachment} =
+ Pleroma.Web.ActivityPub.ActivityPub.upload(file, actor: user.ap_id)
+
+ %{data: %{"attachment" => [%{"url" => [%{"href" => href}]}]}} =
+ note = insert(:note, %{user: user, data: %{"attachment" => [attachment.data]}})
+
+ filename = Path.basename(href)
+
+ assert {:ok, files} = File.ls(uploads_dir)
+ assert filename in files
+
+ Object.delete(note)
+
+ assert Object.get_by_id(attachment.id) == nil
+ assert {:ok, files} = File.ls(uploads_dir)
+ refute filename in files
+ end
+ end
+
describe "normalizer" do
test "fetches unknown objects by default" do
%Object{} =
diff --git a/test/uploaders/local_test.exs b/test/uploaders/local_test.exs
@@ -29,4 +29,25 @@ defmodule Pleroma.Uploaders.LocalTest do
|> File.exists?()
end
end
+
+ describe "delete_file/1" do
+ test "deletes local file" do
+ file_path = "local_upload/files/image.jpg"
+
+ file = %Pleroma.Upload{
+ name: "image.jpg",
+ content_type: "image/jpg",
+ path: file_path,
+ tempfile: Path.absname("test/fixtures/image_tmp.jpg")
+ }
+
+ :ok = Local.put_file(file)
+ local_path = Path.join([Local.upload_path(), file_path])
+ assert File.exists?(local_path)
+
+ Local.delete_file(file_path)
+
+ refute File.exists?(local_path)
+ end
+ end
end
diff --git a/test/uploaders/s3_test.exs b/test/uploaders/s3_test.exs
@@ -79,4 +79,11 @@ defmodule Pleroma.Uploaders.S3Test do
end
end
end
+
+ describe "delete_file/1" do
+ test_with_mock "deletes file", ExAws, request: fn _req -> {:ok, %{status_code: 204}} end do
+ assert :ok = S3.delete_file("image.jpg")
+ assert_called(ExAws.request(:_))
+ end
+ end
end
