commit: 739cb1463ba07513f047b2ac8f7e22a16c89ef4e
parent 3ad7492f9dd1c76cdbc64ad2246f8e9c8c5c4ae6
Author: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 4 Sep 2020 21:48:52 +0400
Add backups deletion
Diffstat:
4 files changed, 91 insertions(+), 10 deletions(-)
diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex
@@ -15,6 +15,7 @@ defmodule Pleroma.Backup do
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.UserView
+ alias Pleroma.Workers.BackupWorker
schema "backups" do
field(:content_type, :string)
@@ -30,7 +31,7 @@ defmodule Pleroma.Backup do
def create(user) do
with :ok <- validate_limit(user),
{:ok, backup} <- user |> new() |> Repo.insert() do
- Pleroma.Workers.BackupWorker.enqueue("process", %{"backup_id" => backup.id})
+ BackupWorker.process(backup)
end
end
@@ -46,6 +47,14 @@ defmodule Pleroma.Backup do
}
end
+ def delete(backup) do
+ uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
+
+ with :ok <- uploader.delete_file("backups/" <> backup.file_name) do
+ Repo.delete(backup)
+ end
+ end
+
defp validate_limit(user) do
case get_last(user.id) do
%__MODULE__{inserted_at: inserted_at} ->
@@ -75,7 +84,8 @@ defmodule Pleroma.Backup do
__MODULE__
|> where(user_id: ^user_id)
|> where([b], b.id != ^latest_id)
- |> Repo.delete_all()
+ |> Repo.all()
+ |> Enum.each(&BackupWorker.delete/1)
end
def get(id), do: Repo.get(__MODULE__, id)
diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex
@@ -3,15 +3,46 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Workers.BackupWorker do
+ use Oban.Worker, queue: :backup, max_attempts: 1
+
+ alias Oban.Job
alias Pleroma.Backup
- use Pleroma.Workers.WorkerHelper, queue: "backup"
+ def process(backup) do
+ %{"op" => "process", "backup_id" => backup.id}
+ |> new()
+ |> Oban.insert()
+ end
+
+ def schedule_deletion(backup) do
+ days = Pleroma.Config.get([Pleroma.Backup, :purge_after_days])
+ time = 60 * 60 * 24 * days
+ scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time)
+
+ %{"op" => "delete", "backup_id" => backup.id}
+ |> new(scheduled_at: scheduled_at)
+ |> Oban.insert()
+ end
+
+ def delete(backup) do
+ %{"op" => "delete", "backup_id" => backup.id}
+ |> new()
+ |> Oban.insert()
+ end
- @impl Oban.Worker
def perform(%Job{args: %{"op" => "process", "backup_id" => backup_id}}) do
with {:ok, %Backup{} = backup} <-
- backup_id |> Backup.get() |> Backup.process() do
+ backup_id |> Backup.get() |> Backup.process(),
+ {:ok, _job} <- schedule_deletion(backup),
+ :ok <- Backup.remove_outdated(backup) do
{:ok, backup}
end
end
+
+ def perform(%Job{args: %{"op" => "delete", "backup_id" => backup_id}}) do
+ case Backup.get(backup_id) do
+ %Backup{} = backup -> Backup.delete(backup)
+ nil -> :ok
+ end
+ end
end
diff --git a/test/backup_test.exs b/test/backup_test.exs
@@ -13,8 +13,12 @@ defmodule Pleroma.BackupTest do
alias Pleroma.Bookmark
alias Pleroma.Web.CommonAPI
alias Pleroma.Workers.BackupWorker
+ alias Pleroma.Tests.ObanHelpers
- setup do: clear_config([Pleroma.Upload, :uploader])
+ setup do
+ clear_config([Pleroma.Upload, :uploader])
+ clear_config([Pleroma.Backup, :limit_days])
+ end
test "it creates a backup record and an Oban job" do
%{id: user_id} = user = insert(:user)
@@ -38,10 +42,34 @@ defmodule Pleroma.BackupTest do
test "it process a backup record" do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
%{id: user_id} = user = insert(:user)
- assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}} = job} = Backup.create(user)
- assert {:ok, backup} = BackupWorker.perform(job)
+
+ assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id} = args}} = Backup.create(user)
+ assert {:ok, backup} = perform_job(BackupWorker, args)
assert backup.file_size > 0
assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
+
+ delete_job_args = %{"op" => "delete", "backup_id" => backup_id}
+
+ assert_enqueued(worker: BackupWorker, args: delete_job_args)
+ assert {:ok, backup} = perform_job(BackupWorker, delete_job_args)
+ refute Backup.get(backup_id)
+ end
+
+ test "it removes outdated backups after creating a fresh one" do
+ Pleroma.Config.put([Pleroma.Backup, :limit_days], -1)
+ Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+ user = insert(:user)
+
+ assert {:ok, job1} = Backup.create(user)
+
+ assert {:ok, %Backup{id: backup1_id}} = ObanHelpers.perform(job1)
+ assert {:ok, job2} = Backup.create(user)
+ assert Pleroma.Repo.aggregate(Backup, :count) == 2
+ assert {:ok, backup2} = ObanHelpers.perform(job2)
+
+ ObanHelpers.perform_all()
+
+ assert [^backup2] = Pleroma.Repo.all(Backup)
end
test "it creates a zip archive with user data" do
@@ -145,7 +173,7 @@ defmodule Pleroma.BackupTest do
File.rm!(path)
end
- describe "it uploads a backup archive" do
+ describe "it uploads and deletes a backup archive" do
setup do
clear_config(Pleroma.Uploaders.S3,
bucket: "test_bucket",
@@ -173,8 +201,16 @@ defmodule Pleroma.BackupTest do
test "S3", %{path: path, backup: backup} do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
- with_mock ExAws, request: fn _ -> {:ok, :ok} end do
+ with_mock ExAws,
+ request: fn
+ %{http_method: :put} -> {:ok, :ok}
+ %{http_method: :delete} -> {:ok, %{status_code: 204}}
+ end do
assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
+ assert {:ok, _backup} = Backup.delete(backup)
+ end
+
+ with_mock ExAws, request: fn %{http_method: :delete} -> {:ok, %{status_code: 204}} end do
end
end
@@ -182,6 +218,7 @@ defmodule Pleroma.BackupTest do
Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
+ assert {:ok, _backup} = Backup.delete(backup)
end
end
end
diff --git a/test/support/oban_helpers.ex b/test/support/oban_helpers.ex
@@ -7,6 +7,8 @@ defmodule Pleroma.Tests.ObanHelpers do
Oban test helpers.
"""
+ require Ecto.Query
+
alias Pleroma.Repo
def wipe_all do
@@ -15,6 +17,7 @@ defmodule Pleroma.Tests.ObanHelpers do
def perform_all do
Oban.Job
+ |> Ecto.Query.where(state: "available")
|> Repo.all()
|> perform()
end
