logo

pleroma

My custom branche(s) on git.pleroma.social/pleroma/pleroma git clone https://hacktivis.me/git/pleroma.git
commit: 4f3a6337454807f4145bbc1830c3d55dd883d46d
parent 75e07ba206b94155c5210151a49e29a11bce6e50
Author: Egor Kislitsyn <egor@kislitsyn.com>
Date:   Wed,  2 Sep 2020 20:21:33 +0400

Add `backups` table

Diffstat:

Alib/pleroma/backup.ex201+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Dlib/pleroma/export.ex139-------------------------------------------------------------------------------
Apriv/repo/migrations/20200831192323_create_backups.exs17+++++++++++++++++
Atest/backup_test.exs179+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Dtest/export_test.exs151------------------------------------------------------------------------------
5 files changed, 397 insertions(+), 290 deletions(-)

diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex @@ -0,0 +1,201 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Backup do + use Ecto.Schema + + import Ecto.Changeset + import Ecto.Query + + alias Pleroma.Activity + alias Pleroma.Bookmark + alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.ActivityPub.UserView + + schema "backups" do + field(:content_type, :string) + field(:file_name, :string) + field(:file_size, :integer, default: 0) + field(:processed, :boolean, default: false) + + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) + + timestamps() + end + + def create(user) do + with :ok <- validate_limit(user), + {:ok, backup} <- user |> new() |> Repo.insert() do + {:ok, backup} + end + end + + def new(user) do + rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false) + datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) + name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip" + + %__MODULE__{ + user_id: user.id, + content_type: "application/zip", + file_name: name + } + end + + defp validate_limit(user) do + case get_last(user.id) do + %__MODULE__{inserted_at: inserted_at} -> + days = 7 + diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days) + + if diff > days do + :ok + else + {:error, "Last export was less than #{days} days ago"} + end + + nil -> + :ok + end + end + + def get_last(user_id) do + __MODULE__ + |> where(user_id: ^user_id) + |> order_by(desc: :id) + |> limit(1) + |> Repo.one() + end + + def process(%__MODULE__{} = backup) do + with {:ok, zip_file} <- zip(backup), + {:ok, %{size: size}} <- File.stat(zip_file), + {:ok, _upload} <- upload(backup, zip_file) do + backup + |> cast(%{file_size: size, processed: true}, [:file_size, :processed]) + |> Repo.update() + end + end + + @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] + def zip(%__MODULE__{} = backup) do + backup = Repo.preload(backup, :user) + name = String.trim_trailing(backup.file_name, ".zip") + dir = Path.join(System.tmp_dir!(), name) + + with :ok <- File.mkdir(dir), + :ok <- actor(dir, backup.user), + :ok <- statuses(dir, backup.user), + :ok <- likes(dir, backup.user), + :ok <- bookmarks(dir, backup.user), + {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir), + {:ok, _} <- File.rm_rf(dir) do + {:ok, :binary.list_to_bin(zip_path)} + end + end + + def upload(%__MODULE__{} = backup, zip_path) do + uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + + upload = %Pleroma.Upload{ + name: backup.file_name, + tempfile: zip_path, + content_type: backup.content_type, + path: "backups/" <> backup.file_name + } + + with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), + :ok <- File.rm(zip_path) do + {:ok, upload} + end + end + + defp actor(dir, user) do + with {:ok, json} <- + UserView.render("user.json", %{user: user}) + |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) + |> Jason.encode() do + File.write(dir <> "/actor.json", json) + end + end + + defp write_header(file, name) do + IO.write( + file, + """ + { + "@context": "https://www.w3.org/ns/activitystreams", + "id": "#{name}.json", + "type": "OrderedCollection", + "orderedItems": [ + """ + ) + end + + defp write(query, dir, name, fun) do + path = dir <> "/#{name}.json" + + with {:ok, file} <- File.open(path, [:write, :utf8]), + :ok <- write_header(file, name) do + counter = :counters.new(1, []) + + query + |> Pleroma.RepoStreamer.chunk_stream(100) + |> Stream.each(fn items -> + Enum.each(items, fn i -> + with {:ok, str} <- fun.(i), + :ok <- IO.write(file, str <> ",\n") do + :counters.add(counter, 1, 1) + end + end) + end) + |> Stream.run() + + total = :counters.get(counter, 1) + + with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do + File.close(file) + end + end + end + + defp bookmarks(dir, %{id: user_id} = _user) do + Bookmark + |> where(user_id: ^user_id) + |> join(:inner, [b], activity in assoc(b, :activity)) + |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)}) + |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end) + end + + defp likes(dir, user) do + user.ap_id + |> Activity.Queries.by_actor() + |> Activity.Queries.by_type("Like") + |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)}) + |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end) + end + + defp statuses(dir, user) do + opts = + %{} + |> Map.put(:type, ["Create", "Announce"]) + |> Map.put(:blocking_user, user) + |> Map.put(:muting_user, user) + |> Map.put(:reply_filtering_user, user) + |> Map.put(:announce_filtering_user, user) + |> Map.put(:user, user) + + [[user.ap_id], User.following(user), Pleroma.List.memberships(user)] + |> Enum.concat() + |> ActivityPub.fetch_activities_query(opts) + |> write(dir, "outbox", fn a -> + with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do + activity |> Map.delete("@context") |> Jason.encode() + end + end) + end +end diff --git a/lib/pleroma/export.ex b/lib/pleroma/export.ex @@ -1,139 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Export do - alias Pleroma.Activity - alias Pleroma.Bookmark - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Transmogrifier - alias Pleroma.Web.ActivityPub.UserView - - import Ecto.Query - - @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] - - def run(user) do - with {:ok, path} <- create_dir(user), - :ok <- actor(path, user), - :ok <- statuses(path, user), - :ok <- likes(path, user), - :ok <- bookmarks(path, user), - {:ok, zip_path} <- :zip.create('#{path}.zip', @files, cwd: path), - {:ok, _} <- File.rm_rf(path) do - {:ok, :binary.list_to_bin(zip_path)} - end - end - - def upload(zip_path) do - uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) - file_name = zip_path |> String.split("/") |> List.last() - id = Ecto.UUID.generate() - - upload = %Pleroma.Upload{ - id: id, - name: file_name, - tempfile: zip_path, - content_type: "application/zip", - path: id <> "/" <> file_name - } - - with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), - :ok <- File.rm(zip_path) do - {:ok, upload} - end - end - - defp actor(dir, user) do - with {:ok, json} <- - UserView.render("user.json", %{user: user}) - |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) - |> Jason.encode() do - File.write(dir <> "/actor.json", json) - end - end - - defp create_dir(user) do - datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) - dir = Path.join(System.tmp_dir!(), "archive-#{user.id}-#{datetime}") - - with :ok <- File.mkdir(dir), do: {:ok, dir} - end - - defp write_header(file, name) do - IO.write( - file, - """ - { - "@context": "https://www.w3.org/ns/activitystreams", - "id": "#{name}.json", - "type": "OrderedCollection", - "orderedItems": [ - """ - ) - end - - defp write(query, dir, name, fun) do - path = dir <> "/#{name}.json" - - with {:ok, file} <- File.open(path, [:write, :utf8]), - :ok <- write_header(file, name) do - counter = :counters.new(1, []) - - query - |> Pleroma.RepoStreamer.chunk_stream(100) - |> Stream.each(fn items -> - Enum.each(items, fn i -> - with {:ok, str} <- fun.(i), - :ok <- IO.write(file, str <> ",\n") do - :counters.add(counter, 1, 1) - end - end) - end) - |> Stream.run() - - total = :counters.get(counter, 1) - - with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do - File.close(file) - end - end - end - - defp bookmarks(dir, %{id: user_id} = _user) do - Bookmark - |> where(user_id: ^user_id) - |> join(:inner, [b], activity in assoc(b, :activity)) - |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)}) - |> write(dir, "bookmarks", fn a -> {:ok, "\"#{a.object}\""} end) - end - - defp likes(dir, user) do - user.ap_id - |> Activity.Queries.by_actor() - |> Activity.Queries.by_type("Like") - |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)}) - |> write(dir, "likes", fn a -> {:ok, "\"#{a.object}\""} end) - end - - defp statuses(dir, user) do - opts = - %{} - |> Map.put(:type, ["Create", "Announce"]) - |> Map.put(:blocking_user, user) - |> Map.put(:muting_user, user) - |> Map.put(:reply_filtering_user, user) - |> Map.put(:announce_filtering_user, user) - |> Map.put(:user, user) - - [[user.ap_id], User.following(user), Pleroma.List.memberships(user)] - |> Enum.concat() - |> ActivityPub.fetch_activities_query(opts) - |> write(dir, "outbox", fn a -> - with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do - activity |> Map.delete("@context") |> Jason.encode() - end - end) - end -end diff --git a/priv/repo/migrations/20200831192323_create_backups.exs b/priv/repo/migrations/20200831192323_create_backups.exs @@ -0,0 +1,17 @@ +defmodule Pleroma.Repo.Migrations.CreateBackups do + use Ecto.Migration + + def change do + create_if_not_exists table(:backups) do + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + add(:file_name, :string, null: false) + add(:content_type, :string, null: false) + add(:processed, :boolean, null: false, default: false) + add(:file_size, :bigint) + + timestamps() + end + + create_if_not_exists(index(:backups, [:user_id])) + end +end diff --git a/test/backup_test.exs b/test/backup_test.exs @@ -0,0 +1,179 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.BackupTest do + use Pleroma.DataCase + import Pleroma.Factory + import Mock + + alias Pleroma.Backup + alias Pleroma.Bookmark + alias Pleroma.Web.CommonAPI + + test "it creates a backup record" do + %{id: user_id} = user = insert(:user) + assert {:ok, backup} = Backup.create(user) + + assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup + end + + test "it return an error if the export limit is over" do + %{id: user_id} = user = insert(:user) + limit_days = 7 + + assert {:ok, backup} = Backup.create(user) + assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup + + assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"} + end + + test "it process a backup record" do + %{id: user_id} = user = insert(:user) + assert {:ok, %{id: backup_id} = backup} = Backup.create(user) + assert {:ok, %Backup{} = backup} = Backup.process(backup) + assert backup.file_size > 0 + assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup + end + + test "it creates a zip archive with user data" do + user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + + {:ok, %{object: %{data: %{"id" => id1}}} = status1} = + CommonAPI.post(user, %{status: "status1"}) + + {:ok, %{object: %{data: %{"id" => id2}}} = status2} = + CommonAPI.post(user, %{status: "status2"}) + + {:ok, %{object: %{data: %{"id" => id3}}} = status3} = + CommonAPI.post(user, %{status: "status3"}) + + CommonAPI.favorite(user, status1.id) + CommonAPI.favorite(user, status2.id) + + Bookmark.create(user.id, status2.id) + Bookmark.create(user.id, status3.id) + + assert {:ok, backup} = user |> Backup.new() |> Repo.insert() + assert {:ok, path} = Backup.zip(backup) + assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory]) + assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) + + assert %{ + "@context" => [ + "https://www.w3.org/ns/activitystreams", + "http://localhost:4001/schemas/litepub-0.1.jsonld", + %{"@language" => "und"} + ], + "bookmarks" => "bookmarks.json", + "followers" => "http://cofe.io/users/cofe/followers", + "following" => "http://cofe.io/users/cofe/following", + "id" => "http://cofe.io/users/cofe", + "inbox" => "http://cofe.io/users/cofe/inbox", + "likes" => "likes.json", + "name" => "Cofe", + "outbox" => "http://cofe.io/users/cofe/outbox", + "preferredUsername" => "cofe", + "publicKey" => %{ + "id" => "http://cofe.io/users/cofe#main-key", + "owner" => "http://cofe.io/users/cofe" + }, + "type" => "Person", + "url" => "http://cofe.io/users/cofe" + } = Jason.decode!(json) + + assert {:ok, {'outbox.json', json}} = :zip.zip_get('outbox.json', zipfile) + + assert %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "outbox.json", + "orderedItems" => [ + %{ + "object" => %{ + "actor" => "http://cofe.io/users/cofe", + "content" => "status1", + "type" => "Note" + }, + "type" => "Create" + }, + %{ + "object" => %{ + "actor" => "http://cofe.io/users/cofe", + "content" => "status2" + } + }, + %{ + "actor" => "http://cofe.io/users/cofe", + "object" => %{ + "content" => "status3" + } + } + ], + "totalItems" => 3, + "type" => "OrderedCollection" + } = Jason.decode!(json) + + assert {:ok, {'likes.json', json}} = :zip.zip_get('likes.json', zipfile) + + assert %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "likes.json", + "orderedItems" => [^id1, ^id2], + "totalItems" => 2, + "type" => "OrderedCollection" + } = Jason.decode!(json) + + assert {:ok, {'bookmarks.json', json}} = :zip.zip_get('bookmarks.json', zipfile) + + assert %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "id" => "bookmarks.json", + "orderedItems" => [^id2, ^id3], + "totalItems" => 2, + "type" => "OrderedCollection" + } = Jason.decode!(json) + + :zip.zip_close(zipfile) + File.rm!(path) + end + + describe "it uploads a backup archive" do + setup do + clear_config(Pleroma.Uploaders.S3, + bucket: "test_bucket", + public_endpoint: "https://s3.amazonaws.com" + ) + + clear_config([Pleroma.Upload, :uploader]) + + user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + + {:ok, status1} = CommonAPI.post(user, %{status: "status1"}) + {:ok, status2} = CommonAPI.post(user, %{status: "status2"}) + {:ok, status3} = CommonAPI.post(user, %{status: "status3"}) + CommonAPI.favorite(user, status1.id) + CommonAPI.favorite(user, status2.id) + Bookmark.create(user.id, status2.id) + Bookmark.create(user.id, status3.id) + + assert {:ok, backup} = user |> Backup.new() |> Repo.insert() + assert {:ok, path} = Backup.zip(backup) + + [path: path, backup: backup] + end + + test "S3", %{path: path, backup: backup} do + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3) + + with_mock ExAws, request: fn _ -> {:ok, :ok} end do + assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) + end + end + + test "Local", %{path: path, backup: backup} do + Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + + assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path) + end + end +end diff --git a/test/export_test.exs b/test/export_test.exs @@ -1,151 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.ExportTest do - use Pleroma.DataCase - import Pleroma.Factory - import Mock - - alias Pleroma.Bookmark - alias Pleroma.Web.CommonAPI - - test "it exports user data" do - user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) - - {:ok, %{object: %{data: %{"id" => id1}}} = status1} = - CommonAPI.post(user, %{status: "status1"}) - - {:ok, %{object: %{data: %{"id" => id2}}} = status2} = - CommonAPI.post(user, %{status: "status2"}) - - {:ok, %{object: %{data: %{"id" => id3}}} = status3} = - CommonAPI.post(user, %{status: "status3"}) - - CommonAPI.favorite(user, status1.id) - CommonAPI.favorite(user, status2.id) - - Bookmark.create(user.id, status2.id) - Bookmark.create(user.id, status3.id) - - assert {:ok, path} = Pleroma.Export.run(user) - assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory]) - assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) - - assert %{ - "@context" => [ - "https://www.w3.org/ns/activitystreams", - "http://localhost:4001/schemas/litepub-0.1.jsonld", - %{"@language" => "und"} - ], - "bookmarks" => "bookmarks.json", - "followers" => "http://cofe.io/users/cofe/followers", - "following" => "http://cofe.io/users/cofe/following", - "id" => "http://cofe.io/users/cofe", - "inbox" => "http://cofe.io/users/cofe/inbox", - "likes" => "likes.json", - "name" => "Cofe", - "outbox" => "http://cofe.io/users/cofe/outbox", - "preferredUsername" => "cofe", - "publicKey" => %{ - "id" => "http://cofe.io/users/cofe#main-key", - "owner" => "http://cofe.io/users/cofe" - }, - "type" => "Person", - "url" => "http://cofe.io/users/cofe" - } = Jason.decode!(json) - - assert {:ok, {'outbox.json', json}} = :zip.zip_get('outbox.json', zipfile) - - assert %{ - "@context" => "https://www.w3.org/ns/activitystreams", - "id" => "outbox.json", - "orderedItems" => [ - %{ - "object" => %{ - "actor" => "http://cofe.io/users/cofe", - "content" => "status1", - "type" => "Note" - }, - "type" => "Create" - }, - %{ - "object" => %{ - "actor" => "http://cofe.io/users/cofe", - "content" => "status2" - } - }, - %{ - "actor" => "http://cofe.io/users/cofe", - "object" => %{ - "content" => "status3" - } - } - ], - "totalItems" => 3, - "type" => "OrderedCollection" - } = Jason.decode!(json) - - assert {:ok, {'likes.json', json}} = :zip.zip_get('likes.json', zipfile) - - assert %{ - "@context" => "https://www.w3.org/ns/activitystreams", - "id" => "likes.json", - "orderedItems" => [^id1, ^id2], - "totalItems" => 2, - "type" => "OrderedCollection" - } = Jason.decode!(json) - - assert {:ok, {'bookmarks.json', json}} = :zip.zip_get('bookmarks.json', zipfile) - - assert %{ - "@context" => "https://www.w3.org/ns/activitystreams", - "id" => "bookmarks.json", - "orderedItems" => [^id2, ^id3], - "totalItems" => 2, - "type" => "OrderedCollection" - } = Jason.decode!(json) - - :zip.zip_close(zipfile) - File.rm!(path) - end - - describe "it uploads an exported backup archive" do - setup do - clear_config(Pleroma.Uploaders.S3, - bucket: "test_bucket", - public_endpoint: "https://s3.amazonaws.com" - ) - - clear_config([Pleroma.Upload, :uploader]) - - user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) - - {:ok, status1} = CommonAPI.post(user, %{status: "status1"}) - {:ok, status2} = CommonAPI.post(user, %{status: "status2"}) - {:ok, status3} = CommonAPI.post(user, %{status: "status3"}) - CommonAPI.favorite(user, status1.id) - CommonAPI.favorite(user, status2.id) - Bookmark.create(user.id, status2.id) - Bookmark.create(user.id, status3.id) - - assert {:ok, path} = Pleroma.Export.run(user) - - [path: path] - end - - test "S3", %{path: path} do - Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3) - - with_mock ExAws, request: fn _ -> {:ok, :ok} end do - assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) - end - end - - test "Local", %{path: path} do - Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) - - assert {:ok, %Pleroma.Upload{}} = Pleroma.Export.upload(path) - end - end -end