logo

pleroma

My custom branche(s) on git.pleroma.social/pleroma/pleroma git clone https://hacktivis.me/git/pleroma.git
commit: 173e977e283789a814278c63bc81f40a13942e21
parent ebcc28fef07236e6094998c230b10345abf8b307
Author: Haelwenn <contact+git.pleroma.social@hacktivis.me>
Date:   Mon, 12 Jul 2021 05:05:49 +0000

Merge branch 'features/ingestion-page' into 'develop'

Pipeline Ingestion: Page

See merge request pleroma/pleroma!3097

Diffstat:

Mlib/pleroma/web/activity_pub/activity_pub.ex2+-
Mlib/pleroma/web/activity_pub/object_validator.ex15++++++++-------
Alib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex123+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Dlib/pleroma/web/activity_pub/object_validators/article_note_validator.ex123-------------------------------------------------------------------------------
Mlib/pleroma/web/activity_pub/side_effects.ex2+-
Mlib/pleroma/web/activity_pub/transmogrifier.ex62+-------------------------------------------------------------
Atest/fixtures/tesla_mock/lemmy-page.json17+++++++++++++++++
Atest/fixtures/tesla_mock/lemmy-user.json27+++++++++++++++++++++++++++
Atest/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs35+++++++++++++++++++++++++++++++++++
Dtest/pleroma/web/activity_pub/object_validators/article_note_validator_test.exs35-----------------------------------
Atest/pleroma/web/activity_pub/transmogrifier/page_handling_test.exs36++++++++++++++++++++++++++++++++++++
11 files changed, 249 insertions(+), 228 deletions(-)

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -91,7 +91,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp increase_replies_count_if_reply(_create_data), do: :noop - @object_types ~w[ChatMessage Question Answer Audio Video Event Article Note] + @object_types ~w[ChatMessage Question Answer Audio Video Event Article Note Page] @impl true def persist(%{"type" => type} = object, meta) when type in @object_types do with {:ok, object} <- Object.create(object) do diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator @@ -102,7 +102,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity, meta ) - when objtype in ~w[Question Answer Audio Video Event Article Note] do + when objtype in ~w[Question Answer Audio Video Event Article Note Page] do with {:ok, object_data} <- cast_and_apply(object), meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), {:ok, create_activity} <- @@ -115,15 +115,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end def validate(%{"type" => type} = object, meta) - when type in ~w[Event Question Audio Video Article Note] do + when type in ~w[Event Question Audio Video Article Note Page] do validator = case type do "Event" -> EventValidator "Question" -> QuestionValidator "Audio" -> AudioVideoValidator "Video" -> AudioVideoValidator - "Article" -> ArticleNoteValidator - "Note" -> ArticleNoteValidator + "Article" -> ArticleNotePageValidator + "Note" -> ArticleNotePageValidator + "Page" -> ArticleNotePageValidator end with {:ok, object} <- @@ -197,8 +198,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do EventValidator.cast_and_apply(object) end - def cast_and_apply(%{"type" => type} = object) when type in ~w[Article Note] do - ArticleNoteValidator.cast_and_apply(object) + def cast_and_apply(%{"type" => type} = object) when type in ~w[Article Note Page] do + ArticleNotePageValidator.cast_and_apply(object) end def cast_and_apply(o), do: {:error, {:validator_not_set, o}} diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -0,0 +1,123 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do + use Ecto.Schema + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Ecto.Changeset + + @primary_key false + @derive Jason.Encoder + + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:bto, ObjectValidators.Recipients, default: []) + field(:bcc, ObjectValidators.Recipients, default: []) + embeds_many(:tag, TagValidator) + field(:type, :string) + + field(:name, :string) + field(:summary, :string) + field(:content, :string) + + field(:context, :string) + # short identifier for PleromaFE to group statuses by context + field(:context_id, :integer) + + # TODO: Remove actor on objects + field(:actor, ObjectValidators.ObjectID) + + field(:attributedTo, ObjectValidators.ObjectID) + field(:published, ObjectValidators.DateTime) + field(:emoji, ObjectValidators.Emoji, default: %{}) + field(:sensitive, :boolean, default: false) + embeds_many(:attachment, AttachmentValidator) + field(:replies_count, :integer, default: 0) + field(:like_count, :integer, default: 0) + field(:announcement_count, :integer, default: 0) + field(:inReplyTo, ObjectValidators.ObjectID) + field(:url, ObjectValidators.Uri) + + field(:likes, {:array, ObjectValidators.ObjectID}, default: []) + field(:announcements, {:array, ObjectValidators.ObjectID}, default: []) + + field(:replies, {:array, ObjectValidators.ObjectID}, default: []) + end + + def cast_and_apply(data) do + data + |> cast_data + |> apply_action(:insert) + end + + def cast_and_validate(data) do + data + |> cast_data() + |> validate_data() + end + + def cast_data(data) do + %__MODULE__{} + |> changeset(data) + end + + defp fix_url(%{"url" => url} = data) when is_bitstring(url), do: data + defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"]) + defp fix_url(data), do: data + + defp fix_tag(%{"tag" => tag} = data) when is_list(tag), do: data + defp fix_tag(%{"tag" => tag} = data) when is_map(tag), do: Map.put(data, "tag", [tag]) + defp fix_tag(data), do: Map.drop(data, ["tag"]) + + defp fix_replies(%{"replies" => %{"first" => %{"items" => replies}}} = data) + when is_list(replies), + do: Map.put(data, "replies", replies) + + defp fix_replies(%{"replies" => %{"items" => replies}} = data) when is_list(replies), + do: Map.put(data, "replies", replies) + + defp fix_replies(%{"replies" => replies} = data) when is_bitstring(replies), + do: Map.drop(data, ["replies"]) + + defp fix_replies(data), do: data + + defp fix(data) do + data + |> CommonFixes.fix_actor() + |> CommonFixes.fix_object_defaults() + |> fix_url() + |> fix_tag() + |> fix_replies() + |> Transmogrifier.fix_emoji() + |> Transmogrifier.fix_content_map() + end + + def changeset(struct, data) do + data = fix(data) + + struct + |> cast(data, __schema__(:fields) -- [:attachment, :tag]) + |> cast_embed(:attachment) + |> cast_embed(:tag) + end + + defp validate_data(data_cng) do + data_cng + |> validate_inclusion(:type, ["Article", "Note", "Page"]) + |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) + |> CommonValidations.validate_any_presence([:cc, :to]) + |> CommonValidations.validate_fields_match([:actor, :attributedTo]) + |> CommonValidations.validate_actor_presence() + |> CommonValidations.validate_host_match() + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex @@ -1,123 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do - use Ecto.Schema - - alias Pleroma.EctoType.ActivityPub.ObjectValidators - alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes - alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations - alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator - alias Pleroma.Web.ActivityPub.Transmogrifier - - import Ecto.Changeset - - @primary_key false - @derive Jason.Encoder - - embedded_schema do - field(:id, ObjectValidators.ObjectID, primary_key: true) - field(:to, ObjectValidators.Recipients, default: []) - field(:cc, ObjectValidators.Recipients, default: []) - field(:bto, ObjectValidators.Recipients, default: []) - field(:bcc, ObjectValidators.Recipients, default: []) - embeds_many(:tag, TagValidator) - field(:type, :string) - - field(:name, :string) - field(:summary, :string) - field(:content, :string) - - field(:context, :string) - # short identifier for PleromaFE to group statuses by context - field(:context_id, :integer) - - # TODO: Remove actor on objects - field(:actor, ObjectValidators.ObjectID) - - field(:attributedTo, ObjectValidators.ObjectID) - field(:published, ObjectValidators.DateTime) - field(:emoji, ObjectValidators.Emoji, default: %{}) - field(:sensitive, :boolean, default: false) - embeds_many(:attachment, AttachmentValidator) - field(:replies_count, :integer, default: 0) - field(:like_count, :integer, default: 0) - field(:announcement_count, :integer, default: 0) - field(:inReplyTo, ObjectValidators.ObjectID) - field(:url, ObjectValidators.Uri) - - field(:likes, {:array, ObjectValidators.ObjectID}, default: []) - field(:announcements, {:array, ObjectValidators.ObjectID}, default: []) - - field(:replies, {:array, ObjectValidators.ObjectID}, default: []) - end - - def cast_and_apply(data) do - data - |> cast_data - |> apply_action(:insert) - end - - def cast_and_validate(data) do - data - |> cast_data() - |> validate_data() - end - - def cast_data(data) do - %__MODULE__{} - |> changeset(data) - end - - defp fix_url(%{"url" => url} = data) when is_bitstring(url), do: data - defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"]) - defp fix_url(data), do: data - - defp fix_tag(%{"tag" => tag} = data) when is_list(tag), do: data - defp fix_tag(%{"tag" => tag} = data) when is_map(tag), do: Map.put(data, "tag", [tag]) - defp fix_tag(data), do: Map.drop(data, ["tag"]) - - defp fix_replies(%{"replies" => %{"first" => %{"items" => replies}}} = data) - when is_list(replies), - do: Map.put(data, "replies", replies) - - defp fix_replies(%{"replies" => %{"items" => replies}} = data) when is_list(replies), - do: Map.put(data, "replies", replies) - - defp fix_replies(%{"replies" => replies} = data) when is_bitstring(replies), - do: Map.drop(data, ["replies"]) - - defp fix_replies(data), do: data - - defp fix(data) do - data - |> CommonFixes.fix_actor() - |> CommonFixes.fix_object_defaults() - |> fix_url() - |> fix_tag() - |> fix_replies() - |> Transmogrifier.fix_emoji() - |> Transmogrifier.fix_content_map() - end - - def changeset(struct, data) do - data = fix(data) - - struct - |> cast(data, __schema__(:fields) -- [:attachment, :tag]) - |> cast_embed(:attachment) - |> cast_embed(:tag) - end - - defp validate_data(data_cng) do - data_cng - |> validate_inclusion(:type, ["Article", "Note"]) - |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) - |> CommonValidations.validate_any_presence([:cc, :to]) - |> CommonValidations.validate_fields_match([:actor, :attributedTo]) - |> CommonValidations.validate_actor_presence() - |> CommonValidations.validate_host_match() - end -end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex @@ -437,7 +437,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end def handle_object_creation(%{"type" => objtype} = object, meta) - when objtype in ~w[Audio Video Question Event Article Note] do + when objtype in ~w[Audio Video Question Event Article Note Page] do with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do {:ok, object, meta} end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -353,29 +353,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end) end - # Compatibility wrapper for Mastodon votes - defp handle_create(%{"object" => %{"type" => "Answer"}} = data, _user) do - handle_incoming(data) - end - - defp handle_create(%{"object" => object} = data, user) do - %{ - to: data["to"], - object: object, - actor: user, - context: object["context"], - local: false, - published: data["published"], - additional: - Map.take(data, [ - "cc", - "directMessage", - "id" - ]) - } - |> ActivityPub.create() - end - def handle_incoming(data, options \\ []) # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them @@ -407,43 +384,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming(%{"id" => id}, _options) when is_binary(id) and byte_size(id) < 8, do: :error - # TODO: validate those with a Ecto scheme - # - tags - # - emoji - def handle_incoming( - %{"type" => "Create", "object" => %{"type" => "Page"} = object} = data, - options - ) do - actor = Containment.get_actor(data) - - with nil <- Activity.get_create_by_object_ap_id(object["id"]), - {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(actor) do - data = - data - |> Map.put("object", fix_object(object, options)) - |> Map.put("actor", actor) - |> fix_addressing() - - with {:ok, created_activity} <- handle_create(data, user) do - reply_depth = (options[:depth] || 0) + 1 - - if Federator.allowed_thread_distance?(reply_depth) do - for reply_id <- replies(object) do - Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{ - "id" => reply_id, - "depth" => reply_depth - }) - end - end - - {:ok, created_activity} - end - else - %Activity{} = activity -> {:ok, activity} - _e -> :error - end - end - def handle_incoming( %{"type" => "Listen", "object" => %{"type" => "Audio"} = object} = data, options @@ -507,7 +447,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do %{"type" => "Create", "object" => %{"type" => objtype, "id" => obj_id}} = data, options ) - when objtype in ~w{Question Answer ChatMessage Audio Video Event Article Note} do + when objtype in ~w{Question Answer ChatMessage Audio Video Event Article Note Page} do fetch_options = Keyword.put(options, :depth, (options[:depth] || 0) + 1) object = diff --git a/test/fixtures/tesla_mock/lemmy-page.json b/test/fixtures/tesla_mock/lemmy-page.json @@ -0,0 +1,17 @@ +{ + "commentsEnabled": true, + "sensitive": false, + "stickied": false, + "attributedTo": "https://enterprise.lemmy.ml/u/nutomic", + "summary": "Hello Federation!", + "url": "https://enterprise.lemmy.ml/pictrs/image/US52d9DPvf.jpg", + "image": { + "type": "Image", + "url": "https://enterprise.lemmy.ml/pictrs/image/lwFAcXHUjS.jpg" + }, + "published": "2020-09-14T15:03:11.909105+00:00", + "to": "https://enterprise.lemmy.ml/c/main", + "@context": "https://www.w3.org/ns/activitystreams", + "id": "https://enterprise.lemmy.ml/post/3", + "type": "Page" +} diff --git a/test/fixtures/tesla_mock/lemmy-user.json b/test/fixtures/tesla_mock/lemmy-user.json @@ -0,0 +1,27 @@ +{ + "publicKey": { + "id": "https://enterprise.lemmy.ml/u/nutomic#main-key", + "owner": "https://enterprise.lemmy.ml/u/nutomic", + "publicKeyPem": "-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvfwAYPxp1gOk2HcCRoUd\nupoecvmnpzRc5Gu6/N3YQyOyRsrYuiYLNQq2cgM3kcU80ZeEetkwkYgXkRJOKu/b\nBWb7i1zt2tdr5k6lUdW8dfCyjht8ooFPQdov8J3QYHfgBHyUYxuCNfSujryxx2wu\nLQcdjRQa5NIWcomSO8OXmCF5/Yhg2XWCbtnlxEq6Y+AFddr1mAlTOy5pBr5d+xZz\njLw/U3CioNJ79yGi/sJhgp6IyJqtUSoN3b4BgRIEts2QVvn44W1rQy9wCbRYQrO1\nBcB9Wel4k3rJJK8uHg+LpHVMaZppkNaWGkMBhMbzr8qmIlcNWNi7cbMK/p5vyviy\nSwIDAQAB\n-----END PUBLIC KEY-----\n" + }, + "inbox": "https://enterprise.lemmy.ml/u/nutomic/inbox", + "preferredUsername": "Nutomic", + "endpoints": { + "sharedInbox": "https://enterprise.lemmy.ml/inbox" + }, + "summary": "some bio", + "icon": { + "type": "Image", + "url": "https://enterprise.lemmy.ml/pictrs/image/F6Z7QcWZRJ.jpg" + }, + "image": { + "type": "Image", + "url": "https://enterprise.lemmy.ml:/pictrs/image/Q79N9oCDEG.png" + }, + "published": "2020-09-14T14:54:53.080949+00:00", + "updated": "2020-10-14T10:58:28.139178+00:00", + "@context": "https://www.w3.org/ns/activitystreams", + "id": "https://enterprise.lemmy.ml/u/nutomic", + "type": "Person", + "name": "nutomic" +} diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -0,0 +1,35 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest do + use Pleroma.DataCase, async: true + + alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator + alias Pleroma.Web.ActivityPub.Utils + + import Pleroma.Factory + + describe "Notes" do + setup do + user = insert(:user) + + note = %{ + "id" => Utils.generate_activity_id(), + "type" => "Note", + "actor" => user.ap_id, + "to" => [user.follower_address], + "cc" => [], + "content" => "Hellow this is content.", + "context" => "xxx", + "summary" => "a post" + } + + %{user: user, note: note} + end + + test "a basic note validates", %{note: note} do + %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note) + end + end +end diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_validator_test.exs @@ -1,35 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidatorTest do - use Pleroma.DataCase, async: true - - alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator - alias Pleroma.Web.ActivityPub.Utils - - import Pleroma.Factory - - describe "Notes" do - setup do - user = insert(:user) - - note = %{ - "id" => Utils.generate_activity_id(), - "type" => "Note", - "actor" => user.ap_id, - "to" => [user.follower_address], - "cc" => [], - "content" => "Hellow this is content.", - "context" => "xxx", - "summary" => "a post" - } - - %{user: user, note: note} - end - - test "a basic note validates", %{note: note} do - %{valid?: true} = ArticleNoteValidator.cast_and_validate(note) - end - end -end diff --git a/test/pleroma/web/activity_pub/transmogrifier/page_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/page_handling_test.exs @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.PageHandlingTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Object.Fetcher + + test "Lemmy Page" do + Tesla.Mock.mock(fn + %{url: "https://enterprise.lemmy.ml/post/3"} -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: File.read!("test/fixtures/tesla_mock/lemmy-page.json") + } + + %{url: "https://enterprise.lemmy.ml/u/nutomic"} -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: File.read!("test/fixtures/tesla_mock/lemmy-user.json") + } + end) + + {:ok, object} = Fetcher.fetch_object_from_id("https://enterprise.lemmy.ml/post/3") + + assert object.data["summary"] == "Hello Federation!" + assert object.data["published"] == "2020-09-14T15:03:11.909105Z" + + # WAT + assert object.data["url"] == "https://enterprise.lemmy.ml/pictrs/image/US52d9DPvf.jpg" + end +end