commit: d02805c6466dfd937c9ad486a1f01f4a9d39cd48 parent 44b2a137d12a4ded75586f919ad1c927c6677a70 Author: Haelwenn (lanodan) Monnier <contact@hacktivis.me> Date: Wed, 21 Oct 2020 01:57:43 +0200 Merge branch 'features/validators-note-hashtag'Diffstat:
48 files changed, 891 insertions(+), 703 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md @@ -18,6 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - **Breaking** Requires `libmagic` (or `file`) to guess file types. - **Breaking:** Pleroma Admin API: emoji packs and files routes changed. - **Breaking:** Sensitive/NSFW statuses no longer disable link previews. +- **Breaking:** Changed storage of hashtags in plain-text to `object->hashtags`, run [`pleroma.database fill_old_hashtags` mix task](docs/administration/CLI_tasks/database.md) for old objects (works while pleroma is running). - Search: Users are now findable by their urls. - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated. - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated. @@ -348,7 +349,6 @@ switched to a new configuration mechanism, however it was not officially removed - Static-FE: Fix remote posts not being sanitized ### Fixed -======= - Rate limiter crashes when there is no explicitly specified ip in the config - 500 errors when no `Accept` header is present if Static-FE is enabled - Instance panel not being updated immediately due to wrong `Cache-Control` headers diff --git a/docs/administration/CLI_tasks/database.md b/docs/administration/CLI_tasks/database.md @@ -91,6 +91,16 @@ Can be safely re-run mix pleroma.database fix_likes_collections ``` +## Fill hashtags for old objects + +```sh tab="OTP" +./bin/pleroma_ctl database fill_old_hashtags +``` + +```sh tab="From Source" +mix pleroma.database fill_old_hashtags +``` + ## Vacuum the database ### Analyze diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex @@ -122,6 +122,39 @@ defmodule Mix.Tasks.Pleroma.Database do |> Stream.run() end + def run(["fill_old_hashtags"]) do + import Ecto.Query + + start_pleroma() + + from( + o in Object, + where: fragment("(?)->>'hashtags' is null", o.data), + where: fragment("(?)->>'tag' != '[]'", o.data), + select: %{id: o.id, tag: fragment("(?)->>'tag'", o.data)} + ) + |> Pleroma.Repo.chunk_stream(200, :batches) + |> Stream.each(fn objects -> + Repo.transaction(fn -> + objects + |> Enum.map(fn object -> + tags = + object.tag + |> Jason.decode!() + |> Enum.filter(&is_bitstring(&1)) + + Object + |> where([o], o.id == ^object.id) + |> update([o], + set: [data: fragment("safe_jsonb_set(?, '{hashtags}', ?, true)", o.data, ^tags)] + ) + |> Repo.update_all([], timeout: :infinity) + end) + end) + end) + |> Stream.run() + end + def run(["vacuum", args]) do start_pleroma() diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex @@ -47,6 +47,10 @@ defmodule Pleroma.Activity.Ir.Topics do tags end + defp hashtags_to_topics(%{data: %{"hashtags" => tags}}) do + Enum.map(tags, fn tag -> "hashtag:" <> tag end) + end + defp hashtags_to_topics(%{data: %{"tag" => tags}}) do tags |> Enum.filter(&is_bitstring(&1)) diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex @@ -18,7 +18,8 @@ defmodule Pleroma.Constants do "emoji", "context_id", "deleted_activity_id", - "pleroma_internal" + "pleroma_internal", + "hashtags" ] ) diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex @@ -13,21 +13,33 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients do cast([object]) end + def cast(object) when is_map(object) do + case ObjectID.cast(object) do + {:ok, data} -> {:ok, [data]} + _ -> :error + end + end + def cast(data) when is_list(data) do - data - |> Enum.reduce_while({:ok, []}, fn element, {:ok, list} -> - case ObjectID.cast(element) do - {:ok, id} -> - {:cont, {:ok, [id | list]}} - - _ -> - {:halt, :error} - end - end) + data = + data + |> Enum.reduce_while([], fn element, list -> + case ObjectID.cast(element) do + {:ok, id} -> + {:cont, [id | list]} + + _ -> + {:cont, list} + end + end) + |> Enum.sort() + |> Enum.uniq() + + {:ok, data} end - def cast(_) do - :error + def cast(data) do + {:error, data} end def dump(data) do diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Object.Fetcher do alias Pleroma.HTTP + alias Pleroma.Maps alias Pleroma.Object alias Pleroma.Object.Containment alias Pleroma.Repo @@ -102,6 +103,9 @@ defmodule Pleroma.Object.Fetcher do {:transmogrifier, {:error, {:reject, e}}} -> {:reject, e} + {:transmogrifier, {:reject, e}} -> + {:reject, e} + {:transmogrifier, _} = e -> {:error, e} @@ -125,12 +129,14 @@ defmodule Pleroma.Object.Fetcher do defp prepare_activity_params(data) do %{ "type" => "Create", - "to" => data["to"] || [], - "cc" => data["cc"] || [], # Should we seriously keep this attributedTo thing? "actor" => data["actor"] || data["attributedTo"], "object" => data } + |> Maps.put_if_present("to", data["to"]) + |> Maps.put_if_present("cc", data["cc"]) + |> Maps.put_if_present("bto", data["bto"]) + |> Maps.put_if_present("bcc", data["bcc"]) end def fetch_object_from_id!(id, options \\ []) do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -84,7 +84,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp increase_replies_count_if_reply(_create_data), do: :noop - @object_types ~w[ChatMessage Question Answer Audio Video Event Article] + @object_types ~w[ChatMessage Question Answer Audio Video Event Article Note] @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()} def persist(%{"type" => type} = object, meta) when type in @object_types do with {:ok, object} <- Object.create(object) do @@ -655,7 +655,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do from( [_activity, object] in query, - where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject) + where: fragment("not (?)->'hashtags' \\?| (?)", object.data, ^tag_reject) ) end @@ -668,7 +668,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do from( [_activity, object] in query, - where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all) + where: fragment("(?)->'hashtags' \\?& (?)", object.data, ^tag_all) ) end @@ -681,14 +681,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_tag(query, %{tag: tag}) when is_list(tag) do from( [_activity, object] in query, - where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag) + where: fragment("(?)->'hashtags' \\?| (?)", object.data, ^tag) ) end defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do from( [_activity, object] in query, - where: fragment("(?)->'tag' \\? (?)", object.data, ^tag) + where: fragment("(?)->'hashtags' \\? (?)", object.data, ^tag) ) end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -74,9 +74,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do object = if MRF.subdomain_match?(media_nsfw, actor_host) do - tags = (child_object["tag"] || []) ++ ["nsfw"] - child_object = Map.put(child_object, "tag", tags) - child_object = Map.put(child_object, "sensitive", true) + child_object = + child_object + |> Map.put("hashtags", (child_object["hashtags"] || []) ++ ["nsfw"]) + |> Map.put("sensitive", true) + Map.put(object, "object", child_object) else object diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex @@ -161,7 +161,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end end - def validate(%{"type" => "Article"} = object, meta) do + def validate(%{"type" => type} = object, meta) when type in ~w[Article Note] do with {:ok, object} <- object |> ArticleNoteValidator.cast_and_validate() @@ -210,7 +210,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity, meta ) - when objtype in ~w[Question Answer Audio Video Event Article] do + when objtype in ~w[Question Answer Audio Video Event Article Note] do with {:ok, object_data} <- cast_and_apply(object), meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), {:ok, create_activity} <- @@ -252,7 +252,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do EventValidator.cast_and_apply(object) end - def cast_and_apply(%{"type" => "Article"} = object) do + def cast_and_apply(%{"type" => type} = object) when type in ~w[Article Note] do ArticleNoteValidator.cast_and_apply(object) end diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do use Ecto.Schema alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations import Ecto.Changeset @@ -23,6 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do field(:name, :string) field(:inReplyTo, ObjectValidators.ObjectID) field(:attributedTo, ObjectValidators.ObjectID) + field(:context, :string) # TODO: Remove actor on objects field(:actor, ObjectValidators.ObjectID) @@ -46,6 +48,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do end def changeset(struct, data) do + data = + data + |> CommonFixes.fix_actor() + |> CommonFixes.fix_object_defaults() + struct |> cast(data, __schema__(:fields)) end diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex @@ -24,6 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do field(:bcc, ObjectValidators.Recipients, default: []) # TODO: Write type field(:tag, {:array, :map}, default: []) + field(:hashtags, {:array, :string}, default: []) field(:type, :string) field(:name, :string) @@ -50,6 +51,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do field(:likes, {:array, ObjectValidators.ObjectID}, default: []) field(:announcements, {:array, ObjectValidators.ObjectID}, default: []) + + field(:replies, {:array, ObjectValidators.ObjectID}, default: []) end def cast_and_apply(data) do @@ -65,25 +68,39 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do end def cast_data(data) do - data = fix(data) - %__MODULE__{} |> changeset(data) end - defp fix_url(%{"url" => url} = data) when is_map(url) do - Map.put(data, "url", url["href"]) - end - + defp fix_url(%{"url" => url} = data) when is_bitstring(url), do: data + defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"]) defp fix_url(data), do: data + defp fix_tag(%{"tag" => tag} = data) when is_list(tag), do: data + defp fix_tag(%{"tag" => tag} = data) when is_map(tag), do: Map.put(data, "tag", [tag]) + defp fix_tag(data), do: Map.drop(data, ["tag"]) + + defp fix_replies(%{"replies" => %{"first" => %{"items" => replies}}} = data) + when is_list(replies), + do: Map.put(data, "replies", replies) + + defp fix_replies(%{"replies" => %{"items" => replies}} = data) when is_list(replies), + do: Map.put(data, "replies", replies) + + defp fix_replies(%{"replies" => replies} = data) when is_bitstring(replies), + do: Map.drop(data, ["replies"]) + + defp fix_replies(data), do: data + defp fix(data) do data - |> CommonFixes.fix_defaults() - |> CommonFixes.fix_attribution() |> CommonFixes.fix_actor() + |> CommonFixes.fix_object_defaults() |> fix_url() + |> fix_tag() + |> fix_replies() |> Transmogrifier.fix_emoji() + |> Transmogrifier.fix_content_map() end def changeset(struct, data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -25,6 +25,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do field(:bcc, ObjectValidators.Recipients, default: []) # TODO: Write type field(:tag, {:array, :map}, default: []) + field(:hashtags, {:array, :string}, default: []) field(:type, :string) field(:name, :string) @@ -106,9 +107,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do defp fix(data) do data - |> CommonFixes.fix_defaults() - |> CommonFixes.fix_attribution() |> CommonFixes.fix_actor() + |> CommonFixes.fix_object_defaults() |> Transmogrifier.fix_emoji() |> fix_url() |> fix_content() diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -3,26 +3,56 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object.Containment + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils - # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults - def fix_defaults(data) do + def cast_and_filter_recipients(message, field, follower_collection, field_fallback \\ []) do + {:ok, data} = ObjectValidators.Recipients.cast(message[field] || field_fallback) + + data = + Enum.reject(data, fn x -> + String.ends_with?(x, "/followers") and x != follower_collection + end) + + Map.put(message, field, data) + end + + def fix_object_defaults(data) do %{data: %{"id" => context}, id: context_id} = Utils.create_context(data["context"] || data["conversation"]) + %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"]) + data |> Map.put("context", context) |> Map.put("context_id", context_id) + |> cast_and_filter_recipients("to", follower_collection) + |> cast_and_filter_recipients("cc", follower_collection) + |> cast_and_filter_recipients("bto", follower_collection) + |> cast_and_filter_recipients("bcc", follower_collection) + |> Transmogrifier.fix_implicit_addressing(follower_collection) + |> Transmogrifier.fix_tag() end - def fix_attribution(data) do - data - |> Map.put_new("actor", data["attributedTo"]) + def fix_activity_addressing(activity, _meta) do + %User{follower_address: follower_collection} = User.get_cached_by_ap_id(activity["actor"]) + + activity + |> cast_and_filter_recipients("to", follower_collection) + |> cast_and_filter_recipients("cc", follower_collection) + |> cast_and_filter_recipients("bto", follower_collection) + |> cast_and_filter_recipients("bcc", follower_collection) + |> Transmogrifier.fix_implicit_addressing(follower_collection) end def fix_actor(data) do - actor = Containment.get_actor(data) + actor = + data + |> Map.put_new("actor", data["attributedTo"]) + |> Containment.get_actor() data |> Map.put("actor", actor) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do fields |> Enum.map(fn field -> get_field(cng, field) end) |> Enum.any?(fn + nil -> false [] -> false _ -> true end) diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -10,8 +10,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object + alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -23,6 +25,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do field(:type, :string) field(:to, ObjectValidators.Recipients, default: []) field(:cc, ObjectValidators.Recipients, default: []) + field(:bto, ObjectValidators.Recipients, default: []) + field(:bcc, ObjectValidators.Recipients, default: []) field(:object, ObjectValidators.ObjectID) field(:expires_at, ObjectValidators.DateTime) @@ -54,39 +58,37 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do |> cast(data, __schema__(:fields)) end - defp fix_context(data, meta) do - if object = meta[:object_data] do - Map.put_new(data, "context", object["context"]) - else - data - end - end + # CommonFixes.fix_activity_addressing adapted for Create specific behavior + defp fix_addressing(data, object) do + %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["actor"]) - defp fix_addressing(data, meta) do - if object = meta[:object_data] do - data - |> Map.put_new("to", object["to"] || []) - |> Map.put_new("cc", object["cc"] || []) - else - data - end + data + |> CommonFixes.cast_and_filter_recipients("to", follower_collection, object["to"]) + |> CommonFixes.cast_and_filter_recipients("cc", follower_collection, object["cc"]) + |> CommonFixes.cast_and_filter_recipients("bto", follower_collection, object["bto"]) + |> CommonFixes.cast_and_filter_recipients("bcc", follower_collection, object["bcc"]) + |> Transmogrifier.fix_implicit_addressing(follower_collection) end - defp fix(data, meta) do + def fix(data, meta) do + object = meta[:object_data] + data - |> fix_context(meta) - |> fix_addressing(meta) |> CommonFixes.fix_actor() + |> Map.put_new("context", object["context"]) + |> fix_addressing(object) end - def validate_data(cng, meta \\ []) do + def validate_data(cng, meta) do + object = meta[:object_data] + cng - |> validate_required([:actor, :type, :object]) + |> validate_required([:actor, :type, :object, :to, :cc]) |> validate_inclusion(:type, ["Create"]) |> CommonValidations.validate_actor_presence() - |> CommonValidations.validate_any_presence([:to, :cc]) - |> validate_actors_match(meta) - |> validate_context_match(meta) + |> validate_actors_match(object) + |> validate_context_match(object) + |> validate_addressing_match(object) |> validate_object_nonexistence() |> validate_object_containment() end @@ -118,8 +120,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do end) end - def validate_actors_match(cng, meta) do - attributed_to = meta[:object_data]["attributedTo"] || meta[:object_data]["actor"] + def validate_actors_match(cng, object) do + attributed_to = object["attributedTo"] || object["actor"] cng |> validate_change(:actor, fn :actor, actor -> @@ -131,7 +133,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do end) end - def validate_context_match(cng, %{object_data: %{"context" => object_context}}) do + def validate_context_match(cng, %{"context" => object_context}) do cng |> validate_change(:context, fn :context, context -> if context == object_context do @@ -142,5 +144,18 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do end) end - def validate_context_match(cng, _), do: cng + def validate_addressing_match(cng, object) do + [:to, :cc, :bcc, :bto] + |> Enum.reduce(cng, fn field, cng -> + object_data = object[to_string(field)] + + validate_change(cng, field, fn field, data -> + if data == object_data do + [] + else + [{field, "field doesn't match with object (#{inspect(object_data)})"}] + end + end) + end) + end end diff --git a/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex @@ -1,29 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do - use Ecto.Schema - - alias Pleroma.EctoType.ActivityPub.ObjectValidators - alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator - - import Ecto.Changeset - - @primary_key false - - embedded_schema do - field(:id, ObjectValidators.ObjectID, primary_key: true) - field(:actor, ObjectValidators.ObjectID) - field(:type, :string) - field(:to, ObjectValidators.Recipients, default: []) - field(:cc, ObjectValidators.Recipients, default: []) - field(:bto, ObjectValidators.Recipients, default: []) - field(:bcc, ObjectValidators.Recipients, default: []) - embeds_one(:object, NoteValidator) - end - - def cast_data(data) do - cast(%__MODULE__{}, data, __schema__(:fields)) - end -end diff --git a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex @@ -25,6 +25,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do field(:bcc, ObjectValidators.Recipients, default: []) # TODO: Write type field(:tag, {:array, :map}, default: []) + field(:hashtags, {:array, :string}, default: []) field(:type, :string) field(:name, :string) @@ -72,8 +73,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do defp fix(data) do data - |> CommonFixes.fix_defaults() - |> CommonFixes.fix_attribution() + |> CommonFixes.fix_actor() + |> CommonFixes.fix_object_defaults() |> Transmogrifier.fix_emoji() end diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -26,6 +26,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do field(:bcc, ObjectValidators.Recipients, default: []) # TODO: Write type field(:tag, {:array, :map}, default: []) + field(:hashtags, {:array, :string}, default: []) field(:type, :string) field(:content, :string) field(:context, :string) @@ -83,8 +84,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do defp fix(data) do data - |> CommonFixes.fix_defaults() - |> CommonFixes.fix_attribution() + |> CommonFixes.fix_actor() + |> CommonFixes.fix_object_defaults() |> Transmogrifier.fix_emoji() |> fix_closed() end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex @@ -191,6 +191,19 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Object.increase_replies_count(in_reply_to) end + reply_depth = (meta[:depth] || 0) + 1 + + # FIXME: Force inReplyTo to replies + if Pleroma.Web.Federator.allowed_thread_distance?(reply_depth) and + object.data["replies"] != nil do + for reply_id <- object.data["replies"] do + Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{ + "id" => reply_id, + "depth" => reply_depth + }) + end + end + BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id}) meta = @@ -340,7 +353,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end def handle_object_creation(%{"type" => objtype} = object, meta) - when objtype in ~w[Audio Video Question Event Article] do + when objtype in ~w[Audio Video Question Event Article Note] do with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do {:ok, object, meta} end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -43,7 +43,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> fix_content_map |> fix_addressing |> fix_summary - |> fix_type(options) end def fix_summary(%{"summary" => nil} = object) do @@ -72,17 +71,21 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def fix_explicit_addressing( - %{"to" => to, "cc" => cc} = object, - explicit_mentions, - follower_collection - ) do - explicit_to = Enum.filter(to, fn x -> x in explicit_mentions end) + # if directMessage flag is set to true, leave the addressing alone + def fix_explicit_addressing(%{"directMessage" => true} = object, _follower_collection), + do: object + + def fix_explicit_addressing(%{"to" => to, "cc" => cc} = object, follower_collection) do + explicit_mentions = + Utils.determine_explicit_mentions(object) ++ + [Pleroma.Constants.as_public(), follower_collection] + explicit_to = Enum.filter(to, fn x -> x in explicit_mentions end) explicit_cc = Enum.filter(to, fn x -> x not in explicit_mentions end) final_cc = (cc ++ explicit_cc) + |> Enum.filter(& &1) |> Enum.reject(fn x -> String.ends_with?(x, "/followers") and x != follower_collection end) |> Enum.uniq() @@ -91,29 +94,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.put("cc", final_cc) end - def fix_explicit_addressing(object, _explicit_mentions, _followers_collection), do: object - - # if directMessage flag is set to true, leave the addressing alone - def fix_explicit_addressing(%{"directMessage" => true} = object), do: object - - def fix_explicit_addressing(object) do - explicit_mentions = Utils.determine_explicit_mentions(object) - - %User{follower_address: follower_collection} = - object - |> Containment.get_actor() - |> User.get_cached_by_ap_id() - - explicit_mentions = - explicit_mentions ++ - [ - Pleroma.Constants.as_public(), - follower_collection - ] - - fix_explicit_addressing(object, explicit_mentions, follower_collection) - end - # if as:Public is addressed, then make sure the followers collection is also addressed # so that the activities will be delivered to local users. def fix_implicit_addressing(%{"to" => to, "cc" => cc} = object, followers_collection) do @@ -137,19 +117,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def fix_implicit_addressing(object, _), do: object - def fix_addressing(object) do - {:ok, %User{} = user} = User.get_or_fetch_by_ap_id(object["actor"]) - followers_collection = User.ap_followers(user) + {:ok, %User{follower_address: follower_collection}} = + object + |> Containment.get_actor() + |> User.get_or_fetch_by_ap_id() object |> fix_addressing_list("to") |> fix_addressing_list("cc") |> fix_addressing_list("bto") |> fix_addressing_list("bcc") - |> fix_explicit_addressing() - |> fix_implicit_addressing(followers_collection) + |> fix_explicit_addressing(follower_collection) + |> fix_implicit_addressing(follower_collection) end def fix_actor(%{"attributedTo" => actor} = object) do @@ -310,18 +290,16 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_emoji(object), do: object def fix_tag(%{"tag" => tag} = object) when is_list(tag) do - tags = + hashtags = tag |> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end) - |> Enum.map(fn data -> String.slice(data["name"], 1..-1) end) + |> Enum.map(fn %{"name" => hashtag} -> String.trim(hashtag, "#") end) - Map.put(object, "tag", tag ++ tags) + Map.put(object, "hashtags", hashtags) end - def fix_tag(%{"tag" => %{"type" => "Hashtag", "name" => hashtag} = tag} = object) do - combined = [tag, String.slice(hashtag, 1..-1)] - - Map.put(object, "tag", combined) + def fix_tag(%{"tag" => %{"type" => "Hashtag", "name" => hashtag}} = object) do + Map.put(object, "hashtags", [String.trim(hashtag, "#")]) end def fix_tag(%{"tag" => %{} = tag} = object), do: Map.put(object, "tag", [tag]) @@ -338,19 +316,16 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_content_map(object), do: object - def fix_type(object, options \\ []) - - def fix_type(%{"inReplyTo" => reply_id, "name" => _} = object, options) - when is_binary(reply_id) do - with true <- Federator.allowed_thread_distance?(options[:depth]), - {:ok, %{data: %{"type" => "Question"} = _} = _} <- get_obj_helper(reply_id, options) do + defp fix_type(%{"type" => "Note", "inReplyTo" => reply_id, "name" => _} = object, options) + when is_binary(reply_id) do + with %Object{data: %{"type" => "Question"}} <- Object.normalize(reply_id, true, options) do Map.put(object, "type", "Answer") else _ -> object end end - def fix_type(object, _), do: object + defp fix_type(object, _options), do: object # Reduce the object list to find the reported user. defp get_reported(objects) do @@ -421,10 +396,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do # - tags # - emoji def handle_incoming( - %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, + %{"type" => "Create", "object" => %{"type" => "Page"} = object} = data, options - ) - when objtype in ~w{Note Page} do + ) do actor = Containment.get_actor(data) with nil <- Activity.get_create_by_object_ap_id(object["id"]), @@ -516,14 +490,23 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming( %{"type" => "Create", "object" => %{"type" => objtype, "id" => obj_id}} = data, - _options + options ) - when objtype in ~w{Question Answer ChatMessage Audio Video Event Article} do - data = Map.put(data, "object", strip_internal_fields(data["object"])) + when objtype in ~w{Question Answer ChatMessage Audio Video Event Article Note} do + fetch_options = Keyword.put(options, :depth, (options[:depth] || 0) + 1) + + object = + data["object"] + |> strip_internal_fields() + |> fix_type(fetch_options) + |> fix_in_reply_to(fetch_options) + + data = Map.put(data, "object", object) + options = Keyword.put(options, :local, false) with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), nil <- Activity.get_create_by_object_ap_id(obj_id), - {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do + {:ok, activity, _} <- Pipeline.common_pipeline(data, options) do {:ok, activity} else %Activity{} = activity -> {:ok, activity} @@ -861,7 +844,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def add_hashtags(object) do tags = - (object["tag"] || []) + ((object["hashtags"] || []) ++ (object["tag"] || [])) |> Enum.map(fn # Expand internal representation tags into AS2 tags. tag when is_binary(tag) -> @@ -932,7 +915,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def set_sensitive(object) do - tags = object["tag"] || [] + tags = object["hashtags"] || object["tag"] || [] Map.put(object, "sensitive", "nsfw" in tags) end diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex @@ -97,7 +97,10 @@ defmodule Pleroma.Web.ActivityPub.Utils do if need_splice? do cc_list = extract_list(params["cc"]) - Map.put(params, "cc", [ap_id | cc_list]) + + params + |> Map.put("cc", [ap_id | cc_list]) + |> Kernel.put_in(["object", "cc"], [ap_id | cc_list]) else params end diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex @@ -330,6 +330,11 @@ defmodule Pleroma.Web.CommonAPI.Utils do sensitive \\ false, extra_params \\ %{} ) do + tags = + tags + |> Keyword.values() + |> Enum.uniq() + %{ "type" => "Note", "to" => to, @@ -340,7 +345,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do "context" => context, "attachment" => attachments, "actor" => actor, - "tag" => Keyword.values(tags) |> Enum.uniq() + "tag" => Enum.filter(tags, &is_map(&1)), + "hashtags" => Enum.filter(tags, &is_bitstring(&1)) } |> add_in_reply_to(in_reply_to) |> Map.merge(extra_params) diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex @@ -92,6 +92,11 @@ defmodule Pleroma.Web.Federator do Logger.debug("Unhandled actor #{actor}, #{inspect(e)}") {:error, e} + {:error, {:validate_object, _}} = e -> + Logger.error("Incoming AP doc validation error: #{inspect(e)}") + Logger.debug(Jason.encode!(params, pretty: true)) + e + e -> # Just drop those for now Logger.debug(fn -> "Unhandled activity\n" <> Jason.encode!(params, pretty: true) end) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -351,7 +351,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do media_attachments: attachments, poll: render(PollView, "show.json", object: object, for: opts[:for]), mentions: mentions, - tags: build_tags(tags), + tags: build_tags(object.data["hashtags"] || tags), application: %{ name: "Web", website: nil diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex @@ -22,8 +22,8 @@ <link type="text/html" href='<%= @data["external_url"] %>' rel="alternate"/> <% end %> - <%= for tag <- @data["tag"] || [] do %> - <category term="<%= tag %>"></category> + <%= for hashtag <- @data["hashtags"] || [] do %> + <category term="<%= hashtag %>"></category> <% end %> <%= for attachment <- @data["attachment"] || [] do %> diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex @@ -21,8 +21,8 @@ <link><%= @data["external_url"] %></link> <% end %> - <%= for tag <- @data["tag"] || [] do %> - <category term="<%= tag %>"></category> + <%= for hashtag <- @data["hashtags"] || [] do %> + <category term="<%= hashtag %>"></category> <% end %> <%= for attachment <- @data["attachment"] || [] do %> diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex @@ -41,8 +41,8 @@ <% end %> <% end %> - <%= for tag <- @data["tag"] || [] do %> - <category term="<%= tag %>"></category> + <%= for hashtag <- @data["hashtags"] || [] do %> + <category term="<%= hashtag %>"></category> <% end %> <%= for {emoji, file} <- @data["emoji"] || %{} do %> diff --git a/priv/repo/migrations/20200731165800_add_hashtags_index_to_objects.exs b/priv/repo/migrations/20200731165800_add_hashtags_index_to_objects.exs @@ -0,0 +1,11 @@ +defmodule Pleroma.Repo.Migrations.AddHashtagsIndexToObjects do + use Ecto.Migration + + def change do + drop_if_exists(index(:objects, ["(data->'tag')"], using: :gin, name: :objects_tags)) + + create_if_not_exists( + index(:objects, ["(data->'hashtags')"], using: :gin, name: :objects_hashtags) + ) + end +end diff --git a/test/fixtures/activitypub-client-post-activity.json b/test/fixtures/activitypub-client-post-activity.json @@ -3,6 +3,7 @@ "type": "Create", "object": { "type": "Note", + "to": ["https://www.w3.org/ns/activitystreams#Public"], "content": "It's a note" }, "to": ["https://www.w3.org/ns/activitystreams#Public"] diff --git a/test/pleroma/activity/ir/topics_test.exs b/test/pleroma/activity/ir/topics_test.exs @@ -78,7 +78,7 @@ defmodule Pleroma.Activity.Ir.TopicsTest do end test "converts tags to hash tags", %{activity: %{object: %{data: data} = object} = activity} do - tagged_data = Map.put(data, "tag", ["foo", "bar"]) + tagged_data = Map.put(data, "hashtags", ["foo", "bar"]) activity = %{activity | object: %{object | data: tagged_data}} topics = Topics.get_activity_topics(activity) diff --git a/test/pleroma/activity_test.exs b/test/pleroma/activity_test.exs @@ -123,7 +123,8 @@ defmodule Pleroma.ActivityTest do "type" => "Note", "content" => "find me!", "id" => "http://mastodon.example.org/users/admin/objects/1", - "attributedTo" => "http://mastodon.example.org/users/admin" + "attributedTo" => "http://mastodon.example.org/users/admin", + "to" => ["https://www.w3.org/ns/activitystreams#Public"] }, "to" => ["https://www.w3.org/ns/activitystreams#Public"] } @@ -132,6 +133,7 @@ defmodule Pleroma.ActivityTest do {:ok, japanese_activity} = Pleroma.Web.CommonAPI.post(user, %{status: "更新情報"}) {:ok, job} = Pleroma.Web.Federator.incoming_ap_doc(params) {:ok, remote_activity} = ObanHelpers.perform(job) + remote_activity = Activity.get_by_id_with_object(remote_activity.id) %{ japanese_activity: japanese_activity, diff --git a/test/pleroma/ecto_type/activity_pub/object_validators/recipients_test.exs b/test/pleroma/ecto_type/activity_pub/object_validators/recipients_test.exs @@ -6,10 +6,10 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.RecipientsTest do alias Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients use Pleroma.DataCase - test "it asserts that all elements of the list are object ids" do + test "it only keeps elements that are valid object ids" do list = ["https://lain.com/users/lain", "invalid"] - assert :error == Recipients.cast(list) + assert {:ok, ["https://lain.com/users/lain"]} == Recipients.cast(list) end test "it works with a list" do diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs @@ -597,6 +597,8 @@ defmodule Pleroma.NotificationTest do "actor" => user.ap_id, "object" => %{ "type" => "Note", + "id" => Pleroma.Web.ActivityPub.Utils.generate_object_id(), + "to" => ["https://www.w3.org/ns/activitystreams#Public"], "content" => "message with a Mention tag, but no explicit tagging", "tag" => [ %{ @@ -628,6 +630,9 @@ defmodule Pleroma.NotificationTest do "actor" => user.ap_id, "object" => %{ "type" => "Note", + "id" => Pleroma.Web.ActivityPub.Utils.generate_object_id(), + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [other_user.ap_id], "content" => "hi everyone", "attributedTo" => user.ap_id } @@ -924,6 +929,7 @@ defmodule Pleroma.NotificationTest do "cc" => [], "object" => %{ "type" => "Note", + "id" => remote_user.ap_id <> "/objects/test", "content" => "Hello!", "tag" => [ %{ diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs @@ -53,6 +53,11 @@ defmodule Pleroma.Object.FetcherTest do %Tesla.Env{ status: 500 } + + %{method: :get, url: "https://stereophonic.space/objects/02997b83-3ea7-4b63-94af-ef3aa2d4ed17"} -> + %Tesla.Env{ + status: 500 + } end) :ok @@ -112,7 +117,6 @@ defmodule Pleroma.Object.FetcherTest do Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") assert activity = Activity.get_create_by_object_ap_id(object.data["id"]) - assert activity.data["id"] {:ok, object_again} = Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -467,7 +467,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() |> Map.put("actor", user.ap_id) - |> put_in(["object", "attridbutedTo"], user.ap_id) + |> put_in(["object", "attributedTo"], user.ap_id) conn = conn @@ -577,7 +577,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do test "it inserts an incoming activity into the database", %{conn: conn, data: data} do user = insert(:user) - data = Map.put(data, "bcc", [user.ap_id]) + + data = + data + |> Map.put("bcc", [user.ap_id]) + |> Kernel.put_in(["object", "bcc"], [user.ap_id]) conn = conn @@ -594,8 +598,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do user = insert(:user) data = - Map.put(data, "to", user.ap_id) - |> Map.delete("cc") + data + |> Map.put("to", user.ap_id) + |> Map.put("cc", []) + |> Kernel.put_in(["object", "to"], user.ap_id) + |> Kernel.put_in(["object", "cc"], []) conn = conn @@ -612,8 +619,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do user = insert(:user) data = - Map.put(data, "cc", user.ap_id) - |> Map.delete("to") + data + |> Map.put("to", []) + |> Map.put("cc", user.ap_id) + |> Kernel.put_in(["object", "to"], []) + |> Kernel.put_in(["object", "cc"], user.ap_id) conn = conn @@ -631,9 +641,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do user = insert(:user) data = - Map.put(data, "bcc", user.ap_id) - |> Map.delete("to") - |> Map.delete("cc") + data + |> Map.put("to", []) + |> Map.put("cc", []) + |> Map.put("bcc", user.ap_id) + |> Kernel.put_in(["object", "to"], []) + |> Kernel.put_in(["object", "cc"], []) + |> Kernel.put_in(["object", "bcc"], user.ap_id) conn = conn @@ -748,29 +762,34 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert Instances.reachable?(sender_host) end + @tag capture_log: true test "it removes all follower collections but actor's", %{conn: conn} do [actor, recipient] = insert_pair(:user) - data = - File.read!("test/fixtures/activitypub-client-post-activity.json") - |> Poison.decode!() + to = [ + recipient.ap_id, + recipient.follower_address, + "https://www.w3.org/ns/activitystreams#Public" + ] - object = Map.put(data["object"], "attributedTo", actor.ap_id) + cc = [recipient.follower_address, actor.follower_address] - data = - data - |> Map.put("id", Utils.generate_object_id()) - |> Map.put("actor", actor.ap_id) - |> Map.put("object", object) - |> Map.put("cc", [ - recipient.follower_address, - actor.follower_address - ]) - |> Map.put("to", [ - recipient.ap_id, - recipient.follower_address, - "https://www.w3.org/ns/activitystreams#Public" - ]) + data = %{ + "@context" => ["https://www.w3.org/ns/activitystreams"], + "type" => "Create", + "id" => Utils.generate_activity_id(), + "to" => to, + "cc" => cc, + "actor" => actor.ap_id, + "object" => %{ + "type" => "Note", + "to" => to, + "cc" => cc, + "content" => "It's a note", + "attributedTo" => actor.ap_id, + "id" => Utils.generate_object_id() + } + } conn |> assign(:valid_signature, true) @@ -780,7 +799,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) - activity = Activity.get_by_ap_id(data["id"]) + assert activity = Activity.get_by_ap_id(data["id"]) assert activity.id assert actor.follower_address in activity.recipients diff --git a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs @@ -78,7 +78,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do assert SimplePolicy.filter(media_message) == {:ok, media_message - |> put_in(["object", "tag"], ["foo", "nsfw"]) + |> put_in(["object", "hashtags"], ["foo", "nsfw"]) |> put_in(["object", "sensitive"], true)} assert SimplePolicy.filter(local_message) == {:ok, local_message} @@ -92,7 +92,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do assert SimplePolicy.filter(media_message) == {:ok, media_message - |> put_in(["object", "tag"], ["foo", "nsfw"]) + |> put_in(["object", "hashtags"], ["foo", "nsfw"]) |> put_in(["object", "sensitive"], true)} assert SimplePolicy.filter(local_message) == {:ok, local_message} @@ -105,7 +105,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do "type" => "Create", "object" => %{ "attachment" => [%{}], - "tag" => ["foo"], + "hashtags" => ["foo"], "sensitive" => false } } diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_validator_test.exs @@ -5,31 +5,75 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidatorTest do use Pleroma.DataCase + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ObjectValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator alias Pleroma.Web.ActivityPub.Utils import Pleroma.Factory - describe "Notes" do - setup do - user = insert(:user) - - note = %{ - "id" => Utils.generate_activity_id(), - "type" => "Note", - "actor" => user.ap_id, - "to" => [user.follower_address], - "cc" => [], - "content" => "Hellow this is content.", - "context" => "xxx", - "summary" => "a post" - } - - %{user: user, note: note} - end - - test "a basic note validates", %{note: note} do - %{valid?: true} = ArticleNoteValidator.cast_and_validate(note) - end + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + test "a basic note validates" do + user = insert(:user) + + note = %{ + "id" => Utils.generate_activity_id(), + "type" => "Note", + "actor" => user.ap_id, + "to" => [user.follower_address], + "cc" => [], + "content" => "Hellow this is content.", + "context" => "xxx", + "summary" => "a post" + } + + %{valid?: true} = ArticleNoteValidator.cast_and_validate(note) + end + + test "Mastodon Note" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Jason.decode!() + + {:ok, %User{}} = ObjectValidator.fetch_actor(data) + + assert %{changes: changes, valid?: true} = + ArticleNoteValidator.cast_and_validate(data["object"]) + + assert %{ + actor: "http://mastodon.example.org/users/admin", + attributedTo: "http://mastodon.example.org/users/admin", + cc: [ + "http://localtesting.pleroma.lol/users/lain", + "http://mastodon.example.org/users/admin/followers" + ], + content: + "<p><span class=\"h-card\"><a href=\"http://localtesting.pleroma.lol/users/lain\" class=\"u-url mention\">@<span>lain</span></a></span></p>", + context: "tag:mastodon.example.org,2018-02-12:objectId=20:objectType=Conversation", + id: "http://mastodon.example.org/users/admin/statuses/99512778738411822", + published: "2018-02-12T14:08:20Z", + sensitive: true, + summary: "cw", + tag: [ + %{ + "href" => "http://localtesting.pleroma.lol/users/lain", + "name" => "@lain@localtesting.pleroma.lol", + "type" => "Mention" + } + ], + to: ["https://www.w3.org/ns/activitystreams#Public"], + type: "Note", + url: "http://mastodon.example.org/@admin/99512778738411822", + replies: [ + "http://mastodon.example.org/users/admin/statuses/99512778738411823", + "http://mastodon.example.org/users/admin/statuses/99512778738411824" + ] + } = changes + + assert changes[:context_id] end end diff --git a/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs @@ -24,6 +24,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.AudioHandlingTest do "actor" => "http://mastodon.example.org/users/admin", "object" => %{ "type" => "Audio", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "cc" => [], "id" => "http://mastodon.example.org/users/admin/listens/1234", "attributedTo" => "http://mastodon.example.org/users/admin", "title" => "lain radio episode 1", @@ -60,7 +62,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.AudioHandlingTest do assert object.data["to"] == ["https://www.w3.org/ns/activitystreams#Public"] - assert object.data["cc"] == [] + assert object.data["cc"] == [ + "https://channels.tests.funkwhale.audio/federation/actors/compositions/followers" + ] assert object.data["url"] == "https://channels.tests.funkwhale.audio/library/tracks/74" diff --git a/test/pleroma/web/activity_pub/transmogrifier/event_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/event_handling_test.exs @@ -29,7 +29,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.EventHandlingTest do ) assert object.data["to"] == ["https://www.w3.org/ns/activitystreams#Public"] - assert object.data["cc"] == [] + assert object.data["cc"] == ["https://mobilizon.org/@tcit/followers"] assert object.data["url"] == "https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39" diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -0,0 +1,419 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do + use Oban.Testing, repo: Pleroma.Repo + use Pleroma.DataCase + + alias Pleroma.Activity + alias Pleroma.Object + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.CommonAPI + + import Mock + import Pleroma.Factory + + setup_all do + Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + setup do: clear_config([:instance, :max_remote_account_fields]) + + describe "handle_incoming" do + test "it works for incoming notices with tag not being an array (kroeg)" do + data = File.read!("test/fixtures/kroeg-array-less-emoji.json") |> Jason.decode!() + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) + + assert object.data["emoji"] == %{ + "icon_e_smile" => "https://puckipedia.com/forum/images/smilies/icon_e_smile.png" + } + + data = File.read!("test/fixtures/kroeg-array-less-hashtag.json") |> Jason.decode!() + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) + + assert "test" in object.data["hashtags"] + + assert object.data["tag"] == [ + %{ + "href" => "https://puckipedia.com/ur33-818k/note/hashtag", + "id" => "https://puckipedia.com/ur33-818k/note/hashtag", + "name" => "#test", + "type" => "Hashtag" + } + ] + end + + @tag capture_log: true + test "it fetches reply-to activities if we don't have them" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Jason.decode!() + + object = + data["object"] + |> Map.put("inReplyTo", "https://mstdn.io/users/mayuutann/statuses/99568293732299394") + + data = Map.put(data, "object", object) + {:ok, returned_activity} = Transmogrifier.handle_incoming(data) + returned_object = Object.normalize(returned_activity, false) + + assert activity = + Activity.get_create_by_object_ap_id( + "https://mstdn.io/users/mayuutann/statuses/99568293732299394" + ) + + assert returned_object.data["inReplyTo"] == + "https://mstdn.io/users/mayuutann/statuses/99568293732299394" + end + + test "it does not fetch reply-to activities beyond max replies depth limit" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Jason.decode!() + + object = + data["object"] + |> Map.put("inReplyTo", "https://shitposter.club/notice/2827873") + + data = Map.put(data, "object", object) + + with_mock Pleroma.Web.Federator, + allowed_thread_distance?: fn _ -> false end do + {:ok, returned_activity} = Transmogrifier.handle_incoming(data) + + returned_object = Object.normalize(returned_activity, false) + + refute Activity.get_create_by_object_ap_id( + "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" + ) + + assert returned_object.data["inReplyTo"] == "https://shitposter.club/notice/2827873" + end + end + + test "it does not crash if the object in inReplyTo can't be fetched" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Jason.decode!() + + object = + data["object"] + |> Map.put("inReplyTo", "https://404.site/whatever") + + data = + data + |> Map.put("object", object) + + assert {:ok, _returned_activity} = Transmogrifier.handle_incoming(data) + end + + test "it works for incoming notices" do + data = File.read!("test/fixtures/mastodon-post-activity.json") |> Jason.decode!() + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + + assert data["id"] == + "http://mastodon.example.org/users/admin/statuses/99512778738411822/activity" + + assert data["context"] == + "tag:mastodon.example.org,2018-02-12:objectId=20:objectType=Conversation" + + assert data["to"] == ["https://www.w3.org/ns/activitystreams#Public"] + + assert data["cc"] == [ + "http://localtesting.pleroma.lol/users/lain", + "http://mastodon.example.org/users/admin/followers" + ] + + assert data["actor"] == "http://mastodon.example.org/users/admin" + + object_data = Object.normalize(data["object"]).data + + assert object_data["id"] == + "http://mastodon.example.org/users/admin/statuses/99512778738411822" + + assert object_data["to"] == ["https://www.w3.org/ns/activitystreams#Public"] + + assert object_data["cc"] == [ + "http://localtesting.pleroma.lol/users/lain", + "http://mastodon.example.org/users/admin/followers" + ] + + assert object_data["actor"] == "http://mastodon.example.org/users/admin" + assert object_data["attributedTo"] == "http://mastodon.example.org/users/admin" + + assert object_data["context"] == + "tag:mastodon.example.org,2018-02-12:objectId=20:objectType=Conversation" + + assert object_data["sensitive"] == true + + user = User.get_cached_by_ap_id(object_data["actor"]) + + assert user.note_count == 1 + end + + test "it does not work for deactivated users" do + data = File.read!("test/fixtures/mastodon-post-activity.json") |> Jason.decode!() + + insert(:user, ap_id: data["actor"], deactivated: true) + + assert {:error, _} = Transmogrifier.handle_incoming(data) + end + + test "it works for incoming notices with hashtags" do + data = File.read!("test/fixtures/mastodon-post-activity-hashtag.json") |> Jason.decode!() + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) + + assert object.data["hashtags"] == ["moo"] + + assert %{ + "href" => "http://mastodon.example.org/tags/moo", + "name" => "#moo", + "type" => "Hashtag" + } in object.data["tag"] + end + + test "it works for incoming notices with contentMap" do + data = File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Jason.decode!() + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) + + assert object.data["content"] == + "<p><span class=\"h-card\"><a href=\"http://localtesting.pleroma.lol/users/lain\" class=\"u-url mention\">@<span>lain</span></a></span></p>" + end + + test "it works for incoming notices with to/cc not being an array (kroeg)" do + data = File.read!("test/fixtures/kroeg-post-activity.json") |> Jason.decode!() + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) + + assert object.data["content"] == + "<p>henlo from my Psion netBook</p><p>message sent from my Psion netBook</p>" + end + + test "it ensures that as:Public activities make it to their followers collection" do + user = insert(:user) + + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Jason.decode!() + |> Map.put("actor", user.ap_id) + |> Map.put("to", ["https://www.w3.org/ns/activitystreams#Public"]) + |> Map.put("cc", []) + + object = + data["object"] + |> Map.put("attributedTo", user.ap_id) + |> Map.put("to", ["https://www.w3.org/ns/activitystreams#Public"]) + |> Map.put("cc", []) + |> Map.put("id", user.ap_id <> "/activities/12345678") + + data = Map.put(data, "object", object) + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + + assert data["cc"] == [User.ap_followers(user)] + end + + test "it ensures that address fields become lists" do + user = insert(:user) + + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Jason.decode!() + |> Map.put("actor", user.ap_id) + |> Map.put("cc", nil) + + object = + data["object"] + |> Map.put("attributedTo", user.ap_id) + |> Map.put("cc", nil) + |> Map.put("id", user.ap_id <> "/activities/12345678") + + data = Map.put(data, "object", object) + + {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + + refute is_nil(data["cc"]) + end + + test "it strips internal likes" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Jason.decode!() + + likes = %{ + "first" => + "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1", + "id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes", + "totalItems" => 3, + "type" => "OrderedCollection" + } + + object = Map.put(data["object"], "likes", likes) + data = Map.put(data, "object", object) + + {:ok, %Activity{} = activity} = Transmogrifier.handle_incoming(data) + + %Object{data: object} = Object.normalize(activity) + assert object["likes"] == [] + end + + test "it strips internal reactions" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{status: "#cofe"}) + {:ok, _} = CommonAPI.react_with_emoji(activity.id, user, "📢") + + %{object: object} = Activity.get_by_id_with_object(activity.id) + assert Map.has_key?(object.data, "reactions") + assert Map.has_key?(object.data, "reaction_count") + + object_data = Transmogrifier.strip_internal_fields(object.data) + refute Map.has_key?(object_data, "reactions") + refute Map.has_key?(object_data, "reaction_count") + end + + test "it correctly processes messages with non-array to field" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Jason.decode!() + |> Map.put("to", "https://www.w3.org/ns/activitystreams#Public") + |> put_in(["object", "to"], "https://www.w3.org/ns/activitystreams#Public") + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + + assert [ + "http://localtesting.pleroma.lol/users/lain", + "http://mastodon.example.org/users/admin/followers" + ] == activity.data["cc"] + + assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["to"] + end + + test "it correctly processes messages with non-array cc field" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Jason.decode!() + |> Map.put("cc", "http://mastodon.example.org/users/admin/followers") + |> put_in(["object", "cc"], "http://mastodon.example.org/users/admin/followers") + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + + assert ["http://mastodon.example.org/users/admin/followers"] == activity.data["cc"] + assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["to"] + end + + test "it correctly processes messages with weirdness in address fields" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Jason.decode!() + |> Map.put("cc", ["http://mastodon.example.org/users/admin/followers", ["¿"]]) + |> put_in(["object", "cc"], ["http://mastodon.example.org/users/admin/followers", ["¿"]]) + + assert {:ok, activity} = Transmogrifier.handle_incoming(data) + + assert ["http://mastodon.example.org/users/admin/followers"] == activity.data["cc"] + assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["to"] + end + end + + describe "`handle_incoming/2`, Mastodon format `replies` handling" do + setup do: clear_config([:activitypub, :note_replies_output_limit], 5) + setup do: clear_config([:instance, :federation_incoming_replies_max_depth]) + + setup do + data = + "test/fixtures/mastodon-post-activity.json" + |> File.read!() + |> Jason.decode!() + + items = get_in(data, ["object", "replies", "first", "items"]) + assert length(items) > 0 + + %{data: data, items: items} + end + + test "schedules background fetching of `replies` items if max thread depth limit allows", %{ + data: data, + items: items + } do + Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 10) + + {:ok, activity} = Transmogrifier.handle_incoming(data) + + object = Object.normalize(activity.data["object"]) + + assert object.data["replies"] == items + + for id <- items do + job_args = %{"op" => "fetch_remote", "id" => id, "depth" => 1} + assert_enqueued(worker: Pleroma.Workers.RemoteFetcherWorker, args: job_args) + end + end + + test "does NOT schedule background fetching of `replies` beyond max thread depth limit allows", + %{data: data} do + Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 0) + + {:ok, _activity} = Transmogrifier.handle_incoming(data) + + assert all_enqueued(worker: Pleroma.Workers.RemoteFetcherWorker) == [] + end + end + + describe "`handle_incoming/2`, Pleroma format `replies` handling" do + setup do: clear_config([:activitypub, :note_replies_output_limit], 5) + setup do: clear_config([:instance, :federation_incoming_replies_max_depth]) + + setup do + replies = %{ + "type" => "Collection", + "items" => [ + Pleroma.Web.ActivityPub.Utils.generate_object_id(), + Pleroma.Web.ActivityPub.Utils.generate_object_id() + ] + } + + activity = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Jason.decode!() + |> Kernel.put_in(["object", "replies"], replies) + + %{activity: activity} + end + + test "schedules background fetching of `replies` items if max thread depth limit allows", %{ + activity: activity + } do + Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 1) + + assert {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(activity) + object = Object.normalize(data["object"]) + + for id <- object.data["replies"] do + job_args = %{"op" => "fetch_remote", "id" => id, "depth" => 1} + assert_enqueued(worker: Pleroma.Workers.RemoteFetcherWorker, args: job_args) + end + end + + test "does NOT schedule background fetching of `replies` beyond max thread depth limit allows", + %{activity: activity} do + Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 0) + + {:ok, _activity} = Transmogrifier.handle_incoming(activity) + + assert all_enqueued(worker: Pleroma.Workers.RemoteFetcherWorker) == [] + end + end +end diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do alias Pleroma.Tests.ObanHelpers alias Pleroma.User alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.AdminAPI.AccountView alias Pleroma.Web.CommonAPI @@ -26,300 +27,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do setup do: clear_config([:instance, :max_remote_account_fields]) describe "handle_incoming" do - test "it works for incoming notices with tag not being an array (kroeg)" do - data = File.read!("test/fixtures/kroeg-array-less-emoji.json") |> Poison.decode!() - - {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) - object = Object.normalize(data["object"]) - - assert object.data["emoji"] == %{ - "icon_e_smile" => "https://puckipedia.com/forum/images/smilies/icon_e_smile.png" - } - - data = File.read!("test/fixtures/kroeg-array-less-hashtag.json") |> Poison.decode!() - - {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) - object = Object.normalize(data["object"]) - - assert "test" in object.data["tag"] - end - - test "it cleans up incoming notices which are not really DMs" do - user = insert(:user) - other_user = insert(:user) - - to = [user.ap_id, other_user.ap_id] - - data = - File.read!("test/fixtures/mastodon-post-activity.json") - |> Poison.decode!() - |> Map.put("to", to) - |> Map.put("cc", []) - - object = - data["object"] - |> Map.put("to", to) - |> Map.put("cc", []) - - data = Map.put(data, "object", object) - - {:ok, %Activity{data: data, local: false} = activity} = Transmogrifier.handle_incoming(data) - - assert data["to"] == [] - assert data["cc"] == to - - object_data = Object.normalize(activity).data - - assert object_data["to"] == [] - assert object_data["cc"] == to - end - - test "it ignores an incoming notice if we already have it" do - activity = insert(:note_activity) - - data = - File.read!("test/fixtures/mastodon-post-activity.json") - |> Poison.decode!() - |> Map.put("object", Object.normalize(activity).data) - - {:ok, returned_activity} = Transmogrifier.handle_incoming(data) - - assert activity == returned_activity - end - - @tag capture_log: true - test "it fetches reply-to activities if we don't have them" do - data = - File.read!("test/fixtures/mastodon-post-activity.json") - |> Poison.decode!() - - object = - data["object"] - |> Map.put("inReplyTo", "https://mstdn.io/users/mayuutann/statuses/99568293732299394") - - data = Map.put(data, "object", object) - {:ok, returned_activity} = Transmogrifier.handle_incoming(data) - returned_object = Object.normalize(returned_activity, false) - - assert activity = - Activity.get_create_by_object_ap_id( - "https://mstdn.io/users/mayuutann/statuses/99568293732299394" - ) - - assert returned_object.data["inReplyTo"] == - "https://mstdn.io/users/mayuutann/statuses/99568293732299394" - end - - test "it does not fetch reply-to activities beyond max replies depth limit" do - data = - File.read!("test/fixtures/mastodon-post-activity.json") - |> Poison.decode!() - - object = - data["object"] - |> Map.put("inReplyTo", "https://shitposter.club/notice/2827873") - - data = Map.put(data, "object", object) - - with_mock Pleroma.Web.Federator, - allowed_thread_distance?: fn _ -> false end do - {:ok, returned_activity} = Transmogrifier.handle_incoming(data) - - returned_object = Object.normalize(returned_activity, false) - - refute Activity.get_create_by_object_ap_id( - "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" - ) - - assert returned_object.data["inReplyTo"] == "https://shitposter.club/notice/2827873" - end - end - - test "it does not crash if the object in inReplyTo can't be fetched" do - data = - File.read!("test/fixtures/mastodon-post-activity.json") - |> Poison.decode!() - - object = - data["object"] - |> Map.put("inReplyTo", "https://404.site/whatever") - - data = - data - |> Map.put("object", object) - - assert capture_log(fn -> - {:ok, _returned_activity} = Transmogrifier.handle_incoming(data) - end) =~ "[warn] Couldn't fetch \"https://404.site/whatever\", error: nil" - end - - test "it does not work for deactivated users" do - data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() - - insert(:user, ap_id: data["actor"], deactivated: true) - - assert {:error, _} = Transmogrifier.handle_incoming(data) - end - - test "it works for incoming notices" do - data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() - - {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) - - assert data["id"] == - "http://mastodon.example.org/users/admin/statuses/99512778738411822/activity" - - assert data["context"] == - "tag:mastodon.example.org,2018-02-12:objectId=20:objectType=Conversation" - - assert data["to"] == ["https://www.w3.org/ns/activitystreams#Public"] - - assert data["cc"] == [ - "http://mastodon.example.org/users/admin/followers", - "http://localtesting.pleroma.lol/users/lain" - ] - - assert data["actor"] == "http://mastodon.example.org/users/admin" - - object_data = Object.normalize(data["object"]).data - - assert object_data["id"] == - "http://mastodon.example.org/users/admin/statuses/99512778738411822" - - assert object_data["to"] == ["https://www.w3.org/ns/activitystreams#Public"] - - assert object_data["cc"] == [ - "http://mastodon.example.org/users/admin/followers", - "http://localtesting.pleroma.lol/users/lain" - ] - - assert object_data["actor"] == "http://mastodon.example.org/users/admin" - assert object_data["attributedTo"] == "http://mastodon.example.org/users/admin" - - assert object_data["context"] == - "tag:mastodon.example.org,2018-02-12:objectId=20:objectType=Conversation" - - assert object_data["sensitive"] == true - - user = User.get_cached_by_ap_id(object_data["actor"]) - - assert user.note_count == 1 - end - - test "it works for incoming notices with hashtags" do - data = File.read!("test/fixtures/mastodon-post-activity-hashtag.json") |> Poison.decode!() - - {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) - object = Object.normalize(data["object"]) - - assert Enum.at(object.data["tag"], 2) == "moo" - end - - test "it works for incoming notices with contentMap" do - data = - File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Poison.decode!() - - {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) - object = Object.normalize(data["object"]) - - assert object.data["content"] == - "<p><span class=\"h-card\"><a href=\"http://localtesting.pleroma.lol/users/lain\" class=\"u-url mention\">@<span>lain</span></a></span></p>" - end - - test "it works for incoming notices with to/cc not being an array (kroeg)" do - data = File.read!("test/fixtures/kroeg-post-activity.json") |> Poison.decode!() - - {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) - object = Object.normalize(data["object"]) - - assert object.data["content"] == - "<p>henlo from my Psion netBook</p><p>message sent from my Psion netBook</p>" - end - - test "it ensures that as:Public activities make it to their followers collection" do - user = insert(:user) - - data = - File.read!("test/fixtures/mastodon-post-activity.json") - |> Poison.decode!() - |> Map.put("actor", user.ap_id) - |> Map.put("to", ["https://www.w3.org/ns/activitystreams#Public"]) - |> Map.put("cc", []) - - object = - data["object"] - |> Map.put("attributedTo", user.ap_id) - |> Map.put("to", ["https://www.w3.org/ns/activitystreams#Public"]) - |> Map.put("cc", []) - |> Map.put("id", user.ap_id <> "/activities/12345678") - - data = Map.put(data, "object", object) - - {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) - - assert data["cc"] == [User.ap_followers(user)] - end - - test "it ensures that address fields become lists" do - user = insert(:user) - - data = - File.read!("test/fixtures/mastodon-post-activity.json") - |> Poison.decode!() - |> Map.put("actor", user.ap_id) - |> Map.put("to", nil) - |> Map.put("cc", nil) - - object = - data["object"] - |> Map.put("attributedTo", user.ap_id) - |> Map.put("to", nil) - |> Map.put("cc", nil) - |> Map.put("id", user.ap_id <> "/activities/12345678") - - data = Map.put(data, "object", object) - - {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) - - assert !is_nil(data["to"]) - assert !is_nil(data["cc"]) - end - - test "it strips internal likes" do - data = - File.read!("test/fixtures/mastodon-post-activity.json") - |> Poison.decode!() - - likes = %{ - "first" => - "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes?page=1", - "id" => "http://mastodon.example.org/objects/dbdbc507-52c8-490d-9b7c-1e1d52e5c132/likes", - "totalItems" => 3, - "type" => "OrderedCollection" - } - - object = Map.put(data["object"], "likes", likes) - data = Map.put(data, "object", object) - - {:ok, %Activity{object: object}} = Transmogrifier.handle_incoming(data) - - refute Map.has_key?(object.data, "likes") - end - - test "it strips internal reactions" do - user = insert(:user) - {:ok, activity} = CommonAPI.post(user, %{status: "#cofe"}) - {:ok, _} = CommonAPI.react_with_emoji(activity.id, user, "📢") - - %{object: object} = Activity.get_by_id_with_object(activity.id) - assert Map.has_key?(object.data, "reactions") - assert Map.has_key?(object.data, "reaction_count") - - object_data = Transmogrifier.strip_internal_fields(object.data) - refute Map.has_key?(object_data, "reactions") - refute Map.has_key?(object_data, "reaction_count") - end - test "it works for incoming unfollows with an existing follow" do user = insert(:user) @@ -377,73 +84,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert activity.data["cc"] == [user.ap_id] end - test "it correctly processes messages with non-array to field" do - user = insert(:user) - - message = %{ - "@context" => "https://www.w3.org/ns/activitystreams", - "to" => "https://www.w3.org/ns/activitystreams#Public", - "type" => "Create", - "object" => %{ - "content" => "blah blah blah", - "type" => "Note", - "attributedTo" => user.ap_id, - "inReplyTo" => nil - }, - "actor" => user.ap_id - } - - assert {:ok, activity} = Transmogrifier.handle_incoming(message) - - assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["to"] - end - - test "it correctly processes messages with non-array cc field" do - user = insert(:user) - - message = %{ - "@context" => "https://www.w3.org/ns/activitystreams", - "to" => user.follower_address, - "cc" => "https://www.w3.org/ns/activitystreams#Public", - "type" => "Create", - "object" => %{ - "content" => "blah blah blah", - "type" => "Note", - "attributedTo" => user.ap_id, - "inReplyTo" => nil - }, - "actor" => user.ap_id - } - - assert {:ok, activity} = Transmogrifier.handle_incoming(message) - - assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["cc"] - assert [user.follower_address] == activity.data["to"] - end - - test "it correctly processes messages with weirdness in address fields" do - user = insert(:user) - - message = %{ - "@context" => "https://www.w3.org/ns/activitystreams", - "to" => [nil, user.follower_address], - "cc" => ["https://www.w3.org/ns/activitystreams#Public", ["¿"]], - "type" => "Create", - "object" => %{ - "content" => "…", - "type" => "Note", - "attributedTo" => user.ap_id, - "inReplyTo" => nil - }, - "actor" => user.ap_id - } - - assert {:ok, activity} = Transmogrifier.handle_incoming(message) - - assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["cc"] - assert [user.follower_address] == activity.data["to"] - end - test "it accepts Move activities" do old_user = insert(:user) new_user = insert(:user) @@ -469,95 +109,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do end end - describe "`handle_incoming/2`, Mastodon format `replies` handling" do - setup do: clear_config([:activitypub, :note_replies_output_limit], 5) - setup do: clear_config([:instance, :federation_incoming_replies_max_depth]) - - setup do - data = - "test/fixtures/mastodon-post-activity.json" - |> File.read!() - |> Poison.decode!() - - items = get_in(data, ["object", "replies", "first", "items"]) - assert length(items) > 0 - - %{data: data, items: items} - end - - test "schedules background fetching of `replies` items if max thread depth limit allows", %{ - data: data, - items: items - } do - Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 10) - - {:ok, _activity} = Transmogrifier.handle_incoming(data) - - for id <- items do - job_args = %{"op" => "fetch_remote", "id" => id, "depth" => 1} - assert_enqueued(worker: Pleroma.Workers.RemoteFetcherWorker, args: job_args) - end - end - - test "does NOT schedule background fetching of `replies` beyond max thread depth limit allows", - %{data: data} do - Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 0) - - {:ok, _activity} = Transmogrifier.handle_incoming(data) - - assert all_enqueued(worker: Pleroma.Workers.RemoteFetcherWorker) == [] - end - end - - describe "`handle_incoming/2`, Pleroma format `replies` handling" do - setup do: clear_config([:activitypub, :note_replies_output_limit], 5) - setup do: clear_config([:instance, :federation_incoming_replies_max_depth]) - - setup do - user = insert(:user) - - {:ok, activity} = CommonAPI.post(user, %{status: "post1"}) - - {:ok, reply1} = - CommonAPI.post(user, %{status: "reply1", in_reply_to_status_id: activity.id}) - - {:ok, reply2} = - CommonAPI.post(user, %{status: "reply2", in_reply_to_status_id: activity.id}) - - replies_uris = Enum.map([reply1, reply2], fn a -> a.object.data["id"] end) - - {:ok, federation_output} = Transmogrifier.prepare_outgoing(activity.data) - - Repo.delete(activity.object) - Repo.delete(activity) - - %{federation_output: federation_output, replies_uris: replies_uris} - end - - test "schedules background fetching of `replies` items if max thread depth limit allows", %{ - federation_output: federation_output, - replies_uris: replies_uris - } do - Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 1) - - {:ok, _activity} = Transmogrifier.handle_incoming(federation_output) - - for id <- replies_uris do - job_args = %{"op" => "fetch_remote", "id" => id, "depth" => 1} - assert_enqueued(worker: Pleroma.Workers.RemoteFetcherWorker, args: job_args) - end - end - - test "does NOT schedule background fetching of `replies` beyond max thread depth limit allows", - %{federation_output: federation_output} do - Pleroma.Config.put([:instance, :federation_incoming_replies_max_depth], 0) - - {:ok, _activity} = Transmogrifier.handle_incoming(federation_output) - - assert all_enqueued(worker: Pleroma.Workers.RemoteFetcherWorker) == [] - end - end - describe "prepare outgoing" do test "it inlines private announced objects" do user = insert(:user) @@ -618,8 +169,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do {:ok, activity} = CommonAPI.post(user, %{status: "hey"}) {:ok, modified} = Transmogrifier.prepare_outgoing(activity.data) - assert modified["@context"] == - Pleroma.Web.ActivityPub.Utils.make_json_ld_header()["@context"] + assert modified["@context"] == Utils.make_json_ld_header()["@context"] assert modified["object"]["conversation"] == modified["context"] end @@ -866,6 +416,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do "object" => %{ "to" => ["https://www.w3.org/ns/activitystreams#Public"], "cc" => [], + "id" => Utils.generate_object_id(), "type" => "Note", "content" => "Hi", "inReplyTo" => nil, @@ -890,6 +441,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do "object" => %{ "to" => ["https://www.w3.org/ns/activitystreams#Public"], "cc" => [], + "id" => Utils.generate_object_id(), "type" => "Note", "content" => "Hi", "inReplyTo" => nil, @@ -930,7 +482,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do end) } - fixed_object = Transmogrifier.fix_explicit_addressing(object) + fixed_object = Transmogrifier.fix_explicit_addressing(object, user.follower_address) assert Enum.all?(explicitly_mentioned_actors, &(&1 in fixed_object["to"])) refute "https://social.beepboop.ga/users/dirb" in fixed_object["to"] assert "https://social.beepboop.ga/users/dirb" in fixed_object["cc"] @@ -943,7 +495,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do "cc" => [] } - fixed_object = Transmogrifier.fix_explicit_addressing(object) + fixed_object = Transmogrifier.fix_explicit_addressing(object, user.follower_address) assert user.follower_address in fixed_object["to"] refute user.follower_address in fixed_object["cc"] end @@ -957,7 +509,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do "cc" => [user.follower_address, recipient.follower_address] } - fixed_object = Transmogrifier.fix_explicit_addressing(object) + fixed_object = Transmogrifier.fix_explicit_addressing(object, user.follower_address) assert user.follower_address in fixed_object["cc"] refute recipient.follower_address in fixed_object["cc"] diff --git a/test/pleroma/web/common_api/utils_test.exs b/test/pleroma/web/common_api/utils_test.exs @@ -559,7 +559,8 @@ defmodule Pleroma.Web.CommonAPI.UtilsTest do "context" => "2hu", "sensitive" => false, "summary" => "test summary", - "tag" => ["jimm"], + "hashtags" => ["jimm"], + "tag" => [], "to" => [user2.ap_id], "type" => "Note", "custom_tag" => "test" diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs @@ -479,7 +479,7 @@ defmodule Pleroma.Web.CommonAPITest do object = Object.normalize(activity) - assert object.data["tag"] == ["2hu"] + assert object.data["hashtags"] == ["2hu"] end test "it adds emoji in the object" do diff --git a/test/pleroma/web/federator_test.exs b/test/pleroma/web/federator_test.exs @@ -123,7 +123,8 @@ defmodule Pleroma.Web.FederatorTest do "type" => "Note", "content" => "hi world!", "id" => "http://mastodon.example.org/users/admin/objects/1", - "attributedTo" => "http://mastodon.example.org/users/admin" + "attributedTo" => "http://mastodon.example.org/users/admin", + "to" => ["https://www.w3.org/ns/activitystreams#Public"] }, "to" => ["https://www.w3.org/ns/activitystreams#Public"] } @@ -145,7 +146,8 @@ defmodule Pleroma.Web.FederatorTest do "type" => "Note", "content" => "hi world!", "id" => "http://mastodon.example.org/users/admin/objects/1", - "attributedTo" => "http://mastodon.example.org/users/admin" + "attributedTo" => "http://mastodon.example.org/users/admin", + "to" => ["https://www.w3.org/ns/activitystreams#Public"] }, "to" => ["https://www.w3.org/ns/activitystreams#Public"] } diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -216,12 +216,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do visibility: "public", media_attachments: [], mentions: [], - tags: [ - %{ - name: "#{object_data["tag"]}", - url: "/tag/#{object_data["tag"]}" - } - ], + tags: [], application: %{ name: "Web", website: nil diff --git a/test/pleroma/web/static_fe/static_fe_controller_test.exs b/test/pleroma/web/static_fe/static_fe_controller_test.exs @@ -8,6 +8,7 @@ defmodule Pleroma.Web.StaticFE.StaticFEControllerTest do alias Pleroma.Activity alias Pleroma.Config alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -168,16 +169,16 @@ defmodule Pleroma.Web.StaticFE.StaticFEControllerTest do test "302 for remote cached status", %{conn: conn, user: user} do message = %{ "@context" => "https://www.w3.org/ns/activitystreams", - "to" => user.follower_address, - "cc" => "https://www.w3.org/ns/activitystreams#Public", "type" => "Create", + "actor" => user.ap_id, "object" => %{ + "to" => user.follower_address, + "cc" => "https://www.w3.org/ns/activitystreams#Public", + "id" => Utils.generate_object_id(), "content" => "blah blah blah", "type" => "Note", - "attributedTo" => user.ap_id, - "inReplyTo" => nil - }, - "actor" => user.ap_id + "attributedTo" => user.ap_id + } } assert {:ok, activity} = Transmogrifier.handle_incoming(message) diff --git a/test/support/factory.ex b/test/support/factory.ex @@ -77,7 +77,13 @@ defmodule Pleroma.Factory do "like_count" => 0, "context" => "2hu", "summary" => "2hu", - "tag" => ["2hu"], + "tag" => [ + %{ + "type" => "Hashtag", + "name" => "#2hu", + "href" => Pleroma.Web.Endpoint.url() <> "/tags/2hu" + } + ], "emoji" => %{ "2hu" => "corndog.png" }