logo

pleroma

My custom branche(s) on git.pleroma.social/pleroma/pleroma
commit: 9da8b287f8a2c1fda2ca05351d5418321510472b
parent: cc93775ebf539598075c0b9afb96c60c8bede6b5
Author: kaniini <nenolod@gmail.com>
Date:   Fri, 19 Apr 2019 16:55:33 +0000

Merge branch 'feature/database-compaction' into 'develop'

database compaction

See merge request pleroma/pleroma!473

Diffstat:

MCHANGELOG.md1+
Alib/mix/tasks/pleroma/database.ex51+++++++++++++++++++++++++++++++++++++++++++++++++++
Mlib/pleroma/activity.ex79++++++++++++++++++++++---------------------------------------------------------
Mlib/pleroma/gopher/server.ex5+++--
Mlib/pleroma/html.ex3++-
Mlib/pleroma/object.ex22+++++++++++++---------
Alib/pleroma/object/containment.ex61+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Alib/pleroma/object/fetcher.ex75+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mlib/pleroma/web/activity_pub/activity_pub.ex103+++++++++++++++++++++++++------------------------------------------------------
Mlib/pleroma/web/activity_pub/activity_pub_controller.ex3++-
Mlib/pleroma/web/activity_pub/transmogrifier.ex93++++++++++++++++++-------------------------------------------------------------
Mlib/pleroma/web/activity_pub/utils.ex10+++++++---
Mlib/pleroma/web/activity_pub/visibility.ex23++++++++++++++---------
Mlib/pleroma/web/common_api/common_api.ex11++++++++---
Mlib/pleroma/web/common_api/utils.ex8+++++---
Mlib/pleroma/web/federator/federator.ex3++-
Mlib/pleroma/web/mastodon_api/mastodon_api_controller.ex26+++++++++++++++-----------
Mlib/pleroma/web/mastodon_api/views/status_view.ex81+++++++++++++++++++++++++++++++++++++++++++++----------------------------------
Mlib/pleroma/web/ostatus/activity_representer.ex42++++++++++++++++++------------------------
Mlib/pleroma/web/ostatus/handlers/note_handler.ex5+++--
Mlib/pleroma/web/twitter_api/controllers/util_controller.ex4++--
Mlib/pleroma/web/twitter_api/twitter_api.ex7++++---
Mlib/pleroma/web/twitter_api/views/activity_view.ex30++++++++++++++++--------------
Mtest/activity_test.exs14--------------
Atest/object/containment_test.exs58++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Atest/object/fetcher_test.exs90+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mtest/object_test.exs28++++++++++++++++++++++++++++
Mtest/scheduled_activity_worker_test.exs2+-
Mtest/user_test.exs7++++---
Mtest/web/activity_pub/activity_pub_test.exs98++++++++++++++++++++++++-------------------------------------------------------
Mtest/web/activity_pub/transmogrifier_test.exs126+++++++++++++++++++++----------------------------------------------------------
Mtest/web/common_api/common_api_test.exs17+++++++++++------
Mtest/web/mastodon_api/mastodon_api_controller_test.exs6+++---
Mtest/web/mastodon_api/status_view_test.exs15++++++++-------
Mtest/web/ostatus/ostatus_test.exs89++++++++++++++++++++++++++++++++++++++++++++-----------------------------------
Mtest/web/twitter_api/twitter_api_test.exs24+++++++++++++-----------
Mtest/web/twitter_api/views/activity_view_test.exs19+++++++++++--------
37 files changed, 762 insertions(+), 577 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - A [job queue](https://git.pleroma.social/pleroma/pleroma_job_queue) for federation, emails, web push, etc. - [Prometheus](https://prometheus.io/) metrics - Support for Mastodon's remote interaction +- Mix Tasks: `mix pleroma.database remove_embedded_objects` - Federation: Support for reports - Configuration: `safe_dm_mentions` option - Configuration: `link_name` option diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex @@ -0,0 +1,51 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Database do + alias Mix.Tasks.Pleroma.Common + require Logger + use Mix.Task + + @shortdoc "A collection of database related tasks" + @moduledoc """ + A collection of database related tasks + + ## Replace embedded objects with their references + + Replaces embedded objects with references to them in the `objects` table. Only needs to be ran once. The reason why this is not a migration is because it could significantly increase the database size after being ran, however after this `VACUUM FULL` will be able to reclaim about 20% (really depends on what is in the database, your mileage may vary) of the db size before the migration. + + mix pleroma.database remove_embedded_objects + + Options: + - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references + """ + def run(["remove_embedded_objects" | args]) do + {options, [], []} = + OptionParser.parse( + args, + strict: [ + vacuum: :boolean + ] + ) + + Common.start_pleroma() + Logger.info("Removing embedded objects") + + Pleroma.Repo.query!( + "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;", + [], + timeout: :infinity + ) + + if Keyword.get(options, :vacuum) do + Logger.info("Runnning VACUUM FULL") + + Pleroma.Repo.query!( + "vacuum full;", + [], + timeout: :infinity + ) + end + end +end diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Activity do alias Pleroma.Object alias Pleroma.Repo + import Ecto.Changeset import Ecto.Query @type t :: %__MODULE__{} @@ -79,6 +80,13 @@ defmodule Pleroma.Activity do ) end + def change(struct, params \\ %{}) do + struct + |> cast(params, [:data]) + |> validate_required([:data]) + |> unique_constraint(:ap_id, name: :activities_unique_apid_index) + end + def get_by_ap_id_with_object(ap_id) do Repo.one( from( @@ -196,21 +204,27 @@ defmodule Pleroma.Activity do def create_by_object_ap_id_with_object(_), do: nil - def get_create_by_object_ap_id_with_object(ap_id) do + def get_create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do ap_id |> create_by_object_ap_id_with_object() |> Repo.one() end - def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"]) - def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id_with_object(ap_id) - def normalize(_), do: nil + def get_create_by_object_ap_id_with_object(_), do: nil - def get_in_reply_to_activity(%Activity{data: %{"object" => %{"inReplyTo" => ap_id}}}) do - get_create_by_object_ap_id(ap_id) + defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do + get_create_by_object_ap_id_with_object(ap_id) end - def get_in_reply_to_activity(_), do: nil + defp get_in_reply_to_activity_from_object(_), do: nil + + def get_in_reply_to_activity(%Activity{data: %{"object" => object}}) do + get_in_reply_to_activity_from_object(Object.normalize(object)) + end + + def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"]) + def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id_with_object(ap_id) + def normalize(_), do: nil def delete_by_ap_id(id) when is_binary(id) do by_object_ap_id(id) @@ -218,6 +232,7 @@ defmodule Pleroma.Activity do |> Repo.delete_all() |> elem(1) |> Enum.find(fn + %{data: %{"type" => "Create", "object" => ap_id}} when is_binary(ap_id) -> ap_id == id %{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id _ -> nil end) @@ -245,54 +260,4 @@ defmodule Pleroma.Activity do |> where([s], s.actor == ^actor) |> Repo.all() end - - def increase_replies_count(nil), do: nil - - def increase_replies_count(object_ap_id) do - from(a in create_by_object_ap_id(object_ap_id), - update: [ - set: [ - data: - fragment( - """ - jsonb_set(?, '{object, repliesCount}', - (coalesce((?->'object'->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true) - """, - a.data, - a.data - ) - ] - ] - ) - |> Repo.update_all([]) - |> case do - {1, [activity]} -> activity - _ -> {:error, "Not found"} - end - end - - def decrease_replies_count(nil), do: nil - - def decrease_replies_count(object_ap_id) do - from(a in create_by_object_ap_id(object_ap_id), - update: [ - set: [ - data: - fragment( - """ - jsonb_set(?, '{object, repliesCount}', - (greatest(0, (?->'object'->>'repliesCount')::int - 1))::varchar::jsonb, true) - """, - a.data, - a.data - ) - ] - ] - ) - |> Repo.update_all([]) - |> case do - {1, [activity]} -> activity - _ -> {:error, "Not found"} - end - end end diff --git a/lib/pleroma/gopher/server.ex b/lib/pleroma/gopher/server.ex @@ -38,6 +38,7 @@ end defmodule Pleroma.Gopher.Server.ProtocolHandler do alias Pleroma.Activity alias Pleroma.HTML + alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Visibility @@ -75,14 +76,14 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do |> Enum.map(fn activity -> user = User.get_cached_by_ap_id(activity.data["actor"]) - object = activity.data["object"] + object = Object.normalize(activity.data["object"]) like_count = object["like_count"] || 0 announcement_count = object["announcement_count"] || 0 link("Post ##{activity.id} by #{user.nickname}", "/notices/#{activity.id}") <> info("#{like_count} likes, #{announcement_count} repeats") <> "i\tfake\t(NULL)\t0\r\n" <> - info(HTML.strip_tags(String.replace(activity.data["object"]["content"], "<br>", "\r"))) + info(HTML.strip_tags(String.replace(object["content"], "<br>", "\r"))) end) |> Enum.join("i\tfake\t(NULL)\t0\r\n") end diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex @@ -32,7 +32,8 @@ defmodule Pleroma.HTML do key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}" Cachex.fetch!(:scrubber_cache, key, fn _key -> - ensure_scrubbed_html(content, scrubbers, activity.data["object"]["fake"] || false) + object = Pleroma.Object.normalize(activity) + ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false) end) end diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Object do alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.Object.Fetcher alias Pleroma.ObjectTombstone alias Pleroma.Repo alias Pleroma.User @@ -40,41 +41,44 @@ defmodule Pleroma.Object do Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id))) end + def normalize(_, fetch_remote \\ true) # If we pass an Activity to Object.normalize(), we can try to use the preloaded object. # Use this whenever possible, especially when walking graphs in an O(N) loop! - def normalize(%Activity{object: %Object{} = object}), do: object + def normalize(%Object{} = object, _), do: object + def normalize(%Activity{object: %Object{} = object}, _), do: object # A hack for fake activities - def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}) do + def normalize(%Activity{data: %{"object" => %{"fake" => true} = data}}, _) do %Object{id: "pleroma:fake_object_id", data: data} end # Catch and log Object.normalize() calls where the Activity's child object is not # preloaded. - def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}) do + def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}, fetch_remote) do Logger.debug( "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" ) Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}") - normalize(ap_id) + normalize(ap_id, fetch_remote) end - def normalize(%Activity{data: %{"object" => ap_id}}) do + def normalize(%Activity{data: %{"object" => ap_id}}, fetch_remote) do Logger.debug( "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" ) Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}") - normalize(ap_id) + normalize(ap_id, fetch_remote) end # Old way, try fetching the object through cache. - def normalize(%{"id" => ap_id}), do: normalize(ap_id) - def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) - def normalize(_), do: nil + def normalize(%{"id" => ap_id}, fetch_remote), do: normalize(ap_id, fetch_remote) + def normalize(ap_id, false) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) + def normalize(ap_id, true) when is_binary(ap_id), do: Fetcher.fetch_object_from_id!(ap_id) + def normalize(_, _), do: nil # Owned objects can only be mutated by their owner def authorize_mutation(%Object{data: %{"actor" => actor}}, %User{ap_id: ap_id}), diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex @@ -0,0 +1,61 @@ +defmodule Pleroma.Object.Containment do + @moduledoc """ + # Object Containment + + This module contains some useful functions for containing objects to specific + origins and determining those origins. They previously lived in the + ActivityPub `Transmogrifier` module. + + Object containment is an important step in validating remote objects to prevent + spoofing, therefore removal of object containment functions is NOT recommended. + """ + def get_actor(%{"actor" => actor}) when is_binary(actor) do + actor + end + + def get_actor(%{"actor" => actor}) when is_list(actor) do + if is_binary(Enum.at(actor, 0)) do + Enum.at(actor, 0) + else + Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end) + |> Map.get("id") + end + end + + def get_actor(%{"actor" => %{"id" => id}}) when is_bitstring(id) do + id + end + + def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do + get_actor(%{"actor" => actor}) + end + + @doc """ + Checks that an imported AP object's actor matches the domain it came from. + """ + def contain_origin(_id, %{"actor" => nil}), do: :error + + def contain_origin(id, %{"actor" => _actor} = params) do + id_uri = URI.parse(id) + actor_uri = URI.parse(get_actor(params)) + + if id_uri.host == actor_uri.host do + :ok + else + :error + end + end + + def contain_origin_from_id(_id, %{"id" => nil}), do: :error + + def contain_origin_from_id(id, %{"id" => other_id} = _params) do + id_uri = URI.parse(id) + other_uri = URI.parse(other_id) + + if id_uri.host == other_uri.host do + :ok + else + :error + end + end +end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex @@ -0,0 +1,75 @@ +defmodule Pleroma.Object.Fetcher do + alias Pleroma.Object + alias Pleroma.Object.Containment + alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.OStatus + + require Logger + + @httpoison Application.get_env(:pleroma, :httpoison) + + # TODO: + # This will create a Create activity, which we need internally at the moment. + def fetch_object_from_id(id) do + if object = Object.get_cached_by_ap_id(id) do + {:ok, object} + else + Logger.info("Fetching #{id} via AP") + + with {:ok, data} <- fetch_and_contain_remote_object_from_id(id), + nil <- Object.normalize(data, false), + params <- %{ + "type" => "Create", + "to" => data["to"], + "cc" => data["cc"], + "actor" => data["actor"] || data["attributedTo"], + "object" => data + }, + :ok <- Containment.contain_origin(id, params), + {:ok, activity} <- Transmogrifier.handle_incoming(params) do + {:ok, Object.normalize(activity, false)} + else + {:error, {:reject, nil}} -> + {:reject, nil} + + object = %Object{} -> + {:ok, object} + + _e -> + Logger.info("Couldn't get object via AP, trying out OStatus fetching...") + + case OStatus.fetch_activity_from_url(id) do + {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"], false)} + e -> e + end + end + end + end + + def fetch_object_from_id!(id) do + with {:ok, object} <- fetch_object_from_id(id) do + object + else + _e -> + nil + end + end + + def fetch_and_contain_remote_object_from_id(id) do + Logger.info("Fetching object #{id} via AP") + + with true <- String.starts_with?(id, "http"), + {:ok, %{body: body, status: code}} when code in 200..299 <- + @httpoison.get( + id, + [{:Accept, "application/activity+json"}] + ), + {:ok, data} <- Jason.decode(body), + :ok <- Containment.contain_origin_from_id(id, data) do + {:ok, data} + else + e -> + {:error, e} + end + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Instances alias Pleroma.Notification alias Pleroma.Object + alias Pleroma.Object.Fetcher alias Pleroma.Pagination alias Pleroma.Repo alias Pleroma.Upload @@ -14,7 +15,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.Federator - alias Pleroma.Web.OStatus alias Pleroma.Web.WebFinger import Ecto.Query @@ -95,7 +95,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do "type" => "Create" }) do if is_public?(object) do - Activity.increase_replies_count(reply_ap_id) Object.increase_replies_count(reply_ap_id) end end @@ -106,7 +105,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do data: %{"inReplyTo" => reply_ap_id} = object }) do if is_public?(object) do - Activity.decrease_replies_count(reply_ap_id) Object.decrease_replies_count(reply_ap_id) end end @@ -121,7 +119,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {:ok, map} <- MRF.filter(map), {recipients, _, _} = get_recipients(map), {:fake, false, map, recipients} <- {:fake, fake, map, recipients}, - {:ok, object} <- insert_full_object(map) do + {:ok, map, object} <- insert_full_object(map) do {:ok, activity} = Repo.insert(%Activity{ data: map, @@ -170,6 +168,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do public = "https://www.w3.org/ns/activitystreams#Public" if activity.data["type"] in ["Create", "Announce", "Delete"] do + object = Object.normalize(activity.data["object"]) Pleroma.Web.Streamer.stream("user", activity) Pleroma.Web.Streamer.stream("list", activity) @@ -181,12 +180,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end if activity.data["type"] in ["Create"] do - activity.data["object"] + object.data |> Map.get("tag", []) |> Enum.filter(fn tag -> is_bitstring(tag) end) |> Enum.each(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end) - if activity.data["object"]["attachment"] != [] do + if object.data["attachment"] != [] do Pleroma.Web.Streamer.stream("public:media", activity) if activity.local do @@ -572,37 +571,49 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_since(query, _), do: query + defp restrict_tag_reject(_query, %{"tag_reject" => _tag_reject, "skip_preload" => true}) do + raise "Can't use the child object without preloading!" + end + defp restrict_tag_reject(query, %{"tag_reject" => tag_reject}) when is_list(tag_reject) and tag_reject != [] do from( - activity in query, - where: fragment(~s(\(not \(? #> '{"object","tag"}'\) \\?| ?\)), activity.data, ^tag_reject) + [_activity, object] in query, + where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject) ) end defp restrict_tag_reject(query, _), do: query + defp restrict_tag_all(_query, %{"tag_all" => _tag_all, "skip_preload" => true}) do + raise "Can't use the child object without preloading!" + end + defp restrict_tag_all(query, %{"tag_all" => tag_all}) when is_list(tag_all) and tag_all != [] do from( - activity in query, - where: fragment(~s(\(? #> '{"object","tag"}'\) \\?& ?), activity.data, ^tag_all) + [_activity, object] in query, + where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all) ) end defp restrict_tag_all(query, _), do: query + defp restrict_tag(_query, %{"tag" => _tag, "skip_preload" => true}) do + raise "Can't use the child object without preloading!" + end + defp restrict_tag(query, %{"tag" => tag}) when is_list(tag) do from( - activity in query, - where: fragment(~s(\(? #> '{"object","tag"}'\) \\?| ?), activity.data, ^tag) + [_activity, object] in query, + where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag) ) end defp restrict_tag(query, %{"tag" => tag}) when is_binary(tag) do from( - activity in query, - where: fragment(~s(? <@ (? #> '{"object","tag"}'\)), ^tag, activity.data) + [_activity, object] in query, + where: fragment("(?)->'tag' \\? (?)", object.data, ^tag) ) end @@ -667,10 +678,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_favorited_by(query, _), do: query + defp restrict_media(_query, %{"only_media" => _val, "skip_preload" => true}) do + raise "Can't use the child object without preloading!" + end + defp restrict_media(query, %{"only_media" => val}) when val == "true" or val == "1" do from( - activity in query, - where: fragment(~s(not (? #> '{"object","attachment"}' = ?\)), activity.data, ^[]) + [_activity, object] in query, + where: fragment("not (?)->'attachment' = (?)", object.data, ^[]) ) end @@ -866,7 +881,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end def fetch_and_prepare_user_from_ap_id(ap_id) do - with {:ok, data} <- fetch_and_contain_remote_object_from_id(ap_id) do + with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do user_data_from_user_object(data) else e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}") @@ -976,60 +991,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - # TODO: - # This will create a Create activity, which we need internally at the moment. - def fetch_object_from_id(id) do - if object = Object.get_cached_by_ap_id(id) do - {:ok, object} - else - with {:ok, data} <- fetch_and_contain_remote_object_from_id(id), - nil <- Object.normalize(data), - params <- %{ - "type" => "Create", - "to" => data["to"], - "cc" => data["cc"], - "actor" => data["actor"] || data["attributedTo"], - "object" => data - }, - :ok <- Transmogrifier.contain_origin(id, params), - {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, Object.normalize(activity)} - else - {:error, {:reject, nil}} -> - {:reject, nil} - - object = %Object{} -> - {:ok, object} - - _e -> - Logger.info("Couldn't get object via AP, trying out OStatus fetching...") - - case OStatus.fetch_activity_from_url(id) do - {:ok, [activity | _]} -> {:ok, Object.normalize(activity)} - e -> e - end - end - end - end - - def fetch_and_contain_remote_object_from_id(id) do - Logger.info("Fetching object #{id} via AP") - - with true <- String.starts_with?(id, "http"), - {:ok, %{body: body, status: code}} when code in 200..299 <- - @httpoison.get( - id, - [{:Accept, "application/activity+json"}] - ), - {:ok, data} <- Jason.decode(body), - :ok <- Transmogrifier.contain_origin_from_id(id, data) do - {:ok, data} - else - e -> - {:error, e} - end - end - # filter out broken threads def contain_broken_threads(%Activity{} = activity, %User{} = user) do entire_thread_visible_for_user?(activity, user) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.Object.Fetcher alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.ObjectView @@ -173,7 +174,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do "Signature missing or not from author, relayed Create message, fetching object from source" ) - ActivityPub.fetch_object_from_id(params["object"]["id"]) + Fetcher.fetch_object_from_id(params["object"]["id"]) json(conn, "ok") end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -8,8 +8,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do """ alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.Object.Containment alias Pleroma.Repo alias Pleroma.User + alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility @@ -18,56 +20,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do require Logger - def get_actor(%{"actor" => actor}) when is_binary(actor) do - actor - end - - def get_actor(%{"actor" => actor}) when is_list(actor) do - if is_binary(Enum.at(actor, 0)) do - Enum.at(actor, 0) - else - Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end) - |> Map.get("id") - end - end - - def get_actor(%{"actor" => %{"id" => id}}) when is_bitstring(id) do - id - end - - def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do - get_actor(%{"actor" => actor}) - end - - @doc """ - Checks that an imported AP object's actor matches the domain it came from. - """ - def contain_origin(_id, %{"actor" => nil}), do: :error - - def contain_origin(id, %{"actor" => _actor} = params) do - id_uri = URI.parse(id) - actor_uri = URI.parse(get_actor(params)) - - if id_uri.host == actor_uri.host do - :ok - else - :error - end - end - - def contain_origin_from_id(_id, %{"id" => nil}), do: :error - - def contain_origin_from_id(id, %{"id" => other_id} = _params) do - id_uri = URI.parse(id) - other_uri = URI.parse(other_id) - - if id_uri.host == other_uri.host do - :ok - else - :error - end - end - @doc """ Modifies an incoming AP object (mastodon format) to our internal format. """ @@ -188,7 +140,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_actor(%{"attributedTo" => actor} = object) do object - |> Map.put("actor", get_actor(%{"actor" => actor})) + |> Map.put("actor", Containment.get_actor(%{"actor" => actor})) end # Check for standardisation @@ -223,7 +175,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "" end - case fetch_obj_helper(in_reply_to_id) do + case get_obj_helper(in_reply_to_id) do {:ok, replied_object} -> with %Activity{} = _activity <- Activity.get_create_by_object_ap_id(replied_object.data["id"]) do @@ -448,7 +400,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do # - emoji def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data) when objtype in ["Article", "Note", "Video", "Page"] do - actor = get_actor(data) + actor = Containment.get_actor(data) data = Map.put(data, "actor", actor) @@ -506,7 +458,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming( %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = followed <- User.get_or_fetch_by_ap_id(actor), {:ok, follow_activity} <- get_follow_activity(follow_object, followed), {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "accept"), @@ -532,7 +484,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming( %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = followed <- User.get_or_fetch_by_ap_id(actor), {:ok, follow_activity} <- get_follow_activity(follow_object, followed), {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"), @@ -556,9 +508,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming( %{"type" => "Like", "object" => object_id, "actor" => _actor, "id" => id} = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), + {:ok, object} <- get_obj_helper(object_id), {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do {:ok, activity} else @@ -569,9 +521,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming( %{"type" => "Announce", "object" => object_id, "actor" => _actor, "id" => id} = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), + {:ok, object} <- get_obj_helper(object_id), public <- Visibility.is_public?(data), {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false, public) do {:ok, activity} @@ -624,10 +576,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do ) do object_id = Utils.get_ap_id(object_id) - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), - :ok <- contain_origin(actor.ap_id, object.data), + {:ok, object} <- get_obj_helper(object_id), + :ok <- Containment.contain_origin(actor.ap_id, object.data), {:ok, activity} <- ActivityPub.delete(object, false) do {:ok, activity} else @@ -643,9 +595,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "id" => id } = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), + {:ok, object} <- get_obj_helper(object_id), {:ok, activity, _} <- ActivityPub.unannounce(actor, object, id, false) do {:ok, activity} else @@ -713,9 +665,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do "id" => id } = data ) do - with actor <- get_actor(data), + with actor <- Containment.get_actor(data), %User{} = actor <- User.get_or_fetch_by_ap_id(actor), - {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id), + {:ok, object} <- get_obj_helper(object_id), {:ok, activity, _, _} <- ActivityPub.unlike(actor, object, id, false) do {:ok, activity} else @@ -725,9 +677,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def handle_incoming(_), do: :error - def fetch_obj_helper(id) when is_bitstring(id), do: ActivityPub.fetch_object_from_id(id) - def fetch_obj_helper(obj) when is_map(obj), do: ActivityPub.fetch_object_from_id(obj["id"]) - def get_obj_helper(id) do if object = Object.normalize(id), do: {:ok, object}, else: nil end @@ -764,9 +713,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do # internal -> Mastodon # """ - def prepare_outgoing(%{"type" => "Create", "object" => object} = data) do + def prepare_outgoing(%{"type" => "Create", "object" => object_id} = data) do object = - object + Object.normalize(object_id).data |> prepare_object data = @@ -827,7 +776,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def maybe_fix_object_url(data) do if is_binary(data["object"]) and not String.starts_with?(data["object"], "http") do - case fetch_obj_helper(data["object"]) do + case get_obj_helper(data["object"]) do {:ok, relative_object} -> if relative_object.data["external_url"] do _data = diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex @@ -234,14 +234,18 @@ defmodule Pleroma.Web.ActivityPub.Utils do @doc """ Inserts a full object if it is contained in an activity. """ - def insert_full_object(%{"object" => %{"type" => type} = object_data}) + def insert_full_object(%{"object" => %{"type" => type} = object_data} = map) when is_map(object_data) and type in @supported_object_types do with {:ok, object} <- Object.create(object_data) do - {:ok, object} + map = + map + |> Map.put("object", object.data["id"]) + + {:ok, map, object} end end - def insert_full_object(_), do: {:ok, nil} + def insert_full_object(map), do: {:ok, map, nil} def update_object_in_activities(%{data: %{"id" => id}} = object) do # TODO diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex @@ -41,16 +41,21 @@ defmodule Pleroma.Web.ActivityPub.Visibility do # guard def entire_thread_visible_for_user?(nil, _user), do: false - # child + # XXX: Probably even more inefficient than the previous implementation intended to be a placeholder untill https://git.pleroma.social/pleroma/pleroma/merge_requests/971 is in develop + # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength + def entire_thread_visible_for_user?( - %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail, + %Activity{} = tail, + # %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail, user - ) - when is_binary(parent_id) do - parent = Activity.get_in_reply_to_activity(tail) - visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user) + ) do + case Object.normalize(tail) do + %{data: %{"inReplyTo" => parent_id}} when is_binary(parent_id) -> + parent = Activity.get_in_reply_to_activity(tail) + visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user) + + _ -> + visible_for_user?(tail, user) + end end - - # root - def entire_thread_visible_for_user?(tail, user), do: visible_for_user?(tail, user) end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex @@ -125,7 +125,10 @@ defmodule Pleroma.Web.CommonAPI do "public" in_reply_to -> - Pleroma.Web.MastodonAPI.StatusView.get_visibility(in_reply_to.data["object"]) + # XXX: these heuristics should be moved out of MastodonAPI. + with %Object{} = object <- Object.normalize(in_reply_to) do + Pleroma.Web.MastodonAPI.StatusView.get_visibility(object) + end end end @@ -214,8 +217,10 @@ defmodule Pleroma.Web.CommonAPI do with %Activity{ actor: ^user_ap_id, data: %{ - "type" => "Create", - "object" => %{ + "type" => "Create" + }, + object: %Object{ + data: %{ "to" => object_to, "type" => "Note" } diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex @@ -208,7 +208,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do context, content_html, attachments, - inReplyTo, + in_reply_to, tags, cw \\ nil, cc \\ [] @@ -225,9 +225,11 @@ defmodule Pleroma.Web.CommonAPI.Utils do "tag" => tags |> Enum.map(fn {_, tag} -> tag end) |> Enum.uniq() } - if inReplyTo do + if in_reply_to do + in_reply_to_object = Object.normalize(in_reply_to.data["object"]) + object - |> Map.put("inReplyTo", inReplyTo.data["object"]["id"]) + |> Map.put("inReplyTo", in_reply_to_object.data["id"]) else object end diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Activity + alias Pleroma.Object.Containment alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Relay @@ -136,7 +137,7 @@ defmodule Pleroma.Web.Federator do # actor shouldn't be acting on objects outside their own AP server. with {:ok, _user} <- ap_enabled_actor(params["actor"]), nil <- Activity.normalize(params["id"]), - :ok <- Transmogrifier.contain_origin_from_id(params["actor"], params), + :ok <- Containment.contain_origin_from_id(params["actor"], params), {:ok, activity} <- Transmogrifier.handle_incoming(params) do {:ok, activity} else diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -4,13 +4,13 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do use Pleroma.Web, :controller - alias Ecto.Changeset alias Pleroma.Activity alias Pleroma.Config alias Pleroma.Filter alias Pleroma.Notification alias Pleroma.Object + alias Pleroma.Object.Fetcher alias Pleroma.Pagination alias Pleroma.Repo alias Pleroma.ScheduledActivity @@ -543,10 +543,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with %Activity{} = activity <- Activity.get_by_id(id), + with %Activity{} = activity <- Activity.get_by_id_with_object(id), + %Object{} = object <- Object.normalize(activity), %User{} = user <- User.get_by_nickname(user.nickname), true <- Visibility.visible_for_user?(activity, user), - {:ok, user} <- User.bookmark(user, activity.data["object"]["id"]) do + {:ok, user} <- User.bookmark(user, object.data["id"]) do conn |> put_view(StatusView) |> try_render("status.json", %{activity: activity, for: user, as: :activity}) @@ -554,10 +555,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end def unbookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with %Activity{} = activity <- Activity.get_by_id(id), + with %Activity{} = activity <- Activity.get_by_id_with_object(id), + %Object{} = object <- Object.normalize(activity), %User{} = user <- User.get_by_nickname(user.nickname), true <- Visibility.visible_for_user?(activity, user), - {:ok, user} <- User.unbookmark(user, activity.data["object"]["id"]) do + {:ok, user} <- User.unbookmark(user, object.data["id"]) do conn |> put_view(StatusView) |> try_render("status.json", %{activity: activity, for: user, as: :activity}) @@ -681,7 +683,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end def favourited_by(conn, %{"id" => id}) do - with %Activity{data: %{"object" => %{"likes" => likes}}} <- Activity.get_by_id(id) do + with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id), + %Object{data: %{"likes" => likes}} <- Object.normalize(object) do q = from(u in User, where: u.ap_id in ^likes) users = Repo.all(q) @@ -694,7 +697,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end def reblogged_by(conn, %{"id" => id}) do - with %Activity{data: %{"object" => %{"announcements" => announces}}} <- Activity.get_by_id(id) do + with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id), + %Object{data: %{"announcements" => announces}} <- Object.normalize(object) do q = from(u in User, where: u.ap_id in ^announces) users = Repo.all(q) @@ -997,7 +1001,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do def status_search(user, query) do fetched = if Regex.match?(~r/https?:/, query) do - with {:ok, object} <- ActivityPub.fetch_object_from_id(query), + with {:ok, object} <- Fetcher.fetch_object_from_id(query), %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), true <- Visibility.visible_for_user?(activity, user) do [activity] @@ -1008,13 +1012,13 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do q = from( - a in Activity, + [a, o] in Activity.with_preloaded_object(Activity), where: fragment("?->>'type' = 'Create'", a.data), where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients, where: fragment( - "to_tsvector('english', ?->'object'->>'content') @@ plainto_tsquery('english', ?)", - a.data, + "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", + o.data, ^query ), limit: 20, diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.Activity alias Pleroma.HTML + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.CommonAPI @@ -19,8 +20,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do defp get_replied_to_activities(activities) do activities |> Enum.map(fn - %{data: %{"type" => "Create", "object" => %{"inReplyTo" => in_reply_to}}} -> - in_reply_to != "" && in_reply_to + %{data: %{"type" => "Create", "object" => object}} -> + object = Object.normalize(object) + object.data["inReplyTo"] != "" && object.data["inReplyTo"] _ -> nil @@ -29,7 +31,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do |> Activity.create_by_object_ap_id() |> Repo.all() |> Enum.reduce(%{}, fn activity, acc -> - Map.put(acc, activity.data["object"]["id"], activity) + object = Object.normalize(activity.data["object"]) + Map.put(acc, object.data["id"], activity) end) end @@ -55,8 +58,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do defp get_context_id(_), do: nil defp reblogged?(activity, user) do - object = activity.data["object"] || %{} - present?(user && user.ap_id in (object["announcements"] || [])) + object = Object.normalize(activity) || %{} + present?(user && user.ap_id in (object.data["announcements"] || [])) end def render("index.json", opts) do @@ -122,14 +125,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do } end - def render("status.json", %{activity: %{data: %{"object" => object}} = activity} = opts) do + def render("status.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do + object = Object.normalize(activity) + user = get_user(activity.data["actor"]) - like_count = object["like_count"] || 0 - announcement_count = object["announcement_count"] || 0 + like_count = object.data["like_count"] || 0 + announcement_count = object.data["announcement_count"] || 0 - tags = object["tag"] || [] - sensitive = object["sensitive"] || Enum.member?(tags, "nsfw") + tags = object.data["tag"] || [] + sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw") mentions = activity.recipients @@ -137,15 +142,17 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do |> Enum.filter(& &1) |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end) - favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || []) - bookmarked = opts[:for] && object["id"] in opts[:for].bookmarks + favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || []) + + bookmarked = opts[:for] && object.data["id"] in opts[:for].bookmarks - attachment_data = object["attachment"] || [] + attachment_data = object.data["attachment"] || [] attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment) - created_at = Utils.to_masto_date(object["published"]) + created_at = Utils.to_masto_date(object.data["published"]) reply_to = get_reply_to(activity, opts) + reply_to_user = reply_to && get_user(reply_to.data["actor"]) content = @@ -167,7 +174,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do "mastoapi:content" ) - summary = object["summary"] || "" + summary = object.data["summary"] || "" summary_html = summary @@ -190,12 +197,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do if user.local do Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity) else - object["external_url"] || object["id"] + object.data["external_url"] || object.data["id"] end %{ id: to_string(activity.id), - uri: object["id"], + uri: object.data["id"], url: url, account: AccountView.render("account.json", %{user: user}), in_reply_to_id: reply_to && to_string(reply_to.id), @@ -205,7 +212,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do content: content_html, created_at: created_at, reblogs_count: announcement_count, - replies_count: object["repliesCount"] || 0, + replies_count: object.data["repliesCount"] || 0, favourites_count: like_count, reblogged: reblogged?(activity, opts[:for]), favourited: present?(favorited), @@ -223,7 +230,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do website: nil }, language: nil, - emojis: build_emojis(activity.data["object"]["emoji"]), + emojis: build_emojis(object.data["emoji"]), pleroma: %{ local: activity.local, conversation_id: get_context_id(activity), @@ -305,15 +312,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do - with nil <- replied_to_activities[activity.data["object"]["inReplyTo"]] do + object = Object.normalize(activity.data["object"]) + + with nil <- replied_to_activities[object.data["inReplyTo"]] do # If user didn't participate in the thread Activity.get_in_reply_to_activity(activity) end end - def get_reply_to(%{data: %{"object" => object}}, _) do - if object["inReplyTo"] && object["inReplyTo"] != "" do - Activity.get_create_by_object_ap_id(object["inReplyTo"]) + def get_reply_to(%{data: %{"object" => _object}} = activity, _) do + object = Object.normalize(activity) + + if object.data["inReplyTo"] && object.data["inReplyTo"] != "" do + Activity.get_create_by_object_ap_id(object.data["inReplyTo"]) else nil end @@ -321,8 +332,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do def get_visibility(object) do public = "https://www.w3.org/ns/activitystreams#Public" - to = object["to"] || [] - cc = object["cc"] || [] + to = object.data["to"] || [] + cc = object.data["cc"] || [] cond do public in to -> @@ -343,25 +354,25 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end end - def render_content(%{"type" => "Video"} = object) do - with name when not is_nil(name) and name != "" <- object["name"] do - "<p><a href=\"#{object["id"]}\">#{name}</a></p>#{object["content"]}" + def render_content(%{data: %{"type" => "Video"}} = object) do + with name when not is_nil(name) and name != "" <- object.data["name"] do + "<p><a href=\"#{object.data["id"]}\">#{name}</a></p>#{object.data["content"]}" else - _ -> object["content"] || "" + _ -> object.data["content"] || "" end end - def render_content(%{"type" => object_type} = object) + def render_content(%{data: %{"type" => object_type}} = object) when object_type in ["Article", "Page"] do - with summary when not is_nil(summary) and summary != "" <- object["name"], - url when is_bitstring(url) <- object["url"] do - "<p><a href=\"#{url}\">#{summary}</a></p>#{object["content"]}" + with summary when not is_nil(summary) and summary != "" <- object.data["name"], + url when is_bitstring(url) <- object.data["url"] do + "<p><a href=\"#{url}\">#{summary}</a></p>#{object.data["content"]}" else - _ -> object["content"] || "" + _ -> object.data["content"] || "" end end - def render_content(object), do: object["content"] || "" + def render_content(object), do: object.data["content"] || "" @doc """ Builds a dictionary tags. diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex @@ -54,23 +54,16 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do end) end - defp get_links(%{local: true, data: data}) do + defp get_links(%{local: true}, %{"id" => object_id}) do h = fn str -> [to_charlist(str)] end [ - {:link, [type: ['application/atom+xml'], href: h.(data["object"]["id"]), rel: 'self'], []}, - {:link, [type: ['text/html'], href: h.(data["object"]["id"]), rel: 'alternate'], []} + {:link, [type: ['application/atom+xml'], href: h.(object_id), rel: 'self'], []}, + {:link, [type: ['text/html'], href: h.(object_id), rel: 'alternate'], []} ] end - defp get_links(%{ - local: false, - data: %{ - "object" => %{ - "external_url" => external_url - } - } - }) do + defp get_links(%{local: false}, %{"external_url" => external_url}) do h = fn str -> [to_charlist(str)] end [ @@ -78,7 +71,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do ] end - defp get_links(_activity), do: [] + defp get_links(_activity, _object_data), do: [] defp get_emoji_links(emojis) do Enum.map(emojis, fn {emoji, file} -> @@ -88,14 +81,16 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do def to_simple_form(activity, user, with_author \\ false) - def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do + def to_simple_form(%{data: %{"type" => "Create"}} = activity, user, with_author) do h = fn str -> [to_charlist(str)] end - updated_at = activity.data["object"]["published"] - inserted_at = activity.data["object"]["published"] + object = Object.normalize(activity.data["object"]) + + updated_at = object.data["published"] + inserted_at = object.data["published"] attachments = - Enum.map(activity.data["object"]["attachment"] || [], fn attachment -> + Enum.map(object.data["attachment"] || [], fn attachment -> url = hd(attachment["url"]) {:link, @@ -108,7 +103,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do mentions = activity.recipients |> get_mentions categories = - (activity.data["object"]["tag"] || []) + (object.data["tag"] || []) |> Enum.map(fn tag -> if is_binary(tag) do {:category, [term: to_charlist(tag)], []} @@ -118,11 +113,11 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do end) |> Enum.filter(& &1) - emoji_links = get_emoji_links(activity.data["object"]["emoji"] || %{}) + emoji_links = get_emoji_links(object.data["emoji"] || %{}) summary = - if activity.data["object"]["summary"] do - [{:summary, [], h.(activity.data["object"]["summary"])}] + if object.data["summary"] do + [{:summary, [], h.(object.data["summary"])}] else [] end @@ -131,10 +126,9 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']}, {:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']}, # For notes, federate the object id. - {:id, h.(activity.data["object"]["id"])}, + {:id, h.(object.data["id"])}, {:title, ['New note by #{user.nickname}']}, - {:content, [type: 'html'], - h.(activity.data["object"]["content"] |> String.replace(~r/[\n\r]/, ""))}, + {:content, [type: 'html'], h.(object.data["content"] |> String.replace(~r/[\n\r]/, ""))}, {:published, h.(inserted_at)}, {:updated, h.(updated_at)}, {:"ostatus:conversation", [ref: h.(activity.data["context"])], @@ -142,7 +136,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do {:link, [ref: h.(activity.data["context"]), rel: 'ostatus:conversation'], []} ] ++ summary ++ - get_links(activity) ++ + get_links(activity, object.data) ++ categories ++ attachments ++ in_reply_to ++ author ++ mentions ++ emoji_links end diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex @@ -113,8 +113,9 @@ defmodule Pleroma.Web.OStatus.NoteHandler do cw <- OStatus.get_cw(entry), in_reply_to <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry), in_reply_to_activity <- fetch_replied_to_activity(entry, in_reply_to), - in_reply_to <- - (in_reply_to_activity && in_reply_to_activity.data["object"]["id"]) || in_reply_to, + in_reply_to_object <- + (in_reply_to_activity && Object.normalize(in_reply_to_activity)) || nil, + in_reply_to <- (in_reply_to_object && in_reply_to_object.data["id"]) || in_reply_to, attachments <- OStatus.get_attachments(entry), context <- get_context(entry, in_reply_to), tags <- OStatus.get_tags(entry), diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -75,7 +75,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do def remote_follow(%{assigns: %{user: user}} = conn, %{"acct" => acct}) do if is_status?(acct) do - {:ok, object} = ActivityPub.fetch_object_from_id(acct) + {:ok, object} = Pleroma.Object.Fetcher.fetch_object_from_id(acct) %Activity{id: activity_id} = Activity.get_create_by_object_ap_id(object.data["id"]) redirect(conn, to: "/notice/#{activity_id}") else @@ -101,7 +101,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end defp is_status?(acct) do - case ActivityPub.fetch_and_contain_remote_object_from_id(acct) do + case Pleroma.Object.Fetcher.fetch_and_contain_remote_object_from_id(acct) do {:ok, %{"type" => type}} when type in ["Article", "Note", "Video", "Page", "Question"] -> true diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -266,6 +266,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do defp parse_int(_, default), do: default + # TODO: unify the search query with MastoAPI one and do only pagination here def search(_user, %{"q" => query} = params) do limit = parse_int(params["rpp"], 20) page = parse_int(params["page"], 1) @@ -273,13 +274,13 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do q = from( - a in Activity, + [a, o] in Activity.with_preloaded_object(Activity), where: fragment("?->>'type' = 'Create'", a.data), where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients, where: fragment( - "to_tsvector('english', ?->'object'->>'content') @@ plainto_tsquery('english', ?)", - a.data, + "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", + o.data, ^query ), limit: ^limit, diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -224,15 +224,17 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do def render( "activity.json", - %{activity: %{data: %{"type" => "Create", "object" => object}} = activity} = opts + %{activity: %{data: %{"type" => "Create", "object" => object_id}} = activity} = opts ) do user = get_user(activity.data["actor"], opts) - created_at = object["published"] |> Utils.date_to_asctime() - like_count = object["like_count"] || 0 - announcement_count = object["announcement_count"] || 0 - favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || []) - repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || []) + object = Object.normalize(object_id) + + created_at = object.data["published"] |> Utils.date_to_asctime() + like_count = object.data["like_count"] || 0 + announcement_count = object.data["announcement_count"] || 0 + favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || []) + repeated = opts[:for] && opts[:for].ap_id in (object.data["announcements"] || []) pinned = activity.id in user.info.pinned_activities attentions = @@ -245,12 +247,12 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do conversation_id = get_context_id(activity, opts) - tags = activity.data["object"]["tag"] || [] - possibly_sensitive = activity.data["object"]["sensitive"] || Enum.member?(tags, "nsfw") + tags = object.data["tag"] || [] + possibly_sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw") tags = if possibly_sensitive, do: Enum.uniq(["nsfw" | tags]), else: tags - {summary, content} = render_content(object) + {summary, content} = render_content(object.data) html = content @@ -259,7 +261,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do activity, "twitterapi:content" ) - |> Formatter.emojify(object["emoji"]) + |> Formatter.emojify(object.data["emoji"]) text = if content do @@ -284,7 +286,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do %{ "id" => activity.id, - "uri" => activity.data["object"]["id"], + "uri" => object.data["id"], "user" => UserView.render("show.json", %{user: user, for: opts[:for]}), "statusnet_html" => html, "text" => text, @@ -297,20 +299,20 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do "in_reply_to_ostatus_uri" => reply_user && reply_user.ap_id, "in_reply_to_user_id" => reply_user && reply_user.id, "statusnet_conversation_id" => conversation_id, - "attachments" => (object["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts), + "attachments" => (object.data["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts), "attentions" => attentions, "fave_num" => like_count, "repeat_num" => announcement_count, "favorited" => !!favorited, "repeated" => !!repeated, "pinned" => pinned, - "external_url" => object["external_url"] || object["id"], + "external_url" => object.data["external_url"] || object.data["id"], "tags" => tags, "activity_type" => "post", "possibly_sensitive" => possibly_sensitive, "visibility" => StatusView.get_visibility(object), "summary" => summary, - "summary_html" => summary |> Formatter.emojify(object["emoji"]), + "summary_html" => summary |> Formatter.emojify(object.data["emoji"]), "card" => card, "muted" => CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user) } diff --git a/test/activity_test.exs b/test/activity_test.exs @@ -28,18 +28,4 @@ defmodule Pleroma.ActivityTest do assert activity == found_activity end - - test "reply count" do - %{id: id, data: %{"object" => %{"id" => object_ap_id}}} = activity = insert(:note_activity) - - replies_count = activity.data["object"]["repliesCount"] || 0 - expected_increase = replies_count + 1 - Activity.increase_replies_count(object_ap_id) - %{data: %{"object" => %{"repliesCount" => actual_increase}}} = Activity.get_by_id(id) - assert expected_increase == actual_increase - expected_decrease = expected_increase - 1 - Activity.decrease_replies_count(object_ap_id) - %{data: %{"object" => %{"repliesCount" => actual_decrease}}} = Activity.get_by_id(id) - assert expected_decrease == actual_decrease - end end diff --git a/test/object/containment_test.exs b/test/object/containment_test.exs @@ -0,0 +1,58 @@ +defmodule Pleroma.Object.ContainmentTest do + use Pleroma.DataCase + + alias Pleroma.Object.Containment + alias Pleroma.User + + import Pleroma.Factory + + describe "general origin containment" do + test "contain_origin_from_id() catches obvious spoofing attempts" do + data = %{ + "id" => "http://example.com/~alyssa/activities/1234.json" + } + + :error = + Containment.contain_origin_from_id( + "http://example.org/~alyssa/activities/1234.json", + data + ) + end + + test "contain_origin_from_id() allows alternate IDs within the same origin domain" do + data = %{ + "id" => "http://example.com/~alyssa/activities/1234.json" + } + + :ok = + Containment.contain_origin_from_id( + "http://example.com/~alyssa/activities/1234", + data + ) + end + + test "contain_origin_from_id() allows matching IDs" do + data = %{ + "id" => "http://example.com/~alyssa/activities/1234.json" + } + + :ok = + Containment.contain_origin_from_id( + "http://example.com/~alyssa/activities/1234.json", + data + ) + end + + test "users cannot be collided through fake direction spoofing attempts" do + _user = + insert(:user, %{ + nickname: "rye@niu.moe", + local: false, + ap_id: "https://niu.moe/users/rye", + follower_address: User.ap_followers(%User{nickname: "rye@niu.moe"}) + }) + + {:error, _} = User.get_or_fetch_by_ap_id("https://n1u.moe/users/rye") + end + end +end diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs @@ -0,0 +1,90 @@ +defmodule Pleroma.Object.FetcherTest do + use Pleroma.DataCase + + alias Pleroma.Activity + alias Pleroma.Object + alias Pleroma.Object.Fetcher + import Tesla.Mock + + setup do + mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + + describe "actor origin containment" do + test "it rejects objects with a bogus origin" do + {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity.json") + end + + test "it rejects objects when attributedTo is wrong (variant 1)" do + {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity2.json") + end + + test "it rejects objects when attributedTo is wrong (variant 2)" do + {:error, _} = Fetcher.fetch_object_from_id("https://info.pleroma.site/activity3.json") + end + end + + describe "fetching an object" do + test "it fetches an object" do + {:ok, object} = + Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") + + assert activity = Activity.get_create_by_object_ap_id(object.data["id"]) + assert activity.data["id"] + + {:ok, object_again} = + Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") + + assert [attachment] = object.data["attachment"] + assert is_list(attachment["url"]) + + assert object == object_again + end + + test "it works with objects only available via Ostatus" do + {:ok, object} = Fetcher.fetch_object_from_id("https://shitposter.club/notice/2827873") + assert activity = Activity.get_create_by_object_ap_id(object.data["id"]) + assert activity.data["id"] + + {:ok, object_again} = Fetcher.fetch_object_from_id("https://shitposter.club/notice/2827873") + + assert object == object_again + end + + test "it correctly stitches up conversations between ostatus and ap" do + last = "https://mstdn.io/users/mayuutann/statuses/99568293732299394" + {:ok, object} = Fetcher.fetch_object_from_id(last) + + object = Object.get_by_ap_id(object.data["inReplyTo"]) + assert object + end + end + + describe "implementation quirks" do + test "it can fetch plume articles" do + {:ok, object} = + Fetcher.fetch_object_from_id( + "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/" + ) + + assert object + end + + test "it can fetch peertube videos" do + {:ok, object} = + Fetcher.fetch_object_from_id( + "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" + ) + + assert object + end + + test "all objects with fake directions are rejected by the object fetcher" do + {:error, _} = + Fetcher.fetch_and_contain_remote_object_from_id( + "https://info.pleroma.site/activity4.json" + ) + end + end +end diff --git a/test/object_test.exs b/test/object_test.exs @@ -5,9 +5,15 @@ defmodule Pleroma.ObjectTest do use Pleroma.DataCase import Pleroma.Factory + import Tesla.Mock alias Pleroma.Object alias Pleroma.Repo + setup do + mock(fn env -> apply(HttpRequestMock, :request, [env]) end) + :ok + end + test "returns an object by it's AP id" do object = insert(:note) found_object = Object.get_by_ap_id(object.data["id"]) @@ -58,4 +64,26 @@ defmodule Pleroma.ObjectTest do assert cached_object.data["type"] == "Tombstone" end end + + describe "normalizer" do + test "fetches unknown objects by default" do + %Object{} = + object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367") + + assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367" + end + + test "fetches unknown objects when fetch_remote is explicitly true" do + %Object{} = + object = Object.normalize("http://mastodon.example.org/@admin/99541947525187367", true) + + assert object.data["url"] == "http://mastodon.example.org/@admin/99541947525187367" + end + + test "does not fetch unknown objects when fetch_remote is false" do + assert is_nil( + Object.normalize("http://mastodon.example.org/@admin/99541947525187367", false) + ) + end + end end diff --git a/test/scheduled_activity_worker_test.exs b/test/scheduled_activity_worker_test.exs @@ -14,6 +14,6 @@ defmodule Pleroma.ScheduledActivityWorkerTest do refute Repo.get(ScheduledActivity, scheduled_activity.id) activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) - assert activity.data["object"]["content"] == "hi" + assert Pleroma.Object.normalize(activity).data["content"] == "hi" end end diff --git a/test/user_test.exs b/test/user_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.UserTest do alias Pleroma.Activity alias Pleroma.Builders.UserBuilder + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.CommonAPI @@ -256,7 +257,7 @@ defmodule Pleroma.UserTest do activity = Repo.one(Pleroma.Activity) assert registered_user.ap_id in activity.recipients - assert activity.data["object"]["content"] =~ "cool site" + assert Object.normalize(activity).data["content"] =~ "cool site" assert activity.actor == welcome_user.ap_id Pleroma.Config.put([:instance, :welcome_user_nickname], nil) @@ -1132,14 +1133,14 @@ defmodule Pleroma.UserTest do "status" => "heweoo!" }) - id1 = activity1.data["object"]["id"] + id1 = Object.normalize(activity1).data["id"] {:ok, activity2} = CommonAPI.post(user, %{ "status" => "heweoo!" }) - id2 = activity2.data["object"]["id"] + id2 = Object.normalize(activity2).data["id"] assert {:ok, user_state1} = User.bookmark(user, id1) assert user_state1.bookmarks == [id1] diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs @@ -84,17 +84,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do {:ok, status_two} = CommonAPI.post(user, %{"status" => ". #essais"}) {:ok, status_three} = CommonAPI.post(user, %{"status" => ". #test #reject"}) - fetch_one = ActivityPub.fetch_activities([], %{"tag" => "test"}) - fetch_two = ActivityPub.fetch_activities([], %{"tag" => ["test", "essais"]}) + fetch_one = ActivityPub.fetch_activities([], %{"type" => "Create", "tag" => "test"}) + + fetch_two = + ActivityPub.fetch_activities([], %{"type" => "Create", "tag" => ["test", "essais"]}) fetch_three = ActivityPub.fetch_activities([], %{ + "type" => "Create", "tag" => ["test", "essais"], "tag_reject" => ["reject"] }) fetch_four = ActivityPub.fetch_activities([], %{ + "type" => "Create", "tag" => ["test"], "tag_all" => ["test", "reject"] }) @@ -192,8 +196,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do } {:ok, %Activity{} = activity} = ActivityPub.insert(data) - assert is_binary(activity.data["object"]["id"]) - assert %Object{} = Object.get_by_ap_id(activity.data["object"]["id"]) + object = Object.normalize(activity.data["object"]) + + assert is_binary(object.data["id"]) + assert %Object{} = Object.get_by_ap_id(activity.data["object"]) end end @@ -206,7 +212,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do to: ["user1", "user1", "user2"], actor: user, context: "", - object: %{} + object: %{ + "to" => ["user1", "user1", "user2"], + "type" => "Note", + "content" => "testing" + } }) assert activity.data["to"] == ["user1", "user2"] @@ -244,25 +254,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do # public {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "public")) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 1 assert object.data["repliesCount"] == 1 # unlisted {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "unlisted")) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 2 assert object.data["repliesCount"] == 2 # private {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "private")) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 2 assert object.data["repliesCount"] == 2 # direct {:ok, _} = CommonAPI.post(user2, Map.put(reply_data, "visibility", "direct")) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 2 assert object.data["repliesCount"] == 2 end end @@ -680,40 +686,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do end end - describe "fetching an object" do - test "it fetches an object" do - {:ok, object} = - ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") - - assert activity = Activity.get_create_by_object_ap_id(object.data["id"]) - assert activity.data["id"] - - {:ok, object_again} = - ActivityPub.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367") - - assert [attachment] = object.data["attachment"] - assert is_list(attachment["url"]) - - assert object == object_again - end - - test "it works with objects only available via Ostatus" do - {:ok, object} = ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873") - assert activity = Activity.get_create_by_object_ap_id(object.data["id"]) - assert activity.data["id"] - - {:ok, object_again} = - ActivityPub.fetch_object_from_id("https://shitposter.club/notice/2827873") + describe "fetch the latest Follow" do + test "fetches the latest Follow activity" do + %Activity{data: %{"type" => "Follow"}} = activity = insert(:follow_activity) + follower = Repo.get_by(User, ap_id: activity.data["actor"]) + followed = Repo.get_by(User, ap_id: activity.data["object"]) - assert object == object_again - end - - test "it correctly stitches up conversations between ostatus and ap" do - last = "https://mstdn.io/users/mayuutann/statuses/99568293732299394" - {:ok, object} = ActivityPub.fetch_object_from_id(last) - - object = Object.get_by_ap_id(object.data["inReplyTo"]) - assert object + assert activity == Utils.fetch_latest_follow(follower, followed) end end @@ -804,10 +783,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do {:ok, a4} = CommonAPI.post(User.get_by_id(user.id), %{"status" => "yeah", "visibility" => "direct"}) - {:ok, _} = a1.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete() - {:ok, _} = a2.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete() - {:ok, _} = a3.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete() - {:ok, _} = a4.data["object"]["id"] |> Object.get_by_ap_id() |> ActivityPub.delete() + {:ok, _} = Object.normalize(a1) |> ActivityPub.delete() + {:ok, _} = Object.normalize(a2) |> ActivityPub.delete() + {:ok, _} = Object.normalize(a3) |> ActivityPub.delete() + {:ok, _} = Object.normalize(a4) |> ActivityPub.delete() user = User.get_by_id(user.id) assert user.info.note_count == 10 @@ -849,22 +828,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do _ = CommonAPI.delete(direct_reply.id, user2) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 2 assert object.data["repliesCount"] == 2 _ = CommonAPI.delete(private_reply.id, user2) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 2 assert object.data["repliesCount"] == 2 _ = CommonAPI.delete(public_reply.id, user2) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 1 assert object.data["repliesCount"] == 1 _ = CommonAPI.delete(unlisted_reply.id, user2) assert %{data: data, object: object} = Activity.get_by_ap_id_with_object(ap_id) - assert data["object"]["repliesCount"] == 0 assert object.data["repliesCount"] == 0 end end @@ -906,7 +881,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do activities = ActivityPub.fetch_activities([user1.ap_id | user1.following]) private_activity_1 = Activity.get_by_ap_id_with_object(private_activity_1.data["id"]) - assert [public_activity, private_activity_1, private_activity_3] == activities + + assert [public_activity, private_activity_1, private_activity_3] == + activities + assert length(activities) == 3 activities = ActivityPub.contain_timeline(activities, user1) @@ -916,15 +894,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do end end - test "it can fetch plume articles" do - {:ok, object} = - ActivityPub.fetch_object_from_id( - "https://baptiste.gelez.xyz/~/PlumeDevelopment/this-month-in-plume-june-2018/" - ) - - assert object - end - describe "update" do test "it creates an update activity with the new user data" do user = insert(:user) @@ -946,15 +915,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do end end - test "it can fetch peertube videos" do - {:ok, object} = - ActivityPub.fetch_object_from_id( - "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" - ) - - assert object - end - test "returned pinned statuses" do Pleroma.Config.put([:instance, :max_pinned_statuses], 3) user = insert(:user) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do use Pleroma.DataCase alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.Object.Fetcher alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -50,14 +51,14 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do |> Map.put("object", object) {:ok, returned_activity} = Transmogrifier.handle_incoming(data) + returned_object = Object.normalize(returned_activity.data["object"]) assert activity = Activity.get_create_by_object_ap_id( "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" ) - assert returned_activity.data["object"]["inReplyToAtomUri"] == - "https://shitposter.club/notice/2827873" + assert returned_object.data["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873" end test "it works for incoming notices" do @@ -80,7 +81,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert data["actor"] == "http://mastodon.example.org/users/admin" - object = data["object"] + object = Object.normalize(data["object"]).data assert object["id"] == "http://mastodon.example.org/users/admin/statuses/99512778738411822" assert object["to"] == ["https://www.w3.org/ns/activitystreams#Public"] @@ -107,7 +108,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do data = File.read!("test/fixtures/mastodon-post-activity-hashtag.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) - assert Enum.at(data["object"]["tag"], 2) == "moo" + object = Object.normalize(data["object"]) + + assert Enum.at(object.data["tag"], 2) == "moo" end test "it works for incoming notices with contentMap" do @@ -115,8 +118,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do File.read!("test/fixtures/mastodon-post-activity-contentmap.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) - assert data["object"]["content"] == + assert object.data["content"] == "<p><span class=\"h-card\"><a href=\"http://localtesting.pleroma.lol/users/lain\" class=\"u-url mention\">@<span>lain</span></a></span></p>" end @@ -124,8 +128,9 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do data = File.read!("test/fixtures/kroeg-post-activity.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) - assert data["object"]["content"] == + assert object.data["content"] == "<p>henlo from my Psion netBook</p><p>message sent from my Psion netBook</p>" end @@ -141,24 +146,27 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do data = File.read!("test/fixtures/kroeg-array-less-emoji.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) - assert data["object"]["emoji"] == %{ + assert object.data["emoji"] == %{ "icon_e_smile" => "https://puckipedia.com/forum/images/smilies/icon_e_smile.png" } data = File.read!("test/fixtures/kroeg-array-less-hashtag.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) - assert "test" in data["object"]["tag"] + assert "test" in object.data["tag"] end test "it works for incoming notices with url not being a string (prismo)" do data = File.read!("test/fixtures/prismo-url-map.json") |> Poison.decode!() {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + object = Object.normalize(data["object"]) - assert data["object"]["url"] == "https://prismo.news/posts/83" + assert object.data["url"] == "https://prismo.news/posts/83" end test "it cleans up incoming notices which are not really DMs" do @@ -180,15 +188,15 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do data = Map.put(data, "object", object) - {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) + {:ok, %Activity{data: data, local: false} = activity} = Transmogrifier.handle_incoming(data) assert data["to"] == [] assert data["cc"] == to - object = data["object"] + object_data = Object.normalize(activity).data - assert object["to"] == [] - assert object["cc"] == to + assert object_data["to"] == [] + assert object_data["cc"] == to end test "it works for incoming follow requests" do @@ -231,14 +239,14 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do data = File.read!("test/fixtures/mastodon-like.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", activity.data["object"]) {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) assert data["actor"] == "http://mastodon.example.org/users/admin" assert data["type"] == "Like" assert data["id"] == "http://mastodon.example.org/users/admin#likes/2" - assert data["object"] == activity.data["object"]["id"] + assert data["object"] == activity.data["object"] end test "it returns an error for incoming unlikes wihout a like activity" do @@ -248,7 +256,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do data = File.read!("test/fixtures/mastodon-undo-like.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", activity.data["object"]) assert Transmogrifier.handle_incoming(data) == :error end @@ -260,7 +268,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do like_data = File.read!("test/fixtures/mastodon-like.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", activity.data["object"]) {:ok, %Activity{data: like_data, local: false}} = Transmogrifier.handle_incoming(like_data) @@ -302,7 +310,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do data = File.read!("test/fixtures/mastodon-announce.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", activity.data["object"]) {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data) @@ -312,7 +320,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert data["id"] == "http://mastodon.example.org/users/admin/statuses/99542391527669785/activity" - assert data["object"] == activity.data["object"]["id"] + assert data["object"] == activity.data["object"] assert Activity.get_create_by_object_ap_id(data["object"]).id == activity.id end @@ -324,7 +332,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do data = File.read!("test/fixtures/mastodon-announce.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", Object.normalize(activity).data["id"]) |> Map.put("to", ["http://mastodon.example.org/users/admin/followers"]) |> Map.put("cc", []) @@ -450,7 +458,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do object = data["object"] - |> Map.put("id", activity.data["object"]["id"]) + |> Map.put("id", activity.data["object"]) data = data @@ -471,7 +479,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do object = data["object"] - |> Map.put("id", activity.data["object"]["id"]) + |> Map.put("id", activity.data["object"]) data = data @@ -489,7 +497,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do announce_data = File.read!("test/fixtures/mastodon-announce.json") |> Poison.decode!() - |> Map.put("object", activity.data["object"]["id"]) + |> Map.put("object", activity.data["object"]) {:ok, %Activity{data: announce_data, local: false}} = Transmogrifier.handle_incoming(announce_data) @@ -504,7 +512,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do assert data["type"] == "Undo" assert data["object"]["type"] == "Announce" - assert data["object"]["object"] == activity.data["object"]["id"] + assert data["object"]["object"] == activity.data["object"] assert data["object"]["id"] == "http://mastodon.example.org/users/admin/statuses/99542391527669785/activity" @@ -783,7 +791,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do test "it remaps video URLs as attachments if necessary" do {:ok, object} = - ActivityPub.fetch_object_from_id( + Fetcher.fetch_object_from_id( "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" ) @@ -1088,10 +1096,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do end describe "actor origin containment" do - test "it rejects objects with a bogus origin" do - {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity.json") - end - test "it rejects activities which reference objects with bogus origins" do data = %{ "@context" => "https://www.w3.org/ns/activitystreams", @@ -1105,10 +1109,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do :error = Transmogrifier.handle_incoming(data) end - test "it rejects objects when attributedTo is wrong (variant 1)" do - {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity2.json") - end - test "it rejects activities which reference objects that have an incorrect attribution (variant 1)" do data = %{ "@context" => "https://www.w3.org/ns/activitystreams", @@ -1122,10 +1122,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do :error = Transmogrifier.handle_incoming(data) end - test "it rejects objects when attributedTo is wrong (variant 2)" do - {:error, _} = ActivityPub.fetch_object_from_id("https://info.pleroma.site/activity3.json") - end - test "it rejects activities which reference objects that have an incorrect attribution (variant 2)" do data = %{ "@context" => "https://www.w3.org/ns/activitystreams", @@ -1140,62 +1136,6 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do end end - describe "general origin containment" do - test "contain_origin_from_id() catches obvious spoofing attempts" do - data = %{ - "id" => "http://example.com/~alyssa/activities/1234.json" - } - - :error = - Transmogrifier.contain_origin_from_id( - "http://example.org/~alyssa/activities/1234.json", - data - ) - end - - test "contain_origin_from_id() allows alternate IDs within the same origin domain" do - data = %{ - "id" => "http://example.com/~alyssa/activities/1234.json" - } - - :ok = - Transmogrifier.contain_origin_from_id( - "http://example.com/~alyssa/activities/1234", - data - ) - end - - test "contain_origin_from_id() allows matching IDs" do - data = %{ - "id" => "http://example.com/~alyssa/activities/1234.json" - } - - :ok = - Transmogrifier.contain_origin_from_id( - "http://example.com/~alyssa/activities/1234.json", - data - ) - end - - test "users cannot be collided through fake direction spoofing attempts" do - insert(:user, %{ - nickname: "rye@niu.moe", - local: false, - ap_id: "https://niu.moe/users/rye", - follower_address: User.ap_followers(%User{nickname: "rye@niu.moe"}) - }) - - {:error, _} = User.get_or_fetch_by_ap_id("https://n1u.moe/users/rye") - end - - test "all objects with fake directions are rejected by the object fetcher" do - {:error, _} = - ActivityPub.fetch_and_contain_remote_object_from_id( - "https://info.pleroma.site/activity4.json" - ) - end - end - describe "reserialization" do test "successfully reserializes a message with inReplyTo == nil" do user = insert(:user) diff --git a/test/web/common_api/common_api_test.exs b/test/web/common_api/common_api_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.Web.CommonAPITest do use Pleroma.DataCase alias Pleroma.Activity + alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.CommonAPI @@ -32,14 +33,16 @@ defmodule Pleroma.Web.CommonAPITest do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => "#2hu #2HU"}) - assert activity.data["object"]["tag"] == ["2hu"] + object = Object.normalize(activity.data["object"]) + + assert object.data["tag"] == ["2hu"] end test "it adds emoji in the object" do user = insert(:user) {:ok, activity} = CommonAPI.post(user, %{"status" => ":moominmamma:"}) - assert activity.data["object"]["emoji"]["moominmamma"] + assert Object.normalize(activity).data["emoji"]["moominmamma"] end test "it adds emoji when updating profiles" do @@ -64,8 +67,9 @@ defmodule Pleroma.Web.CommonAPITest do "content_type" => "text/html" }) - content = activity.data["object"]["content"] - assert content == "<p><b>2hu</b></p>alert('xss')" + object = Object.normalize(activity.data["object"]) + + assert object.data["content"] == "<p><b>2hu</b></p>alert('xss')" end test "it filters out obviously bad tags when accepting a post as Markdown" do @@ -79,8 +83,9 @@ defmodule Pleroma.Web.CommonAPITest do "content_type" => "text/markdown" }) - content = activity.data["object"]["content"] - assert content == "<p><b>2hu</b></p>alert('xss')" + object = Object.normalize(activity.data["object"]) + + assert object.data["content"] == "<p><b>2hu</b></p>alert('xss')" end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1879,7 +1879,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do capture_log(fn -> conn = conn - |> get("/api/v1/search", %{"q" => activity.data["object"]["id"]}) + |> get("/api/v1/search", %{"q" => Object.normalize(activity).data["id"]}) assert results = json_response(conn, 200) @@ -2791,9 +2791,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert %{"content" => "xD", "id" => id} = json_response(conn1, 200) - activity = Activity.get_by_id(id) + activity = Activity.get_by_id_with_object(id) - assert activity.data["object"]["inReplyTo"] == replied_to.data["object"]["id"] + assert Object.normalize(activity).data["inReplyTo"] == Object.normalize(replied_to).data["id"] assert Activity.get_in_reply_to_activity(activity).id == replied_to.id # Reblog from the third user diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs @@ -6,8 +6,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do use Pleroma.DataCase alias Pleroma.Activity + alias Pleroma.Object + alias Pleroma.Repo alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.AccountView @@ -53,14 +54,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do test "a note with null content" do note = insert(:note_activity) + note_object = Object.normalize(note.data["object"]) data = - note.data - |> put_in(["object", "content"], nil) + note_object.data + |> Map.put("content", nil) - note = - note - |> Map.put(:data, data) + Object.change(note_object, %{data: data}) + |> Object.update_and_set_cache() User.get_cached_by_ap_id(note.data["actor"]) @@ -230,7 +231,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do user = insert(:user) {:ok, object} = - ActivityPub.fetch_object_from_id( + Pleroma.Object.Fetcher.fetch_object_from_id( "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" ) diff --git a/test/web/ostatus/ostatus_test.exs b/test/web/ostatus/ostatus_test.exs @@ -28,34 +28,35 @@ defmodule Pleroma.Web.OStatusTest do test "handle incoming note - GS, Salmon" do incoming = File.read!("test/fixtures/incoming_note_activity.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) user = User.get_by_ap_id(activity.data["actor"]) assert user.info.note_count == 1 assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" + assert object.data["type"] == "Note" - assert activity.data["object"]["id"] == - "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note" + assert object.data["id"] == "tag:gs.example.org:4040,2017-04-23:noticeId=29:objectType=note" assert activity.data["published"] == "2017-04-23T14:51:03+00:00" - assert activity.data["object"]["published"] == "2017-04-23T14:51:03+00:00" + assert object.data["published"] == "2017-04-23T14:51:03+00:00" assert activity.data["context"] == "tag:gs.example.org:4040,2017-04-23:objectType=thread:nonce=f09e22f58abd5c7b" assert "http://pleroma.example.org:4000/users/lain3" in activity.data["to"] - assert activity.data["object"]["emoji"] == %{"marko" => "marko.png", "reimu" => "reimu.png"} + assert object.data["emoji"] == %{"marko" => "marko.png", "reimu" => "reimu.png"} assert activity.local == false end test "handle incoming notes - GS, subscription" do incoming = File.read!("test/fixtures/ostatus_incoming_post.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" - assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"]["content"] == "Will it blend?" + assert object.data["type"] == "Note" + assert object.data["actor"] == "https://social.heldscal.la/user/23211" + assert object.data["content"] == "Will it blend?" user = User.get_cached_by_ap_id(activity.data["actor"]) assert User.ap_followers(user) in activity.data["to"] assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] @@ -64,20 +65,22 @@ defmodule Pleroma.Web.OStatusTest do test "handle incoming notes with attachments - GS, subscription" do incoming = File.read!("test/fixtures/incoming_websub_gnusocial_attachments.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" - assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"]["attachment"] |> length == 2 - assert activity.data["object"]["external_url"] == "https://social.heldscal.la/notice/2020923" + assert object.data["type"] == "Note" + assert object.data["actor"] == "https://social.heldscal.la/user/23211" + assert object.data["attachment"] |> length == 2 + assert object.data["external_url"] == "https://social.heldscal.la/notice/2020923" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] end test "handle incoming notes with tags" do incoming = File.read!("test/fixtures/ostatus_incoming_post_tag.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) - assert activity.data["object"]["tag"] == ["nsfw"] + assert object.data["tag"] == ["nsfw"] assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] end @@ -92,10 +95,11 @@ defmodule Pleroma.Web.OStatusTest do incoming = File.read!("test/fixtures/incoming_reply_mastodon.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" - assert activity.data["object"]["actor"] == "https://mastodon.social/users/lambadalambda" + assert object.data["type"] == "Note" + assert object.data["actor"] == "https://mastodon.social/users/lambadalambda" assert activity.data["context"] == "2hu" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] end @@ -103,42 +107,47 @@ defmodule Pleroma.Web.OStatusTest do test "handle incoming notes - Mastodon, with CW" do incoming = File.read!("test/fixtures/mastodon-note-cw.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" - assert activity.data["object"]["actor"] == "https://mastodon.social/users/lambadalambda" - assert activity.data["object"]["summary"] == "technologic" + assert object.data["type"] == "Note" + assert object.data["actor"] == "https://mastodon.social/users/lambadalambda" + assert object.data["summary"] == "technologic" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] end test "handle incoming unlisted messages, put public into cc" do incoming = File.read!("test/fixtures/mastodon-note-unlisted.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) + refute "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["cc"] - refute "https://www.w3.org/ns/activitystreams#Public" in activity.data["object"]["to"] - assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["object"]["cc"] + refute "https://www.w3.org/ns/activitystreams#Public" in object.data["to"] + assert "https://www.w3.org/ns/activitystreams#Public" in object.data["cc"] end test "handle incoming retweets - Mastodon, with CW" do incoming = File.read!("test/fixtures/cw_retweet.xml") {:ok, [[_activity, retweeted_activity]]} = OStatus.handle_incoming(incoming) + retweeted_object = Object.normalize(retweeted_activity.data["object"]) - assert retweeted_activity.data["object"]["summary"] == "Hey." + assert retweeted_object.data["summary"] == "Hey." end test "handle incoming notes - GS, subscription, reply" do incoming = File.read!("test/fixtures/ostatus_incoming_reply.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" - assert activity.data["object"]["actor"] == "https://social.heldscal.la/user/23211" + assert object.data["type"] == "Note" + assert object.data["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"]["content"] == + assert object.data["content"] == "@<a href=\"https://gs.archae.me/user/4687\" class=\"h-card u-url p-nickname mention\" title=\"shpbot\">shpbot</a> why not indeed." - assert activity.data["object"]["inReplyTo"] == + assert object.data["inReplyTo"] == "tag:gs.archae.me,2017-04-30:noticeId=778260:objectType=note" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] @@ -150,17 +159,18 @@ defmodule Pleroma.Web.OStatusTest do assert activity.data["type"] == "Announce" assert activity.data["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"] == retweeted_activity.data["object"]["id"] + assert activity.data["object"] == retweeted_activity.data["object"] assert "https://pleroma.soykaf.com/users/lain" in activity.data["to"] refute activity.local retweeted_activity = Activity.get_by_id(retweeted_activity.id) + retweeted_object = Object.normalize(retweeted_activity.data["object"]) assert retweeted_activity.data["type"] == "Create" assert retweeted_activity.data["actor"] == "https://pleroma.soykaf.com/users/lain" refute retweeted_activity.local - assert retweeted_activity.data["object"]["announcement_count"] == 1 - assert String.contains?(retweeted_activity.data["object"]["content"], "mastodon") - refute String.contains?(retweeted_activity.data["object"]["content"], "Test account") + assert retweeted_object.data["announcement_count"] == 1 + assert String.contains?(retweeted_object.data["content"], "mastodon") + refute String.contains?(retweeted_object.data["content"], "Test account") end test "handle incoming retweets - GS, subscription - local message" do @@ -192,10 +202,11 @@ defmodule Pleroma.Web.OStatusTest do test "handle incoming retweets - Mastodon, salmon" do incoming = File.read!("test/fixtures/share.xml") {:ok, [[activity, retweeted_activity]]} = OStatus.handle_incoming(incoming) + retweeted_object = Object.normalize(retweeted_activity.data["object"]) assert activity.data["type"] == "Announce" assert activity.data["actor"] == "https://mastodon.social/users/lambadalambda" - assert activity.data["object"] == retweeted_activity.data["object"]["id"] + assert activity.data["object"] == retweeted_activity.data["object"] assert activity.data["id"] == "tag:mastodon.social,2017-05-03:objectId=4934452:objectType=Status" @@ -204,7 +215,7 @@ defmodule Pleroma.Web.OStatusTest do assert retweeted_activity.data["type"] == "Create" assert retweeted_activity.data["actor"] == "https://pleroma.soykaf.com/users/lain" refute retweeted_activity.local - refute String.contains?(retweeted_activity.data["object"]["content"], "Test account") + refute String.contains?(retweeted_object.data["content"], "Test account") end test "handle incoming favorites - GS, websub" do @@ -214,7 +225,7 @@ defmodule Pleroma.Web.OStatusTest do assert activity.data["type"] == "Like" assert activity.data["actor"] == "https://social.heldscal.la/user/23211" - assert activity.data["object"] == favorited_activity.data["object"]["id"] + assert activity.data["object"] == favorited_activity.data["object"] assert activity.data["id"] == "tag:social.heldscal.la,2017-05-05:fave:23211:comment:2061643:2017-05-05T09:12:50+00:00" @@ -223,7 +234,7 @@ defmodule Pleroma.Web.OStatusTest do assert favorited_activity.data["type"] == "Create" assert favorited_activity.data["actor"] == "https://shitposter.club/user/1" - assert favorited_activity.data["object"]["id"] == + assert favorited_activity.data["object"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" refute favorited_activity.local @@ -258,17 +269,17 @@ defmodule Pleroma.Web.OStatusTest do test "handle incoming replies" do incoming = File.read!("test/fixtures/incoming_note_activity_answer.xml") {:ok, [activity]} = OStatus.handle_incoming(incoming) + object = Object.normalize(activity.data["object"]) assert activity.data["type"] == "Create" - assert activity.data["object"]["type"] == "Note" + assert object.data["type"] == "Note" - assert activity.data["object"]["inReplyTo"] == + assert object.data["inReplyTo"] == "http://pleroma.example.org:4000/objects/55bce8fc-b423-46b1-af71-3759ab4670bc" assert "http://pleroma.example.org:4000/users/lain5" in activity.data["to"] - assert activity.data["object"]["id"] == - "tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note" + assert object.data["id"] == "tag:gs.example.org:4040,2017-04-25:noticeId=55:objectType=note" assert "https://www.w3.org/ns/activitystreams#Public" in activity.data["to"] end @@ -495,7 +506,7 @@ defmodule Pleroma.Web.OStatusTest do assert activity.data["actor"] == "https://shitposter.club/user/1" - assert activity.data["object"]["id"] == + assert activity.data["object"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment" end) end @@ -504,7 +515,7 @@ defmodule Pleroma.Web.OStatusTest do url = "https://social.sakamoto.gq/objects/0ccc1a2c-66b0-4305-b23a-7f7f2b040056" {:ok, [activity]} = OStatus.fetch_activity_from_url(url) assert activity.data["actor"] == "https://social.sakamoto.gq/users/eal" - assert activity.data["object"]["id"] == url + assert activity.data["object"] == url end end diff --git a/test/web/twitter_api/twitter_api_test.exs b/test/web/twitter_api/twitter_api_test.exs @@ -46,13 +46,14 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPITest do } {:ok, activity = %Activity{}} = TwitterAPI.create_status(user, input) + object = Object.normalize(activity.data["object"]) expected_text = "Hello again, <span class='h-card'><a data-user='#{mentioned_user.id}' class='u-url mention' href='shp'>@<span>shp</span></a></span>.&lt;script&gt;&lt;/script&gt;<br>This is on another :moominmamma: line. <a class='hashtag' data-tag='2hu' href='http://localhost:4001/tag/2hu' rel='tag'>#2hu</a> <a class='hashtag' data-tag='epic' href='http://localhost:4001/tag/epic' rel='tag'>#epic</a> <a class='hashtag' data-tag='phantasmagoric' href='http://localhost:4001/tag/phantasmagoric' rel='tag'>#phantasmagoric</a><br><a href=\"http://example.org/image.jpg\" class='attachment'>image.jpg</a>" - assert get_in(activity.data, ["object", "content"]) == expected_text - assert get_in(activity.data, ["object", "type"]) == "Note" - assert get_in(activity.data, ["object", "actor"]) == user.ap_id + assert get_in(object.data, ["content"]) == expected_text + assert get_in(object.data, ["type"]) == "Note" + assert get_in(object.data, ["actor"]) == user.ap_id assert get_in(activity.data, ["actor"]) == user.ap_id assert Enum.member?(get_in(activity.data, ["cc"]), User.ap_followers(user)) @@ -65,18 +66,18 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPITest do assert activity.local == true assert %{"moominmamma" => "http://localhost:4001/finmoji/128px/moominmamma-128.png"} = - activity.data["object"]["emoji"] + object.data["emoji"] # hashtags - assert activity.data["object"]["tag"] == ["2hu", "epic", "phantasmagoric"] + assert object.data["tag"] == ["2hu", "epic", "phantasmagoric"] # Add a context assert is_binary(get_in(activity.data, ["context"])) - assert is_binary(get_in(activity.data, ["object", "context"])) + assert is_binary(get_in(object.data, ["context"])) - assert is_list(activity.data["object"]["attachment"]) + assert is_list(object.data["attachment"]) - assert activity.data["object"] == Object.get_by_ap_id(activity.data["object"]["id"]).data + assert activity.data["object"] == object.data["id"] user = User.get_by_ap_id(user.ap_id) @@ -91,6 +92,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPITest do } {:ok, activity = %Activity{}} = TwitterAPI.create_status(user, input) + object = Object.normalize(activity.data["object"]) input = %{ "status" => "Here's your (you).", @@ -98,13 +100,13 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPITest do } {:ok, reply = %Activity{}} = TwitterAPI.create_status(user, input) + reply_object = Object.normalize(reply.data["object"]) assert get_in(reply.data, ["context"]) == get_in(activity.data, ["context"]) - assert get_in(reply.data, ["object", "context"]) == - get_in(activity.data, ["object", "context"]) + assert get_in(reply_object.data, ["context"]) == get_in(object.data, ["context"]) - assert get_in(reply.data, ["object", "inReplyTo"]) == get_in(activity.data, ["object", "id"]) + assert get_in(reply_object.data, ["inReplyTo"]) == get_in(activity.data, ["object"]) assert Activity.get_in_reply_to_activity(reply).id == activity.id end diff --git a/test/web/twitter_api/views/activity_view_test.exs b/test/web/twitter_api/views/activity_view_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do use Pleroma.DataCase alias Pleroma.Activity + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -125,10 +126,11 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do other_user = insert(:user, %{nickname: "shp"}) {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!", "visibility" => "direct"}) + object = Object.normalize(activity.data["object"]) result = ActivityView.render("activity.json", activity: activity) - convo_id = Utils.context_to_conversation_id(activity.data["object"]["context"]) + convo_id = Utils.context_to_conversation_id(object.data["context"]) expected = %{ "activity_type" => "post", @@ -136,8 +138,8 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do "attentions" => [ UserView.render("show.json", %{user: other_user}) ], - "created_at" => activity.data["object"]["published"] |> Utils.date_to_asctime(), - "external_url" => activity.data["object"]["id"], + "created_at" => object.data["published"] |> Utils.date_to_asctime(), + "external_url" => object.data["id"], "fave_num" => 0, "favorited" => false, "id" => activity.id, @@ -161,7 +163,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do }\">@<span>shp</span></a></span>!", "tags" => [], "text" => "Hey @shp!", - "uri" => activity.data["object"]["id"], + "uri" => object.data["id"], "user" => UserView.render("show.json", %{user: user}), "visibility" => "direct", "card" => nil, @@ -175,8 +177,9 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do user = insert(:user) other_user = insert(:user, %{nickname: "shp"}) {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!"}) + object = Object.normalize(activity.data["object"]) - convo_id = Utils.context_to_conversation_id(activity.data["object"]["context"]) + convo_id = Utils.context_to_conversation_id(object.data["context"]) mocks = [ { @@ -277,9 +280,9 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do other_user = insert(:user, %{nickname: "shp"}) {:ok, activity} = CommonAPI.post(user, %{"status" => "Hey @shp!"}) - {:ok, announce, _object} = CommonAPI.repeat(activity.id, other_user) + {:ok, announce, object} = CommonAPI.repeat(activity.id, other_user) - convo_id = Utils.context_to_conversation_id(activity.data["object"]["context"]) + convo_id = Utils.context_to_conversation_id(object.data["context"]) activity = Activity.get_by_id(activity.id) @@ -357,7 +360,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityViewTest do test "a peertube video" do {:ok, object} = - ActivityPub.fetch_object_from_id( + Pleroma.Object.Fetcher.fetch_object_from_id( "https://peertube.moe/videos/watch/df5f464b-be8d-46fb-ad81-2d4c2d1630e3" )