commit: 212bcf8d00e40c70728a5479b0bf6421a0809cd6
parent 8910303f71b6fa0a1fd79531ec983e1a9c5dc5ac
Author: Haelwenn (lanodan) Monnier <contact@hacktivis.me>
Date: Tue, 16 Feb 2021 17:55:17 +0100
Merge branch 'features/validators-apc2s'
Diffstat:
82 files changed, 1689 insertions(+), 562 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -33,6 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Admin API: Reports now ordered by newest
</details>
+- Improved hashtag timeline performance (requires a background migration).
### Added
diff --git a/config/description.exs b/config/description.exs
@@ -461,6 +461,34 @@ config :pleroma, :config_description, [
},
%{
group: :pleroma,
+ key: :database,
+ type: :group,
+ description: "Database-related settings",
+ children: [
+ %{
+ key: :improved_hashtag_timeline,
+ type: :keyword,
+ description:
+ "If `true`, hashtags will be fetched from `hashtags` table for hashtags timeline. When `false`, object-embedded hashtags will be used (slower). Is auto-set to `true` (unless overridden) when HashtagsTableMigrator completes."
+ }
+ ]
+ },
+ %{
+ group: :pleroma,
+ key: :populate_hashtags_table,
+ type: :group,
+ description: "`populate_hashtags_table` background migration settings",
+ children: [
+ %{
+ key: :sleep_interval_ms,
+ type: :integer,
+ description:
+ "Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances)."
+ }
+ ]
+ },
+ %{
+ group: :pleroma,
key: :instance,
type: :group,
description: "Instance-related settings",
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
@@ -65,6 +65,12 @@ To add configuration to your config file, you can copy it from the base config.
* `show_reactions`: Let favourites and emoji reactions be viewed through the API (default: `true`).
* `password_reset_token_validity`: The time after which reset tokens aren't accepted anymore, in seconds (default: one day).
+## :database
+* `improved_hashtag_timeline`: If `true`, hashtags will be fetched from `hashtags` table for hashtags timeline. When `false`, object-embedded hashtags will be used (slower). Is auto-set to `true` (unless overridden) when `HashtagsTableMigrator` completes.
+
+## Background migrations
+* `populate_hashtags_table/sleep_interval_ms`: Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances).
+
## Welcome
* `direct_message`: - welcome message sent as a direct message.
* `enabled`: Enables the send a direct message to a newly registered user. Defaults to `false`.
diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
@@ -8,10 +8,13 @@ defmodule Mix.Tasks.Pleroma.Database do
alias Pleroma.Object
alias Pleroma.Repo
alias Pleroma.User
+
require Logger
require Pleroma.Constants
+
import Ecto.Query
import Mix.Pleroma
+
use Mix.Task
@shortdoc "A collection of database related tasks"
@@ -214,4 +217,33 @@ defmodule Mix.Tasks.Pleroma.Database do
shell_info('Done.')
end
end
+
+ # Rolls back a specific migration (leaving subsequent migrations applied).
+ # WARNING: imposes a risk of unrecoverable data loss — proceed at your own responsibility.
+ # Based on https://stackoverflow.com/a/53825840
+ def run(["rollback", version]) do
+ prompt = "SEVERE WARNING: this operation may result in unrecoverable data loss. Continue?"
+
+ if shell_prompt(prompt, "n") in ~w(Yn Y y) do
+ {_, result, _} =
+ Ecto.Migrator.with_repo(Pleroma.Repo, fn repo ->
+ version = String.to_integer(version)
+ re = ~r/^#{version}_.*\.exs/
+ path = Ecto.Migrator.migrations_path(repo)
+
+ with {:find, "" <> file} <- {:find, Enum.find(File.ls!(path), &String.match?(&1, re))},
+ {:compile, [{mod, _} | _]} <- {:compile, Code.compile_file(Path.join(path, file))},
+ {:rollback, :ok} <- {:rollback, Ecto.Migrator.down(repo, version, mod)} do
+ {:ok, "Reversed migration: #{file}"}
+ else
+ {:find, _} -> {:error, "No migration found with version prefix: #{version}"}
+ {:compile, e} -> {:error, "Problem compiling migration module: #{inspect(e)}"}
+ {:rollback, e} -> {:error, "Problem reversing migration: #{inspect(e)}"}
+ e -> {:error, "Something unexpected happened: #{inspect(e)}"}
+ end
+ end)
+
+ IO.inspect(result)
+ end
+ end
end
diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
@@ -113,6 +113,7 @@ defmodule Pleroma.Activity do
from([a] in query,
left_join: b in Bookmark,
on: b.user_id == ^user.id and b.activity_id == a.id,
+ as: :bookmark,
preload: [bookmark: b]
)
end
@@ -123,6 +124,7 @@ defmodule Pleroma.Activity do
from([a] in query,
left_join: r in ReportNote,
on: a.id == r.activity_id,
+ as: :report_note,
preload: [report_notes: r]
)
end
@@ -277,7 +279,8 @@ defmodule Pleroma.Activity do
get_in_reply_to_activity_from_object(Object.normalize(activity, fetch: false))
end
- def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"])
+ def normalize(%Activity{data: %{"id" => ap_id}}), do: get_by_ap_id_with_object(ap_id)
+ def normalize(%{"id" => ap_id}), do: get_by_ap_id_with_object(ap_id)
def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id_with_object(ap_id)
def normalize(_), do: nil
diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex
@@ -48,14 +48,12 @@ defmodule Pleroma.Activity.Ir.Topics do
tags
end
- defp hashtags_to_topics(%{data: %{"tag" => tags}}) do
- tags
- |> Enum.filter(&is_bitstring(&1))
- |> Enum.map(fn tag -> "hashtag:" <> tag end)
+ defp hashtags_to_topics(object) do
+ object
+ |> Object.hashtags()
+ |> Enum.map(fn hashtag -> "hashtag:" <> hashtag end)
end
- defp hashtags_to_topics(_), do: []
-
defp remote_topics(%{local: true}), do: []
defp remote_topics(%{actor: actor}) when is_binary(actor),
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
@@ -104,7 +104,8 @@ defmodule Pleroma.Application do
chat_child(chat_enabled?()) ++
[
Pleroma.Web.Endpoint,
- Pleroma.Gopher.Server
+ Pleroma.Gopher.Server,
+ Pleroma.Migrators.HashtagsTableMigrator
]
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
diff --git a/lib/pleroma/data_migration.ex b/lib/pleroma/data_migration.ex
@@ -0,0 +1,46 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.DataMigration do
+ use Ecto.Schema
+
+ alias Pleroma.DataMigration
+ alias Pleroma.DataMigration.State
+ alias Pleroma.Repo
+
+ import Ecto.Changeset
+
+ schema "data_migrations" do
+ field(:name, :string)
+ field(:state, State, default: :pending)
+ field(:feature_lock, :boolean, default: false)
+ field(:params, :map, default: %{})
+ field(:data, :map, default: %{})
+
+ timestamps()
+ end
+
+ def changeset(data_migration, params \\ %{}) do
+ data_migration
+ |> cast(params, [:name, :state, :feature_lock, :params, :data])
+ |> validate_required([:name])
+ |> unique_constraint(:name)
+ end
+
+ def update(data_migration, params \\ %{}) do
+ data_migration
+ |> changeset(params)
+ |> Repo.update()
+ end
+
+ def update_state(data_migration, new_state) do
+ update(data_migration, %{state: new_state})
+ end
+
+ def get_by_name(name) do
+ Repo.get_by(DataMigration, name: name)
+ end
+
+ def populate_hashtags_table, do: get_by_name("populate_hashtags_table")
+end
diff --git a/lib/pleroma/delivery.ex b/lib/pleroma/delivery.ex
@@ -9,7 +9,6 @@ defmodule Pleroma.Delivery do
alias Pleroma.Object
alias Pleroma.Repo
alias Pleroma.User
- alias Pleroma.User
import Ecto.Changeset
import Ecto.Query
diff --git a/lib/pleroma/ecto_enums.ex b/lib/pleroma/ecto_enums.ex
@@ -17,3 +17,11 @@ defenum(Pleroma.FollowingRelationship.State,
follow_accept: 2,
follow_reject: 3
)
+
+defenum(Pleroma.DataMigration.State,
+ pending: 1,
+ running: 2,
+ complete: 3,
+ failed: 4,
+ manual: 5
+)
diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex
@@ -13,21 +13,33 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients do
cast([object])
end
+ def cast(object) when is_map(object) do
+ case ObjectID.cast(object) do
+ {:ok, data} -> {:ok, [data]}
+ _ -> :error
+ end
+ end
+
def cast(data) when is_list(data) do
- data
- |> Enum.reduce_while({:ok, []}, fn element, {:ok, list} ->
- case ObjectID.cast(element) do
- {:ok, id} ->
- {:cont, {:ok, [id | list]}}
-
- _ ->
- {:halt, :error}
- end
- end)
+ data =
+ data
+ |> Enum.reduce_while([], fn element, list ->
+ case ObjectID.cast(element) do
+ {:ok, id} ->
+ {:cont, [id | list]}
+
+ _ ->
+ {:cont, list}
+ end
+ end)
+ |> Enum.sort()
+ |> Enum.uniq()
+
+ {:ok, data}
end
- def cast(_) do
- :error
+ def cast(data) do
+ {:error, data}
end
def dump(data) do
diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex
@@ -0,0 +1,104 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Hashtag do
+ use Ecto.Schema
+
+ import Ecto.Changeset
+ import Ecto.Query
+
+ alias Ecto.Multi
+ alias Pleroma.Hashtag
+ alias Pleroma.Object
+ alias Pleroma.Repo
+
+ schema "hashtags" do
+ field(:name, :string)
+
+ many_to_many(:objects, Object, join_through: "hashtags_objects", on_replace: :delete)
+
+ timestamps()
+ end
+
+ def get_by_name(name) do
+ from(h in Hashtag)
+ |> where([h], fragment("name = ?::citext", ^String.downcase(name)))
+ |> Repo.one()
+ end
+
+ def get_or_create_by_name(name) when is_bitstring(name) do
+ with %Hashtag{} = hashtag <- get_by_name(name) do
+ {:ok, hashtag}
+ else
+ _ ->
+ %Hashtag{}
+ |> changeset(%{name: name})
+ |> Repo.insert()
+ end
+ end
+
+ def get_or_create_by_names(names) when is_list(names) do
+ names = Enum.map(names, &String.downcase/1)
+ timestamp = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)
+
+ structs =
+ Enum.map(names, fn name ->
+ %Hashtag{}
+ |> changeset(%{name: name})
+ |> Map.get(:changes)
+ |> Map.merge(%{inserted_at: timestamp, updated_at: timestamp})
+ end)
+
+ try do
+ with {:ok, %{query_op: hashtags}} <-
+ Multi.new()
+ |> Multi.insert_all(:insert_all_op, Hashtag, structs, on_conflict: :nothing)
+ |> Multi.run(:query_op, fn _repo, _changes ->
+ {:ok,
+ Repo.all(from(ht in Hashtag, where: ht.name in fragment("?::citext[]", ^names)))}
+ end)
+ |> Repo.transaction() do
+ {:ok, hashtags}
+ else
+ {:error, _name, value, _changes_so_far} -> {:error, value}
+ end
+ rescue
+ e -> {:error, e}
+ end
+ end
+
+ def changeset(%Hashtag{} = struct, params) do
+ struct
+ |> cast(params, [:name])
+ |> update_change(:name, &String.downcase/1)
+ |> validate_required([:name])
+ |> unique_constraint(:name)
+ end
+
+ def unlink(%Object{id: object_id}) do
+ with {_, hashtag_ids} <-
+ from(hto in "hashtags_objects",
+ where: hto.object_id == ^object_id,
+ select: hto.hashtag_id
+ )
+ |> Repo.delete_all(),
+ {:ok, unreferenced_count} <- delete_unreferenced(hashtag_ids) do
+ {:ok, length(hashtag_ids), unreferenced_count}
+ end
+ end
+
+ @delete_unreferenced_query """
+ DELETE FROM hashtags WHERE id IN
+ (SELECT hashtags.id FROM hashtags
+ LEFT OUTER JOIN hashtags_objects
+ ON hashtags_objects.hashtag_id = hashtags.id
+ WHERE hashtags_objects.hashtag_id IS NULL AND hashtags.id = ANY($1));
+ """
+
+ def delete_unreferenced(ids) do
+ with {:ok, %{num_rows: deleted_count}} <- Repo.query(@delete_unreferenced_query, [ids]) do
+ {:ok, deleted_count}
+ end
+ end
+end
diff --git a/lib/pleroma/maps.ex b/lib/pleroma/maps.ex
@@ -12,4 +12,10 @@ defmodule Pleroma.Maps do
_ -> map
end
end
+
+ def safe_put_in(data, keys, value) when is_map(data) and is_list(keys) do
+ Kernel.put_in(data, keys, value)
+ rescue
+ _ -> data
+ end
end
diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex
@@ -0,0 +1,338 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Migrators.HashtagsTableMigrator do
+ use GenServer
+
+ require Logger
+
+ import Ecto.Query
+
+ alias __MODULE__.State
+ alias Pleroma.Config
+ alias Pleroma.DataMigration
+ alias Pleroma.Hashtag
+ alias Pleroma.Object
+ alias Pleroma.Repo
+
+ defdelegate state(), to: State, as: :get
+ defdelegate put_stat(key, value), to: State, as: :put
+ defdelegate increment_stat(key, increment), to: State, as: :increment
+
+ defdelegate data_migration(), to: DataMigration, as: :populate_hashtags_table
+
+ @reg_name {:global, __MODULE__}
+
+ def whereis, do: GenServer.whereis(@reg_name)
+
+ def start_link(_) do
+ case whereis() do
+ nil ->
+ GenServer.start_link(__MODULE__, nil, name: @reg_name)
+
+ pid ->
+ {:ok, pid}
+ end
+ end
+
+ @impl true
+ def init(_) do
+ {:ok, nil, {:continue, :init_state}}
+ end
+
+ @impl true
+ def handle_continue(:init_state, _state) do
+ {:ok, _} = State.start_link(nil)
+
+ update_status(:init)
+
+ data_migration = data_migration()
+ manual_migrations = Config.get([:instance, :manual_data_migrations], [])
+
+ cond do
+ Config.get(:env) == :test ->
+ update_status(:noop)
+
+ is_nil(data_migration) ->
+ update_status(:halt, "Data migration does not exist.")
+
+ data_migration.state == :manual or data_migration.name in manual_migrations ->
+ update_status(:noop, "Data migration is in manual execution state.")
+
+ data_migration.state == :complete ->
+ handle_success(data_migration)
+
+ true ->
+ send(self(), :migrate_hashtags)
+ end
+
+ {:noreply, nil}
+ end
+
+ @impl true
+ def handle_info(:migrate_hashtags, state) do
+ State.clear()
+
+ update_status(:running)
+ put_stat(:started_at, NaiveDateTime.utc_now())
+
+ data_migration = data_migration()
+ persistent_data = Map.take(data_migration.data, ["max_processed_id"])
+
+ {:ok, data_migration} =
+ DataMigration.update(data_migration, %{state: :running, data: persistent_data})
+
+ Logger.info("Starting transferring object embedded hashtags to `hashtags` table...")
+
+ max_processed_id = data_migration.data["max_processed_id"] || 0
+
+ query()
+ |> where([object], object.id > ^max_processed_id)
+ |> Repo.chunk_stream(100, :batches, timeout: :infinity)
+ |> Stream.each(fn objects ->
+ object_ids = Enum.map(objects, & &1.id)
+
+ failed_ids =
+ objects
+ |> Enum.map(&transfer_object_hashtags(&1))
+ |> Enum.filter(&(elem(&1, 0) == :error))
+ |> Enum.map(&elem(&1, 1))
+
+ for failed_id <- failed_ids do
+ _ =
+ Repo.query(
+ "INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <>
+ "VALUES ($1, $2) ON CONFLICT DO NOTHING;",
+ [data_migration.id, failed_id]
+ )
+ end
+
+ _ =
+ Repo.query(
+ "DELETE FROM data_migration_failed_ids " <>
+ "WHERE data_migration_id = $1 AND record_id = ANY($2)",
+ [data_migration.id, object_ids -- failed_ids]
+ )
+
+ max_object_id = Enum.at(object_ids, -1)
+
+ put_stat(:max_processed_id, max_object_id)
+ increment_stat(:processed_count, length(object_ids))
+ increment_stat(:failed_count, length(failed_ids))
+
+ put_stat(
+ :records_per_second,
+ state()[:processed_count] /
+ Enum.max([NaiveDateTime.diff(NaiveDateTime.utc_now(), state()[:started_at]), 1])
+ )
+
+ persist_stats(data_migration)
+
+ # A quick and dirty approach to controlling the load this background migration imposes
+ sleep_interval = Config.get([:populate_hashtags_table, :sleep_interval_ms], 0)
+ Process.sleep(sleep_interval)
+ end)
+ |> Stream.run()
+
+ with 0 <- failures_count(data_migration.id) do
+ _ = delete_non_create_activities_hashtags()
+
+ {:ok, data_migration} = DataMigration.update_state(data_migration, :complete)
+
+ handle_success(data_migration)
+ else
+ _ ->
+ _ = DataMigration.update_state(data_migration, :failed)
+
+ update_status(:failed, "Please check data_migration_failed_ids records.")
+ end
+
+ {:noreply, state}
+ end
+
+ @hashtags_objects_cleanup_query """
+ DELETE FROM hashtags_objects WHERE object_id IN
+ (SELECT DISTINCT objects.id FROM objects
+ JOIN hashtags_objects ON hashtags_objects.object_id = objects.id LEFT JOIN activities
+ ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') =
+ (objects.data->>'id')
+ AND activities.data->>'type' = 'Create'
+ WHERE activities.id IS NULL);
+ """
+
+ @hashtags_cleanup_query """
+ DELETE FROM hashtags WHERE id IN
+ (SELECT hashtags.id FROM hashtags
+ LEFT OUTER JOIN hashtags_objects
+ ON hashtags_objects.hashtag_id = hashtags.id
+ WHERE hashtags_objects.hashtag_id IS NULL);
+ """
+
+ def delete_non_create_activities_hashtags do
+ {:ok, %{num_rows: hashtags_objects_count}} =
+ Repo.query(@hashtags_objects_cleanup_query, [], timeout: :infinity)
+
+ {:ok, %{num_rows: hashtags_count}} =
+ Repo.query(@hashtags_cleanup_query, [], timeout: :infinity)
+
+ {:ok, hashtags_objects_count, hashtags_count}
+ end
+
+ defp query do
+ # Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out)
+ # Note: not checking activity type, expecting remove_non_create_objects_hashtags/_ to clean up
+ from(
+ object in Object,
+ where:
+ fragment("(?)->'tag' IS NOT NULL AND (?)->'tag' != '[]'::jsonb", object.data, object.data),
+ select: %{
+ id: object.id,
+ tag: fragment("(?)->'tag'", object.data)
+ }
+ )
+ |> join(:left, [o], hashtags_objects in fragment("SELECT object_id FROM hashtags_objects"),
+ on: hashtags_objects.object_id == o.id
+ )
+ |> where([_o, hashtags_objects], is_nil(hashtags_objects.object_id))
+ end
+
+ defp transfer_object_hashtags(object) do
+ embedded_tags = if Map.has_key?(object, :tag), do: object.tag, else: object.data["tag"]
+ hashtags = Object.object_data_hashtags(%{"tag" => embedded_tags})
+
+ if Enum.any?(hashtags) do
+ transfer_object_hashtags(object, hashtags)
+ else
+ {:ok, object.id}
+ end
+ end
+
+ defp transfer_object_hashtags(object, hashtags) do
+ Repo.transaction(fn ->
+ with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do
+ maps = Enum.map(hashtag_records, &%{hashtag_id: &1.id, object_id: object.id})
+ expected_rows = length(hashtag_records)
+
+ base_error =
+ "ERROR when inserting #{expected_rows} hashtags_objects for obj. #{object.id}"
+
+ try do
+ with {^expected_rows, _} <- Repo.insert_all("hashtags_objects", maps) do
+ object.id
+ else
+ e ->
+ Logger.error("#{base_error}: #{inspect(e)}")
+ Repo.rollback(object.id)
+ end
+ rescue
+ e ->
+ Logger.error("#{base_error}: #{inspect(e)}")
+ Repo.rollback(object.id)
+ end
+ else
+ e ->
+ error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}"
+ Logger.error(error)
+ Repo.rollback(object.id)
+ end
+ end)
+ end
+
+ @doc "Approximate count for current iteration (including processed records count)"
+ def count(force \\ false, timeout \\ :infinity) do
+ stored_count = state()[:count]
+
+ if stored_count && !force do
+ stored_count
+ else
+ processed_count = state()[:processed_count] || 0
+ max_processed_id = data_migration().data["max_processed_id"] || 0
+ query = where(query(), [object], object.id > ^max_processed_id)
+
+ count = Repo.aggregate(query, :count, :id, timeout: timeout) + processed_count
+ put_stat(:count, count)
+ count
+ end
+ end
+
+ defp persist_stats(data_migration) do
+ runner_state = Map.drop(state(), [:status])
+ _ = DataMigration.update(data_migration, %{data: runner_state})
+ end
+
+ defp handle_success(data_migration) do
+ update_status(:complete)
+
+ cond do
+ data_migration.feature_lock ->
+ :noop
+
+ not is_nil(Config.get([:database, :improved_hashtag_timeline])) ->
+ :noop
+
+ true ->
+ Config.put([:database, :improved_hashtag_timeline], true)
+ :ok
+ end
+ end
+
+ def failed_objects_query do
+ from(o in Object)
+ |> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"),
+ on: dmf.record_id == o.id
+ )
+ |> where([_o, dmf], dmf.data_migration_id == ^data_migration().id)
+ |> order_by([o], asc: o.id)
+ end
+
+ def failures_count(data_migration_id \\ nil) do
+ data_migration_id = data_migration_id || data_migration().id
+
+ with {:ok, %{rows: [[count]]}} <-
+ Repo.query(
+ "SELECT COUNT(record_id) FROM data_migration_failed_ids WHERE data_migration_id = $1;",
+ [data_migration_id]
+ ) do
+ count
+ end
+ end
+
+ def retry_failed do
+ data_migration = data_migration()
+
+ failed_objects_query()
+ |> Repo.chunk_stream(100, :one)
+ |> Stream.each(fn object ->
+ with {:ok, _} <- transfer_object_hashtags(object) do
+ _ =
+ Repo.query(
+ "DELETE FROM data_migration_failed_ids " <>
+ "WHERE data_migration_id = $1 AND record_id = $2",
+ [data_migration.id, object.id]
+ )
+ end
+ end)
+ |> Stream.run()
+ end
+
+ def force_continue do
+ send(whereis(), :migrate_hashtags)
+ end
+
+ def force_restart do
+ {:ok, _} = DataMigration.update(data_migration(), %{state: :pending, data: %{}})
+ force_continue()
+ end
+
+ def force_complete do
+ {:ok, data_migration} = DataMigration.update_state(data_migration(), :complete)
+
+ handle_success(data_migration)
+ end
+
+ defp update_status(status, message \\ nil) do
+ put_stat(:status, status)
+ put_stat(:message, message)
+ end
+end
diff --git a/lib/pleroma/migrators/hashtags_table_migrator/state.ex b/lib/pleroma/migrators/hashtags_table_migrator/state.ex
@@ -0,0 +1,35 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Migrators.HashtagsTableMigrator.State do
+ use Agent
+
+ @init_state %{}
+ @reg_name {:global, __MODULE__}
+
+ def start_link(_) do
+ Agent.start_link(fn -> @init_state end, name: @reg_name)
+ end
+
+ def clear do
+ Agent.update(@reg_name, fn _state -> @init_state end)
+ end
+
+ def get do
+ Agent.get(@reg_name, & &1)
+ end
+
+ def put(key, value) do
+ Agent.update(@reg_name, fn state ->
+ Map.put(state, key, value)
+ end)
+ end
+
+ def increment(key, increment \\ 1) do
+ Agent.update(@reg_name, fn state ->
+ updated_value = (state[key] || 0) + increment
+ Map.put(state, key, updated_value)
+ end)
+ end
+end
diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Object do
alias Pleroma.Activity
alias Pleroma.Config
+ alias Pleroma.Hashtag
alias Pleroma.Object
alias Pleroma.Object.Fetcher
alias Pleroma.ObjectTombstone
@@ -28,6 +29,8 @@ defmodule Pleroma.Object do
schema "objects" do
field(:data, :map)
+ many_to_many(:hashtags, Hashtag, join_through: "hashtags_objects", on_replace: :delete)
+
timestamps()
end
@@ -49,7 +52,8 @@ defmodule Pleroma.Object do
end
def create(data) do
- Object.change(%Object{}, %{data: data})
+ %Object{}
+ |> Object.change(%{data: data})
|> Repo.insert()
end
@@ -58,8 +62,41 @@ defmodule Pleroma.Object do
|> cast(params, [:data])
|> validate_required([:data])
|> unique_constraint(:ap_id, name: :objects_unique_apid_index)
+ # Expecting `maybe_handle_hashtags_change/1` to run last:
+ |> maybe_handle_hashtags_change(struct)
+ end
+
+ # Note: not checking activity type (assuming non-legacy objects are associated with Create act.)
+ defp maybe_handle_hashtags_change(changeset, struct) do
+ with %Ecto.Changeset{valid?: true} <- changeset,
+ data_hashtags_change = get_change(changeset, :data),
+ {_, true} <- {:changed, hashtags_changed?(struct, data_hashtags_change)},
+ {:ok, hashtag_records} <-
+ data_hashtags_change
+ |> object_data_hashtags()
+ |> Hashtag.get_or_create_by_names() do
+ put_assoc(changeset, :hashtags, hashtag_records)
+ else
+ %{valid?: false} ->
+ changeset
+
+ {:changed, false} ->
+ changeset
+
+ {:error, _} ->
+ validate_change(changeset, :data, fn _, _ ->
+ [data: "error referencing hashtags"]
+ end)
+ end
+ end
+
+ defp hashtags_changed?(%Object{} = struct, %{"tag" => _} = data) do
+ Enum.sort(embedded_hashtags(struct)) !=
+ Enum.sort(object_data_hashtags(data))
end
+ defp hashtags_changed?(_, _), do: false
+
def get_by_id(nil), do: nil
def get_by_id(id), do: Repo.get(Object, id)
@@ -187,9 +224,13 @@ defmodule Pleroma.Object do
def swap_object_with_tombstone(object) do
tombstone = make_tombstone(object)
- object
- |> Object.change(%{data: tombstone})
- |> Repo.update()
+ with {:ok, object} <-
+ object
+ |> Object.change(%{data: tombstone})
+ |> Repo.update() do
+ Hashtag.unlink(object)
+ {:ok, object}
+ end
end
def delete(%Object{data: %{"id" => id}} = object) do
@@ -349,4 +390,39 @@ defmodule Pleroma.Object do
def self_replies(object, opts \\ []),
do: replies(object, Keyword.put(opts, :self_only, true))
+
+ def tags(%Object{data: %{"tag" => tags}}) when is_list(tags), do: tags
+
+ def tags(_), do: []
+
+ def hashtags(%Object{} = object) do
+ # Note: always using embedded hashtags regardless whether they are migrated to hashtags table
+ # (embedded hashtags stay in sync anyways, and we avoid extra joins and preload hassle)
+ embedded_hashtags(object)
+ end
+
+ def embedded_hashtags(%Object{data: data}) do
+ object_data_hashtags(data)
+ end
+
+ def embedded_hashtags(_), do: []
+
+ def object_data_hashtags(%{"tag" => tags}) when is_list(tags) do
+ tags
+ |> Enum.filter(fn
+ %{"type" => "Hashtag"} = data -> Map.has_key?(data, "name")
+ plain_text when is_bitstring(plain_text) -> true
+ _ -> false
+ end)
+ |> Enum.map(fn
+ %{"name" => "#" <> hashtag} -> String.downcase(hashtag)
+ %{"name" => hashtag} -> String.downcase(hashtag)
+ hashtag when is_bitstring(hashtag) -> String.downcase(hashtag)
+ end)
+ |> Enum.uniq()
+ # Note: "" elements (plain text) might occur in `data.tag` for incoming objects
+ |> Enum.filter(&(&1 not in [nil, ""]))
+ end
+
+ def object_data_hashtags(_), do: []
end
diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
@@ -4,6 +4,7 @@
defmodule Pleroma.Object.Fetcher do
alias Pleroma.HTTP
+ alias Pleroma.Maps
alias Pleroma.Object
alias Pleroma.Object.Containment
alias Pleroma.Repo
@@ -101,6 +102,9 @@ defmodule Pleroma.Object.Fetcher do
{:transmogrifier, {:error, {:reject, e}}} ->
{:reject, e}
+ {:transmogrifier, {:reject, e}} ->
+ {:reject, e}
+
{:transmogrifier, _} = e ->
{:error, e}
@@ -124,12 +128,14 @@ defmodule Pleroma.Object.Fetcher do
defp prepare_activity_params(data) do
%{
"type" => "Create",
- "to" => data["to"] || [],
- "cc" => data["cc"] || [],
# Should we seriously keep this attributedTo thing?
"actor" => data["actor"] || data["attributedTo"],
"object" => data
}
+ |> Maps.put_if_present("to", data["to"])
+ |> Maps.put_if_present("cc", data["cc"])
+ |> Maps.put_if_present("bto", data["bto"])
+ |> Maps.put_if_present("bcc", data["bcc"])
end
def fetch_object_from_id!(id, options \\ []) do
diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex
@@ -63,8 +63,8 @@ defmodule Pleroma.Repo do
iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500, :batches)
"""
@spec chunk_stream(Ecto.Query.t(), integer(), atom()) :: Enumerable.t()
- def chunk_stream(query, chunk_size, returns_as \\ :one) do
- # We don't actually need start and end funcitons of resource streaming,
+ def chunk_stream(query, chunk_size, returns_as \\ :one, query_options \\ []) do
+ # We don't actually need start and end functions of resource streaming,
# but it seems to be the only way to not fetch records one-by-one and
# have individual records be the elements of the stream, instead of
# lists of records
@@ -76,7 +76,7 @@ defmodule Pleroma.Repo do
|> order_by(asc: :id)
|> where([r], r.id > ^last_id)
|> limit(^chunk_size)
- |> all()
+ |> all(query_options)
|> case do
[] ->
{:halt, last_id}
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -87,7 +87,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp increase_replies_count_if_reply(_create_data), do: :noop
- @object_types ~w[ChatMessage Question Answer Audio Video Event Article]
+ @object_types ~w[ChatMessage Question Answer Audio Video Event Article Note]
@impl true
def persist(%{"type" => type} = object, meta) when type in @object_types do
with {:ok, object} <- Object.create(object) do
@@ -693,51 +693,150 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp restrict_since(query, _), do: query
- defp restrict_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
- raise "Can't use the child object without preloading!"
+ defp restrict_embedded_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do
+ raise_on_missing_preload()
+ end
+
+ defp restrict_embedded_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
+ tag_all = Enum.map(tag_all, &String.downcase/1)
+
+ from(
+ [_activity, object] in query,
+ where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
+ )
+ end
+
+ defp restrict_embedded_tag_all(query, %{tag_all: tag}) when is_binary(tag) do
+ restrict_embedded_tag_any(query, %{tag: tag})
+ end
+
+ defp restrict_embedded_tag_all(query, _), do: query
+
+ defp restrict_embedded_tag_any(_query, %{tag: _tag, skip_preload: true}) do
+ raise_on_missing_preload()
+ end
+
+ defp restrict_embedded_tag_any(query, %{tag: [_ | _] = tag_any}) do
+ tag_any = Enum.map(tag_any, &String.downcase/1)
+
+ from(
+ [_activity, object] in query,
+ where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag_any)
+ )
+ end
+
+ defp restrict_embedded_tag_any(query, %{tag: tag}) when is_binary(tag) do
+ restrict_embedded_tag_any(query, %{tag: [tag]})
end
- defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do
+ defp restrict_embedded_tag_any(query, _), do: query
+
+ defp restrict_embedded_tag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
+ raise_on_missing_preload()
+ end
+
+ defp restrict_embedded_tag_reject_any(query, %{tag_reject: [_ | _] = tag_reject}) do
+ tag_reject = Enum.map(tag_reject, &String.downcase/1)
+
from(
[_activity, object] in query,
where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
)
end
- defp restrict_tag_reject(query, _), do: query
+ defp restrict_embedded_tag_reject_any(query, %{tag_reject: tag_reject})
+ when is_binary(tag_reject) do
+ restrict_embedded_tag_reject_any(query, %{tag_reject: [tag_reject]})
+ end
- defp restrict_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do
- raise "Can't use the child object without preloading!"
+ defp restrict_embedded_tag_reject_any(query, _), do: query
+
+ defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do
+ raise_on_missing_preload()
+ end
+
+ defp restrict_hashtag_all(query, %{tag_all: [single_tag]}) do
+ restrict_hashtag_any(query, %{tag: single_tag})
end
- defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
+ defp restrict_hashtag_all(query, %{tag_all: [_ | _] = tags}) do
from(
[_activity, object] in query,
- where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
+ where:
+ fragment(
+ """
+ (SELECT array_agg(hashtags.name) FROM hashtags JOIN hashtags_objects
+ ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?::citext[])
+ AND hashtags_objects.object_id = ?) @> ?
+ """,
+ ^tags,
+ object.id,
+ ^tags
+ )
)
end
- defp restrict_tag_all(query, _), do: query
+ defp restrict_hashtag_all(query, %{tag_all: tag}) when is_binary(tag) do
+ restrict_hashtag_all(query, %{tag_all: [tag]})
+ end
- defp restrict_tag(_query, %{tag: _tag, skip_preload: true}) do
- raise "Can't use the child object without preloading!"
+ defp restrict_hashtag_all(query, _), do: query
+
+ defp restrict_hashtag_any(_query, %{tag: _tag, skip_preload: true}) do
+ raise_on_missing_preload()
end
- defp restrict_tag(query, %{tag: tag}) when is_list(tag) do
+ defp restrict_hashtag_any(query, %{tag: [_ | _] = tags}) do
from(
[_activity, object] in query,
- where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag)
+ where:
+ fragment(
+ """
+ EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects
+ ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?::citext[])
+ AND hashtags_objects.object_id = ? LIMIT 1)
+ """,
+ ^tags,
+ object.id
+ )
)
end
- defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do
+ defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do
+ restrict_hashtag_any(query, %{tag: [tag]})
+ end
+
+ defp restrict_hashtag_any(query, _), do: query
+
+ defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
+ raise_on_missing_preload()
+ end
+
+ defp restrict_hashtag_reject_any(query, %{tag_reject: [_ | _] = tags_reject}) do
from(
[_activity, object] in query,
- where: fragment("(?)->'tag' \\? (?)", object.data, ^tag)
+ where:
+ fragment(
+ """
+ NOT EXISTS (SELECT 1 FROM hashtags JOIN hashtags_objects
+ ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?::citext[])
+ AND hashtags_objects.object_id = ? LIMIT 1)
+ """,
+ ^tags_reject,
+ object.id
+ )
)
end
- defp restrict_tag(query, _), do: query
+ defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do
+ restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]})
+ end
+
+ defp restrict_hashtag_reject_any(query, _), do: query
+
+ defp raise_on_missing_preload do
+ raise "Can't use the child object without preloading!"
+ end
defp restrict_recipients(query, [], _user), do: query
@@ -1128,41 +1227,51 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
skip_thread_containment: Config.get([:instance, :skip_thread_containment])
}
- Activity
- |> maybe_preload_objects(opts)
- |> maybe_preload_bookmarks(opts)
- |> maybe_preload_report_notes(opts)
- |> maybe_set_thread_muted_field(opts)
- |> maybe_order(opts)
- |> restrict_recipients(recipients, opts[:user])
- |> restrict_replies(opts)
- |> restrict_tag(opts)
- |> restrict_tag_reject(opts)
- |> restrict_tag_all(opts)
- |> restrict_since(opts)
- |> restrict_local(opts)
- |> restrict_remote(opts)
- |> restrict_actor(opts)
- |> restrict_type(opts)
- |> restrict_state(opts)
- |> restrict_favorited_by(opts)
- |> restrict_blocked(restrict_blocked_opts)
- |> restrict_muted(restrict_muted_opts)
- |> restrict_filtered(opts)
- |> restrict_media(opts)
- |> restrict_visibility(opts)
- |> restrict_thread_visibility(opts, config)
- |> restrict_reblogs(opts)
- |> restrict_pinned(opts)
- |> restrict_muted_reblogs(restrict_muted_reblogs_opts)
- |> restrict_instance(opts)
- |> restrict_announce_object_actor(opts)
- |> restrict_filtered(opts)
- |> Activity.restrict_deactivated_users()
- |> exclude_poll_votes(opts)
- |> exclude_chat_messages(opts)
- |> exclude_invisible_actors(opts)
- |> exclude_visibility(opts)
+ query =
+ Activity
+ |> maybe_preload_objects(opts)
+ |> maybe_preload_bookmarks(opts)
+ |> maybe_preload_report_notes(opts)
+ |> maybe_set_thread_muted_field(opts)
+ |> maybe_order(opts)
+ |> restrict_recipients(recipients, opts[:user])
+ |> restrict_replies(opts)
+ |> restrict_since(opts)
+ |> restrict_local(opts)
+ |> restrict_remote(opts)
+ |> restrict_actor(opts)
+ |> restrict_type(opts)
+ |> restrict_state(opts)
+ |> restrict_favorited_by(opts)
+ |> restrict_blocked(restrict_blocked_opts)
+ |> restrict_muted(restrict_muted_opts)
+ |> restrict_filtered(opts)
+ |> restrict_media(opts)
+ |> restrict_visibility(opts)
+ |> restrict_thread_visibility(opts, config)
+ |> restrict_reblogs(opts)
+ |> restrict_pinned(opts)
+ |> restrict_muted_reblogs(restrict_muted_reblogs_opts)
+ |> restrict_instance(opts)
+ |> restrict_announce_object_actor(opts)
+ |> restrict_filtered(opts)
+ |> Activity.restrict_deactivated_users()
+ |> exclude_poll_votes(opts)
+ |> exclude_chat_messages(opts)
+ |> exclude_invisible_actors(opts)
+ |> exclude_visibility(opts)
+
+ if Config.get([:database, :improved_hashtag_timeline]) do
+ query
+ |> restrict_hashtag_any(opts)
+ |> restrict_hashtag_all(opts)
+ |> restrict_hashtag_reject_any(opts)
+ else
+ query
+ |> restrict_embedded_tag_any(opts)
+ |> restrict_embedded_tag_all(opts)
+ |> restrict_embedded_tag_reject_any(opts)
+ end
end
def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -11,7 +11,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
alias Pleroma.Object.Fetcher
alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub
- alias Pleroma.Web.ActivityPub.Builder
alias Pleroma.Web.ActivityPub.InternalFetchActor
alias Pleroma.Web.ActivityPub.ObjectView
alias Pleroma.Web.ActivityPub.Pipeline
@@ -403,83 +402,90 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
|> json(err)
end
- defp handle_user_activity(
- %User{} = user,
- %{"type" => "Create", "object" => %{"type" => "Note"} = object} = params
- ) do
- content = if is_binary(object["content"]), do: object["content"], else: ""
- name = if is_binary(object["name"]), do: object["name"], else: ""
- summary = if is_binary(object["summary"]), do: object["summary"], else: ""
- length = String.length(content <> name <> summary)
+ defp fix_user_message(%User{ap_id: actor}, %{"type" => "Create", "object" => object} = activity)
+ when is_map(object) do
+ length =
+ [object["content"], object["summary"], object["name"]]
+ |> Enum.filter(&is_binary(&1))
+ |> Enum.join("")
+ |> String.length()
- if length > Pleroma.Config.get([:instance, :limit]) do
- {:error, dgettext("errors", "Note is over the character limit")}
- else
+ limit = Pleroma.Config.get([:instance, :limit])
+
+ if length < limit do
object =
object
- |> Map.merge(Map.take(params, ["to", "cc"]))
- |> Map.put("attributedTo", user.ap_id)
- |> Transmogrifier.fix_object()
-
- ActivityPub.create(%{
- to: params["to"],
- actor: user,
- context: object["context"],
- object: object,
- additional: Map.take(params, ["cc"])
- })
- end
- end
+ |> Transmogrifier.strip_internal_fields()
+ |> Map.put("attributedTo", actor)
+ |> Map.put("actor", actor)
+ |> Map.put("id", Utils.generate_object_id())
- defp handle_user_activity(%User{} = user, %{"type" => "Delete"} = params) do
- with %Object{} = object <- Object.normalize(params["object"], fetch: false),
- true <- user.is_moderator || user.ap_id == object.data["actor"],
- {:ok, delete_data, _} <- Builder.delete(user, object.data["id"]),
- {:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do
- {:ok, delete}
+ {:ok, Map.put(activity, "object", object)}
else
- _ -> {:error, dgettext("errors", "Can't delete object")}
+ {:error,
+ dgettext(
+ "errors",
+ "Character limit (%{limit} characters) exceeded, contains %{length} characters",
+ limit: limit,
+ length: length
+ )}
end
end
- defp handle_user_activity(%User{} = user, %{"type" => "Like"} = params) do
- with %Object{} = object <- Object.normalize(params["object"], fetch: false),
- {_, {:ok, like_object, meta}} <- {:build_object, Builder.like(user, object)},
- {_, {:ok, %Activity{} = activity, _meta}} <-
- {:common_pipeline,
- Pipeline.common_pipeline(like_object, Keyword.put(meta, :local, true))} do
+ defp fix_user_message(
+ %User{ap_id: actor} = user,
+ %{"type" => "Delete", "object" => object} = activity
+ ) do
+ with {_, %Object{data: object_data}} <- {:normalize, Object.normalize(object, fetch: false)},
+ {_, true} <- {:permission, user.is_moderator || actor == object_data["actor"]} do
{:ok, activity}
else
- _ -> {:error, dgettext("errors", "Can't like object")}
+ {:normalize, _} ->
+ {:error, "No such object found"}
+
+ {:permission, _} ->
+ {:forbidden, "You can't delete this object"}
end
end
- defp handle_user_activity(_, _) do
- {:error, dgettext("errors", "Unhandled activity type")}
+ defp fix_user_message(%User{}, activity) do
+ {:ok, activity}
end
def update_outbox(
- %{assigns: %{user: %User{nickname: nickname} = user}} = conn,
+ %{assigns: %{user: %User{nickname: nickname, ap_id: actor} = user}} = conn,
%{"nickname" => nickname} = params
) do
- actor = user.ap_id
-
params =
params
- |> Map.drop(["id"])
+ |> Map.drop(["nickname"])
+ |> Map.put("id", Utils.generate_activity_id())
|> Map.put("actor", actor)
- |> Transmogrifier.fix_addressing()
- with {:ok, %Activity{} = activity} <- handle_user_activity(user, params) do
+ with {:ok, params} <- fix_user_message(user, params),
+ {:ok, activity, _} <- Pipeline.common_pipeline(params, local: true),
+ %Activity{data: activity_data} <- Activity.normalize(activity) do
conn
|> put_status(:created)
- |> put_resp_header("location", activity.data["id"])
- |> json(activity.data)
+ |> put_resp_header("location", activity_data["id"])
+ |> json(activity_data)
else
+ {:forbidden, message} ->
+ conn
+ |> put_status(:forbidden)
+ |> json(message)
+
{:error, message} ->
conn
|> put_status(:bad_request)
|> json(message)
+
+ e ->
+ Logger.warn(fn -> "AP C2S: #{inspect(e)}" end)
+
+ conn
+ |> put_status(:bad_request)
+ |> json("Bad Request")
end
end
diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -74,9 +74,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
object =
if MRF.subdomain_match?(media_nsfw, actor_host) do
- tags = (child_object["tag"] || []) ++ ["nsfw"]
- child_object = Map.put(child_object, "tag", tags)
- child_object = Map.put(child_object, "sensitive", true)
+ child_object =
+ child_object
+ |> Map.put("tag", (child_object["tag"] || []) ++ ["nsfw"])
+ |> Map.put("sensitive", true)
+
Map.put(object, "object", child_object)
else
object
diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -163,7 +163,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
end
end
- def validate(%{"type" => "Article"} = object, meta) do
+ def validate(%{"type" => type} = object, meta) when type in ~w[Article Note] do
with {:ok, object} <-
object
|> ArticleNoteValidator.cast_and_validate()
@@ -212,7 +212,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
%{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity,
meta
)
- when objtype in ~w[Question Answer Audio Video Event Article] do
+ when objtype in ~w[Question Answer Audio Video Event Article Note] do
with {:ok, object_data} <- cast_and_apply(object),
meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
{:ok, create_activity} <-
@@ -234,6 +234,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
end
end
+ def validate(o, m), do: {:error, {:validator_not_set, {o, m}}}
+
def cast_and_apply(%{"type" => "ChatMessage"} = object) do
ChatMessageValidator.cast_and_apply(object)
end
@@ -254,13 +256,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
EventValidator.cast_and_apply(object)
end
- def cast_and_apply(%{"type" => "Article"} = object) do
+ def cast_and_apply(%{"type" => type} = object) when type in ~w[Article Note] do
ArticleNoteValidator.cast_and_apply(object)
end
def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
- # is_struct/1 isn't present in Elixir 1.8.x
+ # is_struct/1 appears in Elixir 1.11
def stringify_keys(%{__struct__: _} = object) do
object
|> Map.from_struct()
diff --git a/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex b/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex
@@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator do
|> cast(data, __schema__(:fields))
end
- def validate_data(cng) do
+ defp validate_data(cng) do
cng
|> validate_required([:id, :type, :actor, :to, :cc, :object])
|> validate_inclusion(:type, ["Accept", "Reject"])
diff --git a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex
@@ -50,7 +50,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do
cng
end
- def validate_data(data_cng) do
+ defp validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["Announce"])
|> validate_required([:id, :type, :object, :actor, :to, :cc])
diff --git a/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex b/lib/pleroma/web/activity_pub/object_validators/answer_validator.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
use Ecto.Schema
alias Pleroma.EctoType.ActivityPub.ObjectValidators
+ alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
import Ecto.Changeset
@@ -23,6 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
field(:name, :string)
field(:inReplyTo, ObjectValidators.ObjectID)
field(:attributedTo, ObjectValidators.ObjectID)
+ field(:context, :string)
# TODO: Remove actor on objects
field(:actor, ObjectValidators.ObjectID)
@@ -46,11 +48,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
end
def changeset(struct, data) do
+ data =
+ data
+ |> CommonFixes.fix_actor()
+ |> CommonFixes.fix_object_defaults()
+
struct
|> cast(data, __schema__(:fields))
end
- def validate_data(data_cng) do
+ defp validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["Answer"])
|> validate_required([:id, :inReplyTo, :name, :attributedTo, :actor])
diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+ alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator
alias Pleroma.Web.ActivityPub.Transmogrifier
import Ecto.Changeset
@@ -22,8 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
field(:cc, ObjectValidators.Recipients, default: [])
field(:bto, ObjectValidators.Recipients, default: [])
field(:bcc, ObjectValidators.Recipients, default: [])
- # TODO: Write type
- field(:tag, {:array, :map}, default: [])
+ embeds_many(:tag, TagValidator)
field(:type, :string)
field(:name, :string)
@@ -50,6 +50,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
+
+ field(:replies, {:array, ObjectValidators.ObjectID}, default: [])
end
def cast_and_apply(data) do
@@ -65,36 +67,51 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
end
def cast_data(data) do
- data = fix(data)
-
%__MODULE__{}
|> changeset(data)
end
- defp fix_url(%{"url" => url} = data) when is_map(url) do
- Map.put(data, "url", url["href"])
- end
-
+ defp fix_url(%{"url" => url} = data) when is_bitstring(url), do: data
+ defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"])
defp fix_url(data), do: data
+ defp fix_tag(%{"tag" => tag} = data) when is_list(tag), do: data
+ defp fix_tag(%{"tag" => tag} = data) when is_map(tag), do: Map.put(data, "tag", [tag])
+ defp fix_tag(data), do: Map.drop(data, ["tag"])
+
+ defp fix_replies(%{"replies" => %{"first" => %{"items" => replies}}} = data)
+ when is_list(replies),
+ do: Map.put(data, "replies", replies)
+
+ defp fix_replies(%{"replies" => %{"items" => replies}} = data) when is_list(replies),
+ do: Map.put(data, "replies", replies)
+
+ defp fix_replies(%{"replies" => replies} = data) when is_bitstring(replies),
+ do: Map.drop(data, ["replies"])
+
+ defp fix_replies(data), do: data
+
defp fix(data) do
data
- |> CommonFixes.fix_defaults()
- |> CommonFixes.fix_attribution()
|> CommonFixes.fix_actor()
+ |> CommonFixes.fix_object_defaults()
|> fix_url()
+ |> fix_tag()
+ |> fix_replies()
|> Transmogrifier.fix_emoji()
+ |> Transmogrifier.fix_content_map()
end
def changeset(struct, data) do
data = fix(data)
struct
- |> cast(data, __schema__(:fields) -- [:attachment])
+ |> cast(data, __schema__(:fields) -- [:attachment, :tag])
|> cast_embed(:attachment)
+ |> cast_embed(:tag)
end
- def validate_data(data_cng) do
+ defp validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["Article", "Note"])
|> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
@@ -6,7 +6,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
use Ecto.Schema
alias Pleroma.EctoType.ActivityPub.ObjectValidators
- alias Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator
import Ecto.Changeset
@@ -90,7 +89,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
end
end
- def validate_data(cng) do
+ defp validate_data(cng) do
cng
|> validate_inclusion(:type, ~w[Document Audio Image Video])
|> validate_required([:mediaType, :url, :type])
diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+ alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator
alias Pleroma.Web.ActivityPub.Transmogrifier
import Ecto.Changeset
@@ -23,8 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
field(:cc, ObjectValidators.Recipients, default: [])
field(:bto, ObjectValidators.Recipients, default: [])
field(:bcc, ObjectValidators.Recipients, default: [])
- # TODO: Write type
- field(:tag, {:array, :map}, default: [])
+ embeds_many(:tag, TagValidator)
field(:type, :string)
field(:name, :string)
@@ -106,9 +106,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
defp fix(data) do
data
- |> CommonFixes.fix_defaults()
- |> CommonFixes.fix_attribution()
|> CommonFixes.fix_actor()
+ |> CommonFixes.fix_object_defaults()
|> Transmogrifier.fix_emoji()
|> fix_url()
|> fix_content()
@@ -118,11 +117,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
data = fix(data)
struct
- |> cast(data, __schema__(:fields) -- [:attachment])
+ |> cast(data, __schema__(:fields) -- [:attachment, :tag])
|> cast_embed(:attachment)
+ |> cast_embed(:tag)
end
- def validate_data(data_cng) do
+ defp validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["Audio", "Video"])
|> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment])
diff --git a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex
@@ -26,7 +26,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator do
|> cast(data, __schema__(:fields))
end
- def validate_data(cng) do
+ defp validate_data(cng) do
cng
|> validate_required([:id, :type, :actor, :to, :cc, :object])
|> validate_inclusion(:type, ["Block"])
diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -67,7 +67,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do
|> cast_embed(:attachment)
end
- def validate_data(data_cng) do
+ defp validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["ChatMessage"])
|> validate_required([:id, :actor, :to, :type, :published])
diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
@@ -3,26 +3,56 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Object.Containment
+ alias Pleroma.User
+ alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.Utils
- # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults
- def fix_defaults(data) do
+ def cast_and_filter_recipients(message, field, follower_collection, field_fallback \\ []) do
+ {:ok, data} = ObjectValidators.Recipients.cast(message[field] || field_fallback)
+
+ data =
+ Enum.reject(data, fn x ->
+ String.ends_with?(x, "/followers") and x != follower_collection
+ end)
+
+ Map.put(message, field, data)
+ end
+
+ def fix_object_defaults(data) do
%{data: %{"id" => context}, id: context_id} =
Utils.create_context(data["context"] || data["conversation"])
+ %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["attributedTo"])
+
data
|> Map.put("context", context)
|> Map.put("context_id", context_id)
+ |> cast_and_filter_recipients("to", follower_collection)
+ |> cast_and_filter_recipients("cc", follower_collection)
+ |> cast_and_filter_recipients("bto", follower_collection)
+ |> cast_and_filter_recipients("bcc", follower_collection)
+ |> Transmogrifier.fix_implicit_addressing(follower_collection)
+ |> Transmogrifier.set_sensitive()
end
- def fix_attribution(data) do
- data
- |> Map.put_new("actor", data["attributedTo"])
+ def fix_activity_addressing(activity, _meta) do
+ %User{follower_address: follower_collection} = User.get_cached_by_ap_id(activity["actor"])
+
+ activity
+ |> cast_and_filter_recipients("to", follower_collection)
+ |> cast_and_filter_recipients("cc", follower_collection)
+ |> cast_and_filter_recipients("bto", follower_collection)
+ |> cast_and_filter_recipients("bcc", follower_collection)
+ |> Transmogrifier.fix_implicit_addressing(follower_collection)
end
def fix_actor(data) do
- actor = Containment.get_actor(data)
+ actor =
+ data
+ |> Map.put_new("actor", data["attributedTo"])
+ |> Containment.get_actor()
data
|> Map.put("actor", actor)
diff --git a/lib/pleroma/web/activity_pub/object_validators/common_validations.ex b/lib/pleroma/web/activity_pub/object_validators/common_validations.ex
@@ -14,6 +14,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
fields
|> Enum.map(fn field -> get_field(cng, field) end)
|> Enum.any?(fn
+ nil -> false
[] -> false
_ -> true
end)
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex
@@ -39,7 +39,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator do
|> validate_data(meta)
end
- def validate_data(cng, meta \\ []) do
+ defp validate_data(cng, meta) do
cng
|> validate_required([:id, :actor, :to, :type, :object])
|> validate_inclusion(:type, ["Create"])
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
@@ -10,8 +10,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
alias Pleroma.EctoType.ActivityPub.ObjectValidators
alias Pleroma.Object
+ alias Pleroma.User
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+ alias Pleroma.Web.ActivityPub.Transmogrifier
import Ecto.Changeset
@@ -23,6 +25,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
field(:type, :string)
field(:to, ObjectValidators.Recipients, default: [])
field(:cc, ObjectValidators.Recipients, default: [])
+ field(:bto, ObjectValidators.Recipients, default: [])
+ field(:bcc, ObjectValidators.Recipients, default: [])
field(:object, ObjectValidators.ObjectID)
field(:expires_at, ObjectValidators.DateTime)
@@ -54,39 +58,37 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
|> cast(data, __schema__(:fields))
end
- defp fix_context(data, meta) do
- if object = meta[:object_data] do
- Map.put_new(data, "context", object["context"])
- else
- data
- end
- end
+ # CommonFixes.fix_activity_addressing adapted for Create specific behavior
+ defp fix_addressing(data, object) do
+ %User{follower_address: follower_collection} = User.get_cached_by_ap_id(data["actor"])
- defp fix_addressing(data, meta) do
- if object = meta[:object_data] do
- data
- |> Map.put_new("to", object["to"] || [])
- |> Map.put_new("cc", object["cc"] || [])
- else
- data
- end
+ data
+ |> CommonFixes.cast_and_filter_recipients("to", follower_collection, object["to"])
+ |> CommonFixes.cast_and_filter_recipients("cc", follower_collection, object["cc"])
+ |> CommonFixes.cast_and_filter_recipients("bto", follower_collection, object["bto"])
+ |> CommonFixes.cast_and_filter_recipients("bcc", follower_collection, object["bcc"])
+ |> Transmogrifier.fix_implicit_addressing(follower_collection)
end
- defp fix(data, meta) do
+ def fix(data, meta) do
+ object = meta[:object_data]
+
data
- |> fix_context(meta)
- |> fix_addressing(meta)
|> CommonFixes.fix_actor()
+ |> Map.put_new("context", object["context"])
+ |> fix_addressing(object)
end
- def validate_data(cng, meta \\ []) do
+ defp validate_data(cng, meta) do
+ object = meta[:object_data]
+
cng
- |> validate_required([:actor, :type, :object])
+ |> validate_required([:actor, :type, :object, :to, :cc])
|> validate_inclusion(:type, ["Create"])
|> CommonValidations.validate_actor_presence()
- |> CommonValidations.validate_any_presence([:to, :cc])
- |> validate_actors_match(meta)
- |> validate_context_match(meta)
+ |> validate_actors_match(object)
+ |> validate_context_match(object)
+ |> validate_addressing_match(object)
|> validate_object_nonexistence()
|> validate_object_containment()
end
@@ -118,8 +120,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
end)
end
- def validate_actors_match(cng, meta) do
- attributed_to = meta[:object_data]["attributedTo"] || meta[:object_data]["actor"]
+ def validate_actors_match(cng, object) do
+ attributed_to = object["attributedTo"] || object["actor"]
cng
|> validate_change(:actor, fn :actor, actor ->
@@ -131,7 +133,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
end)
end
- def validate_context_match(cng, %{object_data: %{"context" => object_context}}) do
+ def validate_context_match(cng, %{"context" => object_context}) do
cng
|> validate_change(:context, fn :context, context ->
if context == object_context do
@@ -142,5 +144,18 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
end)
end
- def validate_context_match(cng, _), do: cng
+ def validate_addressing_match(cng, object) do
+ [:to, :cc, :bcc, :bto]
+ |> Enum.reduce(cng, fn field, cng ->
+ object_data = object[to_string(field)]
+
+ validate_change(cng, field, fn field, data ->
+ if data == object_data do
+ []
+ else
+ [{field, "field doesn't match with object (#{inspect(object_data)})"}]
+ end
+ end)
+ end)
+ end
end
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex
@@ -1,29 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do
- use Ecto.Schema
-
- alias Pleroma.EctoType.ActivityPub.ObjectValidators
- alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator
-
- import Ecto.Changeset
-
- @primary_key false
-
- embedded_schema do
- field(:id, ObjectValidators.ObjectID, primary_key: true)
- field(:actor, ObjectValidators.ObjectID)
- field(:type, :string)
- field(:to, ObjectValidators.Recipients, default: [])
- field(:cc, ObjectValidators.Recipients, default: [])
- field(:bto, ObjectValidators.Recipients, default: [])
- field(:bcc, ObjectValidators.Recipients, default: [])
- embeds_one(:object, NoteValidator)
- end
-
- def cast_data(data) do
- cast(%__MODULE__{}, data, __schema__(:fields))
- end
-end
diff --git a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex
@@ -53,7 +53,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do
Tombstone
Video
}
- def validate_data(cng) do
+ defp validate_data(cng) do
cng
|> validate_required([:id, :type, :actor, :to, :cc, :object])
|> validate_inclusion(:type, ["Delete"])
diff --git a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex
@@ -70,7 +70,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do
end
end
- def validate_data(data_cng) do
+ defp validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["EmojiReact"])
|> validate_required([:id, :type, :object, :actor, :context, :to, :cc, :content])
diff --git a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+ alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator
alias Pleroma.Web.ActivityPub.Transmogrifier
import Ecto.Changeset
@@ -23,8 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
field(:cc, ObjectValidators.Recipients, default: [])
field(:bto, ObjectValidators.Recipients, default: [])
field(:bcc, ObjectValidators.Recipients, default: [])
- # TODO: Write type
- field(:tag, {:array, :map}, default: [])
+ embeds_many(:tag, TagValidator)
field(:type, :string)
field(:name, :string)
@@ -72,8 +72,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
defp fix(data) do
data
- |> CommonFixes.fix_defaults()
- |> CommonFixes.fix_attribution()
+ |> CommonFixes.fix_actor()
+ |> CommonFixes.fix_object_defaults()
|> Transmogrifier.fix_emoji()
end
@@ -81,11 +81,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
data = fix(data)
struct
- |> cast(data, __schema__(:fields) -- [:attachment])
+ |> cast(data, __schema__(:fields) -- [:attachment, :tag])
|> cast_embed(:attachment)
+ |> cast_embed(:tag)
end
- def validate_data(data_cng) do
+ defp validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["Event"])
|> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
diff --git a/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex
@@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator do
|> cast(data, __schema__(:fields))
end
- def validate_data(cng) do
+ defp validate_data(cng) do
cng
|> validate_required([:id, :type, :actor, :to, :cc, :object])
|> validate_inclusion(:type, ["Follow"])
diff --git a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex
@@ -76,7 +76,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do
end
end
- def validate_data(data_cng) do
+ defp validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["Like"])
|> validate_required([:id, :type, :object, :actor, :context, :to, :cc])
diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator
+ alias Pleroma.Web.ActivityPub.ObjectValidators.TagValidator
alias Pleroma.Web.ActivityPub.Transmogrifier
import Ecto.Changeset
@@ -24,8 +25,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
field(:cc, ObjectValidators.Recipients, default: [])
field(:bto, ObjectValidators.Recipients, default: [])
field(:bcc, ObjectValidators.Recipients, default: [])
- # TODO: Write type
- field(:tag, {:array, :map}, default: [])
+ embeds_many(:tag, TagValidator)
field(:type, :string)
field(:content, :string)
field(:context, :string)
@@ -83,8 +83,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
defp fix(data) do
data
- |> CommonFixes.fix_defaults()
- |> CommonFixes.fix_attribution()
+ |> CommonFixes.fix_actor()
+ |> CommonFixes.fix_object_defaults()
|> Transmogrifier.fix_emoji()
|> fix_closed()
end
@@ -93,13 +93,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
data = fix(data)
struct
- |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment])
+ |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment, :tag])
|> cast_embed(:attachment)
|> cast_embed(:anyOf)
|> cast_embed(:oneOf)
+ |> cast_embed(:tag)
end
- def validate_data(data_cng) do
+ defp validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["Question"])
|> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex
@@ -0,0 +1,77 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do
+ use Ecto.Schema
+
+ alias Pleroma.EctoType.ActivityPub.ObjectValidators
+
+ import Ecto.Changeset
+
+ @primary_key false
+ embedded_schema do
+ # Common
+ field(:type, :string)
+ field(:name, :string)
+
+ # Mention, Hashtag
+ field(:href, ObjectValidators.Uri)
+
+ # Emoji
+ embeds_one :icon, IconObjectValidator, primary_key: false do
+ field(:type, :string)
+ field(:url, ObjectValidators.Uri)
+ end
+
+ field(:updated, ObjectValidators.DateTime)
+ field(:id, ObjectValidators.Uri)
+ end
+
+ def cast_and_validate(data) do
+ data
+ |> cast_data()
+ end
+
+ def cast_data(data) do
+ %__MODULE__{}
+ |> changeset(data)
+ end
+
+ def changeset(struct, %{"type" => "Mention"} = data) do
+ struct
+ |> cast(data, [:type, :name, :href])
+ |> validate_required([:type, :href])
+ end
+
+ def changeset(struct, %{"type" => "Hashtag", "name" => name} = data) do
+ name =
+ cond do
+ "#" <> name -> name
+ name -> name
+ end
+ |> String.downcase()
+
+ data = Map.put(data, "name", name)
+
+ struct
+ |> cast(data, [:type, :name, :href])
+ |> validate_required([:type, :name])
+ end
+
+ def changeset(struct, %{"type" => "Emoji"} = data) do
+ data = Map.put(data, "name", String.trim(data["name"], ":"))
+
+ struct
+ |> cast(data, [:type, :name, :updated, :id])
+ |> cast_embed(:icon, with: &icon_changeset/2)
+ |> validate_required([:type, :name, :icon])
+ end
+
+ def icon_changeset(struct, data) do
+ struct
+ |> cast(data, [:type, :url])
+ |> validate_inclusion(:type, ~w[Image])
+ |> validate_required([:type, :url])
+ end
+end
diff --git a/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex b/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex
@@ -38,7 +38,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator do
|> cast(data, __schema__(:fields))
end
- def validate_data(data_cng) do
+ defp validate_data(data_cng) do
data_cng
|> validate_inclusion(:type, ["Undo"])
|> validate_required([:id, :type, :object, :actor, :to, :cc])
diff --git a/lib/pleroma/web/activity_pub/object_validators/update_validator.ex b/lib/pleroma/web/activity_pub/object_validators/update_validator.ex
@@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator do
|> cast(data, __schema__(:fields))
end
- def validate_data(cng) do
+ defp validate_data(cng) do
cng
|> validate_required([:id, :type, :actor, :to, :cc, :object])
|> validate_inclusion(:type, ["Update"])
diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex
@@ -40,23 +40,34 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
end
end
- def do_common_pipeline(object, meta) do
- with {_, {:ok, validated_object, meta}} <-
- {:validate_object, @object_validator.validate(object, meta)},
- {_, {:ok, mrfd_object, meta}} <-
- {:mrf_object, @mrf.pipeline_filter(validated_object, meta)},
- {_, {:ok, activity, meta}} <-
- {:persist_object, @activity_pub.persist(mrfd_object, meta)},
- {_, {:ok, activity, meta}} <-
- {:execute_side_effects, @side_effects.handle(activity, meta)},
- {_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do
- {:ok, activity, meta}
+ def do_common_pipeline(%{__struct__: _}, _meta), do: {:error, :is_struct}
+
+ def do_common_pipeline(message, meta) do
+ with {_, {:ok, message, meta}} <- {:validate, @object_validator.validate(message, meta)},
+ {_, {:ok, message, meta}} <- {:fixup, validation_fixups(message, meta)},
+ {_, {:ok, message, meta}} <- {:mrf, @mrf.pipeline_filter(message, meta)},
+ {_, {:ok, message, meta}} <- {:persist, @activity_pub.persist(message, meta)},
+ {_, {:ok, message, meta}} <- {:side_effects, @side_effects.handle(message, meta)},
+ {_, {:ok, _}} <- {:federation, maybe_federate(message, meta)} do
+ {:ok, message, meta}
else
- {:mrf_object, {:reject, message, _}} -> {:reject, message}
+ {:mrf, {:reject, message, _}} -> {:reject, message}
e -> {:error, e}
end
end
+ defp validation_fixups(message, meta) do
+ message =
+ if message["tag"] do
+ tag = (message["tag"] || []) ++ Object.hashtags(%Object{data: message})
+ Map.put(message, "tag", tag)
+ else
+ message
+ end
+
+ {:ok, message, meta}
+ end
+
defp maybe_federate(%Object{}, _), do: {:ok, :not_federated}
defp maybe_federate(%Activity{} = activity, meta) do
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -203,6 +203,19 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
Object.increase_replies_count(in_reply_to)
end
+ reply_depth = (meta[:depth] || 0) + 1
+
+ # FIXME: Force inReplyTo to replies
+ if Pleroma.Web.Federator.allowed_thread_distance?(reply_depth) and
+ object.data["replies"] != nil do
+ for reply_id <- object.data["replies"] do
+ Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{
+ "id" => reply_id,
+ "depth" => reply_depth
+ })
+ end
+ end
+
ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn ->
Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
end)
@@ -366,7 +379,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
end
def handle_object_creation(%{"type" => objtype} = object, meta)
- when objtype in ~w[Audio Video Question Event Article] do
+ when objtype in ~w[Audio Video Question Event Article Note] do
with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
{:ok, object, meta}
end
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -32,19 +32,18 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
"""
def fix_object(object, options \\ []) do
object
- |> strip_internal_fields
- |> fix_actor
- |> fix_url
- |> fix_attachments
- |> fix_context
+ |> strip_internal_fields()
+ |> fix_actor()
+ |> fix_url()
+ |> fix_attachments()
+ |> fix_context()
|> fix_in_reply_to(options)
- |> fix_emoji
- |> fix_tag
- |> set_sensitive
- |> fix_content_map
- |> fix_addressing
- |> fix_summary
- |> fix_type(options)
+ |> fix_emoji()
+ |> fix_tag()
+ |> set_sensitive()
+ |> fix_content_map()
+ |> fix_addressing()
+ |> fix_summary()
end
def fix_summary(%{"summary" => nil} = object) do
@@ -73,17 +72,21 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
end
end
- def fix_explicit_addressing(
- %{"to" => to, "cc" => cc} = object,
- explicit_mentions,
- follower_collection
- ) do
- explicit_to = Enum.filter(to, fn x -> x in explicit_mentions end)
+ # if directMessage flag is set to true, leave the addressing alone
+ def fix_explicit_addressing(%{"directMessage" => true} = object, _follower_collection),
+ do: object
+
+ def fix_explicit_addressing(%{"to" => to, "cc" => cc} = object, follower_collection) do
+ explicit_mentions =
+ Utils.determine_explicit_mentions(object) ++
+ [Pleroma.Constants.as_public(), follower_collection]
+ explicit_to = Enum.filter(to, fn x -> x in explicit_mentions end)
explicit_cc = Enum.filter(to, fn x -> x not in explicit_mentions end)
final_cc =
(cc ++ explicit_cc)
+ |> Enum.filter(& &1)
|> Enum.reject(fn x -> String.ends_with?(x, "/followers") and x != follower_collection end)
|> Enum.uniq()
@@ -92,29 +95,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
|> Map.put("cc", final_cc)
end
- def fix_explicit_addressing(object, _explicit_mentions, _followers_collection), do: object
-
- # if directMessage flag is set to true, leave the addressing alone
- def fix_explicit_addressing(%{"directMessage" => true} = object), do: object
-
- def fix_explicit_addressing(object) do
- explicit_mentions = Utils.determine_explicit_mentions(object)
-
- %User{follower_address: follower_collection} =
- object
- |> Containment.get_actor()
- |> User.get_cached_by_ap_id()
-
- explicit_mentions =
- explicit_mentions ++
- [
- Pleroma.Constants.as_public(),
- follower_collection
- ]
-
- fix_explicit_addressing(object, explicit_mentions, follower_collection)
- end
-
# if as:Public is addressed, then make sure the followers collection is also addressed
# so that the activities will be delivered to local users.
def fix_implicit_addressing(%{"to" => to, "cc" => cc} = object, followers_collection) do
@@ -138,19 +118,19 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
end
end
- def fix_implicit_addressing(object, _), do: object
-
def fix_addressing(object) do
- {:ok, %User{} = user} = User.get_or_fetch_by_ap_id(object["actor"])
- followers_collection = User.ap_followers(user)
+ {:ok, %User{follower_address: follower_collection}} =
+ object
+ |> Containment.get_actor()
+ |> User.get_or_fetch_by_ap_id()
object
|> fix_addressing_list("to")
|> fix_addressing_list("cc")
|> fix_addressing_list("bto")
|> fix_addressing_list("bcc")
- |> fix_explicit_addressing()
- |> fix_implicit_addressing(followers_collection)
+ |> fix_explicit_addressing(follower_collection)
+ |> fix_implicit_addressing(follower_collection)
end
def fix_actor(%{"attributedTo" => actor} = object) do
@@ -315,10 +295,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
tags =
tag
|> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end)
- |> Enum.map(fn %{"name" => name} ->
- name
- |> String.slice(1..-1)
- |> String.downcase()
+ |> Enum.map(fn
+ %{"name" => "#" <> hashtag} -> String.downcase(hashtag)
+ %{"name" => hashtag} -> String.downcase(hashtag)
end)
Map.put(object, "tag", tag ++ tags)
@@ -342,19 +321,18 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
def fix_content_map(object), do: object
- def fix_type(object, options \\ [])
+ defp fix_type(%{"type" => "Note", "inReplyTo" => reply_id, "name" => _} = object, options)
+ when is_binary(reply_id) do
+ options = Keyword.put(options, :fetch, true)
- def fix_type(%{"inReplyTo" => reply_id, "name" => _} = object, options)
- when is_binary(reply_id) do
- with true <- Federator.allowed_thread_distance?(options[:depth]),
- {:ok, %{data: %{"type" => "Question"} = _} = _} <- get_obj_helper(reply_id, options) do
+ with %Object{data: %{"type" => "Question"}} <- Object.normalize(reply_id, options) do
Map.put(object, "type", "Answer")
else
_ -> object
end
end
- def fix_type(object, _), do: object
+ defp fix_type(object, _options), do: object
# Reduce the object list to find the reported user.
defp get_reported(objects) do
@@ -425,10 +403,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
# - tags
# - emoji
def handle_incoming(
- %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
+ %{"type" => "Create", "object" => %{"type" => "Page"} = object} = data,
options
- )
- when objtype in ~w{Note Page} do
+ ) do
actor = Containment.get_actor(data)
with nil <- Activity.get_create_by_object_ap_id(object["id"]),
@@ -520,14 +497,23 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
def handle_incoming(
%{"type" => "Create", "object" => %{"type" => objtype, "id" => obj_id}} = data,
- _options
+ options
)
- when objtype in ~w{Question Answer ChatMessage Audio Video Event Article} do
- data = Map.put(data, "object", strip_internal_fields(data["object"]))
+ when objtype in ~w{Question Answer ChatMessage Audio Video Event Article Note} do
+ fetch_options = Keyword.put(options, :depth, (options[:depth] || 0) + 1)
+
+ object =
+ data["object"]
+ |> strip_internal_fields()
+ |> fix_type(fetch_options)
+ |> fix_in_reply_to(fetch_options)
+
+ data = Map.put(data, "object", object)
+ options = Keyword.put(options, :local, false)
with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
nil <- Activity.get_create_by_object_ap_id(obj_id),
- {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
+ {:ok, activity, _} <- Pipeline.common_pipeline(data, options) do
{:ok, activity}
else
%Activity{} = activity -> {:ok, activity}
@@ -566,7 +552,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
Pipeline.common_pipeline(data, local: false) do
{:ok, activity}
else
- {:error, {:validate_object, _}} = e ->
+ {:error, {:validate, _}} = e ->
# Check if we have a create activity for this
with {:ok, object_id} <- ObjectValidators.ObjectID.cast(data["object"]),
%Activity{data: %{"actor" => actor}} <-
@@ -933,12 +919,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
Map.put(object, "conversation", object["context"])
end
- def set_sensitive(%{"sensitive" => _} = object) do
+ def set_sensitive(%{"sensitive" => sensitive} = object) when is_boolean(sensitive) do
object
end
def set_sensitive(object) do
- tags = object["tag"] || []
+ tags = Object.hashtags(%Object{data: object})
Map.put(object, "sensitive", "nsfw" in tags)
end
diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex
@@ -96,8 +96,11 @@ defmodule Pleroma.Web.ActivityPub.Utils do
!label_in_collection?(ap_id, params["cc"])
if need_splice? do
- cc_list = extract_list(params["cc"])
- Map.put(params, "cc", [ap_id | cc_list])
+ cc = [ap_id | extract_list(params["cc"])]
+
+ params
+ |> Map.put("cc", cc)
+ |> Maps.safe_put_in(["object", "cc"], cc)
else
params
end
diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex
@@ -228,17 +228,7 @@ defmodule Pleroma.Web.CommonAPI do
{:find_object, _} ->
{:error, :not_found}
- {:common_pipeline,
- {
- :error,
- {
- :validate_object,
- {
- :error,
- changeset
- }
- }
- }} = e ->
+ {:common_pipeline, {:error, {:validate, {:error, changeset}}}} = e ->
if {:object, {"already liked by this actor", []}} in changeset.errors do
{:ok, :already_liked}
else
diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex
@@ -96,6 +96,11 @@ defmodule Pleroma.Web.Federator do
Logger.debug("Unhandled actor #{actor}, #{inspect(e)}")
{:error, e}
+ {:error, {:validate_object, _}} = e ->
+ Logger.error("Incoming AP doc validation error: #{inspect(e)}")
+ Logger.debug(Jason.encode!(params, pretty: true))
+ e
+
e ->
# Just drop those for now
Logger.debug(fn -> "Unhandled activity\n" <> Jason.encode!(params, pretty: true) end)
diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex
@@ -32,6 +32,7 @@ defmodule Pleroma.Web.Feed.FeedView do
%{
activity: activity,
+ object: object,
data: Map.get(object, :data),
actor: actor
}
diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
@@ -136,9 +136,9 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do
tags =
[params[:tag], params[:any]]
|> List.flatten()
- |> Enum.uniq()
|> Enum.reject(&is_nil/1)
|> Enum.map(&String.downcase/1)
+ |> Enum.uniq()
tag_all =
params
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -201,8 +201,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
like_count = object.data["like_count"] || 0
announcement_count = object.data["announcement_count"] || 0
- tags = object.data["tag"] || []
- sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw")
+ hashtags = Object.hashtags(object)
+ sensitive = object.data["sensitive"] || Enum.member?(hashtags, "nsfw")
+
+ tags = Object.tags(object)
tag_mentions =
tags
diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex
@@ -22,7 +22,7 @@
<link type="text/html" href='<%= @data["external_url"] %>' rel="alternate"/>
<% end %>
- <%= for tag <- @data["tag"] || [] do %>
+ <%= for tag <- Pleroma.Object.hashtags(@object) do %>
<category term="<%= tag %>"></category>
<% end %>
diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex
@@ -22,7 +22,7 @@
<link rel="ostatus:conversation"><%= activity_context(@activity) %></link>
- <%= for tag <- @data["tag"] || [] do %>
+ <%= for tag <- Pleroma.Object.hashtags(@object) do %>
<category term="<%= tag %>"></category>
<% end %>
diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex
@@ -41,7 +41,7 @@
<% end %>
<% end %>
- <%= for tag <- @data["tag"] || [] do %>
+ <%= for tag <- Pleroma.Object.hashtags(@object) do %>
<category term="<%= tag %>"></category>
<% end %>
diff --git a/priv/repo/migrations/20190711042021_create_safe_jsonb_set.exs b/priv/repo/migrations/20190711042021_create_safe_jsonb_set.exs
@@ -9,7 +9,7 @@ defmodule Pleroma.Repo.Migrations.CreateSafeJsonbSet do
begin
result := jsonb_set(target, path, coalesce(new_value, 'null'::jsonb), create_missing);
if result is NULL then
- raise 'jsonb_set tried to wipe the object, please report this incindent to Pleroma bug tracker. https://git.pleroma.social/pleroma/pleroma/issues/new';
+ raise 'jsonb_set tried to wipe the object, please report this incident to Pleroma bug tracker. https://git.pleroma.social/pleroma/pleroma/issues/new';
return target;
else
return result;
diff --git a/priv/repo/migrations/20201221202251_create_hashtags.exs b/priv/repo/migrations/20201221202251_create_hashtags.exs
@@ -0,0 +1,13 @@
+defmodule Pleroma.Repo.Migrations.CreateHashtags do
+ use Ecto.Migration
+
+ def change do
+ create_if_not_exists table(:hashtags) do
+ add(:name, :citext, null: false)
+
+ timestamps()
+ end
+
+ create_if_not_exists(unique_index(:hashtags, [:name]))
+ end
+end
diff --git a/priv/repo/migrations/20201221202252_remove_data_from_hashtags.exs b/priv/repo/migrations/20201221202252_remove_data_from_hashtags.exs
@@ -0,0 +1,15 @@
+defmodule Pleroma.Repo.Migrations.RemoveDataFromHashtags do
+ use Ecto.Migration
+
+ def up do
+ alter table(:hashtags) do
+ remove_if_exists(:data, :map)
+ end
+ end
+
+ def down do
+ alter table(:hashtags) do
+ add_if_not_exists(:data, :map, default: %{})
+ end
+ end
+end
diff --git a/priv/repo/migrations/20201221203824_create_hashtags_objects.exs b/priv/repo/migrations/20201221203824_create_hashtags_objects.exs
@@ -0,0 +1,13 @@
+defmodule Pleroma.Repo.Migrations.CreateHashtagsObjects do
+ use Ecto.Migration
+
+ def change do
+ create_if_not_exists table(:hashtags_objects, primary_key: false) do
+ add(:hashtag_id, references(:hashtags), null: false, primary_key: true)
+ add(:object_id, references(:objects), null: false, primary_key: true)
+ end
+
+ create_if_not_exists(unique_index(:hashtags_objects, [:hashtag_id, :object_id]))
+ create_if_not_exists(index(:hashtags_objects, [:object_id]))
+ end
+end
diff --git a/priv/repo/migrations/20210105195018_create_data_migrations.exs b/priv/repo/migrations/20210105195018_create_data_migrations.exs
@@ -0,0 +1,17 @@
+defmodule Pleroma.Repo.Migrations.CreateDataMigrations do
+ use Ecto.Migration
+
+ def change do
+ create_if_not_exists table(:data_migrations) do
+ add(:name, :string, null: false)
+ add(:state, :integer, default: 1)
+ add(:feature_lock, :boolean, default: false)
+ add(:params, :map, default: %{})
+ add(:data, :map, default: %{})
+
+ timestamps()
+ end
+
+ create_if_not_exists(unique_index(:data_migrations, [:name]))
+ end
+end
diff --git a/priv/repo/migrations/20210106183301_data_migration_create_populate_hashtags_table.exs b/priv/repo/migrations/20210106183301_data_migration_create_populate_hashtags_table.exs
@@ -0,0 +1,16 @@
+defmodule Pleroma.Repo.Migrations.DataMigrationCreatePopulateHashtagsTable do
+ use Ecto.Migration
+
+ def up do
+ dt = NaiveDateTime.utc_now()
+
+ execute(
+ "INSERT INTO data_migrations(name, inserted_at, updated_at) " <>
+ "VALUES ('populate_hashtags_table', '#{dt}', '#{dt}') ON CONFLICT DO NOTHING;"
+ )
+ end
+
+ def down do
+ execute("DELETE FROM data_migrations WHERE name = 'populate_hashtags_table';")
+ end
+end
diff --git a/priv/repo/migrations/20210111172254_create_data_migration_failed_ids.exs b/priv/repo/migrations/20210111172254_create_data_migration_failed_ids.exs
@@ -0,0 +1,14 @@
+defmodule Pleroma.Repo.Migrations.CreateDataMigrationFailedIds do
+ use Ecto.Migration
+
+ def change do
+ create_if_not_exists table(:data_migration_failed_ids, primary_key: false) do
+ add(:data_migration_id, references(:data_migrations), null: false, primary_key: true)
+ add(:record_id, :bigint, null: false, primary_key: true)
+ end
+
+ create_if_not_exists(
+ unique_index(:data_migration_failed_ids, [:data_migration_id, :record_id])
+ )
+ end
+end
diff --git a/test/fixtures/activitypub-client-post-activity.json b/test/fixtures/activitypub-client-post-activity.json
@@ -3,6 +3,7 @@
"type": "Create",
"object": {
"type": "Note",
+ "to": ["https://www.w3.org/ns/activitystreams#Public"],
"content": "It's a note"
},
"to": ["https://www.w3.org/ns/activitystreams#Public"]
diff --git a/test/pleroma/activity/ir/topics_test.exs b/test/pleroma/activity/ir/topics_test.exs
@@ -11,6 +11,8 @@ defmodule Pleroma.Activity.Ir.TopicsTest do
require Pleroma.Constants
+ import Mock
+
describe "poll answer" do
test "produce no topics" do
activity = %Activity{object: %Object{data: %{"type" => "Answer"}}}
@@ -77,14 +79,13 @@ defmodule Pleroma.Activity.Ir.TopicsTest do
refute Enum.member?(topics, "public:local:media")
end
- test "converts tags to hash tags", %{activity: %{object: %{data: data} = object} = activity} do
- tagged_data = Map.put(data, "tag", ["foo", "bar"])
- activity = %{activity | object: %{object | data: tagged_data}}
-
- topics = Topics.get_activity_topics(activity)
+ test "converts tags to hash tags", %{activity: activity} do
+ with_mock(Object, [:passthrough], hashtags: fn _ -> ["foo", "bar"] end) do
+ topics = Topics.get_activity_topics(activity)
- assert Enum.member?(topics, "hashtag:foo")
- assert Enum.member?(topics, "hashtag:bar")
+ assert Enum.member?(topics, "hashtag:foo")
+ assert Enum.member?(topics, "hashtag:bar")
+ end
end
test "only converts strings to hash tags", %{
diff --git a/test/pleroma/activity_test.exs b/test/pleroma/activity_test.exs
@@ -123,7 +123,8 @@ defmodule Pleroma.ActivityTest do
"type" => "Note",
"content" => "find me!",
"id" => "http://mastodon.example.org/users/admin/objects/1",
- "attributedTo" => "http://mastodon.example.org/users/admin"
+ "attributedTo" => "http://mastodon.example.org/users/admin",
+ "to" => ["https://www.w3.org/ns/activitystreams#Public"]
},
"to" => ["https://www.w3.org/ns/activitystreams#Public"]
}
@@ -132,6 +133,7 @@ defmodule Pleroma.ActivityTest do
{:ok, japanese_activity} = Pleroma.Web.CommonAPI.post(user, %{status: "更新情報"})
{:ok, job} = Pleroma.Web.Federator.incoming_ap_doc(params)
{:ok, remote_activity} = ObanHelpers.perform(job)
+ remote_activity = Activity.get_by_id_with_object(remote_activity.id)
%{
japanese_activity: japanese_activity,
diff --git a/test/pleroma/ecto_type/activity_pub/object_validators/recipients_test.exs b/test/pleroma/ecto_type/activity_pub/object_validators/recipients_test.exs
@@ -6,10 +6,10 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.RecipientsTest do
alias Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients
use Pleroma.DataCase, async: true
- test "it asserts that all elements of the list are object ids" do
+ test "it only keeps elements that are valid object ids" do
list = ["https://lain.com/users/lain", "invalid"]
- assert :error == Recipients.cast(list)
+ assert {:ok, ["https://lain.com/users/lain"]} == Recipients.cast(list)
end
test "it works with a list" do
diff --git a/test/pleroma/hashtag_test.exs b/test/pleroma/hashtag_test.exs
@@ -0,0 +1,17 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HashtagTest do
+ use Pleroma.DataCase
+
+ alias Pleroma.Hashtag
+
+ describe "changeset validations" do
+ test "ensure non-blank :name" do
+ changeset = Hashtag.changeset(%Hashtag{}, %{name: ""})
+
+ assert {:name, {"can't be blank", [validation: :required]}} in changeset.errors
+ end
+ end
+end
diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs
@@ -624,6 +624,8 @@ defmodule Pleroma.NotificationTest do
"actor" => user.ap_id,
"object" => %{
"type" => "Note",
+ "id" => Pleroma.Web.ActivityPub.Utils.generate_object_id(),
+ "to" => ["https://www.w3.org/ns/activitystreams#Public"],
"content" => "message with a Mention tag, but no explicit tagging",
"tag" => [
%{
@@ -655,6 +657,9 @@ defmodule Pleroma.NotificationTest do
"actor" => user.ap_id,
"object" => %{
"type" => "Note",
+ "id" => Pleroma.Web.ActivityPub.Utils.generate_object_id(),
+ "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+ "cc" => [other_user.ap_id],
"content" => "hi everyone",
"attributedTo" => user.ap_id
}
@@ -951,6 +956,7 @@ defmodule Pleroma.NotificationTest do
"cc" => [],
"object" => %{
"type" => "Note",
+ "id" => remote_user.ap_id <> "/objects/test",
"content" => "Hello!",
"tag" => [
%{
diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs
@@ -66,6 +66,14 @@ defmodule Pleroma.Object.FetcherTest do
%Tesla.Env{
status: 500
}
+
+ %{
+ method: :get,
+ url: "https://stereophonic.space/objects/02997b83-3ea7-4b63-94af-ef3aa2d4ed17"
+ } ->
+ %Tesla.Env{
+ status: 500
+ }
end)
:ok
@@ -125,7 +133,6 @@ defmodule Pleroma.Object.FetcherTest do
Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
assert activity = Activity.get_create_by_object_ap_id(object.data["id"])
- assert activity.data["id"]
{:ok, object_again} =
Fetcher.fetch_object_from_id("http://mastodon.example.org/@admin/99541947525187367")
diff --git a/test/pleroma/object_test.exs b/test/pleroma/object_test.exs
@@ -5,10 +5,13 @@
defmodule Pleroma.ObjectTest do
use Pleroma.DataCase
use Oban.Testing, repo: Pleroma.Repo
+
import ExUnit.CaptureLog
import Pleroma.Factory
import Tesla.Mock
+
alias Pleroma.Activity
+ alias Pleroma.Hashtag
alias Pleroma.Object
alias Pleroma.Repo
alias Pleroma.Tests.ObanHelpers
@@ -417,4 +420,28 @@ defmodule Pleroma.ObjectTest do
assert updated_object.data["like_count"] == 1
end
end
+
+ describe ":hashtags association" do
+ test "Hashtag records are created with Object record and updated on its change" do
+ user = insert(:user)
+
+ {:ok, %{object: object}} =
+ CommonAPI.post(user, %{status: "some text #hashtag1 #hashtag2 ..."})
+
+ assert [%Hashtag{name: "hashtag1"}, %Hashtag{name: "hashtag2"}] =
+ Enum.sort_by(object.hashtags, & &1.name)
+
+ {:ok, object} = Object.update_data(object, %{"tag" => []})
+
+ assert [] = object.hashtags
+
+ object = Object.get_by_id(object.id) |> Repo.preload(:hashtags)
+ assert [] = object.hashtags
+
+ {:ok, object} = Object.update_data(object, %{"tag" => ["abc", "def"]})
+
+ assert [%Hashtag{name: "abc"}, %Hashtag{name: "def"}] =
+ Enum.sort_by(object.hashtags, & &1.name)
+ end
+ end
end
diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs
@@ -539,7 +539,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
File.read!("test/fixtures/mastodon-post-activity.json")
|> Jason.decode!()
|> Map.put("actor", user.ap_id)
- |> put_in(["object", "attridbutedTo"], user.ap_id)
+ |> put_in(["object", "attributedTo"], user.ap_id)
conn =
conn
@@ -649,7 +649,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
test "it inserts an incoming activity into the database", %{conn: conn, data: data} do
user = insert(:user)
- data = Map.put(data, "bcc", [user.ap_id])
+
+ data =
+ data
+ |> Map.put("bcc", [user.ap_id])
+ |> Kernel.put_in(["object", "bcc"], [user.ap_id])
conn =
conn
@@ -666,8 +670,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
user = insert(:user)
data =
- Map.put(data, "to", user.ap_id)
- |> Map.delete("cc")
+ data
+ |> Map.put("to", user.ap_id)
+ |> Map.put("cc", [])
+ |> Kernel.put_in(["object", "to"], user.ap_id)
+ |> Kernel.put_in(["object", "cc"], [])
conn =
conn
@@ -684,8 +691,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
user = insert(:user)
data =
- Map.put(data, "cc", user.ap_id)
- |> Map.delete("to")
+ data
+ |> Map.put("to", [])
+ |> Map.put("cc", user.ap_id)
+ |> Kernel.put_in(["object", "to"], [])
+ |> Kernel.put_in(["object", "cc"], user.ap_id)
conn =
conn
@@ -703,9 +713,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
user = insert(:user)
data =
- Map.put(data, "bcc", user.ap_id)
- |> Map.delete("to")
- |> Map.delete("cc")
+ data
+ |> Map.put("to", [])
+ |> Map.put("cc", [])
+ |> Map.put("bcc", user.ap_id)
+ |> Kernel.put_in(["object", "to"], [])
+ |> Kernel.put_in(["object", "cc"], [])
+ |> Kernel.put_in(["object", "bcc"], user.ap_id)
conn =
conn
@@ -820,29 +834,34 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
assert Instances.reachable?(sender_host)
end
+ @tag capture_log: true
test "it removes all follower collections but actor's", %{conn: conn} do
[actor, recipient] = insert_pair(:user)
- data =
- File.read!("test/fixtures/activitypub-client-post-activity.json")
- |> Jason.decode!()
+ to = [
+ recipient.ap_id,
+ recipient.follower_address,
+ "https://www.w3.org/ns/activitystreams#Public"
+ ]
- object = Map.put(data["object"], "attributedTo", actor.ap_id)
+ cc = [recipient.follower_address, actor.follower_address]
- data =
- data
- |> Map.put("id", Utils.generate_object_id())
- |> Map.put("actor", actor.ap_id)
- |> Map.put("object", object)
- |> Map.put("cc", [
- recipient.follower_address,
- actor.follower_address
- ])
- |> Map.put("to", [
- recipient.ap_id,
- recipient.follower_address,
- "https://www.w3.org/ns/activitystreams#Public"
- ])
+ data = %{
+ "@context" => ["https://www.w3.org/ns/activitystreams"],
+ "type" => "Create",
+ "id" => Utils.generate_activity_id(),
+ "to" => to,
+ "cc" => cc,
+ "actor" => actor.ap_id,
+ "object" => %{
+ "type" => "Note",
+ "to" => to,
+ "cc" => cc,
+ "content" => "It's a note",
+ "attributedTo" => actor.ap_id,
+ "id" => Utils.generate_object_id()
+ }
+ }
conn
|> assign(:valid_signature, true)
@@ -852,7 +871,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
ObanHelpers.perform(all_enqueued(worker: ReceiverWorker))
- activity = Activity.get_by_ap_id(data["id"])
+ assert activity = Activity.get_by_ap_id(data["id"])
assert activity.id
assert actor.follower_address in activity.recipients
@@ -984,7 +1003,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
"actor" => remote_actor,
"content" => "test report",
"id" => "https://#{remote_domain}/e3b12fd1-948c-446e-b93b-a5e67edbe1d8",
- "nickname" => reported_user.nickname,
"object" => [
reported_user.ap_id,
note.data["object"]
@@ -1136,9 +1154,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
activity: %{
"@context" => "https://www.w3.org/ns/activitystreams",
"type" => "Create",
- "object" => %{"type" => "Note", "content" => "AP C2S test"},
- "to" => "https://www.w3.org/ns/activitystreams#Public",
- "cc" => []
+ "object" => %{
+ "type" => "Note",
+ "content" => "AP C2S test",
+ "to" => "https://www.w3.org/ns/activitystreams#Public",
+ "cc" => []
+ }
}
]
end
@@ -1244,19 +1265,19 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
user = User.get_cached_by_ap_id(note_activity.data["actor"])
data = %{
- type: "Delete",
- object: %{
- id: note_object.data["id"]
+ "type" => "Delete",
+ "object" => %{
+ "id" => note_object.data["id"]
}
}
- conn =
+ result =
conn
|> assign(:user, user)
|> put_req_header("content-type", "application/activity+json")
|> post("/users/#{user.nickname}/outbox", data)
+ |> json_response(201)
- result = json_response(conn, 201)
assert Activity.get_by_ap_id(result["id"])
assert object = Object.get_by_ap_id(note_object.data["id"])
@@ -1281,7 +1302,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
|> put_req_header("content-type", "application/activity+json")
|> post("/users/#{user.nickname}/outbox", data)
- assert json_response(conn, 400)
+ assert json_response(conn, 403)
end
test "it increases like count when receiving a like action", %{conn: conn} do
@@ -1359,7 +1380,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
|> post("/users/#{user.nickname}/outbox", activity)
|> json_response(400)
- assert result == "Note is over the character limit"
+ assert result == "Character limit (5 characters) exceeded, contains 11 characters"
end
end
@@ -1736,10 +1757,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do
"object" => %{
"type" => "Note",
"content" => "AP C2S test, attachment",
- "attachment" => [object]
- },
- "to" => "https://www.w3.org/ns/activitystreams#Public",
- "cc" => []
+ "attachment" => [object],
+ "to" => "https://www.w3.org/ns/activitystreams#Public",
+ "cc" => []
+ }
}
activity_response =
diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs
@@ -213,32 +213,64 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do
test "it fetches the appropriate tag-restricted posts" do
user = insert(:user)
- {:ok, status_one} = CommonAPI.post(user, %{status: ". #test"})
+ {:ok, status_one} = CommonAPI.post(user, %{status: ". #TEST"})
{:ok, status_two} = CommonAPI.post(user, %{status: ". #essais"})
- {:ok, status_three} = CommonAPI.post(user, %{status: ". #test #reject"})
+ {:ok, status_three} = CommonAPI.post(user, %{status: ". #test #Reject"})
- fetch_one = ActivityPub.fetch_activities([], %{type: "Create", tag: "test"})
+ {:ok, status_four} = CommonAPI.post(user, %{status: ". #Any1 #any2"})
+ {:ok, status_five} = CommonAPI.post(user, %{status: ". #Any2 #any1"})
- fetch_two = ActivityPub.fetch_activities([], %{type: "Create", tag: ["test", "essais"]})
+ for hashtag_timeline_strategy <- [true, false] do
+ clear_config([:database, :improved_hashtag_timeline], hashtag_timeline_strategy)
- fetch_three =
- ActivityPub.fetch_activities([], %{
- type: "Create",
- tag: ["test", "essais"],
- tag_reject: ["reject"]
- })
+ fetch_one = ActivityPub.fetch_activities([], %{type: "Create", tag: "test"})
- fetch_four =
- ActivityPub.fetch_activities([], %{
- type: "Create",
- tag: ["test"],
- tag_all: ["test", "reject"]
- })
+ fetch_two = ActivityPub.fetch_activities([], %{type: "Create", tag: ["TEST", "essais"]})
+
+ fetch_three =
+ ActivityPub.fetch_activities([], %{
+ type: "Create",
+ tag: ["test", "Essais"],
+ tag_reject: ["reject"]
+ })
+
+ fetch_four =
+ ActivityPub.fetch_activities([], %{
+ type: "Create",
+ tag: ["test"],
+ tag_all: ["test", "REJECT"]
+ })
- assert fetch_one == [status_one, status_three]
- assert fetch_two == [status_one, status_two, status_three]
- assert fetch_three == [status_one, status_two]
- assert fetch_four == [status_three]
+ # Testing that deduplication (if needed) is done on DB (not Ecto) level; :limit is important
+ fetch_five =
+ ActivityPub.fetch_activities([], %{
+ type: "Create",
+ tag: ["ANY1", "any2"],
+ limit: 2
+ })
+
+ fetch_six =
+ ActivityPub.fetch_activities([], %{
+ type: "Create",
+ tag: ["any1", "Any2"],
+ tag_all: [],
+ tag_reject: []
+ })
+
+ # Regression test: passing empty lists as filter options shouldn't affect the results
+ assert fetch_five == fetch_six
+
+ [fetch_one, fetch_two, fetch_three, fetch_four, fetch_five] =
+ Enum.map([fetch_one, fetch_two, fetch_three, fetch_four, fetch_five], fn statuses ->
+ Enum.map(statuses, fn s -> Repo.preload(s, object: :hashtags) end)
+ end)
+
+ assert fetch_one == [status_one, status_three]
+ assert fetch_two == [status_one, status_two, status_three]
+ assert fetch_three == [status_one, status_two]
+ assert fetch_four == [status_three]
+ assert fetch_five == [status_four, status_five]
+ end
end
describe "insertion" do
diff --git a/test/pleroma/web/activity_pub/pipeline_test.exs b/test/pleroma/web/activity_pub/pipeline_test.exs
@@ -25,9 +25,6 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
MRFMock
|> expect(:pipeline_filter, fn o, m -> {:ok, o, m} end)
- ActivityPubMock
- |> expect(:persist, fn o, m -> {:ok, o, m} end)
-
SideEffectsMock
|> expect(:handle, fn o, m -> {:ok, o, m} end)
|> expect(:handle_after_transaction, fn m -> m end)
@@ -42,6 +39,9 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
activity_with_object = %{activity | data: Map.put(activity.data, "object", object)}
+ ActivityPubMock
+ |> expect(:persist, fn _, m -> {:ok, activity, m} end)
+
FederatorMock
|> expect(:publish, fn ^activity_with_object -> :ok end)
@@ -50,7 +50,7 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
assert {:ok, ^activity, ^meta} =
Pleroma.Web.ActivityPub.Pipeline.common_pipeline(
- activity,
+ activity.data,
meta
)
end
@@ -59,6 +59,9 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
activity = insert(:note_activity)
meta = [local: true]
+ ActivityPubMock
+ |> expect(:persist, fn _, m -> {:ok, activity, m} end)
+
FederatorMock
|> expect(:publish, fn ^activity -> :ok end)
@@ -66,29 +69,35 @@ defmodule Pleroma.Web.ActivityPub.PipelineTest do
|> expect(:get, fn [:instance, :federating] -> true end)
assert {:ok, ^activity, ^meta} =
- Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
+ Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity.data, meta)
end
test "it goes through validation, filtering, persisting, side effects without federation for remote activities" do
activity = insert(:note_activity)
meta = [local: false]
+ ActivityPubMock
+ |> expect(:persist, fn _, m -> {:ok, activity, m} end)
+
ConfigMock
|> expect(:get, fn [:instance, :federating] -> true end)
assert {:ok, ^activity, ^meta} =
- Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
+ Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity.data, meta)
end
test "it goes through validation, filtering, persisting, side effects without federation for local activities if federation is deactivated" do
activity = insert(:note_activity)
meta = [local: true]
+ ActivityPubMock
+ |> expect(:persist, fn _, m -> {:ok, activity, m} end)
+
ConfigMock
|> expect(:get, fn [:instance, :federating] -> false end)
assert {:ok, ^activity, ^meta} =
- Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity, meta)
+ Pleroma.Web.ActivityPub.Pipeline.common_pipeline(activity.data, meta)
end
end
end
diff --git a/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/audio_handling_test.exs
@@ -24,6 +24,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.AudioHandlingTest do
"actor" => "http://mastodon.example.org/users/admin",
"object" => %{
"type" => "Audio",
+ "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+ "cc" => [],
"id" => "http://mastodon.example.org/users/admin/listens/1234",
"attributedTo" => "http://mastodon.example.org/users/admin",
"title" => "lain radio episode 1",
@@ -61,7 +63,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.AudioHandlingTest do
assert object.data["to"] == ["https://www.w3.org/ns/activitystreams#Public"]
- assert object.data["cc"] == []
+ assert object.data["cc"] == [
+ "https://channels.tests.funkwhale.audio/federation/actors/compositions/followers"
+ ]
assert object.data["url"] == "https://channels.tests.funkwhale.audio/library/tracks/74"
diff --git a/test/pleroma/web/activity_pub/transmogrifier/event_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/event_handling_test.exs
@@ -31,7 +31,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.EventHandlingTest do
)
assert object.data["to"] == ["https://www.w3.org/ns/activitystreams#Public"]
- assert object.data["cc"] == []
+ assert object.data["cc"] == ["https://mobilizon.org/@tcit/followers"]
assert object.data["url"] ==
"https://mobilizon.org/events/252d5816-00a3-4a89-a66f-15bf65c33e39"
diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs
@@ -10,11 +10,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
alias Pleroma.Object
alias Pleroma.User
alias Pleroma.Web.ActivityPub.Transmogrifier
+ alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.CommonAPI
import Mock
import Pleroma.Factory
- import ExUnit.CaptureLog
setup_all do
Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
@@ -39,37 +39,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
object = Object.normalize(data["object"], fetch: false)
- assert "test" in object.data["tag"]
- end
-
- test "it cleans up incoming notices which are not really DMs" do
- user = insert(:user)
- other_user = insert(:user)
-
- to = [user.ap_id, other_user.ap_id]
-
- data =
- File.read!("test/fixtures/mastodon-post-activity.json")
- |> Jason.decode!()
- |> Map.put("to", to)
- |> Map.put("cc", [])
-
- object =
- data["object"]
- |> Map.put("to", to)
- |> Map.put("cc", [])
-
- data = Map.put(data, "object", object)
-
- {:ok, %Activity{data: data, local: false} = activity} = Transmogrifier.handle_incoming(data)
-
- assert data["to"] == []
- assert data["cc"] == to
-
- object_data = Object.normalize(activity, fetch: false).data
-
- assert object_data["to"] == []
- assert object_data["cc"] == to
+ assert "test" in Object.tags(object)
+ assert Object.hashtags(object) == ["test"]
end
test "it ignores an incoming notice if we already have it" do
@@ -146,9 +117,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
data
|> Map.put("object", object)
- assert capture_log(fn ->
- {:ok, _returned_activity} = Transmogrifier.handle_incoming(data)
- end) =~ "[warn] Couldn't fetch \"https://404.site/whatever\", error: nil"
+ assert {:ok, _returned_activity} = Transmogrifier.handle_incoming(data)
end
test "it does not work for deactivated users" do
@@ -173,8 +142,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
assert data["to"] == ["https://www.w3.org/ns/activitystreams#Public"]
assert data["cc"] == [
- "http://mastodon.example.org/users/admin/followers",
- "http://localtesting.pleroma.lol/users/lain"
+ "http://localtesting.pleroma.lol/users/lain",
+ "http://mastodon.example.org/users/admin/followers"
]
assert data["actor"] == "http://mastodon.example.org/users/admin"
@@ -187,8 +156,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
assert object_data["to"] == ["https://www.w3.org/ns/activitystreams#Public"]
assert object_data["cc"] == [
- "http://mastodon.example.org/users/admin/followers",
- "http://localtesting.pleroma.lol/users/lain"
+ "http://localtesting.pleroma.lol/users/lain",
+ "http://mastodon.example.org/users/admin/followers"
]
assert object_data["actor"] == "http://mastodon.example.org/users/admin"
@@ -220,7 +189,25 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
object = Object.normalize(data["object"], fetch: false)
- assert Enum.at(object.data["tag"], 2) == "moo"
+ assert match?(
+ %{
+ "href" => "http://localtesting.pleroma.lol/users/lain",
+ "name" => "@lain@localtesting.pleroma.lol",
+ "type" => "Mention"
+ },
+ Enum.at(object.data["tag"], 0)
+ )
+
+ assert match?(
+ %{
+ "href" => "http://mastodon.example.org/tags/moo",
+ "name" => "#moo",
+ "type" => "Hashtag"
+ },
+ Enum.at(object.data["tag"], 1)
+ )
+
+ assert "moo" == Enum.at(object.data["tag"], 2)
end
test "it works for incoming notices with contentMap" do
@@ -274,13 +261,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
File.read!("test/fixtures/mastodon-post-activity.json")
|> Jason.decode!()
|> Map.put("actor", user.ap_id)
- |> Map.put("to", nil)
|> Map.put("cc", nil)
object =
data["object"]
|> Map.put("attributedTo", user.ap_id)
- |> Map.put("to", nil)
|> Map.put("cc", nil)
|> Map.put("id", user.ap_id <> "/activities/12345678")
@@ -288,8 +273,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
{:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(data)
- assert !is_nil(data["to"])
- assert !is_nil(data["cc"])
+ refute is_nil(data["cc"])
end
test "it strips internal likes" do
@@ -308,9 +292,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
object = Map.put(data["object"], "likes", likes)
data = Map.put(data, "object", object)
- {:ok, %Activity{object: object}} = Transmogrifier.handle_incoming(data)
+ {:ok, %Activity{} = activity} = Transmogrifier.handle_incoming(data)
+
+ object = Object.normalize(activity)
- refute Map.has_key?(object.data, "likes")
+ assert object.data["likes"] == []
end
test "it strips internal reactions" do
@@ -328,70 +314,46 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
end
test "it correctly processes messages with non-array to field" do
- user = insert(:user)
+ data =
+ File.read!("test/fixtures/mastodon-post-activity.json")
+ |> Poison.decode!()
+ |> Map.put("to", "https://www.w3.org/ns/activitystreams#Public")
+ |> put_in(["object", "to"], "https://www.w3.org/ns/activitystreams#Public")
- message = %{
- "@context" => "https://www.w3.org/ns/activitystreams",
- "to" => "https://www.w3.org/ns/activitystreams#Public",
- "type" => "Create",
- "object" => %{
- "content" => "blah blah blah",
- "type" => "Note",
- "attributedTo" => user.ap_id,
- "inReplyTo" => nil
- },
- "actor" => user.ap_id
- }
+ assert {:ok, activity} = Transmogrifier.handle_incoming(data)
- assert {:ok, activity} = Transmogrifier.handle_incoming(message)
+ assert [
+ "http://localtesting.pleroma.lol/users/lain",
+ "http://mastodon.example.org/users/admin/followers"
+ ] == activity.data["cc"]
assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["to"]
end
test "it correctly processes messages with non-array cc field" do
- user = insert(:user)
-
- message = %{
- "@context" => "https://www.w3.org/ns/activitystreams",
- "to" => user.follower_address,
- "cc" => "https://www.w3.org/ns/activitystreams#Public",
- "type" => "Create",
- "object" => %{
- "content" => "blah blah blah",
- "type" => "Note",
- "attributedTo" => user.ap_id,
- "inReplyTo" => nil
- },
- "actor" => user.ap_id
- }
+ data =
+ File.read!("test/fixtures/mastodon-post-activity.json")
+ |> Poison.decode!()
+ |> Map.put("cc", "http://mastodon.example.org/users/admin/followers")
+ |> put_in(["object", "cc"], "http://mastodon.example.org/users/admin/followers")
- assert {:ok, activity} = Transmogrifier.handle_incoming(message)
+ assert {:ok, activity} = Transmogrifier.handle_incoming(data)
- assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["cc"]
- assert [user.follower_address] == activity.data["to"]
+ assert ["http://mastodon.example.org/users/admin/followers"] == activity.data["cc"]
+ assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["to"]
end
test "it correctly processes messages with weirdness in address fields" do
- user = insert(:user)
-
- message = %{
- "@context" => "https://www.w3.org/ns/activitystreams",
- "to" => [nil, user.follower_address],
- "cc" => ["https://www.w3.org/ns/activitystreams#Public", ["¿"]],
- "type" => "Create",
- "object" => %{
- "content" => "…",
- "type" => "Note",
- "attributedTo" => user.ap_id,
- "inReplyTo" => nil
- },
- "actor" => user.ap_id
- }
+ data =
+ File.read!("test/fixtures/mastodon-post-activity.json")
+ |> Poison.decode!()
+ |> Map.put("cc", ["http://mastodon.example.org/users/admin/followers", ["¿"]])
+ |> put_in(["object", "cc"], ["http://mastodon.example.org/users/admin/followers", ["¿"]])
- assert {:ok, activity} = Transmogrifier.handle_incoming(message)
+ assert {:ok, activity} = Transmogrifier.handle_incoming(data)
- assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["cc"]
- assert [user.follower_address] == activity.data["to"]
+ assert ["http://mastodon.example.org/users/admin/followers"] == activity.data["cc"]
+ assert ["https://www.w3.org/ns/activitystreams#Public"] == activity.data["to"]
end
end
@@ -417,7 +379,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
} do
clear_config([:instance, :federation_incoming_replies_max_depth], 10)
- {:ok, _activity} = Transmogrifier.handle_incoming(data)
+ {:ok, activity} = Transmogrifier.handle_incoming(data)
+
+ object = Object.normalize(activity.data["object"])
+
+ assert object.data["replies"] == items
for id <- items do
job_args = %{"op" => "fetch_remote", "id" => id, "depth" => 1}
@@ -440,45 +406,38 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
setup do: clear_config([:instance, :federation_incoming_replies_max_depth])
setup do
- user = insert(:user)
-
- {:ok, activity} = CommonAPI.post(user, %{status: "post1"})
-
- {:ok, reply1} =
- CommonAPI.post(user, %{status: "reply1", in_reply_to_status_id: activity.id})
-
- {:ok, reply2} =
- CommonAPI.post(user, %{status: "reply2", in_reply_to_status_id: activity.id})
-
- replies_uris = Enum.map([reply1, reply2], fn a -> a.object.data["id"] end)
-
- {:ok, federation_output} = Transmogrifier.prepare_outgoing(activity.data)
+ replies = %{
+ "type" => "Collection",
+ "items" => [Utils.generate_object_id(), Utils.generate_object_id()]
+ }
- Repo.delete(activity.object)
- Repo.delete(activity)
+ activity =
+ File.read!("test/fixtures/mastodon-post-activity.json")
+ |> Poison.decode!()
+ |> Kernel.put_in(["object", "replies"], replies)
- %{federation_output: federation_output, replies_uris: replies_uris}
+ %{activity: activity}
end
test "schedules background fetching of `replies` items if max thread depth limit allows", %{
- federation_output: federation_output,
- replies_uris: replies_uris
+ activity: activity
} do
clear_config([:instance, :federation_incoming_replies_max_depth], 1)
- {:ok, _activity} = Transmogrifier.handle_incoming(federation_output)
+ assert {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(activity)
+ object = Object.normalize(data["object"])
- for id <- replies_uris do
+ for id <- object.data["replies"] do
job_args = %{"op" => "fetch_remote", "id" => id, "depth" => 1}
assert_enqueued(worker: Pleroma.Workers.RemoteFetcherWorker, args: job_args)
end
end
test "does NOT schedule background fetching of `replies` beyond max thread depth limit allows",
- %{federation_output: federation_output} do
+ %{activity: activity} do
clear_config([:instance, :federation_incoming_replies_max_depth], 0)
- {:ok, _activity} = Transmogrifier.handle_incoming(federation_output)
+ {:ok, _activity} = Transmogrifier.handle_incoming(activity)
assert all_enqueued(worker: Pleroma.Workers.RemoteFetcherWorker) == []
end
@@ -496,6 +455,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
"object" => %{
"to" => ["https://www.w3.org/ns/activitystreams#Public"],
"cc" => [],
+ "id" => Utils.generate_object_id(),
"type" => "Note",
"content" => "Hi",
"inReplyTo" => nil,
@@ -520,6 +480,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
"object" => %{
"to" => ["https://www.w3.org/ns/activitystreams#Public"],
"cc" => [],
+ "id" => Utils.generate_object_id(),
"type" => "Note",
"content" => "Hi",
"inReplyTo" => nil,
diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs
@@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
alias Pleroma.Tests.ObanHelpers
alias Pleroma.User
alias Pleroma.Web.ActivityPub.Transmogrifier
+ alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.AdminAPI.AccountView
alias Pleroma.Web.CommonAPI
@@ -168,8 +169,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
{:ok, activity} = CommonAPI.post(user, %{status: "hey"})
{:ok, modified} = Transmogrifier.prepare_outgoing(activity.data)
- assert modified["@context"] ==
- Pleroma.Web.ActivityPub.Utils.make_json_ld_header()["@context"]
+ assert modified["@context"] == Utils.make_json_ld_header()["@context"]
assert modified["object"]["conversation"] == modified["context"]
end
@@ -441,7 +441,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
end)
}
- fixed_object = Transmogrifier.fix_explicit_addressing(object)
+ fixed_object = Transmogrifier.fix_explicit_addressing(object, user.follower_address)
assert Enum.all?(explicitly_mentioned_actors, &(&1 in fixed_object["to"]))
refute "https://social.beepboop.ga/users/dirb" in fixed_object["to"]
assert "https://social.beepboop.ga/users/dirb" in fixed_object["cc"]
@@ -454,7 +454,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
"cc" => []
}
- fixed_object = Transmogrifier.fix_explicit_addressing(object)
+ fixed_object = Transmogrifier.fix_explicit_addressing(object, user.follower_address)
assert user.follower_address in fixed_object["to"]
refute user.follower_address in fixed_object["cc"]
end
@@ -468,7 +468,7 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
"cc" => [user.follower_address, recipient.follower_address]
}
- fixed_object = Transmogrifier.fix_explicit_addressing(object)
+ fixed_object = Transmogrifier.fix_explicit_addressing(object, user.follower_address)
assert user.follower_address in fixed_object["cc"]
refute recipient.follower_address in fixed_object["cc"]
diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs
@@ -493,7 +493,7 @@ defmodule Pleroma.Web.CommonAPITest do
object = Object.normalize(activity, fetch: false)
- assert object.data["tag"] == ["2hu"]
+ assert Object.tags(object) == ["2hu"]
end
test "it adds emoji in the object" do
diff --git a/test/pleroma/web/federator_test.exs b/test/pleroma/web/federator_test.exs
@@ -123,7 +123,8 @@ defmodule Pleroma.Web.FederatorTest do
"type" => "Note",
"content" => "hi world!",
"id" => "http://mastodon.example.org/users/admin/objects/1",
- "attributedTo" => "http://mastodon.example.org/users/admin"
+ "attributedTo" => "http://mastodon.example.org/users/admin",
+ "to" => ["https://www.w3.org/ns/activitystreams#Public"]
},
"to" => ["https://www.w3.org/ns/activitystreams#Public"]
}
@@ -145,7 +146,8 @@ defmodule Pleroma.Web.FederatorTest do
"type" => "Note",
"content" => "hi world!",
"id" => "http://mastodon.example.org/users/admin/objects/1",
- "attributedTo" => "http://mastodon.example.org/users/admin"
+ "attributedTo" => "http://mastodon.example.org/users/admin",
+ "to" => ["https://www.w3.org/ns/activitystreams#Public"]
},
"to" => ["https://www.w3.org/ns/activitystreams#Public"]
}
diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs
@@ -262,8 +262,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
mentions: [],
tags: [
%{
- name: "#{object_data["tag"]}",
- url: "http://localhost:4001/tag/#{object_data["tag"]}"
+ name: "#{hd(object_data["tag"])}",
+ url: "http://localhost:4001/tag/#{hd(object_data["tag"])}"
}
],
application: %{
diff --git a/test/pleroma/web/static_fe/static_fe_controller_test.exs b/test/pleroma/web/static_fe/static_fe_controller_test.exs
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.StaticFE.StaticFEControllerTest do
alias Pleroma.Activity
alias Pleroma.Web.ActivityPub.Transmogrifier
+ alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.CommonAPI
import Pleroma.Factory
@@ -185,16 +186,16 @@ defmodule Pleroma.Web.StaticFE.StaticFEControllerTest do
test "302 for remote cached status", %{conn: conn, user: user} do
message = %{
"@context" => "https://www.w3.org/ns/activitystreams",
- "to" => user.follower_address,
- "cc" => "https://www.w3.org/ns/activitystreams#Public",
"type" => "Create",
+ "actor" => user.ap_id,
"object" => %{
+ "to" => user.follower_address,
+ "cc" => "https://www.w3.org/ns/activitystreams#Public",
+ "id" => Utils.generate_object_id(),
"content" => "blah blah blah",
"type" => "Note",
- "attributedTo" => user.ap_id,
- "inReplyTo" => nil
- },
- "actor" => user.ap_id
+ "attributedTo" => user.ap_id
+ }
}
assert {:ok, activity} = Transmogrifier.handle_incoming(message)