commit: 2b880c8a65b8597d8cf2cdc63c819fcc3c119666
parent: 5af159f1d420963ebc041811efdd85f8e14186cf
Author: Haelwenn (lanodan) Monnier <contact@hacktivis.me>
Date: Mon, 18 May 2020 10:09:30 +0200
Merge branch 'pleroma/features/emoji_stealer'
Diffstat:
5 files changed, 171 insertions(+), 0 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Mix task to create trusted OAuth App.
- Notifications: Added `follow_request` notification type.
- Added `:reject_deletes` group to SimplePolicy
+- MRF (`EmojiStealPolicy`): New MRF Policy which allows to automatically download emojis from remote instances
<details>
<summary>API Changes</summary>
- Mastodon API: Extended `/api/v1/instance`.
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
@@ -149,6 +149,11 @@ config :pleroma, :mrf_user_allowlist,
* `:strip_followers` removes followers from the ActivityPub recipient list, ensuring they won't be delivered to home timelines
* `:reject` rejects the message entirely
+#### mrf_steal_emoji
+* `hosts`: List of hosts to steal emojis from
+* `rejected_shortcodes`: Regex-list of shortcodes to reject
+* `size_limit`: File size limit (in bytes), checked before an emoji is saved to the disk
+
### :activitypub
* `unfollow_blocked`: Whether blocks result in people getting unfollowed
* `outgoing_blocks`: Whether to federate blocks to other instances
diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex
@@ -0,0 +1,97 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do
+ require Logger
+
+ alias Pleroma.Config
+
+ @moduledoc "Detect new emojis by their shortcode and steals them"
+ @behaviour Pleroma.Web.ActivityPub.MRF
+
+ defp remote_host?(host), do: host != Config.get([Pleroma.Web.Endpoint, :url, :host])
+
+ defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], [])
+
+ defp steal_emoji({shortcode, url}) do
+ url = Pleroma.Web.MediaProxy.url(url)
+ {:ok, response} = Pleroma.HTTP.get(url)
+ size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000)
+
+ if byte_size(response.body) <= size_limit do
+ emoji_dir_path =
+ Config.get(
+ [:mrf_steal_emoji, :path],
+ Path.join(Config.get([:instance, :static_dir]), "emoji/stolen")
+ )
+
+ extension =
+ url
+ |> URI.parse()
+ |> Map.get(:path)
+ |> Path.basename()
+ |> Path.extname()
+
+ file_path = Path.join([emoji_dir_path, shortcode <> (extension || ".png")])
+
+ try do
+ :ok = File.write(file_path, response.body)
+
+ shortcode
+ rescue
+ e ->
+ Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}")
+ nil
+ end
+ else
+ Logger.debug(
+ "MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{
+ size_limit
+ } B)"
+ )
+
+ nil
+ end
+ rescue
+ e ->
+ Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}")
+ nil
+ end
+
+ @impl true
+ def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = message) do
+ host = URI.parse(actor).host
+
+ if remote_host?(host) and accept_host?(host) do
+ installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
+
+ new_emojis =
+ foreign_emojis
+ |> Enum.filter(fn {shortcode, _url} -> shortcode not in installed_emoji end)
+ |> Enum.filter(fn {shortcode, _url} ->
+ reject_emoji? =
+ Config.get([:mrf_steal_emoji, :rejected_shortcodes], [])
+ |> Enum.find(false, fn regex -> String.match?(shortcode, regex) end)
+
+ !reject_emoji?
+ end)
+ |> Enum.map(&steal_emoji(&1))
+ |> Enum.filter(& &1)
+
+ if !Enum.empty?(new_emojis) do
+ Logger.info("Stole new emojis: #{inspect(new_emojis)}")
+ Pleroma.Emoji.reload()
+ end
+ end
+
+ {:ok, message}
+ end
+
+ def filter(message), do: {:ok, message}
+
+ @impl true
+ def describe do
+ {:ok, %{}}
+ end
+end
diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex
@@ -1291,6 +1291,10 @@ defmodule HttpRequestMock do
}}
end
+ def get("https://example.org/emoji/firedfox.png", _, _, _) do
+ {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/image.jpg")}}
+ end
+
def get("https://skippers-bin.com/users/7v1w1r8ce6", _, _, _) do
{:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sjw.json")}}
end
diff --git a/test/web/activity_pub/mrf/steal_emoji_policy_test.exs b/test/web/activity_pub/mrf/steal_emoji_policy_test.exs
@@ -0,0 +1,64 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicyTest do
+ use Pleroma.DataCase
+
+ alias Pleroma.Config
+ alias Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy
+
+ setup_all do
+ Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+ :ok
+ end
+
+ setup do
+ clear_config(:mrf_steal_emoji)
+
+ emoji_path = Path.join(Config.get([:instance, :static_dir]), "emoji/stolen")
+ File.rm_rf!(emoji_path)
+ File.mkdir!(emoji_path)
+
+ Pleroma.Emoji.reload()
+ end
+
+ test "does nothing by default" do
+ installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
+ refute "firedfox" in installed_emoji
+
+ message = %{
+ "type" => "Create",
+ "object" => %{
+ "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}],
+ "actor" => "https://example.org/users/admin"
+ }
+ }
+
+ assert {:ok, message} == StealEmojiPolicy.filter(message)
+
+ installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
+ refute "firedfox" in installed_emoji
+ end
+
+ test "Steals emoji on unknown shortcode from allowed remote host" do
+ installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
+ refute "firedfox" in installed_emoji
+
+ message = %{
+ "type" => "Create",
+ "object" => %{
+ "emoji" => [{"firedfox", "https://example.org/emoji/firedfox.png"}],
+ "actor" => "https://example.org/users/admin"
+ }
+ }
+
+ Config.put([:mrf_steal_emoji, :hosts], ["example.org"])
+ Config.put([:mrf_steal_emoji, :size_limit], 284_468)
+
+ assert {:ok, message} == StealEmojiPolicy.filter(message)
+
+ installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end)
+ assert "firedfox" in installed_emoji
+ end
+end
