commit: 4745a41393cddd9bbc5a14affa77595204488b8f
parent b729a8b140306fd67817442bdde1299e05aca5b2
Author: marcin mikołajczak <git@mkljczk.pl>
Date: Thu, 10 Aug 2023 23:03:19 +0200
Allow to specify post language
Signed-off-by: marcin mikołajczak <git@mkljczk.pl>
Diffstat:
14 files changed, 280 insertions(+), 25 deletions(-)
diff --git a/changelog.d/post-languages.add b/changelog.d/post-languages.add
@@ -0,0 +1 @@
+Allow to specify post language
+\ No newline at end of file
diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex
@@ -19,7 +19,8 @@ defmodule Pleroma.Constants do
"context_id",
"deleted_activity_id",
"pleroma_internal",
- "generator"
+ "generator",
+ "language"
]
)
@@ -38,7 +39,8 @@ defmodule Pleroma.Constants do
"summary",
"sensitive",
"attachment",
- "generator"
+ "generator",
+ "language"
]
)
diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex
@@ -86,6 +86,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do
|> fix_attachments()
|> Transmogrifier.fix_emoji()
|> Transmogrifier.fix_content_map()
+ |> Transmogrifier.maybe_add_language()
end
def changeset(struct, data) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex
@@ -57,6 +57,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do
field(:replies_count, :integer, default: 0)
field(:like_count, :integer, default: 0)
field(:announcement_count, :integer, default: 0)
+ field(:language, :string)
field(:inReplyTo, ObjectValidators.ObjectID)
field(:url, ObjectValidators.BareUri)
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -22,6 +22,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
alias Pleroma.Web.Federator
import Ecto.Query
+ import Pleroma.Web.CommonAPI.Utils, only: [is_good_locale_code?: 1]
+ import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
require Logger
require Pleroma.Constants
@@ -42,6 +44,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
|> fix_content_map()
|> fix_addressing()
|> fix_summary()
+ |> maybe_add_language()
end
def fix_summary(%{"summary" => nil} = object) do
@@ -318,6 +321,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
def fix_tag(object), do: object
+ def fix_content_map(%{"content" => content} = object) when not_empty_string(content), do: object
+
# content map usually only has one language so this will do for now.
def fix_content_map(%{"contentMap" => content_map} = object) do
content_groups = Map.to_list(content_map)
@@ -454,6 +459,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
|> strip_internal_fields()
|> fix_type(fetch_options)
|> fix_in_reply_to(fetch_options)
+ |> maybe_add_language_from_activity(data)
data = Map.put(data, "object", object)
options = Keyword.put(options, :local, false)
@@ -679,6 +685,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
|> add_mention_tags
|> add_emoji_tags
|> add_attributed_to
+ |> maybe_add_content_map
|> prepare_attachments
|> set_conversation
|> set_reply_to_uri
@@ -722,7 +729,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
data =
data
|> Map.put("object", object)
- |> Map.merge(Utils.make_json_ld_header())
+ |> Map.merge(Utils.make_json_ld_header(data))
|> Map.delete("bcc")
{:ok, data}
@@ -737,7 +744,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
data =
data
|> Map.put("object", object)
- |> Map.merge(Utils.make_json_ld_header())
+ |> Map.merge(Utils.make_json_ld_header(data))
|> Map.delete("bcc")
{:ok, data}
@@ -758,7 +765,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
data =
data
|> strip_internal_fields
- |> Map.merge(Utils.make_json_ld_header())
+ |> Map.merge(Utils.make_json_ld_header(data))
|> Map.delete("bcc")
{:ok, data}
@@ -778,7 +785,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
data =
data
|> Map.put("object", object)
- |> Map.merge(Utils.make_json_ld_header())
+ |> Map.merge(Utils.make_json_ld_header(data))
{:ok, data}
end
@@ -796,7 +803,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
data =
data
|> Map.put("object", object)
- |> Map.merge(Utils.make_json_ld_header())
+ |> Map.merge(Utils.make_json_ld_header(data))
{:ok, data}
end
@@ -807,7 +814,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
data
|> strip_internal_fields
|> maybe_fix_object_url
- |> Map.merge(Utils.make_json_ld_header())
+ |> Map.merge(Utils.make_json_ld_header(data))
{:ok, data}
end
@@ -952,4 +959,63 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
def maybe_fix_user_url(data), do: data
def maybe_fix_user_object(data), do: maybe_fix_user_url(data)
+
+ defp maybe_add_content_map(%{"language" => language, "content" => content} = object)
+ when not_empty_string(language) do
+ Map.put(object, "contentMap", Map.put(%{}, language, content))
+ end
+
+ defp maybe_add_content_map(object), do: object
+
+ def maybe_add_language(object) do
+ language =
+ [
+ get_language_from_context(object),
+ get_language_from_content_map(object),
+ get_language_from_content(object)
+ ]
+ |> Enum.find(&is_good_locale_code?(&1))
+
+ if language do
+ Map.put(object, "language", language)
+ else
+ object
+ end
+ end
+
+ def maybe_add_language_from_activity(object, activity) do
+ language = get_language_from_context(activity)
+
+ if is_good_locale_code?(language) do
+ Map.put(object, "language", language)
+ else
+ object
+ end
+ end
+
+ defp get_language_from_context(%{"@context" => context}) when is_list(context) do
+ case context
+ |> Enum.find(fn
+ %{"@language" => language} -> language != "und"
+ _ -> nil
+ end) do
+ %{"@language" => language} -> language
+ _ -> nil
+ end
+ end
+
+ defp get_language_from_context(_), do: nil
+
+ defp get_language_from_content_map(%{"contentMap" => content_map, "content" => source_content}) do
+ content_groups = Map.to_list(content_map)
+
+ case Enum.find(content_groups, fn {_, content} -> content == source_content end) do
+ {language, _} -> language
+ _ -> nil
+ end
+ end
+
+ defp get_language_from_content_map(_), do: nil
+
+ defp get_language_from_content(_), do: nil
end
diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex
@@ -19,6 +19,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
alias Pleroma.Web.Router.Helpers
import Ecto.Query
+ import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
require Logger
require Pleroma.Constants
@@ -108,18 +109,24 @@ defmodule Pleroma.Web.ActivityPub.Utils do
end
end
- def make_json_ld_header do
+ def make_json_ld_header(data \\ %{}) do
%{
"@context" => [
"https://www.w3.org/ns/activitystreams",
"#{Endpoint.url()}/schemas/litepub-0.1.jsonld",
%{
- "@language" => "und"
+ "@language" => get_language(data)
}
]
}
end
+ defp get_language(%{"language" => language}) when not_empty_string(language) do
+ language
+ end
+
+ defp get_language(_), do: "und"
+
def make_date do
DateTime.utc_now() |> DateTime.to_iso8601()
end
diff --git a/lib/pleroma/web/activity_pub/views/object_view.ex b/lib/pleroma/web/activity_pub/views/object_view.ex
@@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
alias Pleroma.Web.ActivityPub.Transmogrifier
def render("object.json", %{object: %Object{} = object}) do
- base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
+ base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header(object.data)
additional = Transmogrifier.prepare_object(object.data)
Map.merge(base, additional)
@@ -17,7 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
def render("object.json", %{object: %Activity{data: %{"type" => activity_type}} = activity})
when activity_type in ["Create", "Listen"] do
- base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
+ base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header(activity.data)
object = Object.normalize(activity, fetch: false)
additional =
@@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do
end
def render("object.json", %{object: %Activity{} = activity}) do
- base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header()
+ base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header(activity.data)
object_id = Object.normalize(activity, id_only: true)
additional =
diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex
@@ -33,6 +33,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
cc: [],
context: nil,
sensitive: false,
+ language: nil,
object: nil,
preview?: false,
changes: %{}
@@ -57,6 +58,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
|> content()
|> with_valid(&to_and_cc/1)
|> with_valid(&context/1)
+ |> with_valid(&language/1)
|> sensitive()
|> with_valid(&object/1)
|> preview?()
@@ -190,6 +192,16 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
%__MODULE__{draft | sensitive: sensitive}
end
+ defp language(draft) do
+ language = draft.params[:language]
+
+ if Utils.is_good_locale_code?(language) do
+ %__MODULE__{draft | language: language}
+ else
+ draft
+ end
+ end
+
defp object(draft) do
emoji = Map.merge(Pleroma.Emoji.Formatter.get_emoji_map(draft.full_payload), draft.emoji)
@@ -229,6 +241,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
"mediaType" => Utils.get_content_type(draft.params[:content_type])
})
|> Map.put("generator", draft.params[:generator])
+ |> Map.put("language", draft.language)
%__MODULE__{draft | object: object}
end
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
@@ -494,4 +494,20 @@ defmodule Pleroma.Web.CommonAPI.Utils do
{:error, dgettext("errors", "Too many attachments")}
end
end
+
+ def is_good_locale_code?(code) when is_binary(code) do
+ code
+ |> String.codepoints()
+ |> Enum.all?(&valid_char?/1)
+ end
+
+ def is_good_locale_code?(_code), do: false
+
+ # [a-zA-Z0-9-]
+ defp valid_char?(char) do
+ ("a" <= char and char <= "z") or
+ ("A" <= char and char <= "Z") or
+ ("0" <= char and char <= "9") or
+ char == "-"
+ end
end
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -200,7 +200,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
mentions: mentions,
tags: reblogged[:tags] || [],
application: build_application(object.data["generator"]),
- language: nil,
+ language: object.data["language"],
emojis: [],
pleroma: %{
local: activity.local,
@@ -391,7 +391,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
mentions: mentions,
tags: build_tags(tags),
application: build_application(object.data["generator"]),
- language: nil,
+ language: object.data["language"],
emojis: build_emojis(object.data["emoji"]),
pleroma: %{
local: activity.local,
diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs
@@ -221,6 +221,36 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
"<p><span class=\"h-card\"><a href=\"http://localtesting.pleroma.lol/users/lain\" class=\"u-url mention\">@<span>lain</span></a></span></p>"
end
+ test "it only uses contentMap if content is not present" do
+ user = insert(:user)
+
+ message = %{
+ "@context" => "https://www.w3.org/ns/activitystreams",
+ "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+ "cc" => [],
+ "type" => "Create",
+ "object" => %{
+ "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+ "cc" => [],
+ "id" => Utils.generate_object_id(),
+ "type" => "Note",
+ "content" => "Hi",
+ "contentMap" => %{
+ "de" => "Hallo",
+ "uk" => "Привіт"
+ },
+ "inReplyTo" => nil,
+ "attributedTo" => user.ap_id
+ },
+ "actor" => user.ap_id
+ }
+
+ {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(message)
+ object = Object.normalize(data["object"], fetch: false)
+
+ assert object.data["content"] == "Hi"
+ end
+
test "it works for incoming notices with to/cc not being an array (kroeg)" do
data = File.read!("test/fixtures/kroeg-post-activity.json") |> Jason.decode!()
@@ -358,6 +388,87 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier.NoteHandlingTest do
end
end
+ test "it detects language from context" do
+ user = insert(:user)
+
+ message = %{
+ "@context" => ["https://www.w3.org/ns/activitystreams", %{"@language" => "pl"}],
+ "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+ "cc" => [],
+ "type" => "Create",
+ "object" => %{
+ "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+ "cc" => [],
+ "id" => Utils.generate_object_id(),
+ "type" => "Note",
+ "content" => "Szczęść Boże",
+ "attributedTo" => user.ap_id
+ },
+ "actor" => user.ap_id
+ }
+
+ {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(message)
+ object = Object.normalize(data["object"], fetch: false)
+
+ assert object.data["language"] == "pl"
+ end
+
+ test "it detects language from contentMap" do
+ user = insert(:user)
+
+ message = %{
+ "@context" => "https://www.w3.org/ns/activitystreams",
+ "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+ "cc" => [],
+ "type" => "Create",
+ "object" => %{
+ "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+ "cc" => [],
+ "id" => Utils.generate_object_id(),
+ "type" => "Note",
+ "content" => "Szczęść Boże",
+ "contentMap" => %{
+ "de" => "Gott segne",
+ "pl" => "Szczęść Boże"
+ },
+ "attributedTo" => user.ap_id
+ },
+ "actor" => user.ap_id
+ }
+
+ {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(message)
+ object = Object.normalize(data["object"], fetch: false)
+
+ assert object.data["language"] == "pl"
+ end
+
+ test "it detects language from content" do
+ clear_config([Pleroma.Language.LanguageDetector, :provider], LanguageDetectorMock)
+
+ user = insert(:user)
+
+ message = %{
+ "@context" => ["https://www.w3.org/ns/activitystreams"],
+ "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+ "cc" => [],
+ "type" => "Create",
+ "object" => %{
+ "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+ "cc" => [],
+ "id" => Utils.generate_object_id(),
+ "type" => "Note",
+ "content" => "Dieu vous bénisse, Fédivers.",
+ "attributedTo" => user.ap_id
+ },
+ "actor" => user.ap_id
+ }
+
+ {:ok, %Activity{data: data, local: false}} = Transmogrifier.handle_incoming(message)
+ object = Object.normalize(data["object"], fetch: false)
+
+ assert object.data["language"] == "fr"
+ end
+
describe "`handle_incoming/2`, Mastodon format `replies` handling" do
setup do: clear_config([:activitypub, :note_replies_output_limit], 5)
setup do: clear_config([:instance, :federation_incoming_replies_max_depth])
diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs
@@ -352,6 +352,18 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
end
end
+ test "it adds contentMap if language is specified" do
+ user = insert(:user)
+
+ {:ok, activity} = CommonAPI.post(user, %{status: "тест", language: "uk"})
+
+ {:ok, prepared} = Transmogrifier.prepare_outgoing(activity.data)
+
+ assert prepared["object"]["contentMap"] == %{
+ "uk" => "тест"
+ }
+ end
+
describe "actor rewriting" do
test "it fixes the actor URL property to be a proper URI" do
data = %{
diff --git a/test/pleroma/web/activity_pub/utils_test.exs b/test/pleroma/web/activity_pub/utils_test.exs
@@ -138,16 +138,30 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do
end
end
- test "make_json_ld_header/0" do
- assert Utils.make_json_ld_header() == %{
- "@context" => [
- "https://www.w3.org/ns/activitystreams",
- "http://localhost:4001/schemas/litepub-0.1.jsonld",
- %{
- "@language" => "und"
- }
- ]
- }
+ describe "make_json_ld_header/1" do
+ test "makes jsonld header" do
+ assert Utils.make_json_ld_header() == %{
+ "@context" => [
+ "https://www.w3.org/ns/activitystreams",
+ "http://localhost:4001/schemas/litepub-0.1.jsonld",
+ %{
+ "@language" => "und"
+ }
+ ]
+ }
+ end
+
+ test "includes language if specified" do
+ assert Utils.make_json_ld_header(%{"language" => "pl"}) == %{
+ "@context" => [
+ "https://www.w3.org/ns/activitystreams",
+ "http://localhost:4001/schemas/litepub-0.1.jsonld",
+ %{
+ "@language" => "pl"
+ }
+ ]
+ }
+ end
end
describe "get_existing_votes" do
diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs
@@ -770,6 +770,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
assert status.edited_at
end
+ test "it shows post language" do
+ user = insert(:user)
+
+ {:ok, post} = CommonAPI.post(user, %{status: "Szczęść Boże", language: "pl"})
+
+ status = StatusView.render("show.json", activity: post)
+
+ assert status.language == "pl"
+ end
+
test "with a source object" do
note =
insert(:note,
