commit: 130fc9eae63ef5860e383c8f31c968103ba3b894
parent: 3c04460c9c0ae5119a08e50389bf6efd984d22a1
Author: kaniini <nenolod@gmail.com>
Date: Fri, 8 Mar 2019 22:33:31 +0000
Merge branch 'fetch-posts-on-first-federation' into 'develop'
Fetch user's outbox posts on first federation with that user
Closes #131
See merge request pleroma/pleroma!907
Diffstat:
5 files changed, 149 insertions(+), 10 deletions(-)
diff --git a/config/config.exs b/config/config.exs
@@ -348,6 +348,10 @@ config :pleroma, Pleroma.Jobs,
federator_outgoing: [max_jobs: 50],
mailer: [max_jobs: 10]
+config :pleroma, :fetch_initial_posts,
+ enabled: false,
+ pages: 5
+
config :auto_linker,
opts: [
scheme: true,
diff --git a/docs/config.md b/docs/config.md
@@ -285,6 +285,10 @@ This config contains two queues: `federator_incoming` and `federator_outgoing`.
## :rich_media
* `enabled`: if enabled the instance will parse metadata from attached links to generate link previews
+## :fetch_initial_posts
+* `enabled`: if enabled, when a new user is federated with, fetch some of their latest posts
+* `pages`: the amount of pages to fetch
+
## :hackney_pools
Advanced. Tweaks Hackney (http client) connections pools.
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
@@ -532,6 +532,10 @@ defmodule Pleroma.User do
_e ->
with [_nick, _domain] <- String.split(nickname, "@"),
{:ok, user} <- fetch_by_nickname(nickname) do
+ if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do
+ {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
+ end
+
user
else
_e -> nil
@@ -539,6 +543,17 @@ defmodule Pleroma.User do
end
end
+ @doc "Fetch some posts when the user has just been federated with"
+ def fetch_initial_posts(user) do
+ pages = Pleroma.Config.get!([:fetch_initial_posts, :pages])
+
+ Enum.each(
+ # Insert all the posts in reverse order, so they're in the right order on the timeline
+ Enum.reverse(Utils.fetch_ordered_collection(user.info.source_data["outbox"], pages)),
+ &Pleroma.Web.Federator.incoming_ap_doc/1
+ )
+ end
+
def get_followers_query(%User{id: id, follower_address: follower_address}, nil) do
from(
u in User,
@@ -1121,24 +1136,36 @@ defmodule Pleroma.User do
def html_filter_policy(_), do: @default_scrubbers
+ def fetch_by_ap_id(ap_id) do
+ ap_try = ActivityPub.make_user_from_ap_id(ap_id)
+
+ case ap_try do
+ {:ok, user} ->
+ user
+
+ _ ->
+ case OStatus.make_user(ap_id) do
+ {:ok, user} -> user
+ _ -> {:error, "Could not fetch by AP id"}
+ end
+ end
+ end
+
def get_or_fetch_by_ap_id(ap_id) do
user = get_by_ap_id(ap_id)
if !is_nil(user) and !User.needs_update?(user) do
user
else
- ap_try = ActivityPub.make_user_from_ap_id(ap_id)
-
- case ap_try do
- {:ok, user} ->
- user
+ user = fetch_by_ap_id(ap_id)
- _ ->
- case OStatus.make_user(ap_id) do
- {:ok, user} -> user
- _ -> {:error, "Could not fetch by AP id"}
- end
+ if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do
+ with %User{} = user do
+ {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user])
+ end
end
+
+ user
end
end
diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex
@@ -633,4 +633,43 @@ defmodule Pleroma.Web.ActivityPub.Utils do
}
|> Map.merge(additional)
end
+
+ @doc """
+ Fetches the OrderedCollection/OrderedCollectionPage from `from`, limiting the amount of pages fetched after
+ the first one to `pages_left` pages.
+ If the amount of pages is higher than the collection has, it returns whatever was there.
+ """
+ def fetch_ordered_collection(from, pages_left, acc \\ []) do
+ with {:ok, response} <- Tesla.get(from),
+ {:ok, collection} <- Poison.decode(response.body) do
+ case collection["type"] do
+ "OrderedCollection" ->
+ # If we've encountered the OrderedCollection and not the page,
+ # just call the same function on the page address
+ fetch_ordered_collection(collection["first"], pages_left)
+
+ "OrderedCollectionPage" ->
+ if pages_left > 0 do
+ # There are still more pages
+ if Map.has_key?(collection, "next") do
+ # There are still more pages, go deeper saving what we have into the accumulator
+ fetch_ordered_collection(
+ collection["next"],
+ pages_left - 1,
+ acc ++ collection["orderedItems"]
+ )
+ else
+ # No more pages left, just return whatever we already have
+ acc ++ collection["orderedItems"]
+ end
+ else
+ # Got the amount of pages needed, add them all to the accumulator
+ acc ++ collection["orderedItems"]
+ end
+
+ _ ->
+ {:error, "Not an OrderedCollection or OrderedCollectionPage"}
+ end
+ end
+ end
end
diff --git a/test/web/activity_pub/utils_test.exs b/test/web/activity_pub/utils_test.exs
@@ -104,4 +104,69 @@ defmodule Pleroma.Web.ActivityPub.UtilsTest do
assert Enum.sort(cc) == expected_cc
end
end
+
+ describe "fetch_ordered_collection" do
+ import Tesla.Mock
+
+ test "fetches the first OrderedCollectionPage when an OrderedCollection is encountered" do
+ mock(fn
+ %{method: :get, url: "http://mastodon.com/outbox"} ->
+ json(%{"type" => "OrderedCollection", "first" => "http://mastodon.com/outbox?page=true"})
+
+ %{method: :get, url: "http://mastodon.com/outbox?page=true"} ->
+ json(%{"type" => "OrderedCollectionPage", "orderedItems" => ["ok"]})
+ end)
+
+ assert Utils.fetch_ordered_collection("http://mastodon.com/outbox", 1) == ["ok"]
+ end
+
+ test "fetches several pages in the right order one after another, but only the specified amount" do
+ mock(fn
+ %{method: :get, url: "http://example.com/outbox"} ->
+ json(%{
+ "type" => "OrderedCollectionPage",
+ "orderedItems" => [0],
+ "next" => "http://example.com/outbox?page=1"
+ })
+
+ %{method: :get, url: "http://example.com/outbox?page=1"} ->
+ json(%{
+ "type" => "OrderedCollectionPage",
+ "orderedItems" => [1],
+ "next" => "http://example.com/outbox?page=2"
+ })
+
+ %{method: :get, url: "http://example.com/outbox?page=2"} ->
+ json(%{"type" => "OrderedCollectionPage", "orderedItems" => [2]})
+ end)
+
+ assert Utils.fetch_ordered_collection("http://example.com/outbox", 0) == [0]
+ assert Utils.fetch_ordered_collection("http://example.com/outbox", 1) == [0, 1]
+ end
+
+ test "returns an error if the url doesn't have an OrderedCollection/Page" do
+ mock(fn
+ %{method: :get, url: "http://example.com/not-an-outbox"} ->
+ json(%{"type" => "NotAnOutbox"})
+ end)
+
+ assert {:error, _} = Utils.fetch_ordered_collection("http://example.com/not-an-outbox", 1)
+ end
+
+ test "returns the what was collected if there are less pages than specified" do
+ mock(fn
+ %{method: :get, url: "http://example.com/outbox"} ->
+ json(%{
+ "type" => "OrderedCollectionPage",
+ "orderedItems" => [0],
+ "next" => "http://example.com/outbox?page=1"
+ })
+
+ %{method: :get, url: "http://example.com/outbox?page=1"} ->
+ json(%{"type" => "OrderedCollectionPage", "orderedItems" => [1]})
+ end)
+
+ assert Utils.fetch_ordered_collection("http://example.com/outbox", 5) == [0, 1]
+ end
+ end
end
