commit: e32d099a3a0e701cf2a75775664483f2d72c2dec
parent: 5c78131ee2718d31d264e158607219ab801eb10e
Author: Haelwenn <contact+git.pleroma.social@hacktivis.me>
Date: Wed, 18 Sep 2019 20:58:53 +0000
Merge branch 'feature/poll-refresh' into 'develop'
Poll refetching
See merge request pleroma/pleroma!1694
Diffstat:
10 files changed, 186 insertions(+), 13 deletions(-)
diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex
@@ -6,4 +6,16 @@ defmodule Pleroma.Constants do
use Const
const(as_public, do: "https://www.w3.org/ns/activitystreams#Public")
+
+ const(object_internal_fields,
+ do: [
+ "likes",
+ "like_count",
+ "announcements",
+ "announcement_count",
+ "emoji",
+ "context_id",
+ "deleted_activity_id"
+ ]
+ )
end
diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
@@ -38,6 +38,24 @@ defmodule Pleroma.Object do
def get_by_id(nil), do: nil
def get_by_id(id), do: Repo.get(Object, id)
+ def get_by_id_and_maybe_refetch(id, opts \\ []) do
+ %{updated_at: updated_at} = object = get_by_id(id)
+
+ if opts[:interval] &&
+ NaiveDateTime.diff(NaiveDateTime.utc_now(), updated_at) > opts[:interval] do
+ case Fetcher.refetch_object(object) do
+ {:ok, %Object{} = object} ->
+ object
+
+ e ->
+ Logger.error("Couldn't refresh #{object.data["id"]}:\n#{inspect(e)}")
+ object
+ end
+ else
+ object
+ end
+ end
+
def get_by_ap_id(nil), do: nil
def get_by_ap_id(ap_id) do
diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
@@ -6,18 +6,39 @@ defmodule Pleroma.Object.Fetcher do
alias Pleroma.HTTP
alias Pleroma.Object
alias Pleroma.Object.Containment
+ alias Pleroma.Repo
alias Pleroma.Signature
alias Pleroma.Web.ActivityPub.InternalFetchActor
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.OStatus
require Logger
+ require Pleroma.Constants
- defp reinject_object(data) do
+ defp touch_changeset(changeset) do
+ updated_at =
+ NaiveDateTime.utc_now()
+ |> NaiveDateTime.truncate(:second)
+
+ Ecto.Changeset.put_change(changeset, :updated_at, updated_at)
+ end
+
+ defp maybe_reinject_internal_fields(data, %{data: %{} = old_data}) do
+ internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields())
+
+ Map.merge(data, internal_fields)
+ end
+
+ defp maybe_reinject_internal_fields(data, _), do: data
+
+ defp reinject_object(struct, data) do
Logger.debug("Reinjecting object #{data["id"]}")
with data <- Transmogrifier.fix_object(data),
- {:ok, object} <- Object.create(data) do
+ data <- maybe_reinject_internal_fields(data, struct),
+ changeset <- Object.change(struct, %{data: data}),
+ changeset <- touch_changeset(changeset),
+ {:ok, object} <- Repo.insert_or_update(changeset) do
{:ok, object}
else
e ->
@@ -26,6 +47,17 @@ defmodule Pleroma.Object.Fetcher do
end
end
+ def refetch_object(%Object{data: %{"id" => id}} = object) do
+ with {:local, false} <- {:local, String.starts_with?(id, Pleroma.Web.base_url() <> "/")},
+ {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
+ {:ok, object} <- reinject_object(object, data) do
+ {:ok, object}
+ else
+ {:local, true} -> object
+ e -> {:error, e}
+ end
+ end
+
# TODO:
# This will create a Create activity, which we need internally at the moment.
def fetch_object_from_id(id, options \\ []) do
@@ -57,7 +89,7 @@ defmodule Pleroma.Object.Fetcher do
{:reject, nil}
{:object, data, nil} ->
- reinject_object(data)
+ reinject_object(%Object{}, data)
{:normalize, object = %Object{}} ->
{:ok, object}
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -979,15 +979,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
defp strip_internal_fields(object) do
object
- |> Map.drop([
- "likes",
- "like_count",
- "announcements",
- "announcement_count",
- "emoji",
- "context_id",
- "deleted_activity_id"
- ])
+ |> Map.drop(Pleroma.Constants.object_internal_fields())
end
defp strip_internal_tags(%{"tag" => tags} = object) do
diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex
@@ -485,7 +485,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
end
def get_poll(%{assigns: %{user: user}} = conn, %{"id" => id}) do
- with %Object{} = object <- Object.get_by_id(id),
+ with %Object{} = object <- Object.get_by_id_and_maybe_refetch(id, interval: 60),
%Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
true <- Visibility.visible_for_user?(activity, user) do
conn
diff --git a/test/fixtures/tesla_mock/poll_modified.json b/test/fixtures/tesla_mock/poll_modified.json
@@ -0,0 +1 @@
+{"@context":["https://www.w3.org/ns/activitystreams","https://patch.cx/schemas/litepub-0.1.jsonld",{"@language":"und"}],"actor":"https://patch.cx/users/rin","attachment":[],"attributedTo":"https://patch.cx/users/rin","cc":["https://patch.cx/users/rin/followers"],"closed":"2019-09-19T00:32:36.785333","content":"can you vote on this poll?","context":"https://patch.cx/contexts/626ecafd-3377-46c4-b908-3721a4d4373c","conversation":"https://patch.cx/contexts/626ecafd-3377-46c4-b908-3721a4d4373c","id":"https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d","oneOf":[{"name":"yes","replies":{"totalItems":8,"type":"Collection"},"type":"Note"},{"name":"no","replies":{"totalItems":3,"type":"Collection"},"type":"Note"}],"published":"2019-09-18T14:32:36.802152Z","sensitive":false,"summary":"","tag":[],"to":["https://www.w3.org/ns/activitystreams#Public"],"type":"Question"}+
\ No newline at end of file
diff --git a/test/fixtures/tesla_mock/poll_original.json b/test/fixtures/tesla_mock/poll_original.json
@@ -0,0 +1 @@
+{"@context":["https://www.w3.org/ns/activitystreams","https://patch.cx/schemas/litepub-0.1.jsonld",{"@language":"und"}],"actor":"https://patch.cx/users/rin","attachment":[],"attributedTo":"https://patch.cx/users/rin","cc":["https://patch.cx/users/rin/followers"],"closed":"2019-09-19T00:32:36.785333","content":"can you vote on this poll?","context":"https://patch.cx/contexts/626ecafd-3377-46c4-b908-3721a4d4373c","conversation":"https://patch.cx/contexts/626ecafd-3377-46c4-b908-3721a4d4373c","id":"https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d","oneOf":[{"name":"yes","replies":{"totalItems":4,"type":"Collection"},"type":"Note"},{"name":"no","replies":{"totalItems":0,"type":"Collection"},"type":"Note"}],"published":"2019-09-18T14:32:36.802152Z","sensitive":false,"summary":"","tag":[],"to":["https://www.w3.org/ns/activitystreams#Public"],"type":"Question"}+
\ No newline at end of file
diff --git a/test/fixtures/tesla_mock/rin.json b/test/fixtures/tesla_mock/rin.json
@@ -0,0 +1 @@
+{"@context":["https://www.w3.org/ns/activitystreams","https://patch.cx/schemas/litepub-0.1.jsonld",{"@language":"und"}],"attachment":[],"endpoints":{"oauthAuthorizationEndpoint":"https://patch.cx/oauth/authorize","oauthRegistrationEndpoint":"https://patch.cx/api/v1/apps","oauthTokenEndpoint":"https://patch.cx/oauth/token","sharedInbox":"https://patch.cx/inbox"},"followers":"https://patch.cx/users/rin/followers","following":"https://patch.cx/users/rin/following","icon":{"type":"Image","url":"https://patch.cx/media/4e914f5b84e4a259a3f6c2d2edc9ab642f2ab05f3e3d9c52c81fc2d984b3d51e.jpg"},"id":"https://patch.cx/users/rin","image":{"type":"Image","url":"https://patch.cx/media/f739efddefeee49c6e67e947c4811fdc911785c16ae43da4c3684051fbf8da6a.jpg?name=f739efddefeee49c6e67e947c4811fdc911785c16ae43da4c3684051fbf8da6a.jpg"},"inbox":"https://patch.cx/users/rin/inbox","manuallyApprovesFollowers":false,"name":"rinpatch","outbox":"https://patch.cx/users/rin/outbox","preferredUsername":"rin","publicKey":{"id":"https://patch.cx/users/rin#main-key","owner":"https://patch.cx/users/rin","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5DLtwGXNZElJyxFGfcVc\nXANhaMadj/iYYQwZjOJTV9QsbtiNBeIK54PJrYuU0/0YIdrvS1iqheX5IwXRhcwa\nhm3ZyLz7XeN9st7FBni4BmZMBtMpxAuYuu5p/jbWy13qAiYOhPreCx0wrWgm/lBD\n9mkgaxIxPooBE0S4ZWEJIDIV1Vft3AWcRUyWW1vIBK0uZzs6GYshbQZB952S0yo4\nFzI1hABGHncH8UvuFauh4EZ8tY7/X5I0pGRnDOcRN1dAht5w5yTA+6r5kebiFQjP\nIzN/eCO/a9Flrj9YGW7HDNtjSOH0A31PLRGlJtJO3yK57dnf5ppyCZGfL4emShQo\ncQIDAQAB\n-----END PUBLIC KEY-----\n\n"},"summary":"your friendly neighborhood pleroma developer<br>I like cute things and distributed systems, and really hate delete and redrafts","tag":[],"type":"Person","url":"https://patch.cx/users/rin"}+
\ No newline at end of file
diff --git a/test/object_test.exs b/test/object_test.exs
@@ -4,10 +4,13 @@
defmodule Pleroma.ObjectTest do
use Pleroma.DataCase
+ import ExUnit.CaptureLog
import Pleroma.Factory
import Tesla.Mock
+ alias Pleroma.Activity
alias Pleroma.Object
alias Pleroma.Repo
+ alias Pleroma.Web.CommonAPI
setup do
mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
@@ -89,4 +92,110 @@ defmodule Pleroma.ObjectTest do
)
end
end
+
+ describe "get_by_id_and_maybe_refetch" do
+ setup do
+ mock(fn
+ %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} ->
+ %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/poll_original.json")}
+
+ env ->
+ apply(HttpRequestMock, :request, [env])
+ end)
+
+ mock_modified = fn resp ->
+ mock(fn
+ %{method: :get, url: "https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"} ->
+ resp
+
+ env ->
+ apply(HttpRequestMock, :request, [env])
+ end)
+ end
+
+ on_exit(fn -> mock(fn env -> apply(HttpRequestMock, :request, [env]) end) end)
+
+ [mock_modified: mock_modified]
+ end
+
+ test "refetches if the time since the last refetch is greater than the interval", %{
+ mock_modified: mock_modified
+ } do
+ %Object{} =
+ object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
+
+ assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
+ assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0
+
+ mock_modified.(%Tesla.Env{
+ status: 200,
+ body: File.read!("test/fixtures/tesla_mock/poll_modified.json")
+ })
+
+ updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
+ assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8
+ assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3
+ end
+
+ test "returns the old object if refetch fails", %{mock_modified: mock_modified} do
+ %Object{} =
+ object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
+
+ assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
+ assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0
+
+ assert capture_log(fn ->
+ mock_modified.(%Tesla.Env{status: 404, body: ""})
+
+ updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
+ assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4
+ assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0
+ end) =~
+ "[error] Couldn't refresh https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d"
+ end
+
+ test "does not refetch if the time since the last refetch is greater than the interval", %{
+ mock_modified: mock_modified
+ } do
+ %Object{} =
+ object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
+
+ assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
+ assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0
+
+ mock_modified.(%Tesla.Env{
+ status: 200,
+ body: File.read!("test/fixtures/tesla_mock/poll_modified.json")
+ })
+
+ updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: 100)
+ assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 4
+ assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 0
+ end
+
+ test "preserves internal fields on refetch", %{mock_modified: mock_modified} do
+ %Object{} =
+ object = Object.normalize("https://patch.cx/objects/9a172665-2bc5-452d-8428-2361d4c33b1d")
+
+ assert Enum.at(object.data["oneOf"], 0)["replies"]["totalItems"] == 4
+ assert Enum.at(object.data["oneOf"], 1)["replies"]["totalItems"] == 0
+
+ user = insert(:user)
+ activity = Activity.get_create_by_object_ap_id(object.data["id"])
+ {:ok, _activity, object} = CommonAPI.favorite(activity.id, user)
+
+ assert object.data["like_count"] == 1
+
+ mock_modified.(%Tesla.Env{
+ status: 200,
+ body: File.read!("test/fixtures/tesla_mock/poll_modified.json")
+ })
+
+ updated_object = Object.get_by_id_and_maybe_refetch(object.id, interval: -1)
+ assert Enum.at(updated_object.data["oneOf"], 0)["replies"]["totalItems"] == 8
+ assert Enum.at(updated_object.data["oneOf"], 1)["replies"]["totalItems"] == 3
+
+ assert updated_object.data["like_count"] == 1
+ end
+ end
end
diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex
@@ -1004,6 +1004,10 @@ defmodule HttpRequestMock do
{:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/sjw.json")}}
end
+ def get("https://patch.cx/users/rin", _, _, _) do
+ {:ok, %Tesla.Env{status: 200, body: File.read!("test/fixtures/tesla_mock/rin.json")}}
+ end
+
def get(url, query, body, headers) do
{:error,
"Mock response not implemented for GET #{inspect(url)}, #{query}, #{inspect(body)}, #{
