commit: 908b6ee49a049e60d705e7f923e71326a965f360
parent: d240ce41b55a557366746fb7e313d3810bd8fe16
Author: Haelwenn <contact+git.pleroma.social@hacktivis.me>
Date: Mon, 24 Feb 2020 19:21:29 +0000
Merge branch 'status-visibility-count-stats' into 'develop'
Status count by visibility scope (`GET /api/pleroma/admin/stats` )
See merge request pleroma/pleroma!2097
Diffstat:
11 files changed, 321 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -76,6 +76,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`).
- Add an option `authorized_fetch_mode` to require HTTP signatures for AP fetches.
- ActivityPub: support for `replies` collection (output for outgoing federation & fetching on incoming federation).
+- Mix task to refresh counter cache (`mix pleroma.refresh_counter_cache`)
<details>
<summary>API Changes</summary>
@@ -119,6 +120,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Mastodon API: Add `reacted` property to `emoji_reactions`
- Pleroma API: Add reactions for a single emoji.
- ActivityPub: `[:activitypub, :note_replies_output_limit]` setting sets the number of note self-replies to output on outgoing federation.
+- Admin API: `GET /api/pleroma/admin/stats` to get status count by visibility scope
</details>
### Fixed
diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md
@@ -939,3 +939,20 @@ Loads json generated from `config/descriptions.exs`.
- Params:
- `nicknames`
- Response: Array of user nicknames
+
+## `GET /api/pleroma/admin/stats`
+
+### Stats
+
+- Response:
+
+```json
+{
+ "status_visibility": {
+ "direct": 739,
+ "private": 9,
+ "public": 17,
+ "unlisted": 14
+ }
+}
+```
diff --git a/lib/mix/tasks/pleroma/refresh_counter_cache.ex b/lib/mix/tasks/pleroma/refresh_counter_cache.ex
@@ -0,0 +1,46 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.RefreshCounterCache do
+ @shortdoc "Refreshes counter cache"
+
+ use Mix.Task
+
+ alias Pleroma.Activity
+ alias Pleroma.CounterCache
+ alias Pleroma.Repo
+
+ require Logger
+ import Ecto.Query
+
+ def run([]) do
+ Mix.Pleroma.start_pleroma()
+
+ ["public", "unlisted", "private", "direct"]
+ |> Enum.each(fn visibility ->
+ count = status_visibility_count_query(visibility)
+ name = "status_visibility_#{visibility}"
+ CounterCache.set(name, count)
+ Mix.Pleroma.shell_info("Set #{name} to #{count}")
+ end)
+
+ Mix.Pleroma.shell_info("Done")
+ end
+
+ defp status_visibility_count_query(visibility) do
+ Activity
+ |> where(
+ [a],
+ fragment(
+ "activity_visibility(?, ?, ?) = ?",
+ a.actor,
+ a.recipients,
+ a.data,
+ ^visibility
+ )
+ )
+ |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
+ |> Repo.aggregate(:count, :id, timeout: :timer.minutes(30))
+ end
+end
diff --git a/lib/pleroma/counter_cache.ex b/lib/pleroma/counter_cache.ex
@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.CounterCache do
+ alias Pleroma.CounterCache
+ alias Pleroma.Repo
+ use Ecto.Schema
+ import Ecto.Changeset
+ import Ecto.Query
+
+ schema "counter_cache" do
+ field(:name, :string)
+ field(:count, :integer)
+ end
+
+ def changeset(struct, params) do
+ struct
+ |> cast(params, [:name, :count])
+ |> validate_required([:name])
+ |> unique_constraint(:name)
+ end
+
+ def get_as_map(names) when is_list(names) do
+ CounterCache
+ |> where([cc], cc.name in ^names)
+ |> Repo.all()
+ |> Enum.group_by(& &1.name, & &1.count)
+ |> Map.new(fn {k, v} -> {k, hd(v)} end)
+ end
+
+ def set(name, count) do
+ %CounterCache{}
+ |> changeset(%{"name" => name, "count" => count})
+ |> Repo.insert(
+ on_conflict: [set: [count: count]],
+ returning: true,
+ conflict_target: :name
+ )
+ end
+end
diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex
@@ -4,6 +4,7 @@
defmodule Pleroma.Stats do
import Ecto.Query
+ alias Pleroma.CounterCache
alias Pleroma.Repo
alias Pleroma.User
@@ -96,4 +97,21 @@ defmodule Pleroma.Stats do
}
}
end
+
+ def get_status_visibility_count do
+ counter_cache =
+ CounterCache.get_as_map([
+ "status_visibility_public",
+ "status_visibility_private",
+ "status_visibility_unlisted",
+ "status_visibility_direct"
+ ])
+
+ %{
+ public: counter_cache["status_visibility_public"] || 0,
+ unlisted: counter_cache["status_visibility_unlisted"] || 0,
+ private: counter_cache["status_visibility_private"] || 0,
+ direct: counter_cache["status_visibility_direct"] || 0
+ }
+ end
end
diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -13,6 +13,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
alias Pleroma.ModerationLog
alias Pleroma.Plugs.OAuthScopesPlug
alias Pleroma.ReportNote
+ alias Pleroma.Stats
alias Pleroma.User
alias Pleroma.UserInviteToken
alias Pleroma.Web.ActivityPub.ActivityPub
@@ -98,7 +99,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
plug(
OAuthScopesPlug,
%{scopes: ["read"], admin: true}
- when action in [:config_show, :list_log]
+ when action in [:config_show, :list_log, :stats]
)
plug(
@@ -953,6 +954,13 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
conn |> json("")
end
+ def stats(conn, _) do
+ count = Stats.get_status_visibility_count()
+
+ conn
+ |> json(%{"status_visibility" => count})
+ end
+
def errors(conn, {:error, :not_found}) do
conn
|> put_status(:not_found)
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
@@ -201,6 +201,7 @@ defmodule Pleroma.Web.Router do
get("/moderation_log", AdminAPIController, :list_log)
post("/reload_emoji", AdminAPIController, :reload_emoji)
+ get("/stats", AdminAPIController, :stats)
end
scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do
diff --git a/priv/repo/migrations/20200109123126_add_counter_cache_table.exs b/priv/repo/migrations/20200109123126_add_counter_cache_table.exs
@@ -0,0 +1,55 @@
+defmodule Pleroma.Repo.Migrations.AddCounterCacheTable do
+ use Ecto.Migration
+
+ def up do
+ create_if_not_exists table(:counter_cache) do
+ add(:name, :string, null: false)
+ add(:count, :bigint, null: false, default: 0)
+ end
+
+ create_if_not_exists(unique_index(:counter_cache, [:name]))
+
+ """
+ CREATE OR REPLACE FUNCTION update_status_visibility_counter_cache()
+ RETURNS TRIGGER AS
+ $$
+ DECLARE
+ BEGIN
+ IF TG_OP = 'INSERT' THEN
+ IF NEW.data->>'type' = 'Create' THEN
+ EXECUTE 'INSERT INTO counter_cache (name, count) VALUES (''status_visibility_' || activity_visibility(NEW.actor, NEW.recipients, NEW.data) || ''', 1) ON CONFLICT (name) DO UPDATE SET count = counter_cache.count + 1';
+ END IF;
+ RETURN NEW;
+ ELSIF TG_OP = 'UPDATE' THEN
+ IF (NEW.data->>'type' = 'Create') and (OLD.data->>'type' = 'Create') and activity_visibility(NEW.actor, NEW.recipients, NEW.data) != activity_visibility(OLD.actor, OLD.recipients, OLD.data) THEN
+ EXECUTE 'INSERT INTO counter_cache (name, count) VALUES (''status_visibility_' || activity_visibility(NEW.actor, NEW.recipients, NEW.data) || ''', 1) ON CONFLICT (name) DO UPDATE SET count = counter_cache.count + 1';
+ EXECUTE 'update counter_cache SET count = counter_cache.count - 1 where count > 0 and name = ''status_visibility_' || activity_visibility(OLD.actor, OLD.recipients, OLD.data) || ''';';
+ END IF;
+ RETURN NEW;
+ ELSIF TG_OP = 'DELETE' THEN
+ IF OLD.data->>'type' = 'Create' THEN
+ EXECUTE 'update counter_cache SET count = counter_cache.count - 1 where count > 0 and name = ''status_visibility_' || activity_visibility(OLD.actor, OLD.recipients, OLD.data) || ''';';
+ END IF;
+ RETURN OLD;
+ END IF;
+ END;
+ $$
+ LANGUAGE 'plpgsql';
+ """
+ |> execute()
+
+ """
+ CREATE TRIGGER status_visibility_counter_cache_trigger BEFORE INSERT OR UPDATE of recipients, data OR DELETE ON activities
+ FOR EACH ROW
+ EXECUTE PROCEDURE update_status_visibility_counter_cache();
+ """
+ |> execute()
+ end
+
+ def down do
+ execute("drop trigger if exists status_visibility_counter_cache_trigger on activities")
+ execute("drop function if exists update_status_visibility_counter_cache()")
+ drop_if_exists(unique_index(:counter_cache, [:name]))
+ drop_if_exists(table(:counter_cache))
+ end
+end
diff --git a/test/stat_test.exs b/test/stat_test.exs
@@ -0,0 +1,70 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.StateTest do
+ use Pleroma.DataCase
+ import Pleroma.Factory
+ alias Pleroma.Web.CommonAPI
+
+ describe "status visibility count" do
+ test "on new status" do
+ user = insert(:user)
+ other_user = insert(:user)
+
+ CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+
+ Enum.each(0..1, fn _ ->
+ CommonAPI.post(user, %{
+ "visibility" => "unlisted",
+ "status" => "hey"
+ })
+ end)
+
+ Enum.each(0..2, fn _ ->
+ CommonAPI.post(user, %{
+ "visibility" => "direct",
+ "status" => "hey @#{other_user.nickname}"
+ })
+ end)
+
+ Enum.each(0..3, fn _ ->
+ CommonAPI.post(user, %{
+ "visibility" => "private",
+ "status" => "hey"
+ })
+ end)
+
+ assert %{direct: 3, private: 4, public: 1, unlisted: 2} =
+ Pleroma.Stats.get_status_visibility_count()
+ end
+
+ test "on status delete" do
+ user = insert(:user)
+ {:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+ assert %{public: 1} = Pleroma.Stats.get_status_visibility_count()
+ CommonAPI.delete(activity.id, user)
+ assert %{public: 0} = Pleroma.Stats.get_status_visibility_count()
+ end
+
+ test "on status visibility update" do
+ user = insert(:user)
+ {:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+ assert %{public: 1, private: 0} = Pleroma.Stats.get_status_visibility_count()
+ {:ok, _} = CommonAPI.update_activity_scope(activity.id, %{"visibility" => "private"})
+ assert %{public: 0, private: 1} = Pleroma.Stats.get_status_visibility_count()
+ end
+
+ test "doesn't count unrelated activities" do
+ user = insert(:user)
+ other_user = insert(:user)
+ {:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+ _ = CommonAPI.follow(user, other_user)
+ CommonAPI.favorite(activity.id, other_user)
+ CommonAPI.repeat(activity.id, other_user)
+
+ assert %{direct: 0, private: 0, public: 1, unlisted: 0} =
+ Pleroma.Stats.get_status_visibility_count()
+ end
+ end
+end
diff --git a/test/tasks/refresh_counter_cache_test.exs b/test/tasks/refresh_counter_cache_test.exs
@@ -0,0 +1,43 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.RefreshCounterCacheTest do
+ use Pleroma.DataCase
+ alias Pleroma.Web.CommonAPI
+ import ExUnit.CaptureIO, only: [capture_io: 1]
+ import Pleroma.Factory
+
+ test "counts statuses" do
+ user = insert(:user)
+ other_user = insert(:user)
+
+ CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+
+ Enum.each(0..1, fn _ ->
+ CommonAPI.post(user, %{
+ "visibility" => "unlisted",
+ "status" => "hey"
+ })
+ end)
+
+ Enum.each(0..2, fn _ ->
+ CommonAPI.post(user, %{
+ "visibility" => "direct",
+ "status" => "hey @#{other_user.nickname}"
+ })
+ end)
+
+ Enum.each(0..3, fn _ ->
+ CommonAPI.post(user, %{
+ "visibility" => "private",
+ "status" => "hey"
+ })
+ end)
+
+ assert capture_io(fn -> Mix.Tasks.Pleroma.RefreshCounterCache.run([]) end) =~ "Done\n"
+
+ assert %{direct: 3, private: 4, public: 1, unlisted: 2} =
+ Pleroma.Stats.get_status_visibility_count()
+ end
+end
diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs
@@ -3545,6 +3545,25 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
assert String.starts_with?(child["group"], ":")
assert child["description"]
end
+
+ describe "/api/pleroma/admin/stats" do
+ test "status visibility count", %{conn: conn} do
+ admin = insert(:user, is_admin: true)
+ user = insert(:user)
+ CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+ CommonAPI.post(user, %{"visibility" => "unlisted", "status" => "hey"})
+ CommonAPI.post(user, %{"visibility" => "unlisted", "status" => "hey"})
+
+ response =
+ conn
+ |> assign(:user, admin)
+ |> get("/api/pleroma/admin/stats")
+ |> json_response(200)
+
+ assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 2} =
+ response["status_visibility"]
+ end
+ end
end
# Needed for testing
