commit: c5434dbefc5e6a25ffb72debe8ed9e09d77bf885
parent: 6c6de8e5dfaf245e4e16ddca0c741342eeaa9187
Author: rinpatch <rinpatch@sdf.org>
Date: Mon, 7 Sep 2020 16:49:27 +0000
Merge branch 'feature/1790-oban-overuse' into 'develop'
removing StatsWorker from Oban cron jobs
See merge request pleroma/pleroma!2963
Diffstat:
10 files changed, 129 insertions(+), 50 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,12 +3,21 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+## Unreleased
+
+### Removed
+
+- **Breaking:** Removed `Pleroma.Workers.Cron.StatsWorker` setting from Oban `:crontab`.
+
+
## unreleased-patch - ???
### Added
+
- Rich media failure tracking (along with `:failure_backoff` option)
### Fixed
+
- Possible OOM errors with the default HTTP adapter
- Mastodon API: Search parameter `following` now correctly returns the followings rather than the followers
- Mastodon API: Timelines hanging for (`number of posts with links * rich media timeout`) in the worst case.
@@ -16,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Mastodon API: Cards being wrong for preview statuses due to cache key collision
- Password resets no longer processed for deactivated accounts
+
## [2.1.0] - 2020-08-28
### Changed
diff --git a/config/config.exs b/config/config.exs
@@ -546,7 +546,6 @@ config :pleroma, Oban,
plugins: [Oban.Plugins.Pruner],
crontab: [
{"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker},
- {"0 * * * *", Pleroma.Workers.Cron.StatsWorker},
{"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker},
{"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker},
{"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}
@@ -672,7 +671,7 @@ config :pleroma, :static_fe, enabled: false
# With no frontend configuration, the bundled files from the `static` directory will
# be used.
#
-# config :pleroma, :frontends,
+# config :pleroma, :frontends,
# primary: %{"name" => "pleroma-fe", "ref" => "develop"},
# admin: %{"name" => "admin-fe", "ref" => "stable"},
# available: %{...}
diff --git a/config/description.exs b/config/description.exs
@@ -2291,7 +2291,6 @@ config :pleroma, :config_description, [
description: "Settings for cron background jobs",
suggestions: [
{"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker},
- {"0 * * * *", Pleroma.Workers.Cron.StatsWorker},
{"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker},
{"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker},
{"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}
diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex
@@ -18,6 +18,7 @@ defmodule Mix.Pleroma do
@doc "Common functions to be reused in mix tasks"
def start_pleroma do
Pleroma.Config.Holder.save_default()
+ Pleroma.Config.Oban.warn()
Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
if Pleroma.Config.get(:env) != :test do
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
@@ -50,6 +50,7 @@ defmodule Pleroma.Application do
Pleroma.Telemetry.Logger.attach()
Config.Holder.save_default()
Pleroma.HTML.compile_scrubbers()
+ Pleroma.Config.Oban.warn()
Config.DeprecationWarnings.warn()
Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
Pleroma.ApplicationRequirements.verify!()
diff --git a/lib/pleroma/config/oban.ex b/lib/pleroma/config/oban.ex
@@ -0,0 +1,30 @@
+defmodule Pleroma.Config.Oban do
+ require Logger
+
+ def warn do
+ oban_config = Pleroma.Config.get(Oban)
+
+ crontab =
+ [Pleroma.Workers.Cron.StatsWorker]
+ |> Enum.reduce(oban_config[:crontab], fn removed_worker, acc ->
+ with acc when is_list(acc) <- acc,
+ setting when is_tuple(setting) <-
+ Enum.find(acc, fn {_, worker} -> worker == removed_worker end) do
+ """
+ !!!OBAN CONFIG WARNING!!!
+ You are using old workers in Oban crontab settings, which were removed.
+ Please, remove setting from crontab in your config file (prod.secret.exs): #{
+ inspect(setting)
+ }
+ """
+ |> Logger.warn()
+
+ List.delete(acc, setting)
+ else
+ _ -> acc
+ end
+ end)
+
+ Pleroma.Config.put(Oban, Keyword.put(oban_config, :crontab, crontab))
+ end
+end
diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex
@@ -3,12 +3,15 @@
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Stats do
+ use GenServer
+
import Ecto.Query
+
alias Pleroma.CounterCache
alias Pleroma.Repo
alias Pleroma.User
- use GenServer
+ @interval :timer.seconds(60)
def start_link(_) do
GenServer.start_link(
@@ -18,6 +21,11 @@ defmodule Pleroma.Stats do
)
end
+ @impl true
+ def init(_args) do
+ {:ok, nil, {:continue, :calculate_stats}}
+ end
+
@doc "Performs update stats"
def force_update do
GenServer.call(__MODULE__, :force_update)
@@ -29,7 +37,11 @@ defmodule Pleroma.Stats do
end
@doc "Returns stats data"
- @spec get_stats() :: %{domain_count: integer(), status_count: integer(), user_count: integer()}
+ @spec get_stats() :: %{
+ domain_count: non_neg_integer(),
+ status_count: non_neg_integer(),
+ user_count: non_neg_integer()
+ }
def get_stats do
%{stats: stats} = GenServer.call(__MODULE__, :get_state)
@@ -44,25 +56,14 @@ defmodule Pleroma.Stats do
peers
end
- def init(_args) do
- {:ok, calculate_stat_data()}
- end
-
- def handle_call(:force_update, _from, _state) do
- new_stats = calculate_stat_data()
- {:reply, new_stats, new_stats}
- end
-
- def handle_call(:get_state, _from, state) do
- {:reply, state, state}
- end
-
- def handle_cast(:run_update, _state) do
- new_stats = calculate_stat_data()
-
- {:noreply, new_stats}
- end
-
+ @spec calculate_stat_data() :: %{
+ peers: list(),
+ stats: %{
+ domain_count: non_neg_integer(),
+ status_count: non_neg_integer(),
+ user_count: non_neg_integer()
+ }
+ }
def calculate_stat_data do
peers =
from(
@@ -97,6 +98,7 @@ defmodule Pleroma.Stats do
}
end
+ @spec get_status_visibility_count(String.t() | nil) :: map()
def get_status_visibility_count(instance \\ nil) do
if is_nil(instance) do
CounterCache.get_sum()
@@ -104,4 +106,36 @@ defmodule Pleroma.Stats do
CounterCache.get_by_instance(instance)
end
end
+
+ @impl true
+ def handle_continue(:calculate_stats, _) do
+ stats = calculate_stat_data()
+ Process.send_after(self(), :run_update, @interval)
+ {:noreply, stats}
+ end
+
+ @impl true
+ def handle_call(:force_update, _from, _state) do
+ new_stats = calculate_stat_data()
+ {:reply, new_stats, new_stats}
+ end
+
+ @impl true
+ def handle_call(:get_state, _from, state) do
+ {:reply, state, state}
+ end
+
+ @impl true
+ def handle_cast(:run_update, _state) do
+ new_stats = calculate_stat_data()
+
+ {:noreply, new_stats}
+ end
+
+ @impl true
+ def handle_info(:run_update, _) do
+ new_stats = calculate_stat_data()
+ Process.send_after(self(), :run_update, @interval)
+ {:noreply, new_stats}
+ end
end
diff --git a/lib/pleroma/workers/cron/stats_worker.ex b/lib/pleroma/workers/cron/stats_worker.ex
@@ -1,17 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Workers.Cron.StatsWorker do
- @moduledoc """
- The worker to update peers statistics.
- """
-
- use Oban.Worker, queue: "background"
-
- @impl Oban.Worker
- def perform(_job) do
- Pleroma.Stats.do_collect()
- :ok
- end
-end
diff --git a/priv/repo/migrations/20200906072147_remove_cron_stats_worker_from_oban_config.exs b/priv/repo/migrations/20200906072147_remove_cron_stats_worker_from_oban_config.exs
@@ -0,0 +1,19 @@
+defmodule Pleroma.Repo.Migrations.RemoveCronStatsWorkerFromObanConfig do
+ use Ecto.Migration
+
+ def change do
+ with %Pleroma.ConfigDB{} = config <-
+ Pleroma.ConfigDB.get_by_params(%{group: :pleroma, key: Oban}),
+ crontab when is_list(crontab) <- config.value[:crontab],
+ index when is_integer(index) <-
+ Enum.find_index(crontab, fn {_, worker} ->
+ worker == Pleroma.Workers.Cron.StatsWorker
+ end) do
+ updated_value = Keyword.put(config.value, :crontab, List.delete_at(crontab, index))
+
+ config
+ |> Ecto.Changeset.change(value: updated_value)
+ |> Pleroma.Repo.update()
+ end
+ end
+end
diff --git a/test/stats_test.exs b/test/stats_test.exs
@@ -4,7 +4,10 @@
defmodule Pleroma.StatsTest do
use Pleroma.DataCase
+
import Pleroma.Factory
+
+ alias Pleroma.Stats
alias Pleroma.Web.CommonAPI
describe "user count" do
@@ -13,7 +16,7 @@ defmodule Pleroma.StatsTest do
_internal = insert(:user, local: true, nickname: nil)
_internal = Pleroma.Web.ActivityPub.Relay.get_actor()
- assert match?(%{stats: %{user_count: 1}}, Pleroma.Stats.calculate_stat_data())
+ assert match?(%{stats: %{user_count: 1}}, Stats.calculate_stat_data())
end
end
@@ -47,23 +50,23 @@ defmodule Pleroma.StatsTest do
end)
assert %{"direct" => 3, "private" => 4, "public" => 1, "unlisted" => 2} =
- Pleroma.Stats.get_status_visibility_count()
+ Stats.get_status_visibility_count()
end
test "on status delete" do
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"})
- assert %{"public" => 1} = Pleroma.Stats.get_status_visibility_count()
+ assert %{"public" => 1} = Stats.get_status_visibility_count()
CommonAPI.delete(activity.id, user)
- assert %{"public" => 0} = Pleroma.Stats.get_status_visibility_count()
+ assert %{"public" => 0} = Stats.get_status_visibility_count()
end
test "on status visibility update" do
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{visibility: "public", status: "hey"})
- assert %{"public" => 1, "private" => 0} = Pleroma.Stats.get_status_visibility_count()
+ assert %{"public" => 1, "private" => 0} = Stats.get_status_visibility_count()
{:ok, _} = CommonAPI.update_activity_scope(activity.id, %{visibility: "private"})
- assert %{"public" => 0, "private" => 1} = Pleroma.Stats.get_status_visibility_count()
+ assert %{"public" => 0, "private" => 1} = Stats.get_status_visibility_count()
end
test "doesn't count unrelated activities" do
@@ -75,7 +78,7 @@ defmodule Pleroma.StatsTest do
CommonAPI.repeat(activity.id, other_user)
assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 0} =
- Pleroma.Stats.get_status_visibility_count()
+ Stats.get_status_visibility_count()
end
end
@@ -110,10 +113,10 @@ defmodule Pleroma.StatsTest do
end)
assert %{"direct" => 10, "private" => 0, "public" => 1, "unlisted" => 5} =
- Pleroma.Stats.get_status_visibility_count(local_instance)
+ Stats.get_status_visibility_count(local_instance)
assert %{"direct" => 0, "private" => 20, "public" => 0, "unlisted" => 0} =
- Pleroma.Stats.get_status_visibility_count(instance2)
+ Stats.get_status_visibility_count(instance2)
end
end
end
