pleroma

commit: d8860eaee46c9bc0a079e90dfb008c54923d7330
parent ba40af054cc90417c5b2b347e325b53c7346e29c
Author: feld <feld@feld.me>
Date:   Thu, 21 Jan 2021 14:59:51 +0000

Merge branch 'limiter-setup-fix' into 'develop'

Configurable limits for ConcurrentLimiter for Pleroma.Web.RichMedia.Helpers & Pleroma.Web.MediaProxyWarmingPolicy

See merge request pleroma/pleroma!3248
Diffstat:
M
CHANGELOG.md
2
++
M
config/config.exs
5
+++++
M
config/description.exs
48
++++++++++++++++++++++++++++++++++++++++++++++++
M
docs/configuration/cheatsheet.md
12
++++++++++++
M
lib/pleroma/application.ex
13
+++++++++++--
M
lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
2
+-

6 files changed, 79 insertions(+), 3 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -34,6 +34,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - OAuth form improvements: users are remembered by their cookie, the CSS is overridable by the admin, and the style has been improved.
 - OAuth improvements and fixes: more secure session-based authentication (by token that could be revoked anytime), ability to revoke belonging OAuth token from any client etc.
 - Ability to set ActivityPub aliases for follower migration.
+- Configurable background job limits for RichMedia (link previews) and MediaProxyWarmingPolicy
+
 
 <details>
   <summary>API Changes</summary>
diff --git a/config/config.exs b/config/config.exs
@@ -832,6 +832,11 @@ config :pleroma, Pleroma.User.Backup,
   limit_days: 7,
   dir: nil
 
+config :pleroma, ConcurrentLimiter, [
+  {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]},
+  {Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, [max_running: 5, max_waiting: 5]}
+]
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"
diff --git a/config/description.exs b/config/description.exs
@@ -3330,5 +3330,53 @@ config :pleroma, :config_description, [
         suggestions: [:text, :protobuf]
       }
     ]
+  },
+  %{
+    group: :pleroma,
+    key: ConcurrentLimiter,
+    type: :group,
+    description: "Limits configuration for background tasks.",
+    children: [
+      %{
+        key: Pleroma.Web.RichMedia.Helpers,
+        type: :keyword,
+        description: "Concurrent limits configuration for getting RichMedia for activities.",
+        suggestions: [max_running: 5, max_waiting: 5],
+        children: [
+          %{
+            key: :max_running,
+            type: :integer,
+            description: "Max running concurrently jobs.",
+            suggestion: [5]
+          },
+          %{
+            key: :max_waiting,
+            type: :integer,
+            description: "Max waiting jobs.",
+            suggestion: [5]
+          }
+        ]
+      },
+      %{
+        key: Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy,
+        type: :keyword,
+        description: "Concurrent limits configuration for MediaProxyWarmingPolicy.",
+        suggestions: [max_running: 5, max_waiting: 5],
+        children: [
+          %{
+            key: :max_running,
+            type: :integer,
+            description: "Max running concurrently jobs.",
+            suggestion: [5]
+          },
+          %{
+            key: :max_waiting,
+            type: :integer,
+            description: "Max waiting jobs.",
+            suggestion: [5]
+          }
+        ]
+      }
+    ]
   }
 ]
diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md
@@ -1110,3 +1110,15 @@ Settings to enable and configure expiration for ephemeral activities
 
 * `:enabled` - enables ephemeral activities creation
 * `:min_lifetime` - minimum lifetime for ephemeral activities (in seconds). Default: 10 minutes.
+
+## ConcurrentLimiter
+
+Settings to restrict concurrently running jobs. Jobs which can be configured:
+
+* `Pleroma.Web.RichMedia.Helpers` - generating link previews of URLs in activities
+* `Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy` - warming remote media cache via MediaProxyWarmingPolicy
+
+Each job has these settings:
+
+* `:max_running` - max concurrently runnings jobs
+* `:max_waiting` - max waiting jobs
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
@@ -297,7 +297,16 @@ defmodule Pleroma.Application do
 
   @spec limiters_setup() :: :ok
   def limiters_setup do
-    [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.MediaProxy]
-    |> Enum.each(&ConcurrentLimiter.new(&1, 1, 0))
+    config = Config.get(ConcurrentLimiter, [])
+
+    [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy]
+    |> Enum.each(fn module ->
+      mod_config = Keyword.get(config, module, [])
+
+      max_running = Keyword.get(mod_config, :max_running, 5)
+      max_waiting = Keyword.get(mod_config, :max_waiting, 5)
+
+      ConcurrentLimiter.new(module, max_running, max_waiting)
+    end)
   end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
@@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
       if Pleroma.Config.get(:env) == :test do
         fetch(prefetch_url)
       else
-        ConcurrentLimiter.limit(MediaProxy, fn ->
+        ConcurrentLimiter.limit(__MODULE__, fn ->
           Task.start(fn -> fetch(prefetch_url) end)
         end)
       end