pleroma

commit: b830605577f369d6b1a8730a5b3476ceea4fef5a
parent f0f0f2af00e8b73a7013c1308289795961b23f4b
Author: Ivan Tashkinov <ivantashkinov@gmail.com>
Date:   Tue, 19 Jan 2021 22:03:25 +0300

[#3213] Performance-related stat in HashtagsTableMigrator. Reworked `count/_` to indicate approximate total count for current iteration.

Diffstat:
M
lib/pleroma/migrators/hashtags_table_migrator.ex
14
+++++++++++++-

1 file changed, 13 insertions(+), 1 deletion(-)
diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex
@@ -80,6 +80,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do
       DataMigration.update(data_migration, %{state: :running, data: persistent_data})
 
     update_status(:running)
+    put_stat(:started_at, NaiveDateTime.utc_now())
 
     Logger.info("Starting transferring object embedded hashtags to `hashtags` table...")
 
@@ -118,6 +119,12 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do
       increment_stat(:processed_count, length(object_ids))
       increment_stat(:failed_count, length(failed_ids))
 
+      put_stat(
+        :records_per_second,
+        state()[:processed_count] /
+          Enum.max([NaiveDateTime.diff(NaiveDateTime.utc_now(), state()[:started_at]), 1])
+      )
+
       persist_stats(data_migration)
 
       # A quick and dirty approach to controlling the load this background migration imposes
@@ -192,13 +199,18 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do
     end)
   end
 
+  @doc "Approximate count for current iteration (including processed records count)"
   def count(force \\ false, timeout \\ :infinity) do
     stored_count = state()[:count]
 
     if stored_count && !force do
       stored_count
     else
-      count = Repo.aggregate(query(), :count, :id, timeout: timeout)
+      processed_count = state()[:processed_count] || 0
+      max_processed_id = data_migration().data["max_processed_id"] || 0
+      query = where(query(), [object], object.id > ^max_processed_id)
+
+      count = Repo.aggregate(query, :count, :id, timeout: timeout) + processed_count
       put_stat(:count, count)
       count
     end