commit: 83e5b95102a3192fa29233f9cf89d4fd198912ee
parent: b536e57124fda9adeb4c78739d0eb2be80d47f6e
Author: lain <lain@soykaf.club>
Date: Tue, 16 Jun 2020 21:07:04 +0000
Merge branch 'hotfix/notification-migration' into 'develop'
Chunk the notification type backfill migration
See merge request pleroma/pleroma!2653
Diffstat:
2 files changed, 29 insertions(+), 1 deletion(-)
diff --git a/lib/pleroma/migration_helper/notification_backfill.ex b/lib/pleroma/migration_helper/notification_backfill.ex
@@ -18,7 +18,7 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do
)
query
- |> Repo.all()
+ |> Repo.chunk_stream(100)
|> Enum.each(fn notification ->
type =
notification.activity
diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex
@@ -8,6 +8,7 @@ defmodule Pleroma.Repo do
adapter: Ecto.Adapters.Postgres,
migration_timestamps: [type: :naive_datetime_usec]
+ import Ecto.Query
require Logger
defmodule Instrumenter do
@@ -78,6 +79,33 @@ defmodule Pleroma.Repo do
:ok
end
end
+
+ def chunk_stream(query, chunk_size) do
+ # We don't actually need start and end funcitons of resource streaming,
+ # but it seems to be the only way to not fetch records one-by-one and
+ # have individual records be the elements of the stream, instead of
+ # lists of records
+ Stream.resource(
+ fn -> 0 end,
+ fn
+ last_id ->
+ query
+ |> order_by(asc: :id)
+ |> where([r], r.id > ^last_id)
+ |> limit(^chunk_size)
+ |> all()
+ |> case do
+ [] ->
+ {:halt, last_id}
+
+ records ->
+ last_id = List.last(records).id
+ {records, last_id}
+ end
+ end,
+ fn _ -> :ok end
+ )
+ end
end
defmodule Pleroma.Repo.UnappliedMigrationsError do