commit: e8759cb5baba85709607b167d27bd54cf4822366
parent: 9367b70f37ba11f22c456dc6b8dd9366ced84386
Author: Haelwenn <contact+git.pleroma.social@hacktivis.me>
Date: Mon, 20 Jan 2020 11:26:07 +0000
Merge branch 'hashtag-benchmarks' into 'develop'
Hashtag benchmarks
See merge request pleroma/pleroma!2116
Diffstat:
3 files changed, 139 insertions(+), 6 deletions(-)
diff --git a/benchmarks/load_testing/generator.ex b/benchmarks/load_testing/generator.ex
@@ -9,7 +9,7 @@ defmodule Pleroma.LoadTesting.Generator do
{time, _} =
:timer.tc(fn ->
Task.async_stream(
- Enum.take_random(posts, count_likes),
+ Enum.take_random(posts, count_likes),
fn post -> {:ok, _, _} = CommonAPI.favorite(post.id, user) end,
max_concurrency: 10,
timeout: 30_000
@@ -142,6 +142,48 @@ defmodule Pleroma.LoadTesting.Generator do
CommonAPI.post(Enum.random(users), post)
end
+ def generate_power_intervals(opts \\ []) do
+ count = Keyword.get(opts, :count, 20)
+ power = Keyword.get(opts, :power, 2)
+ IO.puts("Generating #{count} intervals for a power #{power} series...")
+ counts = Enum.map(1..count, fn n -> :math.pow(n, power) end)
+ sum = Enum.sum(counts)
+
+ densities =
+ Enum.map(counts, fn c ->
+ c / sum
+ end)
+
+ densities
+ |> Enum.reduce(0, fn density, acc ->
+ if acc == 0 do
+ [{0, density}]
+ else
+ [{_, lower} | _] = acc
+ [{lower, lower + density} | acc]
+ end
+ end)
+ |> Enum.reverse()
+ end
+
+ def generate_tagged_activities(opts \\ []) do
+ tag_count = Keyword.get(opts, :tag_count, 20)
+ users = Keyword.get(opts, :users, Repo.all(User))
+ activity_count = Keyword.get(opts, :count, 200_000)
+
+ intervals = generate_power_intervals(count: tag_count)
+
+ IO.puts(
+ "Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0"
+ )
+
+ Enum.each(1..activity_count, fn _ ->
+ random = :rand.uniform()
+ i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end)
+ CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"})
+ end)
+ end
+
defp do_generate_activity_with_mention(user, users) do
mentions_cnt = Enum.random([2, 3, 4, 5])
with_user = Enum.random([true, false])
diff --git a/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex b/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex
@@ -0,0 +1,87 @@
+defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
+ use Mix.Task
+ alias Pleroma.Repo
+ alias Pleroma.LoadTesting.Generator
+ import Ecto.Query
+
+ def run(_args) do
+ Mix.Pleroma.start_pleroma()
+ activities_count = Repo.aggregate(from(a in Pleroma.Activity), :count, :id)
+
+ if activities_count == 0 do
+ IO.puts("Did not find any activities, cleaning and generating")
+ clean_tables()
+ Generator.generate_users(users_max: 10)
+ Generator.generate_tagged_activities()
+ else
+ IO.puts("Found #{activities_count} activities, won't generate new ones")
+ end
+
+ tags = Enum.map(0..20, fn i -> {"For #tag_#{i}", "tag_#{i}"} end)
+
+ Enum.each(tags, fn {_, tag} ->
+ query =
+ from(o in Pleroma.Object,
+ where: fragment("(?)->'tag' \\? (?)", o.data, ^tag)
+ )
+
+ count = Repo.aggregate(query, :count, :id)
+ IO.puts("Database contains #{count} posts tagged with #{tag}")
+ end)
+
+ user = Repo.all(Pleroma.User) |> List.first()
+
+ Benchee.run(
+ %{
+ "Hashtag fetching, any" => fn tags ->
+ Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
+ %{
+ "any" => tags
+ },
+ user,
+ false
+ )
+ end,
+ # Will always return zero results because no overlapping hashtags are generated.
+ "Hashtag fetching, all" => fn tags ->
+ Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
+ %{
+ "all" => tags
+ },
+ user,
+ false
+ )
+ end
+ },
+ inputs:
+ tags
+ |> Enum.map(fn {_, v} -> v end)
+ |> Enum.chunk_every(2)
+ |> Enum.map(fn tags -> {"For #{inspect(tags)}", tags} end),
+ time: 5
+ )
+
+ Benchee.run(
+ %{
+ "Hashtag fetching" => fn tag ->
+ Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
+ %{
+ "tag" => tag
+ },
+ user,
+ false
+ )
+ end
+ },
+ inputs: tags,
+ time: 5
+ )
+ end
+
+ defp clean_tables do
+ IO.puts("Deleting old data...\n")
+ Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
+ Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
+ Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
+ end
+end
diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
@@ -77,10 +77,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do
|> render("index.json", activities: activities, for: user, as: :activity)
end
- # GET /api/v1/timelines/tag/:tag
- def hashtag(%{assigns: %{user: user}} = conn, params) do
- local_only = truthy_param?(params["local"])
-
+ def hashtag_fetching(params, user, local_only) do
tags =
[params["tag"], params["any"]]
|> List.flatten()
@@ -98,7 +95,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do
|> Map.get("none", [])
|> Enum.map(&String.downcase(&1))
- activities =
+ _activities =
params
|> Map.put("type", "Create")
|> Map.put("local_only", local_only)
@@ -109,6 +106,13 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do
|> Map.put("tag_all", tag_all)
|> Map.put("tag_reject", tag_reject)
|> ActivityPub.fetch_public_activities()
+ end
+
+ # GET /api/v1/timelines/tag/:tag
+ def hashtag(%{assigns: %{user: user}} = conn, params) do
+ local_only = truthy_param?(params["local"])
+
+ activities = hashtag_fetching(params, user, local_only)
conn
|> add_link_headers(activities, %{"local" => local_only})
