commit: a6946048fbe049aa223d094d36eb767739ab5ff2
parent 0fae71f88d142f64ec18a49ff4292db816dacdc8
Author: Ekaterina Vaartis <vaartis@kotobank.ch>
Date: Wed, 17 Nov 2021 22:29:49 +0300
Rename Activity.Search to Search.DatabaseSearch
Diffstat:
13 files changed, 238 insertions(+), 250 deletions(-)
diff --git a/config/config.exs b/config/config.exs
@@ -870,7 +870,7 @@ config :pleroma, ConcurrentLimiter, [
{Pleroma.Search, [max_running: 30, max_waiting: 50]}
]
-config :pleroma, Pleroma.Search, module: Pleroma.Activity.Search
+config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil
diff --git a/config/test.exs b/config/test.exs
@@ -133,7 +133,7 @@ config :pleroma, :side_effects,
ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock,
logger: Pleroma.LoggerMock
-config :pleroma, Pleroma.Search, module: Pleroma.Activity
+config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", private_key: nil
diff --git a/docs/configuration/search.md b/docs/configuration/search.md
@@ -6,7 +6,7 @@
To use built-in search that has no external dependencies, set the search module to `Pleroma.Activity`:
-> config :pleroma, Pleroma.Search, module: Pleroma.Activity
+> config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
While it has no external dependencies, it has problems with performance and relevancy.
diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
@@ -366,7 +366,7 @@ defmodule Pleroma.Activity do
from(activity in query, where: activity.actor not in subquery(deactivated_users_query))
end
- defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search
+ defdelegate search(user, query, options \\ []), to: Pleroma.Search.DatabaseSearch
def direct_conversation_id(activity, for_user) do
alias Pleroma.Conversation.Participation
diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex
@@ -1,165 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Activity.Search do
- alias Pleroma.Activity
- alias Pleroma.Object.Fetcher
- alias Pleroma.Pagination
- alias Pleroma.User
- alias Pleroma.Web.ActivityPub.Visibility
-
- require Pleroma.Constants
-
- import Ecto.Query
-
- def search(user, search_query, options \\ []) do
- index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin
- limit = Enum.min([Keyword.get(options, :limit), 40])
- offset = Keyword.get(options, :offset, 0)
- author = Keyword.get(options, :author)
-
- search_function =
- if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do
- :websearch
- else
- :plain
- end
-
- try do
- Activity
- |> Activity.with_preloaded_object()
- |> Activity.restrict_deactivated_users()
- |> restrict_public(user)
- |> query_with(index_type, search_query, search_function)
- |> maybe_restrict_local(user)
- |> maybe_restrict_author(author)
- |> maybe_restrict_blocked(user)
- |> Pagination.fetch_paginated(
- %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
- :offset
- )
- |> maybe_fetch(user, search_query)
- rescue
- _ -> maybe_fetch([], user, search_query)
- end
- end
-
- def add_to_index(_activity), do: nil
- def remove_from_index(_object), do: nil
-
- def maybe_restrict_author(query, %User{} = author) do
- Activity.Queries.by_author(query, author)
- end
-
- def maybe_restrict_author(query, _), do: query
-
- def maybe_restrict_blocked(query, %User{} = user) do
- Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user))
- end
-
- def maybe_restrict_blocked(query, _), do: query
-
- defp restrict_public(q, user) when not is_nil(user) do
- intended_recipients = [
- Pleroma.Constants.as_public(),
- Pleroma.Web.ActivityPub.Utils.as_local_public()
- ]
-
- from([a, o] in q,
- where: fragment("?->>'type' = 'Create'", a.data),
- where: fragment("? && ?", ^intended_recipients, a.recipients)
- )
- end
-
- defp restrict_public(q, _user) do
- from([a, o] in q,
- where: fragment("?->>'type' = 'Create'", a.data),
- where: ^Pleroma.Constants.as_public() in a.recipients
- )
- end
-
- defp query_with(q, :gin, search_query, :plain) do
- %{rows: [[tsc]]} =
- Ecto.Adapters.SQL.query!(
- Pleroma.Repo,
- "select current_setting('default_text_search_config')::regconfig::oid;"
- )
-
- from([a, o] in q,
- where:
- fragment(
- "to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)",
- ^tsc,
- o.data,
- ^search_query
- )
- )
- end
-
- defp query_with(q, :gin, search_query, :websearch) do
- %{rows: [[tsc]]} =
- Ecto.Adapters.SQL.query!(
- Pleroma.Repo,
- "select current_setting('default_text_search_config')::regconfig::oid;"
- )
-
- from([a, o] in q,
- where:
- fragment(
- "to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)",
- ^tsc,
- o.data,
- ^search_query
- )
- )
- end
-
- defp query_with(q, :rum, search_query, :plain) do
- from([a, o] in q,
- where:
- fragment(
- "? @@ plainto_tsquery(?)",
- o.fts_content,
- ^search_query
- ),
- order_by: [fragment("? <=> now()::date", o.inserted_at)]
- )
- end
-
- defp query_with(q, :rum, search_query, :websearch) do
- from([a, o] in q,
- where:
- fragment(
- "? @@ websearch_to_tsquery(?)",
- o.fts_content,
- ^search_query
- ),
- order_by: [fragment("? <=> now()::date", o.inserted_at)]
- )
- end
-
- def maybe_restrict_local(q, user) do
- limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
-
- case {limit, user} do
- {:all, _} -> restrict_local(q)
- {:unauthenticated, %User{}} -> q
- {:unauthenticated, _} -> restrict_local(q)
- {false, _} -> q
- end
- end
-
- defp restrict_local(q), do: where(q, local: true)
-
- def maybe_fetch(activities, user, search_query) do
- with true <- Regex.match?(~r/https?:/, search_query),
- {:ok, object} <- Fetcher.fetch_object_from_id(search_query),
- %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
- true <- Visibility.visible_for_user?(activity, user) do
- [activity | activities]
- else
- _ -> activities
- end
- end
-end
diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex
@@ -0,0 +1,24 @@
+defmodule Pleroma.Search do
+ def add_to_index(activity) do
+ search_module = Pleroma.Config.get([Pleroma.Search, :module])
+
+ ConcurrentLimiter.limit(Pleroma.Search, fn ->
+ Task.start(fn -> search_module.add_to_index(activity) end)
+ end)
+ end
+
+ def remove_from_index(object) do
+ # Also delete from search index
+ search_module = Pleroma.Config.get([Pleroma.Search, :module])
+
+ ConcurrentLimiter.limit(Pleroma.Search, fn ->
+ Task.start(fn -> search_module.remove_from_index(object) end)
+ end)
+ end
+
+ def search(query, options) do
+ search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
+
+ search_module.search(options[:for_user], query, options)
+ end
+end
diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex
@@ -1,24 +1,153 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
defmodule Pleroma.Search.DatabaseSearch do
- def add_to_index(activity) do
- search_module = Pleroma.Config.get([Pleroma.Search, :module])
+ alias Pleroma.Activity
+ alias Pleroma.Object.Fetcher
+ alias Pleroma.Pagination
+ alias Pleroma.User
+ alias Pleroma.Web.ActivityPub.Visibility
+
+ require Pleroma.Constants
+
+ import Ecto.Query
+
+ def search(user, search_query, options \\ []) do
+ index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin
+ limit = Enum.min([Keyword.get(options, :limit), 40])
+ offset = Keyword.get(options, :offset, 0)
+ author = Keyword.get(options, :author)
+
+ search_function =
+ if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do
+ :websearch
+ else
+ :plain
+ end
+
+ try do
+ Activity
+ |> Activity.with_preloaded_object()
+ |> Activity.restrict_deactivated_users()
+ |> restrict_public()
+ |> query_with(index_type, search_query, search_function)
+ |> maybe_restrict_local(user)
+ |> maybe_restrict_author(author)
+ |> maybe_restrict_blocked(user)
+ |> Pagination.fetch_paginated(
+ %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
+ :offset
+ )
+ |> maybe_fetch(user, search_query)
+ rescue
+ _ -> maybe_fetch([], user, search_query)
+ end
+ end
+
+ def add_to_index(_activity), do: nil
+ def remove_from_index(_object), do: nil
+
+ def maybe_restrict_author(query, %User{} = author) do
+ Activity.Queries.by_author(query, author)
+ end
+
+ def maybe_restrict_author(query, _), do: query
+
+ def maybe_restrict_blocked(query, %User{} = user) do
+ Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user))
+ end
+
+ def maybe_restrict_blocked(query, _), do: query
+
+ def restrict_public(q) do
+ from([a, o] in q,
+ where: fragment("?->>'type' = 'Create'", a.data),
+ where: ^Pleroma.Constants.as_public() in a.recipients
+ )
+ end
+
+ defp query_with(q, :gin, search_query, :plain) do
+ %{rows: [[tsc]]} =
+ Ecto.Adapters.SQL.query!(
+ Pleroma.Repo,
+ "select current_setting('default_text_search_config')::regconfig::oid;"
+ )
+
+ from([a, o] in q,
+ where:
+ fragment(
+ "to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)",
+ ^tsc,
+ o.data,
+ ^search_query
+ )
+ )
+ end
+
+ defp query_with(q, :gin, search_query, :websearch) do
+ %{rows: [[tsc]]} =
+ Ecto.Adapters.SQL.query!(
+ Pleroma.Repo,
+ "select current_setting('default_text_search_config')::regconfig::oid;"
+ )
+
+ from([a, o] in q,
+ where:
+ fragment(
+ "to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)",
+ ^tsc,
+ o.data,
+ ^search_query
+ )
+ )
+ end
+
+ defp query_with(q, :rum, search_query, :plain) do
+ from([a, o] in q,
+ where:
+ fragment(
+ "? @@ plainto_tsquery(?)",
+ o.fts_content,
+ ^search_query
+ ),
+ order_by: [fragment("? <=> now()::date", o.inserted_at)]
+ )
+ end
- ConcurrentLimiter.limit(Pleroma.Search, fn ->
- Task.start(fn -> search_module.add_to_index(activity) end)
- end)
+ defp query_with(q, :rum, search_query, :websearch) do
+ from([a, o] in q,
+ where:
+ fragment(
+ "? @@ websearch_to_tsquery(?)",
+ o.fts_content,
+ ^search_query
+ ),
+ order_by: [fragment("? <=> now()::date", o.inserted_at)]
+ )
end
- def remove_from_index(object) do
- # Also delete from search index
- search_module = Pleroma.Config.get([Pleroma.Search, :module])
+ def maybe_restrict_local(q, user) do
+ limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
- ConcurrentLimiter.limit(Pleroma.Search, fn ->
- Task.start(fn -> search_module.remove_from_index(object) end)
- end)
+ case {limit, user} do
+ {:all, _} -> restrict_local(q)
+ {:unauthenticated, %User{}} -> q
+ {:unauthenticated, _} -> restrict_local(q)
+ {false, _} -> q
+ end
end
- def search(query, options) do
- search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity)
+ defp restrict_local(q), do: where(q, local: true)
- search_module.search(options[:for_user], query, options)
+ def maybe_fetch(activities, user, search_query) do
+ with true <- Regex.match?(~r/https?:/, search_query),
+ {:ok, object} <- Fetcher.fetch_object_from_id(search_query),
+ %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
+ true <- Visibility.visible_for_user?(activity, user) do
+ [activity | activities]
+ else
+ _ -> activities
+ end
end
end
diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex
@@ -4,7 +4,7 @@ defmodule Pleroma.Search.Meilisearch do
alias Pleroma.Activity
- import Pleroma.Activity.Search
+ import Pleroma.Search.DatabaseSearch
import Ecto.Query
defp meili_headers do
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -141,7 +141,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end)
# Add local posts to search index
- if local, do: Pleroma.Search.DatabaseSearch.add_to_index(activity)
+ if local, do: Pleroma.Search.add_to_index(activity)
{:ok, activity}
else
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -227,7 +227,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
end)
- Pleroma.Search.DatabaseSearch.add_to_index(Map.put(activity, :object, object))
+ Pleroma.Search.add_to_index(Map.put(activity, :object, object))
meta =
meta
@@ -330,7 +330,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
if result == :ok do
Notification.create_notifications(object)
- Pleroma.Search.DatabaseSearch.remove_from_index(deleted_object)
+ Pleroma.Search.remove_from_index(deleted_object)
{:ok, object, meta}
else
diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
@@ -99,7 +99,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
end
defp resource_search(_, "statuses", query, options) do
- statuses = with_fallback(fn -> Pleroma.Search.DatabaseSearch.search(query, options) end)
+ statuses = with_fallback(fn -> Pleroma.Search.search(query, options) end)
StatusView.render("index.json",
activities: statuses,
diff --git a/test/pleroma/activity/search_test.exs b/test/pleroma/activity/search_test.exs
@@ -1,62 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Activity.SearchTest do
- alias Pleroma.Activity.Search
- alias Pleroma.Web.CommonAPI
- import Pleroma.Factory
-
- use Pleroma.DataCase, async: true
-
- test "it finds something" do
- user = insert(:user)
- {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
-
- [result] = Search.search(nil, "wednesday")
-
- assert result.id == post.id
- end
-
- test "it finds local-only posts for authenticated users" do
- user = insert(:user)
- reader = insert(:user)
- {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes", visibility: "local"})
-
- [result] = Search.search(reader, "wednesday")
-
- assert result.id == post.id
- end
-
- test "it does not find local-only posts for anonymous users" do
- user = insert(:user)
- {:ok, _post} = CommonAPI.post(user, %{status: "it's wednesday my dudes", visibility: "local"})
-
- assert [] = Search.search(nil, "wednesday")
- end
-
- test "using plainto_tsquery on postgres < 11" do
- old_version = :persistent_term.get({Pleroma.Repo, :postgres_version})
- :persistent_term.put({Pleroma.Repo, :postgres_version}, 10.0)
- on_exit(fn -> :persistent_term.put({Pleroma.Repo, :postgres_version}, old_version) end)
-
- user = insert(:user)
- {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
- {:ok, _post2} = CommonAPI.post(user, %{status: "it's wednesday my bros"})
-
- # plainto doesn't understand complex queries
- assert [result] = Search.search(nil, "wednesday -dudes")
-
- assert result.id == post.id
- end
-
- test "using websearch_to_tsquery" do
- user = insert(:user)
- {:ok, _post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
- {:ok, other_post} = CommonAPI.post(user, %{status: "it's wednesday my bros"})
-
- assert [result] = Search.search(nil, "wednesday -dudes")
-
- assert result.id == other_post.id
- end
-end
diff --git a/test/pleroma/search/database_search_test.ex b/test/pleroma/search/database_search_test.ex
@@ -0,0 +1,62 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Search.DatabaseSearchTest do
+ alias Pleroma.Search.DatabaseSearch
+ alias Pleroma.Web.CommonAPI
+ import Pleroma.Factory
+
+ use Pleroma.DataCase, async: true
+
+ test "it finds something" do
+ user = insert(:user)
+ {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
+
+ [result] = DatabaseSearch.search(nil, "wednesday")
+
+ assert result.id == post.id
+ end
+
+ test "it finds local-only posts for authenticated users" do
+ user = insert(:user)
+ reader = insert(:user)
+ {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes", visibility: "local"})
+
+ [result] = Search.search(reader, "wednesday")
+
+ assert result.id == post.id
+ end
+
+ test "it does not find local-only posts for anonymous users" do
+ user = insert(:user)
+ {:ok, _post} = CommonAPI.post(user, %{status: "it's wednesday my dudes", visibility: "local"})
+
+ assert [] = Search.search(nil, "wednesday")
+ end
+
+ test "using plainto_tsquery on postgres < 11" do
+ old_version = :persistent_term.get({Pleroma.Repo, :postgres_version})
+ :persistent_term.put({Pleroma.Repo, :postgres_version}, 10.0)
+ on_exit(fn -> :persistent_term.put({Pleroma.Repo, :postgres_version}, old_version) end)
+
+ user = insert(:user)
+ {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
+ {:ok, _post2} = CommonAPI.post(user, %{status: "it's wednesday my bros"})
+
+ # plainto doesn't understand complex queries
+ assert [result] = DatabaseSearch.search(nil, "wednesday -dudes")
+
+ assert result.id == post.id
+ end
+
+ test "using websearch_to_tsquery" do
+ user = insert(:user)
+ {:ok, _post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
+ {:ok, other_post} = CommonAPI.post(user, %{status: "it's wednesday my bros"})
+
+ assert [result] = DatabaseSearch.search(nil, "wednesday -dudes")
+
+ assert result.id == other_post.id
+ end
+end
