commit: 12bdced845a85033280f69a638337566b297754d
parent e07f8f8aa5f53e2746b823559742f597348080f8
Author: rinpatch <rinpatch@sdf.org>
Date: Mon, 23 Nov 2020 14:40:33 +0000
Merge branch 'websearch' into 'develop'
Use `websearch_to_tsvector` by default
See merge request pleroma/pleroma!3163
Diffstat:
6 files changed, 108 insertions(+), 5 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -57,7 +57,7 @@ unit-testing:
policy: pull
services:
- - name: postgres:9.6
+ - name: postgres:13
alias: postgres
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
script:
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Polls now always return a `voters_count`, even if they are single-choice.
- Admin Emails: The ap id is used as the user link in emails now.
+- Search: When using Postgres 11+, Pleroma will use the `websearch_to_tsvector` function to parse search queries.
### Added
diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex
@@ -19,11 +19,18 @@ defmodule Pleroma.Activity.Search do
offset = Keyword.get(options, :offset, 0)
author = Keyword.get(options, :author)
+ search_function =
+ if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do
+ :websearch
+ else
+ :plain
+ end
+
Activity
|> Activity.with_preloaded_object()
|> Activity.restrict_deactivated_users()
|> restrict_public()
- |> query_with(index_type, search_query)
+ |> query_with(index_type, search_query, search_function)
|> maybe_restrict_local(user)
|> maybe_restrict_author(author)
|> maybe_restrict_blocked(user)
@@ -53,7 +60,7 @@ defmodule Pleroma.Activity.Search do
)
end
- defp query_with(q, :gin, search_query) do
+ defp query_with(q, :gin, search_query, :plain) do
from([a, o] in q,
where:
fragment(
@@ -64,7 +71,18 @@ defmodule Pleroma.Activity.Search do
)
end
- defp query_with(q, :rum, search_query) do
+ defp query_with(q, :gin, search_query, :websearch) do
+ from([a, o] in q,
+ where:
+ fragment(
+ "to_tsvector('english', ?->>'content') @@ websearch_to_tsquery('english', ?)",
+ o.data,
+ ^search_query
+ )
+ )
+ end
+
+ defp query_with(q, :rum, search_query, :plain) do
from([a, o] in q,
where:
fragment(
@@ -76,6 +94,18 @@ defmodule Pleroma.Activity.Search do
)
end
+ defp query_with(q, :rum, search_query, :websearch) do
+ from([a, o] in q,
+ where:
+ fragment(
+ "? @@ websearch_to_tsquery('english', ?)",
+ o.fts_content,
+ ^search_query
+ ),
+ order_by: [fragment("? <=> now()::date", o.inserted_at)]
+ )
+ end
+
defp maybe_restrict_local(q, user) do
limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated)
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
@@ -110,7 +110,28 @@ defmodule Pleroma.Application do
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
# for other strategies and supported options
opts = [strategy: :one_for_one, name: Pleroma.Supervisor]
- Supervisor.start_link(children, opts)
+ result = Supervisor.start_link(children, opts)
+
+ set_postgres_server_version()
+
+ result
+ end
+
+ defp set_postgres_server_version do
+ version =
+ with %{rows: [[version]]} <- Ecto.Adapters.SQL.query!(Pleroma.Repo, "show server_version"),
+ {num, _} <- Float.parse(version) do
+ num
+ else
+ e ->
+ Logger.warn(
+ "Could not get the postgres version: #{inspect(e)}.\nSetting the default value of 9.6"
+ )
+
+ 9.6
+ end
+
+ :persistent_term.put({Pleroma.Repo, :postgres_version}, version)
end
def load_custom_modules do
diff --git a/test/pleroma/activity/search_test.exs b/test/pleroma/activity/search_test.exs
@@ -0,0 +1,46 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Activity.SearchTest do
+ alias Pleroma.Activity.Search
+ alias Pleroma.Web.CommonAPI
+ import Pleroma.Factory
+
+ use Pleroma.DataCase
+
+ test "it finds something" do
+ user = insert(:user)
+ {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
+
+ [result] = Search.search(nil, "wednesday")
+
+ assert result.id == post.id
+ end
+
+ test "using plainto_tsquery on postgres < 11" do
+ old_version = :persistent_term.get({Pleroma.Repo, :postgres_version})
+ :persistent_term.put({Pleroma.Repo, :postgres_version}, 10.0)
+
+ user = insert(:user)
+ {:ok, post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
+ {:ok, _post2} = CommonAPI.post(user, %{status: "it's wednesday my bros"})
+
+ # plainto doesn't understand complex queries
+ assert [result] = Search.search(nil, "wednesday -dudes")
+
+ assert result.id == post.id
+
+ :persistent_term.put({Pleroma.Repo, :postgres_version}, old_version)
+ end
+
+ test "using websearch_to_tsquery" do
+ user = insert(:user)
+ {:ok, _post} = CommonAPI.post(user, %{status: "it's wednesday my dudes"})
+ {:ok, other_post} = CommonAPI.post(user, %{status: "it's wednesday my bros"})
+
+ assert [result] = Search.search(nil, "wednesday -dudes")
+
+ assert result.id == other_post.id
+ end
+end
diff --git a/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/search_controller_test.exs
@@ -279,6 +279,9 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do
end
test "search fetches remote statuses and prefers them over other results", %{conn: conn} do
+ old_version = :persistent_term.get({Pleroma.Repo, :postgres_version})
+ :persistent_term.put({Pleroma.Repo, :postgres_version}, 10.0)
+
capture_log(fn ->
{:ok, %{id: activity_id}} =
CommonAPI.post(insert(:user), %{
@@ -295,6 +298,8 @@ defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do
%{"id" => ^activity_id}
] = results["statuses"]
end)
+
+ :persistent_term.put({Pleroma.Repo, :postgres_version}, old_version)
end
test "search doesn't show statuses that it shouldn't", %{conn: conn} do
