commit: a511d2f9003870d267c143ee1aa12a0dfd805f8e
parent: 50afe5ab9e384676a7e64b476f3655ccf94acb5d
Author: kaniini <nenolod@gmail.com>
Date: Wed, 5 Jun 2019 15:51:32 +0000
Merge branch 'improve-search' into 'develop'
[#943] Contain search for unauthenticated users
See merge request pleroma/pleroma!1220
Diffstat:
10 files changed, 305 insertions(+), 178 deletions(-)
diff --git a/lib/mix/tasks/benchmark.ex b/lib/mix/tasks/benchmark.ex
@@ -7,7 +7,7 @@ defmodule Mix.Tasks.Pleroma.Benchmark do
Benchee.run(%{
"search" => fn ->
- Pleroma.Web.MastodonAPI.MastodonAPIController.status_search(nil, "cofe")
+ Pleroma.Activity.search(nil, "cofe")
end
})
end
diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
@@ -343,4 +343,6 @@ defmodule Pleroma.Activity do
)
)
end
+
+ defdelegate search(user, query), to: Pleroma.Activity.Search
end
diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex
@@ -0,0 +1,75 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Activity.Search do
+ alias Pleroma.Activity
+ alias Pleroma.Object.Fetcher
+ alias Pleroma.Repo
+ alias Pleroma.User
+ alias Pleroma.Web.ActivityPub.Visibility
+
+ import Ecto.Query
+
+ def search(user, search_query) do
+ index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin
+
+ Activity
+ |> Activity.with_preloaded_object()
+ |> Activity.restrict_deactivated_users()
+ |> restrict_public()
+ |> query_with(index_type, search_query)
+ |> maybe_restrict_local(user)
+ |> Repo.all()
+ |> maybe_fetch(user, search_query)
+ end
+
+ defp restrict_public(q) do
+ from([a, o] in q,
+ where: fragment("?->>'type' = 'Create'", a.data),
+ where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
+ limit: 40
+ )
+ end
+
+ defp query_with(q, :gin, search_query) do
+ from([a, o] in q,
+ where:
+ fragment(
+ "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
+ o.data,
+ ^search_query
+ ),
+ order_by: [desc: :id]
+ )
+ end
+
+ defp query_with(q, :rum, search_query) do
+ from([a, o] in q,
+ where:
+ fragment(
+ "? @@ plainto_tsquery('english', ?)",
+ o.fts_content,
+ ^search_query
+ ),
+ order_by: [fragment("? <=> now()::date", o.inserted_at)]
+ )
+ end
+
+ # users can search everything
+ defp maybe_restrict_local(q, %User{}), do: q
+
+ # unauthenticated users can only search local activities
+ defp maybe_restrict_local(q, _), do: where(q, local: true)
+
+ defp maybe_fetch(activities, user, search_query) do
+ with true <- Regex.match?(~r/https?:/, search_query),
+ {:ok, object} <- Fetcher.fetch_object_from_id(search_query),
+ %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
+ true <- Visibility.visible_for_user?(activity, user) do
+ activities ++ [activity]
+ else
+ _ -> activities
+ end
+ end
+end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
@@ -735,122 +735,6 @@ defmodule Pleroma.User do
|> Repo.all()
end
- def search(query, resolve \\ false, for_user \\ nil) do
- # Strip the beginning @ off if there is a query
- query = String.trim_leading(query, "@")
-
- if resolve, do: get_or_fetch(query)
-
- {:ok, results} =
- Repo.transaction(fn ->
- Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", [])
- Repo.all(search_query(query, for_user))
- end)
-
- results
- end
-
- def search_query(query, for_user) do
- fts_subquery = fts_search_subquery(query)
- trigram_subquery = trigram_search_subquery(query)
- union_query = from(s in trigram_subquery, union_all: ^fts_subquery)
- distinct_query = from(s in subquery(union_query), order_by: s.search_type, distinct: s.id)
-
- from(s in subquery(boost_search_rank_query(distinct_query, for_user)),
- order_by: [desc: s.search_rank],
- limit: 40
- )
- end
-
- defp boost_search_rank_query(query, nil), do: query
-
- defp boost_search_rank_query(query, for_user) do
- friends_ids = get_friends_ids(for_user)
- followers_ids = get_followers_ids(for_user)
-
- from(u in subquery(query),
- select_merge: %{
- search_rank:
- fragment(
- """
- CASE WHEN (?) THEN (?) * 1.3
- WHEN (?) THEN (?) * 1.2
- WHEN (?) THEN (?) * 1.1
- ELSE (?) END
- """,
- u.id in ^friends_ids and u.id in ^followers_ids,
- u.search_rank,
- u.id in ^friends_ids,
- u.search_rank,
- u.id in ^followers_ids,
- u.search_rank,
- u.search_rank
- )
- }
- )
- end
-
- defp fts_search_subquery(term, query \\ User) do
- processed_query =
- term
- |> String.replace(~r/\W+/, " ")
- |> String.trim()
- |> String.split()
- |> Enum.map(&(&1 <> ":*"))
- |> Enum.join(" | ")
-
- from(
- u in query,
- select_merge: %{
- search_type: ^0,
- search_rank:
- fragment(
- """
- ts_rank_cd(
- setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
- setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'),
- to_tsquery('simple', ?),
- 32
- )
- """,
- u.nickname,
- u.name,
- ^processed_query
- )
- },
- where:
- fragment(
- """
- (setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
- setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')) @@ to_tsquery('simple', ?)
- """,
- u.nickname,
- u.name,
- ^processed_query
- )
- )
- |> restrict_deactivated()
- end
-
- defp trigram_search_subquery(term) do
- from(
- u in User,
- select_merge: %{
- # ^1 gives 'Postgrex expected a binary, got 1' for some weird reason
- search_type: fragment("?", 1),
- search_rank:
- fragment(
- "similarity(?, trim(? || ' ' || coalesce(?, '')))",
- ^term,
- u.nickname,
- u.name
- )
- },
- where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^term)
- )
- |> restrict_deactivated()
- end
-
def mute(muter, %User{ap_id: ap_id}) do
info_cng =
muter.info
@@ -1449,4 +1333,6 @@ defmodule Pleroma.User do
)
|> Repo.all()
end
+
+ defdelegate search(query, opts \\ []), to: User.Search
end
diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex
@@ -0,0 +1,145 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.User.Search do
+ alias Pleroma.Repo
+ alias Pleroma.User
+ import Ecto.Query
+
+ def search(query, opts \\ []) do
+ resolve = Keyword.get(opts, :resolve, false)
+ for_user = Keyword.get(opts, :for_user)
+
+ # Strip the beginning @ off if there is a query
+ query = String.trim_leading(query, "@")
+
+ if match?(%User{}, for_user) and resolve, do: User.get_or_fetch(query)
+
+ {:ok, results} =
+ Repo.transaction(fn ->
+ Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", [])
+
+ query
+ |> search_query(for_user)
+ |> Repo.all()
+ end)
+
+ results
+ end
+
+ defp search_query(query, for_user) do
+ query
+ |> union_query()
+ |> distinct_query()
+ |> boost_search_rank_query(for_user)
+ |> subquery()
+ |> order_by(desc: :search_rank)
+ |> limit(20)
+ |> maybe_restrict_local(for_user)
+ end
+
+ defp union_query(query) do
+ fts_subquery = fts_search_subquery(query)
+ trigram_subquery = trigram_search_subquery(query)
+
+ from(s in trigram_subquery, union_all: ^fts_subquery)
+ end
+
+ defp distinct_query(q) do
+ from(s in subquery(q), order_by: s.search_type, distinct: s.id)
+ end
+
+ # unauthenticated users can only search local activities
+ defp maybe_restrict_local(q, %User{}), do: q
+ defp maybe_restrict_local(q, _), do: where(q, [u], u.local == true)
+
+ defp boost_search_rank_query(query, nil), do: query
+
+ defp boost_search_rank_query(query, for_user) do
+ friends_ids = User.get_friends_ids(for_user)
+ followers_ids = User.get_followers_ids(for_user)
+
+ from(u in subquery(query),
+ select_merge: %{
+ search_rank:
+ fragment(
+ """
+ CASE WHEN (?) THEN (?) * 1.3
+ WHEN (?) THEN (?) * 1.2
+ WHEN (?) THEN (?) * 1.1
+ ELSE (?) END
+ """,
+ u.id in ^friends_ids and u.id in ^followers_ids,
+ u.search_rank,
+ u.id in ^friends_ids,
+ u.search_rank,
+ u.id in ^followers_ids,
+ u.search_rank,
+ u.search_rank
+ )
+ }
+ )
+ end
+
+ defp fts_search_subquery(term, query \\ User) do
+ processed_query =
+ term
+ |> String.replace(~r/\W+/, " ")
+ |> String.trim()
+ |> String.split()
+ |> Enum.map(&(&1 <> ":*"))
+ |> Enum.join(" | ")
+
+ from(
+ u in query,
+ select_merge: %{
+ search_type: ^0,
+ search_rank:
+ fragment(
+ """
+ ts_rank_cd(
+ setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
+ setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'),
+ to_tsquery('simple', ?),
+ 32
+ )
+ """,
+ u.nickname,
+ u.name,
+ ^processed_query
+ )
+ },
+ where:
+ fragment(
+ """
+ (setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
+ setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')) @@ to_tsquery('simple', ?)
+ """,
+ u.nickname,
+ u.name,
+ ^processed_query
+ )
+ )
+ |> User.restrict_deactivated()
+ end
+
+ defp trigram_search_subquery(term) do
+ from(
+ u in User,
+ select_merge: %{
+ # ^1 gives 'Postgrex expected a binary, got 1' for some weird reason
+ search_type: fragment("?", 1),
+ search_rank:
+ fragment(
+ "similarity(?, trim(? || ' ' || coalesce(?, '')))",
+ ^term,
+ u.nickname,
+ u.name
+ )
+ },
+ where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^term)
+ )
+ |> User.restrict_deactivated()
+ end
+end
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -14,7 +14,6 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
alias Pleroma.HTTP
alias Pleroma.Notification
alias Pleroma.Object
- alias Pleroma.Object.Fetcher
alias Pleroma.Pagination
alias Pleroma.Repo
alias Pleroma.ScheduledActivity
@@ -1125,64 +1124,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
end
end
- def status_search_query_with_gin(q, query) do
- from([a, o] in q,
- where:
- fragment(
- "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
- o.data,
- ^query
- ),
- order_by: [desc: :id]
- )
- end
-
- def status_search_query_with_rum(q, query) do
- from([a, o] in q,
- where:
- fragment(
- "? @@ plainto_tsquery('english', ?)",
- o.fts_content,
- ^query
- ),
- order_by: [fragment("? <=> now()::date", o.inserted_at)]
- )
- end
-
- def status_search(user, query) do
- fetched =
- if Regex.match?(~r/https?:/, query) do
- with {:ok, object} <- Fetcher.fetch_object_from_id(query),
- %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
- true <- Visibility.visible_for_user?(activity, user) do
- [activity]
- else
- _e -> []
- end
- end || []
-
- q =
- from([a, o] in Activity.with_preloaded_object(Activity),
- where: fragment("?->>'type' = 'Create'", a.data),
- where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
- limit: 40
- )
-
- q =
- if Pleroma.Config.get([:database, :rum_enabled]) do
- status_search_query_with_rum(q, query)
- else
- status_search_query_with_gin(q, query)
- end
-
- Repo.all(q) ++ fetched
- end
-
def search2(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
accounts = User.search(query, resolve: params["resolve"] == "true", for_user: user)
-
- statuses = status_search(user, query)
-
+ statuses = Activity.search(user, query)
tags_path = Web.base_url() <> "/tag/"
tags =
@@ -1205,8 +1149,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
accounts = User.search(query, resolve: params["resolve"] == "true", for_user: user)
-
- statuses = status_search(user, query)
+ statuses = Activity.search(user, query)
tags =
query
diff --git a/priv/repo/migrations/20190603115238_add_index_on_activities_local.exs b/priv/repo/migrations/20190603115238_add_index_on_activities_local.exs
@@ -0,0 +1,7 @@
+defmodule Pleroma.Repo.Migrations.AddIndexOnActivitiesLocal do
+ use Ecto.Migration
+
+ def change do
+ create(index("activities", [:local]))
+ end
+end
diff --git a/test/activity_test.exs b/test/activity_test.exs
@@ -99,4 +99,44 @@ defmodule Pleroma.ActivityTest do
assert Activity.get_bookmark(queried_activity, user) == bookmark
end
end
+
+ describe "search" do
+ setup do
+ Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+
+ user = insert(:user)
+
+ params = %{
+ "@context" => "https://www.w3.org/ns/activitystreams",
+ "actor" => "http://mastodon.example.org/users/admin",
+ "type" => "Create",
+ "id" => "http://mastodon.example.org/users/admin/activities/1",
+ "object" => %{
+ "type" => "Note",
+ "content" => "find me!",
+ "id" => "http://mastodon.example.org/users/admin/objects/1",
+ "attributedTo" => "http://mastodon.example.org/users/admin"
+ },
+ "to" => ["https://www.w3.org/ns/activitystreams#Public"]
+ }
+
+ {:ok, local_activity} = Pleroma.Web.CommonAPI.post(user, %{"status" => "find me!"})
+ {:ok, remote_activity} = Pleroma.Web.Federator.incoming_ap_doc(params)
+ %{local_activity: local_activity, remote_activity: remote_activity, user: user}
+ end
+
+ test "find local and remote statuses for authenticated users", %{
+ local_activity: local_activity,
+ remote_activity: remote_activity,
+ user: user
+ } do
+ activities = Enum.sort_by(Activity.search(user, "find me"), & &1.id)
+
+ assert [^local_activity, ^remote_activity] = activities
+ end
+
+ test "find only local statuses for unauthenticated users", %{local_activity: local_activity} do
+ assert [^local_activity] = Activity.search(nil, "find me")
+ end
+ end
end
diff --git a/test/user_test.exs b/test/user_test.exs
@@ -1055,7 +1055,7 @@ defmodule Pleroma.UserTest do
u3 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social"})
u4 = insert(:user, %{nickname: "lain@pleroma.soykaf.com"})
- assert [u4.id, u3.id, u1.id] == Enum.map(User.search("lain@ple"), & &1.id)
+ assert [u4.id, u3.id, u1.id] == Enum.map(User.search("lain@ple", for_user: u1), & &1.id)
end
test "finds users, handling misspelled requests" do
@@ -1077,6 +1077,28 @@ defmodule Pleroma.UserTest do
Enum.map(User.search("doe", resolve: false, for_user: u1), & &1.id) == []
end
+ test "find local and remote statuses for authenticated users" do
+ u1 = insert(:user, %{name: "lain"})
+ u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false})
+ u3 = insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false})
+
+ results =
+ "lain"
+ |> User.search(for_user: u1)
+ |> Enum.map(& &1.id)
+ |> Enum.sort()
+
+ assert [u1.id, u2.id, u3.id] == results
+ end
+
+ test "find only local statuses for unauthenticated users" do
+ %{id: id} = insert(:user, %{name: "lain"})
+ insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false})
+ insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false})
+
+ assert [%{id: ^id}] = User.search("lain")
+ end
+
test "finds a user whose name is nil" do
_user = insert(:user, %{name: "notamatch", nickname: "testuser@pleroma.amplifie.red"})
user_two = insert(:user, %{name: nil, nickname: "lain@pleroma.soykaf.com"})
@@ -1097,7 +1119,11 @@ defmodule Pleroma.UserTest do
end
test "works with URIs" do
- results = User.search("http://mastodon.example.org/users/admin", resolve: true)
+ user = insert(:user)
+
+ results =
+ User.search("http://mastodon.example.org/users/admin", resolve: true, for_user: user)
+
result = results |> List.first()
user = User.get_cached_by_ap_id("http://mastodon.example.org/users/admin")
diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs
@@ -2173,8 +2173,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do
end
test "search fetches remote accounts", %{conn: conn} do
+ user = insert(:user)
+
conn =
conn
+ |> assign(:user, user)
|> get("/api/v1/search", %{"q" => "shp@social.heldscal.la", "resolve" => "true"})
assert results = json_response(conn, 200)
