commit: 34b099fffa64c20d42c417abb06e0e32a5ad296a
parent: f06444a2892a2c3d6e764488958e2f3cf259be29
Author: lain <lain@soykaf.club>
Date: Mon, 17 Aug 2020 13:05:20 +0000
Merge branch 'issue/1936' into 'develop'
[#1936] Ability to search for banned MediaProxy URLs 
See merge request pleroma/pleroma!2873
Diffstat:
6 files changed, 121 insertions(+), 79 deletions(-)
diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md
@@ -1266,11 +1266,14 @@ Loads json generated from `config/descriptions.exs`.
- Params:
- *optional* `page`: **integer** page number
- *optional* `page_size`: **integer** number of log entries per page (default is `50`)
+- *optional* `query`: **string** search term
- Response:
``` json
{
+ "page_size": integer,
+ "count": integer,
"urls": [
"http://example.com/media/a688346.jpg",
"http://example.com/media/fb1f4d.jpg"
@@ -1290,12 +1293,7 @@ Loads json generated from `config/descriptions.exs`.
- Response:
``` json
-{
- "urls": [
- "http://example.com/media/a688346.jpg",
- "http://example.com/media/fb1f4d.jpg"
- ]
-}
+{ }
```
@@ -1311,11 +1309,6 @@ Loads json generated from `config/descriptions.exs`.
- Response:
``` json
-{
- "urls": [
- "http://example.com/media/a688346.jpg",
- "http://example.com/media/fb1f4d.jpg"
- ]
-}
+{ }
```
diff --git a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex
@@ -26,29 +26,40 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation
def index(%{assigns: %{user: _}} = conn, params) do
- cursor =
- :banned_urls_cache
- |> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}])
- |> :qlc.cursor()
+ entries = fetch_entries(params)
+ urls = paginate_entries(entries, params.page, params.page_size)
+
+ render(conn, "index.json",
+ urls: urls,
+ page_size: params.page_size,
+ count: length(entries)
+ )
+ end
+
+ defp fetch_entries(params) do
+ MediaProxy.cache_table()
+ |> Cachex.stream!(Cachex.Query.create(true, :key))
+ |> filter_entries(params[:query])
+ end
- urls =
- case params.page do
- 1 ->
- :qlc.next_answers(cursor, params.page_size)
+ defp filter_entries(stream, query) when is_binary(query) do
+ regex = ~r/#{query}/i
- _ ->
- :qlc.next_answers(cursor, (params.page - 1) * params.page_size)
- :qlc.next_answers(cursor, params.page_size)
- end
+ stream
+ |> Enum.filter(fn url -> String.match?(url, regex) end)
+ |> Enum.to_list()
+ end
- :qlc.delete_cursor(cursor)
+ defp filter_entries(stream, _), do: Enum.to_list(stream)
- render(conn, "index.json", urls: urls)
+ defp paginate_entries(entries, page, page_size) do
+ offset = page_size * (page - 1)
+ Enum.slice(entries, offset, page_size)
end
def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do
MediaProxy.remove_from_banned_urls(urls)
- render(conn, "index.json", urls: urls)
+ json(conn, %{})
end
def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do
@@ -58,6 +69,6 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
MediaProxy.put_in_banned_urls(urls)
end
- render(conn, "index.json", urls: urls)
+ json(conn, %{})
end
end
diff --git a/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex b/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex
@@ -5,7 +5,11 @@
defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do
use Pleroma.Web, :view
- def render("index.json", %{urls: urls}) do
- %{urls: urls}
+ def render("index.json", %{urls: urls, page_size: page_size, count: count}) do
+ %{
+ urls: urls,
+ count: count,
+ page_size: page_size
+ }
end
end
diff --git a/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex
@@ -22,6 +22,12 @@ defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do
security: [%{"oAuth" => ["read:media_proxy_caches"]}],
parameters: [
Operation.parameter(
+ :query,
+ :query,
+ %Schema{type: :string, default: nil},
+ "Page"
+ ),
+ Operation.parameter(
:page,
:query,
%Schema{type: :integer, default: 1},
@@ -36,7 +42,26 @@ defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do
| admin_api_params()
],
responses: %{
- 200 => success_response()
+ 200 =>
+ Operation.response(
+ "Array of banned MediaProxy URLs in Cachex",
+ "application/json",
+ %Schema{
+ type: :object,
+ properties: %{
+ count: %Schema{type: :integer},
+ page_size: %Schema{type: :integer},
+ urls: %Schema{
+ type: :array,
+ items: %Schema{
+ type: :string,
+ format: :uri,
+ description: "MediaProxy URLs"
+ }
+ }
+ }
+ }
+ )
}
}
end
@@ -61,7 +86,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do
required: true
),
responses: %{
- 200 => success_response(),
+ 200 => empty_object_response(),
400 => Operation.response("Error", "application/json", ApiError)
}
}
@@ -88,25 +113,9 @@ defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do
required: true
),
responses: %{
- 200 => success_response(),
+ 200 => empty_object_response(),
400 => Operation.response("Error", "application/json", ApiError)
}
}
end
-
- defp success_response do
- Operation.response("Array of banned MediaProxy URLs in Cachex", "application/json", %Schema{
- type: :object,
- properties: %{
- urls: %Schema{
- type: :array,
- items: %Schema{
- type: :string,
- format: :uri,
- description: "MediaProxy URLs"
- }
- }
- }
- })
- end
end
diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -9,28 +9,31 @@ defmodule Pleroma.Web.MediaProxy do
alias Pleroma.Web.MediaProxy.Invalidation
@base64_opts [padding: false]
+ @cache_table :banned_urls_cache
+
+ def cache_table, do: @cache_table
@spec in_banned_urls(String.t()) :: boolean()
- def in_banned_urls(url), do: elem(Cachex.exists?(:banned_urls_cache, url(url)), 1)
+ def in_banned_urls(url), do: elem(Cachex.exists?(@cache_table, url(url)), 1)
def remove_from_banned_urls(urls) when is_list(urls) do
- Cachex.execute!(:banned_urls_cache, fn cache ->
+ Cachex.execute!(@cache_table, fn cache ->
Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1))
end)
end
def remove_from_banned_urls(url) when is_binary(url) do
- Cachex.del(:banned_urls_cache, url(url))
+ Cachex.del(@cache_table, url(url))
end
def put_in_banned_urls(urls) when is_list(urls) do
- Cachex.execute!(:banned_urls_cache, fn cache ->
+ Cachex.execute!(@cache_table, fn cache ->
Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true))
end)
end
def put_in_banned_urls(url) when is_binary(url) do
- Cachex.put(:banned_urls_cache, url(url), true)
+ Cachex.put(@cache_table, url(url), true)
end
def url(url) when is_nil(url) or url == "", do: nil
diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs
@@ -48,6 +48,9 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2")
|> json_response_and_validate_schema(200)
+ assert response["page_size"] == 2
+ assert response["count"] == 5
+
assert response["urls"] == [
"http://localhost:4001/media/fb1f4d.jpg",
"http://localhost:4001/media/a688346.jpg"
@@ -63,6 +66,9 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
"http://localhost:4001/media/tb13f47.jpg"
]
+ assert response["page_size"] == 2
+ assert response["count"] == 5
+
response =
conn
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3")
@@ -70,6 +76,30 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"]
end
+
+ test "search banned MediaProxy URLs", %{conn: conn} do
+ MediaProxy.put_in_banned_urls([
+ "http://localhost:4001/media/a688346.jpg",
+ "http://localhost:4001/media/ff44b1f4d.jpg"
+ ])
+
+ MediaProxy.put_in_banned_urls("http://localhost:4001/media/gb1f44.jpg")
+ MediaProxy.put_in_banned_urls("http://localhost:4001/media/tb13f47.jpg")
+ MediaProxy.put_in_banned_urls("http://localhost:4001/media/wb1f46.jpg")
+
+ response =
+ conn
+ |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&query=F44")
+ |> json_response_and_validate_schema(200)
+
+ assert response["urls"] == [
+ "http://localhost:4001/media/gb1f44.jpg",
+ "http://localhost:4001/media/ff44b1f4d.jpg"
+ ]
+
+ assert response["page_size"] == 2
+ assert response["count"] == 2
+ end
end
describe "POST /api/pleroma/admin/media_proxy_caches/delete" do
@@ -79,15 +109,13 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
"http://localhost:4001/media/fb1f4d.jpg"
])
- response =
- conn
- |> put_req_header("content-type", "application/json")
- |> post("/api/pleroma/admin/media_proxy_caches/delete", %{
- urls: ["http://localhost:4001/media/a688346.jpg"]
- })
- |> json_response_and_validate_schema(200)
+ conn
+ |> put_req_header("content-type", "application/json")
+ |> post("/api/pleroma/admin/media_proxy_caches/delete", %{
+ urls: ["http://localhost:4001/media/a688346.jpg"]
+ })
+ |> json_response_and_validate_schema(200)
- assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"]
refute MediaProxy.in_banned_urls("http://localhost:4001/media/a688346.jpg")
assert MediaProxy.in_banned_urls("http://localhost:4001/media/fb1f4d.jpg")
end
@@ -106,13 +134,10 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
purge: fn _, _ -> {"ok", 0} end
]}
] do
- response =
- conn
- |> put_req_header("content-type", "application/json")
- |> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false})
- |> json_response_and_validate_schema(200)
-
- assert response["urls"] == urls
+ conn
+ |> put_req_header("content-type", "application/json")
+ |> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false})
+ |> json_response_and_validate_schema(200)
refute MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
refute MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
@@ -126,16 +151,13 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
]
with_mocks [{MediaProxy.Invalidation.Script, [], [purge: fn _, _ -> {"ok", 0} end]}] do
- response =
- conn
- |> put_req_header("content-type", "application/json")
- |> post("/api/pleroma/admin/media_proxy_caches/purge", %{
- urls: urls,
- ban: true
- })
- |> json_response_and_validate_schema(200)
-
- assert response["urls"] == urls
+ conn
+ |> put_req_header("content-type", "application/json")
+ |> post(
+ "/api/pleroma/admin/media_proxy_caches/purge",
+ %{urls: urls, ban: true}
+ )
+ |> json_response_and_validate_schema(200)
assert MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
assert MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
