From 345ac512e43fbb127c45552690741088d465d31d Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 11 Aug 2020 10:28:35 +0300 Subject: [PATCH] added paginate+search for admin/MediaProxy URLs --- .../media_proxy_cache_controller.ex | 41 ++++++----- .../admin_api/views/media_proxy_cache_view.ex | 8 ++- .../admin/media_proxy_cache_operation.ex | 47 +++++++----- lib/pleroma/web/media_proxy/media_proxy.ex | 13 ++-- .../media_proxy_cache_controller_test.exs | 72 ++++++++++++------- 5 files changed, 114 insertions(+), 67 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex index e2759d59f..76d3af4ef 100644 --- a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex @@ -26,29 +26,38 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation def index(%{assigns: %{user: _}} = conn, params) do - cursor = - :banned_urls_cache - |> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}]) - |> :qlc.cursor() + entries = fetch_entries(params) + urls = paginate_entries(entries, params.page, params.page_size) - urls = - case params.page do - 1 -> - :qlc.next_answers(cursor, params.page_size) + render(conn, "index.json", + urls: urls, + page_size: params.page_size, + count: length(entries) + ) + end - _ -> - :qlc.next_answers(cursor, (params.page - 1) * params.page_size) - :qlc.next_answers(cursor, params.page_size) - end + defp fetch_entries(params) do + MediaProxy.cache_table() + |> Cachex.export!() + |> filter_urls(params[:query]) + end - :qlc.delete_cursor(cursor) + defp filter_urls(entries, query) when is_binary(query) do + for {_, url, _, _, _} <- entries, String.contains?(url, query), do: url + end - render(conn, "index.json", urls: urls) + defp filter_urls(entries, _) do + Enum.map(entries, fn {_, url, _, _, _} -> url end) + end + + defp paginate_entries(entries, page, page_size) do + offset = page_size * (page - 1) + Enum.slice(entries, offset, page_size) end def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do MediaProxy.remove_from_banned_urls(urls) - render(conn, "index.json", urls: urls) + json(conn, %{}) end def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do @@ -58,6 +67,6 @@ def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _ MediaProxy.put_in_banned_urls(urls) end - render(conn, "index.json", urls: urls) + json(conn, %{}) end end diff --git a/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex b/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex index c97400beb..a803bda0b 100644 --- a/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex +++ b/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex @@ -5,7 +5,11 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do use Pleroma.Web, :view - def render("index.json", %{urls: urls}) do - %{urls: urls} + def render("index.json", %{urls: urls, page_size: page_size, count: count}) do + %{ + urls: urls, + count: count, + page_size: page_size + } end end diff --git a/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex index 20d033f66..ab45d6633 100644 --- a/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex @@ -21,6 +21,12 @@ def index_operation do operationId: "AdminAPI.MediaProxyCacheController.index", security: [%{"oAuth" => ["read:media_proxy_caches"]}], parameters: [ + Operation.parameter( + :query, + :query, + %Schema{type: :string, default: nil}, + "Page" + ), Operation.parameter( :page, :query, @@ -36,7 +42,26 @@ def index_operation do | admin_api_params() ], responses: %{ - 200 => success_response() + 200 => + Operation.response( + "Array of banned MediaProxy URLs in Cachex", + "application/json", + %Schema{ + type: :object, + properties: %{ + count: %Schema{type: :integer}, + page_size: %Schema{type: :integer}, + urls: %Schema{ + type: :array, + items: %Schema{ + type: :string, + format: :uri, + description: "MediaProxy URLs" + } + } + } + } + ) } } end @@ -61,7 +86,7 @@ def delete_operation do required: true ), responses: %{ - 200 => success_response(), + 200 => empty_object_response(), 400 => Operation.response("Error", "application/json", ApiError) } } @@ -88,25 +113,9 @@ def purge_operation do required: true ), responses: %{ - 200 => success_response(), + 200 => empty_object_response(), 400 => Operation.response("Error", "application/json", ApiError) } } end - - defp success_response do - Operation.response("Array of banned MediaProxy URLs in Cachex", "application/json", %Schema{ - type: :object, - properties: %{ - urls: %Schema{ - type: :array, - items: %Schema{ - type: :string, - format: :uri, - description: "MediaProxy URLs" - } - } - } - }) - end end diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index dfbfcea6b..e18dd8224 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -9,28 +9,31 @@ defmodule Pleroma.Web.MediaProxy do alias Pleroma.Web.MediaProxy.Invalidation @base64_opts [padding: false] + @cache_table :banned_urls_cache + + def cache_table, do: @cache_table @spec in_banned_urls(String.t()) :: boolean() - def in_banned_urls(url), do: elem(Cachex.exists?(:banned_urls_cache, url(url)), 1) + def in_banned_urls(url), do: elem(Cachex.exists?(@cache_table, url(url)), 1) def remove_from_banned_urls(urls) when is_list(urls) do - Cachex.execute!(:banned_urls_cache, fn cache -> + Cachex.execute!(@cache_table, fn cache -> Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1)) end) end def remove_from_banned_urls(url) when is_binary(url) do - Cachex.del(:banned_urls_cache, url(url)) + Cachex.del(@cache_table, url(url)) end def put_in_banned_urls(urls) when is_list(urls) do - Cachex.execute!(:banned_urls_cache, fn cache -> + Cachex.execute!(@cache_table, fn cache -> Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true)) end) end def put_in_banned_urls(url) when is_binary(url) do - Cachex.put(:banned_urls_cache, url(url), true) + Cachex.put(@cache_table, url(url), true) end def url(url) when is_nil(url) or url == "", do: nil diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs index 5ab6cb78a..3cf98d7c7 100644 --- a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs +++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs @@ -48,6 +48,9 @@ test "shows banned MediaProxy URLs", %{conn: conn} do |> get("/api/pleroma/admin/media_proxy_caches?page_size=2") |> json_response_and_validate_schema(200) + assert response["page_size"] == 2 + assert response["count"] == 5 + assert response["urls"] == [ "http://localhost:4001/media/fb1f4d.jpg", "http://localhost:4001/media/a688346.jpg" @@ -63,6 +66,9 @@ test "shows banned MediaProxy URLs", %{conn: conn} do "http://localhost:4001/media/tb13f47.jpg" ] + assert response["page_size"] == 2 + assert response["count"] == 5 + response = conn |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3") @@ -70,6 +76,30 @@ test "shows banned MediaProxy URLs", %{conn: conn} do assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"] end + + test "search banned MediaProxy URLs", %{conn: conn} do + MediaProxy.put_in_banned_urls([ + "http://localhost:4001/media/a688346.jpg", + "http://localhost:4001/media/ff44b1f4d.jpg" + ]) + + MediaProxy.put_in_banned_urls("http://localhost:4001/media/gb1f44.jpg") + MediaProxy.put_in_banned_urls("http://localhost:4001/media/tb13f47.jpg") + MediaProxy.put_in_banned_urls("http://localhost:4001/media/wb1f46.jpg") + + response = + conn + |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&query=f44") + |> json_response_and_validate_schema(200) + + assert response["urls"] == [ + "http://localhost:4001/media/gb1f44.jpg", + "http://localhost:4001/media/ff44b1f4d.jpg" + ] + + assert response["page_size"] == 2 + assert response["count"] == 2 + end end describe "POST /api/pleroma/admin/media_proxy_caches/delete" do @@ -79,15 +109,13 @@ test "deleted MediaProxy URLs from banned", %{conn: conn} do "http://localhost:4001/media/fb1f4d.jpg" ]) - response = - conn - |> put_req_header("content-type", "application/json") - |> post("/api/pleroma/admin/media_proxy_caches/delete", %{ - urls: ["http://localhost:4001/media/a688346.jpg"] - }) - |> json_response_and_validate_schema(200) + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/media_proxy_caches/delete", %{ + urls: ["http://localhost:4001/media/a688346.jpg"] + }) + |> json_response_and_validate_schema(200) - assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"] refute MediaProxy.in_banned_urls("http://localhost:4001/media/a688346.jpg") assert MediaProxy.in_banned_urls("http://localhost:4001/media/fb1f4d.jpg") end @@ -106,13 +134,10 @@ test "perform invalidates cache of MediaProxy", %{conn: conn} do purge: fn _, _ -> {"ok", 0} end ]} ] do - response = - conn - |> put_req_header("content-type", "application/json") - |> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false}) - |> json_response_and_validate_schema(200) - - assert response["urls"] == urls + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false}) + |> json_response_and_validate_schema(200) refute MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg") refute MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg") @@ -126,16 +151,13 @@ test "perform invalidates cache of MediaProxy and adds url to banned", %{conn: c ] with_mocks [{MediaProxy.Invalidation.Script, [], [purge: fn _, _ -> {"ok", 0} end]}] do - response = - conn - |> put_req_header("content-type", "application/json") - |> post("/api/pleroma/admin/media_proxy_caches/purge", %{ - urls: urls, - ban: true - }) - |> json_response_and_validate_schema(200) - - assert response["urls"] == urls + conn + |> put_req_header("content-type", "application/json") + |> post( + "/api/pleroma/admin/media_proxy_caches/purge", + %{urls: urls, ban: true} + ) + |> json_response_and_validate_schema(200) assert MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg") assert MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")