Merge branch 'issue/1936' into 'develop'

[#1936] Ability to search for banned MediaProxy URLs 

See merge request pleroma/pleroma!2873

docs/API/admin_api.md

@@ -1266,11 +1266,14 @@ Loads json generated from `config/descriptions.exs`.
 - Params:
 - *optional* `page`: **integer** page number
 - *optional* `page_size`: **integer** number of log entries per page (default is `50`)
+- *optional* `query`: **string** search term
 
 - Response:
 
 ``` json
 {
+  "page_size": integer,
+  "count": integer,
   "urls": [
     "http://example.com/media/a688346.jpg",
     "http://example.com/media/fb1f4d.jpg"
@@ -1290,12 +1293,7 @@ Loads json generated from `config/descriptions.exs`.
 - Response:
 
 ``` json
-{
+{ }
-  "urls": [
-    "http://example.com/media/a688346.jpg",
-    "http://example.com/media/fb1f4d.jpg"
-  ]
-}
 
 ```
 
@@ -1311,11 +1309,6 @@ Loads json generated from `config/descriptions.exs`.
 - Response:
 
 ``` json
-{
+{ }
-  "urls": [
-    "http://example.com/media/a688346.jpg",
-    "http://example.com/media/fb1f4d.jpg"
-  ]
-}
 
 ```

lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex

@@ -26,29 +26,40 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
   defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation
 
   def index(%{assigns: %{user: _}} = conn, params) do
-    cursor =
+    entries = fetch_entries(params)
-      :banned_urls_cache
+    urls = paginate_entries(entries, params.page, params.page_size)
-      |> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}])
-      |> :qlc.cursor()
 
-    urls =
+    render(conn, "index.json",
-      case params.page do
+      urls: urls,
-        1 ->
+      page_size: params.page_size,
-          :qlc.next_answers(cursor, params.page_size)
+      count: length(entries)
-
+    )
-        _ ->
-          :qlc.next_answers(cursor, (params.page - 1) * params.page_size)
-          :qlc.next_answers(cursor, params.page_size)
   end
 
-    :qlc.delete_cursor(cursor)
+  defp fetch_entries(params) do
+    MediaProxy.cache_table()
+    |> Cachex.stream!(Cachex.Query.create(true, :key))
+    |> filter_entries(params[:query])
+  end
 
-    render(conn, "index.json", urls: urls)
+  defp filter_entries(stream, query) when is_binary(query) do
+    regex = ~r/#{query}/i
+
+    stream
+    |> Enum.filter(fn url -> String.match?(url, regex) end)
+    |> Enum.to_list()
+  end
+
+  defp filter_entries(stream, _), do: Enum.to_list(stream)
+
+  defp paginate_entries(entries, page, page_size) do
+    offset = page_size * (page - 1)
+    Enum.slice(entries, offset, page_size)
   end
 
   def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do
     MediaProxy.remove_from_banned_urls(urls)
-    render(conn, "index.json", urls: urls)
+    json(conn, %{})
   end
 
   def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do
@@ -58,6 +69,6 @@ def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _
       MediaProxy.put_in_banned_urls(urls)
     end
 
-    render(conn, "index.json", urls: urls)
+    json(conn, %{})
   end
 end

lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex

@@ -5,7 +5,11 @@
 defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do
   use Pleroma.Web, :view
 
-  def render("index.json", %{urls: urls}) do
+  def render("index.json", %{urls: urls, page_size: page_size, count: count}) do
-    %{urls: urls}
+    %{
+      urls: urls,
+      count: count,
+      page_size: page_size
+    }
   end
 end

lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex

@@ -21,6 +21,12 @@ def index_operation do
       operationId: "AdminAPI.MediaProxyCacheController.index",
       security: [%{"oAuth" => ["read:media_proxy_caches"]}],
       parameters: [
+        Operation.parameter(
+          :query,
+          :query,
+          %Schema{type: :string, default: nil},
+          "Page"
+        ),
         Operation.parameter(
           :page,
           :query,
@@ -36,7 +42,26 @@ def index_operation do
         | admin_api_params()
       ],
       responses: %{
-        200 => success_response()
+        200 =>
+          Operation.response(
+            "Array of banned MediaProxy URLs in Cachex",
+            "application/json",
+            %Schema{
+              type: :object,
+              properties: %{
+                count: %Schema{type: :integer},
+                page_size: %Schema{type: :integer},
+                urls: %Schema{
+                  type: :array,
+                  items: %Schema{
+                    type: :string,
+                    format: :uri,
+                    description: "MediaProxy URLs"
+                  }
+                }
+              }
+            }
+          )
       }
     }
   end
@@ -61,7 +86,7 @@ def delete_operation do
           required: true
         ),
       responses: %{
-        200 => success_response(),
+        200 => empty_object_response(),
         400 => Operation.response("Error", "application/json", ApiError)
       }
     }
@@ -88,25 +113,9 @@ def purge_operation do
           required: true
         ),
       responses: %{
-        200 => success_response(),
+        200 => empty_object_response(),
         400 => Operation.response("Error", "application/json", ApiError)
       }
     }
   end
-
-  defp success_response do
-    Operation.response("Array of banned MediaProxy URLs in Cachex", "application/json", %Schema{
-      type: :object,
-      properties: %{
-        urls: %Schema{
-          type: :array,
-          items: %Schema{
-            type: :string,
-            format: :uri,
-            description: "MediaProxy URLs"
-          }
-        }
-      }
-    })
-  end
 end

lib/pleroma/web/media_proxy/media_proxy.ex

@@ -9,28 +9,31 @@ defmodule Pleroma.Web.MediaProxy do
   alias Pleroma.Web.MediaProxy.Invalidation
 
   @base64_opts [padding: false]
+  @cache_table :banned_urls_cache
+
+  def cache_table, do: @cache_table
 
   @spec in_banned_urls(String.t()) :: boolean()
-  def in_banned_urls(url), do: elem(Cachex.exists?(:banned_urls_cache, url(url)), 1)
+  def in_banned_urls(url), do: elem(Cachex.exists?(@cache_table, url(url)), 1)
 
   def remove_from_banned_urls(urls) when is_list(urls) do
-    Cachex.execute!(:banned_urls_cache, fn cache ->
+    Cachex.execute!(@cache_table, fn cache ->
       Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1))
     end)
   end
 
   def remove_from_banned_urls(url) when is_binary(url) do
-    Cachex.del(:banned_urls_cache, url(url))
+    Cachex.del(@cache_table, url(url))
   end
 
   def put_in_banned_urls(urls) when is_list(urls) do
-    Cachex.execute!(:banned_urls_cache, fn cache ->
+    Cachex.execute!(@cache_table, fn cache ->
       Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true))
     end)
   end
 
   def put_in_banned_urls(url) when is_binary(url) do
-    Cachex.put(:banned_urls_cache, url(url), true)
+    Cachex.put(@cache_table, url(url), true)
   end
 
   def url(url) when is_nil(url) or url == "", do: nil

test/web/admin_api/controllers/media_proxy_cache_controller_test.exs

@@ -48,6 +48,9 @@ test "shows banned MediaProxy URLs", %{conn: conn} do
         |> get("/api/pleroma/admin/media_proxy_caches?page_size=2")
         |> json_response_and_validate_schema(200)
 
+      assert response["page_size"] == 2
+      assert response["count"] == 5
+
       assert response["urls"] == [
                "http://localhost:4001/media/fb1f4d.jpg",
                "http://localhost:4001/media/a688346.jpg"
@@ -63,6 +66,9 @@ test "shows banned MediaProxy URLs", %{conn: conn} do
                "http://localhost:4001/media/tb13f47.jpg"
              ]
 
+      assert response["page_size"] == 2
+      assert response["count"] == 5
+
       response =
         conn
         |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3")
@@ -70,6 +76,30 @@ test "shows banned MediaProxy URLs", %{conn: conn} do
 
       assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"]
     end
+
+    test "search banned MediaProxy URLs", %{conn: conn} do
+      MediaProxy.put_in_banned_urls([
+        "http://localhost:4001/media/a688346.jpg",
+        "http://localhost:4001/media/ff44b1f4d.jpg"
+      ])
+
+      MediaProxy.put_in_banned_urls("http://localhost:4001/media/gb1f44.jpg")
+      MediaProxy.put_in_banned_urls("http://localhost:4001/media/tb13f47.jpg")
+      MediaProxy.put_in_banned_urls("http://localhost:4001/media/wb1f46.jpg")
+
+      response =
+        conn
+        |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&query=F44")
+        |> json_response_and_validate_schema(200)
+
+      assert response["urls"] == [
+               "http://localhost:4001/media/gb1f44.jpg",
+               "http://localhost:4001/media/ff44b1f4d.jpg"
+             ]
+
+      assert response["page_size"] == 2
+      assert response["count"] == 2
+    end
   end
 
   describe "POST /api/pleroma/admin/media_proxy_caches/delete" do
@@ -79,7 +109,6 @@ test "deleted MediaProxy URLs from banned", %{conn: conn} do
         "http://localhost:4001/media/fb1f4d.jpg"
       ])
 
-      response =
       conn
       |> put_req_header("content-type", "application/json")
       |> post("/api/pleroma/admin/media_proxy_caches/delete", %{
@@ -87,7 +116,6 @@ test "deleted MediaProxy URLs from banned", %{conn: conn} do
       })
       |> json_response_and_validate_schema(200)
 
-      assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"]
       refute MediaProxy.in_banned_urls("http://localhost:4001/media/a688346.jpg")
       assert MediaProxy.in_banned_urls("http://localhost:4001/media/fb1f4d.jpg")
     end
@@ -106,14 +134,11 @@ test "perform invalidates cache of MediaProxy", %{conn: conn} do
            purge: fn _, _ -> {"ok", 0} end
          ]}
       ] do
-        response =
         conn
         |> put_req_header("content-type", "application/json")
         |> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false})
         |> json_response_and_validate_schema(200)
 
-        assert response["urls"] == urls
-
         refute MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
         refute MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
       end
@@ -126,17 +151,14 @@ test "perform invalidates cache of MediaProxy and adds url to banned", %{conn: c
       ]
 
       with_mocks [{MediaProxy.Invalidation.Script, [], [purge: fn _, _ -> {"ok", 0} end]}] do
-        response =
         conn
         |> put_req_header("content-type", "application/json")
-          |> post("/api/pleroma/admin/media_proxy_caches/purge", %{
+        |> post(
-            urls: urls,
+          "/api/pleroma/admin/media_proxy_caches/purge",
-            ban: true
+          %{urls: urls, ban: true}
-          })
+        )
         |> json_response_and_validate_schema(200)
 
-        assert response["urls"] == urls
-
         assert MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
         assert MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
       end