From f9dcf15ecb684b4b802d731a216448c76913d462 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 12 Jun 2020 14:49:54 +0300 Subject: [PATCH 01/21] added admin api for MediaProxy cache invalidation --- .../media_proxy_cache_controller.ex | 38 ++++++ .../admin_api/views/media_proxy_cache_view.ex | 11 ++ .../admin/media_proxy_cache_operation.ex | 109 ++++++++++++++++++ lib/pleroma/web/router.ex | 4 + .../media_proxy_cache_controller_test.exs | 66 +++++++++++ 5 files changed, 228 insertions(+) create mode 100644 lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex create mode 100644 lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex create mode 100644 lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex create mode 100644 test/web/admin_api/controllers/media_proxy_cache_controller_test.exs diff --git a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex new file mode 100644 index 000000000..7b28f7c72 --- /dev/null +++ b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex @@ -0,0 +1,38 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do + use Pleroma.Web, :controller + + alias Pleroma.Plugs.OAuthScopesPlug + alias Pleroma.Web.ApiSpec.Admin, as: Spec + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug( + OAuthScopesPlug, + %{scopes: ["read:media_proxy_caches"], admin: true} when action in [:index] + ) + + plug( + OAuthScopesPlug, + %{scopes: ["write:media_proxy_caches"], admin: true} when action in [:purge, :delete] + ) + + action_fallback(Pleroma.Web.AdminAPI.FallbackController) + + defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation + + def index(%{assigns: %{user: _}} = conn, _) do + render(conn, "index.json", urls: []) + end + + def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do + render(conn, "index.json", urls: urls) + end + + def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: _ban}} = conn, _) do + render(conn, "index.json", urls: urls) + end +end diff --git a/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex b/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex new file mode 100644 index 000000000..c97400beb --- /dev/null +++ b/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex @@ -0,0 +1,11 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do + use Pleroma.Web, :view + + def render("index.json", %{urls: urls}) do + %{urls: urls} + end +end diff --git a/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex new file mode 100644 index 000000000..0358cfbad --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex @@ -0,0 +1,109 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Admin", "MediaProxyCache"], + summary: "Fetch a paginated list of all banned MediaProxy URLs in Cachex", + operationId: "AdminAPI.MediaProxyCacheController.index", + security: [%{"oAuth" => ["read:media_proxy_caches"]}], + parameters: [ + Operation.parameter( + :page, + :query, + %Schema{type: :integer, default: 1}, + "Page" + ), + Operation.parameter( + :page_size, + :query, + %Schema{type: :integer, default: 50}, + "Number of statuses to return" + ) + ], + responses: %{ + 200 => success_response() + } + } + end + + def delete_operation do + %Operation{ + tags: ["Admin", "MediaProxyCache"], + summary: "Remove a banned MediaProxy URL from Cachex", + operationId: "AdminAPI.MediaProxyCacheController.delete", + security: [%{"oAuth" => ["write:media_proxy_caches"]}], + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + required: [:urls], + properties: %{ + urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}} + } + }, + required: true + ), + responses: %{ + 200 => success_response(), + 400 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def purge_operation do + %Operation{ + tags: ["Admin", "MediaProxyCache"], + summary: "Purge and optionally ban a MediaProxy URL", + operationId: "AdminAPI.MediaProxyCacheController.purge", + security: [%{"oAuth" => ["write:media_proxy_caches"]}], + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + required: [:urls], + properties: %{ + urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}}, + ban: %Schema{type: :boolean, default: true} + } + }, + required: true + ), + responses: %{ + 200 => success_response(), + 400 => Operation.response("Error", "application/json", ApiError) + } + } + end + + defp success_response do + Operation.response("Array of banned MediaProxy URLs in Cachex", "application/json", %Schema{ + type: :object, + properties: %{ + urls: %Schema{ + type: :array, + items: %Schema{ + type: :string, + format: :uri, + description: "MediaProxy URLs" + } + } + } + }) + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 57570b672..eda74a171 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -209,6 +209,10 @@ defmodule Pleroma.Web.Router do post("/oauth_app", OAuthAppController, :create) patch("/oauth_app/:id", OAuthAppController, :update) delete("/oauth_app/:id", OAuthAppController, :delete) + + get("/media_proxy_caches", MediaProxyCacheController, :index) + post("/media_proxy_caches/delete", MediaProxyCacheController, :delete) + post("/media_proxy_caches/purge", MediaProxyCacheController, :purge) end scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs new file mode 100644 index 000000000..1b1d6bc36 --- /dev/null +++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs @@ -0,0 +1,66 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do + use Pleroma.Web.ConnCase + + import Pleroma.Factory + + setup do + admin = insert(:user, is_admin: true) + token = insert(:oauth_admin_token, user: admin) + + conn = + build_conn() + |> assign(:user, admin) + |> assign(:token, token) + + {:ok, %{admin: admin, token: token, conn: conn}} + end + + describe "GET /api/pleroma/admin/media_proxy_caches" do + test "shows banned MediaProxy URLs", %{conn: conn} do + response = + conn + |> get("/api/pleroma/admin/media_proxy_caches") + |> json_response_and_validate_schema(200) + + assert response["urls"] == [] + end + end + + describe "DELETE /api/pleroma/admin/media_proxy_caches/delete" do + test "deleted MediaProxy URLs from banned", %{conn: conn} do + response = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/media_proxy_caches/delete", %{ + urls: ["http://example.com/media/a688346.jpg", "http://example.com/media/fb1f4d.jpg"] + }) + |> json_response_and_validate_schema(200) + + assert response["urls"] == [ + "http://example.com/media/a688346.jpg", + "http://example.com/media/fb1f4d.jpg" + ] + end + end + + describe "PURGE /api/pleroma/admin/media_proxy_caches/purge" do + test "perform invalidates cache of MediaProxy", %{conn: conn} do + response = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/media_proxy_caches/purge", %{ + urls: ["http://example.com/media/a688346.jpg", "http://example.com/media/fb1f4d.jpg"] + }) + |> json_response_and_validate_schema(200) + + assert response["urls"] == [ + "http://example.com/media/a688346.jpg", + "http://example.com/media/fb1f4d.jpg" + ] + end + end +end From 2e8a236cef28c0b754aecb04a5c60c3b7655c5a6 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Sun, 14 Jun 2020 21:02:57 +0300 Subject: [PATCH 02/21] fix invalidates media url's --- config/config.exs | 7 + config/description.exs | 64 +++++++++ docs/configuration/cheatsheet.md | 6 +- installation/nginx-cache-purge.sh.example | 4 +- lib/pleroma/application.ex | 3 +- lib/pleroma/plugs/uploaded_media.ex | 16 ++- lib/pleroma/web/media_proxy/invalidation.ex | 29 ++-- .../web/media_proxy/invalidations/http.ex | 8 +- .../web/media_proxy/invalidations/script.ex | 36 ++--- lib/pleroma/web/media_proxy/media_proxy.ex | 35 ++++- .../web/media_proxy/media_proxy_controller.ex | 3 +- .../workers/attachments_cleanup_worker.ex | 133 ++++++++++-------- test/web/media_proxy/invalidation_test.exs | 65 +++++++++ .../media_proxy/invalidations/http_test.exs | 13 +- .../media_proxy/invalidations/script_test.exs | 21 ++- .../media_proxy_controller_test.exs | 17 +++ 16 files changed, 346 insertions(+), 114 deletions(-) create mode 100644 test/web/media_proxy/invalidation_test.exs diff --git a/config/config.exs b/config/config.exs index 9508ae077..e299fb8dd 100644 --- a/config/config.exs +++ b/config/config.exs @@ -406,6 +406,13 @@ config :pleroma, :media_proxy, ], whitelist: [] +config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http, + method: :purge, + headers: [], + options: [] + +config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, script_path: nil + config :pleroma, :chat, enabled: true config :phoenix, :format_encoders, json: Jason diff --git a/config/description.exs b/config/description.exs index 807c945e0..857293794 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1637,6 +1637,31 @@ config :pleroma, :config_description, [ "The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.", suggestions: ["https://example.com"] }, + %{ + key: :invalidation, + type: :keyword, + descpiption: "", + suggestions: [ + enabled: true, + provider: Pleroma.Web.MediaProxy.Invalidation.Script + ], + children: [ + %{ + key: :enabled, + type: :boolean, + description: "Enables invalidate media cache" + }, + %{ + key: :provider, + type: :module, + description: "Module which will be used to cache purge.", + suggestions: [ + Pleroma.Web.MediaProxy.Invalidation.Script, + Pleroma.Web.MediaProxy.Invalidation.Http + ] + } + ] + }, %{ key: :proxy_opts, type: :keyword, @@ -1709,6 +1734,45 @@ config :pleroma, :config_description, [ } ] }, + %{ + group: :pleroma, + key: Pleroma.Web.MediaProxy.Invalidation.Http, + type: :group, + description: "HTTP invalidate settings", + children: [ + %{ + key: :method, + type: :atom, + description: "HTTP method of request. Default: :purge" + }, + %{ + key: :headers, + type: {:list, :tuple}, + description: "HTTP headers of request.", + suggestions: [{"x-refresh", 1}] + }, + %{ + key: :options, + type: :keyword, + description: "Request options.", + suggestions: [params: %{ts: "xxx"}] + } + ] + }, + %{ + group: :pleroma, + key: Pleroma.Web.MediaProxy.Invalidation.Script, + type: :group, + description: "Script invalidate settings", + children: [ + %{ + key: :script_path, + type: :string, + description: "Path to shell script. Which will run purge cache.", + suggestions: ["./installation/nginx-cache-purge.sh.example"] + } + ] + }, %{ group: :pleroma, key: :gopher, diff --git a/docs/configuration/cheatsheet.md b/docs/configuration/cheatsheet.md index 505acb293..20bd0ed85 100644 --- a/docs/configuration/cheatsheet.md +++ b/docs/configuration/cheatsheet.md @@ -262,7 +262,7 @@ This section describe PWA manifest instance-specific values. Currently this opti #### Pleroma.Web.MediaProxy.Invalidation.Script -This strategy allow perform external bash script to purge cache. +This strategy allow perform external shell script to purge cache. Urls of attachments pass to script as arguments. * `script_path`: path to external script. @@ -278,8 +278,8 @@ config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, This strategy allow perform custom http request to purge cache. * `method`: http method. default is `purge` -* `headers`: http headers. default is empty -* `options`: request options. default is empty +* `headers`: http headers. +* `options`: request options. Example: ```elixir diff --git a/installation/nginx-cache-purge.sh.example b/installation/nginx-cache-purge.sh.example index b2915321c..5f6cbb128 100755 --- a/installation/nginx-cache-purge.sh.example +++ b/installation/nginx-cache-purge.sh.example @@ -13,7 +13,7 @@ CACHE_DIRECTORY="/tmp/pleroma-media-cache" ## $3 - (optional) the number of parallel processes to run for grep. get_cache_files() { local max_parallel=${3-16} - find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E Rl "^KEY:.*$1" | sort -u + find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E -Rl "^KEY:.*$1" | sort -u } ## Removes an item from the given cache zone. @@ -37,4 +37,4 @@ purge() { } -purge $1 +purge $@ diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 9d3d92b38..adebebc7a 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -148,7 +148,8 @@ defmodule Pleroma.Application do build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500), build_cachex("web_resp", limit: 2500), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), - build_cachex("failed_proxy_url", limit: 2500) + build_cachex("failed_proxy_url", limit: 2500), + build_cachex("deleted_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) ] end diff --git a/lib/pleroma/plugs/uploaded_media.ex b/lib/pleroma/plugs/uploaded_media.ex index 94147e0c4..2f3fde002 100644 --- a/lib/pleroma/plugs/uploaded_media.ex +++ b/lib/pleroma/plugs/uploaded_media.ex @@ -10,6 +10,8 @@ defmodule Pleroma.Plugs.UploadedMedia do import Pleroma.Web.Gettext require Logger + alias Pleroma.Web.MediaProxy + @behaviour Plug # no slashes @path "media" @@ -35,8 +37,7 @@ defmodule Pleroma.Plugs.UploadedMedia do %{query_params: %{"name" => name}} = conn -> name = String.replace(name, "\"", "\\\"") - conn - |> put_resp_header("content-disposition", "filename=\"#{name}\"") + put_resp_header(conn, "content-disposition", "filename=\"#{name}\"") conn -> conn @@ -47,7 +48,8 @@ defmodule Pleroma.Plugs.UploadedMedia do with uploader <- Keyword.fetch!(config, :uploader), proxy_remote = Keyword.get(config, :proxy_remote, false), - {:ok, get_method} <- uploader.get_file(file) do + {:ok, get_method} <- uploader.get_file(file), + false <- media_is_deleted(conn, get_method) do get_media(conn, get_method, proxy_remote, opts) else _ -> @@ -59,6 +61,14 @@ defmodule Pleroma.Plugs.UploadedMedia do def call(conn, _opts), do: conn + defp media_is_deleted(%{request_path: path} = _conn, {:static_dir, _}) do + MediaProxy.in_deleted_urls(Pleroma.Web.base_url() <> path) + end + + defp media_is_deleted(_, {:url, url}), do: MediaProxy.in_deleted_urls(url) + + defp media_is_deleted(_, _), do: false + defp get_media(conn, {:static_dir, directory}, _, opts) do static_opts = Map.get(opts, :static_plug_opts) diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index c037ff13e..83ff8589c 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -5,22 +5,33 @@ defmodule Pleroma.Web.MediaProxy.Invalidation do @moduledoc false - @callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()} + @callback purge(list(String.t()), Keyword.t()) :: {:ok, list(String.t())} | {:error, String.t()} alias Pleroma.Config + alias Pleroma.Web.MediaProxy - @spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()} + @spec enabled?() :: boolean() + def enabled?, do: Config.get([:media_proxy, :invalidation, :enabled]) + + @spec purge(list(String.t()) | String.t()) :: {:ok, list(String.t())} | {:error, String.t()} def purge(urls) do - [:media_proxy, :invalidation, :enabled] - |> Config.get() - |> do_purge(urls) + prepared_urls = prepare_urls(urls) + + if enabled?() do + do_purge(prepared_urls) + else + {:ok, prepared_urls} + end end - defp do_purge(true, urls) do + defp do_purge(urls) do provider = Config.get([:media_proxy, :invalidation, :provider]) - options = Config.get(provider) - provider.purge(urls, options) + provider.purge(urls, Config.get(provider)) end - defp do_purge(_, _), do: :ok + def prepare_urls(urls) do + urls + |> List.wrap() + |> Enum.map(&MediaProxy.url(&1)) + end end diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex index 07248df6e..3694b56e8 100644 --- a/lib/pleroma/web/media_proxy/invalidations/http.ex +++ b/lib/pleroma/web/media_proxy/invalidations/http.ex @@ -10,9 +10,9 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do @impl Pleroma.Web.MediaProxy.Invalidation def purge(urls, opts) do - method = Map.get(opts, :method, :purge) - headers = Map.get(opts, :headers, []) - options = Map.get(opts, :options, []) + method = Keyword.get(opts, :method, :purge) + headers = Keyword.get(opts, :headers, []) + options = Keyword.get(opts, :options, []) Logger.debug("Running cache purge: #{inspect(urls)}") @@ -22,7 +22,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do end end) - {:ok, "success"} + {:ok, urls} end defp do_purge(method, url, headers, options) do diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index 6be782132..d41d647bb 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -10,32 +10,34 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do require Logger @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, %{script_path: script_path} = _options) do + def purge(urls, opts) do args = urls |> List.wrap() |> Enum.uniq() |> Enum.join(" ") - path = Path.expand(script_path) - - Logger.debug("Running cache purge: #{inspect(urls)}, #{path}") - - case do_purge(path, [args]) do - {result, exit_status} when exit_status > 0 -> - Logger.error("Error while cache purge: #{inspect(result)}") - {:error, inspect(result)} - - _ -> - {:ok, "success"} - end + opts + |> Keyword.get(:script_path, nil) + |> do_purge([args]) + |> handle_result(urls) end - def purge(_, _), do: {:error, "not found script path"} - - defp do_purge(path, args) do + defp do_purge(script_path, args) when is_binary(script_path) do + path = Path.expand(script_path) + Logger.debug("Running cache purge: #{inspect(args)}, #{inspect(path)}") System.cmd(path, args) rescue - error -> {inspect(error), 1} + error -> error + end + + defp do_purge(_, _), do: {:error, "not found script path"} + + defp handle_result({_result, 0}, urls), do: {:ok, urls} + defp handle_result({:error, error}, urls), do: handle_result(error, urls) + + defp handle_result(error, _) do + Logger.error("Error while cache purge: #{inspect(error)}") + {:error, inspect(error)} end end diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index b2b524524..59ca217ab 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -6,20 +6,53 @@ defmodule Pleroma.Web.MediaProxy do alias Pleroma.Config alias Pleroma.Upload alias Pleroma.Web + alias Pleroma.Web.MediaProxy.Invalidation @base64_opts [padding: false] + @spec in_deleted_urls(String.t()) :: boolean() + def in_deleted_urls(url), do: elem(Cachex.exists?(:deleted_urls_cache, url(url)), 1) + + def remove_from_deleted_urls(urls) when is_list(urls) do + Cachex.execute!(:deleted_urls_cache, fn cache -> + Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1)) + end) + end + + def remove_from_deleted_urls(url) when is_binary(url) do + Cachex.del(:deleted_urls_cache, url(url)) + end + + def put_in_deleted_urls(urls) when is_list(urls) do + Cachex.execute!(:deleted_urls_cache, fn cache -> + Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true)) + end) + end + + def put_in_deleted_urls(url) when is_binary(url) do + Cachex.put(:deleted_urls_cache, url(url), true) + end + def url(url) when is_nil(url) or url == "", do: nil def url("/" <> _ = url), do: url def url(url) do - if disabled?() or local?(url) or whitelisted?(url) do + if disabled?() or not is_url_proxiable?(url) do url else encode_url(url) end end + @spec is_url_proxiable?(String.t()) :: boolean() + def is_url_proxiable?(url) do + if local?(url) or whitelisted?(url) do + false + else + true + end + end + defp disabled?, do: !Config.get([:media_proxy, :enabled], false) defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url()) diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index 4657a4383..ff0158d83 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -14,10 +14,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do with config <- Pleroma.Config.get([:media_proxy], []), true <- Keyword.get(config, :enabled, false), {:ok, url} <- MediaProxy.decode_url(sig64, url64), + {_, false} <- {:in_deleted_urls, MediaProxy.in_deleted_urls(url)}, :ok <- filename_matches(params, conn.request_path, url) do ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts)) else - false -> + error when error in [false, {:in_deleted_urls, true}] -> send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404)) {:error, :invalid_signature} -> diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 49352db2a..4ad19c0fc 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -23,8 +23,25 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do Enum.map(attachment["url"], & &1["href"]) end) - names = Enum.map(attachments, & &1["name"]) + # find all objects for copies of the attachments, name and actor doesn't matter here + hrefs + |> fetch_objects + |> prepare_objects(actor, Enum.map(attachments, & &1["name"])) + |> Enum.reduce({[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} -> + with 1 <- count do + {ids ++ [id], hrefs ++ [href]} + else + _ -> {ids ++ [id], hrefs} + end + end) + |> do_clean + {:ok, :success} + end + + def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip} + + defp do_clean({object_ids, attachment_urls}) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) prefix = @@ -39,68 +56,60 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do "/" ) - # find all objects for copies of the attachments, name and actor doesn't matter here - object_ids_and_hrefs = - from(o in Object, - where: - fragment( - "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)", - o.data, - o.data, - ^hrefs - ) - ) - # The query above can be time consumptive on large instances until we - # refactor how uploads are stored - |> Repo.all(timeout: :infinity) - # we should delete 1 object for any given attachment, but don't delete - # files if there are more than 1 object for it - |> Enum.reduce(%{}, fn %{ - id: id, - data: %{ - "url" => [%{"href" => href}], - "actor" => obj_actor, - "name" => name - } - }, - acc -> - Map.update(acc, href, %{id: id, count: 1}, fn val -> - case obj_actor == actor and name in names do - true -> - # set id of the actor's object that will be deleted - %{val | id: id, count: val.count + 1} + Enum.each(attachment_urls, fn href -> + href + |> String.trim_leading("#{base_url}/#{prefix}") + |> uploader.delete_file() + end) - false -> - # another actor's object, just increase count to not delete file - %{val | count: val.count + 1} - end - end) - end) - |> Enum.map(fn {href, %{id: id, count: count}} -> - # only delete files that have single instance - with 1 <- count do - href - |> String.trim_leading("#{base_url}/#{prefix}") - |> uploader.delete_file() - - {id, href} - else - _ -> {id, nil} - end - end) - - object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end) - - from(o in Object, where: o.id in ^object_ids) - |> Repo.delete_all() - - object_ids_and_hrefs - |> Enum.filter(fn {_, href} -> not is_nil(href) end) - |> Enum.map(&elem(&1, 1)) - |> Pleroma.Web.MediaProxy.Invalidation.purge() - - {:ok, :success} + delete_objects(object_ids) end - def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip} + defp delete_objects([_ | _] = object_ids) do + Repo.delete_all(from(o in Object, where: o.id in ^object_ids)) + end + + defp delete_objects(_), do: :ok + + # we should delete 1 object for any given attachment, but don't delete + # files if there are more than 1 object for it + def prepare_objects(objects, actor, names) do + objects + |> Enum.reduce(%{}, fn %{ + id: id, + data: %{ + "url" => [%{"href" => href}], + "actor" => obj_actor, + "name" => name + } + }, + acc -> + Map.update(acc, href, %{id: id, count: 1}, fn val -> + case obj_actor == actor and name in names do + true -> + # set id of the actor's object that will be deleted + %{val | id: id, count: val.count + 1} + + false -> + # another actor's object, just increase count to not delete file + %{val | count: val.count + 1} + end + end) + end) + end + + def fetch_objects(hrefs) do + from(o in Object, + where: + fragment( + "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)", + o.data, + o.data, + ^hrefs + ) + ) + # The query above can be time consumptive on large instances until we + # refactor how uploads are stored + |> Repo.all(timeout: :infinity) + end end diff --git a/test/web/media_proxy/invalidation_test.exs b/test/web/media_proxy/invalidation_test.exs new file mode 100644 index 000000000..3a9fa8c88 --- /dev/null +++ b/test/web/media_proxy/invalidation_test.exs @@ -0,0 +1,65 @@ +defmodule Pleroma.Web.MediaProxy.InvalidationTest do + use ExUnit.Case + use Pleroma.Tests.Helpers + + alias Pleroma.Config + alias Pleroma.Web.MediaProxy.Invalidation + + import ExUnit.CaptureLog + import Mock + import Tesla.Mock + + setup do: clear_config([:media_proxy]) + + setup do + on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + :ok + end + + describe "Invalidation.Http" do + test "perform request to clear cache" do + Config.put([:media_proxy, :enabled], false) + Config.put([:media_proxy, :invalidation, :enabled], true) + Config.put([:media_proxy, :invalidation, :provider], Invalidation.Http) + + Config.put([Invalidation.Http], method: :purge, headers: [{"x-refresh", 1}]) + image_url = "http://example.com/media/example.jpg" + Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url) + + mock(fn + %{ + method: :purge, + url: "http://example.com/media/example.jpg", + headers: [{"x-refresh", 1}] + } -> + %Tesla.Env{status: 200} + end) + + assert capture_log(fn -> + assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + assert Invalidation.purge([image_url]) == {:ok, [image_url]} + assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + end) =~ "Running cache purge: [\"#{image_url}\"]" + end + end + + describe "Invalidation.Script" do + test "run script to clear cache" do + Config.put([:media_proxy, :enabled], false) + Config.put([:media_proxy, :invalidation, :enabled], true) + Config.put([:media_proxy, :invalidation, :provider], Invalidation.Script) + Config.put([Invalidation.Script], script_path: "purge-nginx") + + image_url = "http://example.com/media/example.jpg" + Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url) + + with_mocks [{System, [], [cmd: fn _, _ -> {"ok", 0} end]}] do + assert capture_log(fn -> + assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + assert Invalidation.purge([image_url]) == {:ok, [image_url]} + assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + end) =~ "Running cache purge: [\"#{image_url}\"]" + end + end + end +end diff --git a/test/web/media_proxy/invalidations/http_test.exs b/test/web/media_proxy/invalidations/http_test.exs index 8a3b4141c..09e7ca0fb 100644 --- a/test/web/media_proxy/invalidations/http_test.exs +++ b/test/web/media_proxy/invalidations/http_test.exs @@ -5,6 +5,11 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do import ExUnit.CaptureLog import Tesla.Mock + setup do + on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + :ok + end + test "logs hasn't error message when request is valid" do mock(fn %{method: :purge, url: "http://example.com/media/example.jpg"} -> @@ -14,8 +19,8 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do refute capture_log(fn -> assert Invalidation.Http.purge( ["http://example.com/media/example.jpg"], - %{} - ) == {:ok, "success"} + [] + ) == {:ok, ["http://example.com/media/example.jpg"]} end) =~ "Error while cache purge" end @@ -28,8 +33,8 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do assert capture_log(fn -> assert Invalidation.Http.purge( ["http://example.com/media/example1.jpg"], - %{} - ) == {:ok, "success"} + [] + ) == {:ok, ["http://example.com/media/example1.jpg"]} end) =~ "Error while cache purge: url - http://example.com/media/example1.jpg" end end diff --git a/test/web/media_proxy/invalidations/script_test.exs b/test/web/media_proxy/invalidations/script_test.exs index 1358963ab..c69cec07a 100644 --- a/test/web/media_proxy/invalidations/script_test.exs +++ b/test/web/media_proxy/invalidations/script_test.exs @@ -4,17 +4,24 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.ScriptTest do import ExUnit.CaptureLog + setup do + on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + :ok + end + test "it logger error when script not found" do assert capture_log(fn -> assert Invalidation.Script.purge( ["http://example.com/media/example.jpg"], - %{script_path: "./example"} - ) == {:error, "\"%ErlangError{original: :enoent}\""} - end) =~ "Error while cache purge: \"%ErlangError{original: :enoent}\"" + script_path: "./example" + ) == {:error, "%ErlangError{original: :enoent}"} + end) =~ "Error while cache purge: %ErlangError{original: :enoent}" - assert Invalidation.Script.purge( - ["http://example.com/media/example.jpg"], - %{} - ) == {:error, "not found script path"} + capture_log(fn -> + assert Invalidation.Script.purge( + ["http://example.com/media/example.jpg"], + [] + ) == {:error, "\"not found script path\""} + end) end end diff --git a/test/web/media_proxy/media_proxy_controller_test.exs b/test/web/media_proxy/media_proxy_controller_test.exs index da79d38a5..2b6b25221 100644 --- a/test/web/media_proxy/media_proxy_controller_test.exs +++ b/test/web/media_proxy/media_proxy_controller_test.exs @@ -10,6 +10,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do setup do: clear_config(:media_proxy) setup do: clear_config([Pleroma.Web.Endpoint, :secret_key_base]) + setup do + on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + :ok + end + test "it returns 404 when MediaProxy disabled", %{conn: conn} do Config.put([:media_proxy, :enabled], false) @@ -66,4 +71,16 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do assert %Plug.Conn{status: :success} = get(conn, url) end end + + test "it returns 404 when url contains in deleted_urls cache", %{conn: conn} do + Config.put([:media_proxy, :enabled], true) + Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000") + url = Pleroma.Web.MediaProxy.encode_url("https://google.fn/test.png") + Pleroma.Web.MediaProxy.put_in_deleted_urls("https://google.fn/test.png") + + with_mock Pleroma.ReverseProxy, + call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do + assert %Plug.Conn{status: 404, resp_body: "Not Found"} = get(conn, url) + end + end end From 62b8c31b7a84dadb2a46861fe0f2dd1dbf9d40f0 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 15 Jun 2020 14:55:00 +0300 Subject: [PATCH 03/21] added tests --- .../media_proxy_cache_controller.ex | 31 ++++- .../web/media_proxy/invalidations/script.ex | 2 +- .../media_proxy_cache_controller_test.exs | 116 +++++++++++++++--- 3 files changed, 127 insertions(+), 22 deletions(-) diff --git a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex index 7b28f7c72..e3fa0ac28 100644 --- a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.Web.ApiSpec.Admin, as: Spec + alias Pleroma.Web.MediaProxy plug(Pleroma.Web.ApiSpec.CastAndValidate) @@ -24,15 +25,39 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation - def index(%{assigns: %{user: _}} = conn, _) do - render(conn, "index.json", urls: []) + def index(%{assigns: %{user: _}} = conn, params) do + cursor = + :deleted_urls_cache + |> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}]) + |> :qlc.cursor() + + urls = + case params.page do + 1 -> + :qlc.next_answers(cursor, params.page_size) + + _ -> + :qlc.next_answers(cursor, (params.page - 1) * params.page_size) + :qlc.next_answers(cursor, params.page_size) + end + + :qlc.delete_cursor(cursor) + + render(conn, "index.json", urls: urls) end def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do + MediaProxy.remove_from_deleted_urls(urls) render(conn, "index.json", urls: urls) end - def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: _ban}} = conn, _) do + def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do + MediaProxy.Invalidation.purge(urls) + + if ban do + MediaProxy.put_in_deleted_urls(urls) + end + render(conn, "index.json", urls: urls) end end diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index d41d647bb..0217b119d 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do require Logger @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, opts) do + def purge(urls, opts \\ %{}) do args = urls |> List.wrap() diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs index 1b1d6bc36..76a96f46f 100644 --- a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs +++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs @@ -6,6 +6,16 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory + import Mock + + alias Pleroma.Web.MediaProxy + + setup do: clear_config([:media_proxy]) + + setup do + on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + :ok + end setup do admin = insert(:user, is_admin: true) @@ -16,51 +26,121 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do |> assign(:user, admin) |> assign(:token, token) + Config.put([:media_proxy, :enabled], true) + Config.put([:media_proxy, :invalidation, :enabled], true) + Config.put([:media_proxy, :invalidation, :provider], MediaProxy.Invalidation.Script) + {:ok, %{admin: admin, token: token, conn: conn}} end describe "GET /api/pleroma/admin/media_proxy_caches" do test "shows banned MediaProxy URLs", %{conn: conn} do + MediaProxy.put_in_deleted_urls([ + "http://localhost:4001/media/a688346.jpg", + "http://localhost:4001/media/fb1f4d.jpg" + ]) + + MediaProxy.put_in_deleted_urls("http://localhost:4001/media/gb1f44.jpg") + MediaProxy.put_in_deleted_urls("http://localhost:4001/media/tb13f47.jpg") + MediaProxy.put_in_deleted_urls("http://localhost:4001/media/wb1f46.jpg") + response = conn - |> get("/api/pleroma/admin/media_proxy_caches") + |> get("/api/pleroma/admin/media_proxy_caches?page_size=2") |> json_response_and_validate_schema(200) - assert response["urls"] == [] + assert response["urls"] == [ + "http://localhost:4001/media/fb1f4d.jpg", + "http://localhost:4001/media/a688346.jpg" + ] + + response = + conn + |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=2") + |> json_response_and_validate_schema(200) + + assert response["urls"] == [ + "http://localhost:4001/media/gb1f44.jpg", + "http://localhost:4001/media/tb13f47.jpg" + ] + + response = + conn + |> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3") + |> json_response_and_validate_schema(200) + + assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"] end end describe "DELETE /api/pleroma/admin/media_proxy_caches/delete" do test "deleted MediaProxy URLs from banned", %{conn: conn} do + MediaProxy.put_in_deleted_urls([ + "http://localhost:4001/media/a688346.jpg", + "http://localhost:4001/media/fb1f4d.jpg" + ]) + response = conn |> put_req_header("content-type", "application/json") |> post("/api/pleroma/admin/media_proxy_caches/delete", %{ - urls: ["http://example.com/media/a688346.jpg", "http://example.com/media/fb1f4d.jpg"] + urls: ["http://localhost:4001/media/a688346.jpg"] }) |> json_response_and_validate_schema(200) - assert response["urls"] == [ - "http://example.com/media/a688346.jpg", - "http://example.com/media/fb1f4d.jpg" - ] + assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"] + refute MediaProxy.in_deleted_urls("http://localhost:4001/media/a688346.jpg") + assert MediaProxy.in_deleted_urls("http://localhost:4001/media/fb1f4d.jpg") end end describe "PURGE /api/pleroma/admin/media_proxy_caches/purge" do test "perform invalidates cache of MediaProxy", %{conn: conn} do - response = - conn - |> put_req_header("content-type", "application/json") - |> post("/api/pleroma/admin/media_proxy_caches/purge", %{ - urls: ["http://example.com/media/a688346.jpg", "http://example.com/media/fb1f4d.jpg"] - }) - |> json_response_and_validate_schema(200) + urls = [ + "http://example.com/media/a688346.jpg", + "http://example.com/media/fb1f4d.jpg" + ] - assert response["urls"] == [ - "http://example.com/media/a688346.jpg", - "http://example.com/media/fb1f4d.jpg" - ] + with_mocks [ + {MediaProxy.Invalidation.Script, [], + [ + purge: fn _, _ -> {"ok", 0} end + ]} + ] do + response = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false}) + |> json_response_and_validate_schema(200) + + assert response["urls"] == urls + + refute MediaProxy.in_deleted_urls("http://example.com/media/a688346.jpg") + refute MediaProxy.in_deleted_urls("http://example.com/media/fb1f4d.jpg") + end + end + + test "perform invalidates cache of MediaProxy and adds url to banned", %{conn: conn} do + urls = [ + "http://example.com/media/a688346.jpg", + "http://example.com/media/fb1f4d.jpg" + ] + + with_mocks [{MediaProxy.Invalidation.Script, [], [purge: fn _, _ -> {"ok", 0} end]}] do + response = + conn + |> put_req_header("content-type", "application/json") + |> post("/api/pleroma/admin/media_proxy_caches/purge", %{ + urls: urls, + ban: true + }) + |> json_response_and_validate_schema(200) + + assert response["urls"] == urls + + assert MediaProxy.in_deleted_urls("http://example.com/media/a688346.jpg") + assert MediaProxy.in_deleted_urls("http://example.com/media/fb1f4d.jpg") + end end end end From efdfc85c2d8e5118c1aa18e4f04026ec90cd11d2 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 15 Jun 2020 15:24:00 +0300 Subject: [PATCH 04/21] update docs --- docs/API/admin_api.md | 64 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 92816baf9..6659b605d 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -1224,4 +1224,66 @@ Loads json generated from `config/descriptions.exs`. - Response: - On success: `204`, empty response - On failure: - - 400 Bad Request `"Invalid parameters"` when `status` is missing \ No newline at end of file + - 400 Bad Request `"Invalid parameters"` when `status` is missing + +## `GET /api/pleroma/admin/media_proxy_caches` + +### Get a list of all banned MediaProxy URLs in Cachex + +- Authentication: required +- Params: +- *optional* `page`: **integer** page number +- *optional* `page_size`: **integer** number of log entries per page (default is `50`) + +- Response: + +``` json +{ + "urls": [ + "http://example.com/media/a688346.jpg", + "http://example.com/media/fb1f4d.jpg" + ] +} + +``` + +## `POST /api/pleroma/admin/media_proxy_caches/delete` + +### Remove a banned MediaProxy URL from Cachex + +- Authentication: required +- Params: + - `urls` + +- Response: + +``` json +{ + "urls": [ + "http://example.com/media/a688346.jpg", + "http://example.com/media/fb1f4d.jpg" + ] +} + +``` + +## `POST /api/pleroma/admin/media_proxy_caches/purge` + +### Purge a MediaProxy URL + +- Authentication: required +- Params: + - `urls` + - `ban` + +- Response: + +``` json +{ + "urls": [ + "http://example.com/media/a688346.jpg", + "http://example.com/media/fb1f4d.jpg" + ] +} + +``` From b02311079961c5193af1c144516a3caeee72b582 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 15 Jun 2020 20:47:02 +0300 Subject: [PATCH 05/21] fixed a visibility of functions --- .../workers/attachments_cleanup_worker.ex | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 4ad19c0fc..8deeabda0 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -18,22 +18,11 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do }, _job ) do - hrefs = - Enum.flat_map(attachments, fn attachment -> - Enum.map(attachment["url"], & &1["href"]) - end) - - # find all objects for copies of the attachments, name and actor doesn't matter here - hrefs + attachments + |> Enum.flat_map(fn item -> Enum.map(item["url"], & &1["href"]) end) |> fetch_objects |> prepare_objects(actor, Enum.map(attachments, & &1["name"])) - |> Enum.reduce({[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} -> - with 1 <- count do - {ids ++ [id], hrefs ++ [href]} - else - _ -> {ids ++ [id], hrefs} - end - end) + |> filter_objects |> do_clean {:ok, :success} @@ -73,7 +62,17 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do # we should delete 1 object for any given attachment, but don't delete # files if there are more than 1 object for it - def prepare_objects(objects, actor, names) do + defp filter_objects(objects) do + Enum.reduce(objects, {[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} -> + with 1 <- count do + {ids ++ [id], hrefs ++ [href]} + else + _ -> {ids ++ [id], hrefs} + end + end) + end + + defp prepare_objects(objects, actor, names) do objects |> Enum.reduce(%{}, fn %{ id: id, @@ -98,7 +97,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do end) end - def fetch_objects(hrefs) do + defp fetch_objects(hrefs) do from(o in Object, where: fragment( From 90613348ed8078e1906f7ffd18eebfa1a3b7f25a Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:56:13 +0000 Subject: [PATCH 06/21] Apply suggestion to docs/API/admin_api.md --- docs/API/admin_api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 6659b605d..8a3d60187 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -1253,7 +1253,7 @@ Loads json generated from `config/descriptions.exs`. - Authentication: required - Params: - - `urls` + - `urls` (array) - Response: From abfb1c756b62c24589d2881d77bf5974a80809d3 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:56:17 +0000 Subject: [PATCH 07/21] Apply suggestion to docs/API/admin_api.md --- docs/API/admin_api.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 8a3d60187..c7f56cf5f 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -1273,8 +1273,8 @@ Loads json generated from `config/descriptions.exs`. - Authentication: required - Params: - - `urls` - - `ban` + - `urls` (array) + - `ban` (boolean) - Response: From 74fd761637f737822d01aed945b6e0c75ced7008 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:56:30 +0000 Subject: [PATCH 08/21] Apply suggestion to lib/pleroma/web/media_proxy/invalidation.ex --- lib/pleroma/web/media_proxy/invalidation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index 83ff8589c..6da7eb720 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -32,6 +32,6 @@ defmodule Pleroma.Web.MediaProxy.Invalidation do def prepare_urls(urls) do urls |> List.wrap() - |> Enum.map(&MediaProxy.url(&1)) + |> Enum.map(&MediaProxy.url/1) end end From 1b45bc7b2ac53e56a4868da7e2b5b198d16306ab Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:58:08 +0000 Subject: [PATCH 09/21] Apply suggestion to test/web/admin_api/controllers/media_proxy_cache_controller_test.exs --- .../admin_api/controllers/media_proxy_cache_controller_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs index 76a96f46f..ddaf39f14 100644 --- a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs +++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs @@ -14,7 +14,6 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do setup do on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) - :ok end setup do From 793a53f1ec18a42f15f58494e40ed3c37d35f95c Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:58:16 +0000 Subject: [PATCH 10/21] Apply suggestion to test/web/admin_api/controllers/media_proxy_cache_controller_test.exs --- .../admin_api/controllers/media_proxy_cache_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs index ddaf39f14..81e20d001 100644 --- a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs +++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs @@ -72,7 +72,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do end end - describe "DELETE /api/pleroma/admin/media_proxy_caches/delete" do + describe "POST /api/pleroma/admin/media_proxy_caches/delete" do test "deleted MediaProxy URLs from banned", %{conn: conn} do MediaProxy.put_in_deleted_urls([ "http://localhost:4001/media/a688346.jpg", From 6d33a3a51bb8ff0afdf7f4f9880f8f5c5f2dfebc Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:58:28 +0000 Subject: [PATCH 11/21] Apply suggestion to test/web/admin_api/controllers/media_proxy_cache_controller_test.exs --- .../admin_api/controllers/media_proxy_cache_controller_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs index 81e20d001..42a3c0dd8 100644 --- a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs +++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs @@ -93,7 +93,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do end end - describe "PURGE /api/pleroma/admin/media_proxy_caches/purge" do + describe "POST /api/pleroma/admin/media_proxy_caches/purge" do test "perform invalidates cache of MediaProxy", %{conn: conn} do urls = [ "http://example.com/media/a688346.jpg", From 11b22a42293ec2ac0e66897bf4b29b5363913c19 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:58:33 +0000 Subject: [PATCH 12/21] Apply suggestion to test/web/media_proxy/invalidations/http_test.exs --- test/web/media_proxy/invalidations/http_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/web/media_proxy/invalidations/http_test.exs b/test/web/media_proxy/invalidations/http_test.exs index 09e7ca0fb..9d181dd8b 100644 --- a/test/web/media_proxy/invalidations/http_test.exs +++ b/test/web/media_proxy/invalidations/http_test.exs @@ -7,7 +7,6 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do setup do on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) - :ok end test "logs hasn't error message when request is valid" do From 2991aae4c4e4ae430539c1e6fac53fb8a0c991e9 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:58:38 +0000 Subject: [PATCH 13/21] Apply suggestion to test/web/media_proxy/invalidations/script_test.exs --- test/web/media_proxy/invalidations/script_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/web/media_proxy/invalidations/script_test.exs b/test/web/media_proxy/invalidations/script_test.exs index c69cec07a..8e155b705 100644 --- a/test/web/media_proxy/invalidations/script_test.exs +++ b/test/web/media_proxy/invalidations/script_test.exs @@ -6,7 +6,6 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.ScriptTest do setup do on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) - :ok end test "it logger error when script not found" do From 078d687e6ed66f921d7f54114f2dc6bf4abbf237 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:58:50 +0000 Subject: [PATCH 14/21] Apply suggestion to test/web/media_proxy/media_proxy_controller_test.exs --- test/web/media_proxy/media_proxy_controller_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/web/media_proxy/media_proxy_controller_test.exs b/test/web/media_proxy/media_proxy_controller_test.exs index 2b6b25221..72da98a6a 100644 --- a/test/web/media_proxy/media_proxy_controller_test.exs +++ b/test/web/media_proxy/media_proxy_controller_test.exs @@ -12,7 +12,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do setup do on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) - :ok end test "it returns 404 when MediaProxy disabled", %{conn: conn} do From 44ce97a9c9e90d5906386d1b51dea144cd258c32 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 13:12:32 +0000 Subject: [PATCH 15/21] Apply suggestion to lib/pleroma/web/media_proxy/invalidations/script.ex --- lib/pleroma/web/media_proxy/invalidations/script.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index 0217b119d..b0f44e8e2 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do require Logger @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, opts \\ %{}) do + def purge(urls, opts \\ []) do args = urls |> List.wrap() From 9a371bf5f6245b0f372bec7af15e2f4fc41d4ab7 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 13:12:38 +0000 Subject: [PATCH 16/21] Apply suggestion to lib/pleroma/web/media_proxy/invalidations/script.ex --- lib/pleroma/web/media_proxy/invalidations/script.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index b0f44e8e2..d32ffc50b 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -18,7 +18,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do |> Enum.join(" ") opts - |> Keyword.get(:script_path, nil) + |> Keyword.get(:script_path) |> do_purge([args]) |> handle_result(urls) end From 96493da7bdab4ff4a51cbebf18df4127ddc47990 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 13:14:01 +0000 Subject: [PATCH 17/21] Apply suggestion to test/web/media_proxy/invalidation_test.exs --- test/web/media_proxy/invalidation_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/web/media_proxy/invalidation_test.exs b/test/web/media_proxy/invalidation_test.exs index 3a9fa8c88..bf9af251c 100644 --- a/test/web/media_proxy/invalidation_test.exs +++ b/test/web/media_proxy/invalidation_test.exs @@ -13,7 +13,6 @@ defmodule Pleroma.Web.MediaProxy.InvalidationTest do setup do on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) - :ok end describe "Invalidation.Http" do From 71a5d9bffb33d9424ea28900ea006678617e0096 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 17 Jun 2020 12:54:02 -0500 Subject: [PATCH 18/21] Empty list as default --- lib/pleroma/web/media_proxy/invalidations/http.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex index 3694b56e8..bb81d8888 100644 --- a/lib/pleroma/web/media_proxy/invalidations/http.ex +++ b/lib/pleroma/web/media_proxy/invalidations/http.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do require Logger @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, opts) do + def purge(urls, opts \\ []) do method = Keyword.get(opts, :method, :purge) headers = Keyword.get(opts, :headers, []) options = Keyword.get(opts, :options, []) From c08c9db0c137d36896910194a6dc50a391a8fee2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 17 Jun 2020 13:02:01 -0500 Subject: [PATCH 19/21] Remove misleading is_ prefix from boolean function --- lib/pleroma/web/media_proxy/media_proxy.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index 59ca217ab..3dccd6b7f 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -37,15 +37,15 @@ defmodule Pleroma.Web.MediaProxy do def url("/" <> _ = url), do: url def url(url) do - if disabled?() or not is_url_proxiable?(url) do + if disabled?() or not url_proxiable?(url) do url else encode_url(url) end end - @spec is_url_proxiable?(String.t()) :: boolean() - def is_url_proxiable?(url) do + @spec url_proxiable?(String.t()) :: boolean() + def url_proxiable?(url) do if local?(url) or whitelisted?(url) do false else From 2731ea1334c2c91315465659a0874829cb9e1e11 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 17 Jun 2020 13:13:55 -0500 Subject: [PATCH 20/21] Change references from "deleted_urls" to "banned_urls" as nothing is handled via media deletions anymore; all actions are manual operations by an admin to ban the url --- lib/pleroma/application.ex | 2 +- lib/pleroma/plugs/uploaded_media.ex | 10 ++++---- .../media_proxy_cache_controller.ex | 6 ++--- lib/pleroma/web/media_proxy/media_proxy.ex | 20 ++++++++-------- .../web/media_proxy/media_proxy_controller.ex | 4 ++-- .../media_proxy_cache_controller_test.exs | 24 +++++++++---------- test/web/media_proxy/invalidation_test.exs | 14 +++++------ .../media_proxy_controller_test.exs | 6 ++--- 8 files changed, 43 insertions(+), 43 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index adebebc7a..4a21bf138 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -149,7 +149,7 @@ defmodule Pleroma.Application do build_cachex("web_resp", limit: 2500), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("failed_proxy_url", limit: 2500), - build_cachex("deleted_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) + build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) ] end diff --git a/lib/pleroma/plugs/uploaded_media.ex b/lib/pleroma/plugs/uploaded_media.ex index 2f3fde002..40984cfc0 100644 --- a/lib/pleroma/plugs/uploaded_media.ex +++ b/lib/pleroma/plugs/uploaded_media.ex @@ -49,7 +49,7 @@ defmodule Pleroma.Plugs.UploadedMedia do with uploader <- Keyword.fetch!(config, :uploader), proxy_remote = Keyword.get(config, :proxy_remote, false), {:ok, get_method} <- uploader.get_file(file), - false <- media_is_deleted(conn, get_method) do + false <- media_is_banned(conn, get_method) do get_media(conn, get_method, proxy_remote, opts) else _ -> @@ -61,13 +61,13 @@ defmodule Pleroma.Plugs.UploadedMedia do def call(conn, _opts), do: conn - defp media_is_deleted(%{request_path: path} = _conn, {:static_dir, _}) do - MediaProxy.in_deleted_urls(Pleroma.Web.base_url() <> path) + defp media_is_banned(%{request_path: path} = _conn, {:static_dir, _}) do + MediaProxy.in_banned_urls(Pleroma.Web.base_url() <> path) end - defp media_is_deleted(_, {:url, url}), do: MediaProxy.in_deleted_urls(url) + defp media_is_banned(_, {:url, url}), do: MediaProxy.in_banned_urls(url) - defp media_is_deleted(_, _), do: false + defp media_is_banned(_, _), do: false defp get_media(conn, {:static_dir, directory}, _, opts) do static_opts = diff --git a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex index e3fa0ac28..e2759d59f 100644 --- a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do def index(%{assigns: %{user: _}} = conn, params) do cursor = - :deleted_urls_cache + :banned_urls_cache |> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}]) |> :qlc.cursor() @@ -47,7 +47,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do end def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do - MediaProxy.remove_from_deleted_urls(urls) + MediaProxy.remove_from_banned_urls(urls) render(conn, "index.json", urls: urls) end @@ -55,7 +55,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do MediaProxy.Invalidation.purge(urls) if ban do - MediaProxy.put_in_deleted_urls(urls) + MediaProxy.put_in_banned_urls(urls) end render(conn, "index.json", urls: urls) diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index 3dccd6b7f..077fabe47 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -10,27 +10,27 @@ defmodule Pleroma.Web.MediaProxy do @base64_opts [padding: false] - @spec in_deleted_urls(String.t()) :: boolean() - def in_deleted_urls(url), do: elem(Cachex.exists?(:deleted_urls_cache, url(url)), 1) + @spec in_banned_urls(String.t()) :: boolean() + def in_banned_urls(url), do: elem(Cachex.exists?(:banned_urls_cache, url(url)), 1) - def remove_from_deleted_urls(urls) when is_list(urls) do - Cachex.execute!(:deleted_urls_cache, fn cache -> + def remove_from_banned_urls(urls) when is_list(urls) do + Cachex.execute!(:banned_urls_cache, fn cache -> Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1)) end) end - def remove_from_deleted_urls(url) when is_binary(url) do - Cachex.del(:deleted_urls_cache, url(url)) + def remove_from_banned_urls(url) when is_binary(url) do + Cachex.del(:banned_urls_cache, url(url)) end - def put_in_deleted_urls(urls) when is_list(urls) do - Cachex.execute!(:deleted_urls_cache, fn cache -> + def put_in_banned_urls(urls) when is_list(urls) do + Cachex.execute!(:banned_urls_cache, fn cache -> Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true)) end) end - def put_in_deleted_urls(url) when is_binary(url) do - Cachex.put(:deleted_urls_cache, url(url), true) + def put_in_banned_urls(url) when is_binary(url) do + Cachex.put(:banned_urls_cache, url(url), true) end def url(url) when is_nil(url) or url == "", do: nil diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index ff0158d83..9a64b0ef3 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -14,11 +14,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do with config <- Pleroma.Config.get([:media_proxy], []), true <- Keyword.get(config, :enabled, false), {:ok, url} <- MediaProxy.decode_url(sig64, url64), - {_, false} <- {:in_deleted_urls, MediaProxy.in_deleted_urls(url)}, + {_, false} <- {:in_banned_urls, MediaProxy.in_banned_urls(url)}, :ok <- filename_matches(params, conn.request_path, url) do ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts)) else - error when error in [false, {:in_deleted_urls, true}] -> + error when error in [false, {:in_banned_urls, true}] -> send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404)) {:error, :invalid_signature} -> diff --git a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs index 42a3c0dd8..5ab6cb78a 100644 --- a/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs +++ b/test/web/admin_api/controllers/media_proxy_cache_controller_test.exs @@ -13,7 +13,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do setup do: clear_config([:media_proxy]) setup do - on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + on_exit(fn -> Cachex.clear(:banned_urls_cache) end) end setup do @@ -34,14 +34,14 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do describe "GET /api/pleroma/admin/media_proxy_caches" do test "shows banned MediaProxy URLs", %{conn: conn} do - MediaProxy.put_in_deleted_urls([ + MediaProxy.put_in_banned_urls([ "http://localhost:4001/media/a688346.jpg", "http://localhost:4001/media/fb1f4d.jpg" ]) - MediaProxy.put_in_deleted_urls("http://localhost:4001/media/gb1f44.jpg") - MediaProxy.put_in_deleted_urls("http://localhost:4001/media/tb13f47.jpg") - MediaProxy.put_in_deleted_urls("http://localhost:4001/media/wb1f46.jpg") + MediaProxy.put_in_banned_urls("http://localhost:4001/media/gb1f44.jpg") + MediaProxy.put_in_banned_urls("http://localhost:4001/media/tb13f47.jpg") + MediaProxy.put_in_banned_urls("http://localhost:4001/media/wb1f46.jpg") response = conn @@ -74,7 +74,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do describe "POST /api/pleroma/admin/media_proxy_caches/delete" do test "deleted MediaProxy URLs from banned", %{conn: conn} do - MediaProxy.put_in_deleted_urls([ + MediaProxy.put_in_banned_urls([ "http://localhost:4001/media/a688346.jpg", "http://localhost:4001/media/fb1f4d.jpg" ]) @@ -88,8 +88,8 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do |> json_response_and_validate_schema(200) assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"] - refute MediaProxy.in_deleted_urls("http://localhost:4001/media/a688346.jpg") - assert MediaProxy.in_deleted_urls("http://localhost:4001/media/fb1f4d.jpg") + refute MediaProxy.in_banned_urls("http://localhost:4001/media/a688346.jpg") + assert MediaProxy.in_banned_urls("http://localhost:4001/media/fb1f4d.jpg") end end @@ -114,8 +114,8 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do assert response["urls"] == urls - refute MediaProxy.in_deleted_urls("http://example.com/media/a688346.jpg") - refute MediaProxy.in_deleted_urls("http://example.com/media/fb1f4d.jpg") + refute MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg") + refute MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg") end end @@ -137,8 +137,8 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do assert response["urls"] == urls - assert MediaProxy.in_deleted_urls("http://example.com/media/a688346.jpg") - assert MediaProxy.in_deleted_urls("http://example.com/media/fb1f4d.jpg") + assert MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg") + assert MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg") end end end diff --git a/test/web/media_proxy/invalidation_test.exs b/test/web/media_proxy/invalidation_test.exs index bf9af251c..926ae74ca 100644 --- a/test/web/media_proxy/invalidation_test.exs +++ b/test/web/media_proxy/invalidation_test.exs @@ -12,7 +12,7 @@ defmodule Pleroma.Web.MediaProxy.InvalidationTest do setup do: clear_config([:media_proxy]) setup do - on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + on_exit(fn -> Cachex.clear(:banned_urls_cache) end) end describe "Invalidation.Http" do @@ -23,7 +23,7 @@ defmodule Pleroma.Web.MediaProxy.InvalidationTest do Config.put([Invalidation.Http], method: :purge, headers: [{"x-refresh", 1}]) image_url = "http://example.com/media/example.jpg" - Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url) + Pleroma.Web.MediaProxy.put_in_banned_urls(image_url) mock(fn %{ @@ -35,9 +35,9 @@ defmodule Pleroma.Web.MediaProxy.InvalidationTest do end) assert capture_log(fn -> - assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + assert Pleroma.Web.MediaProxy.in_banned_urls(image_url) assert Invalidation.purge([image_url]) == {:ok, [image_url]} - assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + assert Pleroma.Web.MediaProxy.in_banned_urls(image_url) end) =~ "Running cache purge: [\"#{image_url}\"]" end end @@ -50,13 +50,13 @@ defmodule Pleroma.Web.MediaProxy.InvalidationTest do Config.put([Invalidation.Script], script_path: "purge-nginx") image_url = "http://example.com/media/example.jpg" - Pleroma.Web.MediaProxy.put_in_deleted_urls(image_url) + Pleroma.Web.MediaProxy.put_in_banned_urls(image_url) with_mocks [{System, [], [cmd: fn _, _ -> {"ok", 0} end]}] do assert capture_log(fn -> - assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + assert Pleroma.Web.MediaProxy.in_banned_urls(image_url) assert Invalidation.purge([image_url]) == {:ok, [image_url]} - assert Pleroma.Web.MediaProxy.in_deleted_urls(image_url) + assert Pleroma.Web.MediaProxy.in_banned_urls(image_url) end) =~ "Running cache purge: [\"#{image_url}\"]" end end diff --git a/test/web/media_proxy/media_proxy_controller_test.exs b/test/web/media_proxy/media_proxy_controller_test.exs index 72da98a6a..d61cef83b 100644 --- a/test/web/media_proxy/media_proxy_controller_test.exs +++ b/test/web/media_proxy/media_proxy_controller_test.exs @@ -11,7 +11,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do setup do: clear_config([Pleroma.Web.Endpoint, :secret_key_base]) setup do - on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + on_exit(fn -> Cachex.clear(:banned_urls_cache) end) end test "it returns 404 when MediaProxy disabled", %{conn: conn} do @@ -71,11 +71,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do end end - test "it returns 404 when url contains in deleted_urls cache", %{conn: conn} do + test "it returns 404 when url contains in banned_urls cache", %{conn: conn} do Config.put([:media_proxy, :enabled], true) Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000") url = Pleroma.Web.MediaProxy.encode_url("https://google.fn/test.png") - Pleroma.Web.MediaProxy.put_in_deleted_urls("https://google.fn/test.png") + Pleroma.Web.MediaProxy.put_in_banned_urls("https://google.fn/test.png") with_mock Pleroma.ReverseProxy, call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do From 4044f24e2e4935757e038e7f06373ed1c9172560 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 18 Jun 2020 05:02:33 +0300 Subject: [PATCH 21/21] fix test --- lib/pleroma/web/media_proxy/invalidation.ex | 3 ++- test/web/media_proxy/invalidations/http_test.exs | 2 +- test/web/media_proxy/invalidations/script_test.exs | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index 6da7eb720..5808861e6 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -26,7 +26,8 @@ defmodule Pleroma.Web.MediaProxy.Invalidation do defp do_purge(urls) do provider = Config.get([:media_proxy, :invalidation, :provider]) - provider.purge(urls, Config.get(provider)) + options = Config.get(provider) + provider.purge(urls, options) end def prepare_urls(urls) do diff --git a/test/web/media_proxy/invalidations/http_test.exs b/test/web/media_proxy/invalidations/http_test.exs index 9d181dd8b..a1bef5237 100644 --- a/test/web/media_proxy/invalidations/http_test.exs +++ b/test/web/media_proxy/invalidations/http_test.exs @@ -6,7 +6,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do import Tesla.Mock setup do - on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + on_exit(fn -> Cachex.clear(:banned_urls_cache) end) end test "logs hasn't error message when request is valid" do diff --git a/test/web/media_proxy/invalidations/script_test.exs b/test/web/media_proxy/invalidations/script_test.exs index 8e155b705..51833ab18 100644 --- a/test/web/media_proxy/invalidations/script_test.exs +++ b/test/web/media_proxy/invalidations/script_test.exs @@ -5,7 +5,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.ScriptTest do import ExUnit.CaptureLog setup do - on_exit(fn -> Cachex.clear(:deleted_urls_cache) end) + on_exit(fn -> Cachex.clear(:banned_urls_cache) end) end test "it logger error when script not found" do