forked from AkkomaGang/akkoma
Merge branch 'issue/1855' into 'develop'
#1855 MediaProxy cache invalidation via Admin API See merge request pleroma/pleroma!2648
This commit is contained in:
commit
f928267773
22 changed files with 740 additions and 118 deletions
|
@ -407,6 +407,13 @@ config :pleroma, :media_proxy,
|
||||||
],
|
],
|
||||||
whitelist: []
|
whitelist: []
|
||||||
|
|
||||||
|
config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Http,
|
||||||
|
method: :purge,
|
||||||
|
headers: [],
|
||||||
|
options: []
|
||||||
|
|
||||||
|
config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script, script_path: nil
|
||||||
|
|
||||||
config :pleroma, :chat, enabled: true
|
config :pleroma, :chat, enabled: true
|
||||||
|
|
||||||
config :phoenix, :format_encoders, json: Jason
|
config :phoenix, :format_encoders, json: Jason
|
||||||
|
|
|
@ -1650,6 +1650,31 @@ config :pleroma, :config_description, [
|
||||||
"The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.",
|
"The base URL to access a user-uploaded file. Useful when you want to proxy the media files via another host/CDN fronts.",
|
||||||
suggestions: ["https://example.com"]
|
suggestions: ["https://example.com"]
|
||||||
},
|
},
|
||||||
|
%{
|
||||||
|
key: :invalidation,
|
||||||
|
type: :keyword,
|
||||||
|
descpiption: "",
|
||||||
|
suggestions: [
|
||||||
|
enabled: true,
|
||||||
|
provider: Pleroma.Web.MediaProxy.Invalidation.Script
|
||||||
|
],
|
||||||
|
children: [
|
||||||
|
%{
|
||||||
|
key: :enabled,
|
||||||
|
type: :boolean,
|
||||||
|
description: "Enables invalidate media cache"
|
||||||
|
},
|
||||||
|
%{
|
||||||
|
key: :provider,
|
||||||
|
type: :module,
|
||||||
|
description: "Module which will be used to cache purge.",
|
||||||
|
suggestions: [
|
||||||
|
Pleroma.Web.MediaProxy.Invalidation.Script,
|
||||||
|
Pleroma.Web.MediaProxy.Invalidation.Http
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
%{
|
%{
|
||||||
key: :proxy_opts,
|
key: :proxy_opts,
|
||||||
type: :keyword,
|
type: :keyword,
|
||||||
|
@ -1722,6 +1747,45 @@ config :pleroma, :config_description, [
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
%{
|
||||||
|
group: :pleroma,
|
||||||
|
key: Pleroma.Web.MediaProxy.Invalidation.Http,
|
||||||
|
type: :group,
|
||||||
|
description: "HTTP invalidate settings",
|
||||||
|
children: [
|
||||||
|
%{
|
||||||
|
key: :method,
|
||||||
|
type: :atom,
|
||||||
|
description: "HTTP method of request. Default: :purge"
|
||||||
|
},
|
||||||
|
%{
|
||||||
|
key: :headers,
|
||||||
|
type: {:list, :tuple},
|
||||||
|
description: "HTTP headers of request.",
|
||||||
|
suggestions: [{"x-refresh", 1}]
|
||||||
|
},
|
||||||
|
%{
|
||||||
|
key: :options,
|
||||||
|
type: :keyword,
|
||||||
|
description: "Request options.",
|
||||||
|
suggestions: [params: %{ts: "xxx"}]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
%{
|
||||||
|
group: :pleroma,
|
||||||
|
key: Pleroma.Web.MediaProxy.Invalidation.Script,
|
||||||
|
type: :group,
|
||||||
|
description: "Script invalidate settings",
|
||||||
|
children: [
|
||||||
|
%{
|
||||||
|
key: :script_path,
|
||||||
|
type: :string,
|
||||||
|
description: "Path to shell script. Which will run purge cache.",
|
||||||
|
suggestions: ["./installation/nginx-cache-purge.sh.example"]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
%{
|
%{
|
||||||
group: :pleroma,
|
group: :pleroma,
|
||||||
key: :gopher,
|
key: :gopher,
|
||||||
|
|
|
@ -1225,3 +1225,65 @@ Loads json generated from `config/descriptions.exs`.
|
||||||
- On success: `204`, empty response
|
- On success: `204`, empty response
|
||||||
- On failure:
|
- On failure:
|
||||||
- 400 Bad Request `"Invalid parameters"` when `status` is missing
|
- 400 Bad Request `"Invalid parameters"` when `status` is missing
|
||||||
|
|
||||||
|
## `GET /api/pleroma/admin/media_proxy_caches`
|
||||||
|
|
||||||
|
### Get a list of all banned MediaProxy URLs in Cachex
|
||||||
|
|
||||||
|
- Authentication: required
|
||||||
|
- Params:
|
||||||
|
- *optional* `page`: **integer** page number
|
||||||
|
- *optional* `page_size`: **integer** number of log entries per page (default is `50`)
|
||||||
|
|
||||||
|
- Response:
|
||||||
|
|
||||||
|
``` json
|
||||||
|
{
|
||||||
|
"urls": [
|
||||||
|
"http://example.com/media/a688346.jpg",
|
||||||
|
"http://example.com/media/fb1f4d.jpg"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
## `POST /api/pleroma/admin/media_proxy_caches/delete`
|
||||||
|
|
||||||
|
### Remove a banned MediaProxy URL from Cachex
|
||||||
|
|
||||||
|
- Authentication: required
|
||||||
|
- Params:
|
||||||
|
- `urls` (array)
|
||||||
|
|
||||||
|
- Response:
|
||||||
|
|
||||||
|
``` json
|
||||||
|
{
|
||||||
|
"urls": [
|
||||||
|
"http://example.com/media/a688346.jpg",
|
||||||
|
"http://example.com/media/fb1f4d.jpg"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
## `POST /api/pleroma/admin/media_proxy_caches/purge`
|
||||||
|
|
||||||
|
### Purge a MediaProxy URL
|
||||||
|
|
||||||
|
- Authentication: required
|
||||||
|
- Params:
|
||||||
|
- `urls` (array)
|
||||||
|
- `ban` (boolean)
|
||||||
|
|
||||||
|
- Response:
|
||||||
|
|
||||||
|
``` json
|
||||||
|
{
|
||||||
|
"urls": [
|
||||||
|
"http://example.com/media/a688346.jpg",
|
||||||
|
"http://example.com/media/fb1f4d.jpg"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
```
|
||||||
|
|
|
@ -268,7 +268,7 @@ This section describe PWA manifest instance-specific values. Currently this opti
|
||||||
|
|
||||||
#### Pleroma.Web.MediaProxy.Invalidation.Script
|
#### Pleroma.Web.MediaProxy.Invalidation.Script
|
||||||
|
|
||||||
This strategy allow perform external bash script to purge cache.
|
This strategy allow perform external shell script to purge cache.
|
||||||
Urls of attachments pass to script as arguments.
|
Urls of attachments pass to script as arguments.
|
||||||
|
|
||||||
* `script_path`: path to external script.
|
* `script_path`: path to external script.
|
||||||
|
@ -284,8 +284,8 @@ config :pleroma, Pleroma.Web.MediaProxy.Invalidation.Script,
|
||||||
This strategy allow perform custom http request to purge cache.
|
This strategy allow perform custom http request to purge cache.
|
||||||
|
|
||||||
* `method`: http method. default is `purge`
|
* `method`: http method. default is `purge`
|
||||||
* `headers`: http headers. default is empty
|
* `headers`: http headers.
|
||||||
* `options`: request options. default is empty
|
* `options`: request options.
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
```elixir
|
```elixir
|
||||||
|
|
|
@ -13,7 +13,7 @@ CACHE_DIRECTORY="/tmp/pleroma-media-cache"
|
||||||
## $3 - (optional) the number of parallel processes to run for grep.
|
## $3 - (optional) the number of parallel processes to run for grep.
|
||||||
get_cache_files() {
|
get_cache_files() {
|
||||||
local max_parallel=${3-16}
|
local max_parallel=${3-16}
|
||||||
find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E Rl "^KEY:.*$1" | sort -u
|
find $2 -maxdepth 2 -type d | xargs -P $max_parallel -n 1 grep -E -Rl "^KEY:.*$1" | sort -u
|
||||||
}
|
}
|
||||||
|
|
||||||
## Removes an item from the given cache zone.
|
## Removes an item from the given cache zone.
|
||||||
|
@ -37,4 +37,4 @@ purge() {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
purge $1
|
purge $@
|
||||||
|
|
|
@ -148,7 +148,8 @@ defmodule Pleroma.Application do
|
||||||
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
|
build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
|
||||||
build_cachex("web_resp", limit: 2500),
|
build_cachex("web_resp", limit: 2500),
|
||||||
build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
|
build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
|
||||||
build_cachex("failed_proxy_url", limit: 2500)
|
build_cachex("failed_proxy_url", limit: 2500),
|
||||||
|
build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000)
|
||||||
]
|
]
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
@ -10,6 +10,8 @@ defmodule Pleroma.Plugs.UploadedMedia do
|
||||||
import Pleroma.Web.Gettext
|
import Pleroma.Web.Gettext
|
||||||
require Logger
|
require Logger
|
||||||
|
|
||||||
|
alias Pleroma.Web.MediaProxy
|
||||||
|
|
||||||
@behaviour Plug
|
@behaviour Plug
|
||||||
# no slashes
|
# no slashes
|
||||||
@path "media"
|
@path "media"
|
||||||
|
@ -35,8 +37,7 @@ defmodule Pleroma.Plugs.UploadedMedia do
|
||||||
%{query_params: %{"name" => name}} = conn ->
|
%{query_params: %{"name" => name}} = conn ->
|
||||||
name = String.replace(name, "\"", "\\\"")
|
name = String.replace(name, "\"", "\\\"")
|
||||||
|
|
||||||
conn
|
put_resp_header(conn, "content-disposition", "filename=\"#{name}\"")
|
||||||
|> put_resp_header("content-disposition", "filename=\"#{name}\"")
|
|
||||||
|
|
||||||
conn ->
|
conn ->
|
||||||
conn
|
conn
|
||||||
|
@ -47,7 +48,8 @@ defmodule Pleroma.Plugs.UploadedMedia do
|
||||||
|
|
||||||
with uploader <- Keyword.fetch!(config, :uploader),
|
with uploader <- Keyword.fetch!(config, :uploader),
|
||||||
proxy_remote = Keyword.get(config, :proxy_remote, false),
|
proxy_remote = Keyword.get(config, :proxy_remote, false),
|
||||||
{:ok, get_method} <- uploader.get_file(file) do
|
{:ok, get_method} <- uploader.get_file(file),
|
||||||
|
false <- media_is_banned(conn, get_method) do
|
||||||
get_media(conn, get_method, proxy_remote, opts)
|
get_media(conn, get_method, proxy_remote, opts)
|
||||||
else
|
else
|
||||||
_ ->
|
_ ->
|
||||||
|
@ -59,6 +61,14 @@ defmodule Pleroma.Plugs.UploadedMedia do
|
||||||
|
|
||||||
def call(conn, _opts), do: conn
|
def call(conn, _opts), do: conn
|
||||||
|
|
||||||
|
defp media_is_banned(%{request_path: path} = _conn, {:static_dir, _}) do
|
||||||
|
MediaProxy.in_banned_urls(Pleroma.Web.base_url() <> path)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp media_is_banned(_, {:url, url}), do: MediaProxy.in_banned_urls(url)
|
||||||
|
|
||||||
|
defp media_is_banned(_, _), do: false
|
||||||
|
|
||||||
defp get_media(conn, {:static_dir, directory}, _, opts) do
|
defp get_media(conn, {:static_dir, directory}, _, opts) do
|
||||||
static_opts =
|
static_opts =
|
||||||
Map.get(opts, :static_plug_opts)
|
Map.get(opts, :static_plug_opts)
|
||||||
|
|
|
@ -0,0 +1,63 @@
|
||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
|
||||||
|
use Pleroma.Web, :controller
|
||||||
|
|
||||||
|
alias Pleroma.Plugs.OAuthScopesPlug
|
||||||
|
alias Pleroma.Web.ApiSpec.Admin, as: Spec
|
||||||
|
alias Pleroma.Web.MediaProxy
|
||||||
|
|
||||||
|
plug(Pleroma.Web.ApiSpec.CastAndValidate)
|
||||||
|
|
||||||
|
plug(
|
||||||
|
OAuthScopesPlug,
|
||||||
|
%{scopes: ["read:media_proxy_caches"], admin: true} when action in [:index]
|
||||||
|
)
|
||||||
|
|
||||||
|
plug(
|
||||||
|
OAuthScopesPlug,
|
||||||
|
%{scopes: ["write:media_proxy_caches"], admin: true} when action in [:purge, :delete]
|
||||||
|
)
|
||||||
|
|
||||||
|
action_fallback(Pleroma.Web.AdminAPI.FallbackController)
|
||||||
|
|
||||||
|
defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation
|
||||||
|
|
||||||
|
def index(%{assigns: %{user: _}} = conn, params) do
|
||||||
|
cursor =
|
||||||
|
:banned_urls_cache
|
||||||
|
|> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}])
|
||||||
|
|> :qlc.cursor()
|
||||||
|
|
||||||
|
urls =
|
||||||
|
case params.page do
|
||||||
|
1 ->
|
||||||
|
:qlc.next_answers(cursor, params.page_size)
|
||||||
|
|
||||||
|
_ ->
|
||||||
|
:qlc.next_answers(cursor, (params.page - 1) * params.page_size)
|
||||||
|
:qlc.next_answers(cursor, params.page_size)
|
||||||
|
end
|
||||||
|
|
||||||
|
:qlc.delete_cursor(cursor)
|
||||||
|
|
||||||
|
render(conn, "index.json", urls: urls)
|
||||||
|
end
|
||||||
|
|
||||||
|
def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do
|
||||||
|
MediaProxy.remove_from_banned_urls(urls)
|
||||||
|
render(conn, "index.json", urls: urls)
|
||||||
|
end
|
||||||
|
|
||||||
|
def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do
|
||||||
|
MediaProxy.Invalidation.purge(urls)
|
||||||
|
|
||||||
|
if ban do
|
||||||
|
MediaProxy.put_in_banned_urls(urls)
|
||||||
|
end
|
||||||
|
|
||||||
|
render(conn, "index.json", urls: urls)
|
||||||
|
end
|
||||||
|
end
|
11
lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex
Normal file
11
lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex
Normal file
|
@ -0,0 +1,11 @@
|
||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do
|
||||||
|
use Pleroma.Web, :view
|
||||||
|
|
||||||
|
def render("index.json", %{urls: urls}) do
|
||||||
|
%{urls: urls}
|
||||||
|
end
|
||||||
|
end
|
|
@ -0,0 +1,109 @@
|
||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do
|
||||||
|
alias OpenApiSpex.Operation
|
||||||
|
alias OpenApiSpex.Schema
|
||||||
|
alias Pleroma.Web.ApiSpec.Schemas.ApiError
|
||||||
|
|
||||||
|
import Pleroma.Web.ApiSpec.Helpers
|
||||||
|
|
||||||
|
def open_api_operation(action) do
|
||||||
|
operation = String.to_existing_atom("#{action}_operation")
|
||||||
|
apply(__MODULE__, operation, [])
|
||||||
|
end
|
||||||
|
|
||||||
|
def index_operation do
|
||||||
|
%Operation{
|
||||||
|
tags: ["Admin", "MediaProxyCache"],
|
||||||
|
summary: "Fetch a paginated list of all banned MediaProxy URLs in Cachex",
|
||||||
|
operationId: "AdminAPI.MediaProxyCacheController.index",
|
||||||
|
security: [%{"oAuth" => ["read:media_proxy_caches"]}],
|
||||||
|
parameters: [
|
||||||
|
Operation.parameter(
|
||||||
|
:page,
|
||||||
|
:query,
|
||||||
|
%Schema{type: :integer, default: 1},
|
||||||
|
"Page"
|
||||||
|
),
|
||||||
|
Operation.parameter(
|
||||||
|
:page_size,
|
||||||
|
:query,
|
||||||
|
%Schema{type: :integer, default: 50},
|
||||||
|
"Number of statuses to return"
|
||||||
|
)
|
||||||
|
],
|
||||||
|
responses: %{
|
||||||
|
200 => success_response()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def delete_operation do
|
||||||
|
%Operation{
|
||||||
|
tags: ["Admin", "MediaProxyCache"],
|
||||||
|
summary: "Remove a banned MediaProxy URL from Cachex",
|
||||||
|
operationId: "AdminAPI.MediaProxyCacheController.delete",
|
||||||
|
security: [%{"oAuth" => ["write:media_proxy_caches"]}],
|
||||||
|
requestBody:
|
||||||
|
request_body(
|
||||||
|
"Parameters",
|
||||||
|
%Schema{
|
||||||
|
type: :object,
|
||||||
|
required: [:urls],
|
||||||
|
properties: %{
|
||||||
|
urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
required: true
|
||||||
|
),
|
||||||
|
responses: %{
|
||||||
|
200 => success_response(),
|
||||||
|
400 => Operation.response("Error", "application/json", ApiError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
def purge_operation do
|
||||||
|
%Operation{
|
||||||
|
tags: ["Admin", "MediaProxyCache"],
|
||||||
|
summary: "Purge and optionally ban a MediaProxy URL",
|
||||||
|
operationId: "AdminAPI.MediaProxyCacheController.purge",
|
||||||
|
security: [%{"oAuth" => ["write:media_proxy_caches"]}],
|
||||||
|
requestBody:
|
||||||
|
request_body(
|
||||||
|
"Parameters",
|
||||||
|
%Schema{
|
||||||
|
type: :object,
|
||||||
|
required: [:urls],
|
||||||
|
properties: %{
|
||||||
|
urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}},
|
||||||
|
ban: %Schema{type: :boolean, default: true}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
required: true
|
||||||
|
),
|
||||||
|
responses: %{
|
||||||
|
200 => success_response(),
|
||||||
|
400 => Operation.response("Error", "application/json", ApiError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp success_response do
|
||||||
|
Operation.response("Array of banned MediaProxy URLs in Cachex", "application/json", %Schema{
|
||||||
|
type: :object,
|
||||||
|
properties: %{
|
||||||
|
urls: %Schema{
|
||||||
|
type: :array,
|
||||||
|
items: %Schema{
|
||||||
|
type: :string,
|
||||||
|
format: :uri,
|
||||||
|
description: "MediaProxy URLs"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
end
|
||||||
|
end
|
|
@ -5,22 +5,34 @@
|
||||||
defmodule Pleroma.Web.MediaProxy.Invalidation do
|
defmodule Pleroma.Web.MediaProxy.Invalidation do
|
||||||
@moduledoc false
|
@moduledoc false
|
||||||
|
|
||||||
@callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()}
|
@callback purge(list(String.t()), Keyword.t()) :: {:ok, list(String.t())} | {:error, String.t()}
|
||||||
|
|
||||||
alias Pleroma.Config
|
alias Pleroma.Config
|
||||||
|
alias Pleroma.Web.MediaProxy
|
||||||
|
|
||||||
@spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()}
|
@spec enabled?() :: boolean()
|
||||||
|
def enabled?, do: Config.get([:media_proxy, :invalidation, :enabled])
|
||||||
|
|
||||||
|
@spec purge(list(String.t()) | String.t()) :: {:ok, list(String.t())} | {:error, String.t()}
|
||||||
def purge(urls) do
|
def purge(urls) do
|
||||||
[:media_proxy, :invalidation, :enabled]
|
prepared_urls = prepare_urls(urls)
|
||||||
|> Config.get()
|
|
||||||
|> do_purge(urls)
|
if enabled?() do
|
||||||
|
do_purge(prepared_urls)
|
||||||
|
else
|
||||||
|
{:ok, prepared_urls}
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
defp do_purge(true, urls) do
|
defp do_purge(urls) do
|
||||||
provider = Config.get([:media_proxy, :invalidation, :provider])
|
provider = Config.get([:media_proxy, :invalidation, :provider])
|
||||||
options = Config.get(provider)
|
options = Config.get(provider)
|
||||||
provider.purge(urls, options)
|
provider.purge(urls, options)
|
||||||
end
|
end
|
||||||
|
|
||||||
defp do_purge(_, _), do: :ok
|
def prepare_urls(urls) do
|
||||||
|
urls
|
||||||
|
|> List.wrap()
|
||||||
|
|> Enum.map(&MediaProxy.url/1)
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -9,10 +9,10 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do
|
||||||
require Logger
|
require Logger
|
||||||
|
|
||||||
@impl Pleroma.Web.MediaProxy.Invalidation
|
@impl Pleroma.Web.MediaProxy.Invalidation
|
||||||
def purge(urls, opts) do
|
def purge(urls, opts \\ []) do
|
||||||
method = Map.get(opts, :method, :purge)
|
method = Keyword.get(opts, :method, :purge)
|
||||||
headers = Map.get(opts, :headers, [])
|
headers = Keyword.get(opts, :headers, [])
|
||||||
options = Map.get(opts, :options, [])
|
options = Keyword.get(opts, :options, [])
|
||||||
|
|
||||||
Logger.debug("Running cache purge: #{inspect(urls)}")
|
Logger.debug("Running cache purge: #{inspect(urls)}")
|
||||||
|
|
||||||
|
@ -22,7 +22,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do
|
||||||
end
|
end
|
||||||
end)
|
end)
|
||||||
|
|
||||||
{:ok, "success"}
|
{:ok, urls}
|
||||||
end
|
end
|
||||||
|
|
||||||
defp do_purge(method, url, headers, options) do
|
defp do_purge(method, url, headers, options) do
|
||||||
|
|
|
@ -10,32 +10,34 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do
|
||||||
require Logger
|
require Logger
|
||||||
|
|
||||||
@impl Pleroma.Web.MediaProxy.Invalidation
|
@impl Pleroma.Web.MediaProxy.Invalidation
|
||||||
def purge(urls, %{script_path: script_path} = _options) do
|
def purge(urls, opts \\ []) do
|
||||||
args =
|
args =
|
||||||
urls
|
urls
|
||||||
|> List.wrap()
|
|> List.wrap()
|
||||||
|> Enum.uniq()
|
|> Enum.uniq()
|
||||||
|> Enum.join(" ")
|
|> Enum.join(" ")
|
||||||
|
|
||||||
|
opts
|
||||||
|
|> Keyword.get(:script_path)
|
||||||
|
|> do_purge([args])
|
||||||
|
|> handle_result(urls)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp do_purge(script_path, args) when is_binary(script_path) do
|
||||||
path = Path.expand(script_path)
|
path = Path.expand(script_path)
|
||||||
|
Logger.debug("Running cache purge: #{inspect(args)}, #{inspect(path)}")
|
||||||
Logger.debug("Running cache purge: #{inspect(urls)}, #{path}")
|
|
||||||
|
|
||||||
case do_purge(path, [args]) do
|
|
||||||
{result, exit_status} when exit_status > 0 ->
|
|
||||||
Logger.error("Error while cache purge: #{inspect(result)}")
|
|
||||||
{:error, inspect(result)}
|
|
||||||
|
|
||||||
_ ->
|
|
||||||
{:ok, "success"}
|
|
||||||
end
|
|
||||||
end
|
|
||||||
|
|
||||||
def purge(_, _), do: {:error, "not found script path"}
|
|
||||||
|
|
||||||
defp do_purge(path, args) do
|
|
||||||
System.cmd(path, args)
|
System.cmd(path, args)
|
||||||
rescue
|
rescue
|
||||||
error -> {inspect(error), 1}
|
error -> error
|
||||||
|
end
|
||||||
|
|
||||||
|
defp do_purge(_, _), do: {:error, "not found script path"}
|
||||||
|
|
||||||
|
defp handle_result({_result, 0}, urls), do: {:ok, urls}
|
||||||
|
defp handle_result({:error, error}, urls), do: handle_result(error, urls)
|
||||||
|
|
||||||
|
defp handle_result(error, _) do
|
||||||
|
Logger.error("Error while cache purge: #{inspect(error)}")
|
||||||
|
{:error, inspect(error)}
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -6,20 +6,53 @@ defmodule Pleroma.Web.MediaProxy do
|
||||||
alias Pleroma.Config
|
alias Pleroma.Config
|
||||||
alias Pleroma.Upload
|
alias Pleroma.Upload
|
||||||
alias Pleroma.Web
|
alias Pleroma.Web
|
||||||
|
alias Pleroma.Web.MediaProxy.Invalidation
|
||||||
|
|
||||||
@base64_opts [padding: false]
|
@base64_opts [padding: false]
|
||||||
|
|
||||||
|
@spec in_banned_urls(String.t()) :: boolean()
|
||||||
|
def in_banned_urls(url), do: elem(Cachex.exists?(:banned_urls_cache, url(url)), 1)
|
||||||
|
|
||||||
|
def remove_from_banned_urls(urls) when is_list(urls) do
|
||||||
|
Cachex.execute!(:banned_urls_cache, fn cache ->
|
||||||
|
Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1))
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def remove_from_banned_urls(url) when is_binary(url) do
|
||||||
|
Cachex.del(:banned_urls_cache, url(url))
|
||||||
|
end
|
||||||
|
|
||||||
|
def put_in_banned_urls(urls) when is_list(urls) do
|
||||||
|
Cachex.execute!(:banned_urls_cache, fn cache ->
|
||||||
|
Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true))
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
def put_in_banned_urls(url) when is_binary(url) do
|
||||||
|
Cachex.put(:banned_urls_cache, url(url), true)
|
||||||
|
end
|
||||||
|
|
||||||
def url(url) when is_nil(url) or url == "", do: nil
|
def url(url) when is_nil(url) or url == "", do: nil
|
||||||
def url("/" <> _ = url), do: url
|
def url("/" <> _ = url), do: url
|
||||||
|
|
||||||
def url(url) do
|
def url(url) do
|
||||||
if disabled?() or local?(url) or whitelisted?(url) do
|
if disabled?() or not url_proxiable?(url) do
|
||||||
url
|
url
|
||||||
else
|
else
|
||||||
encode_url(url)
|
encode_url(url)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@spec url_proxiable?(String.t()) :: boolean()
|
||||||
|
def url_proxiable?(url) do
|
||||||
|
if local?(url) or whitelisted?(url) do
|
||||||
|
false
|
||||||
|
else
|
||||||
|
true
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
defp disabled?, do: !Config.get([:media_proxy, :enabled], false)
|
defp disabled?, do: !Config.get([:media_proxy, :enabled], false)
|
||||||
|
|
||||||
defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
|
defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
|
||||||
|
|
|
@ -14,10 +14,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
|
||||||
with config <- Pleroma.Config.get([:media_proxy], []),
|
with config <- Pleroma.Config.get([:media_proxy], []),
|
||||||
true <- Keyword.get(config, :enabled, false),
|
true <- Keyword.get(config, :enabled, false),
|
||||||
{:ok, url} <- MediaProxy.decode_url(sig64, url64),
|
{:ok, url} <- MediaProxy.decode_url(sig64, url64),
|
||||||
|
{_, false} <- {:in_banned_urls, MediaProxy.in_banned_urls(url)},
|
||||||
:ok <- filename_matches(params, conn.request_path, url) do
|
:ok <- filename_matches(params, conn.request_path, url) do
|
||||||
ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
|
ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
|
||||||
else
|
else
|
||||||
false ->
|
error when error in [false, {:in_banned_urls, true}] ->
|
||||||
send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
|
send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
|
||||||
|
|
||||||
{:error, :invalid_signature} ->
|
{:error, :invalid_signature} ->
|
||||||
|
|
|
@ -209,6 +209,10 @@ defmodule Pleroma.Web.Router do
|
||||||
post("/oauth_app", OAuthAppController, :create)
|
post("/oauth_app", OAuthAppController, :create)
|
||||||
patch("/oauth_app/:id", OAuthAppController, :update)
|
patch("/oauth_app/:id", OAuthAppController, :update)
|
||||||
delete("/oauth_app/:id", OAuthAppController, :delete)
|
delete("/oauth_app/:id", OAuthAppController, :delete)
|
||||||
|
|
||||||
|
get("/media_proxy_caches", MediaProxyCacheController, :index)
|
||||||
|
post("/media_proxy_caches/delete", MediaProxyCacheController, :delete)
|
||||||
|
post("/media_proxy_caches/purge", MediaProxyCacheController, :purge)
|
||||||
end
|
end
|
||||||
|
|
||||||
scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do
|
scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do
|
||||||
|
|
|
@ -18,13 +18,19 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
|
||||||
},
|
},
|
||||||
_job
|
_job
|
||||||
) do
|
) do
|
||||||
hrefs =
|
attachments
|
||||||
Enum.flat_map(attachments, fn attachment ->
|
|> Enum.flat_map(fn item -> Enum.map(item["url"], & &1["href"]) end)
|
||||||
Enum.map(attachment["url"], & &1["href"])
|
|> fetch_objects
|
||||||
end)
|
|> prepare_objects(actor, Enum.map(attachments, & &1["name"]))
|
||||||
|
|> filter_objects
|
||||||
|
|> do_clean
|
||||||
|
|
||||||
names = Enum.map(attachments, & &1["name"])
|
{:ok, :success}
|
||||||
|
end
|
||||||
|
|
||||||
|
def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}
|
||||||
|
|
||||||
|
defp do_clean({object_ids, attachment_urls}) do
|
||||||
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
|
uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
|
||||||
|
|
||||||
prefix =
|
prefix =
|
||||||
|
@ -39,22 +45,35 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
|
||||||
"/"
|
"/"
|
||||||
)
|
)
|
||||||
|
|
||||||
# find all objects for copies of the attachments, name and actor doesn't matter here
|
Enum.each(attachment_urls, fn href ->
|
||||||
object_ids_and_hrefs =
|
href
|
||||||
from(o in Object,
|
|> String.trim_leading("#{base_url}/#{prefix}")
|
||||||
where:
|
|> uploader.delete_file()
|
||||||
fragment(
|
end)
|
||||||
"to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)",
|
|
||||||
o.data,
|
delete_objects(object_ids)
|
||||||
o.data,
|
end
|
||||||
^hrefs
|
|
||||||
)
|
defp delete_objects([_ | _] = object_ids) do
|
||||||
)
|
Repo.delete_all(from(o in Object, where: o.id in ^object_ids))
|
||||||
# The query above can be time consumptive on large instances until we
|
end
|
||||||
# refactor how uploads are stored
|
|
||||||
|> Repo.all(timeout: :infinity)
|
defp delete_objects(_), do: :ok
|
||||||
|
|
||||||
# we should delete 1 object for any given attachment, but don't delete
|
# we should delete 1 object for any given attachment, but don't delete
|
||||||
# files if there are more than 1 object for it
|
# files if there are more than 1 object for it
|
||||||
|
defp filter_objects(objects) do
|
||||||
|
Enum.reduce(objects, {[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} ->
|
||||||
|
with 1 <- count do
|
||||||
|
{ids ++ [id], hrefs ++ [href]}
|
||||||
|
else
|
||||||
|
_ -> {ids ++ [id], hrefs}
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp prepare_objects(objects, actor, names) do
|
||||||
|
objects
|
||||||
|> Enum.reduce(%{}, fn %{
|
|> Enum.reduce(%{}, fn %{
|
||||||
id: id,
|
id: id,
|
||||||
data: %{
|
data: %{
|
||||||
|
@ -76,31 +95,20 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do
|
||||||
end
|
end
|
||||||
end)
|
end)
|
||||||
end)
|
end)
|
||||||
|> Enum.map(fn {href, %{id: id, count: count}} ->
|
|
||||||
# only delete files that have single instance
|
|
||||||
with 1 <- count do
|
|
||||||
href
|
|
||||||
|> String.trim_leading("#{base_url}/#{prefix}")
|
|
||||||
|> uploader.delete_file()
|
|
||||||
|
|
||||||
{id, href}
|
|
||||||
else
|
|
||||||
_ -> {id, nil}
|
|
||||||
end
|
|
||||||
end)
|
|
||||||
|
|
||||||
object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end)
|
|
||||||
|
|
||||||
from(o in Object, where: o.id in ^object_ids)
|
|
||||||
|> Repo.delete_all()
|
|
||||||
|
|
||||||
object_ids_and_hrefs
|
|
||||||
|> Enum.filter(fn {_, href} -> not is_nil(href) end)
|
|
||||||
|> Enum.map(&elem(&1, 1))
|
|
||||||
|> Pleroma.Web.MediaProxy.Invalidation.purge()
|
|
||||||
|
|
||||||
{:ok, :success}
|
|
||||||
end
|
end
|
||||||
|
|
||||||
def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip}
|
defp fetch_objects(hrefs) do
|
||||||
|
from(o in Object,
|
||||||
|
where:
|
||||||
|
fragment(
|
||||||
|
"to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)",
|
||||||
|
o.data,
|
||||||
|
o.data,
|
||||||
|
^hrefs
|
||||||
|
)
|
||||||
|
)
|
||||||
|
# The query above can be time consumptive on large instances until we
|
||||||
|
# refactor how uploads are stored
|
||||||
|
|> Repo.all(timeout: :infinity)
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -0,0 +1,145 @@
|
||||||
|
# Pleroma: A lightweight social networking server
|
||||||
|
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
|
||||||
|
# SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
|
||||||
|
defmodule Pleroma.Web.AdminAPI.MediaProxyCacheControllerTest do
|
||||||
|
use Pleroma.Web.ConnCase
|
||||||
|
|
||||||
|
import Pleroma.Factory
|
||||||
|
import Mock
|
||||||
|
|
||||||
|
alias Pleroma.Web.MediaProxy
|
||||||
|
|
||||||
|
setup do: clear_config([:media_proxy])
|
||||||
|
|
||||||
|
setup do
|
||||||
|
on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
|
||||||
|
end
|
||||||
|
|
||||||
|
setup do
|
||||||
|
admin = insert(:user, is_admin: true)
|
||||||
|
token = insert(:oauth_admin_token, user: admin)
|
||||||
|
|
||||||
|
conn =
|
||||||
|
build_conn()
|
||||||
|
|> assign(:user, admin)
|
||||||
|
|> assign(:token, token)
|
||||||
|
|
||||||
|
Config.put([:media_proxy, :enabled], true)
|
||||||
|
Config.put([:media_proxy, :invalidation, :enabled], true)
|
||||||
|
Config.put([:media_proxy, :invalidation, :provider], MediaProxy.Invalidation.Script)
|
||||||
|
|
||||||
|
{:ok, %{admin: admin, token: token, conn: conn}}
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "GET /api/pleroma/admin/media_proxy_caches" do
|
||||||
|
test "shows banned MediaProxy URLs", %{conn: conn} do
|
||||||
|
MediaProxy.put_in_banned_urls([
|
||||||
|
"http://localhost:4001/media/a688346.jpg",
|
||||||
|
"http://localhost:4001/media/fb1f4d.jpg"
|
||||||
|
])
|
||||||
|
|
||||||
|
MediaProxy.put_in_banned_urls("http://localhost:4001/media/gb1f44.jpg")
|
||||||
|
MediaProxy.put_in_banned_urls("http://localhost:4001/media/tb13f47.jpg")
|
||||||
|
MediaProxy.put_in_banned_urls("http://localhost:4001/media/wb1f46.jpg")
|
||||||
|
|
||||||
|
response =
|
||||||
|
conn
|
||||||
|
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2")
|
||||||
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
|
assert response["urls"] == [
|
||||||
|
"http://localhost:4001/media/fb1f4d.jpg",
|
||||||
|
"http://localhost:4001/media/a688346.jpg"
|
||||||
|
]
|
||||||
|
|
||||||
|
response =
|
||||||
|
conn
|
||||||
|
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=2")
|
||||||
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
|
assert response["urls"] == [
|
||||||
|
"http://localhost:4001/media/gb1f44.jpg",
|
||||||
|
"http://localhost:4001/media/tb13f47.jpg"
|
||||||
|
]
|
||||||
|
|
||||||
|
response =
|
||||||
|
conn
|
||||||
|
|> get("/api/pleroma/admin/media_proxy_caches?page_size=2&page=3")
|
||||||
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
|
assert response["urls"] == ["http://localhost:4001/media/wb1f46.jpg"]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "POST /api/pleroma/admin/media_proxy_caches/delete" do
|
||||||
|
test "deleted MediaProxy URLs from banned", %{conn: conn} do
|
||||||
|
MediaProxy.put_in_banned_urls([
|
||||||
|
"http://localhost:4001/media/a688346.jpg",
|
||||||
|
"http://localhost:4001/media/fb1f4d.jpg"
|
||||||
|
])
|
||||||
|
|
||||||
|
response =
|
||||||
|
conn
|
||||||
|
|> put_req_header("content-type", "application/json")
|
||||||
|
|> post("/api/pleroma/admin/media_proxy_caches/delete", %{
|
||||||
|
urls: ["http://localhost:4001/media/a688346.jpg"]
|
||||||
|
})
|
||||||
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
|
assert response["urls"] == ["http://localhost:4001/media/a688346.jpg"]
|
||||||
|
refute MediaProxy.in_banned_urls("http://localhost:4001/media/a688346.jpg")
|
||||||
|
assert MediaProxy.in_banned_urls("http://localhost:4001/media/fb1f4d.jpg")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "POST /api/pleroma/admin/media_proxy_caches/purge" do
|
||||||
|
test "perform invalidates cache of MediaProxy", %{conn: conn} do
|
||||||
|
urls = [
|
||||||
|
"http://example.com/media/a688346.jpg",
|
||||||
|
"http://example.com/media/fb1f4d.jpg"
|
||||||
|
]
|
||||||
|
|
||||||
|
with_mocks [
|
||||||
|
{MediaProxy.Invalidation.Script, [],
|
||||||
|
[
|
||||||
|
purge: fn _, _ -> {"ok", 0} end
|
||||||
|
]}
|
||||||
|
] do
|
||||||
|
response =
|
||||||
|
conn
|
||||||
|
|> put_req_header("content-type", "application/json")
|
||||||
|
|> post("/api/pleroma/admin/media_proxy_caches/purge", %{urls: urls, ban: false})
|
||||||
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
|
assert response["urls"] == urls
|
||||||
|
|
||||||
|
refute MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
|
||||||
|
refute MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
test "perform invalidates cache of MediaProxy and adds url to banned", %{conn: conn} do
|
||||||
|
urls = [
|
||||||
|
"http://example.com/media/a688346.jpg",
|
||||||
|
"http://example.com/media/fb1f4d.jpg"
|
||||||
|
]
|
||||||
|
|
||||||
|
with_mocks [{MediaProxy.Invalidation.Script, [], [purge: fn _, _ -> {"ok", 0} end]}] do
|
||||||
|
response =
|
||||||
|
conn
|
||||||
|
|> put_req_header("content-type", "application/json")
|
||||||
|
|> post("/api/pleroma/admin/media_proxy_caches/purge", %{
|
||||||
|
urls: urls,
|
||||||
|
ban: true
|
||||||
|
})
|
||||||
|
|> json_response_and_validate_schema(200)
|
||||||
|
|
||||||
|
assert response["urls"] == urls
|
||||||
|
|
||||||
|
assert MediaProxy.in_banned_urls("http://example.com/media/a688346.jpg")
|
||||||
|
assert MediaProxy.in_banned_urls("http://example.com/media/fb1f4d.jpg")
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
64
test/web/media_proxy/invalidation_test.exs
Normal file
64
test/web/media_proxy/invalidation_test.exs
Normal file
|
@ -0,0 +1,64 @@
|
||||||
|
defmodule Pleroma.Web.MediaProxy.InvalidationTest do
|
||||||
|
use ExUnit.Case
|
||||||
|
use Pleroma.Tests.Helpers
|
||||||
|
|
||||||
|
alias Pleroma.Config
|
||||||
|
alias Pleroma.Web.MediaProxy.Invalidation
|
||||||
|
|
||||||
|
import ExUnit.CaptureLog
|
||||||
|
import Mock
|
||||||
|
import Tesla.Mock
|
||||||
|
|
||||||
|
setup do: clear_config([:media_proxy])
|
||||||
|
|
||||||
|
setup do
|
||||||
|
on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "Invalidation.Http" do
|
||||||
|
test "perform request to clear cache" do
|
||||||
|
Config.put([:media_proxy, :enabled], false)
|
||||||
|
Config.put([:media_proxy, :invalidation, :enabled], true)
|
||||||
|
Config.put([:media_proxy, :invalidation, :provider], Invalidation.Http)
|
||||||
|
|
||||||
|
Config.put([Invalidation.Http], method: :purge, headers: [{"x-refresh", 1}])
|
||||||
|
image_url = "http://example.com/media/example.jpg"
|
||||||
|
Pleroma.Web.MediaProxy.put_in_banned_urls(image_url)
|
||||||
|
|
||||||
|
mock(fn
|
||||||
|
%{
|
||||||
|
method: :purge,
|
||||||
|
url: "http://example.com/media/example.jpg",
|
||||||
|
headers: [{"x-refresh", 1}]
|
||||||
|
} ->
|
||||||
|
%Tesla.Env{status: 200}
|
||||||
|
end)
|
||||||
|
|
||||||
|
assert capture_log(fn ->
|
||||||
|
assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
|
||||||
|
assert Invalidation.purge([image_url]) == {:ok, [image_url]}
|
||||||
|
assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
|
||||||
|
end) =~ "Running cache purge: [\"#{image_url}\"]"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "Invalidation.Script" do
|
||||||
|
test "run script to clear cache" do
|
||||||
|
Config.put([:media_proxy, :enabled], false)
|
||||||
|
Config.put([:media_proxy, :invalidation, :enabled], true)
|
||||||
|
Config.put([:media_proxy, :invalidation, :provider], Invalidation.Script)
|
||||||
|
Config.put([Invalidation.Script], script_path: "purge-nginx")
|
||||||
|
|
||||||
|
image_url = "http://example.com/media/example.jpg"
|
||||||
|
Pleroma.Web.MediaProxy.put_in_banned_urls(image_url)
|
||||||
|
|
||||||
|
with_mocks [{System, [], [cmd: fn _, _ -> {"ok", 0} end]}] do
|
||||||
|
assert capture_log(fn ->
|
||||||
|
assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
|
||||||
|
assert Invalidation.purge([image_url]) == {:ok, [image_url]}
|
||||||
|
assert Pleroma.Web.MediaProxy.in_banned_urls(image_url)
|
||||||
|
end) =~ "Running cache purge: [\"#{image_url}\"]"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
|
@ -5,6 +5,10 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do
|
||||||
import ExUnit.CaptureLog
|
import ExUnit.CaptureLog
|
||||||
import Tesla.Mock
|
import Tesla.Mock
|
||||||
|
|
||||||
|
setup do
|
||||||
|
on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
|
||||||
|
end
|
||||||
|
|
||||||
test "logs hasn't error message when request is valid" do
|
test "logs hasn't error message when request is valid" do
|
||||||
mock(fn
|
mock(fn
|
||||||
%{method: :purge, url: "http://example.com/media/example.jpg"} ->
|
%{method: :purge, url: "http://example.com/media/example.jpg"} ->
|
||||||
|
@ -14,8 +18,8 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do
|
||||||
refute capture_log(fn ->
|
refute capture_log(fn ->
|
||||||
assert Invalidation.Http.purge(
|
assert Invalidation.Http.purge(
|
||||||
["http://example.com/media/example.jpg"],
|
["http://example.com/media/example.jpg"],
|
||||||
%{}
|
[]
|
||||||
) == {:ok, "success"}
|
) == {:ok, ["http://example.com/media/example.jpg"]}
|
||||||
end) =~ "Error while cache purge"
|
end) =~ "Error while cache purge"
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@ -28,8 +32,8 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.HttpTest do
|
||||||
assert capture_log(fn ->
|
assert capture_log(fn ->
|
||||||
assert Invalidation.Http.purge(
|
assert Invalidation.Http.purge(
|
||||||
["http://example.com/media/example1.jpg"],
|
["http://example.com/media/example1.jpg"],
|
||||||
%{}
|
[]
|
||||||
) == {:ok, "success"}
|
) == {:ok, ["http://example.com/media/example1.jpg"]}
|
||||||
end) =~ "Error while cache purge: url - http://example.com/media/example1.jpg"
|
end) =~ "Error while cache purge: url - http://example.com/media/example1.jpg"
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -4,17 +4,23 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.ScriptTest do
|
||||||
|
|
||||||
import ExUnit.CaptureLog
|
import ExUnit.CaptureLog
|
||||||
|
|
||||||
|
setup do
|
||||||
|
on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
|
||||||
|
end
|
||||||
|
|
||||||
test "it logger error when script not found" do
|
test "it logger error when script not found" do
|
||||||
assert capture_log(fn ->
|
assert capture_log(fn ->
|
||||||
assert Invalidation.Script.purge(
|
assert Invalidation.Script.purge(
|
||||||
["http://example.com/media/example.jpg"],
|
["http://example.com/media/example.jpg"],
|
||||||
%{script_path: "./example"}
|
script_path: "./example"
|
||||||
) == {:error, "\"%ErlangError{original: :enoent}\""}
|
) == {:error, "%ErlangError{original: :enoent}"}
|
||||||
end) =~ "Error while cache purge: \"%ErlangError{original: :enoent}\""
|
end) =~ "Error while cache purge: %ErlangError{original: :enoent}"
|
||||||
|
|
||||||
|
capture_log(fn ->
|
||||||
assert Invalidation.Script.purge(
|
assert Invalidation.Script.purge(
|
||||||
["http://example.com/media/example.jpg"],
|
["http://example.com/media/example.jpg"],
|
||||||
%{}
|
[]
|
||||||
) == {:error, "not found script path"}
|
) == {:error, "\"not found script path\""}
|
||||||
|
end)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
@ -10,6 +10,10 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do
|
||||||
setup do: clear_config(:media_proxy)
|
setup do: clear_config(:media_proxy)
|
||||||
setup do: clear_config([Pleroma.Web.Endpoint, :secret_key_base])
|
setup do: clear_config([Pleroma.Web.Endpoint, :secret_key_base])
|
||||||
|
|
||||||
|
setup do
|
||||||
|
on_exit(fn -> Cachex.clear(:banned_urls_cache) end)
|
||||||
|
end
|
||||||
|
|
||||||
test "it returns 404 when MediaProxy disabled", %{conn: conn} do
|
test "it returns 404 when MediaProxy disabled", %{conn: conn} do
|
||||||
Config.put([:media_proxy, :enabled], false)
|
Config.put([:media_proxy, :enabled], false)
|
||||||
|
|
||||||
|
@ -66,4 +70,16 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyControllerTest do
|
||||||
assert %Plug.Conn{status: :success} = get(conn, url)
|
assert %Plug.Conn{status: :success} = get(conn, url)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
test "it returns 404 when url contains in banned_urls cache", %{conn: conn} do
|
||||||
|
Config.put([:media_proxy, :enabled], true)
|
||||||
|
Config.put([Pleroma.Web.Endpoint, :secret_key_base], "00000000000")
|
||||||
|
url = Pleroma.Web.MediaProxy.encode_url("https://google.fn/test.png")
|
||||||
|
Pleroma.Web.MediaProxy.put_in_banned_urls("https://google.fn/test.png")
|
||||||
|
|
||||||
|
with_mock Pleroma.ReverseProxy,
|
||||||
|
call: fn _conn, _url, _opts -> %Plug.Conn{status: :success} end do
|
||||||
|
assert %Plug.Conn{status: 404, resp_body: "Not Found"} = get(conn, url)
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
Loading…
Reference in a new issue