Merge branch 'develop' into feature/account-export

This commit is contained in:
Mark Felder 2020-10-27 12:54:33 -05:00
commit 291189357b
7 changed files with 191 additions and 9 deletions

View file

@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details). - Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
- Pleroma API: Importing the mutes users from CSV files. - Pleroma API: Importing the mutes users from CSV files.
- Experimental websocket-based federation between Pleroma instances. - Experimental websocket-based federation between Pleroma instances.
- App metrics: ability to restrict access to specified IP whitelist.
- Account backup - Account backup
### Changed ### Changed
@ -19,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- **Breaking** Requires `libmagic` (or `file`) to guess file types. - **Breaking** Requires `libmagic` (or `file`) to guess file types.
- **Breaking:** Pleroma Admin API: emoji packs and files routes changed. - **Breaking:** Pleroma Admin API: emoji packs and files routes changed.
- **Breaking:** Sensitive/NSFW statuses no longer disable link previews. - **Breaking:** Sensitive/NSFW statuses no longer disable link previews.
- **Breaking:** App metrics endpoint (`/api/pleroma/app_metrics`) is disabled by default, check `docs/API/prometheus.md` on enabling and configuring.
- Search: Users are now findable by their urls. - Search: Users are now findable by their urls.
- Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated. - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
- Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated. - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.

View file

@ -636,7 +636,12 @@ config :pleroma, Pleroma.Emails.UserEmail,
config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: false config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: false
config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics" config :prometheus, Pleroma.Web.Endpoint.MetricsExporter,
enabled: false,
auth: false,
ip_whitelist: [],
path: "/api/pleroma/app_metrics",
format: :text
config :pleroma, Pleroma.ScheduledActivity, config :pleroma, Pleroma.ScheduledActivity,
daily_user_limit: 25, daily_user_limit: 25,

View file

@ -3742,5 +3742,42 @@ config :pleroma, :config_description, [
suggestions: [7] suggestions: [7]
} }
] ]
},
%{
group: :prometheus,
key: Pleroma.Web.Endpoint.MetricsExporter,
type: :group,
description: "Prometheus app metrics endpoint configuration",
children: [
%{
key: :enabled,
type: :boolean,
description: "[Pleroma extension] Enables app metrics endpoint."
},
%{
key: :ip_whitelist,
type: [{:list, :string}, {:list, :charlist}, {:list, :tuple}],
description:
"[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses."
},
%{
key: :auth,
type: [:boolean, :tuple],
description: "Enables HTTP Basic Auth for app metrics endpoint.",
suggestion: [false, {:basic, "myusername", "mypassword"}]
},
%{
key: :path,
type: :string,
description: "App metrics endpoint URI path.",
suggestions: ["/api/pleroma/app_metrics"]
},
%{
key: :format,
type: :atom,
description: "App metrics endpoint output format.",
suggestions: [:text, :protobuf]
}
]
} }
] ]

View file

@ -2,15 +2,37 @@
Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library. Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library.
Config example:
```
config :prometheus, Pleroma.Web.Endpoint.MetricsExporter,
enabled: true,
auth: {:basic, "myusername", "mypassword"},
ip_whitelist: ["127.0.0.1"],
path: "/api/pleroma/app_metrics",
format: :text
```
* `enabled` (Pleroma extension) enables the endpoint
* `ip_whitelist` (Pleroma extension) could be used to restrict access only to specified IPs
* `auth` sets the authentication (`false` for no auth; configurable to HTTP Basic Auth, see [prometheus-plugs](https://github.com/deadtrickster/prometheus-plugs#exporting) documentation)
* `format` sets the output format (`:text` or `:protobuf`)
* `path` sets the path to app metrics page
## `/api/pleroma/app_metrics` ## `/api/pleroma/app_metrics`
### Exports Prometheus application metrics ### Exports Prometheus application metrics
* Method: `GET` * Method: `GET`
* Authentication: not required * Authentication: not required by default (see configuration options above)
* Params: none * Params: none
* Response: JSON * Response: text
## Grafana ## Grafana
### Config example ### Config example
The following is a config example to use with [Grafana](https://grafana.com) The following is a config example to use with [Grafana](https://grafana.com)
``` ```

View file

@ -0,0 +1,19 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Helpers.InetHelper do
def parse_address(ip) when is_tuple(ip) do
{:ok, ip}
end
def parse_address(ip) when is_binary(ip) do
ip
|> String.to_charlist()
|> parse_address()
end
def parse_address(ip) do
:inet.parse_address(ip)
end
end

View file

@ -7,6 +7,8 @@ defmodule Pleroma.Web.Endpoint do
require Pleroma.Constants require Pleroma.Constants
alias Pleroma.Config
socket("/socket", Pleroma.Web.UserSocket) socket("/socket", Pleroma.Web.UserSocket)
plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint]) plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint])
@ -88,19 +90,19 @@ defmodule Pleroma.Web.Endpoint do
plug(Plug.Parsers, plug(Plug.Parsers,
parsers: [ parsers: [
:urlencoded, :urlencoded,
{:multipart, length: {Pleroma.Config, :get, [[:instance, :upload_limit]]}}, {:multipart, length: {Config, :get, [[:instance, :upload_limit]]}},
:json :json
], ],
pass: ["*/*"], pass: ["*/*"],
json_decoder: Jason, json_decoder: Jason,
length: Pleroma.Config.get([:instance, :upload_limit]), length: Config.get([:instance, :upload_limit]),
body_reader: {Pleroma.Web.Plugs.DigestPlug, :read_body, []} body_reader: {Pleroma.Web.Plugs.DigestPlug, :read_body, []}
) )
plug(Plug.MethodOverride) plug(Plug.MethodOverride)
plug(Plug.Head) plug(Plug.Head)
secure_cookies = Pleroma.Config.get([__MODULE__, :secure_cookie_flag]) secure_cookies = Config.get([__MODULE__, :secure_cookie_flag])
cookie_name = cookie_name =
if secure_cookies, if secure_cookies,
@ -108,7 +110,7 @@ defmodule Pleroma.Web.Endpoint do
else: "pleroma_key" else: "pleroma_key"
extra = extra =
Pleroma.Config.get([__MODULE__, :extra_cookie_attrs]) Config.get([__MODULE__, :extra_cookie_attrs])
|> Enum.join(";") |> Enum.join(";")
# The session will be stored in the cookie and signed, # The session will be stored in the cookie and signed,
@ -118,7 +120,7 @@ defmodule Pleroma.Web.Endpoint do
Plug.Session, Plug.Session,
store: :cookie, store: :cookie,
key: cookie_name, key: cookie_name,
signing_salt: Pleroma.Config.get([__MODULE__, :signing_salt], "CqaoopA2"), signing_salt: Config.get([__MODULE__, :signing_salt], "CqaoopA2"),
http_only: true, http_only: true,
secure: secure_cookies, secure: secure_cookies,
extra: extra extra: extra
@ -138,8 +140,34 @@ defmodule Pleroma.Web.Endpoint do
use Prometheus.PlugExporter use Prometheus.PlugExporter
end end
defmodule MetricsExporterCaller do
@behaviour Plug
def init(opts), do: opts
def call(conn, opts) do
prometheus_config = Application.get_env(:prometheus, MetricsExporter, [])
ip_whitelist = List.wrap(prometheus_config[:ip_whitelist])
cond do
!prometheus_config[:enabled] ->
conn
ip_whitelist != [] and
!Enum.find(ip_whitelist, fn ip ->
Pleroma.Helpers.InetHelper.parse_address(ip) == {:ok, conn.remote_ip}
end) ->
conn
true ->
MetricsExporter.call(conn, opts)
end
end
end
plug(PipelineInstrumenter) plug(PipelineInstrumenter)
plug(MetricsExporter)
plug(MetricsExporterCaller)
plug(Pleroma.Web.Router) plug(Pleroma.Web.Router)

View file

@ -0,0 +1,69 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.Endpoint.MetricsExporterTest do
use Pleroma.Web.ConnCase
alias Pleroma.Web.Endpoint.MetricsExporter
defp config do
Application.get_env(:prometheus, MetricsExporter)
end
describe "with default config" do
test "does NOT expose app metrics", %{conn: conn} do
conn
|> get(config()[:path])
|> json_response(404)
end
end
describe "when enabled" do
setup do
initial_config = config()
on_exit(fn -> Application.put_env(:prometheus, MetricsExporter, initial_config) end)
Application.put_env(
:prometheus,
MetricsExporter,
Keyword.put(initial_config, :enabled, true)
)
end
test "serves app metrics", %{conn: conn} do
conn = get(conn, config()[:path])
assert response = response(conn, 200)
for metric <- [
"http_requests_total",
"http_request_duration_microseconds",
"phoenix_controller_render_duration",
"phoenix_controller_call_duration",
"telemetry_scrape_duration",
"erlang_vm_memory_atom_bytes_total"
] do
assert response =~ ~r/#{metric}/
end
end
test "when IP whitelist configured, " <>
"serves app metrics only if client IP is whitelisted",
%{conn: conn} do
Application.put_env(
:prometheus,
MetricsExporter,
Keyword.put(config(), :ip_whitelist, ["127.127.127.127", {1, 1, 1, 1}, '255.255.255.255'])
)
conn
|> get(config()[:path])
|> json_response(404)
conn
|> Map.put(:remote_ip, {127, 127, 127, 127})
|> get(config()[:path])
|> response(200)
end
end
end