From 524fb0e4c2561f4a2e4c8e58519df991f034c901 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 18 Oct 2020 21:22:21 +0300 Subject: [PATCH 1/4] [#1668] Restricted access to app metrics endpoint by default. Added ability to configure IP whitelist for this endpoint. Added tests and documentation. --- CHANGELOG.md | 2 + config/config.exs | 7 +- docs/API/prometheus.md | 26 ++++++- lib/pleroma/helpers/inet_helper.ex | 19 +++++ lib/pleroma/web/endpoint.ex | 40 +++++++++-- .../web/endpoint/metrics_exporter_test.exs | 69 +++++++++++++++++++ 6 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 lib/pleroma/helpers/inet_helper.ex create mode 100644 test/pleroma/web/endpoint/metrics_exporter_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 05e94581a..9f6a31f23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,12 +12,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details). - Pleroma API: Importing the mutes users from CSV files. - Experimental websocket-based federation between Pleroma instances. +- App metrics: ability to restrict access to specified IP whitelist. ### Changed - **Breaking** Requires `libmagic` (or `file`) to guess file types. - **Breaking:** Pleroma Admin API: emoji packs and files routes changed. - **Breaking:** Sensitive/NSFW statuses no longer disable link previews. +- **Breaking:** App metrics endpoint (`/api/pleroma/app_metrics`) is disabled by default, check `docs/API/prometheus.md` on enabling and configuring. - Search: Users are now findable by their urls. - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated. - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated. diff --git a/config/config.exs b/config/config.exs index 2c6142360..a7aae5802 100644 --- a/config/config.exs +++ b/config/config.exs @@ -636,7 +636,12 @@ config :pleroma, Pleroma.Emails.UserEmail, config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: false -config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics" +config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, + enabled: false, + auth: false, + ip_whitelist: [], + path: "/api/pleroma/app_metrics", + format: :text config :pleroma, Pleroma.ScheduledActivity, daily_user_limit: 25, diff --git a/docs/API/prometheus.md b/docs/API/prometheus.md index 19c564e3c..a5158d905 100644 --- a/docs/API/prometheus.md +++ b/docs/API/prometheus.md @@ -2,15 +2,37 @@ Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library. +Config example: + +``` +config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, + enabled: true, + auth: {:basic, "myusername", "mypassword"}, + ip_whitelist: ["127.0.0.1"], + path: "/api/pleroma/app_metrics", + format: :text +``` + +* `enabled` (Pleroma extension) enables the endpoint +* `ip_whitelist` (Pleroma extension) could be used to restrict access only to specified IPs +* `auth` sets the authentication (`false` for no auth; configurable to HTTP Basic Auth, see [prometheus-plugs](https://github.com/deadtrickster/prometheus-plugs#exporting) documentation) +* `format` sets the output format (`:text` or `:protobuf`) +* `path` sets the path to app metrics page + + ## `/api/pleroma/app_metrics` + ### Exports Prometheus application metrics + * Method: `GET` -* Authentication: not required +* Authentication: not required by default (see configuration options above) * Params: none -* Response: JSON +* Response: text ## Grafana + ### Config example + The following is a config example to use with [Grafana](https://grafana.com) ``` diff --git a/lib/pleroma/helpers/inet_helper.ex b/lib/pleroma/helpers/inet_helper.ex new file mode 100644 index 000000000..126f82381 --- /dev/null +++ b/lib/pleroma/helpers/inet_helper.ex @@ -0,0 +1,19 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Helpers.InetHelper do + def parse_address(ip) when is_tuple(ip) do + {:ok, ip} + end + + def parse_address(ip) when is_binary(ip) do + ip + |> String.to_charlist() + |> parse_address() + end + + def parse_address(ip) do + :inet.parse_address(ip) + end +end diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 56562c12f..1a8fdd8b9 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -7,6 +7,8 @@ defmodule Pleroma.Web.Endpoint do require Pleroma.Constants + alias Pleroma.Config + socket("/socket", Pleroma.Web.UserSocket) plug(Pleroma.Web.Plugs.SetLocalePlug) @@ -86,19 +88,19 @@ defmodule Pleroma.Web.Endpoint do plug(Plug.Parsers, parsers: [ :urlencoded, - {:multipart, length: {Pleroma.Config, :get, [[:instance, :upload_limit]]}}, + {:multipart, length: {Config, :get, [[:instance, :upload_limit]]}}, :json ], pass: ["*/*"], json_decoder: Jason, - length: Pleroma.Config.get([:instance, :upload_limit]), + length: Config.get([:instance, :upload_limit]), body_reader: {Pleroma.Web.Plugs.DigestPlug, :read_body, []} ) plug(Plug.MethodOverride) plug(Plug.Head) - secure_cookies = Pleroma.Config.get([__MODULE__, :secure_cookie_flag]) + secure_cookies = Config.get([__MODULE__, :secure_cookie_flag]) cookie_name = if secure_cookies, @@ -106,7 +108,7 @@ defmodule Pleroma.Web.Endpoint do else: "pleroma_key" extra = - Pleroma.Config.get([__MODULE__, :extra_cookie_attrs]) + Config.get([__MODULE__, :extra_cookie_attrs]) |> Enum.join(";") # The session will be stored in the cookie and signed, @@ -116,7 +118,7 @@ defmodule Pleroma.Web.Endpoint do Plug.Session, store: :cookie, key: cookie_name, - signing_salt: Pleroma.Config.get([__MODULE__, :signing_salt], "CqaoopA2"), + signing_salt: Config.get([__MODULE__, :signing_salt], "CqaoopA2"), http_only: true, secure: secure_cookies, extra: extra @@ -136,8 +138,34 @@ defmodule Pleroma.Web.Endpoint do use Prometheus.PlugExporter end + defmodule MetricsExporterCaller do + @behaviour Plug + + def init(opts), do: opts + + def call(conn, opts) do + prometheus_config = Application.get_env(:prometheus, MetricsExporter, []) + ip_whitelist = List.wrap(prometheus_config[:ip_whitelist]) + + cond do + !prometheus_config[:enabled] -> + conn + + ip_whitelist != [] and + !Enum.find(ip_whitelist, fn ip -> + Pleroma.Helpers.InetHelper.parse_address(ip) == {:ok, conn.remote_ip} + end) -> + conn + + true -> + MetricsExporter.call(conn, opts) + end + end + end + plug(PipelineInstrumenter) - plug(MetricsExporter) + + plug(MetricsExporterCaller) plug(Pleroma.Web.Router) diff --git a/test/pleroma/web/endpoint/metrics_exporter_test.exs b/test/pleroma/web/endpoint/metrics_exporter_test.exs new file mode 100644 index 000000000..f954cc1e7 --- /dev/null +++ b/test/pleroma/web/endpoint/metrics_exporter_test.exs @@ -0,0 +1,69 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Endpoint.MetricsExporterTest do + use Pleroma.Web.ConnCase + + alias Pleroma.Web.Endpoint.MetricsExporter + + defp config do + Application.get_env(:prometheus, MetricsExporter) + end + + describe "with default config" do + test "does NOT expose app metrics", %{conn: conn} do + conn + |> get(config()[:path]) + |> json_response(404) + end + end + + describe "when enabled" do + setup do + initial_config = config() + on_exit(fn -> Application.put_env(:prometheus, MetricsExporter, initial_config) end) + + Application.put_env( + :prometheus, + MetricsExporter, + Keyword.put(initial_config, :enabled, true) + ) + end + + test "serves app metrics", %{conn: conn} do + conn = get(conn, config()[:path]) + assert response = response(conn, 200) + + for metric <- [ + "http_requests_total", + "http_request_duration_microseconds", + "phoenix_controller_render_duration", + "phoenix_controller_call_duration", + "telemetry_scrape_duration", + "erlang_vm_memory_atom_bytes_total" + ] do + assert response =~ ~r/#{metric}/ + end + end + + test "when IP whitelist configured, " <> + "serves app metrics only if client IP is whitelisted", + %{conn: conn} do + Application.put_env( + :prometheus, + MetricsExporter, + Keyword.put(config(), :ip_whitelist, ["127.127.127.127", {1, 1, 1, 1}, '255.255.255.255']) + ) + + conn + |> get(config()[:path]) + |> json_response(404) + + conn + |> Map.put(:remote_ip, {127, 127, 127, 127}) + |> get(config()[:path]) + |> response(200) + end + end +end From 3a28aa8814e186eea67df7da00df1636823291f3 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 20 Oct 2020 15:13:20 +0300 Subject: [PATCH 2/4] [#1668] Added :prometheus group config to config/description.exs. --- config/description.exs | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/config/description.exs b/config/description.exs index 2a1898922..0fe86ded7 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3722,5 +3722,41 @@ config :pleroma, :config_description, [ suggestions: [2] } ] + }, + %{ + group: :prometheus, + key: Pleroma.Web.Endpoint.MetricsExporter, + type: :group, + description: "Prometheus app metrics endpoint configuration", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "[Pleroma extension] Enables app metrics endpoint." + }, + %{ + key: :ip_whitelist, + type: [{:list, :string}, {:list, :charlist}, {:list, :tuple}], + description: "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses." + }, + %{ + key: :auth, + type: [:boolean, :tuple], + description: "Enables HTTP Basic Auth for app metrics endpoint.", + suggestion: [false, {:basic, "myusername", "mypassword"}] + }, + %{ + key: :path, + type: :string, + description: "App metrics endpoint URI path.", + suggestions: ["/api/pleroma/app_metrics"] + }, + %{ + key: :format, + type: :atom, + description: "App metrics endpoint output format.", + suggestions: [:text, :protobuf] + } + ] } ] From 50d428088017e0383d8b35d4ab1b831f40646ab0 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 20 Oct 2020 16:18:24 +0300 Subject: [PATCH 3/4] [#1668] Formatting fix. --- config/description.exs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/description.exs b/config/description.exs index 0fe86ded7..11755d757 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3737,7 +3737,8 @@ config :pleroma, :config_description, [ %{ key: :ip_whitelist, type: [{:list, :string}, {:list, :charlist}, {:list, :tuple}], - description: "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses." + description: + "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses." }, %{ key: :auth, From cbe41408e4f77da55d59ee3d4d26d002a1f20f02 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 27 Oct 2020 14:37:48 -0500 Subject: [PATCH 4/4] phoenix_controller_render_duration is no longer available in telemetry of Phoenix 1.5+ --- test/pleroma/web/endpoint/metrics_exporter_test.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/test/pleroma/web/endpoint/metrics_exporter_test.exs b/test/pleroma/web/endpoint/metrics_exporter_test.exs index f954cc1e7..875addc96 100644 --- a/test/pleroma/web/endpoint/metrics_exporter_test.exs +++ b/test/pleroma/web/endpoint/metrics_exporter_test.exs @@ -38,7 +38,6 @@ defmodule Pleroma.Web.Endpoint.MetricsExporterTest do for metric <- [ "http_requests_total", "http_request_duration_microseconds", - "phoenix_controller_render_duration", "phoenix_controller_call_duration", "telemetry_scrape_duration", "erlang_vm_memory_atom_bytes_total"