From 33243c56e5560f99bd886ed44bcb50c9f679ccc2 Mon Sep 17 00:00:00 2001 From: Tim Buchwaldt Date: Sat, 12 Nov 2022 11:14:16 +0100 Subject: [PATCH 01/22] Start adding telemetry --- lib/pleroma/application.ex | 1 + lib/pleroma/web/router.ex | 2 +- lib/pleroma/web/telemetry.ex | 90 ++++++++++++++++++++++++++++++++++++ mix.exs | 3 ++ mix.lock | 5 +- 5 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 lib/pleroma/web/telemetry.ex diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 48a2623ce..ad768b0e0 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -64,6 +64,7 @@ def start(_type, _args) do Config.TransferTask, Pleroma.Emoji, Pleroma.Web.Plugs.RateLimiter.Supervisor, + Pleroma.Web.Telemetry, {Task.Supervisor, name: Pleroma.TaskSupervisor} ] ++ cachex_children() ++ diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e790b1cdb..bcb5fb15e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -867,7 +867,7 @@ defmodule Pleroma.Web.Router do scope "/" do pipe_through([:pleroma_html, :authenticate, :require_admin]) - live_dashboard("/phoenix/live_dashboard") + live_dashboard("/phoenix/live_dashboard", metrics: Pleroma.Web.Telemetry) end # Test-only routes needed to test action dispatching and plug chain execution diff --git a/lib/pleroma/web/telemetry.ex b/lib/pleroma/web/telemetry.ex new file mode 100644 index 000000000..f7e9d5ca7 --- /dev/null +++ b/lib/pleroma/web/telemetry.ex @@ -0,0 +1,90 @@ +defmodule Pleroma.Web.Telemetry do + use Supervisor + import Telemetry.Metrics + + def start_link(arg) do + Supervisor.start_link(__MODULE__, arg, name: __MODULE__) + end + + @impl true + def init(_arg) do + children = [ + # Telemetry poller will execute the given period measurements + # every 10_000ms. Learn more here: https://hexdocs.pm/telemetry_metrics + {:telemetry_poller, measurements: periodic_measurements(), period: 10_000}, + # Add reporters as children of your supervision tree. + # {Telemetry.Metrics.ConsoleReporter, metrics: metrics()}, + {TelemetryMetricsPrometheus, metrics: metrics()} + ] + + Supervisor.init(children, strategy: :one_for_one) + end + + def metrics do + [ + # Phoenix Metrics + summary("phoenix.endpoint.stop.duration", + unit: {:native, :millisecond} + ), + summary("phoenix.router_dispatch.stop.duration", + tags: [:route], + unit: {:native, :millisecond} + ), + distribution( + "phoenix.router_dispatch.stop.duration", + # event_name: [:pleroma, :repo, :query, :total_time], + measurement: :duration, + unit: {:native, :second}, + tags: [:route], + reporter_options: [ + buckets: [0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10] + ] + ), + + # Database Time Metrics + distribution( + "pleroma.repo.query.total_time", + # event_name: [:pleroma, :repo, :query, :total_time], + measurement: :total_time, + unit: {:native, :millisecond}, + reporter_options: [ + buckets: [0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10] + ] + ), + distribution( + "pleroma.repo.query.queue_time", + # event_name: [:pleroma, :repo, :query, :total_time], + measurement: :queue_time, + unit: {:native, :millisecond}, + reporter_options: [ + buckets: [0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10] + ] + ), + summary("pleroma.repo.query.total_time", unit: {:native, :millisecond}), + summary("pleroma.repo.query.decode_time", unit: {:native, :millisecond}), + summary("pleroma.repo.query.query_time", unit: {:native, :millisecond}), + summary("pleroma.repo.query.queue_time", unit: {:native, :millisecond}), + summary("pleroma.repo.query.idle_time", unit: {:native, :millisecond}), + + # VM Metrics + summary("vm.memory.total", unit: {:byte, :kilobyte}), + summary("vm.total_run_queue_lengths.total"), + summary("vm.total_run_queue_lengths.cpu"), + summary("vm.total_run_queue_lengths.io"), + distribution( + "oban.job.stop", + event_name: [:oban, :job, :stop], + measurement: :duration, + tags: [:job], + unit: {:native, :second}, + reporter_options: [ + buckets: [0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10] + ] + ) + ] + end + + defp periodic_measurements do + [] + end +end diff --git a/mix.exs b/mix.exs index 00a250002..b5b7efd0c 100644 --- a/mix.exs +++ b/mix.exs @@ -161,6 +161,9 @@ defp deps do git: "https://akkoma.dev/AkkomaGang/linkify.git", branch: "bugfix/line-ending-buffer"}, {:http_signatures, "~> 0.1.1"}, {:telemetry, "~> 0.3"}, + {:telemetry_poller, "~> 0.4"}, + {:telemetry_metrics, "~> 0.4"}, + {:telemetry_metrics_prometheus, "~> 1.1.0"}, {:poolboy, "~> 1.5"}, {:recon, "~> 2.5"}, {:joken, "~> 2.0"}, diff --git a/mix.lock b/mix.lock index 0e3ac3514..01c628c29 100644 --- a/mix.lock +++ b/mix.lock @@ -17,7 +17,7 @@ "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.1", "ebd1a1d7aff97f27c66654e78ece187abdc646992714164380d8a041eda16754", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a6efd3366130eab84ca372cbd4a7d3c3a97bdfcfb4911233b035d117063f0af"}, "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"}, - "credo": {:git, "https://github.com/rrrene/credo.git", "1c1b99ea41a457761383d81aaf6a606913996fe7", [ref: "1c1b99ea41a457761383d81aaf6a606913996fe7"]}, + "credo": {:hex, :credo, "1.6.7", "323f5734350fd23a456f2688b9430e7d517afb313fbd38671b8a4449798a7854", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "41e110bfb007f7eda7f897c10bf019ceab9a0b269ce79f015d54b0dcf4fc7dd3"}, "crypt": {:git, "https://github.com/msantos/crypt.git", "f75cd55325e33cbea198fb41fe41871392f8fb76", [ref: "f75cd55325e33cbea198fb41fe41871392f8fb76"]}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, "db_connection": {:hex, :db_connection, "2.4.3", "3b9aac9f27347ec65b271847e6baeb4443d8474289bd18c1d6f4de655b70c94d", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c127c15b0fa6cfb32eed07465e05da6c815b032508d4ed7c116122871df73c12"}, @@ -112,6 +112,9 @@ "table_rex": {:hex, :table_rex, "3.1.1", "0c67164d1714b5e806d5067c1e96ff098ba7ae79413cc075973e17c38a587caa", [:mix], [], "hexpm", "678a23aba4d670419c23c17790f9dcd635a4a89022040df7d5d772cb21012490"}, "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.1", "315d9163a1d4660aedc3fee73f33f1d355dcc76c5c3ab3d59e76e3edf80eef1f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7be9e0871c41732c233be71e4be11b96e56177bf15dde64a8ac9ce72ac9834c6"}, + "telemetry_metrics_prometheus": {:hex, :telemetry_metrics_prometheus, "1.1.0", "1cc23e932c1ef9aa3b91db257ead31ea58d53229d407e059b29bb962c1505a13", [:mix], [{:plug_cowboy, "~> 2.1", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:telemetry_metrics_prometheus_core, "~> 1.0", [hex: :telemetry_metrics_prometheus_core, repo: "hexpm", optional: false]}], "hexpm", "d43b3659b3244da44fe0275b717701542365d4519b79d9ce895b9719c1ce4d26"}, + "telemetry_metrics_prometheus_core": {:hex, :telemetry_metrics_prometheus_core, "1.1.0", "4e15f6d7dbedb3a4e3aed2262b7e1407f166fcb9c30ca3f96635dfbbef99965c", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "0dd10e7fe8070095df063798f82709b0a1224c31b8baf6278b423898d591a069"}, + "telemetry_poller": {:hex, :telemetry_poller, "0.5.1", "21071cc2e536810bac5628b935521ff3e28f0303e770951158c73eaaa01e962a", [:rebar3], [{:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4cab72069210bc6e7a080cec9afffad1b33370149ed5d379b81c7c5f0c663fd4"}, "temple": {:git, "https://akkoma.dev/AkkomaGang/temple.git", "066a699ade472d8fa42a9d730b29a61af9bc8b59", [ref: "066a699ade472d8fa42a9d730b29a61af9bc8b59"]}, "tesla": {:hex, :tesla, "1.4.4", "bb89aa0c9745190930366f6a2ac612cdf2d0e4d7fff449861baa7875afd797b2", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "d5503a49f9dec1b287567ea8712d085947e247cb11b06bc54adb05bfde466457"}, "timex": {:hex, :timex, "3.7.9", "790cdfc4acfce434e442f98c02ea6d84d0239073bfd668968f82ac63e9a6788d", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "64691582e5bb87130f721fc709acfb70f24405833998fabf35be968984860ce1"}, From 1e9c2cd8ef1da5cf6803d690b14e4b956b0b2c38 Mon Sep 17 00:00:00 2001 From: Tim Buchwaldt Date: Sat, 12 Nov 2022 11:23:44 +0100 Subject: [PATCH 02/22] Fix buckets for query timing --- lib/pleroma/web/telemetry.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/telemetry.ex b/lib/pleroma/web/telemetry.ex index f7e9d5ca7..546c39f05 100644 --- a/lib/pleroma/web/telemetry.ex +++ b/lib/pleroma/web/telemetry.ex @@ -37,7 +37,7 @@ def metrics do unit: {:native, :second}, tags: [:route], reporter_options: [ - buckets: [0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10] + buckets: [0.1, 0.2, 0.5, 1, 2.5, 5, 10, 25, 50, 100, 250, 500, 1000] ] ), @@ -48,7 +48,7 @@ def metrics do measurement: :total_time, unit: {:native, :millisecond}, reporter_options: [ - buckets: [0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10] + buckets: [0.1, 0.2, 0.5, 1, 2.5, 5, 10, 25, 50, 100, 250, 500, 1000] ] ), distribution( From a06bb694c1776a9144746237991a43f53c63e634 Mon Sep 17 00:00:00 2001 From: Tim Buchwaldt Date: Sat, 12 Nov 2022 11:31:03 +0100 Subject: [PATCH 03/22] Listen to loopback --- lib/pleroma/web/telemetry.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/telemetry.ex b/lib/pleroma/web/telemetry.ex index 546c39f05..2ca188e1a 100644 --- a/lib/pleroma/web/telemetry.ex +++ b/lib/pleroma/web/telemetry.ex @@ -14,7 +14,7 @@ def init(_arg) do {:telemetry_poller, measurements: periodic_measurements(), period: 10_000}, # Add reporters as children of your supervision tree. # {Telemetry.Metrics.ConsoleReporter, metrics: metrics()}, - {TelemetryMetricsPrometheus, metrics: metrics()} + {TelemetryMetricsPrometheus, metrics: metrics(), plug_cowboy_opts: [ip: {127, 0, 0, 1}]} ] Supervisor.init(children, strategy: :one_for_one) From f8d338317918cd62ebe45c3ea6433425b1cb1b28 Mon Sep 17 00:00:00 2001 From: Tim Buchwaldt Date: Sat, 12 Nov 2022 11:54:35 +0100 Subject: [PATCH 04/22] Fix oban tags --- lib/pleroma/web/telemetry.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/telemetry.ex b/lib/pleroma/web/telemetry.ex index 2ca188e1a..505ee382c 100644 --- a/lib/pleroma/web/telemetry.ex +++ b/lib/pleroma/web/telemetry.ex @@ -75,7 +75,8 @@ def metrics do "oban.job.stop", event_name: [:oban, :job, :stop], measurement: :duration, - tags: [:job], + tags: [:worker], + tag_values: fn tags -> Map.put(tags, :worker, tags.job.worker) end, unit: {:native, :second}, reporter_options: [ buckets: [0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10] From 8f58eb4a183e97e28850d7030beeb7b13294d318 Mon Sep 17 00:00:00 2001 From: Tim Buchwaldt Date: Sat, 12 Nov 2022 12:38:25 +0100 Subject: [PATCH 05/22] Revert "Bump live-dashboard" This reverts commit c196d79aafd51b671aa19032b32e4cd416dab720. --- mix.exs | 2 +- mix.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mix.exs b/mix.exs index b5b7efd0c..cb46e8da3 100644 --- a/mix.exs +++ b/mix.exs @@ -185,7 +185,7 @@ defp deps do git: "https://github.com/FloatingGhost/pleroma-contrib-search-parser.git", ref: "08971a81e68686f9ac465cfb6661d51c5e4e1e7f"}, {:nimble_parsec, "~> 1.0", override: true}, - {:phoenix_live_dashboard, "~> 0.7.2"}, + {:phoenix_live_dashboard, "~> 0.6.2"}, {:ecto_psql_extras, "~> 0.6"}, {:elasticsearch, git: "https://akkoma.dev/AkkomaGang/elasticsearch-elixir.git", ref: "main"}, diff --git a/mix.lock b/mix.lock index 01c628c29..62f481d83 100644 --- a/mix.lock +++ b/mix.lock @@ -85,8 +85,8 @@ "phoenix": {:hex, :phoenix, "1.6.15", "0a1d96bbc10747fd83525370d691953cdb6f3ccbac61aa01b4acb012474b047d", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 1.0 or ~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d70ab9fbf6b394755ea88b644d34d79d8b146e490973151f248cacd122d20672"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.0", "0672ed4e4808b3fbed494dded89958e22fb882de47a97634c0b13e7b0b5f7720", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "09864e558ed31ee00bd48fcc1d4fc58ae9678c9e81649075431e69dbabb43cc1"}, "phoenix_html": {:hex, :phoenix_html, "3.2.0", "1c1219d4b6cb22ac72f12f73dc5fad6c7563104d083f711c3fcd8551a1f4ae11", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "36ec97ba56d25c0136ef1992c37957e4246b649d620958a1f9fa86165f8bc54f"}, - "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.7.2", "97cc4ff2dba1ebe504db72cb45098cb8e91f11160528b980bd282cc45c73b29c", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.18.3", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "0e5fdf063c7a3b620c566a30fcf68b7ee02e5e46fe48ee46a6ec3ba382dc05b7"}, - "phoenix_live_view": {:hex, :phoenix_live_view, "0.18.3", "2e3d009422addf8b15c3dccc65ce53baccbe26f7cfd21d264680b5867789a9c1", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.1", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c8845177a866e017dcb7083365393c8f00ab061b8b6b2bda575891079dce81b2"}, + "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.6.5", "1495bb014be12c9a9252eca04b9af54246f6b5c1e4cd1f30210cd00ec540cf8e", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.3", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.17.7", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "ef4fa50dd78364409039c99cf6f98ab5209b4c5f8796c17f4db118324f0db852"}, + "phoenix_live_view": {:hex, :phoenix_live_view, "0.17.12", "74f4c0ad02d7deac2d04f50b52827a5efdc5c6e7fac5cede145f5f0e4183aedc", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.0 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.1", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "af6dd5e0aac16ff43571f527a8e0616d62cb80b10eb87aac82170243e50d99c8"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.1", "ba04e489ef03763bf28a17eb2eaddc2c20c6d217e2150a61e3298b0f4c2012b5", [:mix], [], "hexpm", "81367c6d1eea5878ad726be80808eb5a787a23dee699f96e72b1109c57cdd8d9"}, "phoenix_swoosh": {:hex, :phoenix_swoosh, "0.3.4", "615f8f393135de7e0cbb4bd00ba238b1e0cd324b0d90efbaee613c2f02ca5e5c", [:mix], [{:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:swoosh, "~> 1.0", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm", "3971221846232021ab5e3c7489fd62ec5bfd6a2e01cae10a317ccf6fb350571c"}, "phoenix_template": {:hex, :phoenix_template, "1.0.0", "c57bc5044f25f007dc86ab21895688c098a9f846a8dda6bc40e2d0ddc146e38f", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "1b066f99a26fd22064c12b2600a9a6e56700f591bf7b20b418054ea38b4d4357"}, From 0995fa141062f2aef1ee017ca03c7f9522dc3da8 Mon Sep 17 00:00:00 2001 From: Tim Buchwaldt Date: Sat, 12 Nov 2022 12:42:53 +0100 Subject: [PATCH 06/22] Track oban failures --- lib/pleroma/web/telemetry.ex | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/telemetry.ex b/lib/pleroma/web/telemetry.ex index 505ee382c..d9c54c28f 100644 --- a/lib/pleroma/web/telemetry.ex +++ b/lib/pleroma/web/telemetry.ex @@ -72,7 +72,7 @@ def metrics do summary("vm.total_run_queue_lengths.cpu"), summary("vm.total_run_queue_lengths.io"), distribution( - "oban.job.stop", + "oban_job_completion", event_name: [:oban, :job, :stop], measurement: :duration, tags: [:worker], @@ -81,6 +81,17 @@ def metrics do reporter_options: [ buckets: [0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10] ] + ), + distribution( + "oban_job_exception", + event_name: [:oban, :job, :exception], + measurement: :duration, + tags: [:worker], + tag_values: fn tags -> Map.put(tags, :worker, tags.job.worker) end, + unit: {:native, :second}, + reporter_options: [ + buckets: [0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10] + ] ) ] end From 63be819661cd28514207bbc19be6d4964235685b Mon Sep 17 00:00:00 2001 From: Tim Buchwaldt Date: Sat, 12 Nov 2022 16:11:38 +0100 Subject: [PATCH 07/22] Take tesla telemetry --- lib/pleroma/http.ex | 2 +- lib/pleroma/web/telemetry.ex | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/http.ex b/lib/pleroma/http.ex index d8028651c..6ae1cdebb 100644 --- a/lib/pleroma/http.ex +++ b/lib/pleroma/http.ex @@ -65,7 +65,7 @@ def request(method, url, body, headers, options) when is_binary(url) do options = put_in(options[:adapter], adapter_opts) params = options[:params] || [] request = build_request(method, headers, options, url, body, params) - client = Tesla.client([Tesla.Middleware.FollowRedirects]) + client = Tesla.client([Tesla.Middleware.FollowRedirects, Tesla.Middleware.Telemetry]) request(client, request) end diff --git a/lib/pleroma/web/telemetry.ex b/lib/pleroma/web/telemetry.ex index d9c54c28f..46b5865f7 100644 --- a/lib/pleroma/web/telemetry.ex +++ b/lib/pleroma/web/telemetry.ex @@ -92,6 +92,17 @@ def metrics do reporter_options: [ buckets: [0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10] ] + ), + distribution( + "tesla_request_completed", + event_name: [:tesla, :request, :stop], + measurement: :duration, + tags: [:response_code], + tag_values: fn tags -> Map.put(tags, :response_code, tags.env.status) end, + unit: {:native, :second}, + reporter_options: [ + buckets: [0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10] + ] ) ] end From 29584197bb53a6fb5f980cc8d2566b0492196def Mon Sep 17 00:00:00 2001 From: Tim Buchwaldt Date: Sat, 12 Nov 2022 17:13:39 +0100 Subject: [PATCH 08/22] Measure stats-data --- lib/pleroma/application.ex | 4 ++-- lib/pleroma/web/telemetry.ex | 17 +++++++++++++++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index ad768b0e0..02336d6d1 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -64,7 +64,6 @@ def start(_type, _args) do Config.TransferTask, Pleroma.Emoji, Pleroma.Web.Plugs.RateLimiter.Supervisor, - Pleroma.Web.Telemetry, {Task.Supervisor, name: Pleroma.TaskSupervisor} ] ++ cachex_children() ++ @@ -74,7 +73,8 @@ def start(_type, _args) do Pleroma.JobQueueMonitor, {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Config.get([:majic_pool, :size], 2)]}, {Oban, Config.get(Oban)}, - Pleroma.Web.Endpoint + Pleroma.Web.Endpoint, + Pleroma.Web.Telemetry ] ++ elasticsearch_children() ++ task_children(@mix_env) ++ diff --git a/lib/pleroma/web/telemetry.ex b/lib/pleroma/web/telemetry.ex index 46b5865f7..73ce6a85c 100644 --- a/lib/pleroma/web/telemetry.ex +++ b/lib/pleroma/web/telemetry.ex @@ -1,6 +1,7 @@ defmodule Pleroma.Web.Telemetry do use Supervisor import Telemetry.Metrics + alias Pleroma.Stats def start_link(arg) do Supervisor.start_link(__MODULE__, arg, name: __MODULE__) @@ -103,11 +104,23 @@ def metrics do reporter_options: [ buckets: [0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10] ] - ) + ), + last_value("pleroma.local_users.total"), + last_value("pleroma.domains.total"), + last_value("pleroma.local_statuses.total") ] end defp periodic_measurements do - [] + [ + {__MODULE__, :instance_stats, []} + ] + end + + def instance_stats do + stats = Stats.get_stats() + :telemetry.execute([:pleroma, :local_users], %{total: stats.user_count}, %{}) + :telemetry.execute([:pleroma, :domains], %{total: stats.domain_count}, %{}) + :telemetry.execute([:pleroma, :local_statuses], %{total: stats.status_count}, %{}) end end From e2320f870e6ef4e8fc9f63282cbd459e819d740a Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Thu, 15 Dec 2022 02:02:07 +0000 Subject: [PATCH 09/22] Add prometheus metrics to router --- .../controllers/metrics_controller.ex | 16 ++++ lib/pleroma/web/endpoint.ex | 1 + lib/pleroma/web/plugs/csp_nonce_plug.ex | 21 +++++ lib/pleroma/web/plugs/http_security_plug.ex | 23 ++++-- lib/pleroma/web/router.ex | 3 +- lib/pleroma/web/telemetry.ex | 82 +++++++++++-------- mix.exs | 2 +- mix.lock | 6 +- 8 files changed, 104 insertions(+), 50 deletions(-) create mode 100644 lib/pleroma/web/akkoma_api/controllers/metrics_controller.ex create mode 100644 lib/pleroma/web/plugs/csp_nonce_plug.ex diff --git a/lib/pleroma/web/akkoma_api/controllers/metrics_controller.ex b/lib/pleroma/web/akkoma_api/controllers/metrics_controller.ex new file mode 100644 index 000000000..c8d3d8948 --- /dev/null +++ b/lib/pleroma/web/akkoma_api/controllers/metrics_controller.ex @@ -0,0 +1,16 @@ +defmodule Pleroma.Web.AkkomaAPI.MetricsController do + use Pleroma.Web, :controller + + alias Pleroma.Web.Plugs.OAuthScopesPlug + + @unauthenticated_access %{fallback: :proceed_unauthenticated, scopes: []} + plug(:skip_auth) + + + def show(conn, _params) do + stats = TelemetryMetricsPrometheus.Core.scrape() + + conn + |> text(stats) + end +end diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index baf0c5651..e3a251ca1 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.Endpoint do plug(Pleroma.Web.Plugs.SetLocalePlug) plug(CORSPlug) + plug(Pleroma.Web.Plugs.CSPNoncePlug) plug(Pleroma.Web.Plugs.HTTPSecurityPlug) plug(Pleroma.Web.Plugs.UploadedMedia) diff --git a/lib/pleroma/web/plugs/csp_nonce_plug.ex b/lib/pleroma/web/plugs/csp_nonce_plug.ex new file mode 100644 index 000000000..bc2c6fcd8 --- /dev/null +++ b/lib/pleroma/web/plugs/csp_nonce_plug.ex @@ -0,0 +1,21 @@ +defmodule Pleroma.Web.Plugs.CSPNoncePlug do + import Plug.Conn + + def init(opts) do + opts + end + + def call(conn, _opts) do + assign_csp_nonce(conn) + end + + defp assign_csp_nonce(conn) do + nonce = + :crypto.strong_rand_bytes(128) + |> Base.url_encode64() + |> binary_part(0, 15) + + conn + |> assign(:csp_nonce, nonce) + end +end diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index 47874a980..6593347ca 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -13,7 +13,7 @@ def init(opts), do: opts def call(conn, _options) do if Config.get([:http_security, :enabled]) do conn - |> merge_resp_headers(headers()) + |> merge_resp_headers(headers(conn)) |> maybe_send_sts_header(Config.get([:http_security, :sts])) else conn @@ -36,7 +36,8 @@ def custom_http_frontend_headers do end end - def headers do + @spec headers(Plug.Conn.t()) :: [{String.t(), String.t()}] + def headers(conn) do referrer_policy = Config.get([:http_security, :referrer_policy]) report_uri = Config.get([:http_security, :report_uri]) custom_http_frontend_headers = custom_http_frontend_headers() @@ -47,7 +48,7 @@ def headers do {"x-frame-options", "DENY"}, {"x-content-type-options", "nosniff"}, {"referrer-policy", referrer_policy}, - {"content-security-policy", csp_string()}, + {"content-security-policy", csp_string(conn)}, {"permissions-policy", "interest-cohort=()"} ] @@ -77,19 +78,18 @@ def headers do "default-src 'none'", "base-uri 'none'", "frame-ancestors 'none'", - "style-src 'self' 'unsafe-inline'", - "font-src 'self'", "manifest-src 'self'" ] @csp_start [Enum.join(static_csp_rules, ";") <> ";"] - defp csp_string do + defp csp_string(conn) do scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme] static_url = Pleroma.Web.Endpoint.static_url() websocket_url = Pleroma.Web.Endpoint.websocket_url() report_uri = Config.get([:http_security, :report_uri]) - + %{assigns: %{csp_nonce: nonce}} = conn + nonce_tag = "nonce-" <> nonce img_src = "img-src 'self' data: blob:" media_src = "media-src 'self'" @@ -111,11 +111,14 @@ defp csp_string do ["connect-src 'self' blob: ", static_url, ?\s, websocket_url] end + style_src = "style-src 'self' '#{nonce_tag}'" + font_src = "font-src 'self' '#{nonce_tag}' data:" + script_src = if Config.get(:env) == :dev do - "script-src 'self' 'unsafe-eval'" + "script-src 'self' 'unsafe-eval' '#{nonce_tag}'" else - "script-src 'self'" + "script-src 'self' '#{nonce_tag}'" end report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"] @@ -126,6 +129,8 @@ defp csp_string do |> add_csp_param(media_src) |> add_csp_param(connect_src) |> add_csp_param(script_src) + |> add_csp_param(font_src) + |> add_csp_param(style_src) |> add_csp_param(insecure) |> add_csp_param(report) |> :erlang.iolist_to_binary() diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index bcb5fb15e..c0ce645c4 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -467,6 +467,7 @@ defmodule Pleroma.Web.Router do scope "/api/v1/akkoma", Pleroma.Web.AkkomaAPI do pipe_through(:authenticated_api) + get("/metrics", MetricsController, :show) get("/translation/languages", TranslationController, :languages) get("/frontend_settings/:frontend_name", FrontendSettingsController, :list_profiles) @@ -867,7 +868,7 @@ defmodule Pleroma.Web.Router do scope "/" do pipe_through([:pleroma_html, :authenticate, :require_admin]) - live_dashboard("/phoenix/live_dashboard", metrics: Pleroma.Web.Telemetry) + live_dashboard("/phoenix/live_dashboard", metrics: {Pleroma.Web.Telemetry, :live_dashboard_metrics}, csp_nonce_assign_key: :csp_nonce) end # Test-only routes needed to test action dispatching and plug chain execution diff --git a/lib/pleroma/web/telemetry.ex b/lib/pleroma/web/telemetry.ex index 73ce6a85c..435f55799 100644 --- a/lib/pleroma/web/telemetry.ex +++ b/lib/pleroma/web/telemetry.ex @@ -10,27 +10,19 @@ def start_link(arg) do @impl true def init(_arg) do children = [ - # Telemetry poller will execute the given period measurements - # every 10_000ms. Learn more here: https://hexdocs.pm/telemetry_metrics {:telemetry_poller, measurements: periodic_measurements(), period: 10_000}, - # Add reporters as children of your supervision tree. - # {Telemetry.Metrics.ConsoleReporter, metrics: metrics()}, - {TelemetryMetricsPrometheus, metrics: metrics(), plug_cowboy_opts: [ip: {127, 0, 0, 1}]} + {TelemetryMetricsPrometheus, metrics: prometheus_metrics(), plug_cowboy_opts: [ip: {127, 0, 0, 1}]} ] Supervisor.init(children, strategy: :one_for_one) end - def metrics do + @doc """ + A seperate set of metrics for distributions because phoenix dashboard does NOT handle + them well + """ + defp distribution_metrics do [ - # Phoenix Metrics - summary("phoenix.endpoint.stop.duration", - unit: {:native, :millisecond} - ), - summary("phoenix.router_dispatch.stop.duration", - tags: [:route], - unit: {:native, :millisecond} - ), distribution( "phoenix.router_dispatch.stop.duration", # event_name: [:pleroma, :repo, :query, :total_time], @@ -61,28 +53,6 @@ def metrics do buckets: [0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10] ] ), - summary("pleroma.repo.query.total_time", unit: {:native, :millisecond}), - summary("pleroma.repo.query.decode_time", unit: {:native, :millisecond}), - summary("pleroma.repo.query.query_time", unit: {:native, :millisecond}), - summary("pleroma.repo.query.queue_time", unit: {:native, :millisecond}), - summary("pleroma.repo.query.idle_time", unit: {:native, :millisecond}), - - # VM Metrics - summary("vm.memory.total", unit: {:byte, :kilobyte}), - summary("vm.total_run_queue_lengths.total"), - summary("vm.total_run_queue_lengths.cpu"), - summary("vm.total_run_queue_lengths.io"), - distribution( - "oban_job_completion", - event_name: [:oban, :job, :stop], - measurement: :duration, - tags: [:worker], - tag_values: fn tags -> Map.put(tags, :worker, tags.job.worker) end, - unit: {:native, :second}, - reporter_options: [ - buckets: [0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10] - ] - ), distribution( "oban_job_exception", event_name: [:oban, :job, :exception], @@ -105,12 +75,52 @@ def metrics do buckets: [0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10] ] ), + distribution( + "oban_job_completion", + event_name: [:oban, :job, :stop], + measurement: :duration, + tags: [:worker], + tag_values: fn tags -> Map.put(tags, :worker, tags.job.worker) end, + unit: {:native, :second}, + reporter_options: [ + buckets: [0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2.5, 5, 10] + ] + ) + ] + end + + defp summary_metrics do + [ + # Phoenix Metrics + summary("phoenix.endpoint.stop.duration", + unit: {:native, :millisecond} + ), + summary("phoenix.router_dispatch.stop.duration", + tags: [:route], + unit: {:native, :millisecond} + ), + summary("pleroma.repo.query.total_time", unit: {:native, :millisecond}), + summary("pleroma.repo.query.decode_time", unit: {:native, :millisecond}), + summary("pleroma.repo.query.query_time", unit: {:native, :millisecond}), + summary("pleroma.repo.query.queue_time", unit: {:native, :millisecond}), + summary("pleroma.repo.query.idle_time", unit: {:native, :millisecond}), + + # VM Metrics + summary("vm.memory.total", unit: {:byte, :kilobyte}), + summary("vm.total_run_queue_lengths.total"), + summary("vm.total_run_queue_lengths.cpu"), + summary("vm.total_run_queue_lengths.io"), + + last_value("pleroma.local_users.total"), last_value("pleroma.domains.total"), last_value("pleroma.local_statuses.total") ] end + def prometheus_metrics, do: summary_metrics() ++ distribution_metrics() + def live_dashboard_metrics, do: summary_metrics() + defp periodic_measurements do [ {__MODULE__, :instance_stats, []} diff --git a/mix.exs b/mix.exs index cb46e8da3..b5b7efd0c 100644 --- a/mix.exs +++ b/mix.exs @@ -185,7 +185,7 @@ defp deps do git: "https://github.com/FloatingGhost/pleroma-contrib-search-parser.git", ref: "08971a81e68686f9ac465cfb6661d51c5e4e1e7f"}, {:nimble_parsec, "~> 1.0", override: true}, - {:phoenix_live_dashboard, "~> 0.6.2"}, + {:phoenix_live_dashboard, "~> 0.7.2"}, {:ecto_psql_extras, "~> 0.6"}, {:elasticsearch, git: "https://akkoma.dev/AkkomaGang/elasticsearch-elixir.git", ref: "main"}, diff --git a/mix.lock b/mix.lock index 62f481d83..4fa4c05ec 100644 --- a/mix.lock +++ b/mix.lock @@ -17,7 +17,7 @@ "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.3.1", "ebd1a1d7aff97f27c66654e78ece187abdc646992714164380d8a041eda16754", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3a6efd3366130eab84ca372cbd4a7d3c3a97bdfcfb4911233b035d117063f0af"}, "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"}, - "credo": {:hex, :credo, "1.6.7", "323f5734350fd23a456f2688b9430e7d517afb313fbd38671b8a4449798a7854", [:mix], [{:bunt, "~> 0.2.1", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2.8", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "41e110bfb007f7eda7f897c10bf019ceab9a0b269ce79f015d54b0dcf4fc7dd3"}, + "credo": {:git, "https://github.com/rrrene/credo.git", "1c1b99ea41a457761383d81aaf6a606913996fe7", [ref: "1c1b99ea41a457761383d81aaf6a606913996fe7"]}, "crypt": {:git, "https://github.com/msantos/crypt.git", "f75cd55325e33cbea198fb41fe41871392f8fb76", [ref: "f75cd55325e33cbea198fb41fe41871392f8fb76"]}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, "db_connection": {:hex, :db_connection, "2.4.3", "3b9aac9f27347ec65b271847e6baeb4443d8474289bd18c1d6f4de655b70c94d", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c127c15b0fa6cfb32eed07465e05da6c815b032508d4ed7c116122871df73c12"}, @@ -85,8 +85,8 @@ "phoenix": {:hex, :phoenix, "1.6.15", "0a1d96bbc10747fd83525370d691953cdb6f3ccbac61aa01b4acb012474b047d", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.0", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 1.0 or ~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.2", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d70ab9fbf6b394755ea88b644d34d79d8b146e490973151f248cacd122d20672"}, "phoenix_ecto": {:hex, :phoenix_ecto, "4.4.0", "0672ed4e4808b3fbed494dded89958e22fb882de47a97634c0b13e7b0b5f7720", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "09864e558ed31ee00bd48fcc1d4fc58ae9678c9e81649075431e69dbabb43cc1"}, "phoenix_html": {:hex, :phoenix_html, "3.2.0", "1c1219d4b6cb22ac72f12f73dc5fad6c7563104d083f711c3fcd8551a1f4ae11", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "36ec97ba56d25c0136ef1992c37957e4246b649d620958a1f9fa86165f8bc54f"}, - "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.6.5", "1495bb014be12c9a9252eca04b9af54246f6b5c1e4cd1f30210cd00ec540cf8e", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.3", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.17.7", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "ef4fa50dd78364409039c99cf6f98ab5209b4c5f8796c17f4db118324f0db852"}, - "phoenix_live_view": {:hex, :phoenix_live_view, "0.17.12", "74f4c0ad02d7deac2d04f50b52827a5efdc5c6e7fac5cede145f5f0e4183aedc", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.0 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.1", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "af6dd5e0aac16ff43571f527a8e0616d62cb80b10eb87aac82170243e50d99c8"}, + "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.7.2", "97cc4ff2dba1ebe504db72cb45098cb8e91f11160528b980bd282cc45c73b29c", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.18.3", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "0e5fdf063c7a3b620c566a30fcf68b7ee02e5e46fe48ee46a6ec3ba382dc05b7"}, + "phoenix_live_view": {:hex, :phoenix_live_view, "0.18.3", "2e3d009422addf8b15c3dccc65ce53baccbe26f7cfd21d264680b5867789a9c1", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.1", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c8845177a866e017dcb7083365393c8f00ab061b8b6b2bda575891079dce81b2"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.1", "ba04e489ef03763bf28a17eb2eaddc2c20c6d217e2150a61e3298b0f4c2012b5", [:mix], [], "hexpm", "81367c6d1eea5878ad726be80808eb5a787a23dee699f96e72b1109c57cdd8d9"}, "phoenix_swoosh": {:hex, :phoenix_swoosh, "0.3.4", "615f8f393135de7e0cbb4bd00ba238b1e0cd324b0d90efbaee613c2f02ca5e5c", [:mix], [{:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:phoenix, "~> 1.4", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:swoosh, "~> 1.0", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm", "3971221846232021ab5e3c7489fd62ec5bfd6a2e01cae10a317ccf6fb350571c"}, "phoenix_template": {:hex, :phoenix_template, "1.0.0", "c57bc5044f25f007dc86ab21895688c098a9f846a8dda6bc40e2d0ddc146e38f", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "1b066f99a26fd22064c12b2600a9a6e56700f591bf7b20b418054ea38b4d4357"}, From b8be8192fbfe8c27b457094e8d64ccb1c3a29f7f Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 16 Dec 2022 03:25:14 +0000 Subject: [PATCH 10/22] do not allow non-admins to register tokens with admin scopes this didn't actually _do_ anything in the past, the users would be prevented from accessing the resource, but they shouldn't be able to even create them --- lib/pleroma/web/o_auth/o_auth_controller.ex | 17 +-- lib/pleroma/web/o_auth/scopes.ex | 16 ++- .../web/o_auth/o_auth_controller_test.exs | 119 +++++++++++++----- 3 files changed, 110 insertions(+), 42 deletions(-) diff --git a/lib/pleroma/web/o_auth/o_auth_controller.ex b/lib/pleroma/web/o_auth/o_auth_controller.ex index 8f32e7219..3943ca449 100644 --- a/lib/pleroma/web/o_auth/o_auth_controller.ex +++ b/lib/pleroma/web/o_auth/o_auth_controller.ex @@ -211,11 +211,11 @@ defp handle_create_authorization_error( {:error, scopes_issue}, %{"authorization" => _} = params ) - when scopes_issue in [:unsupported_scopes, :missing_scopes] do + when scopes_issue in [:unsupported_scopes, :missing_scopes, :user_is_not_an_admin] do # Per https://github.com/tootsuite/mastodon/blob/ # 51e154f5e87968d6bb115e053689767ab33e80cd/app/controllers/api/base_controller.rb#L39 conn - |> put_flash(:error, dgettext("errors", "This action is outside the authorized scopes")) + |> put_flash(:error, dgettext("errors", "This action is outside of authorized scopes")) |> put_status(:unauthorized) |> authorize(params) end @@ -605,7 +605,7 @@ defp do_create_authorization( defp do_create_authorization(%User{} = user, %App{} = app, requested_scopes) when is_list(requested_scopes) do with {:account_status, :active} <- {:account_status, User.account_status(user)}, - {:ok, scopes} <- validate_scopes(app, requested_scopes), + {:ok, scopes} <- validate_scopes(user, app, requested_scopes), {:ok, auth} <- Authorization.create_authorization(app, user, scopes) do {:ok, auth} end @@ -637,15 +637,16 @@ defp build_and_response_mfa_token(user, auth) do end end - @spec validate_scopes(App.t(), map() | list()) :: + @spec validate_scopes(User.t(), App.t(), map() | list()) :: {:ok, list()} | {:error, :missing_scopes | :unsupported_scopes} - defp validate_scopes(%App{} = app, params) when is_map(params) do + defp validate_scopes(%User{} = user, %App{} = app, params) when is_map(params) do requested_scopes = Scopes.fetch_scopes(params, app.scopes) - validate_scopes(app, requested_scopes) + validate_scopes(user, app, requested_scopes) end - defp validate_scopes(%App{} = app, requested_scopes) when is_list(requested_scopes) do - Scopes.validate(requested_scopes, app.scopes) + defp validate_scopes(%User{} = user, %App{} = app, requested_scopes) + when is_list(requested_scopes) do + Scopes.validate(requested_scopes, app.scopes, user) end def default_redirect_uri(%App{} = app) do diff --git a/lib/pleroma/web/o_auth/scopes.ex b/lib/pleroma/web/o_auth/scopes.ex index ada43eae9..7fe04b912 100644 --- a/lib/pleroma/web/o_auth/scopes.ex +++ b/lib/pleroma/web/o_auth/scopes.ex @@ -56,12 +56,20 @@ def to_string(scopes), do: Enum.join(scopes, " ") @doc """ Validates scopes. """ - @spec validate(list() | nil, list()) :: - {:ok, list()} | {:error, :missing_scopes | :unsupported_scopes} - def validate(blank_scopes, _app_scopes) when blank_scopes in [nil, []], + @spec validate(list() | nil, list(), Pleroma.User.t()) :: + {:ok, list()} | {:error, :missing_scopes | :unsupported_scopes, :user_is_not_an_admin} + def validate(blank_scopes, _app_scopes, _user) when blank_scopes in [nil, []], do: {:error, :missing_scopes} - def validate(scopes, app_scopes) do + def validate(scopes, app_scopes, %Pleroma.User{is_admin: is_admin}) do + if !is_admin && contains_admin_scopes?(scopes) do + {:error, :user_is_not_an_admin} + else + validate_scopes_are_supported(scopes, app_scopes) + end + end + + defp validate_scopes_are_supported(scopes, app_scopes) do case OAuthScopesPlug.filter_descendants(scopes, app_scopes) do ^scopes -> {:ok, scopes} _ -> {:error, :unsupported_scopes} diff --git a/test/pleroma/web/o_auth/o_auth_controller_test.exs b/test/pleroma/web/o_auth/o_auth_controller_test.exs index 5a1258ec3..19a7dea60 100644 --- a/test/pleroma/web/o_auth/o_auth_controller_test.exs +++ b/test/pleroma/web/o_auth/o_auth_controller_test.exs @@ -693,45 +693,76 @@ test "with existing authentication and OOB `redirect_uri`, redirects to app with describe "POST /oauth/authorize" do test "redirects with oauth authorization, " <> - "granting requested app-supported scopes to both admin- and non-admin users" do + "granting requested app-supported scopes to both admin users" do app_scopes = ["read", "write", "admin", "secret_scope"] app = insert(:oauth_app, scopes: app_scopes) redirect_uri = OAuthController.default_redirect_uri(app) + scopes_subset = ["read:subscope", "write", "admin"] + admin = insert(:user, is_admin: true) + + # In case scope param is missing, expecting _all_ app-supported scopes to be granted + conn = + post( + build_conn(), + "/oauth/authorize", + %{ + "authorization" => %{ + "name" => admin.nickname, + "password" => "test", + "client_id" => app.client_id, + "redirect_uri" => redirect_uri, + "scope" => scopes_subset, + "state" => "statepassed" + } + } + ) + + target = redirected_to(conn) + assert target =~ redirect_uri + + query = URI.parse(target).query |> URI.query_decoder() |> Map.new() + + assert %{"state" => "statepassed", "code" => code} = query + auth = Repo.get_by(Authorization, token: code) + assert auth + assert auth.scopes == scopes_subset + end + + test "redirects with oauth authorization, " <> + "granting requested app-supported scopes for non-admin users" do + app_scopes = ["read", "write", "secret_scope", "admin"] + app = insert(:oauth_app, scopes: app_scopes) + redirect_uri = OAuthController.default_redirect_uri(app) non_admin = insert(:user, is_admin: false) - admin = insert(:user, is_admin: true) - scopes_subset = ["read:subscope", "write", "admin"] + scopes_subset = ["read:subscope", "write"] # In case scope param is missing, expecting _all_ app-supported scopes to be granted - for user <- [non_admin, admin], - {requested_scopes, expected_scopes} <- - %{scopes_subset => scopes_subset, nil: app_scopes} do - conn = - post( - build_conn(), - "/oauth/authorize", - %{ - "authorization" => %{ - "name" => user.nickname, - "password" => "test", - "client_id" => app.client_id, - "redirect_uri" => redirect_uri, - "scope" => requested_scopes, - "state" => "statepassed" - } + conn = + post( + build_conn(), + "/oauth/authorize", + %{ + "authorization" => %{ + "name" => non_admin.nickname, + "password" => "test", + "client_id" => app.client_id, + "redirect_uri" => redirect_uri, + "scope" => scopes_subset, + "state" => "statepassed" } - ) + } + ) - target = redirected_to(conn) - assert target =~ redirect_uri + target = redirected_to(conn) + assert target =~ redirect_uri - query = URI.parse(target).query |> URI.query_decoder() |> Map.new() + query = URI.parse(target).query |> URI.query_decoder() |> Map.new() - assert %{"state" => "statepassed", "code" => code} = query - auth = Repo.get_by(Authorization, token: code) - assert auth - assert auth.scopes == expected_scopes - end + assert %{"state" => "statepassed", "code" => code} = query + auth = Repo.get_by(Authorization, token: code) + assert auth + assert auth.scopes == scopes_subset end test "authorize from cookie" do @@ -739,6 +770,7 @@ test "authorize from cookie" do app = insert(:oauth_app) oauth_token = insert(:oauth_token, user: user, app: app) redirect_uri = OAuthController.default_redirect_uri(app) + IO.inspect(app) conn = build_conn() @@ -831,6 +863,33 @@ test "returns 401 for wrong credentials", %{conn: conn} do assert result =~ "Invalid Username/Password" end + test "returns 401 when attempting to use an admin scope with a non-admin", %{conn: conn} do + user = insert(:user) + app = insert(:oauth_app, scopes: ["admin"]) + redirect_uri = OAuthController.default_redirect_uri(app) + + result = + conn + |> post("/oauth/authorize", %{ + "authorization" => %{ + "name" => user.nickname, + "password" => "test", + "client_id" => app.client_id, + "redirect_uri" => redirect_uri, + "state" => "statepassed", + "scope" => Enum.join(app.scopes, " ") + } + }) + |> html_response(:unauthorized) + + # Keep the details + assert result =~ app.client_id + assert result =~ redirect_uri + + # Error message + assert result =~ "outside of authorized scopes" + end + test "returns 401 for missing scopes" do user = insert(:user, is_admin: false) app = insert(:oauth_app, scopes: ["read", "write", "admin"]) @@ -855,7 +914,7 @@ test "returns 401 for missing scopes" do assert result =~ redirect_uri # Error message - assert result =~ "This action is outside the authorized scopes" + assert result =~ "This action is outside of authorized scopes" end test "returns 401 for scopes beyond app scopes hierarchy", %{conn: conn} do @@ -882,7 +941,7 @@ test "returns 401 for scopes beyond app scopes hierarchy", %{conn: conn} do assert result =~ redirect_uri # Error message - assert result =~ "This action is outside the authorized scopes" + assert result =~ "This action is outside of authorized scopes" end end From c2054f82abbd67045de21e232e0731e6dbac55a0 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 16 Dec 2022 03:32:51 +0000 Subject: [PATCH 11/22] allow users with admin:metrics to read app metrics --- .gitignore | 1 + .../controllers/metrics_controller.ex | 10 +- lib/pleroma/web/plugs/rate_limiter.ex | 6 +- lib/pleroma/web/router.ex | 7 +- lib/pleroma/web/telemetry.ex | 9 +- lib/pleroma/web/templates/layout/app.html.eex | 32 +++-- .../web/templates/layout/static_fe.html.eex | 4 +- .../o_auth/oob_authorization_created.html.eex | 10 +- .../web/templates/o_auth/o_auth/show.html.eex | 4 +- mix.exs | 2 +- priv/static/static-fe/forms.css | 114 ++++++++++++++++++ priv/static/static-fe/static-fe.css | 61 +++++++--- scripts/create_metrics_app.sh | 65 ++++++++++ test/support/factory.ex | 2 +- 14 files changed, 283 insertions(+), 44 deletions(-) create mode 100644 priv/static/static-fe/forms.css create mode 100755 scripts/create_metrics_app.sh diff --git a/.gitignore b/.gitignore index 14373fb8c..95b236af6 100644 --- a/.gitignore +++ b/.gitignore @@ -76,3 +76,4 @@ docs/site # docker stuff docker-db +*.iml diff --git a/lib/pleroma/web/akkoma_api/controllers/metrics_controller.ex b/lib/pleroma/web/akkoma_api/controllers/metrics_controller.ex index c8d3d8948..8d413bf58 100644 --- a/lib/pleroma/web/akkoma_api/controllers/metrics_controller.ex +++ b/lib/pleroma/web/akkoma_api/controllers/metrics_controller.ex @@ -3,9 +3,13 @@ defmodule Pleroma.Web.AkkomaAPI.MetricsController do alias Pleroma.Web.Plugs.OAuthScopesPlug - @unauthenticated_access %{fallback: :proceed_unauthenticated, scopes: []} - plug(:skip_auth) - + plug( + OAuthScopesPlug, + %{scopes: ["admin:metrics"]} + when action in [ + :show + ] + ) def show(conn, _params) do stats = TelemetryMetricsPrometheus.Core.scrape() diff --git a/lib/pleroma/web/plugs/rate_limiter.ex b/lib/pleroma/web/plugs/rate_limiter.ex index 5bebe0ad5..f5ca27f0c 100644 --- a/lib/pleroma/web/plugs/rate_limiter.ex +++ b/lib/pleroma/web/plugs/rate_limiter.ex @@ -197,7 +197,11 @@ defp incorporate_conn_info(action_settings, %{params: params} = conn) do }) end - defp ip(%{remote_ip: remote_ip}) do + defp ip(%{remote_ip: remote_ip}) when is_binary(remote_ip) do + remote_ip + end + + defp ip(%{remote_ip: remote_ip}) when is_tuple(remote_ip) do remote_ip |> Tuple.to_list() |> Enum.join(".") diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index c0ce645c4..f47041b0b 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -868,7 +868,11 @@ defmodule Pleroma.Web.Router do scope "/" do pipe_through([:pleroma_html, :authenticate, :require_admin]) - live_dashboard("/phoenix/live_dashboard", metrics: {Pleroma.Web.Telemetry, :live_dashboard_metrics}, csp_nonce_assign_key: :csp_nonce) + + live_dashboard("/phoenix/live_dashboard", + metrics: {Pleroma.Web.Telemetry, :live_dashboard_metrics}, + csp_nonce_assign_key: :csp_nonce + ) end # Test-only routes needed to test action dispatching and plug chain execution @@ -907,6 +911,7 @@ defmodule Pleroma.Web.Router do scope "/", Pleroma.Web.Fallback do get("/registration/:token", RedirectController, :registration_page) get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta) + get("/api/*path", RedirectController, :api_not_implemented) get("/*path", RedirectController, :redirector_with_preload) options("/*path", RedirectController, :empty) diff --git a/lib/pleroma/web/telemetry.ex b/lib/pleroma/web/telemetry.ex index 435f55799..5b01ee14d 100644 --- a/lib/pleroma/web/telemetry.ex +++ b/lib/pleroma/web/telemetry.ex @@ -11,16 +11,13 @@ def start_link(arg) do def init(_arg) do children = [ {:telemetry_poller, measurements: periodic_measurements(), period: 10_000}, - {TelemetryMetricsPrometheus, metrics: prometheus_metrics(), plug_cowboy_opts: [ip: {127, 0, 0, 1}]} + {TelemetryMetricsPrometheus.Core, metrics: prometheus_metrics()} ] Supervisor.init(children, strategy: :one_for_one) end - @doc """ - A seperate set of metrics for distributions because phoenix dashboard does NOT handle - them well - """ + # A seperate set of metrics for distributions because phoenix dashboard does NOT handle them well defp distribution_metrics do [ distribution( @@ -110,8 +107,6 @@ defp summary_metrics do summary("vm.total_run_queue_lengths.total"), summary("vm.total_run_queue_lengths.cpu"), summary("vm.total_run_queue_lengths.io"), - - last_value("pleroma.local_users.total"), last_value("pleroma.domains.total"), last_value("pleroma.local_statuses.total") diff --git a/lib/pleroma/web/templates/layout/app.html.eex b/lib/pleroma/web/templates/layout/app.html.eex index e33bada85..31e6ec52b 100644 --- a/lib/pleroma/web/templates/layout/app.html.eex +++ b/lib/pleroma/web/templates/layout/app.html.eex @@ -4,17 +4,33 @@ <%= Pleroma.Config.get([:instance, :name]) %> - + + - + +
+
- <%= @inner_content %> +
+
+
+ <%= @inner_content %> +
+
+ + diff --git a/lib/pleroma/web/templates/layout/static_fe.html.eex b/lib/pleroma/web/templates/layout/static_fe.html.eex index 3d55393f0..d159eb901 100644 --- a/lib/pleroma/web/templates/layout/static_fe.html.eex +++ b/lib/pleroma/web/templates/layout/static_fe.html.eex @@ -20,8 +20,8 @@
-
- <%= @inner_content %> +
+ <%= @inner_content %>