From 2e5affce61a9255602d3a5d4c5caced9f09b1f5a Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 11 Jun 2019 14:27:41 +0700 Subject: [PATCH 1/5] Add RateLimiter --- docs/config.md | 11 +++ lib/pleroma/plugs/rate_limiter.ex | 87 ++++++++++++++++++++++++ mix.exs | 2 +- test/plugs/rate_limiter_test.exs | 108 ++++++++++++++++++++++++++++++ 4 files changed, 207 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/plugs/rate_limiter.ex create mode 100644 test/plugs/rate_limiter_test.exs diff --git a/docs/config.md b/docs/config.md index c61a5d8a3..e31e2b90f 100644 --- a/docs/config.md +++ b/docs/config.md @@ -616,3 +616,14 @@ To enable them, both the `rum_enabled` flag has to be set and the following spec `mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/` This will probably take a long time. + +## :rate_limit + +A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where: + +* The first element: `scale` (Integer). The time scale in milliseconds. +* The second element: `limit` (Integer). How many requests to limit in the time scale provided. + +It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated. + +See [`Pleroma.Plugs.RateLimiter`](Pleroma.Plugs.RateLimiter.html) documentation for examples. diff --git a/lib/pleroma/plugs/rate_limiter.ex b/lib/pleroma/plugs/rate_limiter.ex new file mode 100644 index 000000000..e02ba4213 --- /dev/null +++ b/lib/pleroma/plugs/rate_limiter.ex @@ -0,0 +1,87 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Plugs.RateLimiter do + @moduledoc """ + + ## Configuration + + A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where: + + * The first element: `scale` (Integer). The time scale in milliseconds. + * The second element: `limit` (Integer). How many requests to limit in the time scale provided. + + It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated. + + ### Example + + config :pleroma, :rate_limit, + one: {1000, 10}, + two: [{10_000, 10}, {10_000, 50}] + + Here we have two limiters: `one` which is not over 10req/1s and `two` which has two limits 10req/10s for unauthenticated users and 50req/10s for authenticated users. + + ## Usage + + Inside a controller: + + plug(Pleroma.Plugs.RateLimiter, :one when action == :one) + plug(Pleroma.Plugs.RateLimiter, :two when action in [:two, :three]) + + or inside a router pipiline: + + pipeline :api do + ... + plug(Pleroma.Plugs.RateLimiter, :one) + ... + end + """ + + import Phoenix.Controller, only: [json: 2] + import Plug.Conn + + alias Pleroma.User + + def init(limiter_name) do + case Pleroma.Config.get([:rate_limit, limiter_name]) do + nil -> nil + config -> {limiter_name, config} + end + end + + # do not limit if there is no limiter configuration + def call(conn, nil), do: conn + + def call(conn, opts) do + case check_rate(conn, opts) do + {:ok, _count} -> conn + {:error, _count} -> render_error(conn) + end + end + + defp check_rate(%{assigns: %{user: %User{id: user_id}}}, {limiter_name, [_, {scale, limit}]}) do + ExRated.check_rate("#{limiter_name}:#{user_id}", scale, limit) + end + + defp check_rate(conn, {limiter_name, [{scale, limit} | _]}) do + ExRated.check_rate("#{limiter_name}:#{ip(conn)}", scale, limit) + end + + defp check_rate(conn, {limiter_name, {scale, limit}}) do + check_rate(conn, {limiter_name, [{scale, limit}]}) + end + + def ip(%{remote_ip: remote_ip}) do + remote_ip + |> Tuple.to_list() + |> Enum.join(".") + end + + defp render_error(conn) do + conn + |> put_status(:too_many_requests) + |> json(%{error: "Throttled"}) + |> halt() + end +end diff --git a/mix.exs b/mix.exs index 9447a2e4f..1b78c5ca8 100644 --- a/mix.exs +++ b/mix.exs @@ -129,7 +129,7 @@ defp deps do {:quack, "~> 0.1.1"}, {:benchee, "~> 1.0"}, {:esshd, "~> 0.1.0", runtime: Application.get_env(:esshd, :enabled, false)}, - {:ex_rated, "~> 1.2"}, + {:ex_rated, "~> 1.3"}, {:plug_static_index_html, "~> 1.0.0"}, {:excoveralls, "~> 0.11.1", only: :test} ] ++ oauth_deps() diff --git a/test/plugs/rate_limiter_test.exs b/test/plugs/rate_limiter_test.exs new file mode 100644 index 000000000..b3798bf03 --- /dev/null +++ b/test/plugs/rate_limiter_test.exs @@ -0,0 +1,108 @@ +defmodule Pleroma.Plugs.RateLimiterTest do + use ExUnit.Case, async: true + use Plug.Test + + alias Pleroma.Plugs.RateLimiter + + import Pleroma.Factory + + @limiter_name :testing + + test "init/1" do + Pleroma.Config.put([:rate_limit, @limiter_name], {1, 1}) + + assert {@limiter_name, {1, 1}} == RateLimiter.init(@limiter_name) + assert nil == RateLimiter.init(:foo) + end + + test "ip/1" do + assert "127.0.0.1" == RateLimiter.ip(%{remote_ip: {127, 0, 0, 1}}) + end + + test "it restricts by opts" do + scale = 100 + limit = 5 + + Pleroma.Config.put([:rate_limit, @limiter_name], {scale, limit}) + + opts = RateLimiter.init(@limiter_name) + conn = conn(:get, "/") + bucket_name = "#{@limiter_name}:#{RateLimiter.ip(conn)}" + + conn = RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {2, 3, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {3, 2, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {4, 1, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {5, 0, to_reset, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + + assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) + assert conn.halted + + Process.sleep(to_reset) + + conn = conn(:get, "/") + + conn = RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + refute conn.status == Plug.Conn.Status.code(:too_many_requests) + refute conn.resp_body + refute conn.halted + end + + test "optional limits for authenticated users" do + Ecto.Adapters.SQL.Sandbox.checkout(Pleroma.Repo) + + scale = 100 + limit = 5 + Pleroma.Config.put([:rate_limit, @limiter_name], [{1, 10}, {scale, limit}]) + + opts = RateLimiter.init(@limiter_name) + + user = insert(:user) + conn = conn(:get, "/") |> assign(:user, user) + bucket_name = "#{@limiter_name}:#{user.id}" + + conn = RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {2, 3, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {3, 2, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {4, 1, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {5, 0, to_reset, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + + assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) + assert conn.halted + + Process.sleep(to_reset) + + conn = conn(:get, "/") |> assign(:user, user) + + conn = RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + refute conn.status == Plug.Conn.Status.code(:too_many_requests) + refute conn.resp_body + refute conn.halted + end +end From bc8f0593670452851d5e9d97bea1ae90f10db354 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 11 Jun 2019 14:28:39 +0700 Subject: [PATCH 2/5] Add rate limiting for search endpoints --- config/config.exs | 2 ++ lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 2 ++ 2 files changed, 4 insertions(+) diff --git a/config/config.exs b/config/config.exs index 4e2b1703b..d20d4fda9 100644 --- a/config/config.exs +++ b/config/config.exs @@ -503,6 +503,8 @@ config :http_signatures, adapter: Pleroma.Signature +config :pleroma, :rate_limit, search: [{1000, 10}, {1000, 30}] + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 92cd77f62..20b08fda4 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -55,6 +55,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do when action in [:account_register] ) + plug(Pleroma.Plugs.RateLimiter, :search when action in [:search, :search2, :account_search]) + @local_mastodon_name "Mastodon-Local" action_fallback(:errors) From f26013cf2ee9c04d4b99819991aaa5936e41cba4 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 11 Jun 2019 14:36:51 +0700 Subject: [PATCH 3/5] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf2232b09..cbdaa9c6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF: Support for stripping avatars and banner images from specific instances (`mrf_simple`) - MRF: Support for running subchains. - Configuration: `skip_thread_containment` option +- Configuration: `rate_limit` option. See `Pleroma.Plugs.RateLimiter` documentation for details. ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer From ad04d12de63d559cc6398c58296afd04321adfbc Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 11 Jun 2019 16:06:03 +0700 Subject: [PATCH 4/5] Replace `MastodonAPIController.account_register/2` rate limiter --- config/config.exs | 6 +-- config/test.exs | 2 +- docs/config.md | 8 +-- lib/pleroma/plugs/rate_limit_plug.ex | 36 ------------- .../mastodon_api/mastodon_api_controller.ex | 10 +--- test/plugs/rate_limit_plug_test.exs | 50 ------------------- .../mastodon_api_controller_test.exs | 20 +------- 7 files changed, 7 insertions(+), 125 deletions(-) delete mode 100644 lib/pleroma/plugs/rate_limit_plug.ex delete mode 100644 test/plugs/rate_limit_plug_test.exs diff --git a/config/config.exs b/config/config.exs index d20d4fda9..3d2c6d48e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -247,8 +247,6 @@ skip_thread_containment: false, limit_unauthenticated_to_local_content: true -config :pleroma, :app_account_creation, enabled: true, max_requests: 25, interval: 1800 - config :pleroma, :markup, # XXX - unfortunately, inline images must be enabled by default right now, because # of custom emoji. Issue #275 discusses defanging that somehow. @@ -503,7 +501,9 @@ config :http_signatures, adapter: Pleroma.Signature -config :pleroma, :rate_limit, search: [{1000, 10}, {1000, 30}] +config :pleroma, :rate_limit, + search: [{1000, 10}, {1000, 30}], + app_account_creation: {1_800_000, 25} # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/config/test.exs b/config/test.exs index 7861b9598..95129f409 100644 --- a/config/test.exs +++ b/config/test.exs @@ -59,7 +59,7 @@ total_user_limit: 3, enabled: false -config :pleroma, :app_account_creation, max_requests: 5 +config :pleroma, :rate_limit, app_account_creation: {1000, 5} config :pleroma, :http_security, report_uri: "https://endpoint.com" diff --git a/docs/config.md b/docs/config.md index e31e2b90f..b62b80490 100644 --- a/docs/config.md +++ b/docs/config.md @@ -114,12 +114,6 @@ config :pleroma, Pleroma.Emails.Mailer, * `skip_thread_containment`: Skip filter out broken threads. The default is `false`. * `limit_unauthenticated_to_local_content`: Limit unauthenticated users to search for local statutes and users only. The default is `true`. -## :app_account_creation -REST API for creating an account settings -* `enabled`: Enable/disable registration -* `max_requests`: Number of requests allowed for creating accounts -* `interval`: Interval for restricting requests for one ip (seconds) - ## :logger * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack @@ -568,7 +562,7 @@ config :ueberauth, Ueberauth, providers: [ microsoft: {Ueberauth.Strategy.Microsoft, [callback_params: []]} ] - + # Keycloak # Note: make sure to add `keycloak:ueberauth_keycloak_strategy` entry to `OAUTH_CONSUMER_STRATEGIES` environment variable keycloak_url = "https://publicly-reachable-keycloak-instance.org:8080" diff --git a/lib/pleroma/plugs/rate_limit_plug.ex b/lib/pleroma/plugs/rate_limit_plug.ex deleted file mode 100644 index 466f64a79..000000000 --- a/lib/pleroma/plugs/rate_limit_plug.ex +++ /dev/null @@ -1,36 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Plugs.RateLimitPlug do - import Phoenix.Controller, only: [json: 2] - import Plug.Conn - - def init(opts), do: opts - - def call(conn, opts) do - enabled? = Pleroma.Config.get([:app_account_creation, :enabled]) - - case check_rate(conn, Map.put(opts, :enabled, enabled?)) do - {:ok, _count} -> conn - {:error, _count} -> render_error(conn) - %Plug.Conn{} = conn -> conn - end - end - - defp check_rate(conn, %{enabled: true} = opts) do - max_requests = opts[:max_requests] - bucket_name = conn.remote_ip |> Tuple.to_list() |> Enum.join(".") - - ExRated.check_rate(bucket_name, opts[:interval] * 1000, max_requests) - end - - defp check_rate(conn, _), do: conn - - defp render_error(conn) do - conn - |> put_status(:forbidden) - |> json(%{error: "Rate limit exceeded."}) - |> halt() - end -end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 20b08fda4..46049dd24 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -46,15 +46,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do require Logger - plug( - Pleroma.Plugs.RateLimitPlug, - %{ - max_requests: Config.get([:app_account_creation, :max_requests]), - interval: Config.get([:app_account_creation, :interval]) - } - when action in [:account_register] - ) - + plug(Pleroma.Plugs.RateLimiter, :app_account_creation when action == :account_register) plug(Pleroma.Plugs.RateLimiter, :search when action in [:search, :search2, :account_search]) @local_mastodon_name "Mastodon-Local" diff --git a/test/plugs/rate_limit_plug_test.exs b/test/plugs/rate_limit_plug_test.exs deleted file mode 100644 index 2ec9a8fb7..000000000 --- a/test/plugs/rate_limit_plug_test.exs +++ /dev/null @@ -1,50 +0,0 @@ -defmodule Pleroma.Plugs.RateLimitPlugTest do - use ExUnit.Case, async: true - use Plug.Test - - alias Pleroma.Plugs.RateLimitPlug - - @opts RateLimitPlug.init(%{max_requests: 5, interval: 1}) - - setup do - enabled = Pleroma.Config.get([:app_account_creation, :enabled]) - - Pleroma.Config.put([:app_account_creation, :enabled], true) - - on_exit(fn -> - Pleroma.Config.put([:app_account_creation, :enabled], enabled) - end) - - :ok - end - - test "it restricts by opts" do - conn = conn(:get, "/") - bucket_name = conn.remote_ip |> Tuple.to_list() |> Enum.join(".") - ms = 1000 - - conn = RateLimitPlug.call(conn, @opts) - {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - conn = RateLimitPlug.call(conn, @opts) - {2, 3, _, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - conn = RateLimitPlug.call(conn, @opts) - {3, 2, _, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - conn = RateLimitPlug.call(conn, @opts) - {4, 1, _, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - conn = RateLimitPlug.call(conn, @opts) - {5, 0, to_reset, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - conn = RateLimitPlug.call(conn, @opts) - assert conn.status == 403 - assert conn.halted - assert conn.resp_body == "{\"error\":\"Rate limit exceeded.\"}" - - Process.sleep(to_reset) - - conn = conn(:get, "/") - conn = RateLimitPlug.call(conn, @opts) - {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - refute conn.status == 403 - refute conn.halted - refute conn.resp_body - end -end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 33c8e209a..c569ae9dd 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -3501,24 +3501,6 @@ test "Repeated posts that are replies incorrectly have in_reply_to_id null", %{c end describe "create account by app" do - setup do - enabled = Pleroma.Config.get([:app_account_creation, :enabled]) - max_requests = Pleroma.Config.get([:app_account_creation, :max_requests]) - interval = Pleroma.Config.get([:app_account_creation, :interval]) - - Pleroma.Config.put([:app_account_creation, :enabled], true) - Pleroma.Config.put([:app_account_creation, :max_requests], 5) - Pleroma.Config.put([:app_account_creation, :interval], 1) - - on_exit(fn -> - Pleroma.Config.put([:app_account_creation, :enabled], enabled) - Pleroma.Config.put([:app_account_creation, :max_requests], max_requests) - Pleroma.Config.put([:app_account_creation, :interval], interval) - end) - - :ok - end - test "Account registration via Application", %{conn: conn} do conn = conn @@ -3621,7 +3603,7 @@ test "rate limit", %{conn: conn} do agreement: true }) - assert json_response(conn, 403) == %{"error" => "Rate limit exceeded."} + assert json_response(conn, :too_many_requests) == %{"error" => "Throttled"} end end From 7c063a898d60cadd324087999226314586b72bd3 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 11 Jun 2019 16:25:47 +0700 Subject: [PATCH 5/5] Update `ex_rated` dependency --- mix.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.lock b/mix.lock index 857bfca79..0b21d9761 100644 --- a/mix.lock +++ b/mix.lock @@ -29,7 +29,7 @@ "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.1", "9e09366e77f25d3d88c5393824e613344631be8db0d1839faca49686e99b6704", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"}, "ex_doc": {:hex, :ex_doc, "0.20.2", "1bd0dfb0304bade58beb77f20f21ee3558cc3c753743ae0ddbb0fd7ba2912331", [:mix], [{:earmark, "~> 1.3", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"}, - "ex_rated": {:hex, :ex_rated, "1.3.2", "6aeb32abb46ea6076f417a9ce8cb1cf08abf35fb2d42375beaad4dd72b550bf1", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, + "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, "excoveralls": {:hex, :excoveralls, "0.11.1", "dd677fbdd49114fdbdbf445540ec735808250d56b011077798316505064edb2c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "floki": {:hex, :floki, "0.20.4", "be42ac911fece24b4c72f3b5846774b6e61b83fe685c2fc9d62093277fb3bc86", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}, {:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"},