diff --git a/CHANGELOG.md b/CHANGELOG.md index cf2232b09..7ecdfe939 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - [MongooseIM](https://github.com/esl/MongooseIM) http authentication support. - LDAP authentication - External OAuth provider authentication +- Support for building a release using [`mix release`](https://hexdocs.pm/mix/master/Mix.Tasks.Release.html) - A [job queue](https://git.pleroma.social/pleroma/pleroma_job_queue) for federation, emails, web push, etc. - [Prometheus](https://prometheus.io/) metrics - Support for Mastodon's remote interaction @@ -27,7 +28,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Configuration: `notify_email` option - Configuration: Media proxy `whitelist` option - Configuration: `report_uri` option -- Configuration: `limit_unauthenticated_to_local_content` option +- Configuration: `limit_to_local_content` option - Pleroma API: User subscriptions - Pleroma API: Healthcheck endpoint - Pleroma API: `/api/v1/pleroma/mascot` per-user frontend mascot configuration endpoints @@ -53,9 +54,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF: Support for stripping avatars and banner images from specific instances (`mrf_simple`) - MRF: Support for running subchains. - Configuration: `skip_thread_containment` option +- Configuration: `rate_limit` option. See `Pleroma.Plugs.RateLimiter` documentation for details. ### Changed - **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer +- Thread containment / test for complete visibility will be skipped by default. - Enforcement of OAuth scopes - Add multiple use/time expiring invite token - Restyled OAuth pages to fit with Pleroma's default theme @@ -64,6 +67,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Federation: Expand the audience of delete activities to all recipients of the deleted object - Federation: Removed `inReplyToStatusId` from objects - Configuration: Dedupe enabled by default +- Configuration: Default log level in `prod` environment is now set to `warn` - Configuration: Added `extra_cookie_attrs` for setting non-standard cookie attributes. Defaults to ["SameSite=Lax"] so that remote follows work. - Timelines: Messages involving people you have blocked will be excluded from the timeline in all cases instead of just repeats. - Admin API: Move the user related API to `api/pleroma/admin/users` diff --git a/config/config.exs b/config/config.exs index 4e2b1703b..f866e8d2b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -244,10 +244,8 @@ safe_dm_mentions: false, healthcheck: false, remote_post_retention_days: 90, - skip_thread_containment: false, - limit_unauthenticated_to_local_content: true - -config :pleroma, :app_account_creation, enabled: true, max_requests: 25, interval: 1800 + skip_thread_containment: true, + limit_to_local_content: :unauthenticated config :pleroma, :markup, # XXX - unfortunately, inline images must be enabled by default right now, because @@ -362,8 +360,8 @@ third_party_engine: "http://vinayaka.distsn.org/cgi-bin/vinayaka-user-match-suggestions-api.cgi?{{host}}+{{user}}", timeout: 300_000, - limit: 23, - web: "https://vinayaka.distsn.org/?{{host}}+{{user}}" + limit: 40, + web: "https://vinayaka.distsn.org" config :pleroma, :http_security, enabled: true, @@ -500,9 +498,15 @@ config :pleroma, :database, rum_enabled: false +config :pleroma, :env, Mix.env() + config :http_signatures, adapter: Pleroma.Signature +config :pleroma, :rate_limit, + search: [{1000, 10}, {1000, 30}], + app_account_creation: {1_800_000, 25} + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/prod.exs b/config/prod.exs index d0cfd1ac2..bf1a97de0 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -17,8 +17,10 @@ http: [port: 4000], protocol: "http" +config :phoenix, serve_endpoints: true + # Do not print debug messages in production -config :logger, level: :info +config :logger, level: :warn # ## SSL Support # diff --git a/config/releases.exs b/config/releases.exs new file mode 100644 index 000000000..f8494dd34 --- /dev/null +++ b/config/releases.exs @@ -0,0 +1,16 @@ +import Config + +config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs" + +if File.exists?(config_path) do + import_config config_path +else + warning = [ + IO.ANSI.red(), + IO.ANSI.bright(), + "!!! #{config_path} not found! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file", + IO.ANSI.reset() + ] + + IO.puts(warning) +end diff --git a/config/test.exs b/config/test.exs index 7861b9598..1c5eff794 100644 --- a/config/test.exs +++ b/config/test.exs @@ -27,7 +27,8 @@ config :pleroma, :instance, email: "admin@example.com", - notify_email: "noreply@example.com" + notify_email: "noreply@example.com", + skip_thread_containment: false # Configure your database config :pleroma, Pleroma.Repo, @@ -59,7 +60,7 @@ total_user_limit: 3, enabled: false -config :pleroma, :app_account_creation, max_requests: 5 +config :pleroma, :rate_limit, app_account_creation: {1000, 5} config :pleroma, :http_security, report_uri: "https://endpoint.com" diff --git a/docs/config.md b/docs/config.md index c61a5d8a3..2b0f5726b 100644 --- a/docs/config.md +++ b/docs/config.md @@ -86,6 +86,7 @@ config :pleroma, Pleroma.Emails.Mailer, * `Pleroma.Web.ActivityPub.MRF.NoOpPolicy`: Doesn’t modify activities (default) * `Pleroma.Web.ActivityPub.MRF.DropPolicy`: Drops all activities. It generally doesn’t makes sense to use in production * `Pleroma.Web.ActivityPub.MRF.SimplePolicy`: Restrict the visibility of activities from certains instances (See ``:mrf_simple`` section) + * `Pleroma.Web.ActivityPub.MRF.TagPolicy`: Applies policies to individual users based on tags, which can be set using pleroma-fe/admin-fe/any other app that supports Pleroma Admin API. For example it allows marking posts from individual users nsfw (sensitive) * `Pleroma.Web.ActivityPub.MRF.SubchainPolicy`: Selectively runs other MRF policies when messages match (see ``:mrf_subchain`` section) * `Pleroma.Web.ActivityPub.MRF.RejectNonPublic`: Drops posts with non-public visibility settings (See ``:mrf_rejectnonpublic`` section) * `Pleroma.Web.ActivityPub.MRF.EnsureRePrepended`: Rewrites posts to ensure that replies to posts with subjects do not have an identical subject and instead begin with re:. @@ -112,13 +113,8 @@ config :pleroma, Pleroma.Emails.Mailer, * `healthcheck`: If set to true, system data will be shown on ``/api/pleroma/healthcheck``. * `remote_post_retention_days`: The default amount of days to retain remote posts when pruning the database. * `skip_thread_containment`: Skip filter out broken threads. The default is `false`. -* `limit_unauthenticated_to_local_content`: Limit unauthenticated users to search for local statutes and users only. The default is `true`. +* `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`. -## :app_account_creation -REST API for creating an account settings -* `enabled`: Enable/disable registration -* `max_requests`: Number of requests allowed for creating accounts -* `interval`: Interval for restricting requests for one ip (seconds) ## :logger * `backends`: `:console` is used to send logs to stdout, `{ExSyslogger, :ex_syslogger}` to log to syslog, and `Quack.Logger` to log to Slack @@ -568,7 +564,7 @@ config :ueberauth, Ueberauth, providers: [ microsoft: {Ueberauth.Strategy.Microsoft, [callback_params: []]} ] - + # Keycloak # Note: make sure to add `keycloak:ueberauth_keycloak_strategy` entry to `OAUTH_CONSUMER_STRATEGIES` environment variable keycloak_url = "https://publicly-reachable-keycloak-instance.org:8080" @@ -616,3 +612,14 @@ To enable them, both the `rum_enabled` flag has to be set and the following spec `mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/` This will probably take a long time. + +## :rate_limit + +A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where: + +* The first element: `scale` (Integer). The time scale in milliseconds. +* The second element: `limit` (Integer). How many requests to limit in the time scale provided. + +It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated. + +See [`Pleroma.Plugs.RateLimiter`](Pleroma.Plugs.RateLimiter.html) documentation for examples. diff --git a/docs/config/howto_user_recomendation.md b/docs/config/howto_user_recomendation.md index 27c0760dd..c4d749d0c 100644 --- a/docs/config/howto_user_recomendation.md +++ b/docs/config/howto_user_recomendation.md @@ -9,8 +9,8 @@ config :pleroma, :suggestions, third_party_engine: "http://vinayaka.distsn.org/cgi-bin/vinayaka-user-match-suggestions-api.cgi?{{host}}+{{user}}", timeout: 300_000, - limit: 23, - web: "https://vinayaka.distsn.org/?{{host}}+{{user}}" + limit: 40, + web: "https://vinayaka.distsn.org" ``` @@ -26,6 +26,6 @@ config :pleroma, :suggestions, third_party_engine: "http://vinayaka.distsn.org/cgi-bin/vinayaka-user-new-suggestions-api.cgi?{{host}}+{{user}}", timeout: 60_000, - limit: 23, + limit: 40, web: "https://vinayaka.distsn.org/user-new.html" ``` diff --git a/lib/mix/tasks/pleroma/common.ex b/lib/mix/tasks/pleroma/common.ex index 48c0c1346..7d50605af 100644 --- a/lib/mix/tasks/pleroma/common.ex +++ b/lib/mix/tasks/pleroma/common.ex @@ -5,23 +5,58 @@ defmodule Mix.Tasks.Pleroma.Common do @doc "Common functions to be reused in mix tasks" def start_pleroma do - Mix.Task.run("app.start") + Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) + {:ok, _} = Application.ensure_all_started(:pleroma) end def get_option(options, opt, prompt, defval \\ nil, defname \\ nil) do - Keyword.get(options, opt) || - case Mix.shell().prompt("#{prompt} [#{defname || defval}]") do - "\n" -> - case defval do - nil -> get_option(options, opt, prompt, defval) - defval -> defval - end - - opt -> - opt |> String.trim() - end + Keyword.get(options, opt) || shell_prompt(prompt, defval, defname) end + def shell_prompt(prompt, defval \\ nil, defname \\ nil) do + prompt_message = "#{prompt} [#{defname || defval}]" + + input = + if mix_shell?(), + do: Mix.shell().prompt(prompt_message), + else: :io.get_line(prompt_message) + + case input do + "\n" -> + case defval do + nil -> + shell_prompt(prompt, defval, defname) + + defval -> + defval + end + + input -> + String.trim(input) + end + end + + def shell_yes?(message) do + if mix_shell?(), + do: Mix.shell().yes?("Continue?"), + else: shell_prompt(message, "Continue?") in ~w(Yn Y y) + end + + def shell_info(message) do + if mix_shell?(), + do: Mix.shell().info(message), + else: IO.puts(message) + end + + def shell_error(message) do + if mix_shell?(), + do: Mix.shell().error(message), + else: IO.puts(:stderr, message) + end + + @doc "Performs a safe check whether `Mix.shell/0` is available (does not raise if Mix is not loaded)" + def mix_shell?, do: :erlang.function_exported(Mix, :shell, 0) + def escape_sh_path(path) do ~S(') <> String.replace(path, ~S('), ~S(\')) <> ~S(') end diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 6cee8d630..88925dbaf 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -155,17 +155,17 @@ def run(["gen" | rest]) do dbpass: dbpass ) - Mix.shell().info( + Common.shell_info( "Writing config to #{config_path}. You should rename it to config/prod.secret.exs or config/dev.secret.exs." ) File.write(config_path, result_config) - Mix.shell().info("Writing #{psql_path}.") + Common.shell_info("Writing #{psql_path}.") File.write(psql_path, result_psql) write_robots_txt(indexable) - Mix.shell().info( + Common.shell_info( "\n" <> """ To get started: @@ -179,7 +179,7 @@ def run(["gen" | rest]) do end ) else - Mix.shell().error( + Common.shell_error( "The task would have overwritten the following files:\n" <> (Enum.map(paths, &"- #{&1}\n") |> Enum.join("")) <> "Rerun with `--force` to overwrite them." @@ -204,10 +204,10 @@ defp write_robots_txt(indexable) do if File.exists?(robots_txt_path) do File.cp!(robots_txt_path, "#{robots_txt_path}.bak") - Mix.shell().info("Backing up existing robots.txt to #{robots_txt_path}.bak") + Common.shell_info("Backing up existing robots.txt to #{robots_txt_path}.bak") end File.write(robots_txt_path, robots_txt) - Mix.shell().info("Writing #{robots_txt_path}.") + Common.shell_info("Writing #{robots_txt_path}.") end end diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex index fbec473c5..213ae24d2 100644 --- a/lib/mix/tasks/pleroma/relay.ex +++ b/lib/mix/tasks/pleroma/relay.ex @@ -30,7 +30,7 @@ def run(["follow", target]) do # put this task to sleep to allow the genserver to push out the messages :timer.sleep(500) else - {:error, e} -> Mix.shell().error("Error while following #{target}: #{inspect(e)}") + {:error, e} -> Common.shell_error("Error while following #{target}: #{inspect(e)}") end end @@ -41,7 +41,7 @@ def run(["unfollow", target]) do # put this task to sleep to allow the genserver to push out the messages :timer.sleep(500) else - {:error, e} -> Mix.shell().error("Error while following #{target}: #{inspect(e)}") + {:error, e} -> Common.shell_error("Error while following #{target}: #{inspect(e)}") end end end diff --git a/lib/mix/tasks/pleroma/uploads.ex b/lib/mix/tasks/pleroma/uploads.ex index 106fcf443..8855b5538 100644 --- a/lib/mix/tasks/pleroma/uploads.ex +++ b/lib/mix/tasks/pleroma/uploads.ex @@ -38,10 +38,10 @@ def run(["migrate_local", target_uploader | args]) do Pleroma.Config.put([Upload, :uploader], uploader) end - Mix.shell().info("Migrating files from local #{local_path} to #{to_string(uploader)}") + Common.shell_info("Migrating files from local #{local_path} to #{to_string(uploader)}") if delete? do - Mix.shell().info( + Common.shell_info( "Attention: uploaded files will be deleted, hope you have backups! (--delete ; cancel with ^C)" ) @@ -78,7 +78,7 @@ def run(["migrate_local", target_uploader | args]) do |> Enum.filter(& &1) total_count = length(uploads) - Mix.shell().info("Found #{total_count} uploads") + Common.shell_info("Found #{total_count} uploads") uploads |> Task.async_stream( @@ -90,7 +90,7 @@ def run(["migrate_local", target_uploader | args]) do :ok error -> - Mix.shell().error("failed to upload #{inspect(upload.path)}: #{inspect(error)}") + Common.shell_error("failed to upload #{inspect(upload.path)}: #{inspect(error)}") end end, timeout: 150_000 @@ -99,10 +99,10 @@ def run(["migrate_local", target_uploader | args]) do # credo:disable-for-next-line Credo.Check.Warning.UnusedEnumOperation |> Enum.reduce(0, fn done, count -> count = count + length(done) - Mix.shell().info("Uploaded #{count}/#{total_count} files") + Common.shell_info("Uploaded #{count}/#{total_count} files") count end) - Mix.shell().info("Done!") + Common.shell_info("Done!") end end diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index 25fc40ea7..7eaa49836 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -115,7 +115,7 @@ def run(["new", nickname, email | rest]) do admin? = Keyword.get(options, :admin, false) assume_yes? = Keyword.get(options, :assume_yes, false) - Mix.shell().info(""" + Common.shell_info(""" A user will be created with the following information: - nickname: #{nickname} - email: #{email} @@ -128,7 +128,7 @@ def run(["new", nickname, email | rest]) do - admin: #{if(admin?, do: "true", else: "false")} """) - proceed? = assume_yes? or Mix.shell().yes?("Continue?") + proceed? = assume_yes? or Common.shell_yes?("Continue?") if proceed? do Common.start_pleroma() @@ -145,7 +145,7 @@ def run(["new", nickname, email | rest]) do changeset = User.register_changeset(%User{}, params, need_confirmation: false) {:ok, _user} = User.register(changeset) - Mix.shell().info("User #{nickname} created") + Common.shell_info("User #{nickname} created") if moderator? do run(["set", nickname, "--moderator"]) @@ -159,7 +159,7 @@ def run(["new", nickname, email | rest]) do run(["reset_password", nickname]) end else - Mix.shell().info("User will not be created.") + Common.shell_info("User will not be created.") end end @@ -168,10 +168,10 @@ def run(["rm", nickname]) do with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do User.perform(:delete, user) - Mix.shell().info("User #{nickname} deleted.") + Common.shell_info("User #{nickname} deleted.") else _ -> - Mix.shell().error("No local user #{nickname}") + Common.shell_error("No local user #{nickname}") end end @@ -181,12 +181,12 @@ def run(["toggle_activated", nickname]) do with %User{} = user <- User.get_cached_by_nickname(nickname) do {:ok, user} = User.deactivate(user, !user.info.deactivated) - Mix.shell().info( + Common.shell_info( "Activation status of #{nickname}: #{if(user.info.deactivated, do: "de", else: "")}activated" ) else _ -> - Mix.shell().error("No user #{nickname}") + Common.shell_error("No user #{nickname}") end end @@ -195,7 +195,7 @@ def run(["reset_password", nickname]) do with %User{local: true} = user <- User.get_cached_by_nickname(nickname), {:ok, token} <- Pleroma.PasswordResetToken.create_token(user) do - Mix.shell().info("Generated password reset token for #{user.nickname}") + Common.shell_info("Generated password reset token for #{user.nickname}") IO.puts( "URL: #{ @@ -208,7 +208,7 @@ def run(["reset_password", nickname]) do ) else _ -> - Mix.shell().error("No local user #{nickname}") + Common.shell_error("No local user #{nickname}") end end @@ -216,7 +216,7 @@ def run(["unsubscribe", nickname]) do Common.start_pleroma() with %User{} = user <- User.get_cached_by_nickname(nickname) do - Mix.shell().info("Deactivating #{user.nickname}") + Common.shell_info("Deactivating #{user.nickname}") User.deactivate(user) {:ok, friends} = User.get_friends(user) @@ -224,7 +224,7 @@ def run(["unsubscribe", nickname]) do Enum.each(friends, fn friend -> user = User.get_cached_by_id(user.id) - Mix.shell().info("Unsubscribing #{friend.nickname} from #{user.nickname}") + Common.shell_info("Unsubscribing #{friend.nickname} from #{user.nickname}") User.unfollow(user, friend) end) @@ -233,11 +233,11 @@ def run(["unsubscribe", nickname]) do user = User.get_cached_by_id(user.id) if Enum.empty?(user.following) do - Mix.shell().info("Successfully unsubscribed all followers from #{user.nickname}") + Common.shell_info("Successfully unsubscribed all followers from #{user.nickname}") end else _ -> - Mix.shell().error("No user #{nickname}") + Common.shell_error("No user #{nickname}") end end @@ -274,7 +274,7 @@ def run(["set", nickname | rest]) do end else _ -> - Mix.shell().error("No local user #{nickname}") + Common.shell_error("No local user #{nickname}") end end @@ -284,10 +284,10 @@ def run(["tag", nickname | tags]) do with %User{} = user <- User.get_cached_by_nickname(nickname) do user = user |> User.tag(tags) - Mix.shell().info("Tags of #{user.nickname}: #{inspect(tags)}") + Common.shell_info("Tags of #{user.nickname}: #{inspect(tags)}") else _ -> - Mix.shell().error("Could not change user tags for #{nickname}") + Common.shell_error("Could not change user tags for #{nickname}") end end @@ -297,10 +297,10 @@ def run(["untag", nickname | tags]) do with %User{} = user <- User.get_cached_by_nickname(nickname) do user = user |> User.untag(tags) - Mix.shell().info("Tags of #{user.nickname}: #{inspect(tags)}") + Common.shell_info("Tags of #{user.nickname}: #{inspect(tags)}") else _ -> - Mix.shell().error("Could not change user tags for #{nickname}") + Common.shell_error("Could not change user tags for #{nickname}") end end @@ -326,7 +326,7 @@ def run(["invite" | rest]) do with {:ok, val} <- options[:expires_at], options = Map.put(options, :expires_at, val), {:ok, invite} <- UserInviteToken.create_invite(options) do - Mix.shell().info( + Common.shell_info( "Generated user invite token " <> String.replace(invite.invite_type, "_", " ") ) @@ -340,14 +340,14 @@ def run(["invite" | rest]) do IO.puts(url) else error -> - Mix.shell().error("Could not create invite token: #{inspect(error)}") + Common.shell_error("Could not create invite token: #{inspect(error)}") end end def run(["invites"]) do Common.start_pleroma() - Mix.shell().info("Invites list:") + Common.shell_info("Invites list:") UserInviteToken.list_invites() |> Enum.each(fn invite -> @@ -361,7 +361,7 @@ def run(["invites"]) do " | Max use: #{max_use} Left use: #{max_use - invite.uses}" end - Mix.shell().info( + Common.shell_info( "ID: #{invite.id} | Token: #{invite.token} | Token type: #{invite.invite_type} | Used: #{ invite.used }#{expire_info}#{using_info}" @@ -374,9 +374,9 @@ def run(["revoke_invite", token]) do with {:ok, invite} <- UserInviteToken.find_by_token(token), {:ok, _} <- UserInviteToken.update_invite(invite, %{used: true}) do - Mix.shell().info("Invite for token #{token} was revoked.") + Common.shell_info("Invite for token #{token} was revoked.") else - _ -> Mix.shell().error("No invite found with token #{token}") + _ -> Common.shell_error("No invite found with token #{token}") end end @@ -385,10 +385,10 @@ def run(["delete_activities", nickname]) do with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do {:ok, _} = User.delete_user_activities(user) - Mix.shell().info("User #{nickname} statuses deleted.") + Common.shell_info("User #{nickname} statuses deleted.") else _ -> - Mix.shell().error("No local user #{nickname}") + Common.shell_error("No local user #{nickname}") end end @@ -400,10 +400,10 @@ def run(["toggle_confirmed", nickname]) do message = if user.info.confirmation_pending, do: "needs", else: "doesn't need" - Mix.shell().info("#{nickname} #{message} confirmation.") + Common.shell_info("#{nickname} #{message} confirmation.") else _ -> - Mix.shell().error("No local user #{nickname}") + Common.shell_error("No local user #{nickname}") end end @@ -416,7 +416,7 @@ defp set_moderator(user, value) do {:ok, user} = User.update_and_set_cache(user_cng) - Mix.shell().info("Moderator status of #{user.nickname}: #{user.info.is_moderator}") + Common.shell_info("Moderator status of #{user.nickname}: #{user.info.is_moderator}") user end @@ -429,7 +429,7 @@ defp set_admin(user, value) do {:ok, user} = User.update_and_set_cache(user_cng) - Mix.shell().info("Admin status of #{user.nickname}: #{user.info.is_admin}") + Common.shell_info("Admin status of #{user.nickname}: #{user.info.is_admin}") user end @@ -442,7 +442,7 @@ defp set_locked(user, value) do {:ok, user} = User.update_and_set_cache(user_cng) - Mix.shell().info("Locked status of #{user.nickname}: #{user.info.locked}") + Common.shell_info("Locked status of #{user.nickname}: #{user.info.locked}") user end end diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex index 9ccedcd13..0aa2aab23 100644 --- a/lib/pleroma/activity/search.ex +++ b/lib/pleroma/activity/search.ex @@ -39,8 +39,7 @@ defp query_with(q, :gin, search_query) do "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", o.data, ^search_query - ), - order_by: [desc: :id] + ) ) end @@ -56,18 +55,19 @@ defp query_with(q, :rum, search_query) do ) end - # users can search everything - defp maybe_restrict_local(q, %User{}), do: q + defp maybe_restrict_local(q, user) do + limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) - # unauthenticated users can only search local activities - defp maybe_restrict_local(q, _) do - if Pleroma.Config.get([:instance, :limit_unauthenticated_to_local_content], true) do - where(q, local: true) - else - q + case {limit, user} do + {:all, _} -> restrict_local(q) + {:unauthenticated, %User{}} -> q + {:unauthenticated, _} -> restrict_local(q) + {false, _} -> q end end + defp restrict_local(q), do: where(q, local: true) + defp maybe_fetch(activities, user, search_query) do with true <- Regex.match?(~r/https?:/, search_query), {:ok, object} <- Fetcher.fetch_object_from_id(search_query), diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 69a8a5761..5627d20af 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -194,7 +194,7 @@ def enabled_hackney_pools do end end - if Mix.env() == :test do + if Pleroma.Config.get(:env) == :test do defp streamer_child, do: [] defp chat_child, do: [] else diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 2c13c4b40..5883e4183 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -59,10 +59,10 @@ def mark_as_unread(participation) do def for_user(user, params \\ %{}) do from(p in __MODULE__, where: p.user_id == ^user.id, - order_by: [desc: p.updated_at] + order_by: [desc: p.updated_at], + preload: [conversation: [:users]] ) |> Pleroma.Pagination.fetch_paginated(params) - |> Repo.preload(conversation: [:users]) end def for_user_with_last_activity_id(user, params \\ %{}) do diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index de7fcc1ce..b77b26f7f 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -98,7 +98,9 @@ defp load do Logger.error("Could not access the custom emoji directory #{emoji_dir_path}: #{e}") {:ok, results} -> - grouped = Enum.group_by(results, &File.dir?/1) + grouped = + Enum.group_by(results, fn file -> File.dir?(Path.join(emoji_dir_path, file)) end) + packs = grouped[true] || [] files = grouped[false] || [] diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index 485ddfbc7..a7cc22831 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -56,14 +56,14 @@ defp csp_string do connect_src = "connect-src 'self' #{static_url} #{websocket_url}" connect_src = - if Mix.env() == :dev do + if Pleroma.Config.get(:env) == :dev do connect_src <> " http://localhost:3035/" else connect_src end script_src = - if Mix.env() == :dev do + if Pleroma.Config.get(:env) == :dev do "script-src 'self' 'unsafe-eval'" else "script-src 'self'" diff --git a/lib/pleroma/plugs/rate_limit_plug.ex b/lib/pleroma/plugs/rate_limit_plug.ex deleted file mode 100644 index 466f64a79..000000000 --- a/lib/pleroma/plugs/rate_limit_plug.ex +++ /dev/null @@ -1,36 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2019 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Plugs.RateLimitPlug do - import Phoenix.Controller, only: [json: 2] - import Plug.Conn - - def init(opts), do: opts - - def call(conn, opts) do - enabled? = Pleroma.Config.get([:app_account_creation, :enabled]) - - case check_rate(conn, Map.put(opts, :enabled, enabled?)) do - {:ok, _count} -> conn - {:error, _count} -> render_error(conn) - %Plug.Conn{} = conn -> conn - end - end - - defp check_rate(conn, %{enabled: true} = opts) do - max_requests = opts[:max_requests] - bucket_name = conn.remote_ip |> Tuple.to_list() |> Enum.join(".") - - ExRated.check_rate(bucket_name, opts[:interval] * 1000, max_requests) - end - - defp check_rate(conn, _), do: conn - - defp render_error(conn) do - conn - |> put_status(:forbidden) - |> json(%{error: "Rate limit exceeded."}) - |> halt() - end -end diff --git a/lib/pleroma/plugs/rate_limiter.ex b/lib/pleroma/plugs/rate_limiter.ex new file mode 100644 index 000000000..e02ba4213 --- /dev/null +++ b/lib/pleroma/plugs/rate_limiter.ex @@ -0,0 +1,87 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Plugs.RateLimiter do + @moduledoc """ + + ## Configuration + + A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where: + + * The first element: `scale` (Integer). The time scale in milliseconds. + * The second element: `limit` (Integer). How many requests to limit in the time scale provided. + + It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated. + + ### Example + + config :pleroma, :rate_limit, + one: {1000, 10}, + two: [{10_000, 10}, {10_000, 50}] + + Here we have two limiters: `one` which is not over 10req/1s and `two` which has two limits 10req/10s for unauthenticated users and 50req/10s for authenticated users. + + ## Usage + + Inside a controller: + + plug(Pleroma.Plugs.RateLimiter, :one when action == :one) + plug(Pleroma.Plugs.RateLimiter, :two when action in [:two, :three]) + + or inside a router pipiline: + + pipeline :api do + ... + plug(Pleroma.Plugs.RateLimiter, :one) + ... + end + """ + + import Phoenix.Controller, only: [json: 2] + import Plug.Conn + + alias Pleroma.User + + def init(limiter_name) do + case Pleroma.Config.get([:rate_limit, limiter_name]) do + nil -> nil + config -> {limiter_name, config} + end + end + + # do not limit if there is no limiter configuration + def call(conn, nil), do: conn + + def call(conn, opts) do + case check_rate(conn, opts) do + {:ok, _count} -> conn + {:error, _count} -> render_error(conn) + end + end + + defp check_rate(%{assigns: %{user: %User{id: user_id}}}, {limiter_name, [_, {scale, limit}]}) do + ExRated.check_rate("#{limiter_name}:#{user_id}", scale, limit) + end + + defp check_rate(conn, {limiter_name, [{scale, limit} | _]}) do + ExRated.check_rate("#{limiter_name}:#{ip(conn)}", scale, limit) + end + + defp check_rate(conn, {limiter_name, {scale, limit}}) do + check_rate(conn, {limiter_name, [{scale, limit}]}) + end + + def ip(%{remote_ip: remote_ip}) do + remote_ip + |> Tuple.to_list() + |> Enum.join(".") + end + + defp render_error(conn) do + conn + |> put_status(:too_many_requests) + |> json(%{error: "Throttled"}) + |> halt() + end +end diff --git a/lib/pleroma/release_tasks.ex b/lib/pleroma/release_tasks.ex new file mode 100644 index 000000000..7726bc635 --- /dev/null +++ b/lib/pleroma/release_tasks.ex @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ReleaseTasks do + @repo Pleroma.Repo + + def run(args) do + Mix.Tasks.Pleroma.Common.start_pleroma() + [task | args] = String.split(args) + + case task do + "migrate" -> migrate() + "create" -> create() + "rollback" -> rollback(String.to_integer(Enum.at(args, 0))) + task -> mix_task(task, args) + end + end + + defp mix_task(task, args) do + {:ok, modules} = :application.get_key(:pleroma, :modules) + + module = + Enum.find(modules, fn module -> + module = Module.split(module) + + match?(["Mix", "Tasks", "Pleroma" | _], module) and + String.downcase(List.last(module)) == task + end) + + if module do + module.run(args) + else + IO.puts("The task #{task} does not exist") + end + end + + def migrate do + {:ok, _, _} = Ecto.Migrator.with_repo(@repo, &Ecto.Migrator.run(&1, :up, all: true)) + end + + def rollback(version) do + {:ok, _, _} = Ecto.Migrator.with_repo(@repo, &Ecto.Migrator.run(&1, :down, to: version)) + end + + def create do + case @repo.__adapter__.storage_up(@repo.config) do + :ok -> + IO.puts("The database for #{inspect(@repo)} has been created") + + {:error, :already_up} -> + IO.puts("The database for #{inspect(@repo)} has already been created") + + {:error, term} when is_binary(term) -> + IO.puts(:stderr, "The database for #{inspect(@repo)} couldn't be created: #{term}") + + {:error, term} -> + IO.puts( + :stderr, + "The database for #{inspect(@repo)} couldn't be created: #{inspect(term)}" + ) + end + end +end diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index add6a0bbf..f88dffa7b 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -28,16 +28,6 @@ def search(query, opts \\ []) do results end - defp maybe_resolve(true, %User{}, query) do - User.get_or_fetch(query) - end - - defp maybe_resolve(true, _, query) do - unless restrict_local?(), do: User.get_or_fetch(query) - end - - defp maybe_resolve(_, _, _), do: :noop - defp search_query(query, for_user) do query |> union_query() @@ -49,10 +39,6 @@ defp search_query(query, for_user) do |> maybe_restrict_local(for_user) end - defp restrict_local? do - Pleroma.Config.get([:instance, :limit_unauthenticated_to_local_content], true) - end - defp union_query(query) do fts_subquery = fts_search_subquery(query) trigram_subquery = trigram_search_subquery(query) @@ -64,17 +50,30 @@ defp distinct_query(q) do from(s in subquery(q), order_by: s.search_type, distinct: s.id) end - # unauthenticated users can only search local activities - defp maybe_restrict_local(q, %User{}), do: q - - defp maybe_restrict_local(q, _) do - if restrict_local?() do - where(q, [u], u.local == true) - else - q + defp maybe_resolve(true, user, query) do + case {limit(), user} do + {:all, _} -> :noop + {:unauthenticated, %User{}} -> User.get_or_fetch(query) + {:unauthenticated, _} -> :noop + {false, _} -> User.get_or_fetch(query) end end + defp maybe_resolve(_, _, _), do: :noop + + defp maybe_restrict_local(q, user) do + case {limit(), user} do + {:all, _} -> restrict_local(q) + {:unauthenticated, %User{}} -> q + {:unauthenticated, _} -> restrict_local(q) + {false, _} -> q + end + end + + defp limit, do: Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) + + defp restrict_local(q), do: where(q, [u], u.local == true) + defp boost_search_rank_query(query, nil), do: query defp boost_search_rank_query(query, for_user) do diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index ff031a16e..3bb8b40b5 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -339,7 +339,7 @@ def fix_content_map(object), do: object def fix_type(%{"inReplyTo" => reply_id} = object) when is_binary(reply_id) do reply = Object.normalize(reply_id) - if reply.data["type"] == "Question" and object["name"] do + if reply && (reply.data["type"] == "Question" and object["name"]) do Map.put(object, "type", "Answer") else object diff --git a/lib/pleroma/web/federator/retry_queue.ex b/lib/pleroma/web/federator/retry_queue.ex index 71e49494f..3db948c2e 100644 --- a/lib/pleroma/web/federator/retry_queue.ex +++ b/lib/pleroma/web/federator/retry_queue.ex @@ -15,7 +15,9 @@ def init(args) do def start_link do enabled = - if Mix.env() == :test, do: true, else: Pleroma.Config.get([__MODULE__, :enabled], false) + if Pleroma.Config.get(:env) == :test, + do: true, + else: Pleroma.Config.get([__MODULE__, :enabled], false) if enabled do Logger.info("Starting retry queue") @@ -219,7 +221,7 @@ def handle_info(unknown, state) do {:noreply, state} end - if Mix.env() == :test do + if Pleroma.Config.get(:env) == :test do defp growth_function(_retries) do _shutit = Pleroma.Config.get([__MODULE__, :initial_timeout]) DateTime.to_unix(DateTime.utc_now()) - 1 diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 92cd77f62..46049dd24 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -46,14 +46,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do require Logger - plug( - Pleroma.Plugs.RateLimitPlug, - %{ - max_requests: Config.get([:app_account_creation, :max_requests]), - interval: Config.get([:app_account_creation, :interval]) - } - when action in [:account_register] - ) + plug(Pleroma.Plugs.RateLimiter, :app_account_creation when action == :account_register) + plug(Pleroma.Plugs.RateLimiter, :search when action in [:search, :search2, :account_search]) @local_mastodon_name "Mastodon-Local" diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex index 26eb614a6..d376e2069 100644 --- a/lib/pleroma/web/rel_me.ex +++ b/lib/pleroma/web/rel_me.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.RelMe do with_body: true ] - if Mix.env() == :test do + if Pleroma.Config.get(:env) == :test do def parse(url) when is_binary(url), do: parse_url(url) else def parse(url) when is_binary(url) do diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 9bc8f2559..f377125d7 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do alias Pleroma.Web.RichMedia.Parser defp validate_page_url(page_url) when is_binary(page_url) do - if AutoLinker.Parser.is_url?(page_url, true) do + if AutoLinker.Parser.url?(page_url, true) do URI.parse(page_url) |> validate_page_url else :error diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index e4595800c..21cd47890 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -18,7 +18,7 @@ defmodule Pleroma.Web.RichMedia.Parser do def parse(nil), do: {:error, "No URL provided"} - if Mix.env() == :test do + if Pleroma.Config.get(:env) == :test do def parse(url), do: parse_url(url) else def parse(url) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e699f6ae2..1b37d6a93 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -699,7 +699,7 @@ defmodule Pleroma.Web.Router do get("/:sig/:url/:filename", MediaProxyController, :remote) end - if Mix.env() == :dev do + if Pleroma.Config.get(:env) == :dev do scope "/dev" do pipe_through([:mailbox_preview]) diff --git a/lib/pleroma/web/views/error_view.ex b/lib/pleroma/web/views/error_view.ex index f4c04131c..5cb8669fe 100644 --- a/lib/pleroma/web/views/error_view.ex +++ b/lib/pleroma/web/views/error_view.ex @@ -13,7 +13,7 @@ def render("404.json", _assigns) do def render("500.json", assigns) do Logger.error("Internal server error: #{inspect(assigns[:reason])}") - if Mix.env() != :prod do + if Pleroma.Config.get(:env) != :prod do %{errors: %{detail: "Internal server error", reason: inspect(assigns[:reason])}} else %{errors: %{detail: "Internal server error"}} diff --git a/mix.exs b/mix.exs index 9447a2e4f..db7a30f99 100644 --- a/mix.exs +++ b/mix.exs @@ -32,10 +32,22 @@ def project do ], main: "readme", output: "priv/static/doc" + ], + releases: [ + pleroma: [ + include_executables_for: [:unix], + applications: [ex_syslogger: :load, syslog: :load], + steps: [:assemble, ©_pleroma_ctl/1] + ] ] ] end + def copy_pleroma_ctl(%{path: target_path} = release) do + File.cp!("./rel/pleroma_ctl", Path.join([target_path, "bin", "pleroma_ctl"])) + release + end + # Configuration for the OTP application. # # Type `mix help compile.app` for more information. @@ -114,7 +126,7 @@ defp deps do {:ueberauth, "~> 0.4"}, {:auto_linker, git: "https://git.pleroma.social/pleroma/auto_linker.git", - ref: "c00c4e75b35367fa42c95ffd9b8c455bf9995829"}, + ref: "e2385402bcd24fc659fee83b3eb8863b0528ad42"}, {:http_signatures, git: "https://git.pleroma.social/pleroma/http_signatures.git", ref: "9789401987096ead65646b52b5a2ca6bf52fc531"}, @@ -129,7 +141,7 @@ defp deps do {:quack, "~> 0.1.1"}, {:benchee, "~> 1.0"}, {:esshd, "~> 0.1.0", runtime: Application.get_env(:esshd, :enabled, false)}, - {:ex_rated, "~> 1.2"}, + {:ex_rated, "~> 1.3"}, {:plug_static_index_html, "~> 1.0.0"}, {:excoveralls, "~> 0.11.1", only: :test} ] ++ oauth_deps() diff --git a/mix.lock b/mix.lock index 857bfca79..90cdf41f7 100644 --- a/mix.lock +++ b/mix.lock @@ -1,6 +1,6 @@ %{ "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm"}, - "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "c00c4e75b35367fa42c95ffd9b8c455bf9995829", [ref: "c00c4e75b35367fa42c95ffd9b8c455bf9995829"]}, + "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "e2385402bcd24fc659fee83b3eb8863b0528ad42", [ref: "e2385402bcd24fc659fee83b3eb8863b0528ad42"]}, "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"}, "bbcode": {:hex, :bbcode, "0.1.0", "400e618b640b635261611d7fb7f79d104917fc5b084aae371ab6b08477cb035b", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"}, "benchee": {:hex, :benchee, "1.0.1", "66b211f9bfd84bd97e6d1beaddf8fc2312aaabe192f776e8931cb0c16f53a521", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}], "hexpm"}, @@ -29,7 +29,7 @@ "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.1", "9e09366e77f25d3d88c5393824e613344631be8db0d1839faca49686e99b6704", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm"}, "ex_doc": {:hex, :ex_doc, "0.20.2", "1bd0dfb0304bade58beb77f20f21ee3558cc3c753743ae0ddbb0fd7ba2912331", [:mix], [{:earmark, "~> 1.3", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"}, "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"}, - "ex_rated": {:hex, :ex_rated, "1.3.2", "6aeb32abb46ea6076f417a9ce8cb1cf08abf35fb2d42375beaad4dd72b550bf1", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, + "ex_rated": {:hex, :ex_rated, "1.3.3", "30ecbdabe91f7eaa9d37fa4e81c85ba420f371babeb9d1910adbcd79ec798d27", [:mix], [{:ex2ms, "~> 1.5", [hex: :ex2ms, repo: "hexpm", optional: false]}], "hexpm"}, "ex_syslogger": {:git, "https://github.com/slashmili/ex_syslogger.git", "f3963399047af17e038897c69e20d552e6899e1d", [tag: "1.4.0"]}, "excoveralls": {:hex, :excoveralls, "0.11.1", "dd677fbdd49114fdbdbf445540ec735808250d56b011077798316505064edb2c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, "floki": {:hex, :floki, "0.20.4", "be42ac911fece24b4c72f3b5846774b6e61b83fe685c2fc9d62093277fb3bc86", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}, {:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, @@ -60,7 +60,7 @@ "phoenix_html": {:hex, :phoenix_html, "2.13.1", "fa8f034b5328e2dfa0e4131b5569379003f34bc1fafdaa84985b0b9d2f12e68b", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "phoenix_pubsub": {:hex, :phoenix_pubsub, "1.1.1", "6668d787e602981f24f17a5fbb69cc98f8ab085114ebfac6cc36e10a90c8e93c", [:mix], [], "hexpm"}, "pleroma_job_queue": {:hex, :pleroma_job_queue, "0.2.0", "879e660aa1cebe8dc6f0aaaa6aa48b4875e89cd961d4a585fd128e0773b31a18", [:mix], [], "hexpm"}, - "plug": {:hex, :plug, "1.7.2", "d7b7db7fbd755e8283b6c0a50be71ec0a3d67d9213d74422d9372effc8e87fd1", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}], "hexpm"}, + "plug": {:hex, :plug, "1.8.2", "0bcce1daa420f189a6491f3940cc77ea7fb1919761175c9c3b59800d897440fc", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"}, "plug_cowboy": {:hex, :plug_cowboy, "2.0.1", "d798f8ee5acc86b7d42dbe4450b8b0dadf665ce588236eb0a751a132417a980e", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"}, "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, diff --git a/priv/repo/migrations/20170719152213_add_follower_address_to_user.exs b/priv/repo/migrations/20170719152213_add_follower_address_to_user.exs index 4d163ce0b..591164be5 100644 --- a/priv/repo/migrations/20170719152213_add_follower_address_to_user.exs +++ b/priv/repo/migrations/20170719152213_add_follower_address_to_user.exs @@ -1,30 +1,10 @@ defmodule Pleroma.Repo.Migrations.AddFollowerAddressToUser do use Ecto.Migration - import Ecto.Query - import Supervisor.Spec - alias Pleroma.{Repo, User} def up do alter table(:users) do add :follower_address, :string, unique: true end - - # Not needed anymore for new setups. - # flush() - - # children = [ - # # Start the endpoint when the application starts - # supervisor(Pleroma.Web.Endpoint, []) - # ] - # opts = [strategy: :one_for_one, name: Pleroma.Supervisor] - # Supervisor.start_link(children, opts) - - # Enum.each(Repo.all(User), fn (user) -> - # if !user.follower_address do - # cs = Ecto.Changeset.change(user, %{follower_address: User.ap_followers(user)}) - # Repo.update!(cs) - # end - # end) end def down do diff --git a/rel/env.sh.eex b/rel/env.sh.eex new file mode 100644 index 000000000..a4ce25295 --- /dev/null +++ b/rel/env.sh.eex @@ -0,0 +1,12 @@ +#!/bin/sh + +# Sets and enables heart (recommended only in daemon mode) +# if [ "$RELEASE_COMMAND" = "daemon" ] || [ "$RELEASE_COMMAND" = "daemon_iex" ]; then +# HEART_COMMAND="$RELEASE_ROOT/bin/$RELEASE_NAME $RELEASE_COMMAND" +# export HEART_COMMAND +# export ELIXIR_ERL_OPTIONS="-heart" +# fi + +# Set the release to work across nodes +export RELEASE_DISTRIBUTION=name +export RELEASE_NODE=<%= @release.name %>@127.0.0.1 diff --git a/rel/pleroma_ctl b/rel/pleroma_ctl new file mode 100755 index 000000000..ef2717c44 --- /dev/null +++ b/rel/pleroma_ctl @@ -0,0 +1,19 @@ +#!/bin/sh +# XXX: This should be removed when elixir's releases get custom command support +if [ -z "$1" ] || [ "$1" == "help" ]; then + echo "Usage: $(basename "$0") COMMAND [ARGS] + + The known commands are: + + create Create database schema (needs to be executed only once) + migrate Execute database migrations (needs to be done after updates) + rollback [VERSION] Rollback database migrations (needs to be done before downgrading) + + and any mix tasks under Pleroma namespace, for example \`mix pleroma.user COMMAND\` is + equivalent to \`$(basename "$0") user COMMAND\` +" +else + SCRIPT=$(readlink -f "$0") + SCRIPTPATH=$(dirname "$SCRIPT") + $SCRIPTPATH/pleroma eval 'Pleroma.ReleaseTasks.run("'"$*"'")' +fi diff --git a/rel/vm.args.eex b/rel/vm.args.eex new file mode 100644 index 000000000..71e803264 --- /dev/null +++ b/rel/vm.args.eex @@ -0,0 +1,11 @@ +## Customize flags given to the VM: http://erlang.org/doc/man/erl.html +## -mode/-name/-sname/-setcookie are configured via env vars, do not set them here + +## Number of dirty schedulers doing IO work (file, sockets, etc) +##+SDio 5 + +## Increase number of concurrent ports/sockets +##+Q 65536 + +## Tweak GC to run more often +##-env ERL_FULLSWEEP_AFTER 10 diff --git a/test/activity_test.exs b/test/activity_test.exs index e56e39096..7ba4363c8 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -139,18 +139,25 @@ test "find only local statuses for unauthenticated users", %{local_activity: loc assert [^local_activity] = Activity.search(nil, "find me") end - test "find all statuses for unauthenticated users when `limit_unauthenticated_to_local_content` is `false`", + test "find only local statuses for unauthenticated users when `limit_to_local_content` is `:all`", + %{local_activity: local_activity} do + Pleroma.Config.put([:instance, :limit_to_local_content], :all) + assert [^local_activity] = Activity.search(nil, "find me") + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) + end + + test "find all statuses for unauthenticated users when `limit_to_local_content` is `false`", %{ local_activity: local_activity, remote_activity: remote_activity } do - Pleroma.Config.put([:instance, :limit_unauthenticated_to_local_content], false) + Pleroma.Config.put([:instance, :limit_to_local_content], false) activities = Enum.sort_by(Activity.search(nil, "find me"), & &1.id) assert [^local_activity, ^remote_activity] = activities - Pleroma.Config.put([:instance, :limit_unauthenticated_to_local_content], true) + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) end end end diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index 0e60bfca5..2a03e5d67 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -72,8 +72,11 @@ test "gets all the participations for a user, ordered by updated at descending" object2 = Pleroma.Object.normalize(activity_two) object3 = Pleroma.Object.normalize(activity_three) + user = Repo.get(Pleroma.User, user.id) + assert participation_one.conversation.ap_id == object3.data["context"] assert participation_two.conversation.ap_id == object2.data["context"] + assert participation_one.conversation.users == [user] # Pagination assert [participation_one] = Participation.for_user(user, %{"limit" => 1}) diff --git a/test/media_proxy_test.exs b/test/media_proxy_test.exs index 0a02039a6..b23aeb88b 100644 --- a/test/media_proxy_test.exs +++ b/test/media_proxy_test.exs @@ -149,6 +149,21 @@ test "encoding S3 links (must preserve `%2F`)" do encoded = url(url) assert decode_result(encoded) == url end + + test "does not change whitelisted urls" do + upload_config = Pleroma.Config.get([Pleroma.Upload]) + media_url = "https://media.pleroma.social" + Pleroma.Config.put([Pleroma.Upload, :base_url], media_url) + Pleroma.Config.put([:media_proxy, :whitelist], ["media.pleroma.social"]) + Pleroma.Config.put([:media_proxy, :base_url], "https://cache.pleroma.social") + + url = "#{media_url}/static/logo.png" + encoded = url(url) + + assert String.starts_with?(encoded, media_url) + + Pleroma.Config.put([Pleroma.Upload], upload_config) + end end describe "when disabled" do diff --git a/test/plugs/rate_limit_plug_test.exs b/test/plugs/rate_limit_plug_test.exs deleted file mode 100644 index 2ec9a8fb7..000000000 --- a/test/plugs/rate_limit_plug_test.exs +++ /dev/null @@ -1,50 +0,0 @@ -defmodule Pleroma.Plugs.RateLimitPlugTest do - use ExUnit.Case, async: true - use Plug.Test - - alias Pleroma.Plugs.RateLimitPlug - - @opts RateLimitPlug.init(%{max_requests: 5, interval: 1}) - - setup do - enabled = Pleroma.Config.get([:app_account_creation, :enabled]) - - Pleroma.Config.put([:app_account_creation, :enabled], true) - - on_exit(fn -> - Pleroma.Config.put([:app_account_creation, :enabled], enabled) - end) - - :ok - end - - test "it restricts by opts" do - conn = conn(:get, "/") - bucket_name = conn.remote_ip |> Tuple.to_list() |> Enum.join(".") - ms = 1000 - - conn = RateLimitPlug.call(conn, @opts) - {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - conn = RateLimitPlug.call(conn, @opts) - {2, 3, _, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - conn = RateLimitPlug.call(conn, @opts) - {3, 2, _, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - conn = RateLimitPlug.call(conn, @opts) - {4, 1, _, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - conn = RateLimitPlug.call(conn, @opts) - {5, 0, to_reset, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - conn = RateLimitPlug.call(conn, @opts) - assert conn.status == 403 - assert conn.halted - assert conn.resp_body == "{\"error\":\"Rate limit exceeded.\"}" - - Process.sleep(to_reset) - - conn = conn(:get, "/") - conn = RateLimitPlug.call(conn, @opts) - {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, ms, 5) - refute conn.status == 403 - refute conn.halted - refute conn.resp_body - end -end diff --git a/test/plugs/rate_limiter_test.exs b/test/plugs/rate_limiter_test.exs new file mode 100644 index 000000000..b3798bf03 --- /dev/null +++ b/test/plugs/rate_limiter_test.exs @@ -0,0 +1,108 @@ +defmodule Pleroma.Plugs.RateLimiterTest do + use ExUnit.Case, async: true + use Plug.Test + + alias Pleroma.Plugs.RateLimiter + + import Pleroma.Factory + + @limiter_name :testing + + test "init/1" do + Pleroma.Config.put([:rate_limit, @limiter_name], {1, 1}) + + assert {@limiter_name, {1, 1}} == RateLimiter.init(@limiter_name) + assert nil == RateLimiter.init(:foo) + end + + test "ip/1" do + assert "127.0.0.1" == RateLimiter.ip(%{remote_ip: {127, 0, 0, 1}}) + end + + test "it restricts by opts" do + scale = 100 + limit = 5 + + Pleroma.Config.put([:rate_limit, @limiter_name], {scale, limit}) + + opts = RateLimiter.init(@limiter_name) + conn = conn(:get, "/") + bucket_name = "#{@limiter_name}:#{RateLimiter.ip(conn)}" + + conn = RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {2, 3, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {3, 2, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {4, 1, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {5, 0, to_reset, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + + assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) + assert conn.halted + + Process.sleep(to_reset) + + conn = conn(:get, "/") + + conn = RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + refute conn.status == Plug.Conn.Status.code(:too_many_requests) + refute conn.resp_body + refute conn.halted + end + + test "optional limits for authenticated users" do + Ecto.Adapters.SQL.Sandbox.checkout(Pleroma.Repo) + + scale = 100 + limit = 5 + Pleroma.Config.put([:rate_limit, @limiter_name], [{1, 10}, {scale, limit}]) + + opts = RateLimiter.init(@limiter_name) + + user = insert(:user) + conn = conn(:get, "/") |> assign(:user, user) + bucket_name = "#{@limiter_name}:#{user.id}" + + conn = RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {2, 3, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {3, 2, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {4, 1, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + assert {5, 0, to_reset, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + conn = RateLimiter.call(conn, opts) + + assert %{"error" => "Throttled"} = Phoenix.ConnTest.json_response(conn, :too_many_requests) + assert conn.halted + + Process.sleep(to_reset) + + conn = conn(:get, "/") |> assign(:user, user) + + conn = RateLimiter.call(conn, opts) + assert {1, 4, _, _, _} = ExRated.inspect_bucket(bucket_name, scale, limit) + + refute conn.status == Plug.Conn.Status.code(:too_many_requests) + refute conn.resp_body + refute conn.halted + end +end diff --git a/test/user_test.exs b/test/user_test.exs index 8dd672173..473f545ff 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1099,8 +1099,20 @@ test "find only local users for unauthenticated users" do assert [%{id: ^id}] = User.search("lain") end - test "find all users for unauthenticated users when `limit_unauthenticated_to_local_content` is `false`" do - Pleroma.Config.put([:instance, :limit_unauthenticated_to_local_content], false) + test "find only local users for authenticated users when `limit_to_local_content` is `:all`" do + Pleroma.Config.put([:instance, :limit_to_local_content], :all) + + %{id: id} = insert(:user, %{name: "lain"}) + insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) + insert(:user, %{nickname: "lain@pleroma.soykaf.com", local: false}) + + assert [%{id: ^id}] = User.search("lain") + + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) + end + + test "find all users for unauthenticated users when `limit_to_local_content` is `false`" do + Pleroma.Config.put([:instance, :limit_to_local_content], false) u1 = insert(:user, %{name: "lain"}) u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false}) @@ -1114,7 +1126,7 @@ test "find all users for unauthenticated users when `limit_unauthenticated_to_lo assert [u1.id, u2.id, u3.id] == results - Pleroma.Config.put([:instance, :limit_unauthenticated_to_local_content], true) + Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) end test "finds a user whose name is nil" do diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 28971ae45..cc1781403 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -60,6 +60,22 @@ test "it fetches replied-to activities if we don't have them" do assert returned_object.data["inReplyToAtomUri"] == "https://shitposter.club/notice/2827873" end + test "it does not crash if the object in inReplyTo can't be fetched" do + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + + object = + data["object"] + |> Map.put("inReplyTo", "https://404.site/whatever") + + data = + data + |> Map.put("object", object) + + {:ok, _returned_activity} = Transmogrifier.handle_incoming(data) + end + test "it works for incoming notices" do data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 33c8e209a..15d3fdb65 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1421,6 +1421,82 @@ test "returns the relationships for the current user", %{conn: conn} do end end + describe "media upload" do + setup do + upload_config = Pleroma.Config.get([Pleroma.Upload]) + proxy_config = Pleroma.Config.get([:media_proxy]) + + on_exit(fn -> + Pleroma.Config.put([Pleroma.Upload], upload_config) + Pleroma.Config.put([:media_proxy], proxy_config) + end) + + user = insert(:user) + + conn = + build_conn() + |> assign(:user, user) + + image = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + [conn: conn, image: image] + end + + test "returns uploaded image", %{conn: conn, image: image} do + desc = "Description of the image" + + media = + conn + |> post("/api/v1/media", %{"file" => image, "description" => desc}) + |> json_response(:ok) + + assert media["type"] == "image" + assert media["description"] == desc + assert media["id"] + + object = Repo.get(Object, media["id"]) + assert object.data["actor"] == User.ap_id(conn.assigns[:user]) + end + + test "returns proxied url when media proxy is enabled", %{conn: conn, image: image} do + Pleroma.Config.put([Pleroma.Upload, :base_url], "https://media.pleroma.social") + + proxy_url = "https://cache.pleroma.social" + Pleroma.Config.put([:media_proxy, :enabled], true) + Pleroma.Config.put([:media_proxy, :base_url], proxy_url) + + media = + conn + |> post("/api/v1/media", %{"file" => image}) + |> json_response(:ok) + + assert String.starts_with?(media["url"], proxy_url) + end + + test "returns media url when proxy is enabled but media url is whitelisted", %{ + conn: conn, + image: image + } do + media_url = "https://media.pleroma.social" + Pleroma.Config.put([Pleroma.Upload, :base_url], media_url) + + Pleroma.Config.put([:media_proxy, :enabled], true) + Pleroma.Config.put([:media_proxy, :base_url], "https://cache.pleroma.social") + Pleroma.Config.put([:media_proxy, :whitelist], ["media.pleroma.social"]) + + media = + conn + |> post("/api/v1/media", %{"file" => image}) + |> json_response(:ok) + + assert String.starts_with?(media["url"], media_url) + end + end + describe "locked accounts" do test "/api/v1/follow_requests works" do user = insert(:user, %{info: %User.Info{locked: true}}) @@ -1530,32 +1606,6 @@ test "account fetching also works nickname", %{conn: conn} do assert id == user.id end - test "media upload", %{conn: conn} do - file = %Plug.Upload{ - content_type: "image/jpg", - path: Path.absname("test/fixtures/image.jpg"), - filename: "an_image.jpg" - } - - desc = "Description of the image" - - user = insert(:user) - - conn = - conn - |> assign(:user, user) - |> post("/api/v1/media", %{"file" => file, "description" => desc}) - - assert media = json_response(conn, 200) - - assert media["type"] == "image" - assert media["description"] == desc - assert media["id"] - - object = Repo.get(Object, media["id"]) - assert object.data["actor"] == User.ap_id(user) - end - test "mascot upload", %{conn: conn} do user = insert(:user) @@ -3501,24 +3551,6 @@ test "Repeated posts that are replies incorrectly have in_reply_to_id null", %{c end describe "create account by app" do - setup do - enabled = Pleroma.Config.get([:app_account_creation, :enabled]) - max_requests = Pleroma.Config.get([:app_account_creation, :max_requests]) - interval = Pleroma.Config.get([:app_account_creation, :interval]) - - Pleroma.Config.put([:app_account_creation, :enabled], true) - Pleroma.Config.put([:app_account_creation, :max_requests], 5) - Pleroma.Config.put([:app_account_creation, :interval], 1) - - on_exit(fn -> - Pleroma.Config.put([:app_account_creation, :enabled], enabled) - Pleroma.Config.put([:app_account_creation, :max_requests], max_requests) - Pleroma.Config.put([:app_account_creation, :interval], interval) - end) - - :ok - end - test "Account registration via Application", %{conn: conn} do conn = conn @@ -3621,7 +3653,7 @@ test "rate limit", %{conn: conn} do agreement: true }) - assert json_response(conn, 403) == %{"error" => "Rate limit exceeded."} + assert json_response(conn, :too_many_requests) == %{"error" => "Throttled"} end end