From 4a6855d9eedf07159520b2205c554c891e70c7d4 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 1 Jan 2017 03:10:08 +0300 Subject: [PATCH 01/59] Provide plaintext representations of content/cw in MastoAPI --- docs/api/differences_in_mastoapi_responses.md | 2 ++ .../web/mastodon_api/views/status_view.ex | 31 ++++++++++++++++--- test/web/mastodon_api/status_view_test.exs | 6 ++-- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/docs/api/differences_in_mastoapi_responses.md b/docs/api/differences_in_mastoapi_responses.md index 215f43155..923d94db2 100644 --- a/docs/api/differences_in_mastoapi_responses.md +++ b/docs/api/differences_in_mastoapi_responses.md @@ -20,6 +20,8 @@ Has these additional fields under the `pleroma` object: - `local`: true if the post was made on the local instance. - `conversation_id`: the ID of the conversation the status is associated with (if any) +- `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` +- `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain` ## Attachments diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 4c0b53bdd..d4a8e4fff 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -147,20 +147,39 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} content = object |> render_content() + + content_html = + content |> HTML.get_cached_scrubbed_html_for_activity( User.html_filter_policy(opts[:for]), activity, "mastoapi:content" ) - summary = - (object["summary"] || "") + content_plaintext = + content + |> HTML.get_cached_stripped_html_for_activity( + activity, + "mastoapi:content" + ) + + summary = object["summary"] || "" + + summary_html = + summary |> HTML.get_cached_scrubbed_html_for_activity( User.html_filter_policy(opts[:for]), activity, "mastoapi:summary" ) + summary_plaintext = + summary + |> HTML.get_cached_stripped_html_for_activity( + activity, + "mastoapi:summary" + ) + card = render("card.json", Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)) url = @@ -179,7 +198,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), reblog: nil, card: card, - content: content, + content: content_html, created_at: created_at, reblogs_count: announcement_count, replies_count: object["repliesCount"] || 0, @@ -190,7 +209,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} muted: CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user), pinned: pinned?(activity, user), sensitive: sensitive, - spoiler_text: summary, + spoiler_text: summary_html, visibility: get_visibility(object), media_attachments: attachments, mentions: mentions, @@ -203,7 +222,9 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} emojis: build_emojis(activity.data["object"]["emoji"]), pleroma: %{ local: activity.local, - conversation_id: get_context_id(activity) + conversation_id: get_context_id(activity), + content: %{"text/plain" => content_plaintext}, + spoiler_text: %{"text/plain" => summary_plaintext} } } end diff --git a/test/web/mastodon_api/status_view_test.exs b/test/web/mastodon_api/status_view_test.exs index 8db92ac16..db2fdc2f6 100644 --- a/test/web/mastodon_api/status_view_test.exs +++ b/test/web/mastodon_api/status_view_test.exs @@ -101,7 +101,7 @@ test "a note activity" do muted: false, pinned: false, sensitive: false, - spoiler_text: note.data["object"]["summary"], + spoiler_text: HtmlSanitizeEx.basic_html(note.data["object"]["summary"]), visibility: "public", media_attachments: [], mentions: [], @@ -126,7 +126,9 @@ test "a note activity" do ], pleroma: %{ local: true, - conversation_id: convo_id + conversation_id: convo_id, + content: %{"text/plain" => HtmlSanitizeEx.strip_tags(note.data["object"]["content"])}, + spoiler_text: %{"text/plain" => HtmlSanitizeEx.strip_tags(note.data["object"]["summary"])} } } From 63ab61ed3f4988bfaf9080bcdc4fc8d5046fa57e Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 11 Mar 2019 20:37:26 +0300 Subject: [PATCH 02/59] Sign in via Twitter (WIP). --- config/config.exs | 11 +++++++++++ config/dev.exs | 1 - lib/pleroma/web/endpoint.ex | 10 ++++++---- lib/pleroma/web/oauth/oauth_controller.ex | 11 +++++++++++ lib/pleroma/web/oauth/oauth_view.ex | 1 + lib/pleroma/web/router.ex | 12 ++++++++++++ .../web/templates/o_auth/o_auth/show.html.eex | 7 +++++++ mix.exs | 9 +++++++-- mix.lock | 11 ++++++++--- 9 files changed, 63 insertions(+), 10 deletions(-) diff --git a/config/config.exs b/config/config.exs index cd4c8e562..8c754cef3 100644 --- a/config/config.exs +++ b/config/config.exs @@ -369,6 +369,17 @@ rel: false ] +config :ueberauth, + Ueberauth, + base_path: "/oauth", + providers: [ + twitter: {Ueberauth.Strategy.Twitter, []} + ] + +config :ueberauth, Ueberauth.Strategy.Twitter.OAuth, + consumer_key: System.get_env("TWITTER_CONSUMER_KEY"), + consumer_secret: System.get_env("TWITTER_CONSUMER_SECRET") + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/dev.exs b/config/dev.exs index f77bb9976..a7eb4b644 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -12,7 +12,6 @@ protocol_options: [max_request_line_length: 8192, max_header_value_length: 8192] ], protocol: "http", - secure_cookie_flag: false, debug_errors: true, code_reloader: true, check_origin: false, diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 3eed047ca..d906db67d 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -50,23 +50,25 @@ defmodule Pleroma.Web.Endpoint do plug(Plug.MethodOverride) plug(Plug.Head) + secure_cookies = Pleroma.Config.get([__MODULE__, :secure_cookie_flag]) + cookie_name = - if Application.get_env(:pleroma, Pleroma.Web.Endpoint) |> Keyword.get(:secure_cookie_flag), + if secure_cookies, do: "__Host-pleroma_key", else: "pleroma_key" # The session will be stored in the cookie and signed, # this means its contents can be read but not tampered with. # Set :encryption_salt if you would also like to encrypt it. + # Note: "SameSite=Strict" would cause issues with Twitter OAuth plug( Plug.Session, store: :cookie, key: cookie_name, signing_salt: {Pleroma.Config, :get, [[__MODULE__, :signing_salt], "CqaoopA2"]}, http_only: true, - secure: - Application.get_env(:pleroma, Pleroma.Web.Endpoint) |> Keyword.get(:secure_cookie_flag), - extra: "SameSite=Strict" + secure: secure_cookies, + extra: "SameSite=Lax" ) plug(Pleroma.Web.Router) diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index 36318d69b..7b052cb36 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -15,11 +15,22 @@ defmodule Pleroma.Web.OAuth.OAuthController do import Pleroma.Web.ControllerHelper, only: [oauth_scopes: 2] + plug(Ueberauth) plug(:fetch_session) plug(:fetch_flash) action_fallback(Pleroma.Web.OAuth.FallbackController) + def callback(%{assigns: %{ueberauth_failure: _failure}} = conn, _params) do + conn + |> put_flash(:error, "Failed to authenticate.") + |> redirect(to: "/") + end + + def callback(%{assigns: %{ueberauth_auth: _auth}} = _conn, _params) do + raise "Authenticated successfully. Sign up via OAuth is not yet implemented." + end + def authorize(conn, params) do app = Repo.get_by(App, client_id: params["client_id"]) available_scopes = (app && app.scopes) || [] diff --git a/lib/pleroma/web/oauth/oauth_view.ex b/lib/pleroma/web/oauth/oauth_view.ex index 9b37a91c5..1450b5a8d 100644 --- a/lib/pleroma/web/oauth/oauth_view.ex +++ b/lib/pleroma/web/oauth/oauth_view.ex @@ -5,4 +5,5 @@ defmodule Pleroma.Web.OAuth.OAuthView do use Pleroma.Web, :view import Phoenix.HTML.Form + import Phoenix.HTML.Link end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 65a90e31e..7cf7794b3 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -5,6 +5,11 @@ defmodule Pleroma.Web.Router do use Pleroma.Web, :router + pipeline :browser do + plug(:accepts, ["html"]) + plug(:fetch_session) + end + pipeline :api do plug(:accepts, ["json"]) plug(:fetch_session) @@ -197,6 +202,13 @@ defmodule Pleroma.Web.Router do post("/authorize", OAuthController, :create_authorization) post("/token", OAuthController, :token_exchange) post("/revoke", OAuthController, :token_revoke) + + scope [] do + pipe_through(:browser) + + get("/:provider", OAuthController, :request) + get("/:provider/callback", OAuthController, :callback) + end end scope "/api/v1", Pleroma.Web.MastodonAPI do diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index 161333847..d465f06b1 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -4,7 +4,9 @@ <%= if get_flash(@conn, :error) do %> <% end %> +

OAuth Authorization

+ <%= form_for @conn, o_auth_path(@conn, :authorize), [as: "authorization"], fn f -> %>
<%= label f, :name, "Name or email" %> @@ -33,3 +35,8 @@ <%= hidden_input f, :state, value: @state%> <%= submit "Authorize" %> <% end %> + +
+<%= link to: "/oauth/twitter", class: "alert alert-info" do %> + Sign in with Twitter +<% end %> \ No newline at end of file diff --git a/mix.exs b/mix.exs index 70b5e4bd6..dcd273d72 100644 --- a/mix.exs +++ b/mix.exs @@ -41,7 +41,7 @@ def project do def application do [ mod: {Pleroma.Application, []}, - extra_applications: [:logger, :runtime_tools, :comeonin], + extra_applications: [:logger, :runtime_tools, :comeonin, :ueberauth_twitter], included_applications: [:ex_syslogger] ] end @@ -69,7 +69,8 @@ defp deps do {:phoenix_html, "~> 2.10"}, {:calendar, "~> 0.17.4"}, {:cachex, "~> 3.0.2"}, - {:httpoison, "~> 1.2.0"}, + {:httpoison, "~> 1.2.0", override: true}, + {:poison, "~> 3.0", override: true}, {:tesla, "~> 1.2"}, {:jason, "~> 1.0"}, {:mogrify, "~> 0.6.1"}, @@ -90,6 +91,10 @@ defp deps do {:floki, "~> 0.20.0"}, {:ex_syslogger, github: "slashmili/ex_syslogger", tag: "1.4.0"}, {:timex, "~> 3.5"}, + {:oauth, github: "tim/erlang-oauth"}, + # {:oauth2, "~> 0.8", override: true}, + {:ueberauth, "~> 0.4"}, + {:ueberauth_twitter, "~> 0.2"}, {:auto_linker, git: "https://git.pleroma.social/pleroma/auto_linker.git", ref: "94193ca5f97c1f9fdf3d1469653e2d46fac34bcd"} diff --git a/mix.lock b/mix.lock index f43a18564..92660b70a 100644 --- a/mix.lock +++ b/mix.lock @@ -4,7 +4,7 @@ "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"}, "cachex": {:hex, :cachex, "3.0.2", "1351caa4e26e29f7d7ec1d29b53d6013f0447630bbf382b4fb5d5bad0209f203", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm"}, "calendar": {:hex, :calendar, "0.17.4", "22c5e8d98a4db9494396e5727108dffb820ee0d18fed4b0aa8ab76e4f5bc32f1", [:mix], [{:tzdata, "~> 0.5.8 or ~> 0.1.201603", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, - "certifi": {:hex, :certifi, "2.4.2", "75424ff0f3baaccfd34b1214184b6ef616d89e420b258bb0a5ea7d7bc628f7f0", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"}, + "certifi": {:hex, :certifi, "2.5.1", "867ce347f7c7d78563450a18a6a28a8090331e77fa02380b4a21962a65d36ee5", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm"}, "comeonin": {:hex, :comeonin, "4.1.1", "c7304fc29b45b897b34142a91122bc72757bc0c295e9e824999d5179ffc08416", [:mix], [{:argon2_elixir, "~> 1.2", [hex: :argon2_elixir, repo: "hexpm", optional: true]}, {:bcrypt_elixir, "~> 0.12.1 or ~> 1.0", [hex: :bcrypt_elixir, repo: "hexpm", optional: true]}, {:pbkdf2_elixir, "~> 0.12", [hex: :pbkdf2_elixir, repo: "hexpm", optional: true]}], "hexpm"}, "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"}, @@ -26,7 +26,7 @@ "floki": {:hex, :floki, "0.20.4", "be42ac911fece24b4c72f3b5846774b6e61b83fe685c2fc9d62093277fb3bc86", [:mix], [{:html_entities, "~> 0.4.0", [hex: :html_entities, repo: "hexpm", optional: false]}, {:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "gen_smtp": {:hex, :gen_smtp, "0.13.0", "11f08504c4bdd831dc520b8f84a1dce5ce624474a797394e7aafd3c29f5dcd25", [:rebar3], [], "hexpm"}, "gettext": {:hex, :gettext, "0.15.0", "40a2b8ce33a80ced7727e36768499fc9286881c43ebafccae6bab731e2b2b8ce", [:mix], [], "hexpm"}, - "hackney": {:hex, :hackney, "1.14.3", "b5f6f5dcc4f1fba340762738759209e21914516df6be440d85772542d4a5e412", [:rebar3], [{:certifi, "2.4.2", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "1.0.2", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, + "hackney": {:hex, :hackney, "1.15.1", "9f8f471c844b8ce395f7b6d8398139e26ddca9ebc171a8b91342ee15a19963f4", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.4", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, "html_entities": {:hex, :html_entities, "0.4.0", "f2fee876858cf6aaa9db608820a3209e45a087c5177332799592142b50e89a6b", [:mix], [], "hexpm"}, "html_sanitize_ex": {:hex, :html_sanitize_ex, "1.3.0", "f005ad692b717691203f940c686208aa3d8ffd9dd4bb3699240096a51fa9564e", [:mix], [{:mochiweb, "~> 2.15", [hex: :mochiweb, repo: "hexpm", optional: false]}], "hexpm"}, "httpoison": {:hex, :httpoison, "1.2.0", "2702ed3da5fd7a8130fc34b11965c8cfa21ade2f232c00b42d96d4967c39a3a3", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, @@ -38,11 +38,14 @@ "meck": {:hex, :meck, "0.8.13", "ffedb39f99b0b99703b8601c6f17c7f76313ee12de6b646e671e3188401f7866", [:rebar3], [], "hexpm"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"}, "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"}, - "mimerl": {:hex, :mimerl, "1.0.2", "993f9b0e084083405ed8252b99460c4f0563e41729ab42d9074fd5e52439be88", [:rebar3], [], "hexpm"}, + "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm"}, "mochiweb": {:hex, :mochiweb, "2.15.0", "e1daac474df07651e5d17cc1e642c4069c7850dc4508d3db7263a0651330aacc", [:rebar3], [], "hexpm"}, "mock": {:hex, :mock, "0.3.1", "994f00150f79a0ea50dc9d86134cd9ebd0d177ad60bd04d1e46336cdfdb98ff9", [:mix], [{:meck, "~> 0.8.8", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "nimble_parsec": {:hex, :nimble_parsec, "0.4.0", "ee261bb53214943679422be70f1658fff573c5d0b0a1ecd0f18738944f818efe", [:mix], [], "hexpm"}, + "oauth": {:git, "https://github.com/tim/erlang-oauth.git", "bd19896e31125f99ff45bb5850b1c0e74b996743", []}, + "oauth2": {:hex, :oauth2, "0.9.4", "632e8e8826a45e33ac2ea5ac66dcc019ba6bb5a0d2ba77e342d33e3b7b252c6e", [:mix], [{:hackney, "~> 1.7", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, + "oauther": {:hex, :oauther, "1.1.1", "7d8b16167bb587ecbcddd3f8792beb9ec3e7b65c1f8ebd86b8dd25318d535752", [:mix], [], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.3", "6706a148809a29c306062862c803406e88f048277f6e85b68faf73291e820b84", [:mix], [], "hexpm"}, "phoenix": {:hex, :phoenix, "1.4.1", "801f9d632808657f1f7c657c8bbe624caaf2ba91429123ebe3801598aea4c3d9", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, @@ -63,6 +66,8 @@ "timex": {:hex, :timex, "3.5.0", "b0a23167da02d0fe4f1a4e104d1f929a00d348502b52432c05de875d0b9cffa5", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.17", "50793e3d85af49736701da1a040c415c97dc1caf6464112fd9bd18f425d3053b", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, + "ueberauth": {:hex, :ueberauth, "0.5.0", "4570ec94d7f784dc4c4aa94c83391dbd9b9bd7b66baa30e95a666c5ec1b168b1", [:mix], [{:plug, "~> 1.2", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, + "ueberauth_twitter": {:hex, :ueberauth_twitter, "0.2.4", "770ac273cc696cde986582e7a36df0923deb39fa3deff0152fbf150343809f81", [:mix], [{:httpoison, "~> 0.7", [hex: :httpoison, repo: "hexpm", optional: false]}, {:oauther, "~> 1.1", [hex: :oauther, repo: "hexpm", optional: false]}, {:poison, "~> 1.3 or ~> 2.0", [hex: :poison, repo: "hexpm", optional: false]}, {:ueberauth, "~> 0.2", [hex: :ueberauth, repo: "hexpm", optional: false]}], "hexpm"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"}, "unsafe": {:hex, :unsafe, "1.0.0", "7c21742cd05380c7875546b023481d3a26f52df8e5dfedcb9f958f322baae305", [:mix], [], "hexpm"}, "web_push_encryption": {:hex, :web_push_encryption, "0.2.1", "d42cecf73420d9dc0053ba3299cc8c8d6ff2be2487d67ca2a57265868e4d9a98", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, From aacbf0f57053786533df045125dee93ace0daa93 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 15 Mar 2019 17:08:03 +0300 Subject: [PATCH 03/59] [#923] OAuth: prototype of sign in / sign up with Twitter. --- config/config.exs | 6 +- lib/pleroma/user.ex | 46 +++++++++- lib/pleroma/web/auth/authenticator.ex | 9 +- lib/pleroma/web/auth/pleroma_authenticator.ex | 56 ++++++++++++- lib/pleroma/web/endpoint.ex | 11 ++- lib/pleroma/web/oauth/oauth_controller.ex | 83 ++++++++++++++----- lib/pleroma/web/oauth/oauth_view.ex | 1 - .../templates/o_auth/o_auth/consumer.html.eex | 14 ++++ .../web/templates/o_auth/o_auth/show.html.eex | 8 +- ...rovider_and_auth_provider_uid_to_users.exs | 12 +++ 10 files changed, 209 insertions(+), 37 deletions(-) create mode 100644 lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex create mode 100644 priv/repo/migrations/20190315101315_add_auth_provider_and_auth_provider_uid_to_users.exs diff --git a/config/config.exs b/config/config.exs index 8c754cef3..1ddc1bad1 100644 --- a/config/config.exs +++ b/config/config.exs @@ -369,11 +369,15 @@ rel: false ] +config :pleroma, :auth, oauth_consumer_enabled: false + config :ueberauth, Ueberauth, base_path: "/oauth", providers: [ - twitter: {Ueberauth.Strategy.Twitter, []} + twitter: + {Ueberauth.Strategy.Twitter, + [callback_params: ~w[client_id redirect_uri scope scopes]]} ] config :ueberauth, Ueberauth.Strategy.Twitter.OAuth, diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f49ede149..e17df8e34 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -40,6 +40,8 @@ defmodule Pleroma.User do field(:email, :string) field(:name, :string) field(:nickname, :string) + field(:auth_provider, :string) + field(:auth_provider_uid, :string) field(:password_hash, :string) field(:password, :string, virtual: true) field(:password_confirmation, :string, virtual: true) @@ -206,6 +208,36 @@ def reset_password(user, data) do update_and_set_cache(password_update_changeset(user, data)) end + # TODO: FIXME (WIP): + def oauth_register_changeset(struct, params \\ %{}) do + info_change = User.Info.confirmation_changeset(%User.Info{}, :confirmed) + + changeset = + struct + |> cast(params, [:email, :nickname, :name, :bio, :auth_provider, :auth_provider_uid]) + |> validate_required([:auth_provider, :auth_provider_uid]) + |> unique_constraint(:email) + |> unique_constraint(:nickname) + |> validate_exclusion(:nickname, Pleroma.Config.get([Pleroma.User, :restricted_nicknames])) + |> validate_format(:email, @email_regex) + |> validate_length(:bio, max: 1000) + |> put_change(:info, info_change) + + if changeset.valid? do + nickname = changeset.changes[:nickname] + ap_id = (nickname && User.ap_id(%User{nickname: nickname})) || nil + followers = User.ap_followers(%User{nickname: ap_id}) + + changeset + |> put_change(:ap_id, ap_id) + |> unique_constraint(:ap_id) + |> put_change(:following, [followers]) + |> put_change(:follower_address, followers) + else + changeset + end + end + def register_changeset(struct, params \\ %{}, opts \\ []) do confirmation_status = if opts[:confirmed] || !Pleroma.Config.get([:instance, :account_activation_required]) do @@ -504,13 +536,19 @@ def get_by_nickname(nickname) do end end + def get_by_email(email), do: Repo.get_by(User, email: email) + def get_by_nickname_or_email(nickname_or_email) do - case user = Repo.get_by(User, nickname: nickname_or_email) do - %User{} -> user - nil -> Repo.get_by(User, email: nickname_or_email) - end + get_by_nickname(nickname_or_email) || get_by_email(nickname_or_email) end + def get_by_auth_provider_uid(auth_provider, auth_provider_uid), + do: + Repo.get_by(User, + auth_provider: to_string(auth_provider), + auth_provider_uid: to_string(auth_provider_uid) + ) + def get_cached_user_info(user) do key = "user_info:#{user.id}" Cachex.fetch!(:user_cache, key, fn _ -> user_info(user) end) diff --git a/lib/pleroma/web/auth/authenticator.ex b/lib/pleroma/web/auth/authenticator.ex index 82267c595..fa439d562 100644 --- a/lib/pleroma/web/auth/authenticator.ex +++ b/lib/pleroma/web/auth/authenticator.ex @@ -12,8 +12,13 @@ def implementation do ) end - @callback get_user(Plug.Conn.t()) :: {:ok, User.t()} | {:error, any()} - def get_user(plug), do: implementation().get_user(plug) + @callback get_user(Plug.Conn.t(), Map.t()) :: {:ok, User.t()} | {:error, any()} + def get_user(plug, params), do: implementation().get_user(plug, params) + + @callback get_or_create_user_by_oauth(Plug.Conn.t(), Map.t()) :: + {:ok, User.t()} | {:error, any()} + def get_or_create_user_by_oauth(plug, params), + do: implementation().get_or_create_user_by_oauth(plug, params) @callback handle_error(Plug.Conn.t(), any()) :: any() def handle_error(plug, error), do: implementation().handle_error(plug, error) diff --git a/lib/pleroma/web/auth/pleroma_authenticator.ex b/lib/pleroma/web/auth/pleroma_authenticator.ex index 3cc19af01..fb04ef8da 100644 --- a/lib/pleroma/web/auth/pleroma_authenticator.ex +++ b/lib/pleroma/web/auth/pleroma_authenticator.ex @@ -8,9 +8,9 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticator do @behaviour Pleroma.Web.Auth.Authenticator - def get_user(%Plug.Conn{} = conn) do - %{"authorization" => %{"name" => name, "password" => password}} = conn.params - + def get_user(%Plug.Conn{} = _conn, %{ + "authorization" => %{"name" => name, "password" => password} + }) do with {_, %User{} = user} <- {:user, User.get_by_nickname_or_email(name)}, {_, true} <- {:checkpw, Pbkdf2.checkpw(password, user.password_hash)} do {:ok, user} @@ -20,6 +20,56 @@ def get_user(%Plug.Conn{} = conn) do end end + def get_user(%Plug.Conn{} = _conn, _params), do: {:error, :missing_credentials} + + def get_or_create_user_by_oauth( + %Plug.Conn{assigns: %{ueberauth_auth: %{provider: provider, uid: uid} = auth}}, + _params + ) do + user = User.get_by_auth_provider_uid(provider, uid) + + if user do + {:ok, user} + else + info = auth.info + email = info.email + nickname = info.nickname + + # TODO: FIXME: connect to existing (non-oauth) account (need a UI flow for that) / generate a random nickname? + email = + if email && User.get_by_email(email) do + nil + else + email + end + + nickname = + if nickname && User.get_by_nickname(nickname) do + nil + else + nickname + end + + new_user = + User.oauth_register_changeset( + %User{}, + %{ + auth_provider: to_string(provider), + auth_provider_uid: to_string(uid), + name: info.name, + bio: info.description, + email: email, + nickname: nickname + } + ) + + Pleroma.Repo.insert(new_user) + end + end + + def get_or_create_user_by_oauth(%Plug.Conn{} = _conn, _params), + do: {:error, :missing_credentials} + def handle_error(%Plug.Conn{} = _conn, error) do error end diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index d906db67d..31ffdecc0 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -57,10 +57,17 @@ defmodule Pleroma.Web.Endpoint do do: "__Host-pleroma_key", else: "pleroma_key" + same_site = + if Pleroma.Config.get([:auth, :oauth_consumer_enabled]) do + # Note: "SameSite=Strict" prevents sign in with external OAuth provider (no cookies during callback request) + "SameSite=Lax" + else + "SameSite=Strict" + end + # The session will be stored in the cookie and signed, # this means its contents can be read but not tampered with. # Set :encryption_salt if you would also like to encrypt it. - # Note: "SameSite=Strict" would cause issues with Twitter OAuth plug( Plug.Session, store: :cookie, @@ -68,7 +75,7 @@ defmodule Pleroma.Web.Endpoint do signing_salt: {Pleroma.Config, :get, [[__MODULE__, :signing_salt], "CqaoopA2"]}, http_only: true, secure: secure_cookies, - extra: "SameSite=Lax" + extra: same_site ) plug(Pleroma.Web.Router) diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index 7b052cb36..366085a57 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -15,20 +15,57 @@ defmodule Pleroma.Web.OAuth.OAuthController do import Pleroma.Web.ControllerHelper, only: [oauth_scopes: 2] - plug(Ueberauth) + if Pleroma.Config.get([:auth, :oauth_consumer_enabled]), do: plug(Ueberauth) + plug(:fetch_session) plug(:fetch_flash) action_fallback(Pleroma.Web.OAuth.FallbackController) - def callback(%{assigns: %{ueberauth_failure: _failure}} = conn, _params) do + def request(conn, params) do + message = + if params["provider"] do + "Unsupported OAuth provider: #{params["provider"]}." + else + "Bad OAuth request." + end + conn - |> put_flash(:error, "Failed to authenticate.") + |> put_flash(:error, message) |> redirect(to: "/") end - def callback(%{assigns: %{ueberauth_auth: _auth}} = _conn, _params) do - raise "Authenticated successfully. Sign up via OAuth is not yet implemented." + def callback(%{assigns: %{ueberauth_failure: failure}} = conn, %{"redirect_uri" => redirect_uri}) do + messages = for e <- Map.get(failure, :errors, []), do: e.message + message = Enum.join(messages, "; ") + + conn + |> put_flash(:error, "Failed to authenticate: #{message}.") + |> redirect(external: redirect_uri(conn, redirect_uri)) + end + + def callback( + conn, + %{"client_id" => client_id, "redirect_uri" => redirect_uri} = params + ) do + with {:ok, user} <- Authenticator.get_or_create_user_by_oauth(conn, params) do + do_create_authorization( + conn, + %{ + "authorization" => %{ + "client_id" => client_id, + "redirect_uri" => redirect_uri, + "scope" => oauth_scopes(params, nil) + } + }, + user + ) + else + _ -> + conn + |> put_flash(:error, "Failed to set up user account.") + |> redirect(external: redirect_uri(conn, redirect_uri)) + end end def authorize(conn, params) do @@ -47,14 +84,21 @@ def authorize(conn, params) do }) end - def create_authorization(conn, %{ - "authorization" => - %{ - "client_id" => client_id, - "redirect_uri" => redirect_uri - } = auth_params - }) do - with {_, {:ok, %User{} = user}} <- {:get_user, Authenticator.get_user(conn)}, + def create_authorization(conn, params), do: do_create_authorization(conn, params, nil) + + defp do_create_authorization( + conn, + %{ + "authorization" => + %{ + "client_id" => client_id, + "redirect_uri" => redirect_uri + } = auth_params + } = params, + user + ) do + with {_, {:ok, %User{} = user}} <- + {:get_user, (user && {:ok, user}) || Authenticator.get_user(conn, params)}, %App{} = app <- Repo.get_by(App, client_id: client_id), true <- redirect_uri in String.split(app.redirect_uris), scopes <- oauth_scopes(auth_params, []), @@ -63,13 +107,7 @@ def create_authorization(conn, %{ {:missing_scopes, false} <- {:missing_scopes, scopes == []}, {:auth_active, true} <- {:auth_active, User.auth_active?(user)}, {:ok, auth} <- Authorization.create_authorization(app, user, scopes) do - redirect_uri = - if redirect_uri == "." do - # Special case: Local MastodonFE - mastodon_api_url(conn, :login) - else - redirect_uri - end + redirect_uri = redirect_uri(conn, redirect_uri) cond do redirect_uri == "urn:ietf:wg:oauth:2.0:oob" -> @@ -225,4 +263,9 @@ defp get_app_from_request(conn, params) do nil end end + + # Special case: Local MastodonFE + defp redirect_uri(conn, "."), do: mastodon_api_url(conn, :login) + + defp redirect_uri(_conn, redirect_uri), do: redirect_uri end diff --git a/lib/pleroma/web/oauth/oauth_view.ex b/lib/pleroma/web/oauth/oauth_view.ex index 1450b5a8d..9b37a91c5 100644 --- a/lib/pleroma/web/oauth/oauth_view.ex +++ b/lib/pleroma/web/oauth/oauth_view.ex @@ -5,5 +5,4 @@ defmodule Pleroma.Web.OAuth.OAuthView do use Pleroma.Web, :view import Phoenix.HTML.Form - import Phoenix.HTML.Link end diff --git a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex new file mode 100644 index 000000000..e7251bce8 --- /dev/null +++ b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex @@ -0,0 +1,14 @@ +

External OAuth Authorization

+<%= form_for @conn, o_auth_path(@conn, :request, :twitter), [method: "get"], fn f -> %> +
+ <%= label f, :scope, "Permissions" %> +
+ <%= text_input f, :scope, value: Enum.join(@available_scopes, " ") %> +
+
+ + <%= hidden_input f, :client_id, value: @client_id %> + <%= hidden_input f, :redirect_uri, value: @redirect_uri %> + <%= hidden_input f, :state, value: @state%> + <%= submit "Sign in with Twitter" %> +<% end %> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index d465f06b1..2fa7837fc 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -36,7 +36,7 @@ <%= submit "Authorize" %> <% end %> -
-<%= link to: "/oauth/twitter", class: "alert alert-info" do %> - Sign in with Twitter -<% end %> \ No newline at end of file +<%= if Pleroma.Config.get([:auth, :oauth_consumer_enabled]) do %> +
+ <%= render @view_module, "consumer.html", assigns %> +<% end %> diff --git a/priv/repo/migrations/20190315101315_add_auth_provider_and_auth_provider_uid_to_users.exs b/priv/repo/migrations/20190315101315_add_auth_provider_and_auth_provider_uid_to_users.exs new file mode 100644 index 000000000..90947f85a --- /dev/null +++ b/priv/repo/migrations/20190315101315_add_auth_provider_and_auth_provider_uid_to_users.exs @@ -0,0 +1,12 @@ +defmodule Pleroma.Repo.Migrations.AddAuthProviderAndAuthProviderUidToUsers do + use Ecto.Migration + + def change do + alter table(:users) do + add :auth_provider, :string + add :auth_provider_uid, :string + end + + create unique_index(:users, [:auth_provider, :auth_provider_uid]) + end +end From 26b63540953f6a65bb52531b434fd6ab85aaedfe Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 18 Mar 2019 17:23:38 +0300 Subject: [PATCH 04/59] [#923] Support for multiple (external) registrations per user via Registration. --- config/config.exs | 2 +- lib/pleroma/registration.ex | 36 +++++++++++++ lib/pleroma/user.ex | 16 ++---- lib/pleroma/web/auth/authenticator.ex | 6 +-- lib/pleroma/web/auth/ldap_authenticator.ex | 2 +- lib/pleroma/web/auth/pleroma_authenticator.ex | 51 +++++++++++-------- lib/pleroma/web/oauth/oauth_controller.ex | 2 +- ...rovider_and_auth_provider_uid_to_users.exs | 12 ----- .../20190315101315_create_registrations.exs | 16 ++++++ 9 files changed, 93 insertions(+), 50 deletions(-) create mode 100644 lib/pleroma/registration.ex delete mode 100644 priv/repo/migrations/20190315101315_add_auth_provider_and_auth_provider_uid_to_users.exs create mode 100644 priv/repo/migrations/20190315101315_create_registrations.exs diff --git a/config/config.exs b/config/config.exs index 6839b489b..03baf894d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -381,7 +381,7 @@ base: System.get_env("LDAP_BASE") || "dc=example,dc=com", uid: System.get_env("LDAP_UID") || "cn" -config :pleroma, :auth, oauth_consumer_enabled: false +config :pleroma, :auth, oauth_consumer_enabled: System.get_env("OAUTH_CONSUMER_ENABLED") == "true" config :ueberauth, Ueberauth, diff --git a/lib/pleroma/registration.ex b/lib/pleroma/registration.ex new file mode 100644 index 000000000..1bd91a316 --- /dev/null +++ b/lib/pleroma/registration.ex @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Registration do + use Ecto.Schema + + import Ecto.Changeset + + alias Pleroma.Registration + alias Pleroma.Repo + alias Pleroma.User + + schema "registrations" do + belongs_to(:user, User, type: Pleroma.FlakeId) + field(:provider, :string) + field(:uid, :string) + field(:info, :map, default: %{}) + + timestamps() + end + + def changeset(registration, params \\ %{}) do + registration + |> cast(params, [:user_id, :provider, :uid, :info]) + |> foreign_key_constraint(:user_id) + |> unique_constraint(:uid, name: :registrations_provider_uid_index) + end + + def get_by_provider_uid(provider, uid) do + Repo.get_by(Registration, + provider: to_string(provider), + uid: to_string(uid) + ) + end +end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 7f8b282e0..bd742b2fd 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -13,6 +13,7 @@ defmodule Pleroma.User do alias Pleroma.Formatter alias Pleroma.Notification alias Pleroma.Object + alias Pleroma.Registration alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web @@ -41,8 +42,6 @@ defmodule Pleroma.User do field(:email, :string) field(:name, :string) field(:nickname, :string) - field(:auth_provider, :string) - field(:auth_provider_uid, :string) field(:password_hash, :string) field(:password, :string, virtual: true) field(:password_confirmation, :string, virtual: true) @@ -56,6 +55,7 @@ defmodule Pleroma.User do field(:bookmarks, {:array, :string}, default: []) field(:last_refreshed_at, :naive_datetime) has_many(:notifications, Notification) + has_many(:registrations, Registration) embeds_one(:info, Pleroma.User.Info) timestamps() @@ -210,13 +210,12 @@ def reset_password(user, data) do end # TODO: FIXME (WIP): - def oauth_register_changeset(struct, params \\ %{}) do + def external_registration_changeset(struct, params \\ %{}) do info_change = User.Info.confirmation_changeset(%User.Info{}, :confirmed) changeset = struct - |> cast(params, [:email, :nickname, :name, :bio, :auth_provider, :auth_provider_uid]) - |> validate_required([:auth_provider, :auth_provider_uid]) + |> cast(params, [:email, :nickname, :name, :bio]) |> unique_constraint(:email) |> unique_constraint(:nickname) |> validate_exclusion(:nickname, Pleroma.Config.get([Pleroma.User, :restricted_nicknames])) @@ -544,13 +543,6 @@ def get_by_nickname_or_email(nickname_or_email) do get_by_nickname(nickname_or_email) || get_by_email(nickname_or_email) end - def get_by_auth_provider_uid(auth_provider, auth_provider_uid), - do: - Repo.get_by(User, - auth_provider: to_string(auth_provider), - auth_provider_uid: to_string(auth_provider_uid) - ) - def get_cached_user_info(user) do key = "user_info:#{user.id}" Cachex.fetch!(:user_cache, key, fn _ -> user_info(user) end) diff --git a/lib/pleroma/web/auth/authenticator.ex b/lib/pleroma/web/auth/authenticator.ex index fa439d562..11f45eec3 100644 --- a/lib/pleroma/web/auth/authenticator.ex +++ b/lib/pleroma/web/auth/authenticator.ex @@ -15,10 +15,10 @@ def implementation do @callback get_user(Plug.Conn.t(), Map.t()) :: {:ok, User.t()} | {:error, any()} def get_user(plug, params), do: implementation().get_user(plug, params) - @callback get_or_create_user_by_oauth(Plug.Conn.t(), Map.t()) :: + @callback get_by_external_registration(Plug.Conn.t(), Map.t()) :: {:ok, User.t()} | {:error, any()} - def get_or_create_user_by_oauth(plug, params), - do: implementation().get_or_create_user_by_oauth(plug, params) + def get_by_external_registration(plug, params), + do: implementation().get_by_external_registration(plug, params) @callback handle_error(Plug.Conn.t(), any()) :: any() def handle_error(plug, error), do: implementation().handle_error(plug, error) diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index 6c65cff27..51a0f0fa2 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -40,7 +40,7 @@ def get_user(%Plug.Conn{} = conn, params) do end end - def get_or_create_user_by_oauth(conn, params), do: get_user(conn, params) + def get_by_external_registration(conn, params), do: get_user(conn, params) def handle_error(%Plug.Conn{} = _conn, error) do error diff --git a/lib/pleroma/web/auth/pleroma_authenticator.ex b/lib/pleroma/web/auth/pleroma_authenticator.ex index 2e2bcfb70..2d4399490 100644 --- a/lib/pleroma/web/auth/pleroma_authenticator.ex +++ b/lib/pleroma/web/auth/pleroma_authenticator.ex @@ -5,6 +5,8 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticator do alias Comeonin.Pbkdf2 alias Pleroma.User + alias Pleroma.Registration + alias Pleroma.Repo @behaviour Pleroma.Web.Auth.Authenticator @@ -27,20 +29,21 @@ def get_user(%Plug.Conn{} = _conn, params) do end end - def get_or_create_user_by_oauth( + def get_by_external_registration( %Plug.Conn{assigns: %{ueberauth_auth: %{provider: provider, uid: uid} = auth}}, _params ) do - user = User.get_by_auth_provider_uid(provider, uid) + registration = Registration.get_by_provider_uid(provider, uid) - if user do + if registration do + user = Repo.preload(registration, :user).user {:ok, user} else info = auth.info email = info.email nickname = info.nickname - # TODO: FIXME: connect to existing (non-oauth) account (need a UI flow for that) / generate a random nickname? + # Note: nullifying email in case this email is already taken email = if email && User.get_by_email(email) do nil @@ -48,31 +51,39 @@ def get_or_create_user_by_oauth( email end + # Note: generating a random numeric suffix to nickname in case this nickname is already taken nickname = if nickname && User.get_by_nickname(nickname) do - nil + "#{nickname}_#{:os.system_time()}" else nickname end - new_user = - User.oauth_register_changeset( - %User{}, - %{ - auth_provider: to_string(provider), - auth_provider_uid: to_string(uid), - name: info.name, - bio: info.description, - email: email, - nickname: nickname - } - ) - - Pleroma.Repo.insert(new_user) + with {:ok, new_user} <- + User.external_registration_changeset( + %User{}, + %{ + name: info.name, + bio: info.description, + email: email, + nickname: nickname + } + ) + |> Repo.insert(), + {:ok, _} <- + Registration.changeset(%Registration{}, %{ + user_id: new_user.id, + provider: to_string(provider), + uid: to_string(uid), + info: %{nickname: info.nickname, email: info.email} + }) + |> Repo.insert() do + {:ok, new_user} + end end end - def get_or_create_user_by_oauth(%Plug.Conn{} = _conn, _params), + def get_by_external_registration(%Plug.Conn{} = _conn, _params), do: {:error, :missing_credentials} def handle_error(%Plug.Conn{} = _conn, error) do diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index 588933d31..8c864cb1d 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -47,7 +47,7 @@ def callback( conn, %{"client_id" => client_id, "redirect_uri" => redirect_uri} = params ) do - with {:ok, user} <- Authenticator.get_or_create_user_by_oauth(conn, params) do + with {:ok, user} <- Authenticator.get_by_external_registration(conn, params) do do_create_authorization( conn, %{ diff --git a/priv/repo/migrations/20190315101315_add_auth_provider_and_auth_provider_uid_to_users.exs b/priv/repo/migrations/20190315101315_add_auth_provider_and_auth_provider_uid_to_users.exs deleted file mode 100644 index 90947f85a..000000000 --- a/priv/repo/migrations/20190315101315_add_auth_provider_and_auth_provider_uid_to_users.exs +++ /dev/null @@ -1,12 +0,0 @@ -defmodule Pleroma.Repo.Migrations.AddAuthProviderAndAuthProviderUidToUsers do - use Ecto.Migration - - def change do - alter table(:users) do - add :auth_provider, :string - add :auth_provider_uid, :string - end - - create unique_index(:users, [:auth_provider, :auth_provider_uid]) - end -end diff --git a/priv/repo/migrations/20190315101315_create_registrations.exs b/priv/repo/migrations/20190315101315_create_registrations.exs new file mode 100644 index 000000000..dac86b780 --- /dev/null +++ b/priv/repo/migrations/20190315101315_create_registrations.exs @@ -0,0 +1,16 @@ +defmodule Pleroma.Repo.Migrations.CreateRegistrations do + use Ecto.Migration + + def change do + create table(:registrations) do + add :user_id, references(:users, type: :uuid, on_delete: :delete_all) + add :provider, :string + add :uid, :string + add :info, :map, default: %{} + + timestamps() + end + + create unique_index(:registrations, [:provider, :uid]) + end +end From 8d21859717a75e01128f50b0b51efdd0a4748670 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 18 Mar 2019 18:09:53 +0300 Subject: [PATCH 05/59] [#923] External User registration refactoring, password randomization. --- lib/pleroma/user.ex | 38 ++++--------------- lib/pleroma/web/auth/pleroma_authenticator.ex | 14 +++++-- 2 files changed, 18 insertions(+), 34 deletions(-) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index bd742b2fd..558216894 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -209,35 +209,6 @@ def reset_password(user, data) do update_and_set_cache(password_update_changeset(user, data)) end - # TODO: FIXME (WIP): - def external_registration_changeset(struct, params \\ %{}) do - info_change = User.Info.confirmation_changeset(%User.Info{}, :confirmed) - - changeset = - struct - |> cast(params, [:email, :nickname, :name, :bio]) - |> unique_constraint(:email) - |> unique_constraint(:nickname) - |> validate_exclusion(:nickname, Pleroma.Config.get([Pleroma.User, :restricted_nicknames])) - |> validate_format(:email, @email_regex) - |> validate_length(:bio, max: 1000) - |> put_change(:info, info_change) - - if changeset.valid? do - nickname = changeset.changes[:nickname] - ap_id = (nickname && User.ap_id(%User{nickname: nickname})) || nil - followers = User.ap_followers(%User{nickname: ap_id}) - - changeset - |> put_change(:ap_id, ap_id) - |> unique_constraint(:ap_id) - |> put_change(:following, [followers]) - |> put_change(:follower_address, followers) - else - changeset - end - end - def register_changeset(struct, params \\ %{}, opts \\ []) do confirmation_status = if opts[:confirmed] || !Pleroma.Config.get([:instance, :account_activation_required]) do @@ -251,7 +222,7 @@ def register_changeset(struct, params \\ %{}, opts \\ []) do changeset = struct |> cast(params, [:bio, :email, :name, :nickname, :password, :password_confirmation]) - |> validate_required([:email, :name, :nickname, :password, :password_confirmation]) + |> validate_required([:name, :nickname, :password, :password_confirmation]) |> validate_confirmation(:password) |> unique_constraint(:email) |> unique_constraint(:nickname) @@ -262,6 +233,13 @@ def register_changeset(struct, params \\ %{}, opts \\ []) do |> validate_length(:name, min: 1, max: 100) |> put_change(:info, info_change) + changeset = + if opts[:external] do + changeset + else + validate_required(changeset, [:email]) + end + if changeset.valid? do hashed = Pbkdf2.hashpwsalt(changeset.changes[:password]) ap_id = User.ap_id(%User{nickname: changeset.changes[:nickname]}) diff --git a/lib/pleroma/web/auth/pleroma_authenticator.ex b/lib/pleroma/web/auth/pleroma_authenticator.ex index 2d4399490..36ecd0560 100644 --- a/lib/pleroma/web/auth/pleroma_authenticator.ex +++ b/lib/pleroma/web/auth/pleroma_authenticator.ex @@ -54,20 +54,26 @@ def get_by_external_registration( # Note: generating a random numeric suffix to nickname in case this nickname is already taken nickname = if nickname && User.get_by_nickname(nickname) do - "#{nickname}_#{:os.system_time()}" + "#{nickname}#{:os.system_time()}" else nickname end + random_password = :crypto.strong_rand_bytes(64) |> Base.encode64() + with {:ok, new_user} <- - User.external_registration_changeset( + User.register_changeset( %User{}, %{ name: info.name, bio: info.description, email: email, - nickname: nickname - } + nickname: nickname, + password: random_password, + password_confirmation: random_password + }, + external: true, + confirmed: true ) |> Repo.insert(), {:ok, _} <- From 40e9a04c31a9965dee92cb8f07ed6db28f8ccd75 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 18 Mar 2019 20:31:24 +0300 Subject: [PATCH 06/59] [#923] Registration validations & unique index on [:user_id, :provider]. --- lib/pleroma/registration.ex | 1 + priv/repo/migrations/20190315101315_create_registrations.exs | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/pleroma/registration.ex b/lib/pleroma/registration.ex index 1bd91a316..773e25fa6 100644 --- a/lib/pleroma/registration.ex +++ b/lib/pleroma/registration.ex @@ -23,6 +23,7 @@ defmodule Pleroma.Registration do def changeset(registration, params \\ %{}) do registration |> cast(params, [:user_id, :provider, :uid, :info]) + |> validate_required([:provider, :uid]) |> foreign_key_constraint(:user_id) |> unique_constraint(:uid, name: :registrations_provider_uid_index) end diff --git a/priv/repo/migrations/20190315101315_create_registrations.exs b/priv/repo/migrations/20190315101315_create_registrations.exs index dac86b780..c566912f5 100644 --- a/priv/repo/migrations/20190315101315_create_registrations.exs +++ b/priv/repo/migrations/20190315101315_create_registrations.exs @@ -12,5 +12,6 @@ def change do end create unique_index(:registrations, [:provider, :uid]) + create unique_index(:registrations, [:user_id, :provider]) end end From e17a9a1f6680bfc464a1433fcff37b6d61cc5340 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 20 Mar 2019 10:35:31 +0300 Subject: [PATCH 07/59] [#923] Nickname & email selection for external registrations, option to connect to existing account. --- lib/pleroma/registration.ex | 20 ++ lib/pleroma/web/auth/authenticator.ex | 12 +- lib/pleroma/web/auth/ldap_authenticator.ex | 11 +- lib/pleroma/web/auth/pleroma_authenticator.ex | 95 ++++--- lib/pleroma/web/oauth/oauth_controller.ex | 245 +++++++++++++----- lib/pleroma/web/router.ex | 2 + .../templates/o_auth/o_auth/register.html.eex | 48 ++++ .../20190315101315_create_registrations.exs | 3 +- 8 files changed, 309 insertions(+), 127 deletions(-) create mode 100644 lib/pleroma/web/templates/o_auth/o_auth/register.html.eex diff --git a/lib/pleroma/registration.ex b/lib/pleroma/registration.ex index 773e25fa6..21fd1fc3f 100644 --- a/lib/pleroma/registration.ex +++ b/lib/pleroma/registration.ex @@ -11,6 +11,8 @@ defmodule Pleroma.Registration do alias Pleroma.Repo alias Pleroma.User + @primary_key {:id, Pleroma.FlakeId, autogenerate: true} + schema "registrations" do belongs_to(:user, User, type: Pleroma.FlakeId) field(:provider, :string) @@ -20,6 +22,18 @@ defmodule Pleroma.Registration do timestamps() end + def nickname(registration, default \\ nil), + do: Map.get(registration.info, "nickname", default) + + def email(registration, default \\ nil), + do: Map.get(registration.info, "email", default) + + def name(registration, default \\ nil), + do: Map.get(registration.info, "name", default) + + def description(registration, default \\ nil), + do: Map.get(registration.info, "description", default) + def changeset(registration, params \\ %{}) do registration |> cast(params, [:user_id, :provider, :uid, :info]) @@ -28,6 +42,12 @@ def changeset(registration, params \\ %{}) do |> unique_constraint(:uid, name: :registrations_provider_uid_index) end + def bind_to_user(registration, user) do + registration + |> changeset(%{user_id: (user && user.id) || nil}) + |> Repo.update() + end + def get_by_provider_uid(provider, uid) do Repo.get_by(Registration, provider: to_string(provider), diff --git a/lib/pleroma/web/auth/authenticator.ex b/lib/pleroma/web/auth/authenticator.ex index 11f45eec3..1f614668c 100644 --- a/lib/pleroma/web/auth/authenticator.ex +++ b/lib/pleroma/web/auth/authenticator.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.Auth.Authenticator do alias Pleroma.User + alias Pleroma.Registration def implementation do Pleroma.Config.get( @@ -15,10 +16,15 @@ def implementation do @callback get_user(Plug.Conn.t(), Map.t()) :: {:ok, User.t()} | {:error, any()} def get_user(plug, params), do: implementation().get_user(plug, params) - @callback get_by_external_registration(Plug.Conn.t(), Map.t()) :: + @callback create_from_registration(Plug.Conn.t(), Map.t(), Registration.t()) :: {:ok, User.t()} | {:error, any()} - def get_by_external_registration(plug, params), - do: implementation().get_by_external_registration(plug, params) + def create_from_registration(plug, params, registration), + do: implementation().create_from_registration(plug, params, registration) + + @callback get_registration(Plug.Conn.t(), Map.t()) :: + {:ok, Registration.t()} | {:error, any()} + def get_registration(plug, params), + do: implementation().get_registration(plug, params) @callback handle_error(Plug.Conn.t(), any()) :: any() def handle_error(plug, error), do: implementation().handle_error(plug, error) diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index 51a0f0fa2..65abd7f38 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -8,10 +8,15 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do require Logger @behaviour Pleroma.Web.Auth.Authenticator + @base Pleroma.Web.Auth.PleromaAuthenticator @connection_timeout 10_000 @search_timeout 10_000 + defdelegate get_registration(conn, params), to: @base + + defdelegate create_from_registration(conn, params, registration), to: @base + def get_user(%Plug.Conn{} = conn, params) do if Pleroma.Config.get([:ldap, :enabled]) do {name, password} = @@ -29,19 +34,17 @@ def get_user(%Plug.Conn{} = conn, params) do {:error, {:ldap_connection_error, _}} -> # When LDAP is unavailable, try default authenticator - Pleroma.Web.Auth.PleromaAuthenticator.get_user(conn, params) + @base.get_user(conn, params) error -> error end else # Fall back to default authenticator - Pleroma.Web.Auth.PleromaAuthenticator.get_user(conn, params) + @base.get_user(conn, params) end end - def get_by_external_registration(conn, params), do: get_user(conn, params) - def handle_error(%Plug.Conn{} = _conn, error) do error end diff --git a/lib/pleroma/web/auth/pleroma_authenticator.ex b/lib/pleroma/web/auth/pleroma_authenticator.ex index 36ecd0560..60847ce6a 100644 --- a/lib/pleroma/web/auth/pleroma_authenticator.ex +++ b/lib/pleroma/web/auth/pleroma_authenticator.ex @@ -29,68 +29,63 @@ def get_user(%Plug.Conn{} = _conn, params) do end end - def get_by_external_registration( + def get_registration( %Plug.Conn{assigns: %{ueberauth_auth: %{provider: provider, uid: uid} = auth}}, _params ) do registration = Registration.get_by_provider_uid(provider, uid) if registration do - user = Repo.preload(registration, :user).user - {:ok, user} + {:ok, registration} else info = auth.info - email = info.email - nickname = info.nickname - # Note: nullifying email in case this email is already taken - email = - if email && User.get_by_email(email) do - nil - else - email - end - - # Note: generating a random numeric suffix to nickname in case this nickname is already taken - nickname = - if nickname && User.get_by_nickname(nickname) do - "#{nickname}#{:os.system_time()}" - else - nickname - end - - random_password = :crypto.strong_rand_bytes(64) |> Base.encode64() - - with {:ok, new_user} <- - User.register_changeset( - %User{}, - %{ - name: info.name, - bio: info.description, - email: email, - nickname: nickname, - password: random_password, - password_confirmation: random_password - }, - external: true, - confirmed: true - ) - |> Repo.insert(), - {:ok, _} <- - Registration.changeset(%Registration{}, %{ - user_id: new_user.id, - provider: to_string(provider), - uid: to_string(uid), - info: %{nickname: info.nickname, email: info.email} - }) - |> Repo.insert() do - {:ok, new_user} - end + Registration.changeset(%Registration{}, %{ + provider: to_string(provider), + uid: to_string(uid), + info: %{ + "nickname" => info.nickname, + "email" => info.email, + "name" => info.name, + "description" => info.description + } + }) + |> Repo.insert() end end - def get_by_external_registration(%Plug.Conn{} = _conn, _params), - do: {:error, :missing_credentials} + def get_registration(%Plug.Conn{} = _conn, _params), do: {:error, :missing_credentials} + + def create_from_registration(_conn, params, registration) do + nickname = value([params["nickname"], Registration.nickname(registration)]) + email = value([params["email"], Registration.email(registration)]) + name = value([params["name"], Registration.name(registration)]) || nickname + bio = value([params["bio"], Registration.description(registration)]) + + random_password = :crypto.strong_rand_bytes(64) |> Base.encode64() + + with {:ok, new_user} <- + User.register_changeset( + %User{}, + %{ + email: email, + nickname: nickname, + name: name, + bio: bio, + password: random_password, + password_confirmation: random_password + }, + external: true, + confirmed: true + ) + |> Repo.insert(), + {:ok, _} <- + Registration.changeset(registration, %{user_id: new_user.id}) |> Repo.update() do + {:ok, new_user} + end + end + + defp value(list), do: Enum.find(list, &(to_string(&1) != "")) def handle_error(%Plug.Conn{} = _conn, error) do error diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index 8c864cb1d..a2c62ae68 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Registration alias Pleroma.Web.Auth.Authenticator alias Pleroma.Web.OAuth.App alias Pleroma.Web.OAuth.Authorization @@ -21,52 +22,6 @@ defmodule Pleroma.Web.OAuth.OAuthController do action_fallback(Pleroma.Web.OAuth.FallbackController) - def request(conn, params) do - message = - if params["provider"] do - "Unsupported OAuth provider: #{params["provider"]}." - else - "Bad OAuth request." - end - - conn - |> put_flash(:error, message) - |> redirect(to: "/") - end - - def callback(%{assigns: %{ueberauth_failure: failure}} = conn, %{"redirect_uri" => redirect_uri}) do - messages = for e <- Map.get(failure, :errors, []), do: e.message - message = Enum.join(messages, "; ") - - conn - |> put_flash(:error, "Failed to authenticate: #{message}.") - |> redirect(external: redirect_uri(conn, redirect_uri)) - end - - def callback( - conn, - %{"client_id" => client_id, "redirect_uri" => redirect_uri} = params - ) do - with {:ok, user} <- Authenticator.get_by_external_registration(conn, params) do - do_create_authorization( - conn, - %{ - "authorization" => %{ - "client_id" => client_id, - "redirect_uri" => redirect_uri, - "scope" => oauth_scopes(params, nil) - } - }, - user - ) - else - _ -> - conn - |> put_flash(:error, "Failed to set up user account.") - |> redirect(external: redirect_uri(conn, redirect_uri)) - end - end - def authorize(conn, params) do app = Repo.get_by(App, client_id: params["client_id"]) available_scopes = (app && app.scopes) || [] @@ -83,29 +38,16 @@ def authorize(conn, params) do }) end - def create_authorization(conn, params), do: do_create_authorization(conn, params, nil) - - defp do_create_authorization( - conn, - %{ - "authorization" => - %{ - "client_id" => client_id, - "redirect_uri" => redirect_uri - } = auth_params - } = params, - user - ) do - with {_, {:ok, %User{} = user}} <- - {:get_user, (user && {:ok, user}) || Authenticator.get_user(conn, params)}, - %App{} = app <- Repo.get_by(App, client_id: client_id), - true <- redirect_uri in String.split(app.redirect_uris), - scopes <- oauth_scopes(auth_params, []), - {:unsupported_scopes, []} <- {:unsupported_scopes, scopes -- app.scopes}, - # Note: `scope` param is intentionally not optional in this context - {:missing_scopes, false} <- {:missing_scopes, scopes == []}, - {:auth_active, true} <- {:auth_active, User.auth_active?(user)}, - {:ok, auth} <- Authorization.create_authorization(app, user, scopes) do + def create_authorization( + conn, + %{ + "authorization" => %{"redirect_uri" => redirect_uri} = auth_params + } = params, + opts \\ [] + ) do + with {:ok, auth} <- + (opts[:auth] && {:ok, opts[:auth]}) || + do_create_authorization(conn, params, opts[:user]) do redirect_uri = redirect_uri(conn, redirect_uri) cond do @@ -232,6 +174,166 @@ def token_revoke(conn, %{"token" => token} = params) do end end + def request(conn, params) do + message = + if params["provider"] do + "Unsupported OAuth provider: #{params["provider"]}." + else + "Bad OAuth request." + end + + conn + |> put_flash(:error, message) + |> redirect(to: "/") + end + + def callback(%{assigns: %{ueberauth_failure: failure}} = conn, %{"redirect_uri" => redirect_uri}) do + messages = for e <- Map.get(failure, :errors, []), do: e.message + message = Enum.join(messages, "; ") + + conn + |> put_flash(:error, "Failed to authenticate: #{message}.") + |> redirect(external: redirect_uri(conn, redirect_uri)) + end + + def callback( + conn, + %{"client_id" => client_id, "redirect_uri" => redirect_uri} = params + ) do + with {:ok, registration} <- Authenticator.get_registration(conn, params) do + user = Repo.preload(registration, :user).user + + auth_params = %{ + "client_id" => client_id, + "redirect_uri" => redirect_uri, + "scopes" => oauth_scopes(params, nil) + } + + if user do + create_authorization( + conn, + %{"authorization" => auth_params}, + user: user + ) + else + registration_params = + Map.merge(auth_params, %{ + "nickname" => Registration.nickname(registration), + "email" => Registration.email(registration) + }) + + conn + |> put_session(:registration_id, registration.id) + |> redirect(to: o_auth_path(conn, :registration_details, registration_params)) + end + else + _ -> + conn + |> put_flash(:error, "Failed to set up user account.") + |> redirect(external: redirect_uri(conn, redirect_uri)) + end + end + + def registration_details(conn, params) do + render(conn, "register.html", %{ + client_id: params["client_id"], + redirect_uri: params["redirect_uri"], + scopes: oauth_scopes(params, []), + nickname: params["nickname"], + email: params["email"] + }) + end + + def register(conn, %{"op" => "connect"} = params) do + create_authorization_params = %{ + "authorization" => Map.merge(params, %{"name" => params["auth_name"]}) + } + + with registration_id when not is_nil(registration_id) <- get_session_registration_id(conn), + %Registration{} = registration <- Repo.get(Registration, registration_id), + {:ok, auth} <- do_create_authorization(conn, create_authorization_params), + %User{} = user <- Repo.preload(auth, :user).user, + {:ok, _updated_registration} <- Registration.bind_to_user(registration, user) do + conn + |> put_session_registration_id(nil) + |> create_authorization( + create_authorization_params, + auth: auth + ) + else + _ -> + conn + |> put_flash(:error, "Unknown error, please try again.") + |> redirect(to: o_auth_path(conn, :registration_details, params)) + end + end + + def register(conn, params) do + with registration_id when not is_nil(registration_id) <- get_session_registration_id(conn), + %Registration{} = registration <- Repo.get(Registration, registration_id), + {:ok, user} <- Authenticator.create_from_registration(conn, params, registration) do + conn + |> put_session_registration_id(nil) + |> create_authorization( + %{ + "authorization" => %{ + "client_id" => params["client_id"], + "redirect_uri" => params["redirect_uri"], + "scopes" => oauth_scopes(params, nil) + } + }, + user: user + ) + else + {:error, changeset} -> + message = + Enum.map(changeset.errors, fn {field, {error, _}} -> + "#{field} #{error}" + end) + |> Enum.join("; ") + + message = + String.replace( + message, + "ap_id has already been taken", + "nickname has already been taken" + ) + + conn + |> put_flash(:error, "Error: #{message}.") + |> redirect(to: o_auth_path(conn, :registration_details, params)) + + _ -> + conn + |> put_flash(:error, "Unknown error, please try again.") + |> redirect(to: o_auth_path(conn, :registration_details, params)) + end + end + + defp do_create_authorization( + conn, + %{ + "authorization" => + %{ + "client_id" => client_id, + "redirect_uri" => redirect_uri + } = auth_params + } = params, + user \\ nil + ) do + with {_, {:ok, %User{} = user}} <- + {:get_user, (user && {:ok, user}) || Authenticator.get_user(conn, params)}, + %App{} = app <- Repo.get_by(App, client_id: client_id), + true <- redirect_uri in String.split(app.redirect_uris), + scopes <- oauth_scopes(auth_params, []), + {:unsupported_scopes, []} <- {:unsupported_scopes, scopes -- app.scopes}, + # Note: `scope` param is intentionally not optional in this context + {:missing_scopes, false} <- {:missing_scopes, scopes == []}, + {:auth_active, true} <- {:auth_active, User.auth_active?(user)} do + Authorization.create_authorization(app, user, scopes) + end + end + # XXX - for whatever reason our token arrives urlencoded, but Plug.Conn should be # decoding it. Investigate sometime. defp fix_padding(token) do @@ -269,4 +371,9 @@ defp get_app_from_request(conn, params) do defp redirect_uri(conn, "."), do: mastodon_api_url(conn, :login) defp redirect_uri(_conn, redirect_uri), do: redirect_uri + + defp get_session_registration_id(conn), do: get_session(conn, :registration_id) + + defp put_session_registration_id(conn, registration_id), + do: put_session(conn, :registration_id, registration_id) end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 9b6784120..f2cec574b 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -208,12 +208,14 @@ defmodule Pleroma.Web.Router do post("/authorize", OAuthController, :create_authorization) post("/token", OAuthController, :token_exchange) post("/revoke", OAuthController, :token_revoke) + get("/registration_details", OAuthController, :registration_details) scope [] do pipe_through(:browser) get("/:provider", OAuthController, :request) get("/:provider/callback", OAuthController, :callback) + post("/register", OAuthController, :register) end end diff --git a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex new file mode 100644 index 000000000..f4547170c --- /dev/null +++ b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex @@ -0,0 +1,48 @@ +<%= if get_flash(@conn, :info) do %> + +<% end %> +<%= if get_flash(@conn, :error) do %> + +<% end %> + +

Registration Details

+ +

If you'd like to register a new account, +
+please provide the details below.

+
+ +<%= form_for @conn, o_auth_path(@conn, :register), [], fn f -> %> + +
+ <%= label f, :nickname, "Nickname" %> + <%= text_input f, :nickname, value: @nickname %> +
+
+ <%= label f, :email, "Email" %> + <%= text_input f, :email, value: @email %> +
+ +<%= submit "Proceed as new user", name: "op", value: "register" %> + +
+
+
+

Alternatively, sign in to connect to existing account.

+ +
+ <%= label f, :auth_name, "Name or email" %> + <%= text_input f, :auth_name %> +
+
+ <%= label f, :password, "Password" %> + <%= password_input f, :password %> +
+ +<%= submit "Proceed as existing user", name: "op", value: "connect" %> + +<%= hidden_input f, :client_id, value: @client_id %> +<%= hidden_input f, :redirect_uri, value: @redirect_uri %> +<%= hidden_input f, :scope, value: Enum.join(@scopes, " ") %> + +<% end %> diff --git a/priv/repo/migrations/20190315101315_create_registrations.exs b/priv/repo/migrations/20190315101315_create_registrations.exs index c566912f5..fbb22ec7c 100644 --- a/priv/repo/migrations/20190315101315_create_registrations.exs +++ b/priv/repo/migrations/20190315101315_create_registrations.exs @@ -2,7 +2,8 @@ defmodule Pleroma.Repo.Migrations.CreateRegistrations do use Ecto.Migration def change do - create table(:registrations) do + create table(:registrations, primary_key: false) do + add :id, :uuid, primary_key: true add :user_id, references(:users, type: :uuid, on_delete: :delete_all) add :provider, :string add :uid, :string From af68a42ef7841013476831e92d3841088fa875df Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 20 Mar 2019 20:25:48 +0300 Subject: [PATCH 08/59] [#923] Support for multiple OAuth consumer strategies. --- config/config.exs | 24 +++++++++------ lib/pleroma/web/oauth/oauth_controller.ex | 29 +++++++++++++------ .../templates/o_auth/o_auth/consumer.html.eex | 20 +++++-------- .../web/templates/o_auth/o_auth/show.html.eex | 1 - mix.exs | 13 +++++---- mix.lock | 1 + 6 files changed, 52 insertions(+), 36 deletions(-) diff --git a/config/config.exs b/config/config.exs index 03baf894d..7d8de5af6 100644 --- a/config/config.exs +++ b/config/config.exs @@ -381,20 +381,26 @@ base: System.get_env("LDAP_BASE") || "dc=example,dc=com", uid: System.get_env("LDAP_UID") || "cn" -config :pleroma, :auth, oauth_consumer_enabled: System.get_env("OAUTH_CONSUMER_ENABLED") == "true" +oauth_consumer_strategies = String.split(System.get_env("OAUTH_CONSUMER_STRATEGIES" || "")) + +ueberauth_providers = + for strategy <- oauth_consumer_strategies do + strategy_module_name = + System.get_env("UEBERAUTH_#{String.upcase(strategy)}_STRATEGY_MODULE") || + "Elixir.Ueberauth.Strategy.#{String.capitalize(strategy)}" + + strategy_module = String.to_atom(strategy_module_name) + {String.to_atom(strategy), {strategy_module, [callback_params: ["state"]]}} + end config :ueberauth, Ueberauth, base_path: "/oauth", - providers: [ - twitter: - {Ueberauth.Strategy.Twitter, - [callback_params: ~w[client_id redirect_uri scope scopes]]} - ] + providers: ueberauth_providers -config :ueberauth, Ueberauth.Strategy.Twitter.OAuth, - consumer_key: System.get_env("TWITTER_CONSUMER_KEY"), - consumer_secret: System.get_env("TWITTER_CONSUMER_SECRET") +config :pleroma, :auth, + oauth_consumer_strategies: oauth_consumer_strategies, + oauth_consumer_enabled: oauth_consumer_strategies != [] # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index a2c62ae68..b300c96df 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -187,25 +187,25 @@ def request(conn, params) do |> redirect(to: "/") end - def callback(%{assigns: %{ueberauth_failure: failure}} = conn, %{"redirect_uri" => redirect_uri}) do + def callback(%{assigns: %{ueberauth_failure: failure}} = conn, params) do + params = callback_params(params) messages = for e <- Map.get(failure, :errors, []), do: e.message message = Enum.join(messages, "; ") conn |> put_flash(:error, "Failed to authenticate: #{message}.") - |> redirect(external: redirect_uri(conn, redirect_uri)) + |> redirect(external: redirect_uri(conn, params["redirect_uri"])) end - def callback( - conn, - %{"client_id" => client_id, "redirect_uri" => redirect_uri} = params - ) do + def callback(conn, params) do + params = callback_params(params) + with {:ok, registration} <- Authenticator.get_registration(conn, params) do user = Repo.preload(registration, :user).user auth_params = %{ - "client_id" => client_id, - "redirect_uri" => redirect_uri, + "client_id" => params["client_id"], + "redirect_uri" => params["redirect_uri"], "scopes" => oauth_scopes(params, nil) } @@ -230,10 +230,21 @@ def callback( _ -> conn |> put_flash(:error, "Failed to set up user account.") - |> redirect(external: redirect_uri(conn, redirect_uri)) + |> redirect(external: redirect_uri(conn, params["redirect_uri"])) end end + defp callback_params(%{"state" => state} = params) do + [client_id, redirect_uri, scope, state] = String.split(state, "|") + + Map.merge(params, %{ + "client_id" => client_id, + "redirect_uri" => redirect_uri, + "scope" => scope, + "state" => state + }) + end + def registration_details(conn, params) do render(conn, "register.html", %{ client_id: params["client_id"], diff --git a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex index e7251bce8..a64859a49 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex @@ -1,14 +1,10 @@ -

External OAuth Authorization

-<%= form_for @conn, o_auth_path(@conn, :request, :twitter), [method: "get"], fn f -> %> -
- <%= label f, :scope, "Permissions" %> -
- <%= text_input f, :scope, value: Enum.join(@available_scopes, " ") %> -
-
+
+
+

Sign in with external provider

- <%= hidden_input f, :client_id, value: @client_id %> - <%= hidden_input f, :redirect_uri, value: @redirect_uri %> - <%= hidden_input f, :state, value: @state%> - <%= submit "Sign in with Twitter" %> +<%= for strategy <- Pleroma.Config.get([:auth, :oauth_consumer_strategies], []) do %> + <%= form_for @conn, o_auth_path(@conn, :request, strategy), [method: "get"], fn f -> %> + <%= hidden_input f, :state, value: Enum.join([@client_id, @redirect_uri, Enum.join(@available_scopes, " "), @state], "|") %> + <%= submit "Sign in with #{String.capitalize(strategy)}" %> + <% end %> <% end %> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index 2fa7837fc..b2381869a 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -37,6 +37,5 @@ <% end %> <%= if Pleroma.Config.get([:auth, :oauth_consumer_enabled]) do %> -
<%= render @view_module, "consumer.html", assigns %> <% end %> diff --git a/mix.exs b/mix.exs index 25711bc26..f7ab008ac 100644 --- a/mix.exs +++ b/mix.exs @@ -44,7 +44,7 @@ def project do def application do [ mod: {Pleroma.Application, []}, - extra_applications: [:logger, :runtime_tools, :comeonin, :ueberauth_twitter], + extra_applications: [:logger, :runtime_tools, :comeonin], included_applications: [:ex_syslogger] ] end @@ -57,6 +57,12 @@ defp elixirc_paths(_), do: ["lib"] # # Type `mix help deps` for examples and options. defp deps do + oauth_strategies = String.split(System.get_env("OAUTH_CONSUMER_STRATEGIES") || "") + + oauth_deps = + for s <- oauth_strategies, + do: {String.to_atom("ueberauth_#{s}"), ">= 0.0.0"} + [ {:phoenix, "~> 1.4.1"}, {:plug_cowboy, "~> 2.0"}, @@ -94,14 +100,11 @@ defp deps do {:floki, "~> 0.20.0"}, {:ex_syslogger, github: "slashmili/ex_syslogger", tag: "1.4.0"}, {:timex, "~> 3.5"}, - {:oauth, github: "tim/erlang-oauth"}, - # {:oauth2, "~> 0.8", override: true}, {:ueberauth, "~> 0.4"}, - {:ueberauth_twitter, "~> 0.2"}, {:auto_linker, git: "https://git.pleroma.social/pleroma/auto_linker.git", ref: "94193ca5f97c1f9fdf3d1469653e2d46fac34bcd"} - ] + ] ++ oauth_deps end # Aliases are shortcuts or tasks specific to the current project. diff --git a/mix.lock b/mix.lock index 92660b70a..6a6cee1a9 100644 --- a/mix.lock +++ b/mix.lock @@ -67,6 +67,7 @@ "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.17", "50793e3d85af49736701da1a040c415c97dc1caf6464112fd9bd18f425d3053b", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, "ueberauth": {:hex, :ueberauth, "0.5.0", "4570ec94d7f784dc4c4aa94c83391dbd9b9bd7b66baa30e95a666c5ec1b168b1", [:mix], [{:plug, "~> 1.2", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, + "ueberauth_facebook": {:hex, :ueberauth_facebook, "0.8.0", "9ec8571f804dd5c06f4e305d70606b39fc0ac8a8f43ed56ebb76012a97d14729", [:mix], [{:oauth2, "~> 0.9", [hex: :oauth2, repo: "hexpm", optional: false]}, {:ueberauth, "~> 0.4", [hex: :ueberauth, repo: "hexpm", optional: false]}], "hexpm"}, "ueberauth_twitter": {:hex, :ueberauth_twitter, "0.2.4", "770ac273cc696cde986582e7a36df0923deb39fa3deff0152fbf150343809f81", [:mix], [{:httpoison, "~> 0.7", [hex: :httpoison, repo: "hexpm", optional: false]}, {:oauther, "~> 1.1", [hex: :oauther, repo: "hexpm", optional: false]}, {:poison, "~> 1.3 or ~> 2.0", [hex: :poison, repo: "hexpm", optional: false]}, {:ueberauth, "~> 0.2", [hex: :ueberauth, repo: "hexpm", optional: false]}], "hexpm"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"}, "unsafe": {:hex, :unsafe, "1.0.0", "7c21742cd05380c7875546b023481d3a26f52df8e5dfedcb9f958f322baae305", [:mix], [], "hexpm"}, From 81bf6d9e6a92b4af00b3351b043193a3c299ede5 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 20 Mar 2019 20:29:08 +0300 Subject: [PATCH 09/59] [#923] Typo fix. --- config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 7d8de5af6..586844516 100644 --- a/config/config.exs +++ b/config/config.exs @@ -381,7 +381,7 @@ base: System.get_env("LDAP_BASE") || "dc=example,dc=com", uid: System.get_env("LDAP_UID") || "cn" -oauth_consumer_strategies = String.split(System.get_env("OAUTH_CONSUMER_STRATEGIES" || "")) +oauth_consumer_strategies = String.split(System.get_env("OAUTH_CONSUMER_STRATEGIES") || "") ueberauth_providers = for strategy <- oauth_consumer_strategies do From 2a95014b9d7142aa2549e70f428293af78fae8eb Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 27 Mar 2019 15:39:35 +0300 Subject: [PATCH 10/59] [#923] OAuth consumer improvements, fixes, refactoring. --- config/config.exs | 5 +--- lib/pleroma/web/auth/authenticator.ex | 6 ++++ lib/pleroma/web/auth/ldap_authenticator.ex | 2 ++ lib/pleroma/web/auth/pleroma_authenticator.ex | 2 ++ lib/pleroma/web/oauth/oauth_controller.ex | 28 +++++++++++++------ lib/pleroma/web/router.ex | 1 + .../templates/o_auth/o_auth/_scopes.html.eex | 13 +++++++++ .../templates/o_auth/o_auth/consumer.html.eex | 15 ++++++---- .../web/templates/o_auth/o_auth/show.html.eex | 16 ++--------- mix.lock | 7 +---- 10 files changed, 59 insertions(+), 36 deletions(-) create mode 100644 lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex diff --git a/config/config.exs b/config/config.exs index 586844516..bdaf5205a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -385,10 +385,7 @@ ueberauth_providers = for strategy <- oauth_consumer_strategies do - strategy_module_name = - System.get_env("UEBERAUTH_#{String.upcase(strategy)}_STRATEGY_MODULE") || - "Elixir.Ueberauth.Strategy.#{String.capitalize(strategy)}" - + strategy_module_name = "Elixir.Ueberauth.Strategy.#{String.capitalize(strategy)}" strategy_module = String.to_atom(strategy_module_name) {String.to_atom(strategy), {strategy_module, [callback_params: ["state"]]}} end diff --git a/lib/pleroma/web/auth/authenticator.ex b/lib/pleroma/web/auth/authenticator.ex index 1f614668c..bb87b323c 100644 --- a/lib/pleroma/web/auth/authenticator.ex +++ b/lib/pleroma/web/auth/authenticator.ex @@ -33,4 +33,10 @@ def handle_error(plug, error), do: implementation().handle_error(plug, error) def auth_template do implementation().auth_template() || Pleroma.Config.get(:auth_template, "show.html") end + + @callback oauth_consumer_template() :: String.t() | nil + def oauth_consumer_template do + implementation().oauth_consumer_template() || + Pleroma.Config.get(:oauth_consumer_template, "consumer.html") + end end diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index 65abd7f38..8b6d5a77f 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -51,6 +51,8 @@ def handle_error(%Plug.Conn{} = _conn, error) do def auth_template, do: nil + def oauth_consumer_template, do: nil + defp ldap_user(name, password) do ldap = Pleroma.Config.get(:ldap, []) host = Keyword.get(ldap, :host, "localhost") diff --git a/lib/pleroma/web/auth/pleroma_authenticator.ex b/lib/pleroma/web/auth/pleroma_authenticator.ex index 60847ce6a..8b190f97f 100644 --- a/lib/pleroma/web/auth/pleroma_authenticator.ex +++ b/lib/pleroma/web/auth/pleroma_authenticator.ex @@ -92,4 +92,6 @@ def handle_error(%Plug.Conn{} = _conn, error) do end def auth_template, do: nil + + def oauth_consumer_template, do: nil end diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index b300c96df..078839d5c 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -174,6 +174,25 @@ def token_revoke(conn, %{"token" => token} = params) do end end + def prepare_request(conn, %{"provider" => provider} = params) do + scope = + oauth_scopes(params, []) + |> Enum.join(" ") + + state = + params + |> Map.delete("scopes") + |> Map.put("scope", scope) + |> Poison.encode!() + + params = + params + |> Map.drop(~w(scope scopes client_id redirect_uri)) + |> Map.put("state", state) + + redirect(conn, to: o_auth_path(conn, :request, provider, params)) + end + def request(conn, params) do message = if params["provider"] do @@ -235,14 +254,7 @@ def callback(conn, params) do end defp callback_params(%{"state" => state} = params) do - [client_id, redirect_uri, scope, state] = String.split(state, "|") - - Map.merge(params, %{ - "client_id" => client_id, - "redirect_uri" => redirect_uri, - "scope" => scope, - "state" => state - }) + Map.merge(params, Poison.decode!(state)) end def registration_details(conn, params) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index f2cec574b..4d0e04d9f 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -213,6 +213,7 @@ defmodule Pleroma.Web.Router do scope [] do pipe_through(:browser) + get("/prepare_request", OAuthController, :prepare_request) get("/:provider", OAuthController, :request) get("/:provider/callback", OAuthController, :callback) post("/register", OAuthController, :register) diff --git a/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex new file mode 100644 index 000000000..4b8fb5dae --- /dev/null +++ b/lib/pleroma/web/templates/o_auth/o_auth/_scopes.html.eex @@ -0,0 +1,13 @@ +
+ <%= label @form, :scope, "Permissions" %> + +
+ <%= for scope <- @available_scopes do %> + <%# Note: using hidden input with `unchecked_value` in order to distinguish user's empty selection from `scope` param being omitted %> +
+ <%= checkbox @form, :"scope_#{scope}", value: scope in @scopes && scope, checked_value: scope, unchecked_value: "", name: assigns[:scope_param] || "scope[]" %> + <%= label @form, :"scope_#{scope}", String.capitalize(scope) %> +
+ <% end %> +
+
diff --git a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex index a64859a49..002f014e6 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex @@ -2,9 +2,14 @@

Sign in with external provider

-<%= for strategy <- Pleroma.Config.get([:auth, :oauth_consumer_strategies], []) do %> - <%= form_for @conn, o_auth_path(@conn, :request, strategy), [method: "get"], fn f -> %> - <%= hidden_input f, :state, value: Enum.join([@client_id, @redirect_uri, Enum.join(@available_scopes, " "), @state], "|") %> - <%= submit "Sign in with #{String.capitalize(strategy)}" %> - <% end %> +<%= form_for @conn, o_auth_path(@conn, :prepare_request), [method: "get"], fn f -> %> + <%= render @view_module, "_scopes.html", Map.put(assigns, :form, f) %> + + <%= hidden_input f, :client_id, value: @client_id %> + <%= hidden_input f, :redirect_uri, value: @redirect_uri %> + <%= hidden_input f, :state, value: @state %> + + <%= for strategy <- Pleroma.Config.get([:auth, :oauth_consumer_strategies], []) do %> + <%= submit "Sign in with #{String.capitalize(strategy)}", name: "provider", value: strategy %> + <% end %> <% end %> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index b2381869a..e6cf1db45 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -16,18 +16,8 @@ <%= label f, :password, "Password" %> <%= password_input f, :password %>
-
-<%= label f, :scope, "Permissions" %> -
- <%= for scope <- @available_scopes do %> - <%# Note: using hidden input with `unchecked_value` in order to distinguish user's empty selection from `scope` param being omitted %> -
- <%= checkbox f, :"scope_#{scope}", value: scope in @scopes && scope, checked_value: scope, unchecked_value: "", name: "authorization[scope][]" %> - <%= label f, :"scope_#{scope}", String.capitalize(scope) %> -
- <% end %> -
-
+ +<%= render @view_module, "_scopes.html", Map.merge(assigns, %{form: f, scope_param: "authorization[scope][]"}) %> <%= hidden_input f, :client_id, value: @client_id %> <%= hidden_input f, :response_type, value: @response_type %> @@ -37,5 +27,5 @@ <% end %> <%= if Pleroma.Config.get([:auth, :oauth_consumer_enabled]) do %> - <%= render @view_module, "consumer.html", assigns %> + <%= render @view_module, Pleroma.Web.Auth.Authenticator.oauth_consumer_template(), assigns %> <% end %> diff --git a/mix.lock b/mix.lock index 6a6cee1a9..ee8617124 100644 --- a/mix.lock +++ b/mix.lock @@ -43,9 +43,6 @@ "mock": {:hex, :mock, "0.3.1", "994f00150f79a0ea50dc9d86134cd9ebd0d177ad60bd04d1e46336cdfdb98ff9", [:mix], [{:meck, "~> 0.8.8", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm"}, "mogrify": {:hex, :mogrify, "0.6.1", "de1b527514f2d95a7bbe9642eb556061afb337e220cf97adbf3a4e6438ed70af", [:mix], [], "hexpm"}, "nimble_parsec": {:hex, :nimble_parsec, "0.4.0", "ee261bb53214943679422be70f1658fff573c5d0b0a1ecd0f18738944f818efe", [:mix], [], "hexpm"}, - "oauth": {:git, "https://github.com/tim/erlang-oauth.git", "bd19896e31125f99ff45bb5850b1c0e74b996743", []}, - "oauth2": {:hex, :oauth2, "0.9.4", "632e8e8826a45e33ac2ea5ac66dcc019ba6bb5a0d2ba77e342d33e3b7b252c6e", [:mix], [{:hackney, "~> 1.7", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, - "oauther": {:hex, :oauther, "1.1.1", "7d8b16167bb587ecbcddd3f8792beb9ec3e7b65c1f8ebd86b8dd25318d535752", [:mix], [], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.3", "6706a148809a29c306062862c803406e88f048277f6e85b68faf73291e820b84", [:mix], [], "hexpm"}, "phoenix": {:hex, :phoenix, "1.4.1", "801f9d632808657f1f7c657c8bbe624caaf2ba91429123ebe3801598aea4c3d9", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, @@ -66,9 +63,7 @@ "timex": {:hex, :timex, "3.5.0", "b0a23167da02d0fe4f1a4e104d1f929a00d348502b52432c05de875d0b9cffa5", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "tzdata": {:hex, :tzdata, "0.5.17", "50793e3d85af49736701da1a040c415c97dc1caf6464112fd9bd18f425d3053b", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"}, - "ueberauth": {:hex, :ueberauth, "0.5.0", "4570ec94d7f784dc4c4aa94c83391dbd9b9bd7b66baa30e95a666c5ec1b168b1", [:mix], [{:plug, "~> 1.2", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, - "ueberauth_facebook": {:hex, :ueberauth_facebook, "0.8.0", "9ec8571f804dd5c06f4e305d70606b39fc0ac8a8f43ed56ebb76012a97d14729", [:mix], [{:oauth2, "~> 0.9", [hex: :oauth2, repo: "hexpm", optional: false]}, {:ueberauth, "~> 0.4", [hex: :ueberauth, repo: "hexpm", optional: false]}], "hexpm"}, - "ueberauth_twitter": {:hex, :ueberauth_twitter, "0.2.4", "770ac273cc696cde986582e7a36df0923deb39fa3deff0152fbf150343809f81", [:mix], [{:httpoison, "~> 0.7", [hex: :httpoison, repo: "hexpm", optional: false]}, {:oauther, "~> 1.1", [hex: :oauther, repo: "hexpm", optional: false]}, {:poison, "~> 1.3 or ~> 2.0", [hex: :poison, repo: "hexpm", optional: false]}, {:ueberauth, "~> 0.2", [hex: :ueberauth, repo: "hexpm", optional: false]}], "hexpm"}, + "ueberauth": {:hex, :ueberauth, "0.6.1", "9e90d3337dddf38b1ca2753aca9b1e53d8a52b890191cdc55240247c89230412", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"}, "unsafe": {:hex, :unsafe, "1.0.0", "7c21742cd05380c7875546b023481d3a26f52df8e5dfedcb9f958f322baae305", [:mix], [], "hexpm"}, "web_push_encryption": {:hex, :web_push_encryption, "0.2.1", "d42cecf73420d9dc0053ba3299cc8c8d6ff2be2487d67ca2a57265868e4d9a98", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.8", [hex: :jose, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, From 642075b1a935c42181a10ea695b2289883126136 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 27 Mar 2019 16:20:50 +0300 Subject: [PATCH 11/59] [#923] Enabled binding of multiple OAuth provider accounts to single user. --- priv/repo/migrations/20190315101315_create_registrations.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/priv/repo/migrations/20190315101315_create_registrations.exs b/priv/repo/migrations/20190315101315_create_registrations.exs index fbb22ec7c..6b28cbdd3 100644 --- a/priv/repo/migrations/20190315101315_create_registrations.exs +++ b/priv/repo/migrations/20190315101315_create_registrations.exs @@ -13,6 +13,6 @@ def change do end create unique_index(:registrations, [:provider, :uid]) - create unique_index(:registrations, [:user_id, :provider]) + create unique_index(:registrations, [:user_id, :provider, :uid]) end end From 55d086b52077a220aef60c8d9071aea990431d74 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Wed, 27 Mar 2019 22:09:39 +0300 Subject: [PATCH 12/59] Notification controls Allow users to configure whether they want to receive notifications from people they follow / who follow them, people from remote / local instances --- lib/pleroma/notification.ex | 65 ++++++++++++++++++++++++++++++---- lib/pleroma/user/info.ex | 4 +++ test/notification_test.exs | 69 +++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index cac10f24a..caa6b755e 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -122,13 +122,7 @@ def create_notifications(_), do: {:ok, []} # TODO move to sql, too. def create_notification(%Activity{} = activity, %User{} = user) do - unless User.blocks?(user, %{ap_id: activity.data["actor"]}) or - CommonAPI.thread_muted?(user, activity) or user.ap_id == activity.data["actor"] or - (activity.data["type"] == "Follow" and - Enum.any?(Notification.for_user(user), fn notif -> - notif.activity.data["type"] == "Follow" and - notif.activity.data["actor"] == activity.data["actor"] - end)) do + unless skip?(activity, user) do notification = %Notification{user_id: user.id, activity: activity} {:ok, notification} = Repo.insert(notification) Pleroma.Web.Streamer.stream("user", notification) @@ -154,4 +148,61 @@ def get_notified_from_activity( end def get_notified_from_activity(_, _local_only), do: [] + + def skip?(activity, user) do + [:self, :blocked, :local, :muted, :followers, :follows, :recently_followed] + |> Enum.any?(&skip?(&1, activity, user)) + end + + def skip?(:self, activity, user) do + activity.data["actor"] == user.ap_id + end + + def skip?(:blocked, activity, user) do + actor = activity.data["actor"] + User.blocks?(user, %{ap_id: actor}) + end + + def skip?(:local, %{local: true}, user) do + user.info.notification_settings["local"] == false + end + + def skip?(:local, %{local: false}, user) do + user.info.notification_settings["remote"] == false + end + + def skip?(:muted, activity, user) do + actor = activity.data["actor"] + + User.mutes?(user, %{ap_id: actor}) or + CommonAPI.thread_muted?(user, activity) + end + + def skip?( + :followers, + activity, + %{info: %{notification_settings: %{"followers" => false}}} = user + ) do + actor = activity.data["actor"] + follower = User.get_cached_by_ap_id(actor) + User.following?(follower, user) + end + + def skip?(:follows, activity, %{info: %{notification_settings: %{"follows" => false}}} = user) do + actor = activity.data["actor"] + followed = User.get_by_ap_id(actor) + User.following?(user, followed) + end + + def skip?(:recently_followed, activity, user) do + actor = activity.data["actor"] + + Notification.for_user(user) + |> Enum.any?(fn + %{activity: %{data: %{"type" => "Follow", "actor" => ^actor}}} -> true + _ -> false + end) + end + + def skip?(_, _, _), do: false end diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 740a46727..c36efa126 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -40,6 +40,10 @@ defmodule Pleroma.User.Info do field(:pinned_activities, {:array, :string}, default: []) field(:flavour, :string, default: nil) + field(:notification_settings, :map, + default: %{"remote" => true, "local" => true, "followers" => true, "follows" => true} + ) + # Found in the wild # ap_id -> Where is this used? # bio -> Where is this used? diff --git a/test/notification_test.exs b/test/notification_test.exs index 12b4292aa..89d06b3a2 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -41,6 +41,75 @@ test "it doesn't create a notification for user if the user blocks the activity assert nil == Notification.create_notification(activity, user) end + test "it doesn't create a notificatin for the user if the user mutes the activity author" do + muter = insert(:user) + muted = insert(:user) + {:ok, _} = User.mute(muter, muted) + muter = Repo.get(User, muter.id) + {:ok, activity} = CommonAPI.post(muted, %{"status" => "Hi @#{muter.nickname}"}) + + assert nil == Notification.create_notification(activity, muter) + end + + test "it doesn't create a notification for an activity from a muted thread" do + muter = insert(:user) + other_user = insert(:user) + {:ok, activity} = CommonAPI.post(muter, %{"status" => "hey"}) + CommonAPI.add_mute(muter, activity) + + {:ok, activity} = + CommonAPI.post(other_user, %{ + "status" => "Hi @#{muter.nickname}", + "in_reply_to_status_id" => activity.id + }) + + assert nil == Notification.create_notification(activity, muter) + end + + test "it disables notifications from people on remote instances" do + user = insert(:user, info: %{notification_settings: %{"remote" => false}}) + other_user = insert(:user) + + create_activity = %{ + "@context" => "https://www.w3.org/ns/activitystreams", + "type" => "Create", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "actor" => other_user.ap_id, + "object" => %{ + "type" => "Note", + "content" => "Hi @#{user.nickname}", + "attributedTo" => other_user.ap_id + } + } + + {:ok, %{local: false} = activity} = Transmogrifier.handle_incoming(create_activity) + assert nil == Notification.create_notification(activity, user) + end + + test "it disables notifications from people on the local instance" do + user = insert(:user, info: %{notification_settings: %{"local" => false}}) + other_user = insert(:user) + {:ok, activity} = CommonAPI.post(other_user, %{"status" => "hey @#{user.nickname}"}) + assert nil == Notification.create_notification(activity, user) + end + + test "it disables notifications from followers" do + follower = insert(:user) + followed = insert(:user, info: %{notification_settings: %{"followers" => false}}) + User.follow(follower, followed) + {:ok, activity} = CommonAPI.post(follower, %{"status" => "hey @#{followed.nickname}"}) + assert nil == Notification.create_notification(activity, followed) + end + + test "it disables notifications from people the user follows" do + follower = insert(:user, info: %{notification_settings: %{"follows" => false}}) + followed = insert(:user) + User.follow(follower, followed) + follower = Repo.get(User, follower.id) + {:ok, activity} = CommonAPI.post(followed, %{"status" => "hey @#{follower.nickname}"}) + assert nil == Notification.create_notification(activity, follower) + end + test "it doesn't create a notification for user if he is the activity author" do activity = insert(:note_activity) author = User.get_by_ap_id(activity.data["actor"]) From cd90695a349f33b84f287794bae6070e9eec446a Mon Sep 17 00:00:00 2001 From: eugenijm Date: Thu, 28 Mar 2019 14:52:09 +0300 Subject: [PATCH 13/59] Add PUT /api/pleroma/notification_settings endpoint --- docs/Pleroma-API.md | 14 ++++++++++-- lib/pleroma/notification.ex | 12 +++++----- lib/pleroma/user.ex | 8 +++++++ lib/pleroma/user/info.ex | 13 +++++++++++ .../web/mastodon_api/views/account_view.ex | 22 +++++++++++++------ lib/pleroma/web/router.ex | 1 + .../controllers/util_controller.ex | 6 +++++ test/web/mastodon_api/account_view_test.exs | 14 ++++++++++++ test/web/twitter_api/util_controller_test.exs | 21 ++++++++++++++++++ 9 files changed, 95 insertions(+), 16 deletions(-) diff --git a/docs/Pleroma-API.md b/docs/Pleroma-API.md index 478c9d874..bb7ed3744 100644 --- a/docs/Pleroma-API.md +++ b/docs/Pleroma-API.md @@ -27,14 +27,14 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi * Method: `GET` * Authentication: not required * Params: none -* Response: Provider specific JSON, the only guaranteed parameter is `type` +* Response: Provider specific JSON, the only guaranteed parameter is `type` * Example response: `{"type": "kocaptcha", "token": "whatever", "url": "https://captcha.kotobank.ch/endpoint"}` ## `/api/pleroma/delete_account` ### Delete an account * Method `POST` * Authentication: required -* Params: +* Params: * `password`: user's password * Response: JSON. Returns `{"status": "success"}` if the deletion was successful, `{"error": "[error message]"}` otherwise * Example response: `{"error": "Invalid password."}` @@ -116,3 +116,13 @@ See [Admin-API](Admin-API.md) * Params: * `id`: notifications's id * Response: JSON. Returns `{"status": "success"}` if the reading was successful, otherwise returns `{"error": "error_msg"}` +## `/api/pleroma/notification_settings` +### Updates user notification settings +* Method `PUT` +* Authentication: required +* Params: + * `followers`: BOOLEAN field, receives notifications from followers + * `follows`: BOOLEAN field, receives notifications from people the user follows + * `remote`: BOOLEAN field, receives notifications from people on remote instances + * `local`: BOOLEAN field, receives notifications from people on the local instance +* Response: JSON. Returns `{"status": "success"}` if the update was successful, otherwise returns `{"error": "error_msg"}` diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index caa6b755e..14de1a097 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -163,13 +163,11 @@ def skip?(:blocked, activity, user) do User.blocks?(user, %{ap_id: actor}) end - def skip?(:local, %{local: true}, user) do - user.info.notification_settings["local"] == false - end + def skip?(:local, %{local: true}, %{info: %{notification_settings: %{"local" => false}}}), + do: true - def skip?(:local, %{local: false}, user) do - user.info.notification_settings["remote"] == false - end + def skip?(:local, %{local: false}, %{info: %{notification_settings: %{"remote" => false}}}), + do: true def skip?(:muted, activity, user) do actor = activity.data["actor"] @@ -194,7 +192,7 @@ def skip?(:follows, activity, %{info: %{notification_settings: %{"follows" => fa User.following?(user, followed) end - def skip?(:recently_followed, activity, user) do + def skip?(:recently_followed, %{data: %{"type" => "Follow"}} = activity, user) do actor = activity.data["actor"] Notification.for_user(user) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 728b00a56..73c2a82a7 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1082,6 +1082,14 @@ def deactivate(%User{} = user, status \\ true) do update_and_set_cache(cng) end + def update_notification_settings(%User{} = user, settings \\ %{}) do + info_changeset = User.Info.update_notification_settings(user.info, settings) + + change(user) + |> put_embed(:info, info_changeset) + |> update_and_set_cache() + end + def delete(%User{} = user) do {:ok, user} = User.deactivate(user) diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index c36efa126..33fd77b02 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -61,6 +61,19 @@ def set_activation_status(info, deactivated) do |> validate_required([:deactivated]) end + def update_notification_settings(info, settings) do + notification_settings = + info.notification_settings + |> Map.merge(settings) + |> Map.take(["remote", "local", "followers", "follows"]) + + params = %{notification_settings: notification_settings} + + info + |> cast(params, [:notification_settings]) + |> validate_required([:notification_settings]) + end + def add_to_note_count(info, number) do set_note_count(info, info.note_count + number) end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index b5f3bbb9d..25899e491 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -117,13 +117,15 @@ defp do_render("account.json", %{user: user} = opts) do }, # Pleroma extension - pleroma: %{ - confirmation_pending: user_info.confirmation_pending, - tags: user.tags, - is_moderator: user.info.is_moderator, - is_admin: user.info.is_admin, - relationship: relationship - } + pleroma: + %{ + confirmation_pending: user_info.confirmation_pending, + tags: user.tags, + is_moderator: user.info.is_moderator, + is_admin: user.info.is_admin, + relationship: relationship + } + |> with_notification_settings(user, opts[:for]) } end @@ -132,4 +134,10 @@ defp username_from_nickname(string) when is_binary(string) do end defp username_from_nickname(_), do: nil + + defp with_notification_settings(data, %User{id: user_id} = user, %User{id: user_id}) do + Map.put(data, :notification_settings, user.info.notification_settings) + end + + defp with_notification_settings(data, _, _), do: data end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 32e5f7644..36cbf0f57 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -182,6 +182,7 @@ defmodule Pleroma.Web.Router do post("/change_password", UtilController, :change_password) post("/delete_account", UtilController, :delete_account) + put("/notification_settings", UtilController, :update_notificaton_settings) end scope [] do diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index faa733fec..2708299cb 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -269,6 +269,12 @@ def emoji(conn, _params) do json(conn, Enum.into(Emoji.get_all(), %{})) end + def update_notificaton_settings(%{assigns: %{user: user}} = conn, params) do + with {:ok, _} <- User.update_notification_settings(user, params) do + json(conn, %{status: "success"}) + end + end + def follow_import(conn, %{"list" => %Plug.Upload{} = listfile}) do follow_import(conn, %{"list" => File.read!(listfile.path)}) end diff --git a/test/web/mastodon_api/account_view_test.exs b/test/web/mastodon_api/account_view_test.exs index 6dc60afe9..aa6a1e960 100644 --- a/test/web/mastodon_api/account_view_test.exs +++ b/test/web/mastodon_api/account_view_test.exs @@ -71,6 +71,20 @@ test "Represent a user account" do assert expected == AccountView.render("account.json", %{user: user}) end + test "Represent the user account for the account owner" do + user = insert(:user) + + notification_settings = %{ + "remote" => true, + "local" => true, + "followers" => true, + "follows" => true + } + + assert %{pleroma: %{notification_settings: ^notification_settings}} = + AccountView.render("account.json", %{user: user, for: user}) + end + test "Represent a Service(bot) account" do user = insert(:user, %{ diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 832fdc096..426508353 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -3,6 +3,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilControllerTest do alias Pleroma.Notification alias Pleroma.Repo + alias Pleroma.User alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -74,6 +75,26 @@ test "it marks a single notification as read", %{conn: conn} do end end + describe "PUT /api/pleroma/notification_settings" do + test "it updates notification settings", %{conn: conn} do + user = insert(:user) + + conn + |> assign(:user, user) + |> put("/api/pleroma/notification_settings", %{ + "remote" => false, + "followers" => false, + "bar" => 1 + }) + |> json_response(:ok) + + user = Repo.get(User, user.id) + + assert %{"remote" => false, "local" => true, "followers" => false, "follows" => true} == + user.info.notification_settings + end + end + describe "GET /api/statusnet/config.json" do test "returns the state of safe_dm_mentions flag", %{conn: conn} do option = Pleroma.Config.get([:instance, :safe_dm_mentions]) From eadafc88b898879eb50545b700ea13c8596e908b Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 1 Apr 2019 09:28:56 +0300 Subject: [PATCH 14/59] [#923] Deps config adjustment (no `override` for `httpoison`), code analysis issues fixes. --- lib/pleroma/web/auth/pleroma_authenticator.ex | 2 +- lib/pleroma/web/endpoint.ex | 3 ++- lib/pleroma/web/oauth/oauth_controller.ex | 2 +- mix.exs | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/auth/pleroma_authenticator.ex b/lib/pleroma/web/auth/pleroma_authenticator.ex index 8b190f97f..c826adb4c 100644 --- a/lib/pleroma/web/auth/pleroma_authenticator.ex +++ b/lib/pleroma/web/auth/pleroma_authenticator.ex @@ -4,9 +4,9 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticator do alias Comeonin.Pbkdf2 - alias Pleroma.User alias Pleroma.Registration alias Pleroma.Repo + alias Pleroma.User @behaviour Pleroma.Web.Auth.Authenticator diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index f92724d8b..b85b95bf9 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -60,7 +60,8 @@ defmodule Pleroma.Web.Endpoint do same_site = if Pleroma.Config.get([:auth, :oauth_consumer_enabled]) do - # Note: "SameSite=Strict" prevents sign in with external OAuth provider (no cookies during callback request) + # Note: "SameSite=Strict" prevents sign in with external OAuth provider + # (there would be no cookies during callback request from OAuth provider) "SameSite=Lax" else "SameSite=Strict" diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index e54e196aa..54e0a35ba 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -5,9 +5,9 @@ defmodule Pleroma.Web.OAuth.OAuthController do use Pleroma.Web, :controller + alias Pleroma.Registration alias Pleroma.Repo alias Pleroma.User - alias Pleroma.Registration alias Pleroma.Web.Auth.Authenticator alias Pleroma.Web.OAuth.App alias Pleroma.Web.OAuth.Authorization diff --git a/mix.exs b/mix.exs index 34c17bd6b..2b0d25b55 100644 --- a/mix.exs +++ b/mix.exs @@ -76,7 +76,7 @@ defp deps do {:phoenix_html, "~> 2.10"}, {:calendar, "~> 0.17.4"}, {:cachex, "~> 3.0.2"}, - {:httpoison, "~> 1.2.0", override: true}, + {:httpoison, "~> 1.2.0"}, {:poison, "~> 3.0", override: true}, {:tesla, "~> 1.2"}, {:jason, "~> 1.0"}, From 804173fc924ec591558b8ed7671e35b506be9345 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 1 Apr 2019 09:45:44 +0300 Subject: [PATCH 15/59] [#923] Minor code readability fix. --- lib/pleroma/web/auth/authenticator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/auth/authenticator.ex b/lib/pleroma/web/auth/authenticator.ex index bb87b323c..4eeef5034 100644 --- a/lib/pleroma/web/auth/authenticator.ex +++ b/lib/pleroma/web/auth/authenticator.ex @@ -3,8 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Auth.Authenticator do - alias Pleroma.User alias Pleroma.Registration + alias Pleroma.User def implementation do Pleroma.Config.get( From 3601f03147bd104f6acff64e7c8d5d4d3e1f53a2 Mon Sep 17 00:00:00 2001 From: Alex S Date: Mon, 1 Apr 2019 17:17:57 +0700 Subject: [PATCH 16/59] Adding tag to emoji ets table changes in apis --- config/config.exs | 7 ++- config/emoji.txt | 5 +- docs/api/pleroma_api.md | 6 +-- docs/config/custom_emoji.md | 24 ++++++++- lib/pleroma/emoji.ex | 53 ++++++++++++++++--- lib/pleroma/formatter.ex | 8 +-- lib/pleroma/web/common_api/common_api.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 2 +- .../mastodon_api/mastodon_api_controller.ex | 5 +- .../controllers/util_controller.ex | 8 ++- test/emoji_test.exs | 30 +++++++++++ test/formatter_test.exs | 3 +- .../mastodon_api_controller_test.exs | 16 ++++++ test/web/twitter_api/util_controller_test.exs | 21 ++++++++ 14 files changed, 165 insertions(+), 25 deletions(-) create mode 100644 test/emoji_test.exs diff --git a/config/config.exs b/config/config.exs index 0df38d75a..245c7d268 100644 --- a/config/config.exs +++ b/config/config.exs @@ -54,7 +54,12 @@ cgi: "https://mdii.sakura.ne.jp/mdii-post.cgi", files: "https://mdii.sakura.ne.jp" -config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"] +config :pleroma, :emoji, + shortcode_globs: ["/emoji/custom/**/*.png"], + custom_tag: "Custom", + finmoji_tag: "Finmoji", + emoji_tag: "Emoji", + custom_emoji_tag: "Custom" config :pleroma, :uri_schemes, valid_schemes: [ diff --git a/config/emoji.txt b/config/emoji.txt index 7afacb09f..79246f239 100644 --- a/config/emoji.txt +++ b/config/emoji.txt @@ -1,5 +1,5 @@ -firefox, /emoji/Firefox.gif -blank, /emoji/blank.png +firefox, /emoji/Firefox.gif, Gif,Fun +blank, /emoji/blank.png, Fun f_00b, /emoji/f_00b.png f_00b11b, /emoji/f_00b11b.png f_00b33b, /emoji/f_00b33b.png @@ -28,4 +28,3 @@ f_33b00b, /emoji/f_33b00b.png f_33b22b, /emoji/f_33b22b.png f_33h, /emoji/f_33h.png f_33t, /emoji/f_33t.png - diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index 478c9d874..2e8fb04d2 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -10,7 +10,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi * Authentication: not required * Params: none * Response: JSON -* Example response: `{"kalsarikannit_f":"/finmoji/128px/kalsarikannit_f-128.png","perkele":"/finmoji/128px/perkele-128.png","blobdab":"/emoji/blobdab.png","happiness":"/finmoji/128px/happiness-128.png"}` +* Example response: `[{"kalsarikannit_f":{"tags":["Finmoji"],"image_url":"/finmoji/128px/kalsarikannit_f-128.png"}},{"perkele":{"tags":["Finmoji"],"image_url":"/finmoji/128px/perkele-128.png"}},{"blobdab":{"tags":["SomeTag"],"image_url":"/emoji/blobdab.png"}},"happiness":{"tags":["Finmoji"],"image_url":"/finmoji/128px/happiness-128.png"}}]` * Note: Same data as Mastodon API’s `/api/v1/custom_emojis` but in a different format ## `/api/pleroma/follow_import` @@ -27,14 +27,14 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi * Method: `GET` * Authentication: not required * Params: none -* Response: Provider specific JSON, the only guaranteed parameter is `type` +* Response: Provider specific JSON, the only guaranteed parameter is `type` * Example response: `{"type": "kocaptcha", "token": "whatever", "url": "https://captcha.kotobank.ch/endpoint"}` ## `/api/pleroma/delete_account` ### Delete an account * Method `POST` * Authentication: required -* Params: +* Params: * `password`: user's password * Response: JSON. Returns `{"status": "success"}` if the deletion was successful, `{"error": "[error message]"}` otherwise * Example response: `{"error": "Invalid password."}` diff --git a/docs/config/custom_emoji.md b/docs/config/custom_emoji.md index e833d2080..e47a75c8e 100644 --- a/docs/config/custom_emoji.md +++ b/docs/config/custom_emoji.md @@ -11,8 +11,28 @@ image files (in `/priv/static/emoji/custom`): `happy.png` and `sad.png` content of `config/custom_emoji.txt`: ``` -happy, /emoji/custom/happy.png -sad, /emoji/custom/sad.png +happy, /emoji/custom/happy.png, Tag1,Tag2 +sad, /emoji/custom/sad.png, Tag1 +foo, /emoji/custom/foo.png ``` The files should be PNG (APNG is okay with `.png` for `image/png` Content-type) and under 50kb for compatibility with mastodon. + +# Emoji tags + +Changing default tags: + +* For `Finmoji`, `emoji.txt` and `custom_emoji.txt` are added default tags, which can be configured in the `config.exs`: +* For emoji loaded from globs: + - `priv/static/emoji/custom/*.png` - `custom_tag`, can be configured in `config.exs` + - `priv/static/emoji/custom/TagName/*.png` - folder (`TagName`) is used as tag + + +``` +config :pleroma, :emoji, + shortcode_globs: ["/emoji/custom/**/*.png"], + custom_tag: "Custom", # Default tag for emoji in `priv/static/emoji/custom` path + finmoji_tag: "Finmoji", # Default tag for Finmoji + emoji_tag: "Emoji", # Default tag for emoji.txt + custom_emoji_tag: "Custom" # Default tag for custom_emoji.txt +``` diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index f3f08cd9d..c35aed6ee 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Emoji do * the built-in Finmojis (if enabled in configuration), * the files: `config/emoji.txt` and `config/custom_emoji.txt` - * glob paths + * glob paths, nested folder is used as tag name for grouping e.g. priv/static/emoji/custom/nested_folder This GenServer stores in an ETS table the list of the loaded emojis, and also allows to reload the list at runtime. """ @@ -152,8 +152,10 @@ defp load do "woollysocks" ] defp load_finmoji(true) do + tag = Keyword.get(Application.get_env(:pleroma, :emoji), :finmoji_tag) + Enum.map(@finmoji, fn finmoji -> - {finmoji, "/finmoji/128px/#{finmoji}-128.png"} + {finmoji, "/finmoji/128px/#{finmoji}-128.png", tag} end) end @@ -168,31 +170,70 @@ defp load_from_file(file) do end defp load_from_file_stream(stream) do + default_tag = + stream.path + |> Path.basename(".txt") + |> get_default_tag() + stream |> Stream.map(&String.trim/1) |> Stream.map(fn line -> case String.split(line, ~r/,\s*/) do - [name, file] -> {name, file} - _ -> nil + [name, file, tags] -> + {name, file, tags} + + [name, file] -> + {name, file, default_tag} + + _ -> + nil end end) |> Enum.to_list() end + @spec get_default_tag(String.t()) :: String.t() + defp get_default_tag(file_name) when file_name in ["emoji", "custom_emojii"] do + Keyword.get( + Application.get_env(:pleroma, :emoji), + String.to_existing_atom(file_name <> "_tag") + ) + end + + defp get_default_tag(_), do: Keyword.get(Application.get_env(:pleroma, :emoji), :custom_tag) + defp load_from_globs(globs) do static_path = Path.join(:code.priv_dir(:pleroma), "static") paths = Enum.map(globs, fn glob -> + static_part = + Path.dirname(glob) + |> String.replace_trailing("**", "") + Path.join(static_path, glob) |> Path.wildcard() + |> Enum.map(fn path -> + custom_folder = + path + |> Path.relative_to(Path.join(static_path, static_part)) + |> Path.dirname() + + [path, custom_folder] + end) end) |> Enum.concat() - Enum.map(paths, fn path -> + Enum.map(paths, fn [path, custom_folder] -> + tag = + case custom_folder do + "." -> Keyword.get(Application.get_env(:pleroma, :emoji), :custom_tag) + tag -> tag + end + shortcode = Path.basename(path, Path.extname(path)) external_path = Path.join("/", Path.relative_to(path, static_path)) - {shortcode, external_path} + {shortcode, external_path, tag} end) end end diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index e3625383b..8ea9dbd38 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -77,9 +77,9 @@ def emojify(text) do def emojify(text, nil), do: text def emojify(text, emoji, strip \\ false) do - Enum.reduce(emoji, text, fn {emoji, file}, text -> - emoji = HTML.strip_tags(emoji) - file = HTML.strip_tags(file) + Enum.reduce(emoji, text, fn emoji_data, text -> + emoji = HTML.strip_tags(elem(emoji_data, 0)) + file = HTML.strip_tags(elem(emoji_data, 1)) html = if not strip do @@ -101,7 +101,7 @@ def demojify(text) do def demojify(text, nil), do: text def get_emoji(text) when is_binary(text) do - Enum.filter(Emoji.get_all(), fn {emoji, _} -> String.contains?(text, ":#{emoji}:") end) + Enum.filter(Emoji.get_all(), fn {emoji, _, _} -> String.contains?(text, ":#{emoji}:") end) end def get_emoji(_), do: [] diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 25b990677..f910eb1f9 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -167,7 +167,7 @@ def post(user, %{"status" => status} = data) do object, "emoji", (Formatter.get_emoji(status) ++ Formatter.get_emoji(data["spoiler_text"])) - |> Enum.reduce(%{}, fn {name, file}, acc -> + |> Enum.reduce(%{}, fn {name, file, _}, acc -> Map.put(acc, name, "#{Pleroma.Web.Endpoint.static_url()}#{file}") end) ) do diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index f596f703b..49f0170cc 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -285,7 +285,7 @@ def confirm_current_password(user, password) do def emoji_from_profile(%{info: _info} = user) do (Formatter.get_emoji(user.bio) ++ Formatter.get_emoji(user.name)) - |> Enum.map(fn {shortcode, url} -> + |> Enum.map(fn {shortcode, url, _} -> %{ "type" => "Emoji", "icon" => %{"type" => "Image", "url" => "#{Endpoint.url()}#{url}"}, diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index eee4e7678..583e4007c 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -178,14 +178,15 @@ def peers(conn, _params) do defp mastodonized_emoji do Pleroma.Emoji.get_all() - |> Enum.map(fn {shortcode, relative_url} -> + |> Enum.map(fn {shortcode, relative_url, tags} -> url = to_string(URI.merge(Web.base_url(), relative_url)) %{ "shortcode" => shortcode, "static_url" => url, "visible_in_picker" => true, - "url" => url + "url" => url, + "tags" => String.split(tags, ",") } end) end diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index faa733fec..e58d9e4cd 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -266,7 +266,13 @@ def version(conn, _params) do end def emoji(conn, _params) do - json(conn, Enum.into(Emoji.get_all(), %{})) + emoji = + Emoji.get_all() + |> Enum.map(fn {short_code, path, tags} -> + %{short_code => %{image_url: path, tags: String.split(tags, ",")}} + end) + + json(conn, emoji) end def follow_import(conn, %{"list" => %Plug.Upload{} = listfile}) do diff --git a/test/emoji_test.exs b/test/emoji_test.exs new file mode 100644 index 000000000..c9c32e20b --- /dev/null +++ b/test/emoji_test.exs @@ -0,0 +1,30 @@ +defmodule Pleroma.EmojiTest do + use ExUnit.Case, async: true + alias Pleroma.Emoji + + describe "get_all/0" do + setup do + emoji_list = Emoji.get_all() + {:ok, emoji_list: emoji_list} + end + test "first emoji", %{emoji_list: emoji_list} do + [emoji | _others] = emoji_list + {code, path, tags} = emoji + + assert tuple_size(emoji) == 3 + assert is_binary(code) + assert is_binary(path) + assert is_binary(tags) + end + + test "random emoji", %{emoji_list: emoji_list} do + emoji = Enum.random(emoji_list) + {code, path, tags} = emoji + + assert tuple_size(emoji) == 3 + assert is_binary(code) + assert is_binary(path) + assert is_binary(tags) + end + end +end diff --git a/test/formatter_test.exs b/test/formatter_test.exs index fcdf931b7..e67042a5f 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -271,7 +271,8 @@ test "it does not add XSS emoji" do test "it returns the emoji used in the text" do text = "I love :moominmamma:" - assert Formatter.get_emoji(text) == [{"moominmamma", "/finmoji/128px/moominmamma-128.png"}] + tag = Keyword.get(Application.get_env(:pleroma, :emoji), :finmoji_tag) + assert Formatter.get_emoji(text) == [{"moominmamma", "/finmoji/128px/moominmamma-128.png", tag}] end test "it returns a nice empty result when no emojis are present" do diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index d9bcbf5a9..3b10c4a1a 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2265,4 +2265,20 @@ test "preserves parameters in link headers", %{conn: conn} do assert link_header =~ ~r/max_id=#{notification1.id}/ end end + + describe "custom emoji" do + test "with tags", %{conn: conn} do + [emoji | _body] = + conn + |> get("/api/v1/custom_emojis") + |> json_response(200) + + assert Map.has_key?(emoji, "shortcode") + assert Map.has_key?(emoji, "static_url") + assert Map.has_key?(emoji, "tags") + assert is_list(emoji["tags"]) + assert Map.has_key?(emoji, "url") + assert Map.has_key?(emoji, "visible_in_picker") + end + end end diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 832fdc096..1063ad28f 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -164,4 +164,25 @@ test "returns everything in :pleroma, :frontend_configurations", %{conn: conn} d assert response == Jason.encode!(config |> Enum.into(%{})) |> Jason.decode!() end end + + describe "/api/pleroma/emoji" do + test "returns json with custom emoji with tags", %{conn: conn} do + [emoji | _body] = + conn + |> get("/api/pleroma/emoji") + |> json_response(200) + + [key] = Map.keys(emoji) + + %{ + ^key => %{ + "image_url" => url, + "tags" => tags + } + } = emoji + + assert is_binary(url) + assert is_list(tags) + end + end end From 17d3d05a7196140b62dd791af8d7ced8b0ad9fa1 Mon Sep 17 00:00:00 2001 From: Alex S Date: Mon, 1 Apr 2019 17:54:30 +0700 Subject: [PATCH 17/59] code style little fix --- lib/pleroma/emoji.ex | 6 +++--- test/emoji_test.exs | 3 ++- test/formatter_test.exs | 5 ++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index c35aed6ee..ad3170f9a 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -152,7 +152,7 @@ defp load do "woollysocks" ] defp load_finmoji(true) do - tag = Keyword.get(Application.get_env(:pleroma, :emoji), :finmoji_tag) + tag = Application.get_env(:pleroma, :emoji)[:finmoji_tag] Enum.map(@finmoji, fn finmoji -> {finmoji, "/finmoji/128px/#{finmoji}-128.png", tag} @@ -193,14 +193,14 @@ defp load_from_file_stream(stream) do end @spec get_default_tag(String.t()) :: String.t() - defp get_default_tag(file_name) when file_name in ["emoji", "custom_emojii"] do + defp get_default_tag(file_name) when file_name in ["emoji", "custom_emoji"] do Keyword.get( Application.get_env(:pleroma, :emoji), String.to_existing_atom(file_name <> "_tag") ) end - defp get_default_tag(_), do: Keyword.get(Application.get_env(:pleroma, :emoji), :custom_tag) + defp get_default_tag(_), do: Application.get_env(:pleroma, :emoji)[:custom_tag] defp load_from_globs(globs) do static_path = Path.join(:code.priv_dir(:pleroma), "static") diff --git a/test/emoji_test.exs b/test/emoji_test.exs index c9c32e20b..a90213d7d 100644 --- a/test/emoji_test.exs +++ b/test/emoji_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.EmojiTest do emoji_list = Emoji.get_all() {:ok, emoji_list: emoji_list} end + test "first emoji", %{emoji_list: emoji_list} do [emoji | _others] = emoji_list {code, path, tags} = emoji @@ -19,7 +20,7 @@ test "first emoji", %{emoji_list: emoji_list} do test "random emoji", %{emoji_list: emoji_list} do emoji = Enum.random(emoji_list) - {code, path, tags} = emoji + {code, path, tags} = emoji assert tuple_size(emoji) == 3 assert is_binary(code) diff --git a/test/formatter_test.exs b/test/formatter_test.exs index e67042a5f..38430e170 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -272,7 +272,10 @@ test "it returns the emoji used in the text" do text = "I love :moominmamma:" tag = Keyword.get(Application.get_env(:pleroma, :emoji), :finmoji_tag) - assert Formatter.get_emoji(text) == [{"moominmamma", "/finmoji/128px/moominmamma-128.png", tag}] + + assert Formatter.get_emoji(text) == [ + {"moominmamma", "/finmoji/128px/moominmamma-128.png", tag} + ] end test "it returns a nice empty result when no emojis are present" do From 49733f61763091514faa49493fdc20b795c08c1c Mon Sep 17 00:00:00 2001 From: Alex S Date: Mon, 1 Apr 2019 18:28:19 +0700 Subject: [PATCH 18/59] add docs folder to gitignore ref #770 --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 04c61ede7..774893b35 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,6 @@ erl_crash.dump # Editor config /.vscode/ + +# Prevent committing docs files +/priv/static/doc/* From 9b2188da7cab43a162d441294db7d3155e2eeab3 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 2 Apr 2019 15:44:56 +0700 Subject: [PATCH 19/59] refactoring of emoji tags config to use groups --- config/config.exs | 9 +++-- lib/pleroma/emoji.ex | 92 +++++++++++++++++++++++--------------------- test/emoji_test.exs | 75 ++++++++++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 47 deletions(-) diff --git a/config/config.exs b/config/config.exs index 245c7d268..4a22167b2 100644 --- a/config/config.exs +++ b/config/config.exs @@ -56,10 +56,11 @@ config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"], - custom_tag: "Custom", - finmoji_tag: "Finmoji", - emoji_tag: "Emoji", - custom_emoji_tag: "Custom" + groups: [ + # Place here groups, which have more priority on defaults. Example in `docs/config/custom_emoji.md` + Finmoji: "/finmoji/128px/*-128.png", + Custom: ["/emoji/*.png", "/emoji/custom/*.png"] + ] config :pleroma, :uri_schemes, valid_schemes: [ diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index ad3170f9a..b60d19e89 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -13,8 +13,14 @@ defmodule Pleroma.Emoji do This GenServer stores in an ETS table the list of the loaded emojis, and also allows to reload the list at runtime. """ use GenServer + + @type pattern :: Regex.t() | module() | String.t() + @type patterns :: pattern | [pattern] + @type group_patterns :: keyword(patterns) + @ets __MODULE__.Ets @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}] + @groups Application.get_env(:pleroma, :emoji)[:groups] @doc false def start_link do @@ -73,13 +79,14 @@ def code_change(_old_vsn, state, _extra) do end defp load do + finmoji_enabled = Keyword.get(Application.get_env(:pleroma, :instance), :finmoji_enabled) + shortcode_globs = Keyword.get(Application.get_env(:pleroma, :emoji, []), :shortcode_globs, []) + emojis = - (load_finmoji(Keyword.get(Application.get_env(:pleroma, :instance), :finmoji_enabled)) ++ + (load_finmoji(finmoji_enabled) ++ load_from_file("config/emoji.txt") ++ load_from_file("config/custom_emoji.txt") ++ - load_from_globs( - Keyword.get(Application.get_env(:pleroma, :emoji, []), :shortcode_globs, []) - )) + load_from_globs(shortcode_globs)) |> Enum.reject(fn value -> value == nil end) true = :ets.insert(@ets, emojis) @@ -151,11 +158,12 @@ defp load do "white_nights", "woollysocks" ] - defp load_finmoji(true) do - tag = Application.get_env(:pleroma, :emoji)[:finmoji_tag] + defp load_finmoji(true) do Enum.map(@finmoji, fn finmoji -> - {finmoji, "/finmoji/128px/#{finmoji}-128.png", tag} + file_name = "/finmoji/128px/#{finmoji}-128.png" + group = match_extra(@groups, file_name) + {finmoji, file_name, to_string(group)} end) end @@ -170,11 +178,6 @@ defp load_from_file(file) do end defp load_from_file_stream(stream) do - default_tag = - stream.path - |> Path.basename(".txt") - |> get_default_tag() - stream |> Stream.map(&String.trim/1) |> Stream.map(fn line -> @@ -183,7 +186,7 @@ defp load_from_file_stream(stream) do {name, file, tags} [name, file] -> - {name, file, default_tag} + {name, file, to_string(match_extra(@groups, file))} _ -> nil @@ -192,48 +195,51 @@ defp load_from_file_stream(stream) do |> Enum.to_list() end - @spec get_default_tag(String.t()) :: String.t() - defp get_default_tag(file_name) when file_name in ["emoji", "custom_emoji"] do - Keyword.get( - Application.get_env(:pleroma, :emoji), - String.to_existing_atom(file_name <> "_tag") - ) - end - - defp get_default_tag(_), do: Application.get_env(:pleroma, :emoji)[:custom_tag] - defp load_from_globs(globs) do static_path = Path.join(:code.priv_dir(:pleroma), "static") paths = Enum.map(globs, fn glob -> - static_part = - Path.dirname(glob) - |> String.replace_trailing("**", "") - Path.join(static_path, glob) |> Path.wildcard() - |> Enum.map(fn path -> - custom_folder = - path - |> Path.relative_to(Path.join(static_path, static_part)) - |> Path.dirname() - - [path, custom_folder] - end) end) |> Enum.concat() - Enum.map(paths, fn [path, custom_folder] -> - tag = - case custom_folder do - "." -> Keyword.get(Application.get_env(:pleroma, :emoji), :custom_tag) - tag -> tag - end - + Enum.map(paths, fn path -> + tag = match_extra(@groups, Path.join("/", Path.relative_to(path, static_path))) shortcode = Path.basename(path, Path.extname(path)) external_path = Path.join("/", Path.relative_to(path, static_path)) - {shortcode, external_path, tag} + {shortcode, external_path, to_string(tag)} + end) + end + + @doc """ + Finds a matching group for the given extra filename + """ + @spec match_extra(group_patterns(), String.t()) :: atom() | nil + def match_extra(group_patterns, filename) do + match_group_patterns(group_patterns, fn pattern -> + case pattern do + %Regex{} = regex -> Regex.match?(regex, filename) + string when is_binary(string) -> filename == string + end + end) + end + + defp match_group_patterns(group_patterns, matcher) do + Enum.find_value(group_patterns, fn {group, patterns} -> + patterns = + patterns + |> List.wrap() + |> Enum.map(fn pattern -> + if String.contains?(pattern, "*") do + ~r(#{String.replace(pattern, "*", ".*")}) + else + pattern + end + end) + + Enum.any?(patterns, matcher) && group end) end end diff --git a/test/emoji_test.exs b/test/emoji_test.exs index a90213d7d..cb1d62d00 100644 --- a/test/emoji_test.exs +++ b/test/emoji_test.exs @@ -28,4 +28,79 @@ test "random emoji", %{emoji_list: emoji_list} do assert is_binary(tags) end end + + describe "match_extra/2" do + setup do + groups = [ + "list of files": ["/emoji/custom/first_file.png", "/emoji/custom/second_file.png"], + "wildcard folder": "/emoji/custom/*/file.png", + "wildcard files": "/emoji/custom/folder/*.png", + "special file": "/emoji/custom/special.png" + ] + + {:ok, groups: groups} + end + + test "config for list of files", %{groups: groups} do + group = + groups + |> Emoji.match_extra("/emoji/custom/first_file.png") + |> to_string() + + assert group == "list of files" + end + + test "config with wildcard folder", %{groups: groups} do + group = + groups + |> Emoji.match_extra("/emoji/custom/some_folder/file.png") + |> to_string() + + assert group == "wildcard folder" + end + + test "config with wildcard folder and subfolders", %{groups: groups} do + group = + groups + |> Emoji.match_extra("/emoji/custom/some_folder/another_folder/file.png") + |> to_string() + + assert group == "wildcard folder" + end + + test "config with wildcard files", %{groups: groups} do + group = + groups + |> Emoji.match_extra("/emoji/custom/folder/some_file.png") + |> to_string() + + assert group == "wildcard files" + end + + test "config with wildcard files and subfolders", %{groups: groups} do + group = + groups + |> Emoji.match_extra("/emoji/custom/folder/another_folder/some_file.png") + |> to_string() + + assert group == "wildcard files" + end + + test "config for special file", %{groups: groups} do + group = + groups + |> Emoji.match_extra("/emoji/custom/special.png") + |> to_string() + + assert group == "special file" + end + + test "no mathing returns nil", %{groups: groups} do + group = + groups + |> Emoji.match_extra("/emoji/some_undefined.png") + + refute group + end + end end From 851c5bf0936fbc58bf509f79531e6cdc070efde5 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 2 Apr 2019 15:57:57 +0700 Subject: [PATCH 20/59] updating custom_emoji docs --- docs/config/custom_emoji.md | 41 +++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/docs/config/custom_emoji.md b/docs/config/custom_emoji.md index e47a75c8e..d37220a72 100644 --- a/docs/config/custom_emoji.md +++ b/docs/config/custom_emoji.md @@ -18,21 +18,36 @@ foo, /emoji/custom/foo.png The files should be PNG (APNG is okay with `.png` for `image/png` Content-type) and under 50kb for compatibility with mastodon. -# Emoji tags - -Changing default tags: - -* For `Finmoji`, `emoji.txt` and `custom_emoji.txt` are added default tags, which can be configured in the `config.exs`: -* For emoji loaded from globs: - - `priv/static/emoji/custom/*.png` - `custom_tag`, can be configured in `config.exs` - - `priv/static/emoji/custom/TagName/*.png` - folder (`TagName`) is used as tag - +# Emoji tags (groups) +Default tags are set in `config.exs`. ``` config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"], - custom_tag: "Custom", # Default tag for emoji in `priv/static/emoji/custom` path - finmoji_tag: "Finmoji", # Default tag for Finmoji - emoji_tag: "Emoji", # Default tag for emoji.txt - custom_emoji_tag: "Custom" # Default tag for custom_emoji.txt + groups: [ + Finmoji: "/finmoji/128px/*-128.png", + Custom: ["/emoji/*.png", "/emoji/custom/*.png"] + ] ``` + +Order of the `groups` matters, so to override default tags just put your group on the top of the list. E.g: +``` +config :pleroma, :emoji, + shortcode_globs: ["/emoji/custom/**/*.png"], + groups: [ + "Finmoji special": "/finmoji/128px/a_trusted_friend-128.png", # special file + "Cirno": "/emoji/custom/cirno*.png", # png files in /emoji/custom/ which start with `cirno` + "Special group": "/emoji/custom/special_folder/*.png", # png files in /emoji/custom/special_folder/ + "Another group": "/emoji/custom/special_folder/*/.png", # png files in /emoji/custom/special_folder/ subfolders + Finmoji: "/finmoji/128px/*-128.png", + Custom: ["/emoji/*.png", "/emoji/custom/*.png"] + ] +``` + +Priority of tag assign in emoji.txt and custom.txt: + +`tag in file > special group setting in config.exs > default setting in config.exs` + +Priority for globs: + +`special group setting in config.exs > default setting in config.exs` From 08d64b977f74abb7cb42bf985116eba91d9a6166 Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 2 Apr 2019 16:13:34 +0700 Subject: [PATCH 21/59] little changes and typos --- config/config.exs | 2 +- docs/config/custom_emoji.md | 4 ++-- lib/pleroma/emoji.ex | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/config/config.exs b/config/config.exs index 4a22167b2..139ec0ace 100644 --- a/config/config.exs +++ b/config/config.exs @@ -57,7 +57,7 @@ config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"], groups: [ - # Place here groups, which have more priority on defaults. Example in `docs/config/custom_emoji.md` + # Put groups that have higher priority than defaults here. Example in `docs/config/custom_emoji.md` Finmoji: "/finmoji/128px/*-128.png", Custom: ["/emoji/*.png", "/emoji/custom/*.png"] ] diff --git a/docs/config/custom_emoji.md b/docs/config/custom_emoji.md index d37220a72..49a451fcc 100644 --- a/docs/config/custom_emoji.md +++ b/docs/config/custom_emoji.md @@ -30,7 +30,7 @@ config :pleroma, :emoji, ] ``` -Order of the `groups` matters, so to override default tags just put your group on the top of the list. E.g: +Order of the `groups` matters, so to override default tags just put your group on top of the list. E.g: ``` config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"], @@ -44,7 +44,7 @@ config :pleroma, :emoji, ] ``` -Priority of tag assign in emoji.txt and custom.txt: +Priority of tags assigns in emoji.txt and custom.txt: `tag in file > special group setting in config.exs > default setting in config.exs` diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index b60d19e89..7a60f3961 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -15,8 +15,8 @@ defmodule Pleroma.Emoji do use GenServer @type pattern :: Regex.t() | module() | String.t() - @type patterns :: pattern | [pattern] - @type group_patterns :: keyword(patterns) + @type patterns :: pattern() | [pattern()] + @type group_patterns :: keyword(patterns()) @ets __MODULE__.Ets @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}] @@ -80,7 +80,7 @@ def code_change(_old_vsn, state, _extra) do defp load do finmoji_enabled = Keyword.get(Application.get_env(:pleroma, :instance), :finmoji_enabled) - shortcode_globs = Keyword.get(Application.get_env(:pleroma, :emoji, []), :shortcode_globs, []) + shortcode_globs = Application.get_env(:pleroma, :emoji)[:shortcode_globs] || [] emojis = (load_finmoji(finmoji_enabled) ++ From 484162c18774ff28842a517ae0afcaaf824e12bf Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 2 Apr 2019 16:26:40 +0700 Subject: [PATCH 22/59] test fix --- test/formatter_test.exs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 38430e170..e74985c4e 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -271,10 +271,8 @@ test "it does not add XSS emoji" do test "it returns the emoji used in the text" do text = "I love :moominmamma:" - tag = Keyword.get(Application.get_env(:pleroma, :emoji), :finmoji_tag) - assert Formatter.get_emoji(text) == [ - {"moominmamma", "/finmoji/128px/moominmamma-128.png", tag} + {"moominmamma", "/finmoji/128px/moominmamma-128.png", "Finmoji"} ] end From 3465b7ba9ad0e26128f18fd4e36aece767ba269e Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 2 Apr 2019 20:32:37 +0700 Subject: [PATCH 23/59] syntax highlighting --- docs/config/custom_emoji.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/config/custom_emoji.md b/docs/config/custom_emoji.md index 49a451fcc..96fcb2fc6 100644 --- a/docs/config/custom_emoji.md +++ b/docs/config/custom_emoji.md @@ -21,7 +21,7 @@ The files should be PNG (APNG is okay with `.png` for `image/png` Content-type) # Emoji tags (groups) Default tags are set in `config.exs`. -``` +```elixir config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"], groups: [ @@ -31,7 +31,7 @@ config :pleroma, :emoji, ``` Order of the `groups` matters, so to override default tags just put your group on top of the list. E.g: -``` +```elixir config :pleroma, :emoji, shortcode_globs: ["/emoji/custom/**/*.png"], groups: [ From d140738edf75467420b35c500716cf89de66548d Mon Sep 17 00:00:00 2001 From: Alex S Date: Tue, 2 Apr 2019 20:35:41 +0700 Subject: [PATCH 24/59] second level of headertext change in doc --- docs/config/custom_emoji.md | 2 +- lib/pleroma/emoji.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/config/custom_emoji.md b/docs/config/custom_emoji.md index 96fcb2fc6..419a7d0e2 100644 --- a/docs/config/custom_emoji.md +++ b/docs/config/custom_emoji.md @@ -18,7 +18,7 @@ foo, /emoji/custom/foo.png The files should be PNG (APNG is okay with `.png` for `image/png` Content-type) and under 50kb for compatibility with mastodon. -# Emoji tags (groups) +## Emoji tags (groups) Default tags are set in `config.exs`. ```elixir diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 7a60f3961..87c7f2cec 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -214,7 +214,7 @@ defp load_from_globs(globs) do end @doc """ - Finds a matching group for the given extra filename + Finds a matching group for the given emoji filename """ @spec match_extra(group_patterns(), String.t()) :: atom() | nil def match_extra(group_patterns, filename) do From cfa6e7289f5cfdb1fce17eb89bc0513ff624480d Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 4 Apr 2019 16:10:43 +0700 Subject: [PATCH 25/59] Improve Transmogrifier.upgrade_user_from_ap_id/2 --- config/config.exs | 3 ++- docs/config.md | 6 +++-- .../web/activity_pub/transmogrifier.ex | 26 ++++++------------- test/web/activity_pub/transmogrifier_test.exs | 3 --- 4 files changed, 14 insertions(+), 24 deletions(-) diff --git a/config/config.exs b/config/config.exs index dccf7b263..d68edafcb 100644 --- a/config/config.exs +++ b/config/config.exs @@ -351,7 +351,8 @@ config :pleroma_job_queue, :queues, federator_incoming: 50, federator_outgoing: 50, - mailer: 10 + mailer: 10, + transmogrifier: 20 config :pleroma, :fetch_initial_posts, enabled: false, diff --git a/docs/config.md b/docs/config.md index 97a0e6ffa..dd3cc3727 100644 --- a/docs/config.md +++ b/docs/config.md @@ -200,14 +200,14 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i - `port` * `url` - a list containing the configuration for generating urls, accepts - `host` - the host without the scheme and a post (e.g `example.com`, not `https://example.com:2020`) - - `scheme` - e.g `http`, `https` + - `scheme` - e.g `http`, `https` - `port` - `path` **Important note**: if you modify anything inside these lists, default `config.exs` values will be overwritten, which may result in breakage, to make sure this does not happen please copy the default value for the list from `config.exs` and modify/add only what you need -Example: +Example: ```elixir config :pleroma, Pleroma.Web.Endpoint, url: [host: "example.com", port: 2020, scheme: "https"], @@ -296,9 +296,11 @@ curl "http://localhost:4000/api/pleroma/admin/invite_token?admin_token=somerando [Pleroma Job Queue](https://git.pleroma.social/pleroma/pleroma_job_queue) configuration: a list of queues with maximum concurrent jobs. Pleroma has the following queues: + * `federator_outgoing` - Outgoing federation * `federator_incoming` - Incoming federation * `mailer` - Email sender, see [`Pleroma.Mailer`](#pleroma-mailer) +* `transmogrifier` - Transmogrifier Example: diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index f733ae7e1..593ae3188 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -954,7 +954,7 @@ defp strip_internal_tags(%{"tag" => tags} = object) do defp strip_internal_tags(object), do: object - defp user_upgrade_task(user) do + def perform(:user_upgrade, user) do # we pass a fake user so that the followers collection is stripped away old_follower_address = User.ap_followers(%User{nickname: user.nickname}) @@ -999,28 +999,18 @@ defp user_upgrade_task(user) do Repo.update_all(q, []) end - def upgrade_user_from_ap_id(ap_id, async \\ true) do + def upgrade_user_from_ap_id(ap_id) do with %User{local: false} = user <- User.get_by_ap_id(ap_id), - {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id) do - already_ap = User.ap_enabled?(user) - - {:ok, user} = - User.upgrade_changeset(user, data) - |> Repo.update() - - if !already_ap do - # This could potentially take a long time, do it in the background - if async do - Task.start(fn -> - user_upgrade_task(user) - end) - else - user_upgrade_task(user) - end + {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id), + already_ap <- User.ap_enabled?(user), + {:ok, user} <- user |> User.upgrade_changeset(data) |> User.update_and_set_cache() do + unless already_ap do + PleromaJobQueue.enqueue(:transmogrifier, __MODULE__, [:user_upgrade, user]) end {:ok, user} else + %User{} = user -> {:ok, user} e -> e end end diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 62b973c4f..47cffe257 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -1028,9 +1028,6 @@ test "it upgrades a user to activitypub" do assert user.info.note_count == 1 assert user.follower_address == "https://niu.moe/users/rye/followers" - # Wait for the background task - :timer.sleep(1000) - user = User.get_by_id(user.id) assert user.info.note_count == 1 From b655a8ea839d19443f44ff5b300a069d88ec7d58 Mon Sep 17 00:00:00 2001 From: href Date: Wed, 6 Feb 2019 10:33:05 +0100 Subject: [PATCH 26/59] Add recon --- mix.exs | 9 ++++++++- mix.lock | 8 ++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 88da6332a..8cb389248 100644 --- a/mix.exs +++ b/mix.exs @@ -94,7 +94,14 @@ defp deps do {:auto_linker, git: "https://git.pleroma.social/pleroma/auto_linker.git", ref: "479dd343f4e563ff91215c8275f3b5c67e032850"}, - {:pleroma_job_queue, "~> 0.2.0"} + {:pleroma_job_queue, "~> 0.2.0"}, + {:telemetry, "~> 0.3"}, + {:prometheus_ex, "~> 3.0"}, + {:prometheus_plugs, "~> 1.1"}, + {:prometheus_phoenix, "~> 1.2"}, + {:prometheus_ecto, "~> 1.4"}, + {:prometheus_process_collector, "~> 1.4"}, + {:recon, github: "ferd/recon"} ] end diff --git a/mix.lock b/mix.lock index 9c454446a..0ece4b353 100644 --- a/mix.lock +++ b/mix.lock @@ -1,4 +1,5 @@ %{ + "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm"}, "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "479dd343f4e563ff91215c8275f3b5c67e032850", [ref: "479dd343f4e563ff91215c8275f3b5c67e032850"]}, "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"}, "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"}, @@ -57,7 +58,14 @@ "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"}, "postgrex": {:hex, :postgrex, "0.14.1", "63247d4a5ad6b9de57a0bac5d807e1c32d41e39c04b8a4156a26c63bcd8a2e49", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"}, + "prometheus": {:hex, :prometheus, "4.2.2", "a830e77b79dc6d28183f4db050a7cac926a6c58f1872f9ef94a35cd989aceef8", [:mix, :rebar3], [], "hexpm"}, + "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, + "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, + "prometheus_phoenix": {:hex, :prometheus_phoenix, "1.2.1", "964a74dfbc055f781d3a75631e06ce3816a2913976d1df7830283aa3118a797a", [:mix], [{:phoenix, "~> 1.3", [hex: :phoenix, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"}, + "prometheus_plugs": {:hex, :prometheus_plugs, "1.1.5", "25933d48f8af3a5941dd7b621c889749894d8a1082a6ff7c67cc99dec26377c5", [:mix], [{:accept, "~> 0.1", [hex: :accept, repo: "hexpm", optional: false]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}, {:prometheus_process_collector, "~> 1.1", [hex: :prometheus_process_collector, repo: "hexpm", optional: true]}], "hexpm"}, + "prometheus_process_collector": {:hex, :prometheus_process_collector, "1.4.0", "6dbd39e3165b9ef1c94a7a820e9ffe08479f949dcdd431ed4aaea7b250eebfde", [:rebar3], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, + "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", []}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, From bc3618a38d2e37254e27f723d3dd61679eca9be5 Mon Sep 17 00:00:00 2001 From: href Date: Wed, 30 Jan 2019 16:32:30 +0100 Subject: [PATCH 27/59] Set up telemetry and prometheus --- config/config.exs | 5 +++++ lib/pleroma/application.ex | 8 ++++++++ lib/pleroma/repo.ex | 4 ++++ lib/pleroma/web/endpoint.ex | 20 ++++++++++++++++++++ 4 files changed, 37 insertions(+) diff --git a/config/config.exs b/config/config.exs index dccf7b263..1e086f44c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -8,6 +8,10 @@ # General application configuration config :pleroma, ecto_repos: [Pleroma.Repo] +config :pleroma, Pleroma.Repo, + types: Pleroma.PostgresTypes, + loggers: [Pleroma.Repo.Instrumenter, Ecto.LogEntry] + config :pleroma, Pleroma.Captcha, enabled: false, seconds_valid: 60, @@ -87,6 +91,7 @@ # Configures the endpoint config :pleroma, Pleroma.Web.Endpoint, + instrumenters: [Pleroma.Web.Endpoint.Instrumenter], url: [host: "localhost"], http: [ dispatch: [ diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 782d1d589..03dcbab1a 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -25,6 +25,7 @@ def start(_type, _args) do import Cachex.Spec Pleroma.Config.DeprecationWarnings.warn() + setup_instrumenters() # Define workers and child supervisors to be supervised children = @@ -140,6 +141,13 @@ def enabled_hackney_pools do end end + defp setup_instrumenters() do + Pleroma.Web.Endpoint.MetricsExporter.setup() + Pleroma.Web.Endpoint.PipelineInstrumenter.setup() + Pleroma.Web.Endpoint.Instrumenter.setup() + Pleroma.Repo.Instrumenter.setup() + end + if Mix.env() == :test do defp streamer_child, do: [] defp chat_child, do: [] diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index 4af1bde56..aa5d427ae 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -8,6 +8,10 @@ defmodule Pleroma.Repo do adapter: Ecto.Adapters.Postgres, migration_timestamps: [type: :naive_datetime_usec] + defmodule Instrumenter do + use Prometheus.EctoInstrumenter + end + @doc """ Dynamically loads the repository url from the DATABASE_URL environment variable. diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index fa2d1cbe7..6d9528c86 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -70,6 +70,26 @@ defmodule Pleroma.Web.Endpoint do extra: "SameSite=Strict" ) + # Note: the plug and its configuration is compile-time this can't be upstreamed yet + if proxies = Pleroma.Config.get([__MODULE__, :reverse_proxies]) do + plug(RemoteIp, proxies: proxies) + end + + defmodule Instrumenter do + use Prometheus.PhoenixInstrumenter + end + + defmodule PipelineInstrumenter do + use Prometheus.PlugPipelineInstrumenter + end + + defmodule MetricsExporter do + use Prometheus.PlugExporter + end + + plug(PipelineInstrumenter) + plug(MetricsExporter) + plug(Pleroma.Web.Router) @doc """ From 0b5c818cb78b8c23fb2ba7ef372d0688ea9f36b7 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 25 Mar 2019 15:29:04 +0700 Subject: [PATCH 28/59] [#1] fix telemetry --- config/config.exs | 2 +- lib/pleroma/application.ex | 25 ++++++++++++++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/config/config.exs b/config/config.exs index 1e086f44c..4fd63f99d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -10,7 +10,7 @@ config :pleroma, Pleroma.Repo, types: Pleroma.PostgresTypes, - loggers: [Pleroma.Repo.Instrumenter, Ecto.LogEntry] + telemetry_event: [Pleroma.Repo.Instrumenter] config :pleroma, Pleroma.Captcha, enabled: false, diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 03dcbab1a..c3f3126c6 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -127,6 +127,24 @@ def start(_type, _args) do Supervisor.start_link(children, opts) end + defp setup_instrumenters() do + require Prometheus.Registry + + :ok = + :telemetry.attach( + "prometheus-ecto", + [:pleroma, :repo, :query], + &Pleroma.Repo.Instrumenter.handle_event/4, + %{} + ) + + Prometheus.Registry.register_collector(:prometheus_process_collector) + Pleroma.Web.Endpoint.MetricsExporter.setup() + Pleroma.Web.Endpoint.PipelineInstrumenter.setup() + Pleroma.Web.Endpoint.Instrumenter.setup() + Pleroma.Repo.Instrumenter.setup() + end + def enabled_hackney_pools do [:media] ++ if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do @@ -141,13 +159,6 @@ def enabled_hackney_pools do end end - defp setup_instrumenters() do - Pleroma.Web.Endpoint.MetricsExporter.setup() - Pleroma.Web.Endpoint.PipelineInstrumenter.setup() - Pleroma.Web.Endpoint.Instrumenter.setup() - Pleroma.Repo.Instrumenter.setup() - end - if Mix.env() == :test do defp streamer_child, do: [] defp chat_child, do: [] From 5564cd421dfc706208df0b7447b0d692dffe052e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 4 Apr 2019 12:19:31 -0500 Subject: [PATCH 29/59] Document Prometheus --- docs/api/prometheus.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 docs/api/prometheus.md diff --git a/docs/api/prometheus.md b/docs/api/prometheus.md new file mode 100644 index 000000000..19c564e3c --- /dev/null +++ b/docs/api/prometheus.md @@ -0,0 +1,22 @@ +# Prometheus Metrics + +Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library. + +## `/api/pleroma/app_metrics` +### Exports Prometheus application metrics +* Method: `GET` +* Authentication: not required +* Params: none +* Response: JSON + +## Grafana +### Config example +The following is a config example to use with [Grafana](https://grafana.com) + +``` + - job_name: 'beam' + metrics_path: /api/pleroma/app_metrics + scheme: https + static_configs: + - targets: ['pleroma.soykaf.com'] +``` From 7e930559fece1a86891645333cc79a18f440ef1d Mon Sep 17 00:00:00 2001 From: href Date: Wed, 30 Jan 2019 16:44:38 +0100 Subject: [PATCH 30/59] Serve metrics at `/api/pleroma/app_metrics` --- config/config.exs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/config.exs b/config/config.exs index 4fd63f99d..ebaf1aec5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -353,6 +353,7 @@ initial_timeout: 30, max_retries: 5 +<<<<<<< HEAD config :pleroma_job_queue, :queues, federator_incoming: 50, federator_outgoing: 50, @@ -385,6 +386,8 @@ config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Sendmail +config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics" + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" From 7222afe01b13586018b481172731309587191338 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 4 Apr 2019 12:29:10 -0500 Subject: [PATCH 31/59] Clean merge crumbs --- config/config.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index ebaf1aec5..b19b36b22 100644 --- a/config/config.exs +++ b/config/config.exs @@ -353,7 +353,6 @@ initial_timeout: 30, max_retries: 5 -<<<<<<< HEAD config :pleroma_job_queue, :queues, federator_incoming: 50, federator_outgoing: 50, From 69038887b2930072356aa00841b889c59518e264 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 4 Apr 2019 12:36:57 -0500 Subject: [PATCH 32/59] Code readability tweak --- lib/pleroma/application.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index c3f3126c6..1fc3fb728 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -127,7 +127,7 @@ def start(_type, _args) do Supervisor.start_link(children, opts) end - defp setup_instrumenters() do + defp setup_instrumenters do require Prometheus.Registry :ok = From f7cd9131d4aa0da3c4c0174acc56ce1bbdbd284c Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Thu, 4 Apr 2019 22:41:03 +0300 Subject: [PATCH 33/59] [#923] OAuth consumer controller tests. Misc. improvements. --- lib/pleroma/web/oauth/oauth_controller.ex | 4 + .../templates/o_auth/o_auth/register.html.eex | 1 + .../web/templates/o_auth/o_auth/show.html.eex | 2 +- test/support/factory.ex | 16 + test/web/oauth/oauth_controller_test.exs | 329 +++++++++++++++++- 5 files changed, 344 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index 1b467e983..2dcaaabc1 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -253,6 +253,7 @@ def callback(conn, params) do auth_params = %{ "client_id" => params["client_id"], "redirect_uri" => params["redirect_uri"], + "state" => params["state"], "scopes" => oauth_scopes(params, nil) } @@ -289,6 +290,7 @@ def registration_details(conn, params) do render(conn, "register.html", %{ client_id: params["client_id"], redirect_uri: params["redirect_uri"], + state: params["state"], scopes: oauth_scopes(params, []), nickname: params["nickname"], email: params["email"] @@ -313,6 +315,8 @@ def register(conn, %{"op" => "connect"} = params) do ) else _ -> + params = Map.delete(params, "password") + conn |> put_flash(:error, "Unknown error, please try again.") |> redirect(to: o_auth_path(conn, :registration_details, params)) diff --git a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex index f4547170c..2e806e5fb 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex @@ -44,5 +44,6 @@ please provide the details below.

<%= hidden_input f, :client_id, value: @client_id %> <%= hidden_input f, :redirect_uri, value: @redirect_uri %> <%= hidden_input f, :scope, value: Enum.join(@scopes, " ") %> +<%= hidden_input f, :state, value: @state %> <% end %> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index e6cf1db45..0144675ab 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -22,7 +22,7 @@ <%= hidden_input f, :client_id, value: @client_id %> <%= hidden_input f, :response_type, value: @response_type %> <%= hidden_input f, :redirect_uri, value: @redirect_uri %> -<%= hidden_input f, :state, value: @state%> +<%= hidden_input f, :state, value: @state %> <%= submit "Authorize" %> <% end %> diff --git a/test/support/factory.ex b/test/support/factory.ex index e1a08315a..67953931b 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -257,4 +257,20 @@ def notification_factory do user: build(:user) } end + + def registration_factory do + user = insert(:user) + + %Pleroma.Registration{ + user: user, + provider: "twitter", + uid: "171799000", + info: %{ + "name" => "John Doe", + "email" => "john@doe.com", + "nickname" => "johndoe", + "description" => "My bio" + } + } + end end diff --git a/test/web/oauth/oauth_controller_test.exs b/test/web/oauth/oauth_controller_test.exs index a9a0b9ed4..e13f4700d 100644 --- a/test/web/oauth/oauth_controller_test.exs +++ b/test/web/oauth/oauth_controller_test.exs @@ -5,24 +5,339 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do use Pleroma.Web.ConnCase import Pleroma.Factory + import Mock + alias Pleroma.Registration alias Pleroma.Repo alias Pleroma.Web.OAuth.Authorization alias Pleroma.Web.OAuth.Token + @session_opts [ + store: :cookie, + key: "_test", + signing_salt: "cooldude" + ] + + describe "in OAuth consumer mode, " do + setup do + oauth_consumer_enabled_path = [:auth, :oauth_consumer_enabled] + oauth_consumer_strategies_path = [:auth, :oauth_consumer_strategies] + oauth_consumer_enabled = Pleroma.Config.get(oauth_consumer_enabled_path) + oauth_consumer_strategies = Pleroma.Config.get(oauth_consumer_strategies_path) + + Pleroma.Config.put(oauth_consumer_enabled_path, true) + Pleroma.Config.put(oauth_consumer_strategies_path, ~w(twitter facebook)) + + on_exit(fn -> + Pleroma.Config.put(oauth_consumer_enabled_path, oauth_consumer_enabled) + Pleroma.Config.put(oauth_consumer_strategies_path, oauth_consumer_strategies) + end) + + [ + app: insert(:oauth_app), + conn: + build_conn() + |> Plug.Session.call(Plug.Session.init(@session_opts)) + |> fetch_session() + ] + end + + test "GET /oauth/authorize also renders OAuth consumer form", %{ + app: app, + conn: conn + } do + conn = + get( + conn, + "/oauth/authorize", + %{ + "response_type" => "code", + "client_id" => app.client_id, + "redirect_uri" => app.redirect_uris, + "scope" => "read" + } + ) + + assert response = html_response(conn, 200) + assert response =~ "Sign in with Twitter" + assert response =~ o_auth_path(conn, :prepare_request) + end + + test "GET /oauth/prepare_request encodes parameters as `state` and redirects", %{ + app: app, + conn: conn + } do + conn = + get( + conn, + "/oauth/prepare_request", + %{ + "provider" => "twitter", + "scope" => app.scopes, + "client_id" => app.client_id, + "redirect_uri" => app.redirect_uris, + "state" => "a_state" + } + ) + + assert response = html_response(conn, 302) + redirected_to = redirected_to(conn) + [state] = Regex.run(~r/(?<=state=).*?(?=\Z|&)/, redirected_to) + state = URI.decode(state) + assert {:ok, state_params} = Poison.decode(state) + + expected_scope_param = Enum.join(app.scopes, "+") + expected_client_id_param = app.client_id + expected_redirect_uri_param = app.redirect_uris + + assert %{ + "scope" => ^expected_scope_param, + "client_id" => ^expected_client_id_param, + "redirect_uri" => ^expected_redirect_uri_param, + "state" => "a_state" + } = state_params + end + + test "on authentication error, redirects to `redirect_uri`", %{app: app, conn: conn} do + state_params = %{ + "scope" => Enum.join(app.scopes, " "), + "client_id" => app.client_id, + "redirect_uri" => app.redirect_uris, + "state" => "" + } + + conn = + conn + |> assign(:ueberauth_failure, %{errors: [%{message: "unknown error"}]}) + |> get( + "/oauth/twitter/callback", + %{ + "oauth_token" => "G-5a3AAAAAAAwMH9AAABaektfSM", + "oauth_verifier" => "QZl8vUqNvXMTKpdmUnGejJxuHG75WWWs", + "provider" => "twitter", + "state" => Poison.encode!(state_params) + } + ) + + assert response = html_response(conn, 302) + assert redirected_to(conn) == app.redirect_uris + end + + test "with user-bound registration, GET /oauth//callback redirects to `redirect_uri` with `code`", + %{app: app, conn: conn} do + registration = insert(:registration) + + state_params = %{ + "scope" => Enum.join(app.scopes, " "), + "client_id" => app.client_id, + "redirect_uri" => app.redirect_uris, + "state" => "" + } + + with_mock Pleroma.Web.Auth.Authenticator, + get_registration: fn _, _ -> {:ok, registration} end do + conn = + get( + conn, + "/oauth/twitter/callback", + %{ + "oauth_token" => "G-5a3AAAAAAAwMH9AAABaektfSM", + "oauth_verifier" => "QZl8vUqNvXMTKpdmUnGejJxuHG75WWWs", + "provider" => "twitter", + "state" => Poison.encode!(state_params) + } + ) + + assert response = html_response(conn, 302) + assert redirected_to(conn) =~ ~r/#{app.redirect_uris}\?code=.+/ + end + end + + test "with user-unbound registration, GET /oauth//callback redirects to registration_details page", + %{app: app, conn: conn} do + registration = insert(:registration, user: nil) + + state_params = %{ + "scope" => "read", + "client_id" => app.client_id, + "redirect_uri" => app.redirect_uris, + "state" => "a_state" + } + + with_mock Pleroma.Web.Auth.Authenticator, + get_registration: fn _, _ -> {:ok, registration} end do + conn = + get( + conn, + "/oauth/twitter/callback", + %{ + "oauth_token" => "G-5a3AAAAAAAwMH9AAABaektfSM", + "oauth_verifier" => "QZl8vUqNvXMTKpdmUnGejJxuHG75WWWs", + "provider" => "twitter", + "state" => Poison.encode!(state_params) + } + ) + + expected_redirect_params = + state_params + |> Map.delete("scope") + |> Map.merge(%{ + "scopes" => ["read"], + "email" => Registration.email(registration), + "nickname" => Registration.nickname(registration) + }) + + assert response = html_response(conn, 302) + + assert redirected_to(conn) == + o_auth_path(conn, :registration_details, expected_redirect_params) + end + end + + test "GET /oauth/registration_details renders registration details form", %{ + app: app, + conn: conn + } do + conn = + get( + conn, + "/oauth/registration_details", + %{ + "scopes" => app.scopes, + "client_id" => app.client_id, + "redirect_uri" => app.redirect_uris, + "state" => "a_state", + "nickname" => nil, + "email" => "john@doe.com" + } + ) + + assert response = html_response(conn, 200) + assert response =~ ~r/name="op" type="submit" value="register"/ + assert response =~ ~r/name="op" type="submit" value="connect"/ + end + + test "with valid params, POST /oauth/register?op=register redirects to `redirect_uri` with `code`", + %{ + app: app, + conn: conn + } do + registration = insert(:registration, user: nil, info: %{"nickname" => nil, "email" => nil}) + + conn = + conn + |> put_session(:registration_id, registration.id) + |> post( + "/oauth/register", + %{ + "op" => "register", + "scopes" => app.scopes, + "client_id" => app.client_id, + "redirect_uri" => app.redirect_uris, + "state" => "a_state", + "nickname" => "availablenick", + "email" => "available@email.com" + } + ) + + assert response = html_response(conn, 302) + assert redirected_to(conn) =~ ~r/#{app.redirect_uris}\?code=.+/ + end + + test "with invalid params, POST /oauth/register?op=register redirects to registration_details page", + %{ + app: app, + conn: conn + } do + another_user = insert(:user) + registration = insert(:registration, user: nil, info: %{"nickname" => nil, "email" => nil}) + + params = %{ + "op" => "register", + "scopes" => app.scopes, + "client_id" => app.client_id, + "redirect_uri" => app.redirect_uris, + "state" => "a_state", + "nickname" => another_user.nickname, + "email" => another_user.email + } + + conn = + conn + |> put_session(:registration_id, registration.id) + |> post("/oauth/register", params) + + assert response = html_response(conn, 302) + + assert redirected_to(conn) == + o_auth_path(conn, :registration_details, params) + end + + test "with valid params, POST /oauth/register?op=connect redirects to `redirect_uri` with `code`", + %{ + app: app, + conn: conn + } do + user = insert(:user, password_hash: Comeonin.Pbkdf2.hashpwsalt("testpassword")) + registration = insert(:registration, user: nil) + + conn = + conn + |> put_session(:registration_id, registration.id) + |> post( + "/oauth/register", + %{ + "op" => "connect", + "scopes" => app.scopes, + "client_id" => app.client_id, + "redirect_uri" => app.redirect_uris, + "state" => "a_state", + "auth_name" => user.nickname, + "password" => "testpassword" + } + ) + + assert response = html_response(conn, 302) + assert redirected_to(conn) =~ ~r/#{app.redirect_uris}\?code=.+/ + end + + test "with invalid params, POST /oauth/register?op=connect redirects to registration_details page", + %{ + app: app, + conn: conn + } do + user = insert(:user) + registration = insert(:registration, user: nil) + + params = %{ + "op" => "connect", + "scopes" => app.scopes, + "client_id" => app.client_id, + "redirect_uri" => app.redirect_uris, + "state" => "a_state", + "auth_name" => user.nickname, + "password" => "wrong password" + } + + conn = + conn + |> put_session(:registration_id, registration.id) + |> post("/oauth/register", params) + + assert response = html_response(conn, 302) + + assert redirected_to(conn) == + o_auth_path(conn, :registration_details, Map.delete(params, "password")) + end + end + describe "GET /oauth/authorize" do setup do - session_opts = [ - store: :cookie, - key: "_test", - signing_salt: "cooldude" - ] - [ app: insert(:oauth_app, redirect_uris: "https://redirect.url"), conn: build_conn() - |> Plug.Session.call(Plug.Session.init(session_opts)) + |> Plug.Session.call(Plug.Session.init(@session_opts)) |> fetch_session() ] end From 3e7f2bfc2f4769af3cedea3126fa0b3cab3f2b7b Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 5 Apr 2019 09:19:17 +0300 Subject: [PATCH 34/59] [#923] OAuthController#callback adjustments (with tests). --- lib/pleroma/web/oauth/oauth_controller.ex | 8 +------ test/web/oauth/oauth_controller_test.exs | 27 +++++++++++------------ 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index 2dcaaabc1..404728899 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -249,13 +249,7 @@ def callback(conn, params) do with {:ok, registration} <- Authenticator.get_registration(conn, params) do user = Repo.preload(registration, :user).user - - auth_params = %{ - "client_id" => params["client_id"], - "redirect_uri" => params["redirect_uri"], - "state" => params["state"], - "scopes" => oauth_scopes(params, nil) - } + auth_params = Map.take(params, ~w(client_id redirect_uri scope scopes state)) if user do create_authorization( diff --git a/test/web/oauth/oauth_controller_test.exs b/test/web/oauth/oauth_controller_test.exs index e13f4700d..75333f2d5 100644 --- a/test/web/oauth/oauth_controller_test.exs +++ b/test/web/oauth/oauth_controller_test.exs @@ -73,7 +73,7 @@ test "GET /oauth/prepare_request encodes parameters as `state` and redirects", % "/oauth/prepare_request", %{ "provider" => "twitter", - "scope" => app.scopes, + "scope" => "read follow", "client_id" => app.client_id, "redirect_uri" => app.redirect_uris, "state" => "a_state" @@ -81,21 +81,20 @@ test "GET /oauth/prepare_request encodes parameters as `state` and redirects", % ) assert response = html_response(conn, 302) - redirected_to = redirected_to(conn) - [state] = Regex.run(~r/(?<=state=).*?(?=\Z|&)/, redirected_to) - state = URI.decode(state) - assert {:ok, state_params} = Poison.decode(state) - expected_scope_param = Enum.join(app.scopes, "+") - expected_client_id_param = app.client_id - expected_redirect_uri_param = app.redirect_uris + redirect_query = URI.parse(redirected_to(conn)).query + assert %{"state" => state_param} = URI.decode_query(redirect_query) + assert {:ok, state_components} = Poison.decode(state_param) + + expected_client_id = app.client_id + expected_redirect_uri = app.redirect_uris assert %{ - "scope" => ^expected_scope_param, - "client_id" => ^expected_client_id_param, - "redirect_uri" => ^expected_redirect_uri_param, + "scope" => "read follow", + "client_id" => ^expected_client_id, + "redirect_uri" => ^expected_redirect_uri, "state" => "a_state" - } = state_params + } = state_components end test "on authentication error, redirects to `redirect_uri`", %{app: app, conn: conn} do @@ -158,7 +157,7 @@ test "with user-unbound registration, GET /oauth//callback redirects t registration = insert(:registration, user: nil) state_params = %{ - "scope" => "read", + "scope" => "read write", "client_id" => app.client_id, "redirect_uri" => app.redirect_uris, "state" => "a_state" @@ -182,7 +181,7 @@ test "with user-unbound registration, GET /oauth//callback redirects t state_params |> Map.delete("scope") |> Map.merge(%{ - "scopes" => ["read"], + "scope" => "read write", "email" => Registration.email(registration), "nickname" => Registration.nickname(registration) }) From 47a236f7537ad4366d07361d184c84f3912648f1 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 5 Apr 2019 15:12:02 +0300 Subject: [PATCH 35/59] [#923] OAuth consumer mode refactoring, new tests, tests adjustments, readme. --- config/config.exs | 4 +- docs/config.md | 55 +++++++ lib/pleroma/config.ex | 4 + lib/pleroma/web/endpoint.ex | 2 +- lib/pleroma/web/oauth/fallback_controller.ex | 17 ++- lib/pleroma/web/oauth/oauth_controller.ex | 136 +++++++++--------- .../templates/o_auth/o_auth/consumer.html.eex | 2 +- .../web/templates/o_auth/o_auth/show.html.eex | 2 +- test/registration_test.exs | 59 ++++++++ test/web/oauth/oauth_controller_test.exs | 112 +++++++-------- 10 files changed, 258 insertions(+), 135 deletions(-) create mode 100644 test/registration_test.exs diff --git a/config/config.exs b/config/config.exs index 9bc79f939..05b164273 100644 --- a/config/config.exs +++ b/config/config.exs @@ -397,9 +397,7 @@ base_path: "/oauth", providers: ueberauth_providers -config :pleroma, :auth, - oauth_consumer_strategies: oauth_consumer_strategies, - oauth_consumer_enabled: oauth_consumer_strategies != [] +config :pleroma, :auth, oauth_consumer_strategies: oauth_consumer_strategies config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Sendmail diff --git a/docs/config.md b/docs/config.md index 06d6fd757..36d7f1273 100644 --- a/docs/config.md +++ b/docs/config.md @@ -412,3 +412,58 @@ Pleroma account will be created with the same name as the LDAP user name. * `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator * `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication + +## :auth + +Authentication / authorization settings. + +* `oauth_consumer_strategies`: lists enabled OAuth consumer strategies; by default it's set by OAUTH_CONSUMER_STRATEGIES environment variable. + +OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.). +Implementation is based on Ueberauth; see the list of [available strategies](https://github.com/ueberauth/ueberauth/wiki/List-of-Strategies). + +Note: each strategy is shipped as a separate dependency; in order to get the strategies, run `OAUTH_CONSUMER_STRATEGIES="..." mix deps.get`, +e.g. `OAUTH_CONSUMER_STRATEGIES="twitter facebook google microsoft" mix deps.get`. +The server should also be started with `OAUTH_CONSUMER_STRATEGIES="..." mix phx.server` in case you enable any strategies. + +Note: each strategy requires separate setup (on external provider side and Pleroma side). Below are the guidelines on setting up most popular strategies. + +* For Twitter, [register an app](https://developer.twitter.com/en/apps), configure callback URL to https:///oauth/twitter/callback + +* For Facebook, [register an app](https://developers.facebook.com/apps), configure callback URL to https:///oauth/facebook/callback, enable Facebook Login service at https://developers.facebook.com/apps//fb-login/settings/ + +* For Google, [register an app](https://console.developers.google.com), configure callback URL to https:///oauth/google/callback + +* For Microsoft, [register an app](https://portal.azure.com), configure callback URL to https:///oauth/microsoft/callback + +Once the app is configured on external OAuth provider side, add app's credentials and strategy-specific settings (if any — e.g. see Microsoft below) to `config/prod.secret.exs`, +per strategy's documentation (e.g. [ueberauth_twitter](https://github.com/ueberauth/ueberauth_twitter)). Example config basing on environment variables: + +``` +# Twitter +config :ueberauth, Ueberauth.Strategy.Twitter.OAuth, + consumer_key: System.get_env("TWITTER_CONSUMER_KEY"), + consumer_secret: System.get_env("TWITTER_CONSUMER_SECRET") + +# Facebook +config :ueberauth, Ueberauth.Strategy.Facebook.OAuth, + client_id: System.get_env("FACEBOOK_APP_ID"), + client_secret: System.get_env("FACEBOOK_APP_SECRET"), + redirect_uri: System.get_env("FACEBOOK_REDIRECT_URI") + +# Google +config :ueberauth, Ueberauth.Strategy.Google.OAuth, + client_id: System.get_env("GOOGLE_CLIENT_ID"), + client_secret: System.get_env("GOOGLE_CLIENT_SECRET"), + redirect_uri: System.get_env("GOOGLE_REDIRECT_URI") + +# Microsoft +config :ueberauth, Ueberauth.Strategy.Microsoft.OAuth, + client_id: System.get_env("MICROSOFT_CLIENT_ID"), + client_secret: System.get_env("MICROSOFT_CLIENT_SECRET") + +config :ueberauth, Ueberauth, + providers: [ + microsoft: {Ueberauth.Strategy.Microsoft, [callback_params: []]} + ] +``` diff --git a/lib/pleroma/config.ex b/lib/pleroma/config.ex index 21507cd38..189faa15f 100644 --- a/lib/pleroma/config.ex +++ b/lib/pleroma/config.ex @@ -57,4 +57,8 @@ def delete([parent_key | keys]) do def delete(key) do Application.delete_env(:pleroma, key) end + + def oauth_consumer_strategies, do: get([:auth, :oauth_consumer_strategies], []) + + def oauth_consumer_enabled?, do: oauth_consumer_strategies() != [] end diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index b85b95bf9..085f23159 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -59,7 +59,7 @@ defmodule Pleroma.Web.Endpoint do else: "pleroma_key" same_site = - if Pleroma.Config.get([:auth, :oauth_consumer_enabled]) do + if Pleroma.Config.oauth_consumer_enabled?() do # Note: "SameSite=Strict" prevents sign in with external OAuth provider # (there would be no cookies during callback request from OAuth provider) "SameSite=Lax" diff --git a/lib/pleroma/web/oauth/fallback_controller.ex b/lib/pleroma/web/oauth/fallback_controller.ex index f0fe3b578..afaa00242 100644 --- a/lib/pleroma/web/oauth/fallback_controller.ex +++ b/lib/pleroma/web/oauth/fallback_controller.ex @@ -6,8 +6,21 @@ defmodule Pleroma.Web.OAuth.FallbackController do use Pleroma.Web, :controller alias Pleroma.Web.OAuth.OAuthController - # No user/password - def call(conn, _) do + def call(conn, {:register, :generic_error}) do + conn + |> put_status(:internal_server_error) + |> put_flash(:error, "Unknown error, please check the details and try again.") + |> OAuthController.registration_details(conn.params) + end + + def call(conn, {:register, _error}) do + conn + |> put_status(:unauthorized) + |> put_flash(:error, "Invalid Username/Password") + |> OAuthController.registration_details(conn.params) + end + + def call(conn, _error) do conn |> put_status(:unauthorized) |> put_flash(:error, "Invalid Username/Password") diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index 404728899..108303eb2 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do import Pleroma.Web.ControllerHelper, only: [oauth_scopes: 2] - if Pleroma.Config.get([:auth, :oauth_consumer_enabled]), do: plug(Ueberauth) + if Pleroma.Config.oauth_consumer_enabled?(), do: plug(Ueberauth) plug(:fetch_session) plug(:fetch_flash) @@ -62,60 +62,65 @@ defp do_authorize(conn, params) do def create_authorization( conn, - %{ - "authorization" => %{"redirect_uri" => redirect_uri} = auth_params - } = params, + %{"authorization" => auth_params} = params, opts \\ [] ) do - with {:ok, auth} <- - (opts[:auth] && {:ok, opts[:auth]}) || - do_create_authorization(conn, params, opts[:user]) do - redirect_uri = redirect_uri(conn, redirect_uri) - - cond do - redirect_uri == "urn:ietf:wg:oauth:2.0:oob" -> - render(conn, "results.html", %{ - auth: auth - }) - - true -> - connector = if String.contains?(redirect_uri, "?"), do: "&", else: "?" - url = "#{redirect_uri}#{connector}" - url_params = %{:code => auth.token} - - url_params = - if auth_params["state"] do - Map.put(url_params, :state, auth_params["state"]) - else - url_params - end - - url = "#{url}#{Plug.Conn.Query.encode(url_params)}" - - redirect(conn, external: url) - end + with {:ok, auth} <- do_create_authorization(conn, params, opts[:user]) do + after_create_authorization(conn, auth, auth_params) else - {scopes_issue, _} when scopes_issue in [:unsupported_scopes, :missing_scopes] -> - # Per https://github.com/tootsuite/mastodon/blob/ - # 51e154f5e87968d6bb115e053689767ab33e80cd/app/controllers/api/base_controller.rb#L39 - conn - |> put_flash(:error, "This action is outside the authorized scopes") - |> put_status(:unauthorized) - |> authorize(auth_params) - - {:auth_active, false} -> - # Per https://github.com/tootsuite/mastodon/blob/ - # 51e154f5e87968d6bb115e053689767ab33e80cd/app/controllers/api/base_controller.rb#L76 - conn - |> put_flash(:error, "Your login is missing a confirmed e-mail address") - |> put_status(:forbidden) - |> authorize(auth_params) - error -> - Authenticator.handle_error(conn, error) + handle_create_authorization_error(conn, error, auth_params) end end + def after_create_authorization(conn, auth, %{"redirect_uri" => redirect_uri} = auth_params) do + redirect_uri = redirect_uri(conn, redirect_uri) + + if redirect_uri == "urn:ietf:wg:oauth:2.0:oob" do + render(conn, "results.html", %{ + auth: auth + }) + else + connector = if String.contains?(redirect_uri, "?"), do: "&", else: "?" + url = "#{redirect_uri}#{connector}" + url_params = %{:code => auth.token} + + url_params = + if auth_params["state"] do + Map.put(url_params, :state, auth_params["state"]) + else + url_params + end + + url = "#{url}#{Plug.Conn.Query.encode(url_params)}" + + redirect(conn, external: url) + end + end + + defp handle_create_authorization_error(conn, {scopes_issue, _}, auth_params) + when scopes_issue in [:unsupported_scopes, :missing_scopes] do + # Per https://github.com/tootsuite/mastodon/blob/ + # 51e154f5e87968d6bb115e053689767ab33e80cd/app/controllers/api/base_controller.rb#L39 + conn + |> put_flash(:error, "This action is outside the authorized scopes") + |> put_status(:unauthorized) + |> authorize(auth_params) + end + + defp handle_create_authorization_error(conn, {:auth_active, false}, auth_params) do + # Per https://github.com/tootsuite/mastodon/blob/ + # 51e154f5e87968d6bb115e053689767ab33e80cd/app/controllers/api/base_controller.rb#L76 + conn + |> put_flash(:error, "Your login is missing a confirmed e-mail address") + |> put_status(:forbidden) + |> authorize(auth_params) + end + + defp handle_create_authorization_error(conn, error, _auth_params) do + Authenticator.handle_error(conn, error) + end + def token_exchange(conn, %{"grant_type" => "authorization_code"} = params) do with %App{} = app <- get_app_from_request(conn, params), fixed_token = fix_padding(params["code"]), @@ -202,6 +207,7 @@ def token_revoke(conn, %{"token" => token} = params) do end end + @doc "Prepares OAuth request to provider for Ueberauth" def prepare_request(conn, %{"provider" => provider} = params) do scope = oauth_scopes(params, []) @@ -218,6 +224,7 @@ def prepare_request(conn, %{"provider" => provider} = params) do |> Map.drop(~w(scope scopes client_id redirect_uri)) |> Map.put("state", state) + # Handing the request to Ueberauth redirect(conn, to: o_auth_path(conn, :request, provider, params)) end @@ -266,7 +273,7 @@ def callback(conn, params) do conn |> put_session(:registration_id, registration.id) - |> redirect(to: o_auth_path(conn, :registration_details, registration_params)) + |> registration_details(registration_params) end else _ -> @@ -292,32 +299,28 @@ def registration_details(conn, params) do end def register(conn, %{"op" => "connect"} = params) do - create_authorization_params = %{ - "authorization" => Map.merge(params, %{"name" => params["auth_name"]}) - } + authorization_params = Map.put(params, "name", params["auth_name"]) + create_authorization_params = %{"authorization" => authorization_params} with registration_id when not is_nil(registration_id) <- get_session_registration_id(conn), %Registration{} = registration <- Repo.get(Registration, registration_id), - {:ok, auth} <- do_create_authorization(conn, create_authorization_params), + {_, {:ok, auth}} <- + {:create_authorization, do_create_authorization(conn, create_authorization_params)}, %User{} = user <- Repo.preload(auth, :user).user, {:ok, _updated_registration} <- Registration.bind_to_user(registration, user) do conn |> put_session_registration_id(nil) - |> create_authorization( - create_authorization_params, - auth: auth - ) + |> after_create_authorization(auth, authorization_params) else - _ -> - params = Map.delete(params, "password") + {:create_authorization, error} -> + {:register, handle_create_authorization_error(conn, error, create_authorization_params)} - conn - |> put_flash(:error, "Unknown error, please try again.") - |> redirect(to: o_auth_path(conn, :registration_details, params)) + _ -> + {:register, :generic_error} end end - def register(conn, params) do + def register(conn, %{"op" => "register"} = params) do with registration_id when not is_nil(registration_id) <- get_session_registration_id(conn), %Registration{} = registration <- Repo.get(Registration, registration_id), {:ok, user} <- Authenticator.create_from_registration(conn, params, registration) do @@ -349,13 +352,12 @@ def register(conn, params) do ) conn + |> put_status(:forbidden) |> put_flash(:error, "Error: #{message}.") - |> redirect(to: o_auth_path(conn, :registration_details, params)) + |> registration_details(params) _ -> - conn - |> put_flash(:error, "Unknown error, please try again.") - |> redirect(to: o_auth_path(conn, :registration_details, params)) + {:register, :generic_error} end end diff --git a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex index 002f014e6..9365c7c44 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex @@ -9,7 +9,7 @@ <%= hidden_input f, :redirect_uri, value: @redirect_uri %> <%= hidden_input f, :state, value: @state %> - <%= for strategy <- Pleroma.Config.get([:auth, :oauth_consumer_strategies], []) do %> + <%= for strategy <- Pleroma.Config.oauth_consumer_strategies() do %> <%= submit "Sign in with #{String.capitalize(strategy)}", name: "provider", value: strategy %> <% end %> <% end %> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index 0144675ab..87278e636 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -26,6 +26,6 @@ <%= submit "Authorize" %> <% end %> -<%= if Pleroma.Config.get([:auth, :oauth_consumer_enabled]) do %> +<%= if Pleroma.Config.oauth_consumer_enabled?() do %> <%= render @view_module, Pleroma.Web.Auth.Authenticator.oauth_consumer_template(), assigns %> <% end %> diff --git a/test/registration_test.exs b/test/registration_test.exs new file mode 100644 index 000000000..6143b82c7 --- /dev/null +++ b/test/registration_test.exs @@ -0,0 +1,59 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.RegistrationTest do + use Pleroma.DataCase + + import Pleroma.Factory + + alias Pleroma.Registration + alias Pleroma.Repo + + describe "generic changeset" do + test "requires :provider, :uid" do + registration = build(:registration, provider: nil, uid: nil) + + cs = Registration.changeset(registration, %{}) + refute cs.valid? + + assert [ + provider: {"can't be blank", [validation: :required]}, + uid: {"can't be blank", [validation: :required]} + ] == cs.errors + end + + test "ensures uniqueness of [:provider, :uid]" do + registration = insert(:registration) + registration2 = build(:registration, provider: registration.provider, uid: registration.uid) + + cs = Registration.changeset(registration2, %{}) + assert cs.valid? + + assert {:error, + %Ecto.Changeset{ + errors: [ + uid: + {"has already been taken", + [constraint: :unique, constraint_name: "registrations_provider_uid_index"]} + ] + }} = Repo.insert(cs) + + # Note: multiple :uid values per [:user_id, :provider] are intentionally allowed + cs2 = Registration.changeset(registration2, %{uid: "available.uid"}) + assert cs2.valid? + assert {:ok, _} = Repo.insert(cs2) + + cs3 = Registration.changeset(registration2, %{provider: "provider2"}) + assert cs3.valid? + assert {:ok, _} = Repo.insert(cs3) + end + + test "allows `nil` :user_id (user-unbound registration)" do + registration = build(:registration, user_id: nil) + cs = Registration.changeset(registration, %{}) + assert cs.valid? + assert {:ok, _} = Repo.insert(cs) + end + end +end diff --git a/test/web/oauth/oauth_controller_test.exs b/test/web/oauth/oauth_controller_test.exs index 75333f2d5..385896dc6 100644 --- a/test/web/oauth/oauth_controller_test.exs +++ b/test/web/oauth/oauth_controller_test.exs @@ -20,16 +20,11 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do describe "in OAuth consumer mode, " do setup do - oauth_consumer_enabled_path = [:auth, :oauth_consumer_enabled] oauth_consumer_strategies_path = [:auth, :oauth_consumer_strategies] - oauth_consumer_enabled = Pleroma.Config.get(oauth_consumer_enabled_path) oauth_consumer_strategies = Pleroma.Config.get(oauth_consumer_strategies_path) - - Pleroma.Config.put(oauth_consumer_enabled_path, true) Pleroma.Config.put(oauth_consumer_strategies_path, ~w(twitter facebook)) on_exit(fn -> - Pleroma.Config.put(oauth_consumer_enabled_path, oauth_consumer_enabled) Pleroma.Config.put(oauth_consumer_strategies_path, oauth_consumer_strategies) end) @@ -42,7 +37,7 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do ] end - test "GET /oauth/authorize also renders OAuth consumer form", %{ + test "GET /oauth/authorize renders auth forms, including OAuth consumer form", %{ app: app, conn: conn } do @@ -97,31 +92,6 @@ test "GET /oauth/prepare_request encodes parameters as `state` and redirects", % } = state_components end - test "on authentication error, redirects to `redirect_uri`", %{app: app, conn: conn} do - state_params = %{ - "scope" => Enum.join(app.scopes, " "), - "client_id" => app.client_id, - "redirect_uri" => app.redirect_uris, - "state" => "" - } - - conn = - conn - |> assign(:ueberauth_failure, %{errors: [%{message: "unknown error"}]}) - |> get( - "/oauth/twitter/callback", - %{ - "oauth_token" => "G-5a3AAAAAAAwMH9AAABaektfSM", - "oauth_verifier" => "QZl8vUqNvXMTKpdmUnGejJxuHG75WWWs", - "provider" => "twitter", - "state" => Poison.encode!(state_params) - } - ) - - assert response = html_response(conn, 302) - assert redirected_to(conn) == app.redirect_uris - end - test "with user-bound registration, GET /oauth//callback redirects to `redirect_uri` with `code`", %{app: app, conn: conn} do registration = insert(:registration) @@ -152,7 +122,7 @@ test "with user-bound registration, GET /oauth//callback redirects to end end - test "with user-unbound registration, GET /oauth//callback redirects to registration_details page", + test "with user-unbound registration, GET /oauth//callback renders registration_details page", %{app: app, conn: conn} do registration = insert(:registration, user: nil) @@ -177,22 +147,43 @@ test "with user-unbound registration, GET /oauth//callback redirects t } ) - expected_redirect_params = - state_params - |> Map.delete("scope") - |> Map.merge(%{ - "scope" => "read write", - "email" => Registration.email(registration), - "nickname" => Registration.nickname(registration) - }) - - assert response = html_response(conn, 302) - - assert redirected_to(conn) == - o_auth_path(conn, :registration_details, expected_redirect_params) + assert response = html_response(conn, 200) + assert response =~ ~r/name="op" type="submit" value="register"/ + assert response =~ ~r/name="op" type="submit" value="connect"/ + assert response =~ Registration.email(registration) + assert response =~ Registration.nickname(registration) end end + test "on authentication error, GET /oauth//callback redirects to `redirect_uri`", %{ + app: app, + conn: conn + } do + state_params = %{ + "scope" => Enum.join(app.scopes, " "), + "client_id" => app.client_id, + "redirect_uri" => app.redirect_uris, + "state" => "" + } + + conn = + conn + |> assign(:ueberauth_failure, %{errors: [%{message: "(error description)"}]}) + |> get( + "/oauth/twitter/callback", + %{ + "oauth_token" => "G-5a3AAAAAAAwMH9AAABaektfSM", + "oauth_verifier" => "QZl8vUqNvXMTKpdmUnGejJxuHG75WWWs", + "provider" => "twitter", + "state" => Poison.encode!(state_params) + } + ) + + assert response = html_response(conn, 302) + assert redirected_to(conn) == app.redirect_uris + assert get_flash(conn, :error) == "Failed to authenticate: (error description)." + end + test "GET /oauth/registration_details renders registration details form", %{ app: app, conn: conn @@ -243,7 +234,7 @@ test "with valid params, POST /oauth/register?op=register redirects to `redirect assert redirected_to(conn) =~ ~r/#{app.redirect_uris}\?code=.+/ end - test "with invalid params, POST /oauth/register?op=register redirects to registration_details page", + test "with invalid params, POST /oauth/register?op=register renders registration_details page", %{ app: app, conn: conn @@ -257,19 +248,22 @@ test "with invalid params, POST /oauth/register?op=register redirects to registr "client_id" => app.client_id, "redirect_uri" => app.redirect_uris, "state" => "a_state", - "nickname" => another_user.nickname, - "email" => another_user.email + "nickname" => "availablenickname", + "email" => "available@email.com" } - conn = - conn - |> put_session(:registration_id, registration.id) - |> post("/oauth/register", params) + for {bad_param, bad_param_value} <- + [{"nickname", another_user.nickname}, {"email", another_user.email}] do + bad_params = Map.put(params, bad_param, bad_param_value) - assert response = html_response(conn, 302) + conn = + conn + |> put_session(:registration_id, registration.id) + |> post("/oauth/register", bad_params) - assert redirected_to(conn) == - o_auth_path(conn, :registration_details, params) + assert html_response(conn, 403) =~ ~r/name="op" type="submit" value="register"/ + assert get_flash(conn, :error) == "Error: #{bad_param} has already been taken." + end end test "with valid params, POST /oauth/register?op=connect redirects to `redirect_uri` with `code`", @@ -300,7 +294,7 @@ test "with valid params, POST /oauth/register?op=connect redirects to `redirect_ assert redirected_to(conn) =~ ~r/#{app.redirect_uris}\?code=.+/ end - test "with invalid params, POST /oauth/register?op=connect redirects to registration_details page", + test "with invalid params, POST /oauth/register?op=connect renders registration_details page", %{ app: app, conn: conn @@ -323,10 +317,8 @@ test "with invalid params, POST /oauth/register?op=connect redirects to registra |> put_session(:registration_id, registration.id) |> post("/oauth/register", params) - assert response = html_response(conn, 302) - - assert redirected_to(conn) == - o_auth_path(conn, :registration_details, Map.delete(params, "password")) + assert html_response(conn, 401) =~ ~r/name="op" type="submit" value="connect"/ + assert get_flash(conn, :error) == "Invalid Username/Password" end end From f0f30019e1c9992cb420ba54457840cddaeb6a3a Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 5 Apr 2019 15:19:44 +0300 Subject: [PATCH 36/59] Refactor html caching functions to have a key instead of a module, use more correct terminology and fix summaries in mastoapi --- lib/pleroma/html.ex | 15 +++++++-------- lib/pleroma/web/mastodon_api/views/status_view.ex | 14 +++++++++++--- lib/pleroma/web/metadata/utils.ex | 2 +- .../web/twitter_api/views/activity_view.ex | 6 +++--- 4 files changed, 22 insertions(+), 15 deletions(-) diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 1e48749a8..7f1dbe28c 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -28,21 +28,20 @@ def filter_tags(html, scrubber), do: Scrubber.scrub(html, scrubber) def filter_tags(html), do: filter_tags(html, nil) def strip_tags(html), do: Scrubber.scrub(html, Scrubber.StripTags) - # TODO: rename object to activity because that's what it is really working with - def get_cached_scrubbed_html_for_object(content, scrubbers, object, module) do - key = "#{module}#{generate_scrubber_signature(scrubbers)}|#{object.id}" + def get_cached_scrubbed_html_for_activity(content, scrubbers, activity, key \\ "") do + key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}" Cachex.fetch!(:scrubber_cache, key, fn _key -> - ensure_scrubbed_html(content, scrubbers, object.data["object"]["fake"] || false) + ensure_scrubbed_html(content, scrubbers, activity.data["object"]["fake"] || false) end) end - def get_cached_stripped_html_for_object(content, object, module) do - get_cached_scrubbed_html_for_object( + def get_cached_stripped_html_for_activity(content, activity, key) do + get_cached_scrubbed_html_for_activity( content, HtmlSanitizeEx.Scrubber.StripTags, - object, - module + activity, + key ) end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 200bb453d..4c0b53bdd 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -147,10 +147,18 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} content = object |> render_content() - |> HTML.get_cached_scrubbed_html_for_object( + |> HTML.get_cached_scrubbed_html_for_activity( User.html_filter_policy(opts[:for]), activity, - __MODULE__ + "mastoapi:content" + ) + + summary = + (object["summary"] || "") + |> HTML.get_cached_scrubbed_html_for_activity( + User.html_filter_policy(opts[:for]), + activity, + "mastoapi:summary" ) card = render("card.json", Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)) @@ -182,7 +190,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity} muted: CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user), pinned: pinned?(activity, user), sensitive: sensitive, - spoiler_text: object["summary"] || "", + spoiler_text: summary, visibility: get_visibility(object), media_attachments: attachments, mentions: mentions, diff --git a/lib/pleroma/web/metadata/utils.ex b/lib/pleroma/web/metadata/utils.ex index 23bbde1a6..58385a3d1 100644 --- a/lib/pleroma/web/metadata/utils.ex +++ b/lib/pleroma/web/metadata/utils.ex @@ -12,7 +12,7 @@ def scrub_html_and_truncate(%{data: %{"content" => content}} = object) do # html content comes from DB already encoded, decode first and scrub after |> HtmlEntities.decode() |> String.replace(~r//, " ") - |> HTML.get_cached_stripped_html_for_object(object, __MODULE__) + |> HTML.get_cached_stripped_html_for_activity(object, "metadata") |> Formatter.demojify() |> Formatter.truncate() end diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index aa1d41fa2..433322eb8 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -254,10 +254,10 @@ def render( html = content - |> HTML.get_cached_scrubbed_html_for_object( + |> HTML.get_cached_scrubbed_html_for_activity( User.html_filter_policy(opts[:for]), activity, - __MODULE__ + "twitterapi:content" ) |> Formatter.emojify(object["emoji"]) @@ -265,7 +265,7 @@ def render( if content do content |> String.replace(~r//, "\n") - |> HTML.get_cached_stripped_html_for_object(activity, __MODULE__) + |> HTML.get_cached_stripped_html_for_activity(activity, "twitterapi:content") else "" end From f1712cd2f1ec6061f70d259f8f5e2b7e9f408d8c Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 5 Apr 2019 19:38:44 +0700 Subject: [PATCH 37/59] Use PleromaJobQueue in Pleroma.Web.Push --- config/config.exs | 1 + docs/config.md | 5 ++-- lib/pleroma/application.ex | 4 +-- lib/pleroma/web/push/impl.ex | 6 ++--- lib/pleroma/web/push/push.ex | 48 ++++++++---------------------------- test/web/push/impl_test.exs | 6 +++-- 6 files changed, 23 insertions(+), 47 deletions(-) diff --git a/config/config.exs b/config/config.exs index c143f79fc..d97586a61 100644 --- a/config/config.exs +++ b/config/config.exs @@ -356,6 +356,7 @@ config :pleroma_job_queue, :queues, federator_incoming: 50, federator_outgoing: 50, + web_push: 50, mailer: 10 config :pleroma, :fetch_initial_posts, diff --git a/docs/config.md b/docs/config.md index 06d6fd757..6f3119573 100644 --- a/docs/config.md +++ b/docs/config.md @@ -218,14 +218,14 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i - `port` * `url` - a list containing the configuration for generating urls, accepts - `host` - the host without the scheme and a post (e.g `example.com`, not `https://example.com:2020`) - - `scheme` - e.g `http`, `https` + - `scheme` - e.g `http`, `https` - `port` - `path` **Important note**: if you modify anything inside these lists, default `config.exs` values will be overwritten, which may result in breakage, to make sure this does not happen please copy the default value for the list from `config.exs` and modify/add only what you need -Example: +Example: ```elixir config :pleroma, Pleroma.Web.Endpoint, url: [host: "example.com", port: 2020, scheme: "https"], @@ -317,6 +317,7 @@ Pleroma has the following queues: * `federator_outgoing` - Outgoing federation * `federator_incoming` - Incoming federation * `mailer` - Email sender, see [`Pleroma.Mailer`](#pleroma-mailer) +* `web_push` - Web push notifications Example: diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 782d1d589..8f8d26814 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -109,8 +109,8 @@ def start(_type, _args) do [ worker(Pleroma.Web.Federator.RetryQueue, []), worker(Pleroma.Stats, []), - worker(Pleroma.Web.Push, []), - worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary) + worker(Task, [&Pleroma.Web.Push.init/0], restart: :temporary, id: :web_push_init), + worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary, id: :federator_init) ] ++ streamer_child() ++ chat_child() ++ diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index 863573185..2233480c5 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -19,8 +19,8 @@ defmodule Pleroma.Web.Push.Impl do @types ["Create", "Follow", "Announce", "Like"] @doc "Performs sending notifications for user subscriptions" - @spec perform_send(Notification.t()) :: list(any) - def perform_send( + @spec perform(Notification.t()) :: list(any) | :error + def perform( %{activity: %{data: %{"type" => activity_type}, id: activity_id}, user_id: user_id} = notif ) @@ -50,7 +50,7 @@ def perform_send( end end - def perform_send(_) do + def perform(_) do Logger.warn("Unknown notification type") :error end diff --git a/lib/pleroma/web/push/push.ex b/lib/pleroma/web/push/push.ex index 5259e8e33..cdd50005d 100644 --- a/lib/pleroma/web/push/push.ex +++ b/lib/pleroma/web/push/push.ex @@ -3,18 +3,20 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Push do - use GenServer - alias Pleroma.Web.Push.Impl require Logger - ############## - # Client API # - ############## + def init() do + unless enabled() do + Logger.warn(""" + VAPID key pair is not found. If you wish to enabled web push, please run - def start_link do - GenServer.start_link(__MODULE__, :ok, name: __MODULE__) + mix web_push.gen.keypair + + and add the resulting output to your configuration file. + """) + end end def vapid_config do @@ -30,35 +32,5 @@ def enabled do end def send(notification), - do: GenServer.cast(__MODULE__, {:send, notification}) - - #################### - # Server Callbacks # - #################### - - @impl true - def init(:ok) do - if enabled() do - {:ok, nil} - else - Logger.warn(""" - VAPID key pair is not found. If you wish to enabled web push, please run - - mix web_push.gen.keypair - - and add the resulting output to your configuration file. - """) - - :ignore - end - end - - @impl true - def handle_cast({:send, notification}, state) do - if enabled() do - Impl.perform_send(notification) - end - - {:noreply, state} - end + do: PleromaJobQueue.enqueue(:web_push, Impl, [notification]) end diff --git a/test/web/push/impl_test.exs b/test/web/push/impl_test.exs index 3f9f3d809..6bac2c9f6 100644 --- a/test/web/push/impl_test.exs +++ b/test/web/push/impl_test.exs @@ -64,17 +64,19 @@ test "performs sending notifications" do } ) - assert Impl.perform_send(notif) == [:ok, :ok] + assert Impl.perform(notif) == [:ok, :ok] end + @tag capture_log: true test "returns error if notif does not match " do - assert Impl.perform_send(%{}) == :error + assert Impl.perform(%{}) == :error end test "successful message sending" do assert Impl.push_message(@message, @sub, @api_key, %Subscription{}) == :ok end + @tag capture_log: true test "fail message sending" do assert Impl.push_message( @message, From 1c2e4f88d1a707791818014f8bcdedd986c2fa75 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Fri, 5 Apr 2019 19:46:28 +0700 Subject: [PATCH 38/59] fix credo --- lib/pleroma/web/push/push.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/push/push.ex b/lib/pleroma/web/push/push.ex index cdd50005d..729dad02a 100644 --- a/lib/pleroma/web/push/push.ex +++ b/lib/pleroma/web/push/push.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Web.Push do require Logger - def init() do + def init do unless enabled() do Logger.warn(""" VAPID key pair is not found. If you wish to enabled web push, please run From 7895ee37fae82de26b3c06e69a96788d8c88d139 Mon Sep 17 00:00:00 2001 From: Roger Braun Date: Sun, 16 Dec 2018 16:41:56 +0100 Subject: [PATCH 39/59] Add user following / unfollowing to the admin api. --- .../web/admin_api/admin_api_controller.ex | 20 ++++++++ lib/pleroma/web/router.ex | 4 ++ .../admin_api/admin_api_controller_test.exs | 46 +++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index b3a09e49e..84d0aabaf 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -25,6 +25,26 @@ def user_delete(conn, %{"nickname" => nickname}) do |> json(nickname) end + def user_follow(conn, %{"follower" => follower_nick, "followed" => followed_nick}) do + with %User{} = follower <- Repo.get_by(User, %{nickname: follower_nick}), + %User{} = followed <- Repo.get_by(User, %{nickname: followed_nick}) do + User.follow(follower, followed) + end + + conn + |> json("ok") + end + + def user_unfollow(conn, %{"follower" => follower_nick, "followed" => followed_nick}) do + with %User{} = follower <- Repo.get_by(User, %{nickname: follower_nick}), + %User{} = followed <- Repo.get_by(User, %{nickname: followed_nick}) do + User.unfollow(follower, followed) + end + + conn + |> json("ok") + end + def user_create( conn, %{"nickname" => nickname, "email" => email, "password" => password} diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 605a327fc..1c752e44c 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -140,8 +140,12 @@ defmodule Pleroma.Web.Router do scope "/api/pleroma/admin", Pleroma.Web.AdminAPI do pipe_through([:admin_api, :oauth_write]) + post("/user/follow", AdminAPIController, :user_follow) + post("/user/unfollow", AdminAPIController, :user_unfollow) + get("/users", AdminAPIController, :list_users) get("/users/:nickname", AdminAPIController, :user_show) + delete("/user", AdminAPIController, :user_delete) patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation) post("/user", AdminAPIController, :user_create) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index acae64361..cedc907ec 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -74,6 +74,52 @@ test "when the user doesn't exist", %{conn: conn} do end end + describe "/api/pleroma/admin/user/follow" do + test "allows to force-follow another user" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user) + follower = insert(:user) + + conn = + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/user/follow", %{ + "follower" => follower.nickname, + "followed" => user.nickname + }) + + user = Repo.get(User, user.id) + follower = Repo.get(User, follower.id) + + assert User.following?(follower, user) + end + end + + describe "/api/pleroma/admin/user/unfollow" do + test "allows to force-unfollow another user" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user) + follower = insert(:user) + + User.follow(follower, user) + + conn = + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/user/unfollow", %{ + "follower" => follower.nickname, + "followed" => user.nickname + }) + + user = Repo.get(User, user.id) + follower = Repo.get(User, follower.id) + + refute User.following?(follower, user) + end + end + describe "PUT /api/pleroma/admin/users/tag" do setup do admin = insert(:user, info: %{is_admin: true}) From da64a5aece131d6bd8c0d17dcda61c626b44c4d0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 5 Apr 2019 11:29:34 -0500 Subject: [PATCH 40/59] Document the admin API endpoints for controlling follow/unfollow --- docs/api/admin_api.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/api/admin_api.md b/docs/api/admin_api.md index 53b68ffd4..86cacebb1 100644 --- a/docs/api/admin_api.md +++ b/docs/api/admin_api.md @@ -58,6 +58,26 @@ Authentication is required and the user must be an admin. - `password` - Response: User’s nickname +## `/api/pleroma/admin/user/follow` +### Make a user follow another user + +- Methods: `POST` +- Params: + - `follower`: The nickname of the follower + - `followed`: The nickname of the followed +- Response: + - "ok" + +## `/api/pleroma/admin/user/unfollow` +### Make a user unfollow another user + +- Methods: `POST` +- Params: + - `follower`: The nickname of the follower + - `followed`: The nickname of the followed +- Response: + - "ok" + ## `/api/pleroma/admin/users/:nickname/toggle_activation` ### Toggle user activation From b5a2d384f71de9f7ff33d99c95c5db4674141d9a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 5 Apr 2019 11:41:41 -0500 Subject: [PATCH 41/59] Redundant Repo.get_by usage was recently removed from the codebase --- lib/pleroma/web/admin_api/admin_api_controller.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 84d0aabaf..78bf31893 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -26,8 +26,8 @@ def user_delete(conn, %{"nickname" => nickname}) do end def user_follow(conn, %{"follower" => follower_nick, "followed" => followed_nick}) do - with %User{} = follower <- Repo.get_by(User, %{nickname: follower_nick}), - %User{} = followed <- Repo.get_by(User, %{nickname: followed_nick}) do + with %User{} = follower <- User.get_by_nickname(follower_nick), + %User{} = followed <- User.get_by_nickname(followed_nick) do User.follow(follower, followed) end @@ -36,8 +36,8 @@ def user_follow(conn, %{"follower" => follower_nick, "followed" => followed_nick end def user_unfollow(conn, %{"follower" => follower_nick, "followed" => followed_nick}) do - with %User{} = follower <- Repo.get_by(User, %{nickname: follower_nick}), - %User{} = followed <- Repo.get_by(User, %{nickname: followed_nick}) do + with %User{} = follower <- User.get_by_nickname(follower_nick), + %User{} = followed <- User.get_by_nickname(followed_nick) do User.unfollow(follower, followed) end From c746087f570e366976b9b89c2aa6c2a5ff83c9ca Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 5 Apr 2019 11:59:56 -0500 Subject: [PATCH 42/59] Also remove Repo functions in the tests --- test/web/admin_api/admin_api_controller_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index cedc907ec..9c1cae6b7 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -89,8 +89,8 @@ test "allows to force-follow another user" do "followed" => user.nickname }) - user = Repo.get(User, user.id) - follower = Repo.get(User, follower.id) + user = User.get_by_nickname(user.id) + follower = User.get_by_nickname(follower.id) assert User.following?(follower, user) end @@ -113,8 +113,8 @@ test "allows to force-unfollow another user" do "followed" => user.nickname }) - user = Repo.get(User, user.id) - follower = Repo.get(User, follower.id) + user = User.get_by_nickname(user.id) + follower = User.get_by_nickname(follower.id) refute User.following?(follower, user) end From fac76bfa35f735005249111e74ea6be8670f5755 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 5 Apr 2019 12:11:19 -0500 Subject: [PATCH 43/59] We actually want the user id not nickname in the test... --- test/web/admin_api/admin_api_controller_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index 9c1cae6b7..dd2fbfb15 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -89,8 +89,8 @@ test "allows to force-follow another user" do "followed" => user.nickname }) - user = User.get_by_nickname(user.id) - follower = User.get_by_nickname(follower.id) + user = User.get_by_id(user.id) + follower = User.get_by_id(follower.id) assert User.following?(follower, user) end @@ -113,8 +113,8 @@ test "allows to force-unfollow another user" do "followed" => user.nickname }) - user = User.get_by_nickname(user.id) - follower = User.get_by_nickname(follower.id) + user = User.get_by_id(user.id) + follower = User.get_by_id(follower.id) refute User.following?(follower, user) end From fb1be1d79892a72b10af2c24479e81600603a6af Mon Sep 17 00:00:00 2001 From: optikfluffel Date: Fri, 5 Apr 2019 20:12:44 +0200 Subject: [PATCH 44/59] Use --cover option when running CI tests --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c07f1a5d3..0bd657d67 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,7 @@ unit-testing: script: - mix ecto.create - mix ecto.migrate - - mix test --trace --preload-modules + - mix test --trace --preload-modules --cover lint: stage: test From e5df8cadeaa1ee3992e31e6ac00a0c391da7e4bd Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 5 Apr 2019 19:59:03 +0000 Subject: [PATCH 45/59] Revert "Merge branch 'test-coverage' into 'develop'" This reverts merge request !1027 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0bd657d67..c07f1a5d3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -50,7 +50,7 @@ unit-testing: script: - mix ecto.create - mix ecto.migrate - - mix test --trace --preload-modules --cover + - mix test --trace --preload-modules lint: stage: test From e9c075d05c2f11b905d40ed86dd19818acf310ec Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Fri, 5 Apr 2019 22:40:30 +0200 Subject: [PATCH 46/59] Mock :crypt.crypt/2 because otherwise the test fails on Mac OS --- test/plugs/legacy_authentication_plug_test.exs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/plugs/legacy_authentication_plug_test.exs b/test/plugs/legacy_authentication_plug_test.exs index 302662797..8b0b06772 100644 --- a/test/plugs/legacy_authentication_plug_test.exs +++ b/test/plugs/legacy_authentication_plug_test.exs @@ -47,16 +47,18 @@ test "it authenticates the auth_user if present and password is correct and rese |> assign(:auth_user, user) conn = - with_mock User, - reset_password: fn user, %{password: password, password_confirmation: password} -> - send(self(), :reset_password) - {:ok, user} - end do - conn - |> LegacyAuthenticationPlug.call(%{}) + with_mocks([ + {:crypt, [], [crypt: fn _password, password_hash -> password_hash end]}, + {User, [], + [ + reset_password: fn user, %{password: password, password_confirmation: password} -> + {:ok, user} + end + ]} + ]) do + LegacyAuthenticationPlug.call(conn, %{}) end - assert_received :reset_password assert conn.assigns.user == user end From 325a2680173f714a5875ed726f9171e7984f7f7a Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Fri, 5 Apr 2019 23:36:42 +0000 Subject: [PATCH 47/59] Redirect to the referer url after mastofe authorization --- .../mastodon_api/mastodon_api_controller.ex | 19 ++++-- test/support/factory.ex | 10 +++ .../mastodon_api_controller_test.exs | 67 +++++++++++++++++++ 3 files changed, 90 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 89fd7629a..bcc79b08a 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -1091,9 +1091,7 @@ def list_timeline(%{assigns: %{user: user}} = conn, %{"list_id" => id} = params) end def index(%{assigns: %{user: user}} = conn, _params) do - token = - conn - |> get_session(:oauth_token) + token = get_session(conn, :oauth_token) if user && token do mastodon_emoji = mastodonized_emoji() @@ -1194,6 +1192,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do |> render("index.html", %{initial_state: initial_state, flavour: flavour}) else conn + |> put_session(:return_to, conn.request_path) |> redirect(to: "/web/login") end end @@ -1278,12 +1277,20 @@ def login(conn, _) do scope: Enum.join(app.scopes, " ") ) - conn - |> redirect(to: path) + redirect(conn, to: path) end end - defp local_mastodon_root_path(conn), do: mastodon_api_path(conn, :index, ["getting-started"]) + defp local_mastodon_root_path(conn) do + case get_session(conn, :return_to) do + nil -> + mastodon_api_path(conn, :index, ["getting-started"]) + + return_to -> + delete_session(conn, :return_to) + return_to + end + end defp get_or_make_app do find_attrs = %{client_name: @local_mastodon_name, redirect_uris: "."} diff --git a/test/support/factory.ex b/test/support/factory.ex index e1a08315a..b37bc2c07 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -240,6 +240,16 @@ def oauth_token_factory do } end + def oauth_authorization_factory do + %Pleroma.Web.OAuth.Authorization{ + token: :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false), + scopes: ["read", "write", "follow", "push"], + valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), 60 * 10), + user: build(:user), + app: build(:oauth_app) + } + end + def push_subscription_factory do %Pleroma.Web.Push.Subscription{ user: build(:user), diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 6060cc97f..438e9507d 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2340,4 +2340,71 @@ test "accounts fetches correct account for nicknames beginning with numbers", %{ refute acc_one == acc_two assert acc_two == acc_three end + + describe "index/2 redirections" do + setup %{conn: conn} do + session_opts = [ + store: :cookie, + key: "_test", + signing_salt: "cooldude" + ] + + conn = + conn + |> Plug.Session.call(Plug.Session.init(session_opts)) + |> fetch_session() + + test_path = "/web/statuses/test" + %{conn: conn, path: test_path} + end + + test "redirects not logged-in users to the login page", %{conn: conn, path: path} do + conn = get(conn, path) + + assert conn.status == 302 + assert redirected_to(conn) == "/web/login" + end + + test "does not redirect logged in users to the login page", %{conn: conn, path: path} do + token = insert(:oauth_token) + + conn = + conn + |> assign(:user, token.user) + |> put_session(:oauth_token, token.token) + |> get(path) + + assert conn.status == 200 + end + + test "saves referer path to session", %{conn: conn, path: path} do + conn = get(conn, path) + return_to = Plug.Conn.get_session(conn, :return_to) + + assert return_to == path + end + + test "redirects to the saved path after log in", %{conn: conn, path: path} do + app = insert(:oauth_app, client_name: "Mastodon-Local", redirect_uris: ".") + auth = insert(:oauth_authorization, app: app) + + conn = + conn + |> put_session(:return_to, path) + |> get("/web/login", %{code: auth.token}) + + assert conn.status == 302 + assert redirected_to(conn) == path + end + + test "redirects to the getting-started page when referer is not present", %{conn: conn} do + app = insert(:oauth_app, client_name: "Mastodon-Local", redirect_uris: ".") + auth = insert(:oauth_authorization, app: app) + + conn = get(conn, "/web/login", %{code: auth.token}) + + assert conn.status == 302 + assert redirected_to(conn) == "/web/getting-started" + end + end end From b395aebf2489f44bdb1a9c4905a51f0f26bf5fab Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 6 Apr 2019 09:30:36 -0500 Subject: [PATCH 48/59] Pin recon dependency to 2.4.0 --- mix.exs | 2 +- mix.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 6e7cff413..ec0865c4f 100644 --- a/mix.exs +++ b/mix.exs @@ -101,7 +101,7 @@ defp deps do {:prometheus_phoenix, "~> 1.2"}, {:prometheus_ecto, "~> 1.4"}, {:prometheus_process_collector, "~> 1.4"}, - {:recon, github: "ferd/recon"}, + {:recon, github: "ferd/recon", tag: "2.4.0"}, {:quack, "~> 0.1.1"} ] end diff --git a/mix.lock b/mix.lock index 662fd0c6e..7c7e322de 100644 --- a/mix.lock +++ b/mix.lock @@ -66,7 +66,7 @@ "prometheus_process_collector": {:hex, :prometheus_process_collector, "1.4.0", "6dbd39e3165b9ef1c94a7a820e9ffe08479f949dcdd431ed4aaea7b250eebfde", [:rebar3], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"}, "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm"}, "ranch": {:hex, :ranch, "1.7.1", "6b1fab51b49196860b733a49c07604465a47bdb78aa10c1c16a3d199f7f8c881", [:rebar3], [], "hexpm"}, - "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", []}, + "recon": {:git, "https://github.com/ferd/recon.git", "75d70c7c08926d2f24f1ee6de14ee50fe8a52763", [tag: "2.4.0"]}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"}, "swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"}, "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]}, From 7aa53d52bd982b5ab233a65048f5fb1823127d4a Mon Sep 17 00:00:00 2001 From: eugenijm Date: Sat, 6 Apr 2019 00:22:42 +0300 Subject: [PATCH 49/59] Return 403 on oauth token exchange for a deactivated user --- lib/pleroma/web/oauth/oauth_controller.ex | 6 ++++++ test/web/oauth/oauth_controller_test.exs | 26 +++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index 26d53df1a..aac8f97fc 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -152,6 +152,7 @@ def token_exchange( with {_, {:ok, %User{} = user}} <- {:get_user, Authenticator.get_user(conn)}, %App{} = app <- get_app_from_request(conn, params), {:auth_active, true} <- {:auth_active, User.auth_active?(user)}, + {:user_active, true} <- {:user_active, !user.info.deactivated}, scopes <- oauth_scopes(params, app.scopes), [] <- scopes -- app.scopes, true <- Enum.any?(scopes), @@ -175,6 +176,11 @@ def token_exchange( |> put_status(:forbidden) |> json(%{error: "Your login is missing a confirmed e-mail address"}) + {:user_active, false} -> + conn + |> put_status(:forbidden) + |> json(%{error: "Your account is currently disabled"}) + _error -> put_status(conn, 400) |> json(%{error: "Invalid credentials"}) diff --git a/test/web/oauth/oauth_controller_test.exs b/test/web/oauth/oauth_controller_test.exs index a9a0b9ed4..a68528420 100644 --- a/test/web/oauth/oauth_controller_test.exs +++ b/test/web/oauth/oauth_controller_test.exs @@ -327,6 +327,32 @@ test "rejects token exchange for valid credentials belonging to unconfirmed user refute Map.has_key?(resp, "access_token") end + test "rejects token exchange for valid credentials belonging to deactivated user" do + password = "testpassword" + + user = + insert(:user, + password_hash: Comeonin.Pbkdf2.hashpwsalt(password), + info: %{deactivated: true} + ) + + app = insert(:oauth_app) + + conn = + build_conn() + |> post("/oauth/token", %{ + "grant_type" => "password", + "username" => user.nickname, + "password" => password, + "client_id" => app.client_id, + "client_secret" => app.client_secret + }) + + assert resp = json_response(conn, 403) + assert %{"error" => _} = resp + refute Map.has_key?(resp, "access_token") + end + test "rejects an invalid authorization code" do app = insert(:oauth_app) From 7bf622ce736af12db9b4865d8d3c2db5792d6f03 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Thu, 28 Mar 2019 12:39:10 +0300 Subject: [PATCH 50/59] Add scheduled activities --- lib/pleroma/scheduled_activity.ex | 74 +++++++++++++ lib/pleroma/web/mastodon_api/mastodon_api.ex | 7 ++ .../mastodon_api/mastodon_api_controller.ex | 47 ++++++++ .../views/scheduled_activity_view.ex | 23 ++++ lib/pleroma/web/router.ex | 6 + ...0328053912_create_scheduled_activities.exs | 15 +++ test/support/factory.ex | 8 ++ .../mastodon_api_controller_test.exs | 104 ++++++++++++++++++ 8 files changed, 284 insertions(+) create mode 100644 lib/pleroma/scheduled_activity.ex create mode 100644 lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex create mode 100644 priv/repo/migrations/20190328053912_create_scheduled_activities.exs diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex new file mode 100644 index 000000000..0c1b26a33 --- /dev/null +++ b/lib/pleroma/scheduled_activity.ex @@ -0,0 +1,74 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ScheduledActivity do + use Ecto.Schema + + alias Pleroma.Repo + alias Pleroma.ScheduledActivity + alias Pleroma.User + + import Ecto.Query + import Ecto.Changeset + + schema "scheduled_activities" do + belongs_to(:user, User, type: Pleroma.FlakeId) + field(:scheduled_at, :naive_datetime) + field(:params, :map) + + timestamps() + end + + def changeset(%ScheduledActivity{} = scheduled_activity, attrs) do + scheduled_activity + |> cast(attrs, [:scheduled_at, :params]) + end + + def update_changeset(%ScheduledActivity{} = scheduled_activity, attrs) do + scheduled_activity + |> cast(attrs, [:scheduled_at]) + end + + def new(%User{} = user, attrs) do + %ScheduledActivity{user_id: user.id} + |> changeset(attrs) + end + + def create(%User{} = user, attrs) do + user + |> new(attrs) + |> Repo.insert() + end + + def get(%User{} = user, scheduled_activity_id) do + ScheduledActivity + |> where(user_id: ^user.id) + |> where(id: ^scheduled_activity_id) + |> Repo.one() + end + + def update(%User{} = user, scheduled_activity_id, attrs) do + with %ScheduledActivity{} = scheduled_activity <- get(user, scheduled_activity_id) do + scheduled_activity + |> update_changeset(attrs) + |> Repo.update() + else + nil -> {:error, :not_found} + end + end + + def delete(%User{} = user, scheduled_activity_id) do + with %ScheduledActivity{} = scheduled_activity <- get(user, scheduled_activity_id) do + scheduled_activity + |> Repo.delete() + else + nil -> {:error, :not_found} + end + end + + def for_user_query(%User{} = user) do + ScheduledActivity + |> where(user_id: ^user.id) + end +end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index 08ea5f967..382f07e6b 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do alias Pleroma.Activity alias Pleroma.Notification alias Pleroma.Pagination + alias Pleroma.ScheduledActivity alias Pleroma.User def get_followers(user, params \\ %{}) do @@ -28,6 +29,12 @@ def get_notifications(user, params \\ %{}) do |> Pagination.fetch_paginated(params) end + def get_scheduled_activities(user, params \\ %{}) do + user + |> ScheduledActivity.for_user_query() + |> Pagination.fetch_paginated(params) + end + defp cast_params(params) do param_types = %{ exclude_types: {:array, :string} diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index bcc79b08a..0916d84dc 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.ScheduledActivity alias Pleroma.Stats alias Pleroma.User alias Pleroma.Web @@ -25,6 +26,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do alias Pleroma.Web.MastodonAPI.MastodonView alias Pleroma.Web.MastodonAPI.NotificationView alias Pleroma.Web.MastodonAPI.ReportView + alias Pleroma.Web.MastodonAPI.ScheduledActivityView alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.MediaProxy alias Pleroma.Web.OAuth.App @@ -364,6 +366,45 @@ def get_context(%{assigns: %{user: user}} = conn, %{"id" => id}) do end end + def scheduled_statuses(%{assigns: %{user: user}} = conn, params) do + with scheduled_activities <- MastodonAPI.get_scheduled_activities(user, params) do + conn + |> add_link_headers(:scheduled_statuses, scheduled_activities) + |> put_view(ScheduledActivityView) + |> render("index.json", %{scheduled_activities: scheduled_activities}) + end + end + + def show_scheduled_status(%{assigns: %{user: user}} = conn, %{"id" => scheduled_activity_id}) do + with %ScheduledActivity{} = scheduled_activity <- + ScheduledActivity.get(user, scheduled_activity_id) do + conn + |> put_view(ScheduledActivityView) + |> render("show.json", %{scheduled_activity: scheduled_activity}) + else + _ -> {:error, :not_found} + end + end + + def update_scheduled_status( + %{assigns: %{user: user}} = conn, + %{"id" => scheduled_activity_id} = params + ) do + with {:ok, scheduled_activity} <- + ScheduledActivity.update(user, scheduled_activity_id, params) do + conn + |> put_view(ScheduledActivityView) + |> render("show.json", %{scheduled_activity: scheduled_activity}) + end + end + + def delete_scheduled_status(%{assigns: %{user: user}} = conn, %{"id" => scheduled_activity_id}) do + with {:ok, %ScheduledActivity{}} <- ScheduledActivity.delete(user, scheduled_activity_id) do + conn + |> json(%{}) + end + end + def post_status(conn, %{"status" => "", "media_ids" => media_ids} = params) when length(media_ids) > 0 do params = @@ -1406,6 +1447,12 @@ def delete_filter(%{assigns: %{user: user}} = conn, %{"id" => filter_id}) do # fallback action # + def errors(conn, {:error, :not_found}) do + conn + |> put_status(404) + |> json(%{error: "Record not found"}) + end + def errors(conn, _) do conn |> put_status(500) diff --git a/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex b/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex new file mode 100644 index 000000000..87aa3729e --- /dev/null +++ b/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex @@ -0,0 +1,23 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.ScheduledActivityView do + use Pleroma.Web, :view + + alias Pleroma.ScheduledActivity + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.MastodonAPI.ScheduledActivityView + + def render("index.json", %{scheduled_activities: scheduled_activities}) do + render_many(scheduled_activities, ScheduledActivityView, "show.json") + end + + def render("show.json", %{scheduled_activity: %ScheduledActivity{} = scheduled_activity}) do + %{ + id: scheduled_activity.id |> to_string, + scheduled_at: scheduled_activity.scheduled_at |> CommonAPI.Utils.to_masto_date(), + params: scheduled_activity.params + } + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 1c752e44c..3b5ac6fdd 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -244,6 +244,9 @@ defmodule Pleroma.Web.Router do get("/notifications", MastodonAPIController, :notifications) get("/notifications/:id", MastodonAPIController, :get_notification) + get("/scheduled_statuses", MastodonAPIController, :scheduled_statuses) + get("/scheduled_statuses/:id", MastodonAPIController, :show_scheduled_status) + get("/lists", MastodonAPIController, :get_lists) get("/lists/:id", MastodonAPIController, :get_list) get("/lists/:id/accounts", MastodonAPIController, :list_accounts) @@ -278,6 +281,9 @@ defmodule Pleroma.Web.Router do post("/statuses/:id/mute", MastodonAPIController, :mute_conversation) post("/statuses/:id/unmute", MastodonAPIController, :unmute_conversation) + put("/scheduled_statuses/:id", MastodonAPIController, :update_scheduled_status) + delete("/scheduled_statuses/:id", MastodonAPIController, :delete_scheduled_status) + post("/media", MastodonAPIController, :upload) put("/media/:id", MastodonAPIController, :update_media) diff --git a/priv/repo/migrations/20190328053912_create_scheduled_activities.exs b/priv/repo/migrations/20190328053912_create_scheduled_activities.exs new file mode 100644 index 000000000..dc2436dce --- /dev/null +++ b/priv/repo/migrations/20190328053912_create_scheduled_activities.exs @@ -0,0 +1,15 @@ +defmodule Pleroma.Repo.Migrations.CreateScheduledActivities do + use Ecto.Migration + + def change do + create table(:scheduled_activities) do + add(:user_id, references(:users, type: :uuid, on_delete: :delete_all)) + add(:scheduled_at, :naive_datetime, null: false) + add(:params, :map, null: false) + + timestamps() + end + + create(index(:scheduled_activities, [:scheduled_at])) + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index b37bc2c07..667f59e8c 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -23,6 +23,14 @@ def user_factory do } end + def scheduled_activity_factory do + %Pleroma.ScheduledActivity{ + user: build(:user), + scheduled_at: NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(60), :millisecond), + params: build(:note) |> Map.from_struct() |> Map.get(:data) + } + end + def note_factory(attrs \\ %{}) do text = sequence(:text, &"This is :moominmamma: note #{&1}") diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 438e9507d..864c0ad4d 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -10,6 +10,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.CommonAPI @@ -2407,4 +2408,107 @@ test "redirects to the getting-started page when referer is not present", %{conn assert redirected_to(conn) == "/web/getting-started" end end + + describe "scheduled activities" do + test "shows scheduled activities", %{conn: conn} do + user = insert(:user) + scheduled_activity_id1 = insert(:scheduled_activity, user: user).id |> to_string() + scheduled_activity_id2 = insert(:scheduled_activity, user: user).id |> to_string() + scheduled_activity_id3 = insert(:scheduled_activity, user: user).id |> to_string() + scheduled_activity_id4 = insert(:scheduled_activity, user: user).id |> to_string() + + conn = + conn + |> assign(:user, user) + + # min_id + conn_res = + conn + |> get("/api/v1/scheduled_statuses?limit=2&min_id=#{scheduled_activity_id1}") + + result = json_response(conn_res, 200) + assert [%{"id" => ^scheduled_activity_id3}, %{"id" => ^scheduled_activity_id2}] = result + + # since_id + conn_res = + conn + |> get("/api/v1/scheduled_statuses?limit=2&since_id=#{scheduled_activity_id1}") + + result = json_response(conn_res, 200) + assert [%{"id" => ^scheduled_activity_id4}, %{"id" => ^scheduled_activity_id3}] = result + + # max_id + conn_res = + conn + |> get("/api/v1/scheduled_statuses?limit=2&max_id=#{scheduled_activity_id4}") + + result = json_response(conn_res, 200) + assert [%{"id" => ^scheduled_activity_id3}, %{"id" => ^scheduled_activity_id2}] = result + end + + test "shows a scheduled activity", %{conn: conn} do + user = insert(:user) + scheduled_activity = insert(:scheduled_activity, user: user) + + res_conn = + conn + |> assign(:user, user) + |> get("/api/v1/scheduled_statuses/#{scheduled_activity.id}") + + assert %{"id" => scheduled_activity_id} = json_response(res_conn, 200) + assert scheduled_activity_id == scheduled_activity.id |> to_string() + + res_conn = + conn + |> assign(:user, user) + |> get("/api/v1/scheduled_statuses/404") + + assert %{"error" => "Record not found"} = json_response(res_conn, 404) + end + + test "updates a scheduled activity", %{conn: conn} do + user = insert(:user) + scheduled_activity = insert(:scheduled_activity, user: user) + + new_scheduled_at = + NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(120), :millisecond) + + res_conn = + conn + |> assign(:user, user) + |> put("/api/v1/scheduled_statuses/#{scheduled_activity.id}", %{ + scheduled_at: new_scheduled_at + }) + + assert %{"scheduled_at" => expected_scheduled_at} = json_response(res_conn, 200) + assert expected_scheduled_at == Pleroma.Web.CommonAPI.Utils.to_masto_date(new_scheduled_at) + + res_conn = + conn + |> assign(:user, user) + |> put("/api/v1/scheduled_statuses/404", %{scheduled_at: new_scheduled_at}) + + assert %{"error" => "Record not found"} = json_response(res_conn, 404) + end + + test "deletes a scheduled activity", %{conn: conn} do + user = insert(:user) + scheduled_activity = insert(:scheduled_activity, user: user) + + res_conn = + conn + |> assign(:user, user) + |> delete("/api/v1/scheduled_statuses/#{scheduled_activity.id}") + + assert %{} = json_response(res_conn, 200) + assert nil == Repo.get(ScheduledActivity, scheduled_activity.id) + + res_conn = + conn + |> assign(:user, user) + |> delete("/api/v1/scheduled_statuses/#{scheduled_activity.id}") + + assert %{"error" => "Record not found"} = json_response(res_conn, 404) + end + end end From b3870df51fb2f35c3e51bea435134fe3fb692ef8 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Sat, 30 Mar 2019 12:58:40 +0300 Subject: [PATCH 51/59] Handle `scheduled_at` on status creation. --- lib/pleroma/activity.ex | 2 +- lib/pleroma/scheduled_activity.ex | 16 +++++++++ .../mastodon_api/mastodon_api_controller.ex | 27 +++++++++++--- .../mastodon_api_controller_test.exs | 36 +++++++++++++++++++ 4 files changed, 75 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index bc3f8caba..ab8861b27 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Activity do field(:data, :map) field(:local, :boolean, default: true) field(:actor, :string) - field(:recipients, {:array, :string}) + field(:recipients, {:array, :string}, default: []) has_many(:notifications, Notification, on_delete: :delete_all) # Attention: this is a fake relation, don't try to preload it blindly and expect it to work! diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 0c1b26a33..9fdc13990 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -12,6 +12,8 @@ defmodule Pleroma.ScheduledActivity do import Ecto.Query import Ecto.Changeset + @min_offset :timer.minutes(5) + schema "scheduled_activities" do belongs_to(:user, User, type: Pleroma.FlakeId) field(:scheduled_at, :naive_datetime) @@ -30,6 +32,20 @@ def update_changeset(%ScheduledActivity{} = scheduled_activity, attrs) do |> cast(attrs, [:scheduled_at]) end + def far_enough?(scheduled_at) when is_binary(scheduled_at) do + with {:ok, scheduled_at} <- Ecto.Type.cast(:naive_datetime, scheduled_at) do + far_enough?(scheduled_at) + else + _ -> false + end + end + + def far_enough?(scheduled_at) do + now = NaiveDateTime.utc_now() + diff = NaiveDateTime.diff(scheduled_at, now, :millisecond) + diff > @min_offset + end + def new(%User{} = user, attrs) do %ScheduledActivity{user_id: user.id} |> changeset(attrs) diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 0916d84dc..863fc3954 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -425,12 +425,29 @@ def post_status(%{assigns: %{user: user}} = conn, %{"status" => _} = params) do _ -> Ecto.UUID.generate() end - {:ok, activity} = - Cachex.fetch!(:idempotency_cache, idempotency_key, fn _ -> CommonAPI.post(user, params) end) + scheduled_at = params["scheduled_at"] - conn - |> put_view(StatusView) - |> try_render("status.json", %{activity: activity, for: user, as: :activity}) + if scheduled_at && ScheduledActivity.far_enough?(scheduled_at) do + {:ok, scheduled_activity} = + Cachex.fetch!(:idempotency_cache, idempotency_key, fn _ -> + ScheduledActivity.create(user, %{"params" => params, "scheduled_at" => scheduled_at}) + end) + + conn + |> put_view(ScheduledActivityView) + |> render("show.json", %{scheduled_activity: scheduled_activity}) + else + params = Map.drop(params, ["scheduled_at"]) + + {:ok, activity} = + Cachex.fetch!(:idempotency_cache, idempotency_key, fn _ -> + CommonAPI.post(user, params) + end) + + conn + |> put_view(StatusView) + |> try_render("status.json", %{activity: activity, for: user, as: :activity}) + end end def delete_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 864c0ad4d..0ec66ab73 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2410,6 +2410,42 @@ test "redirects to the getting-started page when referer is not present", %{conn end describe "scheduled activities" do + test "creates a scheduled activity", %{conn: conn} do + user = insert(:user) + scheduled_at = NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(120), :millisecond) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "scheduled", + "scheduled_at" => scheduled_at + }) + + assert %{"scheduled_at" => expected_scheduled_at} = json_response(conn, 200) + assert expected_scheduled_at == Pleroma.Web.CommonAPI.Utils.to_masto_date(scheduled_at) + assert [] == Repo.all(Activity) + end + + test "skips the scheduling and creates the activity if scheduled_at is earlier than 5 minutes from now", + %{conn: conn} do + user = insert(:user) + + scheduled_at = + NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(5) - 1, :millisecond) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "status" => "not scheduled", + "scheduled_at" => scheduled_at + }) + + assert %{"content" => "not scheduled"} = json_response(conn, 200) + assert [] == Repo.all(ScheduledActivity) + end + test "shows scheduled activities", %{conn: conn} do user = insert(:user) scheduled_activity_id1 = insert(:scheduled_activity, user: user).id |> to_string() From fc92a0fd8d5be0352f4791b79bda04960f36f707 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Tue, 2 Apr 2019 01:31:01 +0300 Subject: [PATCH 52/59] Added limits and media attachments for scheduled activities. --- config/config.exs | 4 + docs/config.md | 9 +- lib/pleroma/object.ex | 8 ++ lib/pleroma/scheduled_activity.ex | 83 ++++++++++++++--- .../mastodon_api/mastodon_api_controller.ex | 18 +++- .../views/scheduled_activity_view.ex | 32 ++++++- ...0328053912_create_scheduled_activities.exs | 1 + test/scheduled_activity_test.exs | 93 +++++++++++++++++++ test/support/factory.ex | 16 ++-- .../mastodon_api_controller_test.exs | 25 +++++ .../scheduled_activity_view_test.exs | 68 ++++++++++++++ 11 files changed, 327 insertions(+), 30 deletions(-) create mode 100644 test/scheduled_activity_test.exs create mode 100644 test/web/mastodon_api/scheduled_activity_view_test.exs diff --git a/config/config.exs b/config/config.exs index 61e799f33..79cef87e6 100644 --- a/config/config.exs +++ b/config/config.exs @@ -367,6 +367,10 @@ enabled: false, pages: 5 +config :pleroma, Pleroma.ScheduledActivity, + daily_user_limit: 25, + total_user_limit: 100 + config :auto_linker, opts: [ scheme: true, diff --git a/docs/config.md b/docs/config.md index 06d6fd757..df21beff3 100644 --- a/docs/config.md +++ b/docs/config.md @@ -218,14 +218,14 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i - `port` * `url` - a list containing the configuration for generating urls, accepts - `host` - the host without the scheme and a post (e.g `example.com`, not `https://example.com:2020`) - - `scheme` - e.g `http`, `https` + - `scheme` - e.g `http`, `https` - `port` - `path` **Important note**: if you modify anything inside these lists, default `config.exs` values will be overwritten, which may result in breakage, to make sure this does not happen please copy the default value for the list from `config.exs` and modify/add only what you need -Example: +Example: ```elixir config :pleroma, Pleroma.Web.Endpoint, url: [host: "example.com", port: 2020, scheme: "https"], @@ -412,3 +412,8 @@ Pleroma account will be created with the same name as the LDAP user name. * `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator * `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication + +## Pleroma.ScheduledActivity + +* `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day +* `total_user_limit`: the number of scheduled activities a user is allowed to create in total diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 013d62157..786d6296c 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -184,4 +184,12 @@ def decrease_replies_count(ap_id) do _ -> {:error, "Not found"} end end + + def enforce_user_objects(user, object_ids) do + Object + |> where([o], fragment("?->>'actor' = ?", o.data, ^user.ap_id)) + |> where([o], o.id in ^object_ids) + |> select([o], o.id) + |> Repo.all() + end end diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 9fdc13990..723eb6dc3 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -5,9 +5,12 @@ defmodule Pleroma.ScheduledActivity do use Ecto.Schema + alias Pleroma.Config + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User + alias Pleroma.Web.CommonAPI.Utils import Ecto.Query import Ecto.Changeset @@ -25,11 +28,69 @@ defmodule Pleroma.ScheduledActivity do def changeset(%ScheduledActivity{} = scheduled_activity, attrs) do scheduled_activity |> cast(attrs, [:scheduled_at, :params]) + |> validate_required([:scheduled_at, :params]) + |> validate_scheduled_at() + |> with_media_attachments() end + defp with_media_attachments( + %{changes: %{params: %{"media_ids" => media_ids} = params}} = changeset + ) + when is_list(media_ids) do + user = User.get_cached_by_id(changeset.data.user_id) + media_ids = Object.enforce_user_objects(user, media_ids) |> Enum.map(&to_string(&1)) + media_attachments = Utils.attachments_from_ids(%{"media_ids" => media_ids}) + + params = + params + |> Map.put("media_attachments", media_attachments) + |> Map.put("media_ids", media_ids) + + put_change(changeset, :params, params) + end + + defp with_media_attachments(changeset), do: changeset + def update_changeset(%ScheduledActivity{} = scheduled_activity, attrs) do scheduled_activity |> cast(attrs, [:scheduled_at]) + |> validate_required([:scheduled_at]) + |> validate_scheduled_at() + end + + def validate_scheduled_at(changeset) do + validate_change(changeset, :scheduled_at, fn _, scheduled_at -> + cond do + not far_enough?(scheduled_at) -> + [scheduled_at: "must be at least 5 minutes from now"] + + exceeds_daily_user_limit?(changeset.data.user_id, scheduled_at) -> + [scheduled_at: "daily limit exceeded"] + + exceeds_total_user_limit?(changeset.data.user_id) -> + [scheduled_at: "total limit exceeded"] + + true -> + [] + end + end) + end + + def exceeds_daily_user_limit?(user_id, scheduled_at) do + ScheduledActivity + |> where(user_id: ^user_id) + |> where([s], type(s.scheduled_at, :date) == type(^scheduled_at, :date)) + |> select([u], count(u.id)) + |> Repo.one() + |> Kernel.>=(Config.get([ScheduledActivity, :daily_user_limit])) + end + + def exceeds_total_user_limit?(user_id) do + ScheduledActivity + |> where(user_id: ^user_id) + |> select([u], count(u.id)) + |> Repo.one() + |> Kernel.>=(Config.get([ScheduledActivity, :total_user_limit])) end def far_enough?(scheduled_at) when is_binary(scheduled_at) do @@ -64,23 +125,15 @@ def get(%User{} = user, scheduled_activity_id) do |> Repo.one() end - def update(%User{} = user, scheduled_activity_id, attrs) do - with %ScheduledActivity{} = scheduled_activity <- get(user, scheduled_activity_id) do - scheduled_activity - |> update_changeset(attrs) - |> Repo.update() - else - nil -> {:error, :not_found} - end + def update(scheduled_activity, attrs) do + scheduled_activity + |> update_changeset(attrs) + |> Repo.update() end - def delete(%User{} = user, scheduled_activity_id) do - with %ScheduledActivity{} = scheduled_activity <- get(user, scheduled_activity_id) do - scheduled_activity - |> Repo.delete() - else - nil -> {:error, :not_found} - end + def delete(scheduled_activity) do + scheduled_activity + |> Repo.delete() end def for_user_query(%User{} = user) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 863fc3954..6cb5df378 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -390,18 +390,28 @@ def update_scheduled_status( %{assigns: %{user: user}} = conn, %{"id" => scheduled_activity_id} = params ) do - with {:ok, scheduled_activity} <- - ScheduledActivity.update(user, scheduled_activity_id, params) do + with %ScheduledActivity{} = scheduled_activity <- + ScheduledActivity.get(user, scheduled_activity_id), + {:ok, scheduled_activity} <- ScheduledActivity.update(scheduled_activity, params) do conn |> put_view(ScheduledActivityView) |> render("show.json", %{scheduled_activity: scheduled_activity}) + else + nil -> {:error, :not_found} + error -> error end end def delete_scheduled_status(%{assigns: %{user: user}} = conn, %{"id" => scheduled_activity_id}) do - with {:ok, %ScheduledActivity{}} <- ScheduledActivity.delete(user, scheduled_activity_id) do + with %ScheduledActivity{} = scheduled_activity <- + ScheduledActivity.get(user, scheduled_activity_id), + {:ok, scheduled_activity} <- ScheduledActivity.delete(scheduled_activity) do conn - |> json(%{}) + |> put_view(ScheduledActivityView) + |> render("show.json", %{scheduled_activity: scheduled_activity}) + else + nil -> {:error, :not_found} + error -> error end end diff --git a/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex b/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex index 87aa3729e..1ebff7aba 100644 --- a/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex +++ b/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.MastodonAPI.ScheduledActivityView do alias Pleroma.ScheduledActivity alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.ScheduledActivityView + alias Pleroma.Web.MastodonAPI.StatusView def render("index.json", %{scheduled_activities: scheduled_activities}) do render_many(scheduled_activities, ScheduledActivityView, "show.json") @@ -17,7 +18,36 @@ def render("show.json", %{scheduled_activity: %ScheduledActivity{} = scheduled_a %{ id: scheduled_activity.id |> to_string, scheduled_at: scheduled_activity.scheduled_at |> CommonAPI.Utils.to_masto_date(), - params: scheduled_activity.params + params: status_params(scheduled_activity.params) } + |> with_media_attachments(scheduled_activity) + end + + defp with_media_attachments(data, %{params: %{"media_attachments" => media_attachments}}) do + attachments = render_many(media_attachments, StatusView, "attachment.json", as: :attachment) + Map.put(data, :media_attachments, attachments) + end + + defp with_media_attachments(data, _), do: data + + defp status_params(params) do + data = %{ + text: params["status"], + sensitive: params["sensitive"], + spoiler_text: params["spoiler_text"], + visibility: params["visibility"], + scheduled_at: params["scheduled_at"], + poll: params["poll"], + in_reply_to_id: params["in_reply_to_id"] + } + + data = + if media_ids = params["media_ids"] do + Map.put(data, :media_ids, media_ids) + else + data + end + + data end end diff --git a/priv/repo/migrations/20190328053912_create_scheduled_activities.exs b/priv/repo/migrations/20190328053912_create_scheduled_activities.exs index dc2436dce..dd737e25a 100644 --- a/priv/repo/migrations/20190328053912_create_scheduled_activities.exs +++ b/priv/repo/migrations/20190328053912_create_scheduled_activities.exs @@ -11,5 +11,6 @@ def change do end create(index(:scheduled_activities, [:scheduled_at])) + create(index(:scheduled_activities, [:user_id])) end end diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs new file mode 100644 index 000000000..c49c65c0a --- /dev/null +++ b/test/scheduled_activity_test.exs @@ -0,0 +1,93 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ScheduledActivityTest do + use Pleroma.DataCase + alias Pleroma.Config + alias Pleroma.DataCase + alias Pleroma.ScheduledActivity + alias Pleroma.Web.ActivityPub.ActivityPub + import Pleroma.Factory + + setup context do + Config.put([ScheduledActivity, :daily_user_limit], 2) + Config.put([ScheduledActivity, :total_user_limit], 3) + DataCase.ensure_local_uploader(context) + end + + describe "creation" do + test "when daily user limit is exceeded" do + user = insert(:user) + + today = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(6), :millisecond) + |> NaiveDateTime.to_iso8601() + + attrs = %{params: %{}, scheduled_at: today} + {:ok, _} = ScheduledActivity.create(user, attrs) + {:ok, _} = ScheduledActivity.create(user, attrs) + {:error, changeset} = ScheduledActivity.create(user, attrs) + assert changeset.errors == [scheduled_at: {"daily limit exceeded", []}] + end + + test "when total user limit is exceeded" do + user = insert(:user) + + today = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(6), :millisecond) + |> NaiveDateTime.to_iso8601() + + tomorrow = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.hours(24), :millisecond) + |> NaiveDateTime.to_iso8601() + + {:ok, _} = ScheduledActivity.create(user, %{params: %{}, scheduled_at: today}) + {:ok, _} = ScheduledActivity.create(user, %{params: %{}, scheduled_at: today}) + {:ok, _} = ScheduledActivity.create(user, %{params: %{}, scheduled_at: tomorrow}) + {:error, changeset} = ScheduledActivity.create(user, %{params: %{}, scheduled_at: tomorrow}) + assert changeset.errors == [scheduled_at: {"total limit exceeded", []}] + end + + test "when scheduled_at is earlier than 5 minute from now" do + user = insert(:user) + + scheduled_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(4), :millisecond) + |> NaiveDateTime.to_iso8601() + + attrs = %{params: %{}, scheduled_at: scheduled_at} + {:error, changeset} = ScheduledActivity.create(user, attrs) + assert changeset.errors == [scheduled_at: {"must be at least 5 minutes from now", []}] + end + + test "excludes attachments belonging to another user" do + user = insert(:user) + another_user = insert(:user) + + scheduled_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(10), :millisecond) + |> NaiveDateTime.to_iso8601() + + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + {:ok, user_upload} = ActivityPub.upload(file, actor: user.ap_id) + {:ok, another_user_upload} = ActivityPub.upload(file, actor: another_user.ap_id) + + media_ids = [user_upload.id, another_user_upload.id] + attrs = %{params: %{"media_ids" => media_ids}, scheduled_at: scheduled_at} + {:ok, scheduled_activity} = ScheduledActivity.create(user, attrs) + assert to_string(user_upload.id) in scheduled_activity.params["media_ids"] + refute to_string(another_user_upload.id) in scheduled_activity.params["media_ids"] + end + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index 667f59e8c..608f8d46b 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -23,14 +23,6 @@ def user_factory do } end - def scheduled_activity_factory do - %Pleroma.ScheduledActivity{ - user: build(:user), - scheduled_at: NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(60), :millisecond), - params: build(:note) |> Map.from_struct() |> Map.get(:data) - } - end - def note_factory(attrs \\ %{}) do text = sequence(:text, &"This is :moominmamma: note #{&1}") @@ -275,4 +267,12 @@ def notification_factory do user: build(:user) } end + + def scheduled_activity_factory do + %Pleroma.ScheduledActivity{ + user: build(:user), + scheduled_at: NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(60), :millisecond), + params: build(:note) |> Map.from_struct() |> Map.get(:data) + } + end end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index 0ec66ab73..ae2375696 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2427,6 +2427,31 @@ test "creates a scheduled activity", %{conn: conn} do assert [] == Repo.all(Activity) end + test "creates a scheduled activity with a media attachment", %{conn: conn} do + user = insert(:user) + scheduled_at = NaiveDateTime.add(NaiveDateTime.utc_now(), :timer.minutes(120), :millisecond) + + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + {:ok, upload} = ActivityPub.upload(file, actor: user.ap_id) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{ + "media_ids" => [to_string(upload.id)], + "status" => "scheduled", + "scheduled_at" => scheduled_at + }) + + assert %{"media_attachments" => [media_attachment]} = json_response(conn, 200) + assert %{"type" => "image"} = media_attachment + end + test "skips the scheduling and creates the activity if scheduled_at is earlier than 5 minutes from now", %{conn: conn} do user = insert(:user) diff --git a/test/web/mastodon_api/scheduled_activity_view_test.exs b/test/web/mastodon_api/scheduled_activity_view_test.exs new file mode 100644 index 000000000..26747a0c0 --- /dev/null +++ b/test/web/mastodon_api/scheduled_activity_view_test.exs @@ -0,0 +1,68 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.MastodonAPI.ScheduledActivityViewTest do + use Pleroma.DataCase + alias Pleroma.ScheduledActivity + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Web.MastodonAPI.ScheduledActivityView + alias Pleroma.Web.MastodonAPI.StatusView + import Pleroma.Factory + + test "A scheduled activity with a media attachment" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{"status" => "hi"}) + + scheduled_at = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(10), :millisecond) + |> NaiveDateTime.to_iso8601() + + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + {:ok, upload} = ActivityPub.upload(file, actor: user.ap_id) + + attrs = %{ + params: %{ + "media_ids" => [upload.id], + "status" => "hi", + "sensitive" => true, + "spoiler_text" => "spoiler", + "visibility" => "unlisted", + "in_reply_to_id" => to_string(activity.id) + }, + scheduled_at: scheduled_at + } + + {:ok, scheduled_activity} = ScheduledActivity.create(user, attrs) + result = ScheduledActivityView.render("show.json", %{scheduled_activity: scheduled_activity}) + + expected = %{ + id: to_string(scheduled_activity.id), + media_attachments: + %{"media_ids" => [upload.id]} + |> Utils.attachments_from_ids() + |> Enum.map(&StatusView.render("attachment.json", %{attachment: &1})), + params: %{ + in_reply_to_id: to_string(activity.id), + media_ids: [to_string(upload.id)], + poll: nil, + scheduled_at: nil, + sensitive: true, + spoiler_text: "spoiler", + text: "hi", + visibility: "unlisted" + }, + scheduled_at: Utils.to_masto_date(scheduled_activity.scheduled_at) + } + + assert expected == result + end +end From 2056efa714460faaf25f6bc03ab643f5a2e8cd3d Mon Sep 17 00:00:00 2001 From: eugenijm Date: Wed, 3 Apr 2019 18:55:04 +0300 Subject: [PATCH 53/59] Add scheduler for sending scheduled activities to the queue --- config/config.exs | 12 ++-- config/test.exs | 5 ++ docs/config.md | 8 ++- lib/pleroma/application.ex | 3 +- lib/pleroma/object.ex | 8 --- lib/pleroma/scheduled_activity.ex | 34 ++++++++--- lib/pleroma/scheduled_activity_worker.ex | 58 +++++++++++++++++++ .../mastodon_api/mastodon_api_controller.ex | 26 ++++++--- .../views/scheduled_activity_view.ex | 12 ++-- test/scheduled_activity_test.exs | 31 +--------- test/scheduled_activity_worker_test.exs | 19 ++++++ .../mastodon_api_controller_test.exs | 46 +++++++++++++++ .../scheduled_activity_view_test.exs | 2 +- 13 files changed, 196 insertions(+), 68 deletions(-) create mode 100644 lib/pleroma/scheduled_activity_worker.ex create mode 100644 test/scheduled_activity_worker_test.exs diff --git a/config/config.exs b/config/config.exs index 79cef87e6..8a977ece5 100644 --- a/config/config.exs +++ b/config/config.exs @@ -361,16 +361,13 @@ config :pleroma_job_queue, :queues, federator_incoming: 50, federator_outgoing: 50, - mailer: 10 + mailer: 10, + scheduled_activities: 10 config :pleroma, :fetch_initial_posts, enabled: false, pages: 5 -config :pleroma, Pleroma.ScheduledActivity, - daily_user_limit: 25, - total_user_limit: 100 - config :auto_linker, opts: [ scheme: true, @@ -396,6 +393,11 @@ config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics" +config :pleroma, Pleroma.ScheduledActivity, + daily_user_limit: 25, + total_user_limit: 300, + enabled: true + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/test.exs b/config/test.exs index 6a7b9067e..894fa8d3d 100644 --- a/config/test.exs +++ b/config/test.exs @@ -50,6 +50,11 @@ config :pleroma_job_queue, disabled: true +config :pleroma, Pleroma.ScheduledActivity, + daily_user_limit: 2, + total_user_limit: 3, + enabled: false + try do import_config "test.secret.exs" rescue diff --git a/docs/config.md b/docs/config.md index df21beff3..ba0759e87 100644 --- a/docs/config.md +++ b/docs/config.md @@ -317,6 +317,7 @@ Pleroma has the following queues: * `federator_outgoing` - Outgoing federation * `federator_incoming` - Incoming federation * `mailer` - Email sender, see [`Pleroma.Mailer`](#pleroma-mailer) +* `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivities`](#pleromascheduledactivity) Example: @@ -413,7 +414,8 @@ Pleroma account will be created with the same name as the LDAP user name. * `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator * `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication -## Pleroma.ScheduledActivity +## Pleroma.ScheduledActivity -* `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day -* `total_user_limit`: the number of scheduled activities a user is allowed to create in total +* `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day (Default: `25`) +* `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`) +* `enabled`: whether scheduled activities are sent to the job queue to be executed diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 1fc3fb728..f0cb7d9a8 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -104,7 +104,8 @@ def start(_type, _args) do ], id: :cachex_idem ), - worker(Pleroma.FlakeId, []) + worker(Pleroma.FlakeId, []), + worker(Pleroma.ScheduledActivityWorker, []) ] ++ hackney_pool_children() ++ [ diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 786d6296c..013d62157 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -184,12 +184,4 @@ def decrease_replies_count(ap_id) do _ -> {:error, "Not found"} end end - - def enforce_user_objects(user, object_ids) do - Object - |> where([o], fragment("?->>'actor' = ?", o.data, ^user.ap_id)) - |> where([o], o.id in ^object_ids) - |> select([o], o.id) - |> Repo.all() - end end diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 723eb6dc3..de0e54699 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -6,7 +6,6 @@ defmodule Pleroma.ScheduledActivity do use Ecto.Schema alias Pleroma.Config - alias Pleroma.Object alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User @@ -37,8 +36,6 @@ defp with_media_attachments( %{changes: %{params: %{"media_ids" => media_ids} = params}} = changeset ) when is_list(media_ids) do - user = User.get_cached_by_id(changeset.data.user_id) - media_ids = Object.enforce_user_objects(user, media_ids) |> Enum.map(&to_string(&1)) media_attachments = Utils.attachments_from_ids(%{"media_ids" => media_ids}) params = @@ -79,8 +76,8 @@ def validate_scheduled_at(changeset) do def exceeds_daily_user_limit?(user_id, scheduled_at) do ScheduledActivity |> where(user_id: ^user_id) - |> where([s], type(s.scheduled_at, :date) == type(^scheduled_at, :date)) - |> select([u], count(u.id)) + |> where([sa], type(sa.scheduled_at, :date) == type(^scheduled_at, :date)) + |> select([sa], count(sa.id)) |> Repo.one() |> Kernel.>=(Config.get([ScheduledActivity, :daily_user_limit])) end @@ -88,7 +85,7 @@ def exceeds_daily_user_limit?(user_id, scheduled_at) do def exceeds_total_user_limit?(user_id) do ScheduledActivity |> where(user_id: ^user_id) - |> select([u], count(u.id)) + |> select([sa], count(sa.id)) |> Repo.one() |> Kernel.>=(Config.get([ScheduledActivity, :total_user_limit])) end @@ -125,19 +122,40 @@ def get(%User{} = user, scheduled_activity_id) do |> Repo.one() end - def update(scheduled_activity, attrs) do + def update(%ScheduledActivity{} = scheduled_activity, attrs) do scheduled_activity |> update_changeset(attrs) |> Repo.update() end - def delete(scheduled_activity) do + def delete(%ScheduledActivity{} = scheduled_activity) do scheduled_activity |> Repo.delete() end + def delete(id) when is_binary(id) or is_integer(id) do + ScheduledActivity + |> where(id: ^id) + |> select([sa], sa) + |> Repo.delete_all() + |> case do + {1, [scheduled_activity]} -> {:ok, scheduled_activity} + _ -> :error + end + end + def for_user_query(%User{} = user) do ScheduledActivity |> where(user_id: ^user.id) end + + def due_activities(offset \\ 0) do + naive_datetime = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(offset, :millisecond) + + ScheduledActivity + |> where([sa], sa.scheduled_at < ^naive_datetime) + |> Repo.all() + end end diff --git a/lib/pleroma/scheduled_activity_worker.ex b/lib/pleroma/scheduled_activity_worker.ex new file mode 100644 index 000000000..65b38622f --- /dev/null +++ b/lib/pleroma/scheduled_activity_worker.ex @@ -0,0 +1,58 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ScheduledActivityWorker do + @moduledoc """ + Sends scheduled activities to the job queue. + """ + + alias Pleroma.Config + alias Pleroma.ScheduledActivity + alias Pleroma.User + alias Pleroma.Web.CommonAPI + use GenServer + require Logger + + @schedule_interval :timer.minutes(1) + + def start_link do + GenServer.start_link(__MODULE__, nil) + end + + def init(_) do + if Config.get([ScheduledActivity, :enabled]) do + schedule_next() + {:ok, nil} + else + :ignore + end + end + + def perform(:execute, scheduled_activity_id) do + try do + {:ok, scheduled_activity} = ScheduledActivity.delete(scheduled_activity_id) + %User{} = user = User.get_cached_by_id(scheduled_activity.user_id) + {:ok, _result} = CommonAPI.post(user, scheduled_activity.params) + rescue + error -> + Logger.error( + "#{__MODULE__} Couldn't create a status from the scheduled activity: #{inspect(error)}" + ) + end + end + + def handle_info(:perform, state) do + ScheduledActivity.due_activities(@schedule_interval) + |> Enum.each(fn scheduled_activity -> + PleromaJobQueue.enqueue(:scheduled_activities, __MODULE__, [:execute, scheduled_activity.id]) + end) + + schedule_next() + {:noreply, state} + end + + defp schedule_next do + Process.send_after(self(), :perform, @schedule_interval) + end +end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 6cb5df378..fc8a2458c 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do use Pleroma.Web, :controller + alias Ecto.Changeset alias Pleroma.Activity alias Pleroma.Config alias Pleroma.Filter @@ -438,14 +439,12 @@ def post_status(%{assigns: %{user: user}} = conn, %{"status" => _} = params) do scheduled_at = params["scheduled_at"] if scheduled_at && ScheduledActivity.far_enough?(scheduled_at) do - {:ok, scheduled_activity} = - Cachex.fetch!(:idempotency_cache, idempotency_key, fn _ -> - ScheduledActivity.create(user, %{"params" => params, "scheduled_at" => scheduled_at}) - end) - - conn - |> put_view(ScheduledActivityView) - |> render("show.json", %{scheduled_activity: scheduled_activity}) + with {:ok, scheduled_activity} <- + ScheduledActivity.create(user, %{"params" => params, "scheduled_at" => scheduled_at}) do + conn + |> put_view(ScheduledActivityView) + |> render("show.json", %{scheduled_activity: scheduled_activity}) + end else params = Map.drop(params, ["scheduled_at"]) @@ -1474,6 +1473,17 @@ def delete_filter(%{assigns: %{user: user}} = conn, %{"id" => filter_id}) do # fallback action # + def errors(conn, {:error, %Changeset{} = changeset}) do + error_message = + changeset + |> Changeset.traverse_errors(fn {message, _opt} -> message end) + |> Enum.map_join(", ", fn {_k, v} -> v end) + + conn + |> put_status(422) + |> json(%{error: error_message}) + end + def errors(conn, {:error, :not_found}) do conn |> put_status(404) diff --git a/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex b/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex index 1ebff7aba..0aae15ab9 100644 --- a/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex +++ b/lib/pleroma/web/mastodon_api/views/scheduled_activity_view.ex @@ -16,16 +16,20 @@ def render("index.json", %{scheduled_activities: scheduled_activities}) do def render("show.json", %{scheduled_activity: %ScheduledActivity{} = scheduled_activity}) do %{ - id: scheduled_activity.id |> to_string, - scheduled_at: scheduled_activity.scheduled_at |> CommonAPI.Utils.to_masto_date(), + id: to_string(scheduled_activity.id), + scheduled_at: CommonAPI.Utils.to_masto_date(scheduled_activity.scheduled_at), params: status_params(scheduled_activity.params) } |> with_media_attachments(scheduled_activity) end defp with_media_attachments(data, %{params: %{"media_attachments" => media_attachments}}) do - attachments = render_many(media_attachments, StatusView, "attachment.json", as: :attachment) - Map.put(data, :media_attachments, attachments) + try do + attachments = render_many(media_attachments, StatusView, "attachment.json", as: :attachment) + Map.put(data, :media_attachments, attachments) + rescue + _ -> data + end end defp with_media_attachments(data, _), do: data diff --git a/test/scheduled_activity_test.exs b/test/scheduled_activity_test.exs index c49c65c0a..edc7cc3f9 100644 --- a/test/scheduled_activity_test.exs +++ b/test/scheduled_activity_test.exs @@ -4,15 +4,11 @@ defmodule Pleroma.ScheduledActivityTest do use Pleroma.DataCase - alias Pleroma.Config alias Pleroma.DataCase alias Pleroma.ScheduledActivity - alias Pleroma.Web.ActivityPub.ActivityPub import Pleroma.Factory setup context do - Config.put([ScheduledActivity, :daily_user_limit], 2) - Config.put([ScheduledActivity, :total_user_limit], 3) DataCase.ensure_local_uploader(context) end @@ -42,7 +38,7 @@ test "when total user limit is exceeded" do tomorrow = NaiveDateTime.utc_now() - |> NaiveDateTime.add(:timer.hours(24), :millisecond) + |> NaiveDateTime.add(:timer.hours(36), :millisecond) |> NaiveDateTime.to_iso8601() {:ok, _} = ScheduledActivity.create(user, %{params: %{}, scheduled_at: today}) @@ -64,30 +60,5 @@ test "when scheduled_at is earlier than 5 minute from now" do {:error, changeset} = ScheduledActivity.create(user, attrs) assert changeset.errors == [scheduled_at: {"must be at least 5 minutes from now", []}] end - - test "excludes attachments belonging to another user" do - user = insert(:user) - another_user = insert(:user) - - scheduled_at = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(:timer.minutes(10), :millisecond) - |> NaiveDateTime.to_iso8601() - - file = %Plug.Upload{ - content_type: "image/jpg", - path: Path.absname("test/fixtures/image.jpg"), - filename: "an_image.jpg" - } - - {:ok, user_upload} = ActivityPub.upload(file, actor: user.ap_id) - {:ok, another_user_upload} = ActivityPub.upload(file, actor: another_user.ap_id) - - media_ids = [user_upload.id, another_user_upload.id] - attrs = %{params: %{"media_ids" => media_ids}, scheduled_at: scheduled_at} - {:ok, scheduled_activity} = ScheduledActivity.create(user, attrs) - assert to_string(user_upload.id) in scheduled_activity.params["media_ids"] - refute to_string(another_user_upload.id) in scheduled_activity.params["media_ids"] - end end end diff --git a/test/scheduled_activity_worker_test.exs b/test/scheduled_activity_worker_test.exs new file mode 100644 index 000000000..b9c91dda6 --- /dev/null +++ b/test/scheduled_activity_worker_test.exs @@ -0,0 +1,19 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.ScheduledActivityWorkerTest do + use Pleroma.DataCase + alias Pleroma.ScheduledActivity + import Pleroma.Factory + + test "creates a status from the scheduled activity" do + user = insert(:user) + scheduled_activity = insert(:scheduled_activity, user: user, params: %{status: "hi"}) + Pleroma.ScheduledActivityWorker.perform(:execute, scheduled_activity.id) + + refute Repo.get(ScheduledActivity, scheduled_activity.id) + activity = Repo.all(Pleroma.Activity) |> Enum.find(&(&1.actor == user.ap_id)) + assert activity.data["object"]["content"] == "hi" + end +end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index ae2375696..cd01116e2 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -2471,6 +2471,52 @@ test "skips the scheduling and creates the activity if scheduled_at is earlier t assert [] == Repo.all(ScheduledActivity) end + test "returns error when daily user limit is exceeded", %{conn: conn} do + user = insert(:user) + + today = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(6), :millisecond) + |> NaiveDateTime.to_iso8601() + + attrs = %{params: %{}, scheduled_at: today} + {:ok, _} = ScheduledActivity.create(user, attrs) + {:ok, _} = ScheduledActivity.create(user, attrs) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{"status" => "scheduled", "scheduled_at" => today}) + + assert %{"error" => "daily limit exceeded"} == json_response(conn, 422) + end + + test "returns error when total user limit is exceeded", %{conn: conn} do + user = insert(:user) + + today = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.minutes(6), :millisecond) + |> NaiveDateTime.to_iso8601() + + tomorrow = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(:timer.hours(36), :millisecond) + |> NaiveDateTime.to_iso8601() + + attrs = %{params: %{}, scheduled_at: today} + {:ok, _} = ScheduledActivity.create(user, attrs) + {:ok, _} = ScheduledActivity.create(user, attrs) + {:ok, _} = ScheduledActivity.create(user, %{params: %{}, scheduled_at: tomorrow}) + + conn = + conn + |> assign(:user, user) + |> post("/api/v1/statuses", %{"status" => "scheduled", "scheduled_at" => tomorrow}) + + assert %{"error" => "total limit exceeded"} == json_response(conn, 422) + end + test "shows scheduled activities", %{conn: conn} do user = insert(:user) scheduled_activity_id1 = insert(:scheduled_activity, user: user).id |> to_string() diff --git a/test/web/mastodon_api/scheduled_activity_view_test.exs b/test/web/mastodon_api/scheduled_activity_view_test.exs index 26747a0c0..ecbb855d4 100644 --- a/test/web/mastodon_api/scheduled_activity_view_test.exs +++ b/test/web/mastodon_api/scheduled_activity_view_test.exs @@ -52,7 +52,7 @@ test "A scheduled activity with a media attachment" do |> Enum.map(&StatusView.render("attachment.json", %{attachment: &1})), params: %{ in_reply_to_id: to_string(activity.id), - media_ids: [to_string(upload.id)], + media_ids: [upload.id], poll: nil, scheduled_at: nil, sensitive: true, From e3328bc1382315c9067c099995a29db70d9d0433 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 7 Apr 2019 11:08:37 +0300 Subject: [PATCH 54/59] [#923] Removed
elements from auth forms, adjusted docs, minor auth settings refactoring. --- docs/config.md | 16 ++++++++++------ lib/pleroma/web/auth/authenticator.ex | 7 +++++-- .../templates/o_auth/o_auth/consumer.html.eex | 2 -- .../templates/o_auth/o_auth/register.html.eex | 8 +------- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/docs/config.md b/docs/config.md index 36d7f1273..686f1f36b 100644 --- a/docs/config.md +++ b/docs/config.md @@ -390,6 +390,11 @@ config :auto_linker, ] ``` +## Pleroma.Web.Auth.Authenticator + +* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator +* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication + ## :ldap Use LDAP for user authentication. When a user logs in to the Pleroma @@ -408,16 +413,15 @@ Pleroma account will be created with the same name as the LDAP user name. * `base`: LDAP base, e.g. "dc=example,dc=com" * `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base" -## Pleroma.Web.Auth.Authenticator - -* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator -* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication - ## :auth Authentication / authorization settings. -* `oauth_consumer_strategies`: lists enabled OAuth consumer strategies; by default it's set by OAUTH_CONSUMER_STRATEGIES environment variable. +* `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`. +* `oauth_consumer_template`: OAuth consumer mode authentication form template. By default it's `consumer.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex`. +* `oauth_consumer_strategies`: the list of enabled OAuth consumer strategies; by default it's set by OAUTH_CONSUMER_STRATEGIES environment variable. + +# OAuth consumer mode OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.). Implementation is based on Ueberauth; see the list of [available strategies](https://github.com/ueberauth/ueberauth/wiki/List-of-Strategies). diff --git a/lib/pleroma/web/auth/authenticator.ex b/lib/pleroma/web/auth/authenticator.ex index 4eeef5034..89d88af32 100644 --- a/lib/pleroma/web/auth/authenticator.ex +++ b/lib/pleroma/web/auth/authenticator.ex @@ -31,12 +31,15 @@ def handle_error(plug, error), do: implementation().handle_error(plug, error) @callback auth_template() :: String.t() | nil def auth_template do - implementation().auth_template() || Pleroma.Config.get(:auth_template, "show.html") + # Note: `config :pleroma, :auth_template, "..."` support is deprecated + implementation().auth_template() || + Pleroma.Config.get([:auth, :auth_template], Pleroma.Config.get(:auth_template)) || + "show.html" end @callback oauth_consumer_template() :: String.t() | nil def oauth_consumer_template do implementation().oauth_consumer_template() || - Pleroma.Config.get(:oauth_consumer_template, "consumer.html") + Pleroma.Config.get([:auth, :oauth_consumer_template], "consumer.html") end end diff --git a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex index 9365c7c44..85f62ca64 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/consumer.html.eex @@ -1,5 +1,3 @@ -
-

Sign in with external provider

<%= form_for @conn, o_auth_path(@conn, :prepare_request), [method: "get"], fn f -> %> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex index 2e806e5fb..126390391 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex @@ -7,10 +7,7 @@

Registration Details

-

If you'd like to register a new account, -
-please provide the details below.

-
+

If you'd like to register a new account, please provide the details below.

<%= form_for @conn, o_auth_path(@conn, :register), [], fn f -> %> @@ -25,9 +22,6 @@ please provide the details below.

<%= submit "Proceed as new user", name: "op", value: "register" %> -
-
-

Alternatively, sign in to connect to existing account.

From 44829d91818e66da1cbeb13aafecc52a931af17d Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Mon, 8 Apr 2019 12:32:55 +0300 Subject: [PATCH 55/59] AdminApiControllerTest unused variables fix. --- .../admin_api/admin_api_controller_test.exs | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index dd2fbfb15..ca6bd0e97 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -80,14 +80,13 @@ test "allows to force-follow another user" do user = insert(:user) follower = insert(:user) - conn = - build_conn() - |> assign(:user, admin) - |> put_req_header("accept", "application/json") - |> post("/api/pleroma/admin/user/follow", %{ - "follower" => follower.nickname, - "followed" => user.nickname - }) + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/user/follow", %{ + "follower" => follower.nickname, + "followed" => user.nickname + }) user = User.get_by_id(user.id) follower = User.get_by_id(follower.id) @@ -104,14 +103,13 @@ test "allows to force-unfollow another user" do User.follow(follower, user) - conn = - build_conn() - |> assign(:user, admin) - |> put_req_header("accept", "application/json") - |> post("/api/pleroma/admin/user/unfollow", %{ - "follower" => follower.nickname, - "followed" => user.nickname - }) + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/user/unfollow", %{ + "follower" => follower.nickname, + "followed" => user.nickname + }) user = User.get_by_id(user.id) follower = User.get_by_id(follower.id) From 36c0a10fdf47efa5067456030bad3204c2088e93 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 8 Apr 2019 11:03:10 +0000 Subject: [PATCH 56/59] adding language tag --- lib/pleroma/web/activity_pub/utils.ex | 5 ++++- test/web/activity_pub/utils_test.exs | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 32545937e..0b53f71c3 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -99,7 +99,10 @@ def make_json_ld_header do %{ "@context" => [ "https://www.w3.org/ns/activitystreams", - "#{Web.base_url()}/schemas/litepub-0.1.jsonld" + "#{Web.base_url()}/schemas/litepub-0.1.jsonld", + %{ + "@language" => "und" + } ] } end diff --git a/test/web/activity_pub/utils_test.exs b/test/web/activity_pub/utils_test.exs index 6b9961d82..758214e68 100644 --- a/test/web/activity_pub/utils_test.exs +++ b/test/web/activity_pub/utils_test.exs @@ -193,4 +193,16 @@ test "returns the what was collected if there are less pages than specified" do assert Utils.fetch_ordered_collection("http://example.com/outbox", 5) == [0, 1] end end + + test "make_json_ld_header/0" do + assert Utils.make_json_ld_header() == %{ + "@context" => [ + "https://www.w3.org/ns/activitystreams", + "http://localhost:4001/schemas/litepub-0.1.jsonld", + %{ + "@language" => "und" + } + ] + } + end end From 07afb9b537fd8966ceefda74f978cf7cfd2c09ea Mon Sep 17 00:00:00 2001 From: Sachin Joshi Date: Mon, 8 Apr 2019 15:16:18 +0545 Subject: [PATCH 57/59] update the follower count when a follower is blocked --- lib/pleroma/user.ex | 2 ++ test/user_test.exs | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 5012aef77..a6e2c8b97 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -937,6 +937,8 @@ def block(blocker, %User{ap_id: ap_id} = blocked) do unfollow(blocked, blocker) end + {:ok, blocker} = update_follower_count(blocker) + info_cng = blocker.info |> User.Info.add_to_block(ap_id) diff --git a/test/user_test.exs b/test/user_test.exs index 38712cebb..4f98af683 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -1125,4 +1125,21 @@ test "bookmarks" do assert {:ok, user_state3} = User.bookmark(user, id2) assert user_state3.bookmarks == [id2] end + + test "follower count is updated when a follower is blocked" do + user = insert(:user) + follower = insert(:user) + follower2 = insert(:user) + follower3 = insert(:user) + + {:ok, follower} = Pleroma.User.follow(follower, user) + {:ok, _follower2} = Pleroma.User.follow(follower2, user) + {:ok, _follower3} = Pleroma.User.follow(follower3, user) + + {:ok, _} = Pleroma.User.block(user, follower) + + user_show = Pleroma.Web.TwitterAPI.UserView.render("show.json", %{user: user}) + + assert Map.get(user_show, "followers_count") == 2 + end end From b57b43027cf958d3a3a82b95f155ae27b235b543 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 9 Apr 2019 23:20:31 +0300 Subject: [PATCH 58/59] Change response format of /api/pleroma/emoji to the one that actually makes sense --- docs/api/pleroma_api.md | 24 ++++++++++++++++++- .../controllers/util_controller.ex | 3 ++- test/web/twitter_api/util_controller_test.exs | 21 +++++++--------- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index 2e8fb04d2..6f2bc61bc 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -10,7 +10,29 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi * Authentication: not required * Params: none * Response: JSON -* Example response: `[{"kalsarikannit_f":{"tags":["Finmoji"],"image_url":"/finmoji/128px/kalsarikannit_f-128.png"}},{"perkele":{"tags":["Finmoji"],"image_url":"/finmoji/128px/perkele-128.png"}},{"blobdab":{"tags":["SomeTag"],"image_url":"/emoji/blobdab.png"}},"happiness":{"tags":["Finmoji"],"image_url":"/finmoji/128px/happiness-128.png"}}]` +* Example response: +```json +{ + "girlpower": { + "tags": [ + "Finmoji" + ], + "image_url": "/finmoji/128px/girlpower-128.png" + }, + "education": { + "tags": [ + "Finmoji" + ], + "image_url": "/finmoji/128px/education-128.png" + }, + "finnishlove": { + "tags": [ + "Finmoji" + ], + "image_url": "/finmoji/128px/finnishlove-128.png" + } +} +``` * Note: Same data as Mastodon API’s `/api/v1/custom_emojis` but in a different format ## `/api/pleroma/follow_import` diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 26407aebd..e6057e072 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -286,8 +286,9 @@ def emoji(conn, _params) do emoji = Emoji.get_all() |> Enum.map(fn {short_code, path, tags} -> - %{short_code => %{image_url: path, tags: String.split(tags, ",")}} + {short_code, %{image_url: path, tags: String.split(tags, ",")}} end) + |> Enum.into(%{}) json(conn, emoji) end diff --git a/test/web/twitter_api/util_controller_test.exs b/test/web/twitter_api/util_controller_test.exs index 410f20f87..a1a5e3964 100644 --- a/test/web/twitter_api/util_controller_test.exs +++ b/test/web/twitter_api/util_controller_test.exs @@ -172,22 +172,19 @@ test "returns everything in :pleroma, :frontend_configurations", %{conn: conn} d describe "/api/pleroma/emoji" do test "returns json with custom emoji with tags", %{conn: conn} do - [emoji | _body] = + emoji = conn |> get("/api/pleroma/emoji") |> json_response(200) - [key] = Map.keys(emoji) - - %{ - ^key => %{ - "image_url" => url, - "tags" => tags - } - } = emoji - - assert is_binary(url) - assert is_list(tags) + assert Enum.all?(emoji, fn + {_key, + %{ + "image_url" => url, + "tags" => tags + }} -> + is_binary(url) and is_list(tags) + end) end end From cf9ba8fe1b5a701488889c4bfdde46e7f00b9ebc Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 10 Apr 2019 02:32:04 +0200 Subject: [PATCH 59/59] transmogrifier.ex: Add fix_summary/1 MRF.KeywordPolicy assumes that summary is present in a activity, and this would probably simplify code elsewhere too anyway. --- lib/pleroma/web/activity_pub/transmogrifier.ex | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index f733ae7e1..9d4cbab0b 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -83,6 +83,22 @@ def fix_object(object) do |> fix_content_map |> fix_likes |> fix_addressing + |> fix_summary + end + + def fix_summary(%{"summary" => nil} = object) do + object + |> Map.put("summary", "") + end + + def fix_summary(%{"summary" => _} = object) do + # summary is present, nothing to do + object + end + + def fix_summary(object) do + object + |> Map.put("summary", "") end def fix_addressing_list(map, field) do