Merge develop to 788-separate-email-addresses

Merge conflicts:
test/web/twitter_api/twitter_api_test.exs
test/web/twitter_api/twitter_api_controller_test.exs
test/web/admin_api/admin_api_controller_test.exs

Changelog.md

@@ -0,0 +1,8 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
+
+## [unreleased]
+### Changed
+- Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer

config/config.exs

@@ -414,7 +414,7 @@
 
 config :pleroma, :auth, oauth_consumer_strategies: oauth_consumer_strategies
 
-config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Sendmail
+config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Sendmail
 
 config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics"
 

config/dev.exs

@@ -18,7 +18,7 @@
   watchers: [],
   secure_cookie_flag: false
 
-config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Local
+config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Local
 
 # ## SSL Support
 #

config/test.exs

@@ -21,7 +21,7 @@
 
 config :pleroma, Pleroma.Uploaders.Local, uploads: "test/uploads"
 
-config :pleroma, Pleroma.Mailer, adapter: Swoosh.Adapters.Test
+config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Test
 
 config :pleroma, :instance,
   email: "admin@example.com",

docs/config.md

@@ -31,14 +31,14 @@ This filter replaces the filename (not the path) of an upload. For complete obfu
 
 * `text`: Text to replace filenames in links. If empty, `{random}.extension` will be used.
 
-## Pleroma.Mailer
+## Pleroma.Emails.Mailer
 * `adapter`: one of the mail adapters listed in [Swoosh readme](https://github.com/swoosh/swoosh#adapters), or `Swoosh.Adapters.Local` for in-memory mailbox.
 * `api_key` / `password` and / or other adapter-specific settings, per the above documentation.
 
 An example for Sendgrid adapter:
 
 ```exs
-config :pleroma, Pleroma.Mailer,
+config :pleroma, Pleroma.Emails.Mailer,
   adapter: Swoosh.Adapters.Sendgrid,
   api_key: "YOUR_API_KEY"
 ```
@@ -46,7 +46,7 @@ config :pleroma, Pleroma.Mailer,
 An example for SMTP adapter:
 
 ```exs
-config :pleroma, Pleroma.Mailer,
+config :pleroma, Pleroma.Emails.Mailer,
   adapter: Swoosh.Adapters.SMTP,
   relay: "smtp.gmail.com",
   username: "YOUR_USERNAME@gmail.com",
@@ -318,7 +318,7 @@ Pleroma has the following queues:
 
 * `federator_outgoing` - Outgoing federation
 * `federator_incoming` - Incoming federation
-* `mailer` - Email sender, see [`Pleroma.Mailer`](#pleroma-mailer)
+* `mailer` - Email sender, see [`Pleroma.Emails.Mailer`](#pleroma-emails-mailer)
 * `transmogrifier` - Transmogrifier
 * `web_push` - Web push notifications
 * `scheduled_activities` - Scheduled activities, see [`Pleroma.ScheduledActivities`](#pleromascheduledactivity)

installation/pleroma.nginx

@@ -37,7 +37,7 @@ server {
     listen [::]:443 ssl http2;
     ssl_session_timeout 5m;
 
-    ssl_trusted_certificate   /etc/letsencrypt/live/example.tld/fullchain.pem;
+    ssl_trusted_certificate   /etc/letsencrypt/live/example.tld/chain.pem;
     ssl_certificate           /etc/letsencrypt/live/example.tld/fullchain.pem;
     ssl_certificate_key       /etc/letsencrypt/live/example.tld/privkey.pem;
 

lib/pleroma/emails/admin_email.ex

@@ -2,7 +2,7 @@
 # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.AdminEmail do
+defmodule Pleroma.Emails.AdminEmail do
   @moduledoc "Admin emails"
 
   import Swoosh.Email

lib/pleroma/emails/mailer.ex

@@ -2,7 +2,7 @@
 # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Mailer do
+defmodule Pleroma.Emails.Mailer do
   use Swoosh.Mailer, otp_app: :pleroma
 
   def deliver_async(email, config \\ []) do

lib/pleroma/emails/user_email.ex

@@ -2,7 +2,7 @@
 # Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.UserEmail do
+defmodule Pleroma.Emails.UserEmail do
   @moduledoc "User emails"
 
   import Swoosh.Email

lib/pleroma/formatter.ex

@@ -9,20 +9,31 @@ defmodule Pleroma.Formatter do
   alias Pleroma.Web.MediaProxy
 
   @safe_mention_regex ~r/^(\s*(?<mentions>@.+?\s+)+)(?<rest>.*)/
+  @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui
   @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
-  @link_regex ~r{((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+}ui
-  # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
 
   @auto_linker_config hashtag: true,
                       hashtag_handler: &Pleroma.Formatter.hashtag_handler/4,
                       mention: true,
                       mention_handler: &Pleroma.Formatter.mention_handler/4
 
+  def escape_mention_handler("@" <> nickname = mention, buffer, _, _) do
+    case User.get_cached_by_nickname(nickname) do
+      %User{} ->
+        # escape markdown characters with `\\`
+        # (we don't want something like @user__name to be parsed by markdown)
+        String.replace(mention, @markdown_characters_regex, "\\\\\\1")
+
+      _ ->
+        buffer
+    end
+  end
+
   def mention_handler("@" <> nickname, buffer, opts, acc) do
     case User.get_cached_by_nickname(nickname) do
       %User{id: id} = user ->
         ap_id = get_ap_id(user)
-        nickname_text = get_nickname_text(nickname, opts) |> maybe_escape(opts)
+        nickname_text = get_nickname_text(nickname, opts)
 
         link =
           "<span class='h-card'><a data-user='#{id}' class='u-url mention' href='#{ap_id}'>@<span>#{
@@ -70,6 +81,25 @@ def linkify(text, options \\ []) do
     end
   end
 
+  @doc """
+  Escapes a special characters in mention names.
+  """
+  def mentions_escape(text, options \\ []) do
+    options =
+      Keyword.merge(options,
+        mention: true,
+        url: false,
+        mention_handler: &Pleroma.Formatter.escape_mention_handler/4
+      )
+
+    if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do
+      %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text)
+      AutoLinker.link(mentions, options) <> AutoLinker.link(rest, options)
+    else
+      AutoLinker.link(text, options)
+    end
+  end
+
   def emojify(text) do
     emojify(text, Emoji.get_all())
   end
@@ -140,10 +170,4 @@ defp get_ap_id(%User{ap_id: ap_id}), do: ap_id
 
   defp get_nickname_text(nickname, %{mentions_format: :full}), do: User.full_nickname(nickname)
   defp get_nickname_text(nickname, _), do: User.local_nickname(nickname)
-
-  defp maybe_escape(str, %{mentions_escape: true}) do
-    String.replace(str, @markdown_characters_regex, "\\\\\\1")
-  end
-
-  defp maybe_escape(str, _), do: str
 end

lib/pleroma/notification.ex

@@ -98,6 +98,14 @@ def clear(user) do
     |> Repo.delete_all()
   end
 
+  def destroy_multiple(%{id: user_id} = _user, ids) do
+    from(n in Notification,
+      where: n.id in ^ids,
+      where: n.user_id == ^user_id
+    )
+    |> Repo.delete_all()
+  end
+
   def dismiss(%{id: user_id} = _user, id) do
     notification = Repo.get(Notification, id)
 
@@ -173,8 +181,7 @@ def skip?(:local, %{local: false}, %{info: %{notification_settings: %{"remote" =
   def skip?(:muted, activity, user) do
     actor = activity.data["actor"]
 
-    User.mutes?(user, %{ap_id: actor}) or
+    User.mutes?(user, %{ap_id: actor}) or CommonAPI.thread_muted?(user, activity)
-      CommonAPI.thread_muted?(user, activity)
   end
 
   def skip?(

lib/pleroma/pagination.ex

@@ -36,6 +36,12 @@ defp cast_params(params) do
       limit: :integer
     }
 
+    params =
+      Enum.reduce(params, %{}, fn
+        {key, _value}, acc when is_atom(key) -> Map.drop(acc, [key])
+        {key, value}, acc -> Map.put(acc, key, value)
+      end)
+
     changeset = cast({%{}, param_types}, params, Map.keys(param_types))
     changeset.changes
   end

lib/pleroma/user.ex

@@ -279,8 +279,8 @@ def try_send_confirmation_email(%User{} = user) do
     if user.info.confirmation_pending &&
          Pleroma.Config.get([:instance, :account_activation_required]) do
       user
-      |> Pleroma.UserEmail.account_confirmation_email()
+      |> Pleroma.Emails.UserEmail.account_confirmation_email()
-      |> Pleroma.Mailer.deliver_async()
+      |> Pleroma.Emails.Mailer.deliver_async()
     else
       {:ok, :noop}
     end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Instances
   alias Pleroma.Notification
   alias Pleroma.Object
+  alias Pleroma.Pagination
   alias Pleroma.Repo
   alias Pleroma.Upload
   alias Pleroma.User
@@ -449,8 +450,8 @@ def flag(
          :ok <- maybe_federate(activity) do
       Enum.each(User.all_superusers(), fn superuser ->
         superuser
-        |> Pleroma.AdminEmail.report(actor, account, statuses, content)
+        |> Pleroma.Emails.AdminEmail.report(actor, account, statuses, content)
-        |> Pleroma.Mailer.deliver_async()
+        |> Pleroma.Emails.Mailer.deliver_async()
       end)
 
       {:ok, activity}
@@ -493,7 +494,7 @@ def fetch_public_activities(opts \\ %{}) do
 
     q
     |> restrict_unlisted()
-    |> Repo.all()
+    |> Pagination.fetch_paginated(opts)
     |> Enum.reverse()
   end
 
@@ -636,26 +637,12 @@ defp restrict_recipients(query, recipients, user) do
     )
   end
 
-  defp restrict_limit(query, %{"limit" => limit}) do
-    from(activity in query, limit: ^limit)
-  end
-
-  defp restrict_limit(query, _), do: query
-
   defp restrict_local(query, %{"local_only" => true}) do
     from(activity in query, where: activity.local == true)
   end
 
   defp restrict_local(query, _), do: query
 
-  defp restrict_max(query, %{"max_id" => ""}), do: query
-
-  defp restrict_max(query, %{"max_id" => max_id}) do
-    from(activity in query, where: activity.id < ^max_id)
-  end
-
-  defp restrict_max(query, _), do: query
-
   defp restrict_actor(query, %{"actor_id" => actor_id}) do
     from(activity in query, where: activity.actor == ^actor_id)
   end
@@ -776,12 +763,7 @@ defp maybe_preload_objects(query, _) do
   end
 
   def fetch_activities_query(recipients, opts \\ %{}) do
-    base_query =
+    base_query = from(activity in Activity)
-      from(
-        activity in Activity,
-        limit: 20,
-        order_by: [fragment("? desc nulls last", activity.id)]
-      )
 
     base_query
     |> maybe_preload_objects(opts)
@@ -791,8 +773,6 @@ def fetch_activities_query(recipients, opts \\ %{}) do
     |> restrict_tag_all(opts)
     |> restrict_since(opts)
     |> restrict_local(opts)
-    |> restrict_limit(opts)
-    |> restrict_max(opts)
     |> restrict_actor(opts)
     |> restrict_type(opts)
     |> restrict_favorited_by(opts)
@@ -808,14 +788,14 @@ def fetch_activities_query(recipients, opts \\ %{}) do
 
   def fetch_activities(recipients, opts \\ %{}) do
     fetch_activities_query(recipients, opts)
-    |> Repo.all()
+    |> Pagination.fetch_paginated(opts)
     |> Enum.reverse()
   end
 
   def fetch_activities_bounded(recipients_to, recipients_cc, opts \\ %{}) do
     fetch_activities_query([], opts)
     |> restrict_to_cc(recipients_to, recipients_cc)
-    |> Repo.all()
+    |> Pagination.fetch_paginated(opts)
     |> Enum.reverse()
   end
 

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -238,8 +238,13 @@ def email_invite(%{assigns: %{user: user}} = conn, %{"email" => email} = params)
              !Pleroma.Config.get([:instance, :registrations_open]),
          {:ok, invite_token} <- UserInviteToken.create_invite(),
          email <-
-           Pleroma.UserEmail.user_invitation_email(user, invite_token, email, params["name"]),
+           Pleroma.Emails.UserEmail.user_invitation_email(
-         {:ok, _} <- Pleroma.Mailer.deliver(email) do
+             user,
+             invite_token,
+             email,
+             params["name"]
+           ),
+         {:ok, _} <- Pleroma.Emails.Mailer.deliver(email) do
       json_response(conn, :no_content, "")
     end
   end

lib/pleroma/web/common_api/utils.ex

@@ -195,11 +195,10 @@ def format_input(text, "text/html", options) do
   Formatting text to markdown.
   """
   def format_input(text, "text/markdown", options) do
-    options = Keyword.put(options, :mentions_escape, true)
-
     text
+    |> Formatter.mentions_escape(options)
+    |> Earmark.as_html!()
     |> Formatter.linkify(options)
-    |> (fn {text, mentions, tags} -> {Earmark.as_html!(text), mentions, tags} end).()
     |> Formatter.html_escape("text/html")
   end
 

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   alias Pleroma.Filter
   alias Pleroma.Notification
   alias Pleroma.Object
+  alias Pleroma.Pagination
   alias Pleroma.Repo
   alias Pleroma.ScheduledActivity
   alias Pleroma.Stats
@@ -202,15 +203,29 @@ def custom_emojis(conn, _params) do
   defp add_link_headers(conn, method, activities, param \\ nil, params \\ %{}) do
     params =
       conn.params
-      |> Map.drop(["since_id", "max_id"])
+      |> Map.drop(["since_id", "max_id", "min_id"])
       |> Map.merge(params)
 
     last = List.last(activities)
-    first = List.first(activities)
 
     if last do
-      min = last.id
+      max_id = last.id
-      max = first.id
+
+      limit =
+        params
+        |> Map.get("limit", "20")
+        |> String.to_integer()
+
+      min_id =
+        if length(activities) <= limit do
+          activities
+          |> List.first()
+          |> Map.get(:id)
+        else
+          activities
+          |> Enum.at(limit * -1)
+          |> Map.get(:id)
+        end
 
       {next_url, prev_url} =
         if param do
@@ -219,13 +234,13 @@ defp add_link_headers(conn, method, activities, param \\ nil, params \\ %{}) do
               Pleroma.Web.Endpoint,
               method,
               param,
-              Map.merge(params, %{max_id: min})
+              Map.merge(params, %{max_id: max_id})
             ),
             mastodon_api_url(
               Pleroma.Web.Endpoint,
               method,
               param,
-              Map.merge(params, %{since_id: max})
+              Map.merge(params, %{min_id: min_id})
             )
           }
         else
@@ -233,12 +248,12 @@ defp add_link_headers(conn, method, activities, param \\ nil, params \\ %{}) do
             mastodon_api_url(
               Pleroma.Web.Endpoint,
               method,
-              Map.merge(params, %{max_id: min})
+              Map.merge(params, %{max_id: max_id})
             ),
             mastodon_api_url(
               Pleroma.Web.Endpoint,
               method,
-              Map.merge(params, %{since_id: max})
+              Map.merge(params, %{min_id: min_id})
             )
           }
         end
@@ -314,7 +329,7 @@ def dm_timeline(%{assigns: %{user: user}} = conn, params) do
     activities =
       [user.ap_id]
       |> ActivityPub.fetch_activities_query(params)
-      |> Repo.all()
+      |> Pagination.fetch_paginated(params)
 
     conn
     |> add_link_headers(:dm_timeline, activities)
@@ -612,6 +627,11 @@ def dismiss_notification(%{assigns: %{user: user}} = conn, %{"id" => id} = _para
     end
   end
 
+  def destroy_multiple(%{assigns: %{user: user}} = conn, %{"ids" => ids} = _params) do
+    Notification.destroy_multiple(user, ids)
+    json(conn, %{})
+  end
+
   def relationships(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     id = List.wrap(id)
     q = from(u in User, where: u.id in ^id)

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -301,8 +301,10 @@ def render("attachment.json", %{attachment: attachment}) do
   end
 
   def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do
-    _id = activity.data["object"]["inReplyTo"]
+    with nil <- replied_to_activities[activity.data["object"]["inReplyTo"]] do
-    replied_to_activities[activity.data["object"]["inReplyTo"]]
+      # If user didn't participate in the thread
+      Activity.get_in_reply_to_activity(activity)
+    end
   end
 
   def get_reply_to(%{data: %{"object" => object}}, _) do

lib/pleroma/web/rel_me.ex

@@ -6,7 +6,8 @@ defmodule Pleroma.Web.RelMe do
   @hackney_options [
     pool: :media,
     recv_timeout: 2_000,
-    max_body: 2_000_000
+    max_body: 2_000_000,
+    with_body: true
   ]
 
   if Mix.env() == :test do

lib/pleroma/web/rich_media/parser.ex

@@ -12,7 +12,8 @@ defmodule Pleroma.Web.RichMedia.Parser do
   @hackney_options [
     pool: :media,
     recv_timeout: 2_000,
-    max_body: 2_000_000
+    max_body: 2_000_000,
+    with_body: true
   ]
 
   def parse(nil), do: {:error, "No URL provided"}

lib/pleroma/web/router.ex

@@ -261,6 +261,7 @@ defmodule Pleroma.Web.Router do
       post("/notifications/dismiss", MastodonAPIController, :dismiss_notification)
       get("/notifications", MastodonAPIController, :notifications)
       get("/notifications/:id", MastodonAPIController, :get_notification)
+      delete("/notifications/destroy_multiple", MastodonAPIController, :destroy_multiple)
 
       get("/scheduled_statuses", MastodonAPIController, :scheduled_statuses)
       get("/scheduled_statuses/:id", MastodonAPIController, :show_scheduled_status)

lib/pleroma/web/templates/layout/app.html.eex

@@ -179,6 +179,17 @@
           flex-basis: 50%;
         }
       }
+      .form-row {
+        display: flex;
+      }
+      .form-row > label {
+        text-align: left;
+        line-height: 47px;
+        flex: 1;
+      }
+      .form-row > input {
+        flex: 2;
+      }
     </style>
   </head>
   <body>

lib/pleroma/web/templates/twitter_api/util/password_reset.html.eex

@@ -1,12 +1,13 @@
 <h2>Password Reset for <%= @user.nickname %></h2>
 <%= form_for @conn, util_path(@conn, :password_reset), [as: "data"], fn f -> %>
-<%= label f, :password, "Password" %>
+  <div class="form-row">
-<%= password_input f, :password %>
+    <%= label f, :password, "Password" %>
-<br>
+    <%= password_input f, :password %>
-
+  </div>
-<%= label f, :password_confirmation, "Confirmation" %>
+  <div class="form-row">
-<%= password_input f, :password_confirmation %>
+    <%= label f, :password_confirmation, "Confirmation" %>
-<br>
+    <%= password_input f, :password_confirmation %>
-<%= hidden_input f, :token, value: @token.token %>
+  </div>
-<%= submit "Reset" %>
+  <%= hidden_input f, :token, value: @token.token %>
+  <%= submit "Reset" %>
 <% end %>

lib/pleroma/web/twitter_api/twitter_api.ex

@@ -4,10 +4,10 @@
 
 defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
   alias Pleroma.Activity
-  alias Pleroma.Mailer
+  alias Pleroma.Emails.Mailer
+  alias Pleroma.Emails.UserEmail
   alias Pleroma.Repo
   alias Pleroma.User
-  alias Pleroma.UserEmail
   alias Pleroma.UserInviteToken
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.CommonAPI

mix.exs

@@ -101,7 +101,7 @@ defp deps do
       {:ueberauth, "~> 0.4"},
       {:auto_linker,
        git: "https://git.pleroma.social/pleroma/auto_linker.git",
-       ref: "479dd343f4e563ff91215c8275f3b5c67e032850"},
+       ref: "90613b4bae875a3610c275b7056b61ffdd53210d"},
       {:pleroma_job_queue, "~> 0.2.0"},
       {:telemetry, "~> 0.3"},
       {:prometheus_ex, "~> 3.0"},

mix.lock

@@ -1,6 +1,6 @@
 %{
   "accept": {:hex, :accept, "0.3.5", "b33b127abca7cc948bbe6caa4c263369abf1347cfa9d8e699c6d214660f10cd1", [:rebar3], [], "hexpm"},
-  "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "479dd343f4e563ff91215c8275f3b5c67e032850", [ref: "479dd343f4e563ff91215c8275f3b5c67e032850"]},
+  "auto_linker": {:git, "https://git.pleroma.social/pleroma/auto_linker.git", "90613b4bae875a3610c275b7056b61ffdd53210d", [ref: "90613b4bae875a3610c275b7056b61ffdd53210d"]},
   "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"},
   "bunt": {:hex, :bunt, "0.2.0", "951c6e801e8b1d2cbe58ebbd3e616a869061ddadcc4863d0a2182541acae9a38", [:mix], [], "hexpm"},
   "cachex": {:hex, :cachex, "3.0.2", "1351caa4e26e29f7d7ec1d29b53d6013f0447630bbf382b4fb5d5bad0209f203", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm"},