Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into feature/jobs

# Conflicts:
#	lib/pleroma/web/activity_pub/activity_pub.ex
#	lib/pleroma/web/federator/federator.ex

config/config.exs

@@ -146,6 +146,7 @@
   banner_upload_limit: 4_000_000,
   registrations_open: true,
   federating: true,
+  federation_reachability_timeout_days: 7,
   allow_relay: true,
   rewrite_policy: Pleroma.Web.ActivityPub.MRF.NoOpPolicy,
   public: true,
@@ -226,7 +227,9 @@
   allow_followersonly: false,
   allow_direct: false
 
-config :pleroma, :mrf_hellthread, threshold: 10
+config :pleroma, :mrf_hellthread,
+  delist_threshold: 5,
+  reject_threshold: 10
 
 config :pleroma, :mrf_simple,
   media_removal: [],
@@ -235,6 +238,8 @@
   reject: [],
   accept: []
 
+config :pleroma, :rich_media, enabled: true
+
 config :pleroma, :media_proxy,
   enabled: false,
   proxy_opts: [

config/test.exs

@@ -36,6 +36,7 @@
 config :pleroma, :websub, Pleroma.Web.WebsubMock
 config :pleroma, :ostatus, Pleroma.Web.OStatusMock
 config :tesla, adapter: Tesla.Mock
+config :pleroma, :rich_media, enabled: false
 
 config :web_push_encryption, :vapid_details,
   subject: "mailto:administrator@example.com",

docs/Pleroma-API.md

@@ -52,6 +52,7 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
     * `confirm`
     * `captcha_solution`: optional, contains provider-specific captcha solution,
     * `captcha_token`: optional, contains provider-specific captcha token
+    * `token`: invite token required when the registerations aren't public.
 * Response: JSON. Returns a user object on success, otherwise returns `{"error": "error_msg"}`
 * Example response:
 ```

docs/config.md

@@ -17,7 +17,7 @@ Note: `strip_exif` has been replaced by `Pleroma.Upload.Filter.Mogrify`.
 
 ## Pleroma.Upload.Filter.Mogrify
 
-* `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", {"impode", "1"}]`.
+* `args`: List of actions for the `mogrify` command like `"strip"` or `["strip", "auto-orient", {"impode", "1"}]`.
 
 ## Pleroma.Upload.Filter.Dedupe
 
@@ -73,6 +73,7 @@ config :pleroma, Pleroma.Mailer,
 * `invites_enabled`: Enable user invitations for admins (depends on `registrations_open: false`).
 * `account_activation_required`: Require users to confirm their emails before signing in.
 * `federating`: Enable federation with other instances
+* `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it.
 * `allow_relay`: Enable Pleroma’s Relay, which makes it possible to follow a whole instance
 * `rewrite_policy`: Message Rewrite Policy, either one or a list. Here are the ones available by default:
   * `Pleroma.Web.ActivityPub.MRF.NoOpPolicy`: Doesn’t modify activities (default)
@@ -124,7 +125,7 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i
 
 * `theme`: Which theme to use, they are defined in ``styles.json``
 * `logo`: URL of the logo, defaults to Pleroma’s logo
-* `logo_mask`: Whenether to mask the logo
+* `logo_mask`: Whether to use only the logo's shape as a mask (true) or as a regular image (false)
 * `logo_margin`: What margin to use around the logo
 * `background`: URL of the background, unless viewing a user profile with a background that is set
 * `redirect_root_no_login`: relative URL which indicates where to redirect when a user isn’t logged in.
@@ -148,7 +149,8 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i
 * `allow_direct`: whether to allow direct messages
 
 ## :mrf_hellthread
-* `threshold`: Number of mentioned users after which the message gets discarded as spam
+* `delist_threshold`: Number of mentioned users after which the message gets delisted (the message can still be seen, but it will not show up in public timelines and mentioned users won't get notifications about it). Set to 0 to disable.
+* `reject_threshold`: Number of mentioned users after which the messaged gets rejected. Set to 0 to disable.
 
 ## :media_proxy
 * `enabled`: Enables proxying of remote media to the instance’s proxy
@@ -252,6 +254,9 @@ This config contains two queues: `federator_incoming` and `federator_outgoing`.
   * Pleroma.Web.Metadata.Providers.TwitterCard
 * `unfurl_nsfw`: If set to `true` nsfw attachments will be shown in previews
 
+## :rich_media
+* `enabled`: if enabled the instance will parse metadata from attached links to generate link previews
+
 ## :hackney_pools
 
 Advanced. Tweaks Hackney (http client) connections pools.

lib/pleroma/application.ex

@@ -6,11 +6,13 @@ defmodule Pleroma.Application do
   use Application
   import Supervisor.Spec
 
-  @name "Pleroma"
+  @name Mix.Project.config()[:name]
   @version Mix.Project.config()[:version]
+  @repository Mix.Project.config()[:source_url]
   def name, do: @name
   def version, do: @version
   def named_version(), do: @name <> " " <> @version
+  def repository, do: @repository
 
   def user_agent() do
     info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"

lib/pleroma/config/deprecation_warnings.ex

@@ -12,6 +12,13 @@ def check_frontend_config_mechanism() do
       You are using the old configuration mechanism for the frontend. Please check config.md.
       """)
     end
+
+    if Pleroma.Config.get(:mrf_hellthread, :threshold) do
+      Logger.warn("""
+      !!!DEPRECATION WARNING!!!
+      You are using the old configuration mechanism for the hellthread filter. Please check config.md.
+      """)
+    end
   end
 
   def warn do

lib/pleroma/instances.ex

@@ -0,0 +1,36 @@
+defmodule Pleroma.Instances do
+  @moduledoc "Instances context."
+
+  @adapter Pleroma.Instances.Instance
+
+  defdelegate filter_reachable(urls_or_hosts), to: @adapter
+  defdelegate reachable?(url_or_host), to: @adapter
+  defdelegate set_reachable(url_or_host), to: @adapter
+  defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: @adapter
+
+  def set_consistently_unreachable(url_or_host),
+    do: set_unreachable(url_or_host, reachability_datetime_threshold())
+
+  def reachability_datetime_threshold do
+    federation_reachability_timeout_days =
+      Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 0
+
+    if federation_reachability_timeout_days > 0 do
+      NaiveDateTime.add(
+        NaiveDateTime.utc_now(),
+        -federation_reachability_timeout_days * 24 * 3600,
+        :second
+      )
+    else
+      ~N[0000-01-01 00:00:00]
+    end
+  end
+
+  def host(url_or_host) when is_binary(url_or_host) do
+    if url_or_host =~ ~r/^http/i do
+      URI.parse(url_or_host).host
+    else
+      url_or_host
+    end
+  end
+end

lib/pleroma/instances/instance.ex

@@ -0,0 +1,113 @@
+defmodule Pleroma.Instances.Instance do
+  @moduledoc "Instance."
+
+  alias Pleroma.Instances
+  alias Pleroma.Instances.Instance
+
+  use Ecto.Schema
+
+  import Ecto.{Query, Changeset}
+
+  alias Pleroma.Repo
+
+  schema "instances" do
+    field(:host, :string)
+    field(:unreachable_since, :naive_datetime)
+
+    timestamps()
+  end
+
+  defdelegate host(url_or_host), to: Instances
+
+  def changeset(struct, params \\ %{}) do
+    struct
+    |> cast(params, [:host, :unreachable_since])
+    |> validate_required([:host])
+    |> unique_constraint(:host)
+  end
+
+  def filter_reachable([]), do: %{}
+
+  def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
+    hosts =
+      urls_or_hosts
+      |> Enum.map(&(&1 && host(&1)))
+      |> Enum.filter(&(to_string(&1) != ""))
+
+    unreachable_since_by_host =
+      Repo.all(
+        from(i in Instance,
+          where: i.host in ^hosts,
+          select: {i.host, i.unreachable_since}
+        )
+      )
+      |> Map.new(& &1)
+
+    reachability_datetime_threshold = Instances.reachability_datetime_threshold()
+
+    for entry <- Enum.filter(urls_or_hosts, &is_binary/1) do
+      host = host(entry)
+      unreachable_since = unreachable_since_by_host[host]
+
+      if !unreachable_since ||
+           NaiveDateTime.compare(unreachable_since, reachability_datetime_threshold) == :gt do
+        {entry, unreachable_since}
+      end
+    end
+    |> Enum.filter(& &1)
+    |> Map.new(& &1)
+  end
+
+  def reachable?(url_or_host) when is_binary(url_or_host) do
+    !Repo.one(
+      from(i in Instance,
+        where:
+          i.host == ^host(url_or_host) and
+            i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
+        select: true
+      )
+    )
+  end
+
+  def reachable?(_), do: true
+
+  def set_reachable(url_or_host) when is_binary(url_or_host) do
+    with host <- host(url_or_host),
+         %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do
+      {:ok, _instance} =
+        existing_record
+        |> changeset(%{unreachable_since: nil})
+        |> Repo.update()
+    end
+  end
+
+  def set_reachable(_), do: {:error, nil}
+
+  def set_unreachable(url_or_host, unreachable_since \\ nil)
+
+  def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do
+    unreachable_since = unreachable_since || DateTime.utc_now()
+    host = host(url_or_host)
+    existing_record = Repo.get_by(Instance, %{host: host})
+
+    changes = %{unreachable_since: unreachable_since}
+
+    cond do
+      is_nil(existing_record) ->
+        %Instance{}
+        |> changeset(Map.put(changes, :host, host))
+        |> Repo.insert()
+
+      existing_record.unreachable_since &&
+          NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt ->
+        {:ok, existing_record}
+
+      true ->
+        existing_record
+        |> changeset(changes)
+        |> Repo.update()
+    end
+  end
+
+  def set_unreachable(_, _), do: {:error, nil}
+end

lib/pleroma/object.ex

@@ -31,8 +31,8 @@ def get_by_ap_id(ap_id) do
     Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id)))
   end
 
-  def normalize(obj) when is_map(obj), do: Object.get_by_ap_id(obj["id"])
+  def normalize(%{"id" => ap_id}), do: normalize(ap_id)
-  def normalize(ap_id) when is_binary(ap_id), do: Object.get_by_ap_id(ap_id)
+  def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
   def normalize(_), do: nil
 
   # Owned objects can only be mutated by their owner
@@ -42,24 +42,18 @@ def authorize_mutation(%Object{data: %{"actor" => actor}}, %User{ap_id: ap_id}),
   # Legacy objects can be mutated by anybody
   def authorize_mutation(%Object{}, %User{}), do: true
 
-  if Mix.env() == :test do
+  def get_cached_by_ap_id(ap_id) do
-    def get_cached_by_ap_id(ap_id) do
+    key = "object:#{ap_id}"
-      get_by_ap_id(ap_id)
-    end
-  else
-    def get_cached_by_ap_id(ap_id) do
-      key = "object:#{ap_id}"
 
-      Cachex.fetch!(:object_cache, key, fn _ ->
+    Cachex.fetch!(:object_cache, key, fn _ ->
-        object = get_by_ap_id(ap_id)
+      object = get_by_ap_id(ap_id)
 
-        if object do
+      if object do
-          {:commit, object}
+        {:commit, object}
-        else
+      else
-          {:ignore, object}
+        {:ignore, object}
-        end
+      end
-      end)
+    end)
-    end
   end
 
   def context_mapping(context) do
@@ -90,4 +84,17 @@ def delete(%Object{data: %{"id" => id}} = object) do
       {:ok, object}
     end
   end
+
+  def set_cache(%Object{data: %{"id" => ap_id}} = object) do
+    Cachex.put(:object_cache, "object:#{ap_id}", object)
+    {:ok, object}
+  end
+
+  def update_and_set_cache(changeset) do
+    with {:ok, object} <- Repo.update(changeset) do
+      set_cache(object)
+    else
+      e -> e
+    end
+  end
 end

lib/pleroma/plugs/instance_static.ex

@@ -21,7 +21,7 @@ def file_path(path) do
     end
   end
 
-  @only ~w(index.html static emoji packs sounds images instance favicon.png)
+  @only ~w(index.html static emoji packs sounds images instance favicon.png sw.js sw-pleroma.js)
 
   def init(opts) do
     opts

lib/pleroma/upload.ex

@@ -124,10 +124,10 @@ defp get_opts(opts) do
 
           :pleroma, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Mogrify]]
 
-          :pleroma, Pleroma.Upload.Filter.Mogrify, args: "strip"
+          :pleroma, Pleroma.Upload.Filter.Mogrify, args: ["strip", "auto-orient"]
         """)
 
-        Pleroma.Config.put([Pleroma.Upload.Filter.Mogrify], args: "strip")
+        Pleroma.Config.put([Pleroma.Upload.Filter.Mogrify], args: ["strip", "auto-orient"])
         Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Mogrify])
       else
         opts

lib/pleroma/user.ex

@@ -39,6 +39,7 @@ defmodule Pleroma.User do
     field(:follower_address, :string)
     field(:search_rank, :float, virtual: true)
     field(:tags, {:array, :string}, default: [])
+    field(:bookmarks, {:array, :string}, default: [])
     field(:last_refreshed_at, :naive_datetime)
     has_many(:notifications, Notification)
     embeds_one(:info, Pleroma.User.Info)
@@ -314,7 +315,16 @@ def follow_all(follower, followeds) do
     q =
       from(u in User,
         where: u.id == ^follower.id,
-        update: [set: [following: fragment("array_cat(?, ?)", u.following, ^followed_addresses)]]
+        update: [
+          set: [
+            following:
+              fragment(
+                "array(select distinct unnest (array_cat(?, ?)))",
+                u.following,
+                ^followed_addresses
+              )
+          ]
+        ]
       )
 
     {1, [follower]} = Repo.update_all(q, [], returning: true)
@@ -1161,6 +1171,22 @@ defp update_tags(%User{} = user, new_tags) do
     updated_user
   end
 
+  def bookmark(%User{} = user, status_id) do
+    bookmarks = Enum.uniq(user.bookmarks ++ [status_id])
+    update_bookmarks(user, bookmarks)
+  end
+
+  def unbookmark(%User{} = user, status_id) do
+    bookmarks = Enum.uniq(user.bookmarks -- [status_id])
+    update_bookmarks(user, bookmarks)
+  end
+
+  def update_bookmarks(%User{} = user, bookmarks) do
+    user
+    |> change(%{bookmarks: bookmarks})
+    |> update_and_set_cache
+  end
+
   defp normalize_tags(tags) do
     [tags]
     |> List.flatten()

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
-  alias Pleroma.{Activity, Repo, Object, Upload, User, Notification}
+  alias Pleroma.{Activity, Repo, Object, Upload, User, Notification, Instances}
   alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF}
   alias Pleroma.Web.WebFinger
   alias Pleroma.Web.Federator
@@ -734,7 +734,7 @@ def should_federate?(inbox, public) do
   end
 
   def publish(actor, activity) do
-    followers =
+    remote_followers =
       if actor.follower_address in activity.recipients do
         {:ok, followers} = User.get_followers(actor)
         followers |> Enum.filter(&(!&1.local))
@@ -747,24 +747,26 @@ def publish(actor, activity) do
     {:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
     json = Jason.encode!(data)
 
-    (Pleroma.Web.Salmon.remote_users(activity) ++ followers)
+    (Pleroma.Web.Salmon.remote_users(activity) ++ remote_followers)
     |> Enum.filter(fn user -> User.ap_enabled?(user) end)
     |> Enum.map(fn %{info: %{source_data: data}} ->
       (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"]
     end)
     |> Enum.uniq()
     |> Enum.filter(fn inbox -> should_federate?(inbox, public) end)
-    |> Enum.each(fn inbox ->
+    |> Instances.filter_reachable()
+    |> Enum.each(fn {inbox, unreachable_since} ->
       Federator.publish_single_ap(%{
         inbox: inbox,
         json: json,
         actor: actor,
-        id: activity.data["id"]
+        id: activity.data["id"],
+        unreachable_since: unreachable_since
       })
     end)
   end
 
-  def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do
+  def publish_one(%{inbox: inbox, json: json, actor: actor, id: id} = params) do
     Logger.info("Federating #{id} to #{inbox}")
     host = URI.parse(inbox).host
 
@@ -777,15 +779,26 @@ def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do
         digest: digest
       })
 
-    @httpoison.post(
+    with {:ok, %{status: code}} when code in 200..299 <-
-      inbox,
+           result =
-      json,
+             @httpoison.post(
-      [
+               inbox,
-        {"Content-Type", "application/activity+json"},
+               json,
-        {"signature", signature},
+               [
-        {"digest", digest}
+                 {"Content-Type", "application/activity+json"},
-      ]
+                 {"signature", signature},
-    )
+                 {"digest", digest}
+               ]
+             ) do
+      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
+        do: Instances.set_reachable(inbox)
+
+      result
+    else
+      {_post_result, response} ->
+        unless params[:unreachable_since], do: Instances.set_unreachable(inbox)
+        {:error, response}
+    end
   end
 
   # TODO:

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   use Pleroma.Web, :controller
+
   alias Pleroma.{Activity, User, Object}
   alias Pleroma.Web.ActivityPub.{ObjectView, UserView}
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -17,6 +18,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   action_fallback(:errors)
 
   plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay])
+  plug(:set_requester_reachable when action in [:inbox])
   plug(:relay_active? when action in [:relay])
 
   def relay_active?(conn, _) do
@@ -289,4 +291,13 @@ def errors(conn, _e) do
     |> put_status(500)
     |> json("error")
   end
+
+  defp set_requester_reachable(%Plug.Conn{} = conn, _) do
+    with actor <- conn.params["actor"],
+         true <- is_binary(actor) do
+      Pleroma.Instances.set_reachable(actor)
+    end
+
+    conn
+  end
 end

lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex

@@ -3,20 +3,46 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
+  alias Pleroma.User
   @behaviour Pleroma.Web.ActivityPub.MRF
 
-  @impl true
+  defp delist_message(message) do
-  def filter(%{"type" => "Create"} = object) do
+    follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address
-    threshold = Pleroma.Config.get([:mrf_hellthread, :threshold])
-    recipients = (object["to"] || []) ++ (object["cc"] || [])
 
-    if length(recipients) > threshold do
+    message
-      {:reject, nil}
+    |> Map.put("to", [follower_collection])
-    else
+    |> Map.put("cc", ["https://www.w3.org/ns/activitystreams#Public"])
-      {:ok, object}
+  end
+
+  @impl true
+  def filter(%{"type" => "Create"} = message) do
+    delist_threshold = Pleroma.Config.get([:mrf_hellthread, :delist_threshold])
+
+    reject_threshold =
+      Pleroma.Config.get(
+        [:mrf_hellthread, :reject_threshold],
+        Pleroma.Config.get([:mrf_hellthread, :threshold])
+      )
+
+    recipients = (message["to"] || []) ++ (message["cc"] || [])
+
+    cond do
+      length(recipients) > reject_threshold and reject_threshold > 0 ->
+        {:reject, nil}
+
+      length(recipients) > delist_threshold and delist_threshold > 0 ->
+        if Enum.member?(message["to"], "https://www.w3.org/ns/activitystreams#Public") or
+             Enum.member?(message["cc"], "https://www.w3.org/ns/activitystreams#Public") do
+          {:ok, delist_message(message)}
+        else
+          {:ok, message}
+        end
+
+      true ->
+        {:ok, message}
     end
   end
 
   @impl true
-  def filter(object), do: {:ok, object}
+  def filter(message), do: {:ok, message}
 end

lib/pleroma/web/activity_pub/utils.ex

@@ -285,7 +285,7 @@ def update_element_in_object(property, element, object) do
            |> Map.put("#{property}_count", length(element))
            |> Map.put("#{property}s", element),
          changeset <- Changeset.change(object, data: new_data),
-         {:ok, object} <- Repo.update(changeset),
+         {:ok, object} <- Object.update_and_set_cache(changeset),
          _ <- update_object_in_activities(object) do
       {:ok, object}
     end

lib/pleroma/web/endpoint.ex

@@ -25,7 +25,7 @@ defmodule Pleroma.Web.Endpoint do
     at: "/",
     from: :pleroma,
     only:
-      ~w(index.html static finmoji emoji packs sounds images instance sw.js favicon.png schemas doc)
+      ~w(index.html static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc)
   )
 
   # Code reloading can be explicitly enabled under the
@@ -82,4 +82,8 @@ def load_from_system_env(config) do
     port = System.get_env("PORT") || raise "expected the PORT environment variable to be set"
     {:ok, Keyword.put(config, :http, [:inet6, port: port])}
   end
+
+  def websocket_url do
+    String.replace_leading(url(), "http", "ws")
+  end
 end

lib/pleroma/web/federator/federator.ex

@@ -6,7 +6,7 @@ defmodule Pleroma.Web.Federator do
   alias Pleroma.User
   alias Pleroma.Activity
   alias Pleroma.Jobs
-  alias Pleroma.Web.{WebFinger, Websub}
+  alias Pleroma.Web.{WebFinger, Websub, Salmon}
   alias Pleroma.Web.Federator.RetryQueue
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
@@ -58,6 +58,10 @@ def refresh_subscriptions() do
     Jobs.enqueue(:federator_out, __MODULE__, [:refresh_subscriptions])
   end
 
+  def publish_single_salmon(params) do
+    Jobs.enqueue(:federator_out, __MODULE__, [:publish_single_salmon, params])
+  end
+
   # Job Worker Callbacks
 
   def perform(:refresh_subscriptions) do
@@ -145,6 +149,10 @@ def perform(:incoming_ap_doc, params) do
     end
   end
 
+  def perform(:publish_single_salmon, params) do
+    Salmon.send_to_user(params)
+  end
+
   def perform(:publish_single_ap, params) do
     case ActivityPub.publish_one(params) do
       {:ok, _} ->

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -138,7 +138,7 @@ def masto_instance(conn, _params) do
       version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})",
       email: Keyword.get(instance, :email),
       urls: %{
-        streaming_api: String.replace(Pleroma.Web.Endpoint.static_url(), "http", "ws")
+        streaming_api: Pleroma.Web.Endpoint.websocket_url()
       },
       stats: Stats.get_stats(),
       thumbnail: Web.base_url() <> "/instance/thumbnail.jpeg",
@@ -423,6 +423,28 @@ def unpin_status(%{assigns: %{user: user}} = conn, %{"id" => ap_id_or_id}) do
     end
   end
 
+  def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
+    with %Activity{} = activity <- Repo.get(Activity, id),
+         %User{} = user <- User.get_by_nickname(user.nickname),
+         true <- ActivityPub.visible_for_user?(activity, user),
+         {:ok, user} <- User.bookmark(user, activity.data["object"]["id"]) do
+      conn
+      |> put_view(StatusView)
+      |> try_render("status.json", %{activity: activity, for: user, as: :activity})
+    end
+  end
+
+  def unbookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
+    with %Activity{} = activity <- Repo.get(Activity, id),
+         %User{} = user <- User.get_by_nickname(user.nickname),
+         true <- ActivityPub.visible_for_user?(activity, user),
+         {:ok, user} <- User.unbookmark(user, activity.data["object"]["id"]) do
+      conn
+      |> put_view(StatusView)
+      |> try_render("status.json", %{activity: activity, for: user, as: :activity})
+    end
+  end
+
   def notifications(%{assigns: %{user: user}} = conn, params) do
     notifications = Notification.for_user(user, params)
 
@@ -859,6 +881,19 @@ def favourites(%{assigns: %{user: user}} = conn, params) do
     |> render("index.json", %{activities: activities, for: user, as: :activity})
   end
 
+  def bookmarks(%{assigns: %{user: user}} = conn, _) do
+    user = Repo.get(User, user.id)
+
+    activities =
+      user.bookmarks
+      |> Enum.map(fn id -> Activity.get_create_by_object_ap_id(id) end)
+      |> Enum.reverse()
+
+    conn
+    |> put_view(StatusView)
+    |> render("index.json", %{activities: activities, for: user, as: :activity})
+  end
+
   def get_lists(%{assigns: %{user: user}} = conn, opts) do
     lists = Pleroma.List.for_user(user, opts)
     res = ListView.render("lists.json", lists: lists)
@@ -870,7 +905,10 @@ def get_list(%{assigns: %{user: user}} = conn, %{"id" => id}) do
       res = ListView.render("list.json", list: list)
       json(conn, res)
     else
-      _e -> json(conn, "error")
+      _e ->
+        conn
+        |> put_status(404)
+        |> json(%{error: "Record not found"})
     end
   end
 

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -87,6 +87,7 @@ def render(
       favourites_count: 0,
       reblogged: false,
       favourited: false,
+      bookmarked: false,
       muted: false,
       pinned: pinned?(activity, user),
       sensitive: false,
@@ -121,6 +122,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity}
 
     repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || [])
     favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || [])
+    bookmarked = opts[:for] && object["id"] in opts[:for].bookmarks
 
     attachment_data = object["attachment"] || []
     attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment)
@@ -157,6 +159,7 @@ def render("status.json", %{activity: %{data: %{"object" => object}} = activity}
       favourites_count: like_count,
       reblogged: present?(repeated),
       favourited: present?(favorited),
+      bookmarked: present?(bookmarked),
       muted: false,
       pinned: pinned?(activity, user),
       sensitive: sensitive,

lib/pleroma/web/nodeinfo/nodeinfo_controller.ex

@@ -19,6 +19,10 @@ def schemas(conn, _params) do
         %{
           rel: "http://nodeinfo.diaspora.software/ns/schema/2.0",
           href: Web.base_url() <> "/nodeinfo/2.0.json"
+        },
+        %{
+          rel: "http://nodeinfo.diaspora.software/ns/schema/2.1",
+          href: Web.base_url() <> "/nodeinfo/2.1.json"
         }
       ]
     }
@@ -26,8 +30,9 @@ def schemas(conn, _params) do
     json(conn, response)
   end
 
-  # Schema definition: https://github.com/jhass/nodeinfo/blob/master/schemas/2.0/schema.json
+  # returns a nodeinfo 2.0 map, since 2.1 just adds a repository field
-  def nodeinfo(conn, %{"version" => "2.0"}) do
+  # under software.
+  def raw_nodeinfo() do
     instance = Application.get_env(:pleroma, :instance)
     media_proxy = Application.get_env(:pleroma, :media_proxy)
     suggestions = Application.get_env(:pleroma, :suggestions)
@@ -98,10 +103,10 @@ def nodeinfo(conn, %{"version" => "2.0"}) do
       ]
       |> Enum.filter(& &1)
 
-    response = %{
+    %{
       version: "2.0",
       software: %{
-        name: Pleroma.Application.name(),
+        name: Pleroma.Application.name() |> String.downcase(),
         version: Pleroma.Application.version()
       },
       protocols: ["ostatus", "activitypub"],
@@ -142,12 +147,37 @@ def nodeinfo(conn, %{"version" => "2.0"}) do
         restrictedNicknames: Pleroma.Config.get([Pleroma.User, :restricted_nicknames])
       }
     }
+  end
 
+  # Schema definition: https://github.com/jhass/nodeinfo/blob/master/schemas/2.0/schema.json
+  # and https://github.com/jhass/nodeinfo/blob/master/schemas/2.1/schema.json
+  def nodeinfo(conn, %{"version" => "2.0"}) do
     conn
     |> put_resp_header(
       "content-type",
       "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#; charset=utf-8"
     )
+    |> json(raw_nodeinfo())
+  end
+
+  def nodeinfo(conn, %{"version" => "2.1"}) do
+    raw_response = raw_nodeinfo()
+
+    updated_software =
+      raw_response
+      |> Map.get(:software)
+      |> Map.put(:repository, Pleroma.Application.repository())
+
+    response =
+      raw_response
+      |> Map.put(:software, updated_software)
+      |> Map.put(:version, "2.1")
+
+    conn
+    |> put_resp_header(
+      "content-type",
+      "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.1#; charset=utf-8"
+    )
     |> json(response)
   end
 

lib/pleroma/web/ostatus/ostatus.ex

@@ -48,6 +48,9 @@ def remote_follow_path do
 
   def handle_incoming(xml_string) do
     with doc when doc != :error <- parse_document(xml_string) do
+      with {:ok, actor_user} <- find_make_or_update_user(doc),
+           do: Pleroma.Instances.set_reachable(actor_user.ap_id)
+
       entries = :xmerl_xpath.string('//entry', doc)
 
       activities =

lib/pleroma/web/ostatus/ostatus_controller.ex

@@ -14,6 +14,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   alias Pleroma.Web.ActivityPub.ActivityPub
 
   plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming])
+
   action_fallback(:errors)
 
   def feed_redirect(conn, %{"nickname" => nickname}) do

lib/pleroma/web/rich_media/helpers.ex

@@ -7,7 +7,8 @@ defmodule Pleroma.Web.RichMedia.Helpers do
   alias Pleroma.Web.RichMedia.Parser
 
   def fetch_data_for_activity(%Activity{} = activity) do
-    with %Object{} = object <- Object.normalize(activity.data["object"]),
+    with true <- Pleroma.Config.get([:rich_media, :enabled]),
+         %Object{} = object <- Object.normalize(activity.data["object"]),
          {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]),
          {:ok, rich_media} <- Parser.parse(page_url) do
       %{page_url: page_url, rich_media: rich_media}

lib/pleroma/web/rich_media/parser.ex

@@ -30,7 +30,7 @@ defp parse_url(url) do
     try do
       {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: [pool: :media])
 
-      html |> maybe_parse() |> get_parsed_data()
+      html |> maybe_parse() |> clean_parsed_data() |> check_parsed_data()
     rescue
       e ->
         {:error, "Parsing error: #{inspect(e)}"}
@@ -46,11 +46,33 @@ defp maybe_parse(html) do
     end)
   end
 
-  defp get_parsed_data(%{title: title} = data) when is_binary(title) and byte_size(title) > 0 do
+  defp check_parsed_data(%{title: title} = data) when is_binary(title) and byte_size(title) > 0 do
     {:ok, data}
   end
 
-  defp get_parsed_data(data) do
+  defp check_parsed_data(data) do
     {:error, "Found metadata was invalid or incomplete: #{inspect(data)}"}
   end
+
+  defp string_is_valid_unicode(data) when is_binary(data) do
+    data
+    |> :unicode.characters_to_binary()
+    |> clean_string()
+  end
+
+  defp string_is_valid_unicode(data), do: {:ok, data}
+
+  defp clean_string({:error, _, _}), do: {:error, "Invalid data"}
+  defp clean_string(data), do: {:ok, data}
+
+  defp clean_parsed_data(data) do
+    data
+    |> Enum.reject(fn {_, val} ->
+      case string_is_valid_unicode(val) do
+        {:ok, _} -> false
+        _ -> true
+      end
+    end)
+    |> Map.new()
+  end
 end

lib/pleroma/web/router.ex

@@ -185,6 +185,7 @@ defmodule Pleroma.Web.Router do
     get("/timelines/direct", MastodonAPIController, :dm_timeline)
 
     get("/favourites", MastodonAPIController, :favourites)
+    get("/bookmarks", MastodonAPIController, :bookmarks)
 
     post("/statuses", MastodonAPIController, :post_status)
     delete("/statuses/:id", MastodonAPIController, :delete_status)
@@ -195,6 +196,8 @@ defmodule Pleroma.Web.Router do
     post("/statuses/:id/unfavourite", MastodonAPIController, :unfav_status)
     post("/statuses/:id/pin", MastodonAPIController, :pin_status)
     post("/statuses/:id/unpin", MastodonAPIController, :unpin_status)
+    post("/statuses/:id/bookmark", MastodonAPIController, :bookmark_status)
+    post("/statuses/:id/unbookmark", MastodonAPIController, :unbookmark_status)
 
     post("/notifications/clear", MastodonAPIController, :clear_notifications)
     post("/notifications/dismiss", MastodonAPIController, :dismiss_notification)

lib/pleroma/web/salmon/salmon.ex

@@ -6,6 +6,7 @@ defmodule Pleroma.Web.Salmon do
   @httpoison Application.get_env(:pleroma, :httpoison)
 
   use Bitwise
+  alias Pleroma.Instances
   alias Pleroma.Web.XML
   alias Pleroma.Web.OStatus.ActivityRepresenter
   alias Pleroma.User
@@ -161,25 +162,31 @@ def remote_users(%{data: %{"to" => to} = data}) do
     |> Enum.filter(fn user -> user && !user.local end)
   end
 
-  # push an activity to remote accounts
+  @doc "Pushes an activity to remote account."
-  #
+  def send_to_user(%{recipient: %{info: %{salmon: salmon}}} = params),
-  defp send_to_user(%{info: %{salmon: salmon}}, feed, poster),
+    do: send_to_user(Map.put(params, :recipient, salmon))
-    do: send_to_user(salmon, feed, poster)
 
-  defp send_to_user(url, feed, poster) when is_binary(url) do
+  def send_to_user(%{recipient: url, feed: feed, poster: poster} = params) when is_binary(url) do
-    with {:ok, %{status: code}} <-
+    with {:ok, %{status: code}} when code in 200..299 <-
            poster.(
              url,
              feed,
              [{"Content-Type", "application/magic-envelope+xml"}]
            ) do
+      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
+        do: Instances.set_reachable(url)
+
       Logger.debug(fn -> "Pushed to #{url}, code #{code}" end)
+      :ok
     else
-      e -> Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end)
+      e ->
+        unless params[:unreachable_since], do: Instances.set_reachable(url)
+        Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end)
+        :error
     end
   end
 
-  defp send_to_user(_, _, _), do: nil
+  def send_to_user(_), do: :noop
 
   @supported_activities [
     "Create",
@@ -209,12 +216,23 @@ def publish(%{info: %{keys: keys}} = user, %{data: %{"type" => type}} = activity
       {:ok, private, _} = keys_from_pem(keys)
       {:ok, feed} = encode(private, feed)
 
-      remote_users(activity)
+      remote_users = remote_users(activity)
+
+      salmon_urls = Enum.map(remote_users, & &1.info.salmon)
+      reachable_urls_metadata = Instances.filter_reachable(salmon_urls)
+      reachable_urls = Map.keys(reachable_urls_metadata)
+
+      remote_users
+      |> Enum.filter(&(&1.info.salmon in reachable_urls))
       |> Enum.each(fn remote_user ->
-        Task.start(fn ->
+        Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end)
-          Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end)
+
-          send_to_user(remote_user, feed, poster)
+        Pleroma.Web.Federator.publish_single_salmon(%{
-        end)
+          recipient: remote_user,
+          feed: feed,
+          poster: poster,
+          unreachable_since: reachable_urls_metadata[remote_user.info.salmon]
+        })
       end)
     end
   end

lib/pleroma/web/templates/layout/app.html.eex

@@ -1,7 +1,8 @@
 <!DOCTYPE html>
 <html>
   <head>
-    <meta charset=utf-8 />
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width,initial-scale=1,minimal-ui" />
     <title>
     <%= Application.get_env(:pleroma, :instance)[:name] %>
     </title>

lib/pleroma/web/templates/mastodon_api/mastodon/index.html.eex

@@ -1,23 +1,28 @@
 <!DOCTYPE html>
 <html lang='en'>
 <head>
+<meta charset='utf-8'>
+<meta content='width=device-width, initial-scale=1' name='viewport'>
 <title>
 <%= Application.get_env(:pleroma, :instance)[:name] %>
 </title>
-<meta charset='utf-8'>
-<meta content='width=device-width, initial-scale=1' name='viewport'>
 <link rel="icon" type="image/png" href="/favicon.png"/>
-<link rel="stylesheet" media="all" href="/packs/common.css" />
+<script crossorigin='anonymous' src="/packs/locales.js"></script>
-<link rel="stylesheet" media="all" href="/packs/default.css" />
+<script crossorigin='anonymous' src="/packs/locales/glitch/en.js"></script>
 
-<script src="/packs/common.js"></script>
+<link rel='preload' as='script' crossorigin='anonymous' href='/packs/features/getting_started.js'>
-<script src="/packs/locale_en.js"></script>
+<link rel='preload' as='script' crossorigin='anonymous' href='/packs/features/compose.js'>
-<link as='script' crossorigin='anonymous' href='/packs/features/getting_started.js' rel='preload'>
+<link rel='preload' as='script' crossorigin='anonymous' href='/packs/features/home_timeline.js'>
-<link as='script' crossorigin='anonymous' href='/packs/features/compose.js' rel='preload'>
+<link rel='preload' as='script' crossorigin='anonymous' href='/packs/features/notifications.js'>
-<link as='script' crossorigin='anonymous' href='/packs/features/home_timeline.js' rel='preload'>
-<link as='script' crossorigin='anonymous' href='/packs/features/notifications.js' rel='preload'>
 <script id='initial-state' type='application/json'><%= raw @initial_state %></script>
-<script src="/packs/application.js"></script>
+
+<script src="/packs/core/common.js"></script>
+<link rel="stylesheet" media="all" href="/packs/core/common.css" />
+
+<script src="/packs/flavours/glitch/common.js"></script>
+<link rel="stylesheet" media="all" href="/packs/flavours/glitch/common.css" />
+
+<script src="/packs/flavours/glitch/home.js"></script>
 </head>
 <body class='app-body no-reduce-motion system-font'>
   <div class='app-holder' data-props='{&quot;locale&quot;:&quot;en&quot;}' id='mastodon'>

lib/pleroma/web/websub/websub.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.Websub do
   alias Ecto.Changeset
   alias Pleroma.Repo
+  alias Pleroma.Instances
   alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription}
   alias Pleroma.Web.OStatus.FeedRepresenter
   alias Pleroma.Web.{XML, Endpoint, OStatus, Federator}
@@ -53,28 +54,34 @@ def verify(subscription, getter \\ &@httpoison.get/3) do
   ]
   def publish(topic, user, %{data: %{"type" => type}} = activity)
       when type in @supported_activities do
-    # TODO: Only send to still valid subscriptions.
+    response =
+      user
+      |> FeedRepresenter.to_simple_form([activity], [user])
+      |> :xmerl.export_simple(:xmerl_xml)
+      |> to_string
+
     query =
       from(
         sub in WebsubServerSubscription,
         where: sub.topic == ^topic and sub.state == "active",
-        where: fragment("? > NOW()", sub.valid_until)
+        where: fragment("? > (NOW() at time zone 'UTC')", sub.valid_until)
       )
 
     subscriptions = Repo.all(query)
 
-    Enum.each(subscriptions, fn sub ->
+    callbacks = Enum.map(subscriptions, & &1.callback)
-      response =
+    reachable_callbacks_metadata = Instances.filter_reachable(callbacks)
-        user
+    reachable_callbacks = Map.keys(reachable_callbacks_metadata)
-        |> FeedRepresenter.to_simple_form([activity], [user])
-        |> :xmerl.export_simple(:xmerl_xml)
-        |> to_string
 
+    subscriptions
+    |> Enum.filter(&(&1.callback in reachable_callbacks))
+    |> Enum.each(fn sub ->
       data = %{
         xml: response,
         topic: topic,
         callback: sub.callback,
-        secret: sub.secret
+        secret: sub.secret,
+        unreachable_since: reachable_callbacks_metadata[sub.callback]
       }
 
       Federator.publish_single_websub(data)
@@ -263,11 +270,11 @@ def refresh_subscriptions(delta \\ 60 * 60 * 24) do
     end)
   end
 
-  def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret}) do
+  def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret} = params) do
     signature = sign(secret || "", xml)
     Logger.info(fn -> "Pushing #{topic} to #{callback}" end)
 
-    with {:ok, %{status: code}} <-
+    with {:ok, %{status: code}} when code in 200..299 <-
            @httpoison.post(
              callback,
              xml,
@@ -276,12 +283,16 @@ def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret}) d
                {"X-Hub-Signature", "sha1=#{signature}"}
              ]
            ) do
+      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
+        do: Instances.set_reachable(callback)
+
       Logger.info(fn -> "Pushed to #{callback}, code #{code}" end)
       {:ok, code}
     else
-      e ->
+      {_post_result, response} ->
-        Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(e)}" end)
+        unless params[:unreachable_since], do: Instances.set_reachable(callback)
-        {:error, e}
+        Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(response)}" end)
+        {:error, response}
     end
   end
 end

lib/pleroma/web/websub/websub_controller.ex

@@ -4,9 +4,11 @@
 
 defmodule Pleroma.Web.Websub.WebsubController do
   use Pleroma.Web, :controller
+
   alias Pleroma.{Repo, User}
   alias Pleroma.Web.{Websub, Federator}
   alias Pleroma.Web.Websub.WebsubClientSubscription
+
   require Logger
 
   plug(

priv/repo/migrations/20181214121049_add_bookmarks_to_users.exs

@@ -0,0 +1,9 @@
+defmodule Pleroma.Repo.Migrations.AddBookmarksToUsers do
+  use Ecto.Migration
+
+  def change do
+    alter table(:users) do
+      add :bookmarks, {:array, :string}, null: false, default: []
+    end
+  end
+end

priv/repo/migrations/20190123125546_create_instances.exs

@@ -0,0 +1,15 @@
+defmodule Pleroma.Repo.Migrations.CreateInstances do
+  use Ecto.Migration
+
+  def change do
+    create table(:instances) do
+      add :host, :string
+      add :unreachable_since, :naive_datetime
+
+      timestamps()
+    end
+
+    create unique_index(:instances, [:host])
+    create index(:instances, [:unreachable_since])
+  end
+end