Merge branch 'develop' of https://git.pleroma.social/pleroma/pleroma into develop

.gitlab-ci.yml

@@ -45,7 +45,8 @@ docs-build:
 unit-testing:
   stage: test
   services:
-  - name: postgres:9.6.2
+  - name: lainsoykaf/postgres-with-rum
+    alias: postgres
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
     - mix deps.get
@@ -54,6 +55,21 @@ unit-testing:
     - mix test --trace --preload-modules
     - mix coveralls
 
+unit-testing-rum:
+  stage: test
+  services:
+  - name: lainsoykaf/postgres-with-rum
+    alias: postgres
+    command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+  variables:
+    RUM_ENABLED: "true"
+  script:
+    - mix deps.get
+    - mix ecto.create
+    - mix ecto.migrate
+    - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
+    - mix test --trace --preload-modules
+
 lint:
   stage: test
   script:
@@ -65,7 +81,6 @@ analysis:
     - mix deps.get
     - mix credo --strict --only=warnings,todo,fixme,consistency,readability
 
-
 docs-deploy:
   stage: deploy
   image: alpine:3.9

CHANGELOG.md

@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Support for Mastodon's remote interaction
 - Mix Tasks: `mix pleroma.database bump_all_conversations`
 - Mix Tasks: `mix pleroma.database remove_embedded_objects`
+- Mix Tasks: `mix pleroma.database update_users_following_followers_counts`
 - Mix Tasks: `mix pleroma.user toggle_confirmed`
 - Federation: Support for reports
 - Configuration: `safe_dm_mentions` option
@@ -19,8 +20,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: `fetch_initial_posts` option
 - Configuration: `notify_email` option
 - Configuration: Media proxy `whitelist` option
+- Configuration: `report_uri` option
 - Pleroma API: User subscriptions
 - Pleroma API: Healthcheck endpoint
+- Pleroma API: `/api/v1/pleroma/mascot` per-user frontend mascot configuration endpoints
 - Admin API: Endpoints for listing/revoking invite tokens
 - Admin API: Endpoints for making users follow/unfollow each other
 - Admin API: added filters (role, tags, email, name) for users endpoint
@@ -69,6 +72,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Deps: Updated Ecto to 3.0.7
 - Don't ship finmoji by default, they can be installed as an emoji pack
 - Hide deactivated users and their statuses
+- Posts which are marked sensitive or tagged nsfw no longer have link previews.
 
 ### Fixed
 - Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended.
@@ -100,6 +104,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Correct `reblogged`, `favourited`, and `bookmarked` values in the reblog status JSON
 - Mastodon API: Exposing default scope of the user to anyone
 - Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`]
+- User-Agent is now sent correctly for all HTTP requests.
 
 ## Removed
 - Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations`

config/config.exs

@@ -48,7 +48,8 @@
 
 config :pleroma, Pleroma.Repo,
   types: Pleroma.PostgresTypes,
-  telemetry_event: [Pleroma.Repo.Instrumenter]
+  telemetry_event: [Pleroma.Repo.Instrumenter],
+  migration_lock: nil
 
 config :pleroma, Pleroma.Captcha,
   enabled: false,
@@ -191,6 +192,7 @@
 # Configures http settings, upstream proxy etc.
 config :pleroma, :http,
   proxy_url: nil,
+  send_user_agent: true,
   adapter: [
     ssl_options: [
       # We don't support TLS v1.3 yet
@@ -274,6 +276,19 @@
     showInstanceSpecificPanel: true
   }
 
+config :pleroma, :assets,
+  mascots: [
+    pleroma_fox_tan: %{
+      url: "/images/pleroma-fox-tan-smol.png",
+      mime_type: "image/png"
+    },
+    pleroma_fox_tan_shy: %{
+      url: "/images/pleroma-fox-tan-shy.png",
+      mime_type: "image/png"
+    }
+  ],
+  default_mascot: :pleroma_fox_tan
+
 config :pleroma, :activitypub,
   accept_blocks: true,
   unfollow_blocked: true,
@@ -296,6 +311,7 @@
   media_removal: [],
   media_nsfw: [],
   federated_timeline_removal: [],
+  report_removal: [],
   reject: [],
   accept: []
 
@@ -464,6 +480,8 @@
   token_expires_in: 600,
   issue_new_refresh_token: true
 
+config :pleroma, :database, rum_enabled: false
+
 config :http_signatures,
   adapter: Pleroma.Signature
 

config/test.exs

@@ -61,6 +61,14 @@
 
 config :pleroma, :app_account_creation, max_requests: 5
 
+config :pleroma, :http_security, report_uri: "https://endpoint.com"
+
+config :pleroma, :http, send_user_agent: false
+
+rum_enabled = System.get_env("RUM_ENABLED") == "true"
+config :pleroma, :database, rum_enabled: rum_enabled
+IO.puts("RUM enabled: #{rum_enabled}")
+
 try do
   import_config "test.secret.exs"
 rescue

docs/api/pleroma_api.md

@@ -252,6 +252,45 @@ See [Admin-API](Admin-API.md)
 ]
 ```
 
+## `/api/v1/pleroma/mascot`
+### Gets user mascot image
+* Method `GET`
+* Authentication: required
+
+* Response: JSON. Returns a mastodon media attachment entity.
+* Example response:
+```json
+{
+    "id": "abcdefg",
+    "url": "https://pleroma.example.org/media/abcdefg.png",
+    "type": "image",
+    "pleroma": {
+        "mime_type": "image/png"
+    }
+}
+```
+
+### Updates user mascot image
+* Method `PUT`
+* Authentication: required
+* Params:
+    * `image`: Multipart image
+* Response: JSON. Returns a mastodon media attachment entity
+  when successful, otherwise returns HTTP 415 `{"error": "error_msg"}`
+* Example response:
+```json
+{
+    "id": "abcdefg",
+    "url": "https://pleroma.example.org/media/abcdefg.png",
+    "type": "image",
+    "pleroma": {
+        "mime_type": "image/png"
+    }
+}
+```
+* Note: Behaves exactly the same as `POST /api/v1/upload`.
+  Can only accept images - any attempt to upload non-image files will be met with `HTTP 415 Unsupported Media Type`.
+
 ## `/api/pleroma/notification_settings`
 ### Updates user notification settings
 * Method `PUT`

docs/config.md

@@ -203,6 +203,16 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i
 * `hide_post_stats`: Hide notices statistics(repeats, favorites, …)
 * `hide_user_stats`: Hide profile statistics(posts, posts per day, followers, followings, …)
 
+## :assets
+
+This section configures assets to be used with various frontends. Currently the only option
+relates to mascots on the mastodon frontend
+
+* `mascots`: KeywordList of mascots, each element __MUST__ contain both a `url` and a
+  `mime_type` key.
+* `default_mascot`: An element from `mascots` - This will be used as the default mascot
+  on MastoFE (default: `:pleroma_fox_tan`)
+
 ## :mrf_simple
 * `media_removal`: List of instances to remove medias from
 * `media_nsfw`: List of instances to put medias as NSFW(sensitive) from
@@ -286,7 +296,8 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start
 * ``sts``: Whether to additionally send a `Strict-Transport-Security` header
 * ``sts_max_age``: The maximum age for the `Strict-Transport-Security` header if sent
 * ``ct_max_age``: The maximum age for the `Expect-CT` header if sent
-* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`.
+* ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`
+* ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header.
 
 ## :mrf_user_allowlist
 
@@ -543,3 +554,18 @@ Configure OAuth 2 provider capabilities:
 * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]`
 * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]`
 * `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays).
+
+## Database options
+
+### RUM indexing for full text search
+* `rum_enabled`: If RUM indexes should be used. Defaults to `false`.
+
+RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. While they may eventually be mainlined, for now they have to be installed as a PostgreSQL extension from https://github.com/postgrespro/rum.
+
+Their advantage over the standard GIN indexes is that they allow efficient ordering of search results by timestamp, which makes search queries a lot faster on larger servers, by one or two orders of magnitude. They take up around 3 times as much space as GIN indexes.
+
+To enable them, both the `rum_enabled` flag has to be set and the following special migration has to be run:
+
+`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`
+
+This will probably take a long time.

docs/config/mrf.md

@@ -5,6 +5,7 @@ Possible uses include:
 
 * marking incoming messages with media from a given account or instance as sensitive
 * rejecting messages from a specific instance
+* rejecting reports (flags) from a specific instance
 * removing/unlisting messages from the public timelines
 * removing media from messages
 * sending only public messages to a specific instance
@@ -41,12 +42,13 @@ Once `SimplePolicy` is enabled, you can configure various groups in the `:mrf_si
 * `media_nsfw`: Servers in this group will have the #nsfw tag and sensitive setting injected into incoming messages which contain media.
 * `reject`: Servers in this group will have their messages rejected.
 * `federated_timeline_removal`: Servers in this group will have their messages unlisted from the public timelines by flipping the `to` and `cc` fields.
+* `report_removal`: Servers in this group will have their reports (flags) rejected.
 
 Servers should be configured as lists.
 
 ### Example
 
-This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com` and remove messages from `spam.university` from the federated timeline:
+This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com`, remove messages from `spam.university` from the federated timeline and block reports (flags) from `whiny.whiner`:
 
 ```
 config :pleroma, :instance,
@@ -56,7 +58,8 @@ config :pleroma, :mrf_simple,
   media_removal: ["illegalporn.biz"],
   media_nsfw: ["porn.biz", "porn.business"],
   reject: ["spam.com"],
-  federated_timeline_removal: ["spam.university"]
+  federated_timeline_removal: ["spam.university"],
+  report_removal: ["whiny.whiner"]
 
 ```
 

lib/mix/tasks/pleroma/database.ex

@@ -5,6 +5,8 @@
 defmodule Mix.Tasks.Pleroma.Database do
   alias Mix.Tasks.Pleroma.Common
   alias Pleroma.Conversation
+  alias Pleroma.Repo
+  alias Pleroma.User
   require Logger
   use Mix.Task
 
@@ -25,6 +27,9 @@ defmodule Mix.Tasks.Pleroma.Database do
 
       mix pleroma.database bump_all_conversations
 
+  ## Remove duplicated items from following and update followers count for all users
+
+      mix pleroma.database update_users_following_followers_counts
   """
   def run(["remove_embedded_objects" | args]) do
     {options, [], []} =
@@ -38,7 +43,7 @@ def run(["remove_embedded_objects" | args]) do
     Common.start_pleroma()
     Logger.info("Removing embedded objects")
 
-    Pleroma.Repo.query!(
+    Repo.query!(
       "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;",
       [],
       timeout: :infinity
@@ -47,7 +52,7 @@ def run(["remove_embedded_objects" | args]) do
     if Keyword.get(options, :vacuum) do
       Logger.info("Runnning VACUUM FULL")
 
-      Pleroma.Repo.query!(
+      Repo.query!(
         "vacuum full;",
         [],
         timeout: :infinity
@@ -59,4 +64,12 @@ def run(["bump_all_conversations"]) do
     Common.start_pleroma()
     Conversation.bump_for_all_activities()
   end
+
+  def run(["update_users_following_followers_counts"]) do
+    Common.start_pleroma()
+
+    users = Repo.all(User)
+    Enum.each(users, &User.remove_duplicated_following/1)
+    Enum.each(users, &User.update_follower_count/1)
+  end
 end

lib/pleroma/http/request_builder.ex

@@ -45,8 +45,15 @@ def url(request, u) do
   Add headers to the request
   """
   @spec headers(map(), list(tuple)) :: map()
-  def headers(request, h) do
+  def headers(request, header_list) do
-    Map.put_new(request, :headers, h)
+    header_list =
+      if Pleroma.Config.get([:http, :send_user_agent]) do
+        header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}]
+      else
+        header_list
+      end
+
+    Map.put_new(request, :headers, header_list)
   end
 
   @doc """

lib/pleroma/plugs/http_security_plug.ex

@@ -20,8 +20,9 @@ def call(conn, _options) do
 
   defp headers do
     referrer_policy = Config.get([:http_security, :referrer_policy])
+    report_uri = Config.get([:http_security, :report_uri])
 
-    [
+    headers = [
       {"x-xss-protection", "1; mode=block"},
       {"x-permitted-cross-domain-policies", "none"},
       {"x-frame-options", "DENY"},
@@ -30,12 +31,27 @@ defp headers do
       {"x-download-options", "noopen"},
       {"content-security-policy", csp_string() <> ";"}
     ]
+
+    if report_uri do
+      report_group = %{
+        "group" => "csp-endpoint",
+        "max-age" => 10_886_400,
+        "endpoints" => [
+          %{"url" => report_uri}
+        ]
+      }
+
+      headers ++ [{"reply-to", Jason.encode!(report_group)}]
+    else
+      headers
+    end
   end
 
   defp csp_string do
     scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
     static_url = Pleroma.Web.Endpoint.static_url()
     websocket_url = Pleroma.Web.Endpoint.websocket_url()
+    report_uri = Config.get([:http_security, :report_uri])
 
     connect_src = "connect-src 'self' #{static_url} #{websocket_url}"
 
@@ -53,7 +69,7 @@ defp csp_string do
         "script-src 'self'"
       end
 
-    [
+    main_part = [
       "default-src 'none'",
       "base-uri 'self'",
       "frame-ancestors 'none'",
@@ -63,11 +79,14 @@ defp csp_string do
       "font-src 'self'",
       "manifest-src 'self'",
       connect_src,
-      script_src,
+      script_src
-      if scheme == "https" do
-        "upgrade-insecure-requests"
-      end
     ]
+
+    report = if report_uri, do: ["report-uri #{report_uri}; report-to csp-endpoint"], else: []
+
+    insecure = if scheme == "https", do: ["upgrade-insecure-requests"], else: []
+
+    (main_part ++ report ++ insecure)
     |> Enum.join("; ")
   end
 

lib/pleroma/user.ex

@@ -55,7 +55,7 @@ defmodule Pleroma.User do
     field(:last_refreshed_at, :naive_datetime_usec)
     has_many(:notifications, Notification)
     has_many(:registrations, Registration)
-    embeds_one(:info, Pleroma.User.Info)
+    embeds_one(:info, User.Info)
 
     timestamps()
   end
@@ -166,7 +166,7 @@ def remote_user_creation(params) do
 
   def update_changeset(struct, params \\ %{}) do
     struct
-    |> cast(params, [:bio, :name, :avatar])
+    |> cast(params, [:bio, :name, :avatar, :following])
     |> unique_constraint(:nickname)
     |> validate_format(:nickname, local_nickname_regex())
     |> validate_length(:bio, max: 5000)
@@ -233,7 +233,7 @@ def register_changeset(struct, params \\ %{}, opts \\ []) do
       |> validate_confirmation(:password)
       |> unique_constraint(:email)
       |> unique_constraint(:nickname)
-      |> validate_exclusion(:nickname, Pleroma.Config.get([Pleroma.User, :restricted_nicknames]))
+      |> validate_exclusion(:nickname, Pleroma.Config.get([User, :restricted_nicknames]))
       |> validate_format(:nickname, local_nickname_regex())
       |> validate_format(:email, @email_regex)
       |> validate_length(:bio, max: 1000)
@@ -278,7 +278,7 @@ def register(%Ecto.Changeset{} = changeset) do
     with {:ok, user} <- Repo.insert(changeset),
          {:ok, user} <- autofollow_users(user),
          {:ok, user} <- set_cache(user),
-         {:ok, _} <- Pleroma.User.WelcomeMessage.post_welcome_message_to_user(user),
+         {:ok, _} <- User.WelcomeMessage.post_welcome_message_to_user(user),
          {:ok, _} <- try_send_confirmation_email(user) do
       {:ok, user}
     end
@@ -709,6 +709,18 @@ def update_follower_count(%User{} = user) do
     end
   end
 
+  def remove_duplicated_following(%User{following: following} = user) do
+    uniq_following = Enum.uniq(following)
+
+    if length(following) == length(uniq_following) do
+      {:ok, user}
+    else
+      user
+      |> update_changeset(%{following: uniq_following})
+      |> update_and_set_cache()
+    end
+  end
+
   @spec get_users_from_set([String.t()], boolean()) :: [User.t()]
   def get_users_from_set(ap_ids, local_only \\ true) do
     criteria = %{ap_id: ap_ids, deactivated: false}
@@ -1132,7 +1144,6 @@ def delete_user_activities(%User{ap_id: ap_id} = user) do
     stream =
       ap_id
       |> Activity.query_by_actor()
-      |> Activity.with_preloaded_object()
       |> Repo.stream()
 
     Repo.transaction(fn -> Enum.each(stream, &delete_activity(&1)) end, timeout: :infinity)
@@ -1391,4 +1402,24 @@ def toggle_confirmation(%User{} = user) do
     |> put_embed(:info, info_changeset)
     |> update_and_set_cache()
   end
+
+  def get_mascot(%{info: %{mascot: %{} = mascot}}) when not is_nil(mascot) do
+    mascot
+  end
+
+  def get_mascot(%{info: %{mascot: mascot}}) when is_nil(mascot) do
+    # use instance-default
+    config = Pleroma.Config.get([:assets, :mascots])
+    default_mascot = Pleroma.Config.get([:assets, :default_mascot])
+    mascot = Keyword.get(config, default_mascot)
+
+    %{
+      "id" => "default-mascot",
+      "url" => mascot[:url],
+      "preview_url" => mascot[:url],
+      "pleroma" => %{
+        "mime_type" => mascot[:mime_type]
+      }
+    }
+  end
 end

lib/pleroma/user/info.ex

@@ -43,6 +43,7 @@ defmodule Pleroma.User.Info do
     field(:hide_favorites, :boolean, default: true)
     field(:pinned_activities, {:array, :string}, default: [])
     field(:flavour, :string, default: nil)
+    field(:mascot, :map, default: nil)
     field(:emoji, {:array, :map}, default: [])
 
     field(:notification_settings, :map,
@@ -248,6 +249,14 @@ def mastodon_flavour_update(info, flavour) do
     |> validate_required([:flavour])
   end
 
+  def mascot_update(info, url) do
+    params = %{mascot: url}
+
+    info
+    |> cast(params, [:mascot])
+    |> validate_required([:mascot])
+  end
+
   def set_source_data(info, source_data) do
     params = %{source_data: source_data}
 

lib/pleroma/web/activity_pub/mrf/simple_policy.ex

@@ -48,14 +48,13 @@ defp check_media_nsfw(
          %{host: actor_host} = _actor_info,
          %{
            "type" => "Create",
-           "object" => %{"attachment" => child_attachment} = child_object
+           "object" => child_object
          } = object
-       )
+       ) do
-       when length(child_attachment) > 0 do
     object =
       if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_nsfw]), actor_host) do
         tags = (child_object["tag"] || []) ++ ["nsfw"]
-        child_object = Map.put(child_object, "tags", tags)
+        child_object = Map.put(child_object, "tag", tags)
         child_object = Map.put(child_object, "sensitive", true)
         Map.put(object, "object", child_object)
       else
@@ -95,6 +94,16 @@ defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do
     {:ok, object}
   end
 
+  defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do
+    if actor_host in Pleroma.Config.get([:mrf_simple, :report_removal]) do
+      {:reject, nil}
+    else
+      {:ok, object}
+    end
+  end
+
+  defp check_report_removal(_actor_info, object), do: {:ok, object}
+
   @impl true
   def filter(object) do
     actor_info = URI.parse(object["actor"])
@@ -103,7 +112,8 @@ def filter(object) do
          {:ok, object} <- check_reject(actor_info, object),
          {:ok, object} <- check_media_removal(actor_info, object),
          {:ok, object} <- check_media_nsfw(actor_info, object),
-         {:ok, object} <- check_ftl_removal(actor_info, object) do
+         {:ok, object} <- check_ftl_removal(actor_info, object),
+         {:ok, object} <- check_report_removal(actor_info, object) do
       {:ok, object}
     else
       _e -> {:reject, nil}

lib/pleroma/web/activity_pub/mrf/tag_policy.ex

@@ -31,7 +31,7 @@ defp process_tag(
 
     object =
       object
-      |> Map.put("tags", tags)
+      |> Map.put("tag", tags)
       |> Map.put("sensitive", true)
 
     message = Map.put(message, "object", object)

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -11,7 +11,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   alias Pleroma.Object.Containment
   alias Pleroma.Repo
   alias Pleroma.User
-  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility

lib/pleroma/web/common_api/common_api.ex

@@ -157,6 +157,7 @@ def post(user, %{"status" => status} = data) do
          {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility),
          context <- make_context(in_reply_to),
          cw <- data["spoiler_text"] || "",
+         sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}),
          full_payload <- String.trim(status <> cw),
          length when length in 1..limit <- String.length(full_payload),
          object <-
@@ -169,7 +170,8 @@ def post(user, %{"status" => status} = data) do
              in_reply_to,
              tags,
              cw,
-             cc
+             cc,
+             sensitive
            ),
          object <-
            Map.put(
@@ -200,7 +202,7 @@ def update(user) do
     user =
       with emoji <- emoji_from_profile(user),
            source_data <- (user.info.source_data || %{}) |> Map.put("tag", emoji),
-           info_cng <- Pleroma.User.Info.set_source_data(user.info, source_data),
+           info_cng <- User.Info.set_source_data(user.info, source_data),
            change <- Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_cng),
            {:ok, user} <- User.update_and_set_cache(change) do
         user
@@ -233,7 +235,7 @@ def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do
          } = activity <- get_by_id_or_ap_id(id_or_ap_id),
          true <- Enum.member?(object_to, "https://www.w3.org/ns/activitystreams#Public"),
          %{valid?: true} = info_changeset <-
-           Pleroma.User.Info.add_pinnned_activity(user.info, activity),
+           User.Info.add_pinnned_activity(user.info, activity),
          changeset <-
            Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset),
          {:ok, _user} <- User.update_and_set_cache(changeset) do
@@ -250,7 +252,7 @@ def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do
   def unpin(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
          %{valid?: true} = info_changeset <-
-           Pleroma.User.Info.remove_pinnned_activity(user.info, activity),
+           User.Info.remove_pinnned_activity(user.info, activity),
          changeset <-
            Ecto.Changeset.change(user) |> Ecto.Changeset.put_embed(:info, info_changeset),
          {:ok, _user} <- User.update_and_set_cache(changeset) do

lib/pleroma/web/common_api/utils.ex

@@ -223,7 +223,8 @@ def make_note_data(
         in_reply_to,
         tags,
         cw \\ nil,
-        cc \\ []
+        cc \\ [],
+        sensitive \\ false
       ) do
     object = %{
       "type" => "Note",
@@ -231,6 +232,7 @@ def make_note_data(
       "cc" => cc,
       "content" => content_html,
       "summary" => cw,
+      "sensitive" => !Enum.member?(["false", "False", "0", false], sensitive),
       "context" => context,
       "attachment" => attachments,
       "actor" => actor,

lib/pleroma/web/federator/publisher.ex

@@ -52,9 +52,9 @@ def perform(type, _, _) do
   @doc """
   Relays an activity to all specified peers.
   """
-  @callback publish(Pleroma.User.t(), Pleroma.Activity.t()) :: :ok | {:error, any()}
+  @callback publish(User.t(), Activity.t()) :: :ok | {:error, any()}
 
-  @spec publish(Pleroma.User.t(), Pleroma.Activity.t()) :: :ok
+  @spec publish(User.t(), Activity.t()) :: :ok
   def publish(%User{} = user, %Activity{} = activity) do
     Config.get([:instance, :federation_publisher_modules])
     |> Enum.each(fn module ->
@@ -70,9 +70,9 @@ def publish(%User{} = user, %Activity{} = activity) do
   @doc """
   Gathers links used by an outgoing federation module for WebFinger output.
   """
-  @callback gather_webfinger_links(Pleroma.User.t()) :: list()
+  @callback gather_webfinger_links(User.t()) :: list()
 
-  @spec gather_webfinger_links(Pleroma.User.t()) :: list()
+  @spec gather_webfinger_links(User.t()) :: list()
   def gather_webfinger_links(%User{} = user) do
     Config.get([:instance, :federation_publisher_modules])
     |> Enum.reduce([], fn module, links ->

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -707,6 +707,41 @@ def upload(%{assigns: %{user: user}} = conn, %{"file" => file} = data) do
     end
   end
 
+  def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do
+    with {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)),
+         %{} = attachment_data <- Map.put(object.data, "id", object.id),
+         %{type: type} = rendered <-
+           StatusView.render("attachment.json", %{attachment: attachment_data}) do
+      # Reject if not an image
+      if type == "image" do
+        # Sure!
+        # Save to the user's info
+        info_changeset = User.Info.mascot_update(user.info, rendered)
+
+        user_changeset =
+          user
+          |> Ecto.Changeset.change()
+          |> Ecto.Changeset.put_embed(:info, info_changeset)
+
+        {:ok, _user} = User.update_and_set_cache(user_changeset)
+
+        conn
+        |> json(rendered)
+      else
+        conn
+        |> put_resp_content_type("application/json")
+        |> send_resp(415, Jason.encode!(%{"error" => "mascots can only be images"}))
+      end
+    end
+  end
+
+  def get_mascot(%{assigns: %{user: user}} = conn, _params) do
+    mascot = User.get_mascot(user)
+
+    conn
+    |> json(mascot)
+  end
+
   def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
          %Object{data: %{"likes" => likes}} <- Object.normalize(object) do
@@ -1009,6 +1044,30 @@ def unsubscribe(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     end
   end
 
+  def status_search_query_with_gin(q, query) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
+          o.data,
+          ^query
+        ),
+      order_by: [desc: :id]
+    )
+  end
+
+  def status_search_query_with_rum(q, query) do
+    from([a, o] in q,
+      where:
+        fragment(
+          "? @@ plainto_tsquery('english', ?)",
+          o.fts_content,
+          ^query
+        ),
+      order_by: [fragment("? <=> now()::date", o.inserted_at)]
+    )
+  end
+
   def status_search(user, query) do
     fetched =
       if Regex.match?(~r/https?:/, query) do
@@ -1022,20 +1081,19 @@ def status_search(user, query) do
       end || []
 
     q =
-      from(
+      from([a, o] in Activity.with_preloaded_object(Activity),
-        [a, o] in Activity.with_preloaded_object(Activity),
         where: fragment("?->>'type' = 'Create'", a.data),
         where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
-        where:
+        limit: 20
-          fragment(
-            "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
-            o.data,
-            ^query
-          ),
-        limit: 20,
-        order_by: [desc: :id]
       )
 
+    q =
+      if Pleroma.Config.get([:database, :rum_enabled]) do
+        status_search_query_with_rum(q, query)
+      else
+        status_search_query_with_gin(q, query)
+      end
+
     Repo.all(q) ++ fetched
   end
 
@@ -1222,7 +1280,7 @@ def remove_from_list(%{assigns: %{user: user}} = conn, %{"id" => id, "account_id
     accounts
     |> Enum.each(fn account_id ->
       with %Pleroma.List{} = list <- Pleroma.List.get(id, user),
-           %User{} = followed <- Pleroma.User.get_cached_by_id(account_id) do
+           %User{} = followed <- User.get_cached_by_id(account_id) do
         Pleroma.List.unfollow(list, followed)
       end
     end)
@@ -1306,7 +1364,7 @@ def index(%{assigns: %{user: user}} = conn, _params) do
             display_sensitive_media: false,
             reduce_motion: false,
             max_toot_chars: limit,
-            mascot: "/images/pleroma-fox-tan-smol.png"
+            mascot: User.get_mascot(user)["url"]
           },
           rights: %{
             delete_others_notice: present?(user.info.is_moderator),

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -40,7 +40,7 @@ def render("relationship.json", %{user: %User{} = user, target: %User{} = target
     follow_activity = Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(user, target)
 
     requested =
-      if follow_activity do
+      if follow_activity && !User.following?(target, user) do
         follow_activity.data["state"] == "pending"
       else
         false

lib/pleroma/web/oauth/authorization.ex

@@ -20,7 +20,7 @@ defmodule Pleroma.Web.OAuth.Authorization do
     field(:scopes, {:array, :string}, default: [])
     field(:valid_until, :naive_datetime_usec)
     field(:used, :boolean, default: false)
-    belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
     timestamps()

lib/pleroma/web/oauth/token.ex

@@ -22,7 +22,7 @@ defmodule Pleroma.Web.OAuth.Token do
     field(:refresh_token, :string)
     field(:scopes, {:array, :string}, default: [])
     field(:valid_until, :naive_datetime_usec)
-    belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId)
+    belongs_to(:user, User, type: Pleroma.FlakeId)
     belongs_to(:app, App)
 
     timestamps()

lib/pleroma/web/rich_media/helpers.ex

@@ -24,6 +24,7 @@ defp validate_page_url(_), do: :error
   def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do
     with true <- Pleroma.Config.get([:rich_media, :enabled]),
          %Object{} = object <- Object.normalize(activity),
+         false <- object.data["sensitive"] || false,
          {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]),
          :ok <- validate_page_url(page_url),
          {:ok, rich_media} <- Parser.parse(page_url) do

lib/pleroma/web/router.ex

@@ -352,6 +352,9 @@ defmodule Pleroma.Web.Router do
 
       post("/pleroma/flavour/:flavour", MastodonAPIController, :set_flavour)
 
+      get("/pleroma/mascot", MastodonAPIController, :get_mascot)
+      put("/pleroma/mascot", MastodonAPIController, :set_mascot)
+
       post("/reports", MastodonAPIController, :reports)
     end
 

lib/pleroma/web/web_finger/web_finger.ex

@@ -99,7 +99,7 @@ def ensure_keys_present(user) do
 
       info_cng =
         info
-        |> Pleroma.User.Info.set_keys(pem)
+        |> User.Info.set_keys(pem)
 
       cng =
         Ecto.Changeset.change(user)

mix.exs

@@ -66,7 +66,10 @@ defp deps do
       {:plug_cowboy, "~> 2.0"},
       {:phoenix_pubsub, "~> 1.1"},
       {:phoenix_ecto, "~> 4.0"},
-      {:ecto_sql, "~>3.0.5"},
+      {:ecto_sql,
+       git: "https://github.com/elixir-ecto/ecto_sql",
+       ref: "14cb065a74c488d737d973f7a91bc036c6245f78",
+       override: true},
       {:postgrex, ">= 0.13.5"},
       {:gettext, "~> 0.15"},
       {:comeonin, "~> 4.1.1"},

mix.lock

@@ -16,12 +16,12 @@
   "cowlib": {:hex, :cowlib, "2.7.0", "3ef16e77562f9855a2605900cedb15c1462d76fb1be6a32fc3ae91973ee543d2", [:rebar3], [], "hexpm"},
   "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"},
   "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]},
-  "db_connection": {:hex, :db_connection, "2.0.5", "ddb2ba6761a08b2bb9ca0e7d260e8f4dd39067426d835c24491a321b7f92a4da", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"},
+  "db_connection": {:hex, :db_connection, "2.0.6", "bde2f85d047969c5b5800cb8f4b3ed6316c8cb11487afedac4aa5f93fd39abfa", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"},
   "decimal": {:hex, :decimal, "1.7.0", "30d6b52c88541f9a66637359ddf85016df9eb266170d53105f02e4a67e00c5aa", [:mix], [], "hexpm"},
   "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"},
   "earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"},
-  "ecto": {:hex, :ecto, "3.0.7", "44dda84ac6b17bbbdeb8ac5dfef08b7da253b37a453c34ab1a98de7f7e5fec7f", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
+  "ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
-  "ecto_sql": {:hex, :ecto_sql, "3.0.5", "7e44172b4f7aca4469f38d7f6a3da394dbf43a1bcf0ca975e958cb957becd74e", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.0.6", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.3.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
+  "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "14cb065a74c488d737d973f7a91bc036c6245f78", [ref: "14cb065a74c488d737d973f7a91bc036c6245f78"]},
   "esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"},
   "eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"},
   "ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"},
@@ -66,7 +66,7 @@
   "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"},
   "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm"},
-  "postgrex": {:hex, :postgrex, "0.14.1", "63247d4a5ad6b9de57a0bac5d807e1c32d41e39c04b8a4156a26c63bcd8a2e49", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
+  "postgrex": {:hex, :postgrex, "0.14.3", "5754dee2fdf6e9e508cbf49ab138df964278700b764177e8f3871e658b345a1e", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
   "prometheus": {:hex, :prometheus, "4.2.2", "a830e77b79dc6d28183f4db050a7cac926a6c58f1872f9ef94a35cd989aceef8", [:mix, :rebar3], [], "hexpm"},
   "prometheus_ecto": {:hex, :prometheus_ecto, "1.4.1", "6c768ea9654de871e5b32fab2eac348467b3021604ebebbcbd8bcbe806a65ed5", [:mix], [{:ecto, "~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:prometheus_ex, "~> 1.1 or ~> 2.0 or ~> 3.0", [hex: :prometheus_ex, repo: "hexpm", optional: false]}], "hexpm"},
   "prometheus_ex": {:hex, :prometheus_ex, "3.0.5", "fa58cfd983487fc5ead331e9a3e0aa622c67232b3ec71710ced122c4c453a02f", [:mix], [{:prometheus, "~> 4.0", [hex: :prometheus, repo: "hexpm", optional: false]}], "hexpm"},
@@ -79,11 +79,11 @@
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.4", "f0eafff810d2041e93f915ef59899c923f4568f4585904d010387ed74988e77b", [:make, :mix, :rebar3], [], "hexpm"},
   "swoosh": {:hex, :swoosh, "0.20.0", "9a6c13822c9815993c03b6f8fccc370fcffb3c158d9754f67b1fdee6b3a5d928", [:mix], [{:cowboy, "~> 1.0.1 or ~> 1.1 or ~> 2.4", [hex: :cowboy, repo: "hexpm", optional: true]}, {:gen_smtp, "~> 0.12", [hex: :gen_smtp, repo: "hexpm", optional: true]}, {:hackney, "~> 1.9", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 1.1", [hex: :mime, repo: "hexpm", optional: false]}, {:plug, "~> 1.4", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm"},
   "syslog": {:git, "https://github.com/Vagabond/erlang-syslog.git", "4a6c6f2c996483e86c1320e9553f91d337bcb6aa", [tag: "1.0.5"]},
-  "telemetry": {:hex, :telemetry, "0.3.0", "099a7f3ce31e4780f971b4630a3c22ec66d22208bc090fe33a2a3a6a67754a73", [:rebar3], [], "hexpm"},
+  "telemetry": {:hex, :telemetry, "0.4.0", "8339bee3fa8b91cb84d14c2935f8ecf399ccd87301ad6da6b71c09553834b2ab", [:rebar3], [], "hexpm"},
   "tesla": {:hex, :tesla, "1.2.1", "864783cc27f71dd8c8969163704752476cec0f3a51eb3b06393b3971dc9733ff", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "~> 4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}], "hexpm"},
   "timex": {:hex, :timex, "3.5.0", "b0a23167da02d0fe4f1a4e104d1f929a00d348502b52432c05de875d0b9cffa5", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm"},
   "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
-  "tzdata": {:hex, :tzdata, "0.5.17", "50793e3d85af49736701da1a040c415c97dc1caf6464112fd9bd18f425d3053b", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
+  "tzdata": {:hex, :tzdata, "0.5.20", "304b9e98a02840fb32a43ec111ffbe517863c8566eb04a061f1c4dbb90b4d84c", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm"},
   "ueberauth": {:hex, :ueberauth, "0.6.1", "9e90d3337dddf38b1ca2753aca9b1e53d8a52b890191cdc55240247c89230412", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
   "unicode_util_compat": {:hex, :unicode_util_compat, "0.4.1", "d869e4c68901dd9531385bb0c8c40444ebf624e60b6962d95952775cac5e90cd", [:rebar3], [], "hexpm"},
   "unsafe": {:hex, :unsafe, "1.0.0", "7c21742cd05380c7875546b023481d3a26f52df8e5dfedcb9f958f322baae305", [:mix], [], "hexpm"},

priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs

@@ -0,0 +1,34 @@
+defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do
+  use Ecto.Migration
+
+  def up do
+    execute("create extension if not exists rum")
+    drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
+    alter table(:objects) do
+      add(:fts_content, :tsvector)
+    end
+
+    execute("CREATE FUNCTION objects_fts_update() RETURNS trigger AS $$
+    begin
+      new.fts_content := to_tsvector('english', new.data->>'content');
+      return new;
+    end
+    $$ LANGUAGE plpgsql")
+    execute("create index objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');")
+
+    execute("CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE ON objects
+    FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()")
+
+    execute("UPDATE objects SET updated_at = NOW()")
+  end
+
+  def down do
+    execute "drop index objects_fts"
+    execute "drop trigger tsvectorupdate on objects"
+    execute "drop function objects_fts_update()"
+    alter table(:objects) do
+      remove(:fts_content, :tsvector)
+    end
+    create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts)
+  end
+end

test/formatter_test.exs

@@ -125,7 +125,7 @@ test "gives a replacement for user links, using local nicknames in user links te
       archaeme =
         insert(:user, %{
           nickname: "archa_eme_",
-          info: %Pleroma.User.Info{source_data: %{"url" => "https://archeme/@archa_eme_"}}
+          info: %User.Info{source_data: %{"url" => "https://archeme/@archa_eme_"}}
         })
 
       archaeme_remote = insert(:user, %{nickname: "archaeme@archae.me"})

test/plugs/http_security_plug_test.exs

@@ -7,12 +7,21 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do
   alias Pleroma.Config
   alias Plug.Conn
 
-  test "it sends CSP headers when enabled", %{conn: conn} do
+  describe "http security enabled" do
+    setup do
+      enabled = Config.get([:http_securiy, :enabled])
+
       Config.put([:http_security, :enabled], true)
 
-    conn =
+      on_exit(fn ->
-      conn
+        Config.put([:http_security, :enabled], enabled)
-      |> get("/api/v1/instance")
+      end)
+
+      :ok
+    end
+
+    test "it sends CSP headers when enabled", %{conn: conn} do
+      conn = get(conn, "/api/v1/instance")
 
       refute Conn.get_resp_header(conn, "x-xss-protection") == []
       refute Conn.get_resp_header(conn, "x-permitted-cross-domain-policies") == []
@@ -23,52 +32,26 @@ test "it sends CSP headers when enabled", %{conn: conn} do
       refute Conn.get_resp_header(conn, "content-security-policy") == []
     end
 
-  test "it does not send CSP headers when disabled", %{conn: conn} do
-    Config.put([:http_security, :enabled], false)
-
-    conn =
-      conn
-      |> get("/api/v1/instance")
-
-    assert Conn.get_resp_header(conn, "x-xss-protection") == []
-    assert Conn.get_resp_header(conn, "x-permitted-cross-domain-policies") == []
-    assert Conn.get_resp_header(conn, "x-frame-options") == []
-    assert Conn.get_resp_header(conn, "x-content-type-options") == []
-    assert Conn.get_resp_header(conn, "x-download-options") == []
-    assert Conn.get_resp_header(conn, "referrer-policy") == []
-    assert Conn.get_resp_header(conn, "content-security-policy") == []
-  end
-
     test "it sends STS headers when enabled", %{conn: conn} do
-    Config.put([:http_security, :enabled], true)
       Config.put([:http_security, :sts], true)
 
-    conn =
+      conn = get(conn, "/api/v1/instance")
-      conn
-      |> get("/api/v1/instance")
 
       refute Conn.get_resp_header(conn, "strict-transport-security") == []
       refute Conn.get_resp_header(conn, "expect-ct") == []
     end
 
     test "it does not send STS headers when disabled", %{conn: conn} do
-    Config.put([:http_security, :enabled], true)
       Config.put([:http_security, :sts], false)
 
-    conn =
+      conn = get(conn, "/api/v1/instance")
-      conn
-      |> get("/api/v1/instance")
 
       assert Conn.get_resp_header(conn, "strict-transport-security") == []
       assert Conn.get_resp_header(conn, "expect-ct") == []
     end
 
     test "referrer-policy header reflects configured value", %{conn: conn} do
-    Config.put([:http_security, :enabled], true)
+      conn = get(conn, "/api/v1/instance")
-
-    conn =
-      conn
-      |> get("/api/v1/instance")
 
       assert Conn.get_resp_header(conn, "referrer-policy") == ["same-origin"]
 
@@ -80,4 +63,40 @@ test "referrer-policy header reflects configured value", %{conn: conn} do
 
       assert Conn.get_resp_header(conn, "referrer-policy") == ["no-referrer"]
     end
+
+    test "it sends `report-to` & `report-uri` CSP response headers" do
+      conn =
+        build_conn()
+        |> get("/api/v1/instance")
+
+      [csp] = Conn.get_resp_header(conn, "content-security-policy")
+
+      assert csp =~ ~r|report-uri https://endpoint.com; report-to csp-endpoint;|
+
+      [reply_to] = Conn.get_resp_header(conn, "reply-to")
+
+      assert reply_to ==
+               "{\"endpoints\":[{\"url\":\"https://endpoint.com\"}],\"group\":\"csp-endpoint\",\"max-age\":10886400}"
+    end
+  end
+
+  test "it does not send CSP headers when disabled", %{conn: conn} do
+    enabled = Config.get([:http_securiy, :enabled])
+
+    Config.put([:http_security, :enabled], false)
+
+    on_exit(fn ->
+      Config.put([:http_security, :enabled], enabled)
+    end)
+
+    conn = get(conn, "/api/v1/instance")
+
+    assert Conn.get_resp_header(conn, "x-xss-protection") == []
+    assert Conn.get_resp_header(conn, "x-permitted-cross-domain-policies") == []
+    assert Conn.get_resp_header(conn, "x-frame-options") == []
+    assert Conn.get_resp_header(conn, "x-content-type-options") == []
+    assert Conn.get_resp_header(conn, "x-download-options") == []
+    assert Conn.get_resp_header(conn, "referrer-policy") == []
+    assert Conn.get_resp_header(conn, "content-security-policy") == []
+  end
 end

test/plugs/legacy_authentication_plug_test.exs

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.LegacyAuthenticationPlugTest do
-  use Pleroma.Web.ConnCase, async: true
+  use Pleroma.Web.ConnCase
 
   alias Pleroma.Plugs.LegacyAuthenticationPlug
   alias Pleroma.User

test/repo_test.exs

@@ -1,23 +1,24 @@
 defmodule Pleroma.RepoTest do
   use Pleroma.DataCase
   import Pleroma.Factory
+  alias Pleroma.User
 
   describe "find_resource/1" do
     test "returns user" do
       user = insert(:user)
-      query = from(t in Pleroma.User, where: t.id == ^user.id)
+      query = from(t in User, where: t.id == ^user.id)
       assert Repo.find_resource(query) == {:ok, user}
     end
 
     test "returns not_found" do
-      query = from(t in Pleroma.User, where: t.id == ^"9gBuXNpD2NyDmmxxdw")
+      query = from(t in User, where: t.id == ^"9gBuXNpD2NyDmmxxdw")
       assert Repo.find_resource(query) == {:error, :not_found}
     end
   end
 
   describe "get_assoc/2" do
     test "get assoc from preloaded data" do
-      user = %Pleroma.User{name: "Agent Smith"}
+      user = %User{name: "Agent Smith"}
       token = %Pleroma.Web.OAuth.Token{insert(:oauth_token) | user: user}
       assert Repo.get_assoc(token, :user) == {:ok, user}
     end

test/support/factory.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Factory do
   use ExMachina.Ecto, repo: Pleroma.Repo
+  alias Pleroma.User
 
   def participation_factory do
     conversation = insert(:conversation)
@@ -23,7 +24,7 @@ def conversation_factory do
   end
 
   def user_factory do
-    user = %Pleroma.User{
+    user = %User{
       name: sequence(:name, &"Test テスト User #{&1}"),
       email: sequence(:email, &"user#{&1}@example.com"),
       nickname: sequence(:nickname, &"nick#{&1}"),
@@ -34,9 +35,9 @@ def user_factory do
 
     %{
       user
-      | ap_id: Pleroma.User.ap_id(user),
+      | ap_id: User.ap_id(user),
-        follower_address: Pleroma.User.ap_followers(user),
+        follower_address: User.ap_followers(user),
-        following: [Pleroma.User.ap_id(user)]
+        following: [User.ap_id(user)]
     }
   end
 

test/tasks/database_test.exs

@@ -0,0 +1,49 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.DatabaseTest do
+  alias Pleroma.Repo
+  alias Pleroma.User
+  use Pleroma.DataCase
+
+  import Pleroma.Factory
+
+  setup_all do
+    Mix.shell(Mix.Shell.Process)
+
+    on_exit(fn ->
+      Mix.shell(Mix.Shell.IO)
+    end)
+
+    :ok
+  end
+
+  describe "running update_users_following_followers_counts" do
+    test "following and followers count are updated" do
+      [user, user2] = insert_pair(:user)
+      {:ok, %User{following: following, info: info} = user} = User.follow(user, user2)
+
+      assert length(following) == 2
+      assert info.follower_count == 0
+
+      info_cng = Ecto.Changeset.change(info, %{follower_count: 3})
+
+      {:ok, user} =
+        user
+        |> Ecto.Changeset.change(%{following: following ++ following})
+        |> Ecto.Changeset.put_embed(:info, info_cng)
+        |> Repo.update()
+
+      assert length(user.following) == 4
+      assert user.info.follower_count == 3
+
+      assert :ok == Mix.Tasks.Pleroma.Database.run(["update_users_following_followers_counts"])
+
+      user = User.get_by_id(user.id)
+
+      assert length(user.following) == 2
+      assert user.info.follower_count == 0
+    end
+  end
+end

test/tasks/user_test.exs

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.UserTest do
+  alias Pleroma.Repo
   alias Pleroma.User
   use Pleroma.DataCase
 

test/user_test.exs

@@ -277,7 +277,7 @@ test "it requires an email, name, nickname and password, bio is optional" do
     end
 
     test "it restricts certain nicknames" do
-      [restricted_name | _] = Pleroma.Config.get([Pleroma.User, :restricted_nicknames])
+      [restricted_name | _] = Pleroma.Config.get([User, :restricted_nicknames])
 
       assert is_bitstring(restricted_name)
 
@@ -626,6 +626,37 @@ test "it sets the info->follower_count property" do
     end
   end
 
+  describe "remove duplicates from following list" do
+    test "it removes duplicates" do
+      user = insert(:user)
+      follower = insert(:user)
+
+      {:ok, %User{following: following} = follower} = User.follow(follower, user)
+      assert length(following) == 2
+
+      {:ok, follower} =
+        follower
+        |> User.update_changeset(%{following: following ++ following})
+        |> Repo.update()
+
+      assert length(follower.following) == 4
+
+      {:ok, follower} = User.remove_duplicated_following(follower)
+      assert length(follower.following) == 2
+    end
+
+    test "it does nothing when following is uniq" do
+      user = insert(:user)
+      follower = insert(:user)
+
+      {:ok, follower} = User.follow(follower, user)
+      assert length(follower.following) == 2
+
+      {:ok, follower} = User.remove_duplicated_following(follower)
+      assert length(follower.following) == 2
+    end
+  end
+
   describe "follow_import" do
     test "it imports user followings from list" do
       [user1, user2, user3] = insert_list(3, :user)
@@ -1192,11 +1223,11 @@ test "follower count is updated when a follower is blocked" do
     follower2 = insert(:user)
     follower3 = insert(:user)
 
-    {:ok, follower} = Pleroma.User.follow(follower, user)
+    {:ok, follower} = User.follow(follower, user)
-    {:ok, _follower2} = Pleroma.User.follow(follower2, user)
+    {:ok, _follower2} = User.follow(follower2, user)
-    {:ok, _follower3} = Pleroma.User.follow(follower3, user)
+    {:ok, _follower3} = User.follow(follower3, user)
 
-    {:ok, _} = Pleroma.User.block(user, follower)
+    {:ok, _} = User.block(user, follower)
 
     user_show = Pleroma.Web.TwitterAPI.UserView.render("show.json", %{user: user})
 

test/web/activity_pub/mrf/simple_policy_test.exs

@@ -0,0 +1,220 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+  alias Pleroma.Config
+  alias Pleroma.Web.ActivityPub.MRF.SimplePolicy
+
+  setup do
+    orig = Config.get!(:mrf_simple)
+
+    Config.put(:mrf_simple,
+      media_removal: [],
+      media_nsfw: [],
+      federated_timeline_removal: [],
+      report_removal: [],
+      reject: [],
+      accept: []
+    )
+
+    on_exit(fn ->
+      Config.put(:mrf_simple, orig)
+    end)
+  end
+
+  describe "when :media_removal" do
+    test "is empty" do
+      Config.put([:mrf_simple, :media_removal], [])
+      media_message = build_media_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(media_message) == {:ok, media_message}
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :media_removal], ["remote.instance"])
+      media_message = build_media_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(media_message) ==
+               {:ok,
+                media_message
+                |> Map.put("object", Map.delete(media_message["object"], "attachment"))}
+
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+  end
+
+  describe "when :media_nsfw" do
+    test "is empty" do
+      Config.put([:mrf_simple, :media_nsfw], [])
+      media_message = build_media_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(media_message) == {:ok, media_message}
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :media_nsfw], ["remote.instance"])
+      media_message = build_media_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(media_message) ==
+               {:ok,
+                media_message
+                |> put_in(["object", "tag"], ["foo", "nsfw"])
+                |> put_in(["object", "sensitive"], true)}
+
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+  end
+
+  defp build_media_message do
+    %{
+      "actor" => "https://remote.instance/users/bob",
+      "type" => "Create",
+      "object" => %{
+        "attachment" => [%{}],
+        "tag" => ["foo"],
+        "sensitive" => false
+      }
+    }
+  end
+
+  describe "when :report_removal" do
+    test "is empty" do
+      Config.put([:mrf_simple, :report_removal], [])
+      report_message = build_report_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(report_message) == {:ok, report_message}
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :report_removal], ["remote.instance"])
+      report_message = build_report_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(report_message) == {:reject, nil}
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+  end
+
+  defp build_report_message do
+    %{
+      "actor" => "https://remote.instance/users/bob",
+      "type" => "Flag"
+    }
+  end
+
+  describe "when :federated_timeline_removal" do
+    test "is empty" do
+      Config.put([:mrf_simple, :federated_timeline_removal], [])
+      {_, ftl_message} = build_ftl_actor_and_message()
+      local_message = build_local_message()
+
+      assert SimplePolicy.filter(ftl_message) == {:ok, ftl_message}
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+
+    test "has a matching host" do
+      {actor, ftl_message} = build_ftl_actor_and_message()
+
+      ftl_message_actor_host =
+        ftl_message
+        |> Map.fetch!("actor")
+        |> URI.parse()
+        |> Map.fetch!(:host)
+
+      Config.put([:mrf_simple, :federated_timeline_removal], [ftl_message_actor_host])
+      local_message = build_local_message()
+
+      assert {:ok, ftl_message} = SimplePolicy.filter(ftl_message)
+      assert actor.follower_address in ftl_message["to"]
+      refute actor.follower_address in ftl_message["cc"]
+      refute "https://www.w3.org/ns/activitystreams#Public" in ftl_message["to"]
+      assert "https://www.w3.org/ns/activitystreams#Public" in ftl_message["cc"]
+
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+    end
+  end
+
+  defp build_ftl_actor_and_message do
+    actor = insert(:user)
+
+    {actor,
+     %{
+       "actor" => actor.ap_id,
+       "to" => ["https://www.w3.org/ns/activitystreams#Public", "http://foo.bar/baz"],
+       "cc" => [actor.follower_address, "http://foo.bar/qux"]
+     }}
+  end
+
+  describe "when :reject" do
+    test "is empty" do
+      Config.put([:mrf_simple, :reject], [])
+
+      remote_message = build_remote_message()
+
+      assert SimplePolicy.filter(remote_message) == {:ok, remote_message}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :reject], ["remote.instance"])
+
+      remote_message = build_remote_message()
+
+      assert SimplePolicy.filter(remote_message) == {:reject, nil}
+    end
+  end
+
+  describe "when :accept" do
+    test "is empty" do
+      Config.put([:mrf_simple, :accept], [])
+
+      local_message = build_local_message()
+      remote_message = build_remote_message()
+
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+      assert SimplePolicy.filter(remote_message) == {:ok, remote_message}
+    end
+
+    test "is not empty but it doesn't have a matching host" do
+      Config.put([:mrf_simple, :accept], ["non.matching.remote"])
+
+      local_message = build_local_message()
+      remote_message = build_remote_message()
+
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+      assert SimplePolicy.filter(remote_message) == {:reject, nil}
+    end
+
+    test "has a matching host" do
+      Config.put([:mrf_simple, :accept], ["remote.instance"])
+
+      local_message = build_local_message()
+      remote_message = build_remote_message()
+
+      assert SimplePolicy.filter(local_message) == {:ok, local_message}
+      assert SimplePolicy.filter(remote_message) == {:ok, remote_message}
+    end
+  end
+
+  defp build_local_message do
+    %{
+      "actor" => "#{Pleroma.Web.base_url()}/users/alice",
+      "to" => [],
+      "cc" => []
+    }
+  end
+
+  defp build_remote_message do
+    %{"actor" => "https://remote.instance/users/bob"}
+  end
+end

test/web/common_api/common_api_test.exs

@@ -310,14 +310,14 @@ test "does not update report state when state is unsupported" do
     test "add a reblog mute", %{muter: muter, muted: muted} do
       {:ok, muter} = CommonAPI.hide_reblogs(muter, muted)
 
-      assert Pleroma.User.showing_reblogs?(muter, muted) == false
+      assert User.showing_reblogs?(muter, muted) == false
     end
 
     test "remove a reblog mute", %{muter: muter, muted: muted} do
       {:ok, muter} = CommonAPI.hide_reblogs(muter, muted)
       {:ok, muter} = CommonAPI.show_reblogs(muter, muted)
 
-      assert Pleroma.User.showing_reblogs?(muter, muted) == true
+      assert User.showing_reblogs?(muter, muted) == true
     end
   end
 end

test/web/mastodon_api/mastodon_api_controller_test.exs

@@ -446,7 +446,7 @@ test "verify_credentials", %{conn: conn} do
   end
 
   test "verify_credentials default scope unlisted", %{conn: conn} do
-    user = insert(:user, %{info: %Pleroma.User.Info{default_scope: "unlisted"}})
+    user = insert(:user, %{info: %User.Info{default_scope: "unlisted"}})
 
     conn =
       conn
@@ -1322,7 +1322,7 @@ test "returns the relationships for the current user", %{conn: conn} do
 
   describe "locked accounts" do
     test "/api/v1/follow_requests works" do
-      user = insert(:user, %{info: %Pleroma.User.Info{locked: true}})
+      user = insert(:user, %{info: %User.Info{locked: true}})
       other_user = insert(:user)
 
       {:ok, _activity} = ActivityPub.follow(other_user, user)
@@ -1367,7 +1367,7 @@ test "/api/v1/follow_requests/:id/authorize works" do
     end
 
     test "verify_credentials", %{conn: conn} do
-      user = insert(:user, %{info: %Pleroma.User.Info{default_scope: "private"}})
+      user = insert(:user, %{info: %User.Info{default_scope: "private"}})
 
       conn =
         conn
@@ -1379,7 +1379,7 @@ test "verify_credentials", %{conn: conn} do
     end
 
     test "/api/v1/follow_requests/:id/reject works" do
-      user = insert(:user, %{info: %Pleroma.User.Info{locked: true}})
+      user = insert(:user, %{info: %User.Info{locked: true}})
       other_user = insert(:user)
 
       {:ok, _activity} = ActivityPub.follow(other_user, user)
@@ -1455,6 +1455,72 @@ test "media upload", %{conn: conn} do
     assert object.data["actor"] == User.ap_id(user)
   end
 
+  test "mascot upload", %{conn: conn} do
+    user = insert(:user)
+
+    non_image_file = %Plug.Upload{
+      content_type: "audio/mpeg",
+      path: Path.absname("test/fixtures/sound.mp3"),
+      filename: "sound.mp3"
+    }
+
+    conn =
+      conn
+      |> assign(:user, user)
+      |> put("/api/v1/pleroma/mascot", %{"file" => non_image_file})
+
+    assert json_response(conn, 415)
+
+    file = %Plug.Upload{
+      content_type: "image/jpg",
+      path: Path.absname("test/fixtures/image.jpg"),
+      filename: "an_image.jpg"
+    }
+
+    conn =
+      build_conn()
+      |> assign(:user, user)
+      |> put("/api/v1/pleroma/mascot", %{"file" => file})
+
+    assert %{"id" => _, "type" => image} = json_response(conn, 200)
+  end
+
+  test "mascot retrieving", %{conn: conn} do
+    user = insert(:user)
+    # When user hasn't set a mascot, we should just get pleroma tan back
+    conn =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/pleroma/mascot")
+
+    assert %{"url" => url} = json_response(conn, 200)
+    assert url =~ "pleroma-fox-tan-smol"
+
+    # When a user sets their mascot, we should get that back
+    file = %Plug.Upload{
+      content_type: "image/jpg",
+      path: Path.absname("test/fixtures/image.jpg"),
+      filename: "an_image.jpg"
+    }
+
+    conn =
+      build_conn()
+      |> assign(:user, user)
+      |> put("/api/v1/pleroma/mascot", %{"file" => file})
+
+    assert json_response(conn, 200)
+
+    user = User.get_cached_by_id(user.id)
+
+    conn =
+      build_conn()
+      |> assign(:user, user)
+      |> get("/api/v1/pleroma/mascot")
+
+    assert %{"url" => url, "type" => "image"} = json_response(conn, 200)
+    assert url =~ "an_image"
+  end
+
   test "hashtag timeline", %{conn: conn} do
     following = insert(:user)
 

test/web/ostatus/ostatus_test.exs

@@ -355,7 +355,7 @@ test "tries to use the information in poco fields" do
 
       {:ok, user} = OStatus.find_or_make_user(uri)
 
-      user = Pleroma.User.get_cached_by_id(user.id)
+      user = User.get_cached_by_id(user.id)
       assert user.name == "Constance Variable"
       assert user.nickname == "lambadalambda@social.heldscal.la"
       assert user.local == false
@@ -374,7 +374,7 @@ test "find_or_make_user sets all the nessary input fields" do
       {:ok, user} = OStatus.find_or_make_user(uri)
 
       assert user.info ==
-               %Pleroma.User.Info{
+               %User.Info{
                  id: user.info.id,
                  ap_enabled: false,
                  background: %{},
@@ -407,7 +407,7 @@ test "find_make_or_update_user takes an author element and returns an updated us
       {:ok, user} = OStatus.find_or_make_user(uri)
       old_name = user.name
       old_bio = user.bio
-      change = Ecto.Changeset.change(user, %{avatar: nil, bio: nil, old_name: nil})
+      change = Ecto.Changeset.change(user, %{avatar: nil, bio: nil, name: nil})
 
       {:ok, user} = Repo.update(change)
       refute user.avatar

test/web/rich_media/helpers_test.exs

@@ -1,6 +1,7 @@
 defmodule Pleroma.Web.RichMedia.HelpersTest do
   use Pleroma.DataCase
 
+  alias Pleroma.Object
   alias Pleroma.Web.CommonAPI
 
   import Pleroma.Factory
@@ -59,4 +60,43 @@ test "crawls valid, complete URLs" do
 
     Pleroma.Config.put([:rich_media, :enabled], false)
   end
+
+  test "refuses to crawl URLs from posts marked sensitive" do
+    user = insert(:user)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        "status" => "http://example.com/ogp",
+        "sensitive" => true
+      })
+
+    %Object{} = object = Object.normalize(activity)
+
+    assert object.data["sensitive"]
+
+    Pleroma.Config.put([:rich_media, :enabled], true)
+
+    assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+
+    Pleroma.Config.put([:rich_media, :enabled], false)
+  end
+
+  test "refuses to crawl URLs from posts tagged NSFW" do
+    user = insert(:user)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        "status" => "http://example.com/ogp #nsfw"
+      })
+
+    %Object{} = object = Object.normalize(activity)
+
+    assert object.data["sensitive"]
+
+    Pleroma.Config.put([:rich_media, :enabled], true)
+
+    assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+
+    Pleroma.Config.put([:rich_media, :enabled], false)
+  end
 end