Merge remote-tracking branch 'remotes/origin/develop' into auth-improvements

.gitattributes

@@ -1,2 +1,8 @@
 *.ex diff=elixir
 *.exs diff=elixir
+# At the time of writing all js/css files included
+# in the repo are minified bundles, and we don't want
+# to search/diff those as text files.
+*.js binary
+*.js.map binary
+*.css binary

.gitlab-ci.yml

@@ -229,7 +229,7 @@ arm:
   only: *release-only
   tags:
     - arm32
-  image: elixir:1.10.3
+  image: arm32v7/elixir:1.10.3
   cache: *release-cache
   variables: *release-variables
   before_script: *before-release
@@ -241,7 +241,7 @@ arm-musl:
   only: *release-only
   tags:
     - arm32
-  image: elixir:1.10.3-alpine
+  image: arm32v7/elixir:1.10.3-alpine
   cache: *release-cache
   variables: *release-variables
   before_script: *before-release-musl
@@ -253,7 +253,7 @@ arm64:
   only: *release-only
   tags:
     - arm
-  image: elixir:1.10.3
+  image: arm64v8/elixir:1.10.3
   cache: *release-cache
   variables: *release-variables
   before_script: *before-release
@@ -265,8 +265,7 @@ arm64-musl:
   only: *release-only
   tags:
     - arm
-  # TODO: Replace with upstream image when 1.9.0 comes out
+  image: arm64v8/elixir:1.10.3-alpine
-  image: elixir:1.10.3-alpine
   cache: *release-cache
   variables: *release-variables
   before_script: *before-release-musl

CHANGELOG.md

@@ -1,47 +1,90 @@
 # Changelog
+
 All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
+### Changed
+
+- Polls now always return a `voters_count`, even if they are single-choice.
+- Admin Emails: The ap id is used as the user link in emails now.
+
 ### Added
-- Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
+
-- Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
+- Reports now generate notifications for admins and mods.
-- Mix task option for force-unfollowing relays
-- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
-- Pleroma API: Importing the mutes users from CSV files.
 - Experimental websocket-based federation between Pleroma instances.
-- Support pagination of blocks and mutes
+- Support for local-only statuses
-- App metrics: ability to restrict access to specified IP whitelist.
+- Support pagination of blocks and mutes.
-- Account backup
+- Account backup.
 - Configuration: Add `:instance, autofollowing_nicknames` setting to provide a way to make accounts automatically follow new users that register on the local Pleroma instance.
+- Ability to view remote timelines, with ex. `/api/v1/timelines/public?instance=lain.com` and streams `public:remote` and `public:remote:media`.
+- The site title is now injected as a `title` tag like preloads or metadata.
+- Password reset tokens now are not accepted after a certain age.
+
+<details>
+  <summary>API Changes</summary>
+- Admin API: (`GET /api/pleroma/admin/users`) filter users by `unconfirmed` status and `actor_type`.
+- Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending.
+- Pleroma API: (`GET /api/v1/pleroma/federation_status`) Add a way to get a list of unreachable instances.
+- Mastodon API: User and conversation mutes can now auto-expire if `expires_in` parameter was given while adding the mute.
+- Admin API: An endpoint to manage frontends
+
+</details>
+
+### Fixed
+
+- Users with `is_discoverable` field set to false (default value) will appear in in-service search results but be hidden from external services (search bots etc.).
+
+<details>
+  <summary>API Changes</summary>
+  - Mastodon API: Current user is now included in conversation if it's the only participant.
+  - Mastodon API: Fixed last_status.account being not filled with account data.
+</details>
+
+## Unreleased (Patch)
+
+### Changed
+
+### Fixed
+
+- Config generation: rename `Pleroma.Upload.Filter.ExifTool` to `Pleroma.Upload.Filter.Exiftool`.
+- Search: RUM index search speed has been fixed.
+- S3 Uploads with Elixir 1.11.
+- Emoji Reaction activity filtering from blocked and muted accounts.
+- Mix task pleroma.user delete_activities for source installations.
+- Fix ability to update Pleroma Chat push notifications with PUT /api/v1/push/subscription and alert type pleroma:chat_mention
+- Forwarded reports duplication from Pleroma instances.
+
+<details>
+  <summary>API</summary>
+- Statuses were not displayed for Mastodon forwarded reports.
+
+</details>
+
+## [2.2.0] - 2020-11-12
+
+### Security
+
+- Fixed the possibility of using file uploads to spoof posts.
 
 ### Changed
 
 - **Breaking** Requires `libmagic` (or `file`) to guess file types.
+- **Breaking:** App metrics endpoint (`/api/pleroma/app_metrics`) is disabled by default, check `docs/API/prometheus.md` on enabling and configuring.
 - **Breaking:** Pleroma Admin API: emoji packs and files routes changed.
 - **Breaking:** Sensitive/NSFW statuses no longer disable link previews.
-- **Breaking:** App metrics endpoint (`/api/pleroma/app_metrics`) is disabled by default, check `docs/API/prometheus.md` on enabling and configuring. 
 - Search: Users are now findable by their urls.
 - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
 - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.
 - The `discoverable` field in the `User` struct will now add a NOINDEX metatag to profile pages when false.
-- Users with the `discoverable` field set to false will not show up in searches.
+- Users with the `is_discoverable` field set to false will not show up in searches ([bug](https://git.pleroma.social/pleroma/pleroma/-/issues/2301)).
 - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
 - Introduced optional dependencies on `ffmpeg`, `ImageMagick`, `exiftool` software packages. Please refer to `docs/installation/optional/media_graphics_packages.md`.
-- Polls now always return a `voters_count`, even if they are single-choice
+- <details>
-
-<details>
   <summary>API Changes</summary>
-
+- API: Empty parameter values for integer parameters are now ignored in non-strict validaton mode.
-- Pleroma API: Importing the mutes users from CSV files.
-- Admin API: Importing emoji from a zip file
-- Pleroma API: Pagination for remote/local packs and emoji.
-- Admin API: (`GET /api/pleroma/admin/users`) added filters user by `unconfirmed` status
-- Admin API: (`GET /api/pleroma/admin/users`) added filters user by `actor_type`
-- Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending.
-
 </details>
 
 ### Removed
@@ -52,20 +95,38 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Removed `:managed_config` option. In practice, it was accidentally removed with 2.0.0 release when frontends were
 switched to a new configuration mechanism, however it was not officially removed until now.
 
+### Added
+
+- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
+- Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
+- Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
+- Mix task option for force-unfollowing relays
+- App metrics: ability to restrict access to specified IP whitelist.
+
+<details>
+  <summary>API Changes</summary>
+
+- Admin API: Importing emoji from a zip file
+- Pleroma API: Importing the mutes users from CSV files.
+- Pleroma API: Pagination for remote/local packs and emoji.
+
+</details>
+
 ### Fixed
 
 - Add documented-but-missing chat pagination.
 - Allow sending out emails again.
-- Allow sending chat messages to yourself.
+- Allow sending chat messages to yourself
-- Fix remote users with a whitespace name.
 - OStatus / static FE endpoints: fixed inaccessibility for anonymous users on non-federating instances, switched to handling per `:restrict_unauthenticated` setting.
-- Mastodon API: Current user is now included in conversation if it's the only participant
+- Fix remote users with a whitespace name.
-- Mastodon API: Fixed last_status.account being not filled with account data
 
-## Unreleased (Patch)
+### Upgrade notes
 
-### Changed
+1. Install libmagic and development headers (`libmagic-dev` on Ubuntu/Debian, `file-dev` on Alpine Linux)
-- API: Empty parameter values for integer parameters are now ignored in non-strict validaton mode.
+2. Run database migrations (inside Pleroma directory):
+  - OTP: `./bin/pleroma_ctl migrate`
+  - From Source: `mix ecto.migrate`
+3. Restart Pleroma
 
 ## [2.1.2] - 2020-09-17
 

Dockerfile

@@ -4,7 +4,7 @@ COPY . .
 
 ENV MIX_ENV=prod
 
-RUN apk add git gcc g++ musl-dev make cmake &&\
+RUN apk add git gcc g++ musl-dev make cmake file-dev &&\
 	echo "import Mix.Config" > config/prod.secret.exs &&\
 	mix local.hex --force &&\
 	mix local.rebar --force &&\
@@ -31,7 +31,7 @@ LABEL maintainer="ops@pleroma.social" \
 ARG HOME=/opt/pleroma
 ARG DATA=/var/lib/pleroma
 
-RUN echo "https://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\
+RUN echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\
 	apk update &&\
 	apk add exiftool imagemagick ncurses postgresql-client &&\
 	adduser --system --shell /bin/false --home ${HOME} pleroma &&\

SECURITY.md

@@ -6,7 +6,7 @@ Currently, Pleroma offers bugfixes and security patches only for the latest mino
 
 | Version | Support 
 |---------| --------
-| 2.1     | Bugfixes and security patches
+| 2.2     | Bugfixes and security patches
 
 ## Reporting a vulnerability
 

config/config.exs

@@ -129,7 +129,6 @@
     dispatch: [
       {:_,
        [
-         {"/api/fedsocket/v1", Pleroma.Web.FedSockets.IncomingHandler, []},
          {"/api/v1/streaming", Pleroma.Web.MastodonAPI.WebsocketHandler, []},
          {"/websocket", Phoenix.Endpoint.CowboyWebSocket,
           {Phoenix.Transports.WebSocket,
@@ -264,7 +263,8 @@
       length: 16
     ]
   ],
-  show_reactions: true
+  show_reactions: true,
+  password_reset_token_validity: 60 * 60 * 24
 
 config :pleroma, :welcome,
   direct_message: [
@@ -562,7 +562,8 @@
     background: 5,
     remote_fetcher: 2,
     attachments_cleanup: 5,
-    new_users_digest: 1
+    new_users_digest: 1,
+    mute_expire: 5
   ],
   plugins: [Oban.Plugins.Pruner],
   crontab: [
@@ -820,7 +821,7 @@
 config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: false
 
 config :pleroma, :mrf,
-  policies: Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy,
+  policies: [Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy, Pleroma.Web.ActivityPub.MRF.TagPolicy],
   transparency: true,
   transparency_exclusions: []
 

config/description.exs

@@ -1,5 +1,4 @@
 use Mix.Config
-alias Pleroma.Docs.Generator
 
 websocket_config = [
   path: "/websocket",
@@ -273,19 +272,6 @@
       }
     ]
   },
-  %{
-    group: :pleroma,
-    key: :fed_sockets,
-    type: :group,
-    description: "Websocket based federation",
-    children: [
-      %{
-        key: :enabled,
-        type: :boolean,
-        description: "Enable FedSockets"
-      }
-    ]
-  },
   %{
     group: :pleroma,
     key: Pleroma.Emails.Mailer,
@@ -1555,298 +1541,6 @@
       }
     ]
   },
-  %{
-    group: :pleroma,
-    key: :mrf,
-    tab: :mrf,
-    label: "MRF",
-    type: :group,
-    description: "General MRF settings",
-    children: [
-      %{
-        key: :policies,
-        type: [:module, {:list, :module}],
-        description:
-          "A list of MRF policies enabled. Module names are shortened (removed leading `Pleroma.Web.ActivityPub.MRF.` part), but on adding custom module you need to use full name.",
-        suggestions: {:list_behaviour_implementations, Pleroma.Web.ActivityPub.MRF}
-      },
-      %{
-        key: :transparency,
-        label: "MRF transparency",
-        type: :boolean,
-        description:
-          "Make the content of your Message Rewrite Facility settings public (via nodeinfo)"
-      },
-      %{
-        key: :transparency_exclusions,
-        label: "MRF transparency exclusions",
-        type: {:list, :string},
-        description:
-          "Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.",
-        suggestions: [
-          "exclusion.com"
-        ]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_simple,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.SimplePolicy",
-    label: "MRF Simple",
-    type: :group,
-    description: "Simple ingress policies",
-    children: [
-      %{
-        key: :media_removal,
-        type: {:list, :string},
-        description: "List of instances to strip media attachments from",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :media_nsfw,
-        label: "Media NSFW",
-        type: {:list, :string},
-        description: "List of instances to tag all media as NSFW (sensitive) from",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :federated_timeline_removal,
-        type: {:list, :string},
-        description:
-          "List of instances to remove from the Federated (aka The Whole Known Network) Timeline",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :reject,
-        type: {:list, :string},
-        description: "List of instances to reject activities from (except deletes)",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :accept,
-        type: {:list, :string},
-        description: "List of instances to only accept activities from (except deletes)",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :followers_only,
-        type: {:list, :string},
-        description: "Force posts from the given instances to be visible by followers only",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :report_removal,
-        type: {:list, :string},
-        description: "List of instances to reject reports from",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :avatar_removal,
-        type: {:list, :string},
-        description: "List of instances to strip avatars from",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :banner_removal,
-        type: {:list, :string},
-        description: "List of instances to strip banners from",
-        suggestions: ["example.com", "*.example.com"]
-      },
-      %{
-        key: :reject_deletes,
-        type: {:list, :string},
-        description: "List of instances to reject deletions from",
-        suggestions: ["example.com", "*.example.com"]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_activity_expiration,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy",
-    label: "MRF Activity Expiration Policy",
-    type: :group,
-    description: "Adds automatic expiration to all local activities",
-    children: [
-      %{
-        key: :days,
-        type: :integer,
-        description: "Default global expiration time for all local activities (in days)",
-        suggestions: [90, 365]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_subchain,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.SubchainPolicy",
-    label: "MRF Subchain",
-    type: :group,
-    description:
-      "This policy processes messages through an alternate pipeline when a given message matches certain criteria." <>
-        " All criteria are configured as a map of regular expressions to lists of policy modules.",
-    children: [
-      %{
-        key: :match_actor,
-        type: {:map, {:list, :string}},
-        description: "Matches a series of regular expressions against the actor field",
-        suggestions: [
-          %{
-            ~r/https:\/\/example.com/s => [Pleroma.Web.ActivityPub.MRF.DropPolicy]
-          }
-        ]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_rejectnonpublic,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.RejectNonPublic",
-    description: "RejectNonPublic drops posts with non-public visibility settings.",
-    label: "MRF Reject Non Public",
-    type: :group,
-    children: [
-      %{
-        key: :allow_followersonly,
-        label: "Allow followers-only",
-        type: :boolean,
-        description: "Whether to allow followers-only posts"
-      },
-      %{
-        key: :allow_direct,
-        type: :boolean,
-        description: "Whether to allow direct messages"
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_hellthread,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.HellthreadPolicy",
-    label: "MRF Hellthread",
-    type: :group,
-    description: "Block messages with excessive user mentions",
-    children: [
-      %{
-        key: :delist_threshold,
-        type: :integer,
-        description:
-          "Number of mentioned users after which the message gets removed from timelines and" <>
-            "disables notifications. Set to 0 to disable.",
-        suggestions: [10]
-      },
-      %{
-        key: :reject_threshold,
-        type: :integer,
-        description:
-          "Number of mentioned users after which the messaged gets rejected. Set to 0 to disable.",
-        suggestions: [20]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_keyword,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.KeywordPolicy",
-    label: "MRF Keyword",
-    type: :group,
-    description:
-      "Reject or Word-Replace messages matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).",
-    children: [
-      %{
-        key: :reject,
-        type: {:list, :string},
-        description: """
-          A list of patterns which result in message being rejected.
-
-          Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
-        """,
-        suggestions: ["foo", ~r/foo/iu]
-      },
-      %{
-        key: :federated_timeline_removal,
-        type: {:list, :string},
-        description: """
-          A list of patterns which result in message being removed from federated timelines (a.k.a unlisted).
-
-          Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
-        """,
-        suggestions: ["foo", ~r/foo/iu]
-      },
-      %{
-        key: :replace,
-        type: {:list, :tuple},
-        description: """
-          **Pattern**: a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
-
-          **Replacement**: a string. Leaving the field empty is permitted.
-        """
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_mention,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.MentionPolicy",
-    label: "MRF Mention",
-    type: :group,
-    description: "Block messages which mention a specific user",
-    children: [
-      %{
-        key: :actors,
-        type: {:list, :string},
-        description: "A list of actors for which any post mentioning them will be dropped",
-        suggestions: ["actor1", "actor2"]
-      }
-    ]
-  },
-  %{
-    group: :pleroma,
-    key: :mrf_vocabulary,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.VocabularyPolicy",
-    label: "MRF Vocabulary",
-    type: :group,
-    description: "Filter messages which belong to certain activity vocabularies",
-    children: [
-      %{
-        key: :accept,
-        type: {:list, :string},
-        description:
-          "A list of ActivityStreams terms to accept. If empty, all supported messages are accepted.",
-        suggestions: ["Create", "Follow", "Mention", "Announce", "Like"]
-      },
-      %{
-        key: :reject,
-        type: {:list, :string},
-        description:
-          "A list of ActivityStreams terms to reject. If empty, no messages are rejected.",
-        suggestions: ["Create", "Follow", "Mention", "Announce", "Like"]
-      }
-    ]
-  },
-  # %{
-  #   group: :pleroma,
-  #   key: :mrf_user_allowlist,
-  #   tab: :mrf,
-  #   related_policy: "Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy",
-  #   type: :map,
-  #   description:
-  #     "The keys in this section are the domain names that the policy should apply to." <>
-  #       " Each key should be assigned a list of users that should be allowed through by their ActivityPub ID",
-  #     suggestions: [
-  #       %{"example.org" => ["https://example.org/users/admin"]}
-  #     ]
-  #   ]
-  # },
   %{
     group: :pleroma,
     key: :media_proxy,
@@ -3159,22 +2853,6 @@
       }
     ]
   },
-  %{
-    group: :pleroma,
-    key: :mrf_normalize_markup,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.NormalizeMarkup",
-    label: "MRF Normalize Markup",
-    description: "MRF NormalizeMarkup settings. Scrub configured hypertext markup.",
-    type: :group,
-    children: [
-      %{
-        key: :scrub_policy,
-        type: :module,
-        suggestions: [Pleroma.HTML.Scrubber.Default]
-      }
-    ]
-  },
   %{
     group: :pleroma,
     key: Pleroma.User,
@@ -3364,33 +3042,6 @@
       }
     ]
   },
-  %{
-    group: :pleroma,
-    key: :mrf_object_age,
-    tab: :mrf,
-    related_policy: "Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy",
-    label: "MRF Object Age",
-    type: :group,
-    description:
-      "Rejects or delists posts based on their timestamp deviance from your server's clock.",
-    children: [
-      %{
-        key: :threshold,
-        type: :integer,
-        description: "Required age (in seconds) of a post before actions are taken.",
-        suggestions: [172_800]
-      },
-      %{
-        key: :actions,
-        type: {:list, :atom},
-        description:
-          "A list of actions to apply to the post. `:delist` removes the post from public timelines; " <>
-            "`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines; " <>
-            "`:reject` rejects the message entirely",
-        suggestions: [:delist, :strip_followers, :reject]
-      }
-    ]
-  },
   %{
     group: :pleroma,
     key: :modules,

docs/API/admin_api.md

@@ -554,7 +554,7 @@ Response:
   * `show_role`
   * `skip_thread_containment`
   * `fields`
-  * `discoverable`
+  * `is_discoverable`
   * `actor_type`
 
 * Responses:
@@ -1499,3 +1499,66 @@ Returns the content of the document
   "url": "https://example.com/instance/panel.html"
 }
 ```
+
+## `GET /api/pleroma/admin/frontends
+
+### List available frontends
+
+- Response:
+
+```json
+[
+   {
+    "build_url": "https://git.pleroma.social/pleroma/fedi-fe/-/jobs/artifacts/${ref}/download?job=build",
+    "git": "https://git.pleroma.social/pleroma/fedi-fe",
+    "installed": true,
+    "name": "fedi-fe",
+    "ref": "master"
+  },
+  {
+    "build_url": "https://git.pleroma.social/lambadalambda/kenoma/-/jobs/artifacts/${ref}/download?job=build",
+    "git": "https://git.pleroma.social/lambadalambda/kenoma",
+    "installed": false,
+    "name": "kenoma",
+    "ref": "master"
+  }
+]
+```
+
+## `POST /api/pleroma/admin/frontends/install`
+
+### Install a frontend
+
+- Params:
+  - `name`: frontend name, required
+  - `ref`: frontend ref
+  - `file`: path to a frontend zip file
+  - `build_url`: build URL
+  - `build_dir`: build directory
+
+- Response:
+
+```json
+[
+   {
+    "build_url": "https://git.pleroma.social/pleroma/fedi-fe/-/jobs/artifacts/${ref}/download?job=build",
+    "git": "https://git.pleroma.social/pleroma/fedi-fe",
+    "installed": true,
+    "name": "fedi-fe",
+    "ref": "master"
+  },
+  {
+    "build_url": "https://git.pleroma.social/lambadalambda/kenoma/-/jobs/artifacts/${ref}/download?job=build",
+    "git": "https://git.pleroma.social/lambadalambda/kenoma",
+    "installed": false,
+    "name": "kenoma",
+    "ref": "master"
+  }
+]
+```
+
+```json
+{
+  "error": "Could not install frontend"
+}
+```

docs/API/differences_in_mastoapi_responses.md

@@ -9,12 +9,16 @@ Pleroma uses 128-bit ids as opposed to Mastodon's 64 bits. However just like Mas
 ## Timelines
 
 Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users.
+
 Adding the parameter `exclude_visibilities` to the timeline queries will exclude the statuses with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`), e.g., `exclude_visibilities[]=direct&exclude_visibilities[]=private`.
+
 Adding the parameter `reply_visibility` to the public and home timelines queries will filter replies. Possible values: without parameter (default) shows all replies, `following` - replies directed to you or users you follow, `self` - replies directed to you.
 
+Adding the parameter `instance=lain.com` to the public timeline will show only statuses originating from `lain.com` (or any remote instance).
+
 ## Statuses
 
-- `visibility`: has an additional possible value `list`
+- `visibility`: has additional possible values `list` and `local` (for local-only statuses)
 
 Has these additional fields under the `pleroma` object:
 
@@ -80,7 +84,7 @@ Has these additional fields under the `pleroma` object:
 
 - `show_role`: boolean, nullable, true when the user wants his role (e.g admin, moderator) to be shown
 - `no_rich_text` - boolean, nullable, true when html tags are stripped from all statuses requested from the API
-- `discoverable`: boolean, true when the user allows discovery of the account in search results and other services.
+- `discoverable`: boolean, true when the user allows external services (search bots) etc. to index / list the account (regardless of this setting, user will still appear in regular search results)
 - `actor_type`: string, the type of this account.
 
 ## Conversations
@@ -125,12 +129,30 @@ The `type` value is `pleroma:emoji_reaction`. Has these fields:
 - `account`: The account of the user who reacted
 - `status`: The status that was reacted on
 
+### ChatMention Notification (not default)
+
+This notification has to be requested explicitly.
+
+The `type` value is `pleroma:chat_mention`
+
+- `account`: The account who sent the message
+- `chat_message`: The chat message
+
+### Report Notification (not default)
+
+This notification has to be requested explicitly.
+
+The `type` value is `pleroma:report`
+
+- `account`: The account who reported
+- `report`: The report
+
 ## GET `/api/v1/notifications`
 
 Accepts additional parameters:
 
 - `exclude_visibilities`: will exclude the notifications for activities with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`). Usage example: `GET /api/v1/notifications?exclude_visibilities[]=direct&exclude_visibilities[]=private`.
-- `include_types`: will include the notifications for activities with the given types. The parameter accepts an array of types (`mention`, `follow`, `reblog`, `favourite`, `move`, `pleroma:emoji_reaction`). Usage example: `GET /api/v1/notifications?include_types[]=mention&include_types[]=reblog`.
+- `include_types`: will include the notifications for activities with the given types. The parameter accepts an array of types (`mention`, `follow`, `reblog`, `favourite`, `move`, `pleroma:emoji_reaction`, `pleroma:chat_mention`, `pleroma:report`). Usage example: `GET /api/v1/notifications?include_types[]=mention&include_types[]=reblog`.
 
 ## DELETE `/api/v1/notifications/destroy_multiple`
 
@@ -151,7 +173,7 @@ Additional parameters can be added to the JSON body/Form data:
 - `preview`: boolean, if set to `true` the post won't be actually posted, but the status entitiy would still be rendered back. This could be useful for previewing rich text/custom emoji, for example.
 - `content_type`: string, contain the MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint.
 - `to`: A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for for post visibility are not affected by this and will still apply.
-- `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted` or `public`) it can be used to address a List by setting it to `list:LIST_ID`.
+- `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted`, `local` or `public`) it can be used to address a List by setting it to `list:LIST_ID`.
 - `expires_in`: The number of seconds the posted activity should expire in. When a posted activity expires it will be deleted from the server, and a delete request for it will be federated. This needs to be longer than an hour.
 - `in_reply_to_conversation_id`: Will reply to a given conversation, addressing only the people who are part of the recipient set of that conversation. Sets the visibility to `direct`.
 
@@ -185,7 +207,7 @@ Additional parameters can be added to the JSON body/Form data:
 - `skip_thread_containment` - if true, skip filtering out broken threads
 - `allow_following_move` - if true, allows automatically follow moved following accounts
 - `pleroma_background_image` - sets the background image of the user. Can be set to "" (an empty string) to reset.
-- `discoverable` - if true, discovery of this account in search results and other services is allowed.
+- `discoverable` - if true, external services (search bots) etc. are allowed to index / list the account (regardless of this setting, user will still appear in regular search results).
 - `actor_type` - the type of this account.
 - `accepts_chat_messages` - if false, this account will reject all chat messages.
 
@@ -211,7 +233,7 @@ Post here request with `grant_type=refresh_token` to obtain new access token. Re
 
 `POST /api/v1/accounts`
 
-Has theses additional parameters (which are the same as in Pleroma-API):
+Has these additional parameters (which are the same as in Pleroma-API):
 
 - `fullname`: optional
 - `bio`: optional
@@ -239,6 +261,16 @@ Has theses additional parameters (which are the same as in Pleroma-API):
 - `pleroma.metadata.post_formats`: A list of the allowed post format types
 - `vapid_public_key`: The public key needed for push messages
 
+## Push Subscription
+
+`POST /api/v1/push/subscription`
+`PUT /api/v1/push/subscription`
+
+Permits these additional alert types:
+
+- pleroma:chat_mention
+- pleroma:emoji_reaction
+
 ## Markers
 
 Has these additional fields under the `pleroma` object:
@@ -249,6 +281,12 @@ Has these additional fields under the `pleroma` object:
 
 There is an additional `user:pleroma_chat` stream. Incoming chat messages will make the current chat be sent to this `user` stream. The `event` of an incoming chat message is `pleroma:chat_update`. The payload is the updated chat with the incoming chat message in the `last_message` field.
 
+For viewing remote server timelines, there are `public:remote` and `public:remote:media` streams. Each of these accept a parameter like `?instance=lain.com`.
+
+## User muting and thread muting
+
+Both user muting and thread muting can be done for only a certain time by adding an `expires_in` parameter to the API calls and giving the expiration time in seconds.
+
 ## Not implemented
 
 Pleroma is generally compatible with the Mastodon 2.7.2 API, but some newer features and non-essential features are omitted. These features usually return an HTTP 200 status code, but with an empty response. While they may be added in the future, they are considered low priority.

docs/administration/CLI_tasks/release_environments.md

@@ -1,9 +0,0 @@
-# Generate release environment file
-
-```sh tab="OTP"
- ./bin/pleroma_ctl release_env gen
-```
-
-```sh tab="From Source"
-mix pleroma.release_env gen
-```

docs/configuration/cheatsheet.md

@@ -63,6 +63,7 @@ To add configuration to your config file, you can copy it from the base config.
 * `external_user_synchronization`: Enabling following/followers counters synchronization for external users.
 * `cleanup_attachments`: Remove attachments along with statuses. Does not affect duplicate files and attachments without status. Enabling this will increase load to database when deleting statuses on larger instances.
 * `show_reactions`: Let favourites and emoji reactions be viewed through the API (default: `true`).
+* `password_reset_token_validity`: The time after which reset tokens aren't accepted anymore, in seconds (default: one day).
 
 ## Welcome
 * `direct_message`: - welcome message sent as a direct message.
@@ -220,18 +221,6 @@ config :pleroma, :mrf_user_allowlist, %{
 * `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`)
 * `enabled`: whether scheduled activities are sent to the job queue to be executed
 
-## FedSockets
-FedSockets is an experimental feature allowing for Pleroma backends to federate using a persistant websocket connection as opposed to making each federation a seperate http connection. This feature is currently off by default. It is configurable throught he following options.
-
-### :fedsockets
-* `enabled`: Enables FedSockets for this instance. `false` by default.
-* `connection_duration`: Time an idle websocket is kept open.
-* `rejection_duration`: Failures to connect via FedSockets will not be retried for this period of time.
-* `fed_socket_fetches` and `fed_socket_rejections`: Settings passed to `cachex` for the fetch registry, and rejection stacks. See `Pleroma.Web.FedSockets` for more details.
-
-
-## Frontends
-
 ### :frontend_configurations
 
 This can be used to configure a keyword list that keeps the configuration data for any kind of frontend. By default, settings for `pleroma_fe` and `masto_fe` are configured. You can find the documentation for `pleroma_fe` configuration into [Pleroma-FE configuration and customization for instance administrators](/frontend/CONFIGURATION/#options).

docs/configuration/howto_ejabberd.md

@@ -0,0 +1,136 @@
+# Configuring Ejabberd (XMPP Server) to use Pleroma for authentication
+
+If you want to give your Pleroma users an XMPP (chat) account, you can configure [Ejabberd](https://github.com/processone/ejabberd) to use your Pleroma server for user authentication, automatically giving every local user an XMPP account.
+
+In general, you just have to follow the configuration described at [https://docs.ejabberd.im/admin/configuration/authentication/#external-script](https://docs.ejabberd.im/admin/configuration/authentication/#external-script). Please read this section carefully. 
+
+Copy the script below to suitable path on your system and set owner and permissions. Also do not forget adjusting `PLEROMA_HOST` and `PLEROMA_PORT`, if necessary.
+
+```bash
+cp pleroma_ejabberd_auth.py /etc/ejabberd/pleroma_ejabberd_auth.py
+chown ejabberd /etc/ejabberd/pleroma_ejabberd_auth.py
+chmod 700 /etc/ejabberd/pleroma_ejabberd_auth.py
+```
+
+Set external auth params in ejabberd.yaml file:
+
+```bash
+auth_method: [external]
+extauth_program: "python3 /etc/ejabberd/pleroma_ejabberd_auth.py"
+extauth_instances: 3
+auth_use_cache: false
+```
+
+Restart / reload your ejabberd service.
+
+After restarting your Ejabberd server, your users should now be able to connect with their Pleroma credentials.
+
+
+```python
+import sys
+import struct
+import http.client
+from base64 import b64encode
+import logging
+
+
+PLEROMA_HOST = "127.0.0.1"
+PLEROMA_PORT = "4000"
+AUTH_ENDPOINT = "/api/v1/accounts/verify_credentials"
+USER_ENDPOINT = "/api/v1/accounts"
+LOGFILE = "/var/log/ejabberd/pleroma_auth.log"
+
+logging.basicConfig(filename=LOGFILE, level=logging.INFO)
+
+
+# Pleroma functions
+def create_connection():
+    return http.client.HTTPConnection(PLEROMA_HOST, PLEROMA_PORT)
+
+
+def verify_credentials(user: str, password: str) -> bool:
+    user_pass_b64 = b64encode("{}:{}".format(
+        user, password).encode('utf-8')).decode("ascii")
+    params = {}
+    headers = {
+        "Authorization": "Basic {}".format(user_pass_b64)
+    }
+
+    try:
+        conn = create_connection()
+        conn.request("GET", AUTH_ENDPOINT, params, headers)
+
+        response = conn.getresponse()
+        if response.status == 200:
+            return True
+
+        return False
+    except Exception as e:
+        logging.info("Can not connect: %s", str(e))
+        return False
+
+
+def does_user_exist(user: str) -> bool:
+    conn = create_connection()
+    conn.request("GET", "{}/{}".format(USER_ENDPOINT, user))
+
+    response = conn.getresponse()
+    if response.status == 200:
+        return True
+
+    return False
+
+
+def auth(username: str, server: str, password: str) -> bool:
+    return verify_credentials(username, password)
+
+
+def isuser(username, server):
+    return does_user_exist(username)
+
+
+def read():
+    (pkt_size,) = struct.unpack('>H', bytes(sys.stdin.read(2), encoding='utf8'))
+    pkt = sys.stdin.read(pkt_size)
+    cmd = pkt.split(':')[0]
+    if cmd == 'auth':
+        username, server, password = pkt.split(':', 3)[1:]
+        write(auth(username, server, password))
+    elif cmd == 'isuser':
+        username, server = pkt.split(':', 2)[1:]
+        write(isuser(username, server))
+    elif cmd == 'setpass':
+        # u, s, p = pkt.split(':', 3)[1:]
+        write(False)
+    elif cmd == 'tryregister':
+        # u, s, p = pkt.split(':', 3)[1:]
+        write(False)
+    elif cmd == 'removeuser':
+        # u, s = pkt.split(':', 2)[1:]
+        write(False)
+    elif cmd == 'removeuser3':
+        # u, s, p = pkt.split(':', 3)[1:]
+        write(False)
+    else:
+        write(False)
+
+
+def write(result):
+    if result:
+        sys.stdout.write('\x00\x02\x00\x01')
+    else:
+        sys.stdout.write('\x00\x02\x00\x00')
+    sys.stdout.flush()
+
+
+if __name__ == "__main__":
+    logging.info("Starting pleroma ejabberd auth daemon...")
+    while True:
+        try:
+            read()
+        except Exception as e:
+            logging.info(
+                "Error while processing data from ejabberd %s", str(e))
+            pass
+
+```

docs/configuration/optimizing_beam.md

@@ -0,0 +1,66 @@
+# Optimizing the BEAM
+
+Pleroma is built upon the Erlang/OTP VM known as BEAM. The BEAM VM is highly optimized for latency, but this has drawbacks in environments without dedicated hardware. One of the tricks used by the BEAM VM is [busy waiting](https://en.wikipedia.org/wiki/Busy_waiting). This allows the application to pretend to be busy working so the OS kernel does not pause the application process and switch to another process waiting for the CPU to execute its workload. It does this by spinning for a period of time which inflates the apparent CPU usage of the application so it is immediately ready to execute another task. This can be observed with utilities like **top(1)** which will show consistently high CPU usage for the process. Switching between procesess is a rather expensive operation and also clears CPU caches further affecting latency and performance. The goal of busy waiting is to avoid this penalty.
+
+This strategy is very successful in making a performant and responsive application, but is not desirable on Virtual Machines or hardware with few CPU cores. Pleroma instances are often deployed on the same server as the required PostgreSQL database which can lead to situations where the Pleroma application is holding the CPU in a busy-wait loop and as a result the database cannot process requests in a timely manner. The fewer CPUs available, the more this problem is exacerbated. The latency is further amplified by the OS being installed on a Virtual Machine as the Hypervisor uses CPU time-slicing to pause the entire OS and switch between other tasks.
+
+More adventurous admins can be creative with CPU affinity (e.g., *taskset* for Linux and *cpuset* on FreeBSD) to pin processes to specific CPUs and eliminate much of this contention. The most important advice is to run as few processes as possible on your server to achieve the best performance. Even idle background processes can occasionally create [software interrupts](https://en.wikipedia.org/wiki/Interrupt) and take attention away from the executing process creating latency spikes and invalidation of the CPU caches as they must be cleared when switching between processes for security.
+
+Please only change these settings if you are experiencing issues or really know what you are doing. In general, there's no need to change these settings.
+
+## VPS Provider Recommendations
+
+### Good
+
+* Hetzner Cloud
+
+### Bad
+
+* AWS (known to use burst scheduling)
+
+
+## Example configurations
+
+Tuning the BEAM requires you provide a config file normally called [vm.args](http://erlang.org/doc/man/erl.html#emulator-flags). If you are using systemd to manage the service you can modify the unit file as such:
+
+`ExecStart=/usr/bin/elixir --erl '-args_file /opt/pleroma/config/vm.args' -S /usr/bin/mix phx.server`
+
+Check your OS documentation to adopt a similar strategy on other platforms.
+
+### Virtual Machine and/or few CPU cores
+
+Disable the busy-waiting. This should generally only be done if you're on a platform that does burst scheduling, like AWS.
+
+**vm.args:**
+
+```
++sbwt none
++sbwtdcpu none
++sbwtdio none
+```
+
+### Dedicated Hardware
+
+Enable more busy waiting, increase the internal maximum limit of BEAM processes and ports. You can use this if you run on dedicated hardware, but it is not necessary.
+
+**vm.args:**
+
+```
++P 16777216
++Q 16777216
++K true
++A 128
++sbt db
++sbwt very_long
++swt very_low
++sub true
++Mulmbcs 32767
++Mumbcgs 1
++Musmbcs 2047
+```
+
+## Additional Reading
+
+* [WhatsApp: Scaling to Millions of Simultaneous Connections](https://www.erlang-factory.com/upload/presentations/558/efsf2012-whatsapp-scaling.pdf)
+* [Preemptive Scheduling and Spinlocks](https://www.uio.no/studier/emner/matnat/ifi/nedlagte-emner/INF3150/h03/annet/slides/preemptive.pdf)
+* [The Curious Case of BEAM CPU Usage](https://stressgrid.com/blog/beam_cpu_usage/)

docs/dev.md

@@ -21,3 +21,26 @@ This document contains notes and guidelines for Pleroma developers.
 ## Auth-related configuration, OAuth consumer mode etc.
 
 See `Authentication` section of [the configuration cheatsheet](configuration/cheatsheet.md#authentication).
+
+## MRF policies descriptions
+
+If MRF policy depends on config, it can be added into MRF tab to adminFE by adding `config_description/0` method, which returns map with special structure.
+
+Example:
+
+```elixir
+%{
+      key: :mrf_activity_expiration,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy",
+      label: "MRF Activity Expiration Policy",
+      description: "Adds automatic expiration to all local activities",
+      children: [
+        %{
+          key: :days,
+          type: :integer,
+          description: "Default global expiration time for all local activities (in days)",
+          suggestions: [90, 365]
+        }
+      ]
+    }
+```

docs/installation/debian_based_en.md

@@ -35,7 +35,7 @@ sudo apt full-upgrade
 * Install some of the above mentioned programs:
 
 ```shell
-sudo apt install git build-essential postgresql postgresql-contrib cmake libmagic-devel
+sudo apt install git build-essential postgresql postgresql-contrib cmake libmagic-dev
 ```
 
 ### Install Elixir and Erlang
@@ -182,7 +182,6 @@ sudo cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.se
 ```
 
 * Edit the service file and make sure that all paths fit your installation
-* Check that `EnvironmentFile` contains the correct path to the env file. Or generate the env file: `sudo -Hu pleroma mix pleroma.release_env gen`
 * Enable and start `pleroma.service`:
 
 ```shell

docs/installation/otp_en.md

@@ -149,9 +149,6 @@ chown -R pleroma /etc/pleroma
 # Run the config generator
 su pleroma -s $SHELL -lc "./bin/pleroma_ctl instance gen --output /etc/pleroma/config.exs --output-psql /tmp/setup_db.psql"
 
-# Run the environment file generator.
-su pleroma -s $SHELL -lc "./bin/pleroma_ctl release_env gen"
-
 # Create the postgres database
 su postgres -s $SHELL -lc "psql -f /tmp/setup_db.psql"
 

installation/init.d/pleroma

@@ -8,7 +8,6 @@ pidfile="/var/run/pleroma.pid"
 directory=/opt/pleroma
 healthcheck_delay=60
 healthcheck_timer=30
-export $(cat /opt/pleroma/config/pleroma.env)
 
 : ${pleroma_port:-4000}
 

installation/pleroma.nginx

@@ -93,9 +93,4 @@ server {
         chunked_transfer_encoding on;
         proxy_pass         http://phoenix;
     }
-
-    location /api/fedsocket/v1 {
-        proxy_request_buffering off;
-        proxy_pass http://phoenix/api/fedsocket/v1;
-    }
 }

installation/pleroma.service

@@ -17,8 +17,6 @@ Environment="MIX_ENV=prod"
 Environment="HOME=/var/lib/pleroma"
 ; Path to the folder containing the Pleroma installation.
 WorkingDirectory=/opt/pleroma
-; Path to the environment file. the file contains RELEASE_COOKIE and etc 
-EnvironmentFile=/opt/pleroma/config/pleroma.env
 ; Path to the Mix binary.
 ExecStart=/usr/bin/mix phx.server
 

lib/mix/pleroma.ex

@@ -14,11 +14,12 @@ defmodule Mix.Pleroma do
     :swoosh,
     :timex
   ]
-  @cachex_children ["object", "user", "scrubber"]
+  @cachex_children ["object", "user", "scrubber", "web_resp"]
   @doc "Common functions to be reused in mix tasks"
   def start_pleroma do
     Pleroma.Config.Holder.save_default()
     Pleroma.Config.Oban.warn()
+    Pleroma.Application.limiters_setup()
     Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
 
     if Pleroma.Config.get(:env) != :test do

lib/mix/tasks/pleroma/frontend.ex

@@ -17,8 +17,6 @@ def run(["install", "none" | _args]) do
   end
 
   def run(["install", frontend | args]) do
-    log_level = Logger.level()
-    Logger.configure(level: :warn)
     start_pleroma()
 
     {options, [], []} =
@@ -33,109 +31,6 @@ def run(["install", frontend | args]) do
         ]
       )
 
-    instance_static_dir =
+    Pleroma.Frontend.install(frontend, options)
-      with nil <- options[:static_dir] do
-        Pleroma.Config.get!([:instance, :static_dir])
-      end
-
-    cmd_frontend_info = %{
-      "name" => frontend,
-      "ref" => options[:ref],
-      "build_url" => options[:build_url],
-      "build_dir" => options[:build_dir]
-    }
-
-    config_frontend_info = Pleroma.Config.get([:frontends, :available, frontend], %{})
-
-    frontend_info =
-      Map.merge(config_frontend_info, cmd_frontend_info, fn _key, config, cmd ->
-        # This only overrides things that are actually set
-        cmd || config
-      end)
-
-    ref = frontend_info["ref"]
-
-    unless ref do
-      raise "No ref given or configured"
-    end
-
-    dest =
-      Path.join([
-        instance_static_dir,
-        "frontends",
-        frontend,
-        ref
-      ])
-
-    fe_label = "#{frontend} (#{ref})"
-
-    tmp_dir = Path.join([instance_static_dir, "frontends", "tmp"])
-
-    with {_, :ok} <-
-           {:download_or_unzip, download_or_unzip(frontend_info, tmp_dir, options[:file])},
-         shell_info("Installing #{fe_label} to #{dest}"),
-         :ok <- install_frontend(frontend_info, tmp_dir, dest) do
-      File.rm_rf!(tmp_dir)
-      shell_info("Frontend #{fe_label} installed to #{dest}")
-
-      Logger.configure(level: log_level)
-    else
-      {:download_or_unzip, _} ->
-        shell_info("Could not download or unzip the frontend")
-
-      _e ->
-        shell_info("Could not install the frontend")
-    end
-  end
-
-  defp download_or_unzip(frontend_info, temp_dir, file) do
-    if file do
-      with {:ok, zip} <- File.read(Path.expand(file)) do
-        unzip(zip, temp_dir)
-      end
-    else
-      download_build(frontend_info, temp_dir)
-    end
-  end
-
-  def unzip(zip, dest) do
-    with {:ok, unzipped} <- :zip.unzip(zip, [:memory]) do
-      File.rm_rf!(dest)
-      File.mkdir_p!(dest)
-
-      Enum.each(unzipped, fn {filename, data} ->
-        path = filename
-
-        new_file_path = Path.join(dest, path)
-
-        new_file_path
-        |> Path.dirname()
-        |> File.mkdir_p!()
-
-        File.write!(new_file_path, data)
-      end)
-
-      :ok
-    end
-  end
-
-  defp download_build(frontend_info, dest) do
-    shell_info("Downloading pre-built bundle for #{frontend_info["name"]}")
-    url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
-
-    with {:ok, %{status: 200, body: zip_body}} <-
-           Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do
-      unzip(zip_body, dest)
-    else
-      e -> {:error, e}
-    end
-  end
-
-  defp install_frontend(frontend_info, source, dest) do
-    from = frontend_info["build_dir"] || "dist"
-    File.rm_rf!(dest)
-    File.mkdir_p!(dest)
-    File.cp_r!(Path.join([source, from]), dest)
-    :ok
   end
 end

lib/mix/tasks/pleroma/instance.ex

@@ -36,9 +36,7 @@ def run(["gen" | rest]) do
           listen_port: :string,
           strip_uploads: :string,
           anonymize_uploads: :string,
-          dedupe_uploads: :string,
+          dedupe_uploads: :string
-          skip_release_env: :boolean,
-          release_env_file: :string
         ],
         aliases: [
           o: :output,
@@ -243,24 +241,6 @@ def run(["gen" | rest]) do
 
       write_robots_txt(static_dir, indexable, template_dir)
 
-      if Keyword.get(options, :skip_release_env, false) do
-        shell_info("""
-        Release environment file is skip. Please generate the release env file before start.
-        `MIX_ENV=#{Mix.env()} mix pleroma.release_env gen`
-        """)
-      else
-        shell_info("Generation the environment file:")
-
-        release_env_args =
-          with path when not is_nil(path) <- Keyword.get(options, :release_env_file) do
-            ["gen", "--path", path]
-          else
-            _ -> ["gen"]
-          end
-
-        Mix.Tasks.Pleroma.ReleaseEnv.run(release_env_args)
-      end
-
       shell_info(
         "\n All files successfully written! Refer to the installation instructions for your platform for next steps."
       )
@@ -304,7 +284,7 @@ defp write_robots_txt(static_dir, indexable, template_dir) do
   defp upload_filters(filters) when is_map(filters) do
     enabled_filters =
       if filters.strip do
-        [Pleroma.Upload.Filter.ExifTool]
+        [Pleroma.Upload.Filter.Exiftool]
       else
         []
       end

lib/mix/tasks/pleroma/release_env.ex

@@ -1,76 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Mix.Tasks.Pleroma.ReleaseEnv do
-  use Mix.Task
-  import Mix.Pleroma
-
-  @shortdoc "Generate Pleroma environment file."
-  @moduledoc File.read!("docs/administration/CLI_tasks/release_environments.md")
-
-  def run(["gen" | rest]) do
-    {options, [], []} =
-      OptionParser.parse(
-        rest,
-        strict: [
-          force: :boolean,
-          path: :string
-        ],
-        aliases: [
-          p: :path,
-          f: :force
-        ]
-      )
-
-    file_path =
-      get_option(
-        options,
-        :path,
-        "Environment file path",
-        "./config/pleroma.env"
-      )
-
-    env_path = Path.expand(file_path)
-
-    proceed? =
-      if File.exists?(env_path) do
-        get_option(
-          options,
-          :force,
-          "Environment file already exists. Do you want to overwrite the #{env_path} file? (y/n)",
-          "n"
-        ) === "y"
-      else
-        true
-      end
-
-    if proceed? do
-      case do_generate(env_path) do
-        {:error, reason} ->
-          shell_error(
-            File.Error.message(%{action: "write to file", reason: reason, path: env_path})
-          )
-
-        _ ->
-          shell_info("\nThe file generated: #{env_path}.\n")
-
-          shell_info("""
-          WARNING: before start pleroma app please make sure to make the file read-only and non-modifiable.
-            Example:
-              chmod 0444 #{file_path}
-              chattr +i #{file_path}
-          """)
-      end
-    else
-      shell_info("\nThe file is exist. #{env_path}.\n")
-    end
-  end
-
-  def do_generate(path) do
-    content = "RELEASE_COOKIE=#{Base.encode32(:crypto.strong_rand_bytes(32))}"
-
-    File.mkdir_p!(Path.dirname(path))
-    File.write(path, content)
-  end
-end

lib/pleroma/activity.ex

@@ -356,4 +356,15 @@ def pinned_by_actor?(%Activity{} = activity) do
     actor = user_actor(activity)
     activity.id in actor.pinned_activities
   end
+
+  @spec get_by_object_ap_id_with_object(String.t()) :: t() | nil
+  def get_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
+    ap_id
+    |> Queries.by_object_id()
+    |> with_preloaded_object()
+    |> first()
+    |> Repo.one()
+  end
+
+  def get_by_object_ap_id_with_object(_), do: nil
 end

lib/pleroma/activity/ir/topics.ex

@@ -40,7 +40,8 @@ defp visibility_tags(object, activity) do
   end
 
   defp item_creation_tags(tags, object, %{data: %{"type" => "Create"}} = activity) do
-    tags ++ hashtags_to_topics(object) ++ attachment_topics(object, activity)
+    tags ++
+      remote_topics(activity) ++ hashtags_to_topics(object) ++ attachment_topics(object, activity)
   end
 
   defp item_creation_tags(tags, _, _) do
@@ -55,9 +56,19 @@ defp hashtags_to_topics(%{data: %{"tag" => tags}}) do
 
   defp hashtags_to_topics(_), do: []
 
+  defp remote_topics(%{local: true}), do: []
+
+  defp remote_topics(%{actor: actor}) when is_binary(actor),
+    do: ["public:remote:" <> URI.parse(actor).host]
+
+  defp remote_topics(_), do: []
+
   defp attachment_topics(%{data: %{"attachment" => []}}, _act), do: []
 
   defp attachment_topics(_object, %{local: true}), do: ["public:media", "public:local:media"]
 
+  defp attachment_topics(_object, %{actor: actor}) when is_binary(actor),
+    do: ["public:media", "public:remote:media:" <> URI.parse(actor).host]
+
   defp attachment_topics(_object, _act), do: ["public:media"]
 end

lib/pleroma/activity/search.ex

@@ -27,7 +27,10 @@ def search(user, search_query, options \\ []) do
     |> maybe_restrict_local(user)
     |> maybe_restrict_author(author)
     |> maybe_restrict_blocked(user)
-    |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => limit}, :offset)
+    |> Pagination.fetch_paginated(
+      %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
+      :offset
+    )
     |> maybe_fetch(user, search_query)
   end
 

lib/pleroma/application.ex

@@ -57,6 +57,7 @@ def start(_type, _args) do
     setup_instrumenters()
     load_custom_modules()
     Pleroma.Docs.JSON.compile()
+    limiters_setup()
 
     adapter = Application.get_env(:tesla, :adapter)
 
@@ -207,8 +208,7 @@ defp dont_run_in_test(_) do
          name: Pleroma.Web.Streamer.registry(),
          keys: :duplicate,
          partitions: System.schedulers_online()
-       ]},
+       ]}
-      Pleroma.Web.FedSockets.Supervisor
     ]
   end
 
@@ -273,4 +273,10 @@ defp http_children(Tesla.Adapter.Gun, _) do
   end
 
   defp http_children(_, _), do: []
+
+  @spec limiters_setup() :: :ok
+  def limiters_setup do
+    [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.MediaProxy]
+    |> Enum.each(&ConcurrentLimiter.new(&1, 1, 0))
+  end
 end

lib/pleroma/constants.ex

@@ -26,4 +26,6 @@ defmodule Pleroma.Constants do
     do:
       ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css)
   )
+
+  def as_local_public, do: Pleroma.Web.base_url() <> "/#Public"
 end

lib/pleroma/docs/json.ex

@@ -11,7 +11,11 @@ defmodule Pleroma.Docs.JSON do
 
   @spec compile :: :ok
   def compile do
-    :persistent_term.put(@term, Pleroma.Docs.Generator.convert_to_strings(@raw_descriptions))
+    descriptions =
+      Pleroma.Web.ActivityPub.MRF.config_descriptions()
+      |> Enum.reduce(@raw_descriptions, fn description, acc -> [description | acc] end)
+
+    :persistent_term.put(@term, Pleroma.Docs.Generator.convert_to_strings(descriptions))
   end
 
   @spec compiled_descriptions :: Map.t()

lib/pleroma/emails/admin_email.ex

@@ -18,10 +18,6 @@ defp instance_notify_email do
     Keyword.get(instance_config(), :notify_email, instance_config()[:email])
   end
 
-  defp user_url(user) do
-    Helpers.user_feed_url(Pleroma.Web.Endpoint, :feed_redirect, user.id)
-  end
-
   def test_email(mail_to \\ nil) do
     html_body = """
     <h3>Instance Test Email</h3>
@@ -52,6 +48,9 @@ def report(to, reporter, account, statuses, comment) do
               status_url = Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, id)
               "<li><a href=\"#{status_url}\">#{status_url}</li>"
 
+            %{"id" => id} when is_binary(id) ->
+              "<li><a href=\"#{id}\">#{id}</li>"
+
             id when is_binary(id) ->
               "<li><a href=\"#{id}\">#{id}</li>"
           end)
@@ -69,8 +68,8 @@ def report(to, reporter, account, statuses, comment) do
       end
 
     html_body = """
-    <p>Reported by: <a href="#{user_url(reporter)}">#{reporter.nickname}</a></p>
+    <p>Reported by: <a href="#{reporter.ap_id}">#{reporter.nickname}</a></p>
-    <p>Reported Account: <a href="#{user_url(account)}">#{account.nickname}</a></p>
+    <p>Reported Account: <a href="#{account.ap_id}">#{account.nickname}</a></p>
     #{comment_html}
     #{statuses_html}
     <p>
@@ -86,7 +85,7 @@ def report(to, reporter, account, statuses, comment) do
 
   def new_unapproved_registration(to, account) do
     html_body = """
-    <p>New account for review: <a href="#{user_url(account)}">@#{account.nickname}</a></p>
+    <p>New account for review: <a href="#{account.ap_id}">@#{account.nickname}</a></p>
     <blockquote>#{HTML.strip_tags(account.registration_reason)}</blockquote>
     <a href="#{Pleroma.Web.base_url()}/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a>
     """

lib/pleroma/frontend.ex

@@ -0,0 +1,110 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Frontend do
+  alias Pleroma.Config
+
+  require Logger
+
+  def install(name, opts \\ []) do
+    frontend_info = %{
+      "ref" => opts[:ref],
+      "build_url" => opts[:build_url],
+      "build_dir" => opts[:build_dir]
+    }
+
+    frontend_info =
+      [:frontends, :available, name]
+      |> Config.get(%{})
+      |> Map.merge(frontend_info, fn _key, config, cmd ->
+        # This only overrides things that are actually set
+        cmd || config
+      end)
+
+    ref = frontend_info["ref"]
+
+    unless ref do
+      raise "No ref given or configured"
+    end
+
+    dest = Path.join([dir(), name, ref])
+
+    label = "#{name} (#{ref})"
+    tmp_dir = Path.join(dir(), "tmp")
+
+    with {_, :ok} <-
+           {:download_or_unzip, download_or_unzip(frontend_info, tmp_dir, opts[:file])},
+         Logger.info("Installing #{label} to #{dest}"),
+         :ok <- install_frontend(frontend_info, tmp_dir, dest) do
+      File.rm_rf!(tmp_dir)
+      Logger.info("Frontend #{label} installed to #{dest}")
+    else
+      {:download_or_unzip, _} ->
+        Logger.info("Could not download or unzip the frontend")
+        {:error, "Could not download or unzip the frontend"}
+
+      _e ->
+        Logger.info("Could not install the frontend")
+        {:error, "Could not install the frontend"}
+    end
+  end
+
+  def dir(opts \\ []) do
+    if is_nil(opts[:static_dir]) do
+      Pleroma.Config.get!([:instance, :static_dir])
+    else
+      opts[:static_dir]
+    end
+    |> Path.join("frontends")
+  end
+
+  defp download_or_unzip(frontend_info, temp_dir, nil),
+    do: download_build(frontend_info, temp_dir)
+
+  defp download_or_unzip(_frontend_info, temp_dir, file) do
+    with {:ok, zip} <- File.read(Path.expand(file)) do
+      unzip(zip, temp_dir)
+    end
+  end
+
+  def unzip(zip, dest) do
+    with {:ok, unzipped} <- :zip.unzip(zip, [:memory]) do
+      File.rm_rf!(dest)
+      File.mkdir_p!(dest)
+
+      Enum.each(unzipped, fn {filename, data} ->
+        path = filename
+
+        new_file_path = Path.join(dest, path)
+
+        new_file_path
+        |> Path.dirname()
+        |> File.mkdir_p!()
+
+        File.write!(new_file_path, data)
+      end)
+    end
+  end
+
+  defp download_build(frontend_info, dest) do
+    Logger.info("Downloading pre-built bundle for #{frontend_info["name"]}")
+    url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
+
+    with {:ok, %{status: 200, body: zip_body}} <-
+           Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do
+      unzip(zip_body, dest)
+    else
+      {:error, e} -> {:error, e}
+      e -> {:error, e}
+    end
+  end
+
+  defp install_frontend(frontend_info, source, dest) do
+    from = frontend_info["build_dir"] || "dist"
+    File.rm_rf!(dest)
+    File.mkdir_p!(dest)
+    File.cp_r!(Path.join([source, from]), dest)
+    :ok
+  end
+end

lib/pleroma/instances.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Instances do
   defdelegate reachable?(url_or_host), to: @adapter
   defdelegate set_reachable(url_or_host), to: @adapter
   defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: @adapter
+  defdelegate get_consistently_unreachable(), to: @adapter
 
   def set_consistently_unreachable(url_or_host),
     do: set_unreachable(url_or_host, reachability_datetime_threshold())

lib/pleroma/instances/instance.ex

@@ -119,6 +119,17 @@ def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host)
 
   def set_unreachable(_, _), do: {:error, nil}
 
+  def get_consistently_unreachable do
+    reachability_datetime_threshold = Instances.reachability_datetime_threshold()
+
+    from(i in Instance,
+      where: ^reachability_datetime_threshold > i.unreachable_since,
+      order_by: i.unreachable_since,
+      select: {i.host, i.unreachable_since}
+    )
+    |> Repo.all()
+  end
+
   defp parse_datetime(datetime) when is_binary(datetime) do
     NaiveDateTime.from_iso8601(datetime)
   end

lib/pleroma/notification.ex

@@ -70,6 +70,7 @@ def unread_notifications_count(%User{id: user_id}) do
     move
     pleroma:chat_mention
     pleroma:emoji_reaction
+    pleroma:report
     reblog
   }
 
@@ -367,7 +368,7 @@ def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = act
   end
 
   def create_notifications(%Activity{data: %{"type" => type}} = activity, options)
-      when type in ["Follow", "Like", "Announce", "Move", "EmojiReact"] do
+      when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag"] do
     do_create_notifications(activity, options)
   end
 
@@ -410,6 +411,9 @@ defp type_from_activity(%{data: %{"type" => type}} = activity) do
       "EmojiReact" ->
         "pleroma:emoji_reaction"
 
+      "Flag" ->
+        "pleroma:report"
+
       # Compatibility with old reactions
       "EmojiReaction" ->
         "pleroma:emoji_reaction"
@@ -467,7 +471,7 @@ def create_notification(%Activity{} = activity, %User{} = user, do_send \\ true)
   def get_notified_from_activity(activity, local_only \\ true)
 
   def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only)
-      when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact"] do
+      when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact", "Flag"] do
     potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity)
 
     potential_receivers =
@@ -503,6 +507,10 @@ def get_potential_receiver_ap_ids(%{data: %{"type" => "Follow", "object" => obje
     [object_id]
   end
 
+  def get_potential_receiver_ap_ids(%{data: %{"type" => "Flag"}}) do
+    User.all_superusers() |> Enum.map(fn user -> user.ap_id end)
+  end
+
   def get_potential_receiver_ap_ids(activity) do
     []
     |> Utils.maybe_notify_to_recipients(activity)

lib/pleroma/object/fetcher.ex

@@ -12,7 +12,6 @@ defmodule Pleroma.Object.Fetcher do
   alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Federator
-  alias Pleroma.Web.FedSockets
 
   require Logger
   require Pleroma.Constants
@@ -183,16 +182,16 @@ defp maybe_date_fetch(headers, date) do
     end
   end
 
-  def fetch_and_contain_remote_object_from_id(prm, opts \\ [])
+  def fetch_and_contain_remote_object_from_id(id)
 
-  def fetch_and_contain_remote_object_from_id(%{"id" => id}, opts),
+  def fetch_and_contain_remote_object_from_id(%{"id" => id}),
-    do: fetch_and_contain_remote_object_from_id(id, opts)
+    do: fetch_and_contain_remote_object_from_id(id)
 
-  def fetch_and_contain_remote_object_from_id(id, opts) when is_binary(id) do
+  def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
     Logger.debug("Fetching object #{id} via AP")
 
     with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
-         {:ok, body} <- get_object(id, opts),
+         {:ok, body} <- get_object(id),
          {:ok, data} <- safe_json_decode(body),
          :ok <- Containment.contain_origin_from_id(id, data) do
       {:ok, data}
@@ -208,22 +207,10 @@ def fetch_and_contain_remote_object_from_id(id, opts) when is_binary(id) do
     end
   end
 
-  def fetch_and_contain_remote_object_from_id(_id, _opts),
+  def fetch_and_contain_remote_object_from_id(_id),
     do: {:error, "id must be a string"}
 
-  defp get_object(id, opts) do
+  defp get_object(id) do
-    with false <- Keyword.get(opts, :force_http, false),
-         {:ok, fedsocket} <- FedSockets.get_or_create_fed_socket(id) do
-      Logger.debug("fetching via fedsocket - #{inspect(id)}")
-      FedSockets.fetch(fedsocket, id)
-    else
-      _other ->
-        Logger.debug("fetching via http - #{inspect(id)}")
-        get_object_http(id)
-    end
-  end
-
-  defp get_object_http(id) do
     date = Pleroma.Signature.signed_date()
 
     headers =
@@ -232,8 +219,24 @@ defp get_object_http(id) do
       |> sign_fetch(id, date)
 
     case HTTP.get(id, headers) do
-      {:ok, %{body: body, status: code}} when code in 200..299 ->
+      {:ok, %{body: body, status: code, headers: headers}} when code in 200..299 ->
-        {:ok, body}
+        case List.keyfind(headers, "content-type", 0) do
+          {_, content_type} ->
+            case Plug.Conn.Utils.media_type(content_type) do
+              {:ok, "application", "activity+json", _} ->
+                {:ok, body}
+
+              {:ok, "application", "ld+json",
+               %{"profile" => "https://www.w3.org/ns/activitystreams"}} ->
+                {:ok, body}
+
+              _ ->
+                {:error, {:content_type, content_type}}
+            end
+
+          _ ->
+            {:error, {:content_type, nil}}
+        end
 
       {:ok, %{status: code}} when code in [404, 410] ->
         {:error, "Object has been deleted"}

lib/pleroma/password_reset_token.ex

@@ -40,6 +40,7 @@ def used_changeset(struct) do
   @spec reset_password(binary(), map()) :: {:ok, User.t()} | {:error, binary()}
   def reset_password(token, data) do
     with %{used: false} = token <- Repo.get_by(PasswordResetToken, %{token: token}),
+         false <- expired?(token),
          %User{} = user <- User.get_cached_by_id(token.user_id),
          {:ok, _user} <- User.reset_password(user, data),
          {:ok, token} <- Repo.update(used_changeset(token)) do
@@ -48,4 +49,14 @@ def reset_password(token, data) do
       _e -> {:error, token}
     end
   end
+
+  def expired?(%__MODULE__{inserted_at: inserted_at}) do
+    validity = Pleroma.Config.get([:instance, :password_reset_token_validity], 0)
+
+    now = NaiveDateTime.utc_now()
+
+    difference = NaiveDateTime.diff(now, inserted_at)
+
+    difference > validity
+  end
 end

lib/pleroma/signature.ex

@@ -39,7 +39,7 @@ def key_id_to_actor_id(key_id) do
   def fetch_public_key(conn) do
     with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
          {:ok, actor_id} <- key_id_to_actor_id(kid),
-         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id, force_http: true) do
+         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
       {:ok, public_key}
     else
       e ->
@@ -50,8 +50,8 @@ def fetch_public_key(conn) do
   def refetch_public_key(conn) do
     with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
          {:ok, actor_id} <- key_id_to_actor_id(kid),
-         {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id, force_http: true),
+         {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id),
-         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id, force_http: true) do
+         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
       {:ok, public_key}
     else
       e ->

lib/pleroma/stats.ex

@@ -23,7 +23,6 @@ def start_link(_) do
 
   @impl true
   def init(_args) do
-    if Pleroma.Config.get(:env) == :test, do: :ok = Ecto.Adapters.SQL.Sandbox.checkout(Repo)
     {:ok, nil, {:continue, :calculate_stats}}
   end
 
@@ -32,11 +31,6 @@ def force_update do
     GenServer.call(__MODULE__, :force_update)
   end
 
-  @doc "Performs collect stats"
-  def do_collect do
-    GenServer.cast(__MODULE__, :run_update)
-  end
-
   @doc "Returns stats data"
   @spec get_stats() :: %{
           domain_count: non_neg_integer(),
@@ -111,7 +105,11 @@ def get_status_visibility_count(instance \\ nil) do
   @impl true
   def handle_continue(:calculate_stats, _) do
     stats = calculate_stat_data()
-    Process.send_after(self(), :run_update, @interval)
+
+    unless Pleroma.Config.get(:env) == :test do
+      Process.send_after(self(), :run_update, @interval)
+    end
+
     {:noreply, stats}
   end
 
@@ -126,13 +124,6 @@ def handle_call(:get_state, _from, state) do
     {:reply, state, state}
   end
 
-  @impl true
-  def handle_cast(:run_update, _state) do
-    new_stats = calculate_stat_data()
-
-    {:noreply, new_stats}
-  end
-
   @impl true
   def handle_info(:run_update, _) do
     new_stats = calculate_stat_data()

lib/pleroma/user.ex

@@ -245,6 +245,18 @@ def unquote(:"#{outgoing_relation_target}_ap_ids")(user, restrict_deactivated? \
     end
   end
 
+  def cached_blocked_users_ap_ids(user) do
+    Cachex.fetch!(:user_cache, "blocked_users_ap_ids:#{user.ap_id}", fn _ ->
+      blocked_users_ap_ids(user)
+    end)
+  end
+
+  def cached_muted_users_ap_ids(user) do
+    Cachex.fetch!(:user_cache, "muted_users_ap_ids:#{user.ap_id}", fn _ ->
+      muted_users_ap_ids(user)
+    end)
+  end
+
   defdelegate following_count(user), to: FollowingRelationship
   defdelegate following(user), to: FollowingRelationship
   defdelegate following?(follower, followed), to: FollowingRelationship
@@ -1036,6 +1048,8 @@ def invalidate_cache(user) do
     Cachex.del(:user_cache, "ap_id:#{user.ap_id}")
     Cachex.del(:user_cache, "nickname:#{user.nickname}")
     Cachex.del(:user_cache, "friends_ap_ids:#{user.ap_id}")
+    Cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
+    Cachex.del(:user_cache, "muted_users_ap_ids:#{user.ap_id}")
   end
 
   @spec get_cached_by_ap_id(String.t()) :: User.t() | nil
@@ -1324,14 +1338,51 @@ def get_recipients_from_activity(%Activity{recipients: to, actor: actor}) do
     |> Repo.all()
   end
 
-  @spec mute(User.t(), User.t(), boolean()) ::
+  @spec mute(User.t(), User.t(), map()) ::
           {:ok, list(UserRelationship.t())} | {:error, String.t()}
-  def mute(%User{} = muter, %User{} = mutee, notifications? \\ true) do
+  def mute(%User{} = muter, %User{} = mutee, params \\ %{}) do
-    add_to_mutes(muter, mutee, notifications?)
+    notifications? = Map.get(params, :notifications, true)
+    expires_in = Map.get(params, :expires_in, 0)
+
+    with {:ok, user_mute} <- UserRelationship.create_mute(muter, mutee),
+         {:ok, user_notification_mute} <-
+           (notifications? && UserRelationship.create_notification_mute(muter, mutee)) ||
+             {:ok, nil} do
+      if expires_in > 0 do
+        Pleroma.Workers.MuteExpireWorker.enqueue(
+          "unmute_user",
+          %{"muter_id" => muter.id, "mutee_id" => mutee.id},
+          schedule_in: expires_in
+        )
+      end
+
+      Cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}")
+
+      {:ok, Enum.filter([user_mute, user_notification_mute], & &1)}
+    end
   end
 
   def unmute(%User{} = muter, %User{} = mutee) do
-    remove_from_mutes(muter, mutee)
+    with {:ok, user_mute} <- UserRelationship.delete_mute(muter, mutee),
+         {:ok, user_notification_mute} <-
+           UserRelationship.delete_notification_mute(muter, mutee) do
+      Cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}")
+      {:ok, [user_mute, user_notification_mute]}
+    end
+  end
+
+  def unmute(muter_id, mutee_id) do
+    with {:muter, %User{} = muter} <- {:muter, User.get_by_id(muter_id)},
+         {:mutee, %User{} = mutee} <- {:mutee, User.get_by_id(mutee_id)} do
+      unmute(muter, mutee)
+    else
+      {who, result} = error ->
+        Logger.warn(
+          "User.unmute/2 failed. #{who}: #{result}, muter_id: #{muter_id}, mutee_id: #{mutee_id}"
+        )
+
+        {:error, error}
+    end
   end
 
   def subscribe(%User{} = subscriber, %User{} = target) do
@@ -1738,12 +1789,12 @@ def html_filter_policy(%User{no_rich_text: true}) do
 
   def html_filter_policy(_), do: Config.get([:markup, :scrub_policy])
 
-  def fetch_by_ap_id(ap_id, opts \\ []), do: ActivityPub.make_user_from_ap_id(ap_id, opts)
+  def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id)
 
-  def get_or_fetch_by_ap_id(ap_id, opts \\ []) do
+  def get_or_fetch_by_ap_id(ap_id) do
     cached_user = get_cached_by_ap_id(ap_id)
 
-    maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id, opts)
+    maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id)
 
     case {cached_user, maybe_fetched_user} do
       {_, {:ok, %User{} = user}} ->
@@ -1816,8 +1867,8 @@ def public_key(%{public_key: public_key_pem}) when is_binary(public_key_pem) do
 
   def public_key(_), do: {:error, "key not found"}
 
-  def get_public_key_for_ap_id(ap_id, opts \\ []) do
+  def get_public_key_for_ap_id(ap_id) do
-    with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id, opts),
+    with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id),
          {:ok, public_key} <- public_key(user) do
       {:ok, public_key}
     else
@@ -2311,29 +2362,18 @@ def unblock_domain(user, domain_blocked) do
   @spec add_to_block(User.t(), User.t()) ::
           {:ok, UserRelationship.t()} | {:error, Ecto.Changeset.t()}
   defp add_to_block(%User{} = user, %User{} = blocked) do
-    UserRelationship.create_block(user, blocked)
+    with {:ok, relationship} <- UserRelationship.create_block(user, blocked) do
+      Cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
+      {:ok, relationship}
+    end
   end
 
   @spec add_to_block(User.t(), User.t()) ::
           {:ok, UserRelationship.t()} | {:ok, nil} | {:error, Ecto.Changeset.t()}
   defp remove_from_block(%User{} = user, %User{} = blocked) do
-    UserRelationship.delete_block(user, blocked)
+    with {:ok, relationship} <- UserRelationship.delete_block(user, blocked) do
-  end
+      Cachex.del(:user_cache, "blocked_users_ap_ids:#{user.ap_id}")
-
+      {:ok, relationship}
-  defp add_to_mutes(%User{} = user, %User{} = muted_user, notifications?) do
-    with {:ok, user_mute} <- UserRelationship.create_mute(user, muted_user),
-         {:ok, user_notification_mute} <-
-           (notifications? && UserRelationship.create_notification_mute(user, muted_user)) ||
-             {:ok, nil} do
-      {:ok, Enum.filter([user_mute, user_notification_mute], & &1)}
-    end
-  end
-
-  defp remove_from_mutes(user, %User{} = muted_user) do
-    with {:ok, user_mute} <- UserRelationship.delete_mute(user, muted_user),
-         {:ok, user_notification_mute} <-
-           UserRelationship.delete_notification_mute(user, muted_user) do
-      {:ok, [user_mute, user_notification_mute]}
     end
   end
 

lib/pleroma/user/search.ex

@@ -85,7 +85,6 @@ defp search_query(query_string, for_user, following, top_user_ids) do
     |> base_query(following)
     |> filter_blocked_user(for_user)
     |> filter_invisible_users()
-    |> filter_discoverable_users()
     |> filter_internal_users()
     |> filter_blocked_domains(for_user)
     |> fts_search(query_string)
@@ -163,10 +162,6 @@ defp filter_invisible_users(query) do
     from(q in query, where: q.invisible == false)
   end
 
-  defp filter_discoverable_users(query) do
-    from(q in query, where: q.is_discoverable == true)
-  end
-
   defp filter_internal_users(query) do
     from(q in query, where: q.actor_type != "Application")
   end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -123,7 +123,9 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when
       # Splice in the child object if we have one.
       activity = Maps.put_if_present(activity, :object, object)
 
-      BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id})
+      ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn ->
+        Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
+      end)
 
       {:ok, activity}
     else
@@ -332,15 +334,21 @@ defp do_unfollow(follower, followed, activity_id, local) do
   end
 
   @spec flag(map()) :: {:ok, Activity.t()} | {:error, any()}
-  def flag(
+  def flag(params) do
-        %{
+    with {:ok, result} <- Repo.transaction(fn -> do_flag(params) end) do
-          actor: actor,
+      result
-          context: _context,
+    end
-          account: account,
+  end
-          statuses: statuses,
+
-          content: content
+  defp do_flag(
-        } = params
+         %{
-      ) do
+           actor: actor,
+           context: _context,
+           account: account,
+           statuses: statuses,
+           content: content
+         } = params
+       ) do
     # only accept false as false value
     local = !(params[:local] == false)
     forward = !(params[:forward] == false)
@@ -358,7 +366,8 @@ def flag(
          {:ok, activity} <- insert(flag_data, local),
          {:ok, stripped_activity} <- strip_report_status_data(activity),
          _ <- notify_and_stream(activity),
-         :ok <- maybe_federate(stripped_activity) do
+         :ok <-
+           maybe_federate(stripped_activity) do
       User.all_superusers()
       |> Enum.filter(fn user -> not is_nil(user.email) end)
       |> Enum.each(fn superuser ->
@@ -368,6 +377,8 @@ def flag(
       end)
 
       {:ok, activity}
+    else
+      {:error, error} -> Repo.rollback(error)
     end
   end
 
@@ -937,16 +948,11 @@ defp restrict_muted_reblogs(query, %{muting_user: %User{} = user} = opts) do
 
   defp restrict_muted_reblogs(query, _), do: query
 
-  defp restrict_instance(query, %{instance: instance}) do
+  defp restrict_instance(query, %{instance: instance}) when is_binary(instance) do
-    users =
+    from(
-      from(
+      activity in query,
-        u in User,
+      where: fragment("split_part(actor::text, '/'::text, 3) = ?", ^instance)
-        select: u.ap_id,
+    )
-        where: fragment("? LIKE ?", u.nickname, ^"%@#{instance}")
-      )
-      |> Repo.all()
-
-    from(activity in query, where: activity.actor in ^users)
   end
 
   defp restrict_instance(query, _), do: query
@@ -1294,12 +1300,10 @@ defp object_to_user_data(data) do
 
   def fetch_follow_information_for_user(user) do
     with {:ok, following_data} <-
-           Fetcher.fetch_and_contain_remote_object_from_id(user.following_address,
+           Fetcher.fetch_and_contain_remote_object_from_id(user.following_address),
-             force_http: true
-           ),
          {:ok, hide_follows} <- collection_private(following_data),
          {:ok, followers_data} <-
-           Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address, force_http: true),
+           Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address),
          {:ok, hide_followers} <- collection_private(followers_data) do
       {:ok,
        %{
@@ -1373,8 +1377,8 @@ def user_data_from_user_object(data) do
     end
   end
 
-  def fetch_and_prepare_user_from_ap_id(ap_id, opts \\ []) do
+  def fetch_and_prepare_user_from_ap_id(ap_id) do
-    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id, opts),
+    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
          {:ok, data} <- user_data_from_user_object(data) do
       {:ok, maybe_update_follow_information(data)}
     else
@@ -1417,13 +1421,13 @@ def maybe_handle_clashing_nickname(data) do
     end
   end
 
-  def make_user_from_ap_id(ap_id, opts \\ []) do
+  def make_user_from_ap_id(ap_id) do
     user = User.get_cached_by_ap_id(ap_id)
 
     if user && !User.ap_enabled?(user) do
       Transmogrifier.upgrade_user_from_ap_id(ap_id)
     else
-      with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, opts) do
+      with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id) do
         if user do
           user
           |> User.remote_user_changeset(data)

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -82,7 +82,8 @@ def user(conn, %{"nickname" => nickname}) do
   def object(conn, _) do
     with ap_id <- Endpoint.url() <> conn.request_path,
          %Object{} = object <- Object.get_cached_by_ap_id(ap_id),
-         {_, true} <- {:public?, Visibility.is_public?(object)} do
+         {_, true} <- {:public?, Visibility.is_public?(object)},
+         {_, false} <- {:local?, Visibility.is_local_public?(object)} do
       conn
       |> assign(:tracking_fun_data, object.id)
       |> set_cache_ttl_for(object)
@@ -92,6 +93,9 @@ def object(conn, _) do
     else
       {:public?, false} ->
         {:error, :not_found}
+
+      {:local?, true} ->
+        {:error, :not_found}
     end
   end
 
@@ -108,7 +112,8 @@ def track_object_fetch(conn, object_id) do
   def activity(conn, _params) do
     with ap_id <- Endpoint.url() <> conn.request_path,
          %Activity{} = activity <- Activity.normalize(ap_id),
-         {_, true} <- {:public?, Visibility.is_public?(activity)} do
+         {_, true} <- {:public?, Visibility.is_public?(activity)},
+         {_, false} <- {:local?, Visibility.is_local_public?(activity)} do
       conn
       |> maybe_set_tracking_data(activity)
       |> set_cache_ttl_for(activity)
@@ -117,6 +122,7 @@ def activity(conn, _params) do
       |> render("object.json", object: activity)
     else
       {:public?, false} -> {:error, :not_found}
+      {:local?, true} -> {:error, :not_found}
       nil -> {:error, :not_found}
     end
   end

lib/pleroma/web/activity_pub/builder.ex

@@ -222,6 +222,9 @@ def announce(actor, object, options \\ []) do
         actor.ap_id == Relay.ap_id() ->
           [actor.follower_address]
 
+        public? and Visibility.is_local_public?(object) ->
+          [actor.follower_address, object.data["actor"], Pleroma.Constants.as_local_public()]
+
         public? ->
           [actor.follower_address, object.data["actor"], Pleroma.Constants.as_public()]
 

lib/pleroma/web/activity_pub/mrf.ex

@@ -3,7 +3,62 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.MRF do
+  require Logger
+
+  @mrf_config_descriptions [
+    %{
+      group: :pleroma,
+      key: :mrf,
+      tab: :mrf,
+      label: "MRF",
+      type: :group,
+      description: "General MRF settings",
+      children: [
+        %{
+          key: :policies,
+          type: [:module, {:list, :module}],
+          description:
+            "A list of MRF policies enabled. Module names are shortened (removed leading `Pleroma.Web.ActivityPub.MRF.` part), but on adding custom module you need to use full name.",
+          suggestions: {:list_behaviour_implementations, Pleroma.Web.ActivityPub.MRF}
+        },
+        %{
+          key: :transparency,
+          label: "MRF transparency",
+          type: :boolean,
+          description:
+            "Make the content of your Message Rewrite Facility settings public (via nodeinfo)"
+        },
+        %{
+          key: :transparency_exclusions,
+          label: "MRF transparency exclusions",
+          type: {:list, :string},
+          description:
+            "Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.",
+          suggestions: [
+            "exclusion.com"
+          ]
+        }
+      ]
+    }
+  ]
+
+  @default_description %{
+    label: "",
+    description: ""
+  }
+
+  @required_description_keys [:key, :related_policy]
+
   @callback filter(Map.t()) :: {:ok | :reject, Map.t()}
+  @callback describe() :: {:ok | :error, Map.t()}
+  @callback config_description() :: %{
+              optional(:children) => [map()],
+              key: atom(),
+              related_policy: String.t(),
+              label: String.t(),
+              description: String.t()
+            }
+  @optional_callbacks config_description: 0
 
   def filter(policies, %{} = message) do
     policies
@@ -51,8 +106,6 @@ def subdomain_match?(domains, host) do
     Enum.any?(domains, fn domain -> Regex.match?(domain, host) end)
   end
 
-  @callback describe() :: {:ok | :error, Map.t()}
-
   def describe(policies) do
     {:ok, policy_configs} =
       policies
@@ -82,4 +135,41 @@ def describe(policies) do
   end
 
   def describe, do: get_policies() |> describe()
+
+  def config_descriptions do
+    Pleroma.Web.ActivityPub.MRF
+    |> Pleroma.Docs.Generator.list_behaviour_implementations()
+    |> config_descriptions()
+  end
+
+  def config_descriptions(policies) do
+    Enum.reduce(policies, @mrf_config_descriptions, fn policy, acc ->
+      if function_exported?(policy, :config_description, 0) do
+        description =
+          @default_description
+          |> Map.merge(policy.config_description)
+          |> Map.put(:group, :pleroma)
+          |> Map.put(:tab, :mrf)
+          |> Map.put(:type, :group)
+
+        if Enum.all?(@required_description_keys, &Map.has_key?(description, &1)) do
+          [description | acc]
+        else
+          Logger.warn(
+            "#{policy} config description doesn't have one or all required keys #{
+              inspect(@required_description_keys)
+            }"
+          )
+
+          acc
+        end
+      else
+        Logger.debug(
+          "#{policy} is excluded from config descriptions, because does not implement `config_description/0` method."
+        )
+
+        acc
+      end
+    end)
+  end
 end

lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex

@@ -40,4 +40,22 @@ defp maybe_add_expiration(activity) do
       _ -> Map.put(activity, "expires_at", expires_at)
     end
   end
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_activity_expiration,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy",
+      label: "MRF Activity Expiration Policy",
+      description: "Adds automatic expiration to all local activities",
+      children: [
+        %{
+          key: :days,
+          type: :integer,
+          description: "Default global expiration time for all local activities (in days)",
+          suggestions: [90, 365]
+        }
+      ]
+    }
+  end
 end

lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex

@@ -97,4 +97,31 @@ def filter(message), do: {:ok, message}
   @impl true
   def describe,
     do: {:ok, %{mrf_hellthread: Pleroma.Config.get(:mrf_hellthread) |> Enum.into(%{})}}
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_hellthread,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.HellthreadPolicy",
+      label: "MRF Hellthread",
+      description: "Block messages with excessive user mentions",
+      children: [
+        %{
+          key: :delist_threshold,
+          type: :integer,
+          description:
+            "Number of mentioned users after which the message gets removed from timelines and" <>
+              "disables notifications. Set to 0 to disable.",
+          suggestions: [10]
+        },
+        %{
+          key: :reject_threshold,
+          type: :integer,
+          description:
+            "Number of mentioned users after which the messaged gets rejected. Set to 0 to disable.",
+          suggestions: [20]
+        }
+      ]
+    }
+  end
 end

lib/pleroma/web/activity_pub/mrf/keyword_policy.ex

@@ -126,4 +126,46 @@ def describe do
 
     {:ok, %{mrf_keyword: mrf_keyword}}
   end
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_keyword,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.KeywordPolicy",
+      label: "MRF Keyword",
+      description:
+        "Reject or Word-Replace messages matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).",
+      children: [
+        %{
+          key: :reject,
+          type: {:list, :string},
+          description: """
+            A list of patterns which result in message being rejected.
+
+            Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+          """,
+          suggestions: ["foo", ~r/foo/iu]
+        },
+        %{
+          key: :federated_timeline_removal,
+          type: {:list, :string},
+          description: """
+            A list of patterns which result in message being removed from federated timelines (a.k.a unlisted).
+
+            Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+          """,
+          suggestions: ["foo", ~r/foo/iu]
+        },
+        %{
+          key: :replace,
+          type: {:list, :tuple},
+          description: """
+            **Pattern**: a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+
+            **Replacement**: a string. Leaving the field empty is permitted.
+          """
+        }
+      ]
+    }
+  end
 end

lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex

@@ -8,7 +8,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
   alias Pleroma.HTTP
   alias Pleroma.Web.MediaProxy
-  alias Pleroma.Workers.BackgroundWorker
 
   require Logger
 
@@ -17,7 +16,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
     recv_timeout: 10_000
   ]
 
-  def perform(:prefetch, url) do
+  defp prefetch(url) do
     # Fetching only proxiable resources
     if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do
       # If preview proxy is enabled, it'll also hit media proxy (so we're caching both requests)
@@ -25,17 +24,25 @@ def perform(:prefetch, url) do
 
       Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
 
-      HTTP.get(prefetch_url, [], @adapter_options)
+      if Pleroma.Config.get(:env) == :test do
+        fetch(prefetch_url)
+      else
+        ConcurrentLimiter.limit(MediaProxy, fn ->
+          Task.start(fn -> fetch(prefetch_url) end)
+        end)
+      end
     end
   end
 
-  def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do
+  defp fetch(url), do: HTTP.get(url, [], @adapter_options)
+
+  defp preload(%{"object" => %{"attachment" => attachments}} = _message) do
     Enum.each(attachments, fn
       %{"url" => url} when is_list(url) ->
         url
         |> Enum.each(fn
           %{"href" => href} ->
-            BackgroundWorker.enqueue("media_proxy_prefetch", %{"url" => href})
+            prefetch(href)
 
           x ->
             Logger.debug("Unhandled attachment URL object #{inspect(x)}")
@@ -51,7 +58,7 @@ def filter(
         %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message
       )
       when is_list(attachments) and length(attachments) > 0 do
-    BackgroundWorker.enqueue("media_proxy_preload", %{"message" => message})
+    preload(message)
 
     {:ok, message}
   end

lib/pleroma/web/activity_pub/mrf/mention_policy.ex

@@ -25,4 +25,22 @@ def filter(message), do: {:ok, message}
 
   @impl true
   def describe, do: {:ok, %{}}
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_mention,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.MentionPolicy",
+      label: "MRF Mention",
+      description: "Block messages which mention a specific user",
+      children: [
+        %{
+          key: :actors,
+          type: {:list, :string},
+          description: "A list of actors for which any post mentioning them will be dropped",
+          suggestions: ["actor1", "actor2"]
+        }
+      ]
+    }
+  end
 end

lib/pleroma/web/activity_pub/mrf/normalize_markup.ex

@@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do
 
   @behaviour Pleroma.Web.ActivityPub.MRF
 
+  @impl true
   def filter(%{"type" => "Create", "object" => child_object} = object) do
     scrub_policy = Pleroma.Config.get([:mrf_normalize_markup, :scrub_policy])
 
@@ -22,5 +23,23 @@ def filter(%{"type" => "Create", "object" => child_object} = object) do
 
   def filter(object), do: {:ok, object}
 
+  @impl true
   def describe, do: {:ok, %{}}
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_normalize_markup,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.NormalizeMarkup",
+      label: "MRF Normalize Markup",
+      description: "MRF NormalizeMarkup settings. Scrub configured hypertext markup.",
+      children: [
+        %{
+          key: :scrub_policy,
+          type: :module,
+          suggestions: [Pleroma.HTML.Scrubber.Default]
+        }
+      ]
+    }
+  end
 end

lib/pleroma/web/activity_pub/mrf/object_age_policy.ex

@@ -106,4 +106,32 @@ def describe do
 
     {:ok, %{mrf_object_age: mrf_object_age}}
   end
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_object_age,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy",
+      label: "MRF Object Age",
+      description:
+        "Rejects or delists posts based on their timestamp deviance from your server's clock.",
+      children: [
+        %{
+          key: :threshold,
+          type: :integer,
+          description: "Required age (in seconds) of a post before actions are taken.",
+          suggestions: [172_800]
+        },
+        %{
+          key: :actions,
+          type: {:list, :atom},
+          description:
+            "A list of actions to apply to the post. `:delist` removes the post from public timelines; " <>
+              "`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines; " <>
+              "`:reject` rejects the message entirely",
+          suggestions: [:delist, :strip_followers, :reject]
+        }
+      ]
+    }
+  end
 end

lib/pleroma/web/activity_pub/mrf/reject_non_public.ex

@@ -48,4 +48,27 @@ def filter(object), do: {:ok, object}
   @impl true
   def describe,
     do: {:ok, %{mrf_rejectnonpublic: Config.get(:mrf_rejectnonpublic) |> Enum.into(%{})}}
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_rejectnonpublic,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.RejectNonPublic",
+      description: "RejectNonPublic drops posts with non-public visibility settings.",
+      label: "MRF Reject Non Public",
+      children: [
+        %{
+          key: :allow_followersonly,
+          label: "Allow followers-only",
+          type: :boolean,
+          description: "Whether to allow followers-only posts"
+        },
+        %{
+          key: :allow_direct,
+          type: :boolean,
+          description: "Whether to allow direct messages"
+        }
+      ]
+    }
+  end
 end

lib/pleroma/web/activity_pub/mrf/simple_policy.ex

@@ -244,4 +244,78 @@ def describe do
 
     {:ok, %{mrf_simple: mrf_simple}}
   end
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_simple,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.SimplePolicy",
+      label: "MRF Simple",
+      description: "Simple ingress policies",
+      children: [
+        %{
+          key: :media_removal,
+          type: {:list, :string},
+          description: "List of instances to strip media attachments from",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :media_nsfw,
+          label: "Media NSFW",
+          type: {:list, :string},
+          description: "List of instances to tag all media as NSFW (sensitive) from",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :federated_timeline_removal,
+          type: {:list, :string},
+          description:
+            "List of instances to remove from the Federated (aka The Whole Known Network) Timeline",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :reject,
+          type: {:list, :string},
+          description: "List of instances to reject activities from (except deletes)",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :accept,
+          type: {:list, :string},
+          description: "List of instances to only accept activities from (except deletes)",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :followers_only,
+          type: {:list, :string},
+          description: "Force posts from the given instances to be visible by followers only",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :report_removal,
+          type: {:list, :string},
+          description: "List of instances to reject reports from",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :avatar_removal,
+          type: {:list, :string},
+          description: "List of instances to strip avatars from",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :banner_removal,
+          type: {:list, :string},
+          description: "List of instances to strip banners from",
+          suggestions: ["example.com", "*.example.com"]
+        },
+        %{
+          key: :reject_deletes,
+          type: {:list, :string},
+          description: "List of instances to reject deletions from",
+          suggestions: ["example.com", "*.example.com"]
+        }
+      ]
+    }
+  end
 end

lib/pleroma/web/activity_pub/mrf/subchain_policy.ex

@@ -39,4 +39,28 @@ def filter(message), do: {:ok, message}
 
   @impl true
   def describe, do: {:ok, %{}}
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_subchain,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.SubchainPolicy",
+      label: "MRF Subchain",
+      description:
+        "This policy processes messages through an alternate pipeline when a given message matches certain criteria." <>
+          " All criteria are configured as a map of regular expressions to lists of policy modules.",
+      children: [
+        %{
+          key: :match_actor,
+          type: {:map, {:list, :string}},
+          description: "Matches a series of regular expressions against the actor field",
+          suggestions: [
+            %{
+              ~r/https:\/\/example.com/s => [Pleroma.Web.ActivityPub.MRF.DropPolicy]
+            }
+          ]
+        }
+      ]
+    }
+  end
 end

lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex

@@ -41,4 +41,25 @@ def describe do
 
     {:ok, %{mrf_user_allowlist: mrf_user_allowlist}}
   end
+
+  # TODO: change way of getting settings on `lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex:18` to use `hosts` subkey
+  # @impl true
+  # def config_description do
+  #   %{
+  #     key: :mrf_user_allowlist,
+  #     related_policy: "Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy",
+  #     description: "Accept-list of users from specified instances",
+  #     children: [
+  #       %{
+  #         key: :hosts,
+  #         type: :map,
+  #         description:
+  #           "The keys in this section are the domain names that the policy should apply to." <>
+  #             " Each key should be assigned a list of users that should be allowed " <>
+  #             "through by their ActivityPub ID",
+  #         suggestions: [%{"example.org" => ["https://example.org/users/admin"]}]
+  #       }
+  #     ]
+  #   }
+  # end
 end

lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do
 
   @behaviour Pleroma.Web.ActivityPub.MRF
 
+  @impl true
   def filter(%{"type" => "Undo", "object" => child_message} = message) do
     with {:ok, _} <- filter(child_message) do
       {:ok, message}
@@ -36,6 +37,33 @@ def filter(%{"type" => message_type} = message) do
 
   def filter(message), do: {:ok, message}
 
+  @impl true
   def describe,
     do: {:ok, %{mrf_vocabulary: Pleroma.Config.get(:mrf_vocabulary) |> Enum.into(%{})}}
+
+  @impl true
+  def config_description do
+    %{
+      key: :mrf_vocabulary,
+      related_policy: "Pleroma.Web.ActivityPub.MRF.VocabularyPolicy",
+      label: "MRF Vocabulary",
+      description: "Filter messages which belong to certain activity vocabularies",
+      children: [
+        %{
+          key: :accept,
+          type: {:list, :string},
+          description:
+            "A list of ActivityStreams terms to accept. If empty, all supported messages are accepted.",
+          suggestions: ["Create", "Follow", "Mention", "Announce", "Like"]
+        },
+        %{
+          key: :reject,
+          type: {:list, :string},
+          description:
+            "A list of ActivityStreams terms to reject. If empty, no messages are rejected.",
+          suggestions: ["Create", "Follow", "Mention", "Announce", "Like"]
+        }
+      ]
+    }
+  end
 end

lib/pleroma/web/activity_pub/object_validators/announce_validator.ex

@@ -67,7 +67,12 @@ def validate_announcable(cng) do
          %Object{} = object <- Object.get_cached_by_ap_id(object),
          false <- Visibility.is_public?(object) do
       same_actor = object.data["actor"] == actor.ap_id
-      is_public = Pleroma.Constants.as_public() in (get_field(cng, :to) ++ get_field(cng, :cc))
+      recipients = get_field(cng, :to) ++ get_field(cng, :cc)
+      local_public = Pleroma.Constants.as_local_public()
+
+      is_public =
+        Enum.member?(recipients, Pleroma.Constants.as_public()) or
+          Enum.member?(recipients, local_public)
 
       cond do
         same_actor && is_public ->

lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex

@@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
     field(:type, :string)
     field(:mediaType, :string, default: "application/octet-stream")
     field(:name, :string)
+    field(:blurhash, :string)
 
     embeds_many :url, UrlObjectValidator, primary_key: false do
       field(:type, :string)
@@ -41,7 +42,7 @@ def changeset(struct, data) do
       |> fix_url()
 
     struct
-    |> cast(data, [:type, :mediaType, :name])
+    |> cast(data, [:type, :mediaType, :name, :blurhash])
     |> cast_embed(:url, with: &url_changeset/2)
     |> validate_inclusion(:type, ~w[Link Document Audio Image Video])
     |> validate_required([:type, :mediaType, :url])

lib/pleroma/web/activity_pub/pipeline.ex

@@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
   alias Pleroma.Web.ActivityPub.MRF
   alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.SideEffects
+  alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.Federator
 
   @spec common_pipeline(map(), keyword()) ::
@@ -55,7 +56,7 @@ defp maybe_federate(%Activity{} = activity, meta) do
     with {:ok, local} <- Keyword.fetch(meta, :local) do
       do_not_federate = meta[:do_not_federate] || !Config.get([:instance, :federating])
 
-      if !do_not_federate && local do
+      if !do_not_federate and local and not Visibility.is_local_public?(activity) do
         activity =
           if object = Keyword.get(meta, :object_data) do
             %{activity | data: Map.put(activity.data, "object", object)}

lib/pleroma/web/activity_pub/publisher.ex

@@ -13,7 +13,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.ActivityPub.Transmogrifier
-  alias Pleroma.Web.FedSockets
 
   require Pleroma.Constants
 
@@ -50,28 +49,6 @@ def is_representable?(%Activity{} = activity) do
   """
   def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = params) do
     Logger.debug("Federating #{id} to #{inbox}")
-
-    case FedSockets.get_or_create_fed_socket(inbox) do
-      {:ok, fedsocket} ->
-        Logger.debug("publishing via fedsockets - #{inspect(inbox)}")
-        FedSockets.publish(fedsocket, json)
-
-      _ ->
-        Logger.debug("publishing via http - #{inspect(inbox)}")
-        http_publish(inbox, actor, json, params)
-    end
-  end
-
-  def publish_one(%{actor_id: actor_id} = params) do
-    actor = User.get_cached_by_id(actor_id)
-
-    params
-    |> Map.delete(:actor_id)
-    |> Map.put(:actor, actor)
-    |> publish_one()
-  end
-
-  defp http_publish(inbox, actor, json, params) do
     uri = %{path: path} = URI.parse(inbox)
     digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64())
 
@@ -110,6 +87,15 @@ defp http_publish(inbox, actor, json, params) do
     end
   end
 
+  def publish_one(%{actor_id: actor_id} = params) do
+    actor = User.get_cached_by_id(actor_id)
+
+    params
+    |> Map.delete(:actor_id)
+    |> Map.put(:actor, actor)
+    |> publish_one()
+  end
+
   defp signature_host(%URI{port: port, scheme: scheme, host: host}) do
     if port == URI.default_port(scheme) do
       host

lib/pleroma/web/activity_pub/side_effects.ex

@@ -24,7 +24,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Push
   alias Pleroma.Web.Streamer
-  alias Pleroma.Workers.BackgroundWorker
 
   require Logger
 
@@ -191,7 +190,9 @@ def handle(%{data: %{"type" => "Create"}} = activity, meta) do
         Object.increase_replies_count(in_reply_to)
       end
 
-      BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id})
+      ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn ->
+        Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
+      end)
 
       meta =
         meta

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -252,6 +252,7 @@ def fix_attachments(%{"attachment" => attachment} = object) when is_list(attachm
           }
           |> Maps.put_if_present("mediaType", media_type)
           |> Maps.put_if_present("name", data["name"])
+          |> Maps.put_if_present("blurhash", data["blurhash"])
         else
           nil
         end
@@ -1007,7 +1008,7 @@ def perform(:user_upgrade, user) do
 
   def upgrade_user_from_ap_id(ap_id) do
     with %User{local: false} = user <- User.get_cached_by_ap_id(ap_id),
-         {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id, force_http: true),
+         {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id),
          {:ok, user} <- update_user(user, data) do
       TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id})
       {:ok, user}

lib/pleroma/web/activity_pub/utils.ex

@@ -175,7 +175,8 @@ def maybe_federate(%Activity{local: true, data: %{"type" => type}} = activity) d
     outgoing_blocks = Config.get([:activitypub, :outgoing_blocks])
 
     with true <- Config.get!([:instance, :federating]),
-         true <- type != "Block" || outgoing_blocks do
+         true <- type != "Block" || outgoing_blocks,
+         false <- Visibility.is_local_public?(activity) do
       Pleroma.Web.Federator.publish(activity)
     end
 
@@ -701,14 +702,30 @@ def make_flag_data(%{actor: actor, context: context, content: content} = params,
 
   def make_flag_data(_, _), do: %{}
 
-  defp build_flag_object(%{account: account, statuses: statuses} = _) do
+  defp build_flag_object(%{account: account, statuses: statuses}) do
-    [account.ap_id] ++ build_flag_object(%{statuses: statuses})
+    [account.ap_id | build_flag_object(%{statuses: statuses})]
   end
 
   defp build_flag_object(%{statuses: statuses}) do
     Enum.map(statuses || [], &build_flag_object/1)
   end
 
+  defp build_flag_object(%Activity{data: %{"id" => id}, object: %{data: data}}) do
+    activity_actor = User.get_by_ap_id(data["actor"])
+
+    %{
+      "type" => "Note",
+      "id" => id,
+      "content" => data["content"],
+      "published" => data["published"],
+      "actor" =>
+        AccountView.render(
+          "show.json",
+          %{user: activity_actor, skip_visibility_check: true}
+        )
+    }
+  end
+
   defp build_flag_object(act) when is_map(act) or is_binary(act) do
     id =
       case act do
@@ -719,22 +736,14 @@ defp build_flag_object(act) when is_map(act) or is_binary(act) do
 
     case Activity.get_by_ap_id_with_object(id) do
       %Activity{} = activity ->
-        activity_actor = User.get_by_ap_id(activity.object.data["actor"])
+        build_flag_object(activity)
 
-        %{
+      nil ->
-          "type" => "Note",
+        if activity = Activity.get_by_object_ap_id_with_object(id) do
-          "id" => activity.data["id"],
+          build_flag_object(activity)
-          "content" => activity.object.data["content"],
+        else
-          "published" => activity.object.data["published"],
+          %{"id" => id, "deleted" => true}
-          "actor" =>
+        end
-            AccountView.render(
-              "show.json",
-              %{user: activity_actor, skip_visibility_check: true}
-            )
-        }
-
-      _ ->
-        %{"id" => id, "deleted" => true}
     end
   end
 

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -110,6 +110,7 @@ def render("user.json", %{user: user}) do
       "endpoints" => endpoints,
       "attachment" => fields,
       "tag" => emoji_tags,
+      # Note: key name is indeed "discoverable" (not an error)
       "discoverable" => user.is_discoverable,
       "capabilities" => capabilities
     }

lib/pleroma/web/activity_pub/visibility.ex

@@ -17,7 +17,19 @@ def is_public?(%Object{data: data}), do: is_public?(data)
   def is_public?(%Activity{data: %{"type" => "Move"}}), do: true
   def is_public?(%Activity{data: data}), do: is_public?(data)
   def is_public?(%{"directMessage" => true}), do: false
-  def is_public?(data), do: Utils.label_in_message?(Pleroma.Constants.as_public(), data)
+
+  def is_public?(data) do
+    Utils.label_in_message?(Pleroma.Constants.as_public(), data) or
+      Utils.label_in_message?(Pleroma.Constants.as_local_public(), data)
+  end
+
+  def is_local_public?(%Object{data: data}), do: is_local_public?(data)
+  def is_local_public?(%Activity{data: data}), do: is_local_public?(data)
+
+  def is_local_public?(data) do
+    Utils.label_in_message?(Pleroma.Constants.as_local_public(), data) and
+      not Utils.label_in_message?(Pleroma.Constants.as_public(), data)
+  end
 
   def is_private?(activity) do
     with false <- is_public?(activity),
@@ -114,6 +126,9 @@ def get_visibility(object) do
       Pleroma.Constants.as_public() in cc ->
         "unlisted"
 
+      Pleroma.Constants.as_local_public() in to ->
+        "local"
+
       # this should use the sql for the object's activity
       Enum.any?(to, &String.contains?(&1, "/followers")) ->
         "private"

lib/pleroma/web/admin_api/controllers/frontend_controller.ex

@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.FrontendController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Config
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+  plug(OAuthScopesPlug, %{scopes: ["write"], admin: true} when action == :install)
+  plug(OAuthScopesPlug, %{scopes: ["read"], admin: true} when action == :index)
+  action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.FrontendOperation
+
+  def index(conn, _params) do
+    installed = installed()
+
+    frontends =
+      [:frontends, :available]
+      |> Config.get([])
+      |> Enum.map(fn {name, desc} ->
+        Map.put(desc, "installed", name in installed)
+      end)
+
+    render(conn, "index.json", frontends: frontends)
+  end
+
+  def install(%{body_params: params} = conn, _params) do
+    with :ok <- Pleroma.Frontend.install(params.name, Map.delete(params, :name)) do
+      index(conn, %{})
+    end
+  end
+
+  defp installed do
+    File.ls!(Pleroma.Frontend.dir())
+  end
+end

lib/pleroma/web/admin_api/views/frontend_view.ex

@@ -0,0 +1,21 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.FrontendView do
+  use Pleroma.Web, :view
+
+  def render("index.json", %{frontends: frontends}) do
+    render_many(frontends, __MODULE__, "show.json")
+  end
+
+  def render("show.json", %{frontend: frontend}) do
+    %{
+      name: frontend["name"],
+      git: frontend["git"],
+      build_url: frontend["build_url"],
+      ref: frontend["ref"],
+      installed: frontend["installed"]
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/account_operation.ex

@@ -139,6 +139,12 @@ def statuses_operation do
             :query,
             %Schema{type: :array, items: VisibilityScope},
             "Exclude visibilities"
+          ),
+          Operation.parameter(
+            :with_muted,
+            :query,
+            BooleanLike,
+            "Include reactions from muted acccounts."
           )
         ] ++ pagination_params(),
       responses: %{
@@ -262,6 +268,12 @@ def mute_operation do
           :query,
           %Schema{allOf: [BooleanLike], default: true},
           "Mute notifications in addition to statuses? Defaults to `true`."
+        ),
+        Operation.parameter(
+          :expires_in,
+          :query,
+          %Schema{type: :integer, default: 0},
+          "Expire the mute in `expires_in` seconds. Default 0 for infinity"
         )
       ],
       responses: %{
@@ -612,7 +624,7 @@ defp update_credentials_request do
           allOf: [BooleanLike],
           nullable: true,
           description:
-            "Discovery of this account in search results and other services is allowed."
+            "Discovery (listing, indexing) of this account by external services (search bots etc.) is allowed."
         },
         actor_type: ActorType
       },
@@ -723,10 +735,17 @@ defp mute_request do
           nullable: true,
           description: "Mute notifications in addition to statuses? Defaults to true.",
           default: true
+        },
+        expires_in: %Schema{
+          type: :integer,
+          nullable: true,
+          description: "Expire the mute in `expires_in` seconds. Default 0 for infinity",
+          default: 0
         }
       },
       example: %{
-        "notifications" => true
+        "notifications" => true,
+        "expires_in" => 86_400
       }
     }
   end

lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex

@@ -0,0 +1,85 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.FrontendOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def index_operation do
+    %Operation{
+      tags: ["Admin", "Reports"],
+      summary: "Get a list of available frontends",
+      operationId: "AdminAPI.FrontendController.index",
+      security: [%{"oAuth" => ["read"]}],
+      responses: %{
+        200 => Operation.response("Response", "application/json", list_of_frontends()),
+        403 => Operation.response("Forbidden", "application/json", ApiError)
+      }
+    }
+  end
+
+  def install_operation do
+    %Operation{
+      tags: ["Admin", "Reports"],
+      summary: "Install a frontend",
+      operationId: "AdminAPI.FrontendController.install",
+      security: [%{"oAuth" => ["read"]}],
+      requestBody: request_body("Parameters", install_request(), required: true),
+      responses: %{
+        200 => Operation.response("Response", "application/json", list_of_frontends()),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        400 => Operation.response("Error", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp list_of_frontends do
+    %Schema{
+      type: :array,
+      items: %Schema{
+        type: :object,
+        properties: %{
+          name: %Schema{type: :string},
+          git: %Schema{type: :string, format: :uri, nullable: true},
+          build_url: %Schema{type: :string, format: :uri, nullable: true},
+          ref: %Schema{type: :string},
+          installed: %Schema{type: :boolean}
+        }
+      }
+    }
+  end
+
+  defp install_request do
+    %Schema{
+      title: "FrontendInstallRequest",
+      type: :object,
+      required: [:name],
+      properties: %{
+        name: %Schema{
+          type: :string
+        },
+        ref: %Schema{
+          type: :string
+        },
+        file: %Schema{
+          type: :string
+        },
+        build_url: %Schema{
+          type: :string
+        },
+        build_dir: %Schema{
+          type: :string
+        }
+      }
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/emoji_reaction_operation.ex

@@ -24,6 +24,12 @@ def index_operation do
         Operation.parameter(:id, :path, FlakeID, "Status ID", required: true),
         Operation.parameter(:emoji, :path, :string, "Filter by a single unicode emoji",
           required: nil
+        ),
+        Operation.parameter(
+          :with_muted,
+          :query,
+          :boolean,
+          "Include reactions from muted acccounts."
         )
       ],
       security: [%{"oAuth" => ["read:statuses"]}],

lib/pleroma/web/api_spec/operations/notification_operation.ex

@@ -193,6 +193,7 @@ defp notification_type do
         "mention",
         "pleroma:emoji_reaction",
         "pleroma:chat_mention",
+        "pleroma:report",
         "move",
         "follow_request"
       ],
@@ -206,6 +207,8 @@ defp notification_type do
       - `poll` - A poll you have voted in or created has ended
       - `move` - Someone moved their account
       - `pleroma:emoji_reaction` - Someone reacted with emoji to your status
+      - `pleroma:chat_mention` - Someone mentioned you in a chat message
+      - `pleroma:report` - Someone was reported
       """
     }
   end

lib/pleroma/web/api_spec/operations/pleroma_instances_operation.ex

@@ -0,0 +1,40 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.PleromaInstancesOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def show_operation do
+    %Operation{
+      tags: ["PleromaInstances"],
+      summary: "Instances federation status",
+      description: "Information about instances deemed unreachable by the server",
+      operationId: "PleromaInstances.show",
+      responses: %{
+        200 => Operation.response("PleromaInstances", "application/json", pleroma_instances())
+      }
+    }
+  end
+
+  def pleroma_instances do
+    %Schema{
+      type: :object,
+      properties: %{
+        unreachable: %Schema{
+          type: :object,
+          properties: %{hostname: %Schema{type: :string, format: :"date-time"}}
+        }
+      },
+      example: %{
+        "unreachable" => %{"consistently-unreachable.name" => "2020-10-14 22:07:58.216473"}
+      }
+    }
+  end
+end

lib/pleroma/web/api_spec/operations/status_operation.ex

@@ -31,6 +31,12 @@ def index_operation do
           :query,
           %Schema{type: :array, items: FlakeID},
           "Array of status IDs"
+        ),
+        Operation.parameter(
+          :with_muted,
+          :query,
+          BooleanLike,
+          "Include reactions from muted acccounts."
         )
       ],
       operationId: "StatusController.index",
@@ -67,7 +73,15 @@ def show_operation do
       description: "View information about a status",
       operationId: "StatusController.show",
       security: [%{"oAuth" => ["read:statuses"]}],
-      parameters: [id_param()],
+      parameters: [
+        id_param(),
+        Operation.parameter(
+          :with_muted,
+          :query,
+          BooleanLike,
+          "Include reactions from muted acccounts."
+        )
+      ],
       responses: %{
         200 => status_response(),
         404 => Operation.response("Not Found", "application/json", ApiError)
@@ -223,7 +237,27 @@ def mute_conversation_operation do
       security: [%{"oAuth" => ["write:mutes"]}],
       description: "Do not receive notifications for the thread that this status is part of.",
       operationId: "StatusController.mute_conversation",
-      parameters: [id_param()],
+      requestBody:
+        request_body("Parameters", %Schema{
+          type: :object,
+          properties: %{
+            expires_in: %Schema{
+              type: :integer,
+              nullable: true,
+              description: "Expire the mute in `expires_in` seconds. Default 0 for infinity",
+              default: 0
+            }
+          }
+        }),
+      parameters: [
+        id_param(),
+        Operation.parameter(
+          :expires_in,
+          :query,
+          %Schema{type: :integer, default: 0},
+          "Expire the mute in `expires_in` seconds. Default 0 for infinity"
+        )
+      ],
       responses: %{
         200 => status_response(),
         400 => Operation.response("Error", "application/json", ApiError)

lib/pleroma/web/api_spec/operations/subscription_operation.ex

@@ -146,6 +146,11 @@ defp create_request do
                   allOf: [BooleanLike],
                   nullable: true,
                   description: "Receive chat notifications?"
+                },
+                "pleroma:emoji_reaction": %Schema{
+                  allOf: [BooleanLike],
+                  nullable: true,
+                  description: "Receive emoji reaction notifications?"
                 }
               }
             }
@@ -210,6 +215,16 @@ defp update_request do
                   allOf: [BooleanLike],
                   nullable: true,
                   description: "Receive poll notifications?"
+                },
+                "pleroma:chat_mention": %Schema{
+                  allOf: [BooleanLike],
+                  nullable: true,
+                  description: "Receive chat notifications?"
+                },
+                "pleroma:emoji_reaction": %Schema{
+                  allOf: [BooleanLike],
+                  nullable: true,
+                  description: "Receive emoji reaction notifications?"
                 }
               }
             }

lib/pleroma/web/api_spec/operations/timeline_operation.ex

@@ -59,6 +59,7 @@ def public_operation do
       security: [%{"oAuth" => ["read:statuses"]}],
       parameters: [
         local_param(),
+        instance_param(),
         only_media_param(),
         with_muted_param(),
         exclude_visibilities_param(),
@@ -158,6 +159,15 @@ defp local_param do
     )
   end
 
+  defp instance_param do
+    Operation.parameter(
+      :instance,
+      :query,
+      %Schema{type: :string},
+      "Show only statuses from the given domain"
+    )
+  end
+
   defp with_muted_param do
     Operation.parameter(:with_muted, :query, BooleanLike, "Include activities by muted users")
   end

lib/pleroma/web/api_spec/schemas/account.ex

@@ -127,7 +127,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
               discoverable: %Schema{
                 type: :boolean,
                 description:
-                  "whether the user allows discovery of the account in search results and other services."
+                  "whether the user allows indexing / listing of the account by external services (search engines etc.)."
               },
               no_rich_text: %Schema{
                 type: :boolean,

lib/pleroma/web/api_spec/schemas/visibility_scope.ex

@@ -9,6 +9,6 @@ defmodule Pleroma.Web.ApiSpec.Schemas.VisibilityScope do
     title: "VisibilityScope",
     description: "Status visibility",
     type: :string,
-    enum: ["public", "unlisted", "private", "direct", "list"]
+    enum: ["public", "unlisted", "local", "private", "direct", "list"]
   })
 end

lib/pleroma/web/common_api.ex

@@ -15,6 +15,7 @@ defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
+  alias Pleroma.Web.CommonAPI.ActivityDraft
 
   import Pleroma.Web.Gettext
   import Pleroma.Web.CommonAPI.Utils
@@ -358,7 +359,7 @@ def public_announce?(object, _) do
   def get_visibility(_, _, %Participation{}), do: {"direct", "direct"}
 
   def get_visibility(%{visibility: visibility}, in_reply_to, _)
-      when visibility in ~w{public unlisted private direct},
+      when visibility in ~w{public local unlisted private direct},
       do: {visibility, get_replied_to_visibility(in_reply_to)}
 
   def get_visibility(%{visibility: "list:" <> list_id}, in_reply_to, _) do
@@ -399,31 +400,13 @@ def check_expiry_date(expiry_str) do
   end
 
   def listen(user, data) do
-    visibility = Map.get(data, :visibility, "public")
+    with {:ok, draft} <- ActivityDraft.listen(user, data) do
-
+      ActivityPub.listen(draft.changes)
-    with {to, cc} <- get_to_and_cc(user, [], nil, visibility, nil),
-         listen_data <-
-           data
-           |> Map.take([:album, :artist, :title, :length])
-           |> Map.new(fn {key, value} -> {to_string(key), value} end)
-           |> Map.put("type", "Audio")
-           |> Map.put("to", to)
-           |> Map.put("cc", cc)
-           |> Map.put("actor", user.ap_id),
-         {:ok, activity} <-
-           ActivityPub.listen(%{
-             actor: user,
-             to: to,
-             object: listen_data,
-             context: Utils.generate_context_id(),
-             additional: %{"cc" => cc}
-           }) do
-      {:ok, activity}
     end
   end
 
   def post(user, %{status: _} = data) do
-    with {:ok, draft} <- Pleroma.Web.CommonAPI.ActivityDraft.create(user, data) do
+    with {:ok, draft} <- ActivityDraft.create(user, data) do
       ActivityPub.create(draft.changes, draft.preview?)
     end
   end
@@ -454,20 +437,46 @@ def unpin(id, user) do
     end
   end
 
-  def add_mute(user, activity) do
+  def add_mute(user, activity, params \\ %{}) do
+    expires_in = Map.get(params, :expires_in, 0)
+
     with {:ok, _} <- ThreadMute.add_mute(user.id, activity.data["context"]),
          _ <- Pleroma.Notification.mark_context_as_read(user, activity.data["context"]) do
+      if expires_in > 0 do
+        Pleroma.Workers.MuteExpireWorker.enqueue(
+          "unmute_conversation",
+          %{"user_id" => user.id, "activity_id" => activity.id},
+          schedule_in: expires_in
+        )
+      end
+
       {:ok, activity}
     else
       {:error, _} -> {:error, dgettext("errors", "conversation is already muted")}
     end
   end
 
-  def remove_mute(user, activity) do
+  def remove_mute(%User{} = user, %Activity{} = activity) do
     ThreadMute.remove_mute(user.id, activity.data["context"])
     {:ok, activity}
   end
 
+  def remove_mute(user_id, activity_id) do
+    with {:user, %User{} = user} <- {:user, User.get_by_id(user_id)},
+         {:activity, %Activity{} = activity} <- {:activity, Activity.get_by_id(activity_id)} do
+      remove_mute(user, activity)
+    else
+      {what, result} = error ->
+        Logger.warn(
+          "CommonAPI.remove_mute/2 failed. #{what}: #{result}, user_id: #{user_id}, activity_id: #{
+            activity_id
+          }"
+        )
+
+        {:error, error}
+    end
+  end
+
   def thread_muted?(%User{id: user_id}, %{data: %{"context" => context}})
       when is_binary(context) do
     ThreadMute.exists?(user_id, context)

lib/pleroma/web/common_api/activity_draft.ex

@@ -22,7 +22,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
             in_reply_to_conversation: nil,
             visibility: nil,
             expires_at: nil,
-            poll: nil,
+            extra: nil,
             emoji: %{},
             content_html: nil,
             mentions: [],
@@ -35,9 +35,14 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
             preview?: false,
             changes: %{}
 
-  def create(user, params) do
+  def new(user, params) do
     %__MODULE__{user: user}
     |> put_params(params)
+  end
+
+  def create(user, params) do
+    user
+    |> new(params)
     |> status()
     |> summary()
     |> with_valid(&attachments/1)
@@ -57,6 +62,30 @@ def create(user, params) do
     |> validate()
   end
 
+  def listen(user, params) do
+    user
+    |> new(params)
+    |> visibility()
+    |> to_and_cc()
+    |> context()
+    |> listen_object()
+    |> with_valid(&changes/1)
+    |> validate()
+  end
+
+  defp listen_object(draft) do
+    object =
+      draft.params
+      |> Map.take([:album, :artist, :title, :length])
+      |> Map.new(fn {key, value} -> {to_string(key), value} end)
+      |> Map.put("type", "Audio")
+      |> Map.put("to", draft.to)
+      |> Map.put("cc", draft.cc)
+      |> Map.put("actor", draft.user.ap_id)
+
+    %__MODULE__{draft | object: object}
+  end
+
   defp put_params(draft, params) do
     params = Map.put_new(params, :in_reply_to_status_id, params[:in_reply_to_id])
     %__MODULE__{draft | params: params}
@@ -121,7 +150,7 @@ defp expires_at(draft) do
   defp poll(draft) do
     case Utils.make_poll_data(draft.params) do
       {:ok, {poll, poll_emoji}} ->
-        %__MODULE__{draft | poll: poll, emoji: Map.merge(draft.emoji, poll_emoji)}
+        %__MODULE__{draft | extra: poll, emoji: Map.merge(draft.emoji, poll_emoji)}
 
       {:error, message} ->
         add_error(draft, message)
@@ -129,32 +158,18 @@ defp poll(draft) do
   end
 
   defp content(draft) do
-    {content_html, mentions, tags} =
+    {content_html, mentioned_users, tags} = Utils.make_content_html(draft)
-      Utils.make_content_html(
+
-        draft.status,
+    mentions =
-        draft.attachments,
+      mentioned_users
-        draft.params,
+      |> Enum.map(fn {_, mentioned_user} -> mentioned_user.ap_id end)
-        draft.visibility
+      |> Utils.get_addressed_users(draft.params[:to])
-      )
 
     %__MODULE__{draft | content_html: content_html, mentions: mentions, tags: tags}
   end
 
   defp to_and_cc(draft) do
-    addressed_users =
+    {to, cc} = Utils.get_to_and_cc(draft)
-      draft.mentions
-      |> Enum.map(fn {_, mentioned_user} -> mentioned_user.ap_id end)
-      |> Utils.get_addressed_users(draft.params[:to])
-
-    {to, cc} =
-      Utils.get_to_and_cc(
-        draft.user,
-        addressed_users,
-        draft.in_reply_to,
-        draft.visibility,
-        draft.in_reply_to_conversation
-      )
-
     %__MODULE__{draft | to: to, cc: cc}
   end
 
@@ -172,19 +187,7 @@ defp object(draft) do
     emoji = Map.merge(Pleroma.Emoji.Formatter.get_emoji_map(draft.full_payload), draft.emoji)
 
     object =
-      Utils.make_note_data(
+      Utils.make_note_data(draft)
-        draft.user.ap_id,
-        draft.to,
-        draft.context,
-        draft.content_html,
-        draft.attachments,
-        draft.in_reply_to,
-        draft.tags,
-        draft.summary,
-        draft.cc,
-        draft.sensitive,
-        draft.poll
-      )
       |> Map.put("emoji", emoji)
       |> Map.put("source", draft.status)
 

lib/pleroma/web/common_api/utils.ex

@@ -16,6 +16,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
+  alias Pleroma.Web.CommonAPI.ActivityDraft
   alias Pleroma.Web.MediaProxy
   alias Pleroma.Web.Plugs.AuthenticationPlug
 
@@ -50,67 +51,62 @@ def attachments_from_ids_descs(ids, descs_str) do
     {_, descs} = Jason.decode(descs_str)
 
     Enum.map(ids, fn media_id ->
-      case Repo.get(Object, media_id) do
+      with %Object{data: data} <- Repo.get(Object, media_id) do
-        %Object{data: data} ->
+        Map.put(data, "name", descs[media_id])
-          Map.put(data, "name", descs[media_id])
-
-        _ ->
-          nil
       end
     end)
     |> Enum.reject(&is_nil/1)
   end
 
-  @spec get_to_and_cc(
+  @spec get_to_and_cc(ActivityDraft.t()) :: {list(String.t()), list(String.t())}
-          User.t(),
-          list(String.t()),
-          Activity.t() | nil,
-          String.t(),
-          Participation.t() | nil
-        ) :: {list(String.t()), list(String.t())}
 
-  def get_to_and_cc(_, _, _, _, %Participation{} = participation) do
+  def get_to_and_cc(%{in_reply_to_conversation: %Participation{} = participation}) do
     participation = Repo.preload(participation, :recipients)
     {Enum.map(participation.recipients, & &1.ap_id), []}
   end
 
-  def get_to_and_cc(user, mentioned_users, inReplyTo, "public", _) do
+  def get_to_and_cc(%{visibility: visibility} = draft) when visibility in ["public", "local"] do
-    to = [Pleroma.Constants.as_public() | mentioned_users]
+    to =
-    cc = [user.follower_address]
+      case visibility do
+        "public" -> [Pleroma.Constants.as_public() | draft.mentions]
+        "local" -> [Pleroma.Constants.as_local_public() | draft.mentions]
+      end
 
-    if inReplyTo do
+    cc = [draft.user.follower_address]
-      {Enum.uniq([inReplyTo.data["actor"] | to]), cc}
+
+    if draft.in_reply_to do
+      {Enum.uniq([draft.in_reply_to.data["actor"] | to]), cc}
     else
       {to, cc}
     end
   end
 
-  def get_to_and_cc(user, mentioned_users, inReplyTo, "unlisted", _) do
+  def get_to_and_cc(%{visibility: "unlisted"} = draft) do
-    to = [user.follower_address | mentioned_users]
+    to = [draft.user.follower_address | draft.mentions]
     cc = [Pleroma.Constants.as_public()]
 
-    if inReplyTo do
+    if draft.in_reply_to do
-      {Enum.uniq([inReplyTo.data["actor"] | to]), cc}
+      {Enum.uniq([draft.in_reply_to.data["actor"] | to]), cc}
     else
       {to, cc}
     end
   end
 
-  def get_to_and_cc(user, mentioned_users, inReplyTo, "private", _) do
+  def get_to_and_cc(%{visibility: "private"} = draft) do
-    {to, cc} = get_to_and_cc(user, mentioned_users, inReplyTo, "direct", nil)
+    {to, cc} = get_to_and_cc(struct(draft, visibility: "direct"))
-    {[user.follower_address | to], cc}
+    {[draft.user.follower_address | to], cc}
   end
 
-  def get_to_and_cc(_user, mentioned_users, inReplyTo, "direct", _) do
+  def get_to_and_cc(%{visibility: "direct"} = draft) do
     # If the OP is a DM already, add the implicit actor.
-    if inReplyTo && Visibility.is_direct?(inReplyTo) do
+    if draft.in_reply_to && Visibility.is_direct?(draft.in_reply_to) do
-      {Enum.uniq([inReplyTo.data["actor"] | mentioned_users]), []}
+      {Enum.uniq([draft.in_reply_to.data["actor"] | draft.mentions]), []}
     else
-      {mentioned_users, []}
+      {draft.mentions, []}
     end
   end
 
-  def get_to_and_cc(_user, mentions, _inReplyTo, {:list, _}, _), do: {mentions, []}
+  def get_to_and_cc(%{visibility: {:list, _}, mentions: mentions}), do: {mentions, []}
 
   def get_addressed_users(_, to) when is_list(to) do
     User.get_ap_ids_by_nicknames(to)
@@ -203,30 +199,25 @@ defp validate_poll_expiration(expires_in, %{min_expiration: min, max_expiration:
     end
   end
 
-  def make_content_html(
+  def make_content_html(%ActivityDraft{} = draft) do
-        status,
-        attachments,
-        data,
-        visibility
-      ) do
     attachment_links =
-      data
+      draft.params
       |> Map.get("attachment_links", Config.get([:instance, :attachment_links]))
       |> truthy_param?()
 
-    content_type = get_content_type(data[:content_type])
+    content_type = get_content_type(draft.params[:content_type])
 
     options =
-      if visibility == "direct" && Config.get([:instance, :safe_dm_mentions]) do
+      if draft.visibility == "direct" && Config.get([:instance, :safe_dm_mentions]) do
         [safe_mention: true]
       else
         []
       end
 
-    status
+    draft.status
     |> format_input(content_type, options)
-    |> maybe_add_attachments(attachments, attachment_links)
+    |> maybe_add_attachments(draft.attachments, attachment_links)
-    |> maybe_add_nsfw_tag(data)
+    |> maybe_add_nsfw_tag(draft.params)
   end
 
   defp get_content_type(content_type) do
@@ -308,33 +299,21 @@ def format_input(text, "text/markdown", options) do
     |> Formatter.html_escape("text/html")
   end
 
-  def make_note_data(
+  def make_note_data(%ActivityDraft{} = draft) do
-        actor,
-        to,
-        context,
-        content_html,
-        attachments,
-        in_reply_to,
-        tags,
-        summary \\ nil,
-        cc \\ [],
-        sensitive \\ false,
-        extra_params \\ %{}
-      ) do
     %{
       "type" => "Note",
-      "to" => to,
+      "to" => draft.to,
-      "cc" => cc,
+      "cc" => draft.cc,
-      "content" => content_html,
+      "content" => draft.content_html,
-      "summary" => summary,
+      "summary" => draft.summary,
-      "sensitive" => truthy_param?(sensitive),
+      "sensitive" => draft.sensitive,
-      "context" => context,
+      "context" => draft.context,
-      "attachment" => attachments,
+      "attachment" => draft.attachments,
-      "actor" => actor,
+      "actor" => draft.user.ap_id,
-      "tag" => Keyword.values(tags) |> Enum.uniq()
+      "tag" => Keyword.values(draft.tags) |> Enum.uniq()
     }
-    |> add_in_reply_to(in_reply_to)
+    |> add_in_reply_to(draft.in_reply_to)
-    |> Map.merge(extra_params)
+    |> Map.merge(draft.extra)
   end
 
   defp add_in_reply_to(object, nil), do: object

lib/pleroma/web/fallback/redirect_controller.ex

@@ -37,10 +37,11 @@ def redirector_with_meta(conn, params) do
 
     tags = build_tags(conn, params)
     preloads = preload_data(conn, params)
+    title = "<title>#{Pleroma.Config.get([:instance, :name])}</title>"
 
     response =
       index_content
-      |> String.replace("<!--server-generated-meta-->", tags <> preloads)
+      |> String.replace("<!--server-generated-meta-->", tags <> preloads <> title)
 
     conn
     |> put_resp_content_type("text/html")
@@ -54,10 +55,11 @@ def redirector_with_preload(conn, %{"path" => ["pleroma", "admin"]}) do
   def redirector_with_preload(conn, params) do
     {:ok, index_content} = File.read(index_file_path())
     preloads = preload_data(conn, params)
+    title = "<title>#{Pleroma.Config.get([:instance, :name])}</title>"
 
     response =
       index_content
-      |> String.replace("<!--server-generated-meta-->", preloads)
+      |> String.replace("<!--server-generated-meta-->", preloads <> title)
 
     conn
     |> put_resp_content_type("text/html")

lib/pleroma/web/fed_sockets.ex

@@ -1,185 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.FedSockets do
-  @moduledoc """
-  This documents the FedSockets framework. A framework for federating
-  ActivityPub objects between servers via persistant WebSocket connections.
-
-  FedSockets allow servers to authenticate on first contact and maintain that
-  connection, eliminating the need to authenticate every time data needs to be shared.
-
-  ## Protocol
-  FedSockets currently support 2 types of data transfer:
-    * `publish` method which doesn't require a response
-    * `fetch` method requires a response be sent
-
-    ### Publish
-    The publish operation sends a json encoded map of the shape:
-      %{action: :publish, data: json}
-    and accepts (but does not require) a reply of form:
-      %{"action" => "publish_reply"}
-
-    The outgoing params represent
-      * data: ActivityPub object encoded into json
-
-
-    ### Fetch
-    The fetch operation sends a json encoded map of the shape:
-      %{action: :fetch, data: id, uuid: fetch_uuid}
-    and requires a reply of form:
-      %{"action" => "fetch_reply", "uuid" => uuid, "data" => data}
-
-    The outgoing params represent
-      * id: an ActivityPub object URI
-      * uuid: a unique uuid generated by the sender
-
-    The reply params represent
-      * data: an ActivityPub object encoded into json
-      * uuid: the uuid sent along with the fetch request
-
-  ## Examples
-  Clients of FedSocket transfers shouldn't need to use any of the functions outside of this module.
-
-  A typical publish operation can be performed through the following code, and a fetch operation in a similar manner.
-
-    case FedSockets.get_or_create_fed_socket(inbox) do
-      {:ok, fedsocket} ->
-        FedSockets.publish(fedsocket, json)
-
-      _ ->
-        alternative_publish(inbox, actor, json, params)
-    end
-
-  ## Configuration
-  FedSockets have the following config settings
-
-  config :pleroma, :fed_sockets,
-  enabled: true,
-  ping_interval: :timer.seconds(15),
-  connection_duration: :timer.hours(1),
-  rejection_duration: :timer.hours(1),
-  fed_socket_fetches: [
-    default: 12_000,
-    interval: 3_000,
-    lazy: false
-  ]
-    * enabled - turn FedSockets on or off with this flag. Can be toggled at runtime.
-    * connection_duration - How long a FedSocket can sit idle before it's culled.
-    * rejection_duration - After failing to make a FedSocket connection a host will be excluded
-    from further connections for this amount of time
-    * fed_socket_fetches - Use these parameters to pass options to the Cachex queue backing the FetchRegistry
-    * fed_socket_rejections - Use these parameters to pass options to the Cachex queue backing the FedRegistry
-
-    Cachex options are
-      * default: the minimum amount of time a fetch can wait before it times out.
-      * interval: the interval between checks for timed out entries. This plus the default represent the maximum time allowed
-      * lazy: leave at false for consistant and fast lookups, set to true for stricter timeout enforcement
-
-  """
-  require Logger
-
-  alias Pleroma.Web.FedSockets.FedRegistry
-  alias Pleroma.Web.FedSockets.FedSocket
-  alias Pleroma.Web.FedSockets.SocketInfo
-
-  @doc """
-  returns a FedSocket for the given origin. Will reuse an existing one or create a new one.
-
-  address is expected to be a fully formed URL such as:
-  "http://www.example.com" or "http://www.example.com:8080"
-
-  It can and usually does include additional path parameters,
-  but these are ignored as the FedSockets are organized by host and port info alone.
-  """
-  def get_or_create_fed_socket(address) do
-    with {:cache, {:error, :missing}} <- {:cache, get_fed_socket(address)},
-         {:connect, {:ok, _pid}} <- {:connect, FedSocket.connect_to_host(address)},
-         {:cache, {:ok, fed_socket}} <- {:cache, get_fed_socket(address)} do
-      Logger.debug("fedsocket created for - #{inspect(address)}")
-      {:ok, fed_socket}
-    else
-      {:cache, {:ok, socket}} ->
-        Logger.debug("fedsocket found in cache - #{inspect(address)}")
-        {:ok, socket}
-
-      {:cache, {:error, :rejected} = e} ->
-        e
-
-      {:connect, {:error, _host}} ->
-        Logger.debug("set host rejected for - #{inspect(address)}")
-        FedRegistry.set_host_rejected(address)
-        {:error, :rejected}
-
-      {_, {:error, :disabled}} ->
-        {:error, :disabled}
-
-      {_, {:error, reason}} ->
-        Logger.warn("get_or_create_fed_socket error - #{inspect(reason)}")
-        {:error, reason}
-    end
-  end
-
-  @doc """
-  returns a FedSocket for the given origin. Will not create a new FedSocket if one does not exist.
-
-  address is expected to be a fully formed URL such as:
-    "http://www.example.com" or "http://www.example.com:8080"
-  """
-  def get_fed_socket(address) do
-    origin = SocketInfo.origin(address)
-
-    with {:config, true} <- {:config, Pleroma.Config.get([:fed_sockets, :enabled], false)},
-         {:ok, socket} <- FedRegistry.get_fed_socket(origin) do
-      {:ok, socket}
-    else
-      {:config, _} ->
-        {:error, :disabled}
-
-      {:error, :rejected} ->
-        Logger.debug("FedSocket previously rejected - #{inspect(origin)}")
-        {:error, :rejected}
-
-      {:error, reason} ->
-        {:error, reason}
-    end
-  end
-
-  @doc """
-  Sends the supplied data via the publish protocol.
-  It will not block waiting for a reply.
-  Returns :ok but this is not an indication of a successful transfer.
-
-  the data is expected to be JSON encoded binary data.
-  """
-  def publish(%SocketInfo{} = fed_socket, json) do
-    FedSocket.publish(fed_socket, json)
-  end
-
-  @doc """
-  Sends the supplied data via the fetch protocol.
-  It will block waiting for a reply or timeout.
-
-  Returns {:ok, object} where object is the requested object (or nil)
-          {:error, :timeout} in the event the message was not responded to
-
-  the id is expected to be the URI of an ActivityPub object.
-  """
-  def fetch(%SocketInfo{} = fed_socket, id) do
-    FedSocket.fetch(fed_socket, id)
-  end
-
-  @doc """
-  Disconnect all and restart FedSockets.
-  This is mainly used in development and testing but could be useful in production.
-  """
-  def reset do
-    FedRegistry
-    |> Process.whereis()
-    |> Process.exit(:testing)
-  end
-
-  def uri_for_origin(origin),
-    do: "ws://#{origin}/api/fedsocket/v1"
-end

lib/pleroma/web/fed_sockets/fed_registry.ex

@@ -1,185 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.FedSockets.FedRegistry do
-  @moduledoc """
-  The FedRegistry stores the active FedSockets for quick retrieval.
-
-  The storage and retrieval portion of the FedRegistry is done in process through
-  elixir's `Registry` module for speed and its ability to monitor for terminated processes.
-
-  Dropped connections will be caught by `Registry` and deleted. Since the next
-  message will initiate a new connection there is no reason to try and reconnect at that point.
-
-  Normally outside modules should have no need to call or use the FedRegistry themselves.
-  """
-
-  alias Pleroma.Web.FedSockets.FedSocket
-  alias Pleroma.Web.FedSockets.SocketInfo
-
-  require Logger
-
-  @default_rejection_duration 15 * 60 * 1000
-  @rejections :fed_socket_rejections
-
-  @doc """
-  Retrieves a FedSocket from the Registry given it's origin.
-
-  The origin is expected to be a string identifying the endpoint "example.com" or "example2.com:8080"
-
-  Will return:
-    * {:ok, fed_socket} for working FedSockets
-    * {:error, :rejected} for origins that have been tried and refused within the rejection duration interval
-    * {:error, some_reason} usually :missing for unknown origins
-  """
-  def get_fed_socket(origin) do
-    case get_registry_data(origin) do
-      {:error, reason} ->
-        {:error, reason}
-
-      {:ok, %{state: :connected} = socket_info} ->
-        {:ok, socket_info}
-    end
-  end
-
-  @doc """
-  Adds a connected FedSocket to the Registry.
-
-  Always returns {:ok, fed_socket}
-  """
-  def add_fed_socket(origin, pid \\ nil) do
-    origin
-    |> SocketInfo.build(pid)
-    |> SocketInfo.connect()
-    |> add_socket_info
-  end
-
-  defp add_socket_info(%{origin: origin, state: :connected} = socket_info) do
-    case Registry.register(FedSockets.Registry, origin, socket_info) do
-      {:ok, _owner} ->
-        clear_prior_rejection(origin)
-        Logger.debug("fedsocket added: #{inspect(origin)}")
-
-        {:ok, socket_info}
-
-      {:error, {:already_registered, _pid}} ->
-        FedSocket.close(socket_info)
-        existing_socket_info = Registry.lookup(FedSockets.Registry, origin)
-
-        {:ok, existing_socket_info}
-
-      _ ->
-        {:error, :error_adding_socket}
-    end
-  end
-
-  @doc """
-  Mark this origin as having rejected a connection attempt.
-  This will keep it from getting additional connection attempts
-  for a period of time specified in the config.
-
-  Always returns {:ok, new_reg_data}
-  """
-  def set_host_rejected(uri) do
-    new_reg_data =
-      uri
-      |> SocketInfo.origin()
-      |> get_or_create_registry_data()
-      |> set_to_rejected()
-      |> save_registry_data()
-
-    {:ok, new_reg_data}
-  end
-
-  @doc """
-  Retrieves the FedRegistryData from the Registry given it's origin.
-
-  The origin is expected to be a string identifying the endpoint "example.com" or "example2.com:8080"
-
-  Will return:
-    * {:ok, fed_registry_data} for known origins
-    * {:error, :missing} for uniknown origins
-    * {:error, :cache_error} indicating some low level runtime issues
-  """
-  def get_registry_data(origin) do
-    case Registry.lookup(FedSockets.Registry, origin) do
-      [] ->
-        if is_rejected?(origin) do
-          Logger.debug("previously rejected fedsocket requested")
-          {:error, :rejected}
-        else
-          {:error, :missing}
-        end
-
-      [{_pid, %{state: :connected} = socket_info}] ->
-        {:ok, socket_info}
-
-      _ ->
-        {:error, :cache_error}
-    end
-  end
-
-  @doc """
-  Retrieves a map of all sockets from the Registry. The keys are the origins and the values are the corresponding SocketInfo
-  """
-  def list_all do
-    (list_all_connected() ++ list_all_rejected())
-    |> Enum.into(%{})
-  end
-
-  defp list_all_connected do
-    FedSockets.Registry
-    |> Registry.select([{{:"$1", :_, :"$3"}, [], [{{:"$1", :"$3"}}]}])
-  end
-
-  defp list_all_rejected do
-    {:ok, keys} = Cachex.keys(@rejections)
-
-    {:ok, registry_data} =
-      Cachex.execute(@rejections, fn worker ->
-        Enum.map(keys, fn k -> {k, Cachex.get!(worker, k)} end)
-      end)
-
-    registry_data
-  end
-
-  defp clear_prior_rejection(origin),
-    do: Cachex.del(@rejections, origin)
-
-  defp is_rejected?(origin) do
-    case Cachex.get(@rejections, origin) do
-      {:ok, nil} ->
-        false
-
-      {:ok, _} ->
-        true
-    end
-  end
-
-  defp get_or_create_registry_data(origin) do
-    case get_registry_data(origin) do
-      {:error, :missing} ->
-        %SocketInfo{origin: origin}
-
-      {:ok, socket_info} ->
-        socket_info
-    end
-  end
-
-  defp save_registry_data(%SocketInfo{origin: origin, state: :connected} = socket_info) do
-    {:ok, true} = Registry.update_value(FedSockets.Registry, origin, fn _ -> socket_info end)
-    socket_info
-  end
-
-  defp save_registry_data(%SocketInfo{origin: origin, state: :rejected} = socket_info) do
-    rejection_expiration =
-      Pleroma.Config.get([:fed_sockets, :rejection_duration], @default_rejection_duration)
-
-    {:ok, true} = Cachex.put(@rejections, origin, socket_info, ttl: rejection_expiration)
-    socket_info
-  end
-
-  defp set_to_rejected(%SocketInfo{} = socket_info),
-    do: %SocketInfo{socket_info | state: :rejected}
-end

lib/pleroma/web/fed_sockets/fed_socket.ex

@@ -1,137 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.FedSockets.FedSocket do
-  @moduledoc """
-  The FedSocket module abstracts the actions to be taken taken on connections regardless of
-  whether the connection started as inbound or outbound.
-
-
-  Normally outside modules will have no need to call the FedSocket module directly.
-  """
-
-  alias Pleroma.Object
-  alias Pleroma.Object.Containment
-  alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.ObjectView
-  alias Pleroma.Web.ActivityPub.UserView
-  alias Pleroma.Web.ActivityPub.Visibility
-  alias Pleroma.Web.FedSockets.FetchRegistry
-  alias Pleroma.Web.FedSockets.IngesterWorker
-  alias Pleroma.Web.FedSockets.OutgoingHandler
-  alias Pleroma.Web.FedSockets.SocketInfo
-
-  require Logger
-
-  @shake "61dd18f7-f1e6-49a4-939a-a749fcdc1103"
-
-  def connect_to_host(uri) do
-    case OutgoingHandler.start_link(uri) do
-      {:ok, pid} ->
-        {:ok, pid}
-
-      error ->
-        {:error, error}
-    end
-  end
-
-  def close(%SocketInfo{pid: socket_pid}),
-    do: Process.send(socket_pid, :close, [])
-
-  def publish(%SocketInfo{pid: socket_pid}, json) do
-    %{action: :publish, data: json}
-    |> Jason.encode!()
-    |> send_packet(socket_pid)
-  end
-
-  def fetch(%SocketInfo{pid: socket_pid}, id) do
-    fetch_uuid = FetchRegistry.register_fetch(id)
-
-    %{action: :fetch, data: id, uuid: fetch_uuid}
-    |> Jason.encode!()
-    |> send_packet(socket_pid)
-
-    wait_for_fetch_to_return(fetch_uuid, 0)
-  end
-
-  def receive_package(%SocketInfo{} = fed_socket, json) do
-    json
-    |> Jason.decode!()
-    |> process_package(fed_socket)
-  end
-
-  defp wait_for_fetch_to_return(uuid, cntr) do
-    case FetchRegistry.check_fetch(uuid) do
-      {:error, :waiting} ->
-        Process.sleep(:math.pow(cntr, 3) |> Kernel.trunc())
-        wait_for_fetch_to_return(uuid, cntr + 1)
-
-      {:error, :missing} ->
-        Logger.error("FedSocket fetch timed out - #{inspect(uuid)}")
-        {:error, :timeout}
-
-      {:ok, _fr} ->
-        FetchRegistry.pop_fetch(uuid)
-    end
-  end
-
-  defp process_package(%{"action" => "publish", "data" => data}, %{origin: origin} = _fed_socket) do
-    if Containment.contain_origin(origin, data) do
-      IngesterWorker.enqueue("ingest", %{"object" => data})
-    end
-
-    {:reply, %{"action" => "publish_reply", "status" => "processed"}}
-  end
-
-  defp process_package(%{"action" => "fetch_reply", "uuid" => uuid, "data" => data}, _fed_socket) do
-    FetchRegistry.register_fetch_received(uuid, data)
-    {:noreply, nil}
-  end
-
-  defp process_package(%{"action" => "fetch", "uuid" => uuid, "data" => ap_id}, _fed_socket) do
-    {:ok, data} = render_fetched_data(ap_id, uuid)
-    {:reply, data}
-  end
-
-  defp process_package(%{"action" => "publish_reply"}, _fed_socket) do
-    {:noreply, nil}
-  end
-
-  defp process_package(other, _fed_socket) do
-    Logger.warn("unknown json packages received #{inspect(other)}")
-    {:noreply, nil}
-  end
-
-  defp render_fetched_data(ap_id, uuid) do
-    {:ok,
-     %{
-       "action" => "fetch_reply",
-       "status" => "processed",
-       "uuid" => uuid,
-       "data" => represent_item(ap_id)
-     }}
-  end
-
-  defp represent_item(ap_id) do
-    case User.get_by_ap_id(ap_id) do
-      nil ->
-        object = Object.get_cached_by_ap_id(ap_id)
-
-        if Visibility.is_public?(object) do
-          Phoenix.View.render_to_string(ObjectView, "object.json", object: object)
-        else
-          nil
-        end
-
-      user ->
-        Phoenix.View.render_to_string(UserView, "user.json", user: user)
-    end
-  end
-
-  defp send_packet(data, socket_pid) do
-    Process.send(socket_pid, {:send, data}, [])
-  end
-
-  def shake, do: @shake
-end

lib/pleroma/web/fed_sockets/fetch_registry.ex

@@ -1,151 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.FedSockets.FetchRegistry do
-  @moduledoc """
-  The FetchRegistry acts as a broker for fetch requests and return values.
-  This allows calling processes to block while waiting for a reply.
-  It doesn't impose it's own process instead using `Cachex` to handle fetches in process, allowing
-  multi threaded processes to avoid bottlenecking.
-
-  Normally outside modules will have no need to call or use the FetchRegistry themselves.
-
-  The `Cachex` parameters can be controlled from the config. Since exact timeout intervals
-  aren't necessary the following settings are used by default:
-
-  config :pleroma, :fed_sockets,
-    fed_socket_fetches: [
-      default: 12_000,
-      interval: 3_000,
-      lazy: false
-    ]
-
-  """
-
-  defmodule FetchRegistryData do
-    defstruct uuid: nil,
-              sent_json: nil,
-              received_json: nil,
-              sent_at: nil,
-              received_at: nil
-  end
-
-  alias Ecto.UUID
-
-  require Logger
-
-  @fetches :fed_socket_fetches
-
-  @doc """
-  Registers a json request wth the FetchRegistry and returns the identifying UUID.
-  """
-  def register_fetch(json) do
-    %FetchRegistryData{uuid: uuid} =
-      json
-      |> new_registry_data
-      |> save_registry_data
-
-    uuid
-  end
-
-  @doc """
-  Reports on the status of a Fetch given the identifying UUID.
-
-  Will return
-    * {:ok, fetched_object} if a fetch has completed
-    * {:error, :waiting} if a fetch is still pending
-    * {:error, other_error} usually :missing to indicate a fetch that has timed out
-  """
-  def check_fetch(uuid) do
-    case get_registry_data(uuid) do
-      {:ok, %FetchRegistryData{received_at: nil}} ->
-        {:error, :waiting}
-
-      {:ok, %FetchRegistryData{} = reg_data} ->
-        {:ok, reg_data}
-
-      e ->
-        e
-    end
-  end
-
-  @doc """
-  Retrieves the response to a fetch given the identifying UUID.
-  The completed fetch will be deleted from the FetchRegistry
-
-  Will return
-    * {:ok, fetched_object} if a fetch has completed
-    * {:error, :waiting} if a fetch is still pending
-    * {:error, other_error} usually :missing to indicate a fetch that has timed out
-  """
-  def pop_fetch(uuid) do
-    case check_fetch(uuid) do
-      {:ok, %FetchRegistryData{received_json: received_json}} ->
-        delete_registry_data(uuid)
-        {:ok, received_json}
-
-      e ->
-        e
-    end
-  end
-
-  @doc """
-  This is called to register a fetch has returned.
-  It expects the result data along with the UUID that was sent in the request
-
-  Will return the fetched object or :error
-  """
-  def register_fetch_received(uuid, data) do
-    case get_registry_data(uuid) do
-      {:ok, %FetchRegistryData{received_at: nil} = reg_data} ->
-        reg_data
-        |> set_fetch_received(data)
-        |> save_registry_data()
-
-      {:ok, %FetchRegistryData{} = reg_data} ->
-        Logger.warn("tried to add fetched data twice - #{uuid}")
-        reg_data
-
-      {:error, _} ->
-        Logger.warn("Error adding fetch to registry - #{uuid}")
-        :error
-    end
-  end
-
-  defp new_registry_data(json) do
-    %FetchRegistryData{
-      uuid: UUID.generate(),
-      sent_json: json,
-      sent_at: :erlang.monotonic_time(:millisecond)
-    }
-  end
-
-  defp get_registry_data(origin) do
-    case Cachex.get(@fetches, origin) do
-      {:ok, nil} ->
-        {:error, :missing}
-
-      {:ok, reg_data} ->
-        {:ok, reg_data}
-
-      _ ->
-        {:error, :cache_error}
-    end
-  end
-
-  defp set_fetch_received(%FetchRegistryData{} = reg_data, data),
-    do: %FetchRegistryData{
-      reg_data
-      | received_at: :erlang.monotonic_time(:millisecond),
-        received_json: data
-    }
-
-  defp save_registry_data(%FetchRegistryData{uuid: uuid} = reg_data) do
-    {:ok, true} = Cachex.put(@fetches, uuid, reg_data)
-    reg_data
-  end
-
-  defp delete_registry_data(origin),
-    do: {:ok, true} = Cachex.del(@fetches, origin)
-end

lib/pleroma/web/fed_sockets/incoming_handler.ex

@@ -1,88 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.FedSockets.IncomingHandler do
-  require Logger
-
-  alias Pleroma.Web.FedSockets.FedRegistry
-  alias Pleroma.Web.FedSockets.FedSocket
-  alias Pleroma.Web.FedSockets.SocketInfo
-
-  import HTTPSignatures, only: [validate_conn: 1, split_signature: 1]
-
-  @behaviour :cowboy_websocket
-
-  def init(req, state) do
-    shake = FedSocket.shake()
-
-    with true <- Pleroma.Config.get([:fed_sockets, :enabled]),
-         sec_protocol <- :cowboy_req.header("sec-websocket-protocol", req, nil),
-         headers = %{"(request-target)" => ^shake} <- :cowboy_req.headers(req),
-         true <- validate_conn(%{req_headers: headers}),
-         %{"keyId" => origin} <- split_signature(headers["signature"]) do
-      req =
-        if is_nil(sec_protocol) do
-          req
-        else
-          :cowboy_req.set_resp_header("sec-websocket-protocol", sec_protocol, req)
-        end
-
-      {:cowboy_websocket, req, %{origin: origin}, %{}}
-    else
-      _ ->
-        {:ok, req, state}
-    end
-  end
-
-  def websocket_init(%{origin: origin}) do
-    case FedRegistry.add_fed_socket(origin) do
-      {:ok, socket_info} ->
-        {:ok, socket_info}
-
-      e ->
-        Logger.error("FedSocket websocket_init failed - #{inspect(e)}")
-        {:error, inspect(e)}
-    end
-  end
-
-  # Use the ping to  check if the connection should be expired
-  def websocket_handle(:ping, socket_info) do
-    if SocketInfo.expired?(socket_info) do
-      {:stop, socket_info}
-    else
-      {:ok, socket_info, :hibernate}
-    end
-  end
-
-  def websocket_handle({:text, data}, socket_info) do
-    socket_info = SocketInfo.touch(socket_info)
-
-    case FedSocket.receive_package(socket_info, data) do
-      {:noreply, _} ->
-        {:ok, socket_info}
-
-      {:reply, reply} ->
-        {:reply, {:text, Jason.encode!(reply)}, socket_info}
-
-      {:error, reason} ->
-        Logger.error("incoming error - receive_package: #{inspect(reason)}")
-        {:ok, socket_info}
-    end
-  end
-
-  def websocket_info({:send, message}, socket_info) do
-    socket_info = SocketInfo.touch(socket_info)
-
-    {:reply, {:text, message}, socket_info}
-  end
-
-  def websocket_info(:close, state) do
-    {:stop, state}
-  end
-
-  def websocket_info(message, state) do
-    Logger.debug("#{__MODULE__} unknown message #{inspect(message)}")
-    {:ok, state}
-  end
-end

lib/pleroma/web/fed_sockets/ingester_worker.ex

@@ -1,33 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.FedSockets.IngesterWorker do
-  use Pleroma.Workers.WorkerHelper, queue: "ingestion_queue"
-  require Logger
-
-  alias Pleroma.Web.Federator
-
-  @impl Oban.Worker
-  def perform(%Job{args: %{"op" => "ingest", "object" => ingestee}}) do
-    try do
-      ingestee
-      |> Jason.decode!()
-      |> do_ingestion()
-    rescue
-      e ->
-        Logger.error("IngesterWorker error - #{inspect(e)}")
-        e
-    end
-  end
-
-  defp do_ingestion(params) do
-    case Federator.incoming_ap_doc(params) do
-      {:error, reason} ->
-        {:error, reason}
-
-      {:ok, object} ->
-        {:ok, object}
-    end
-  end
-end

lib/pleroma/web/fed_sockets/outgoing_handler.ex

@@ -1,151 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.FedSockets.OutgoingHandler do
-  use GenServer
-
-  require Logger
-
-  alias Pleroma.Application
-  alias Pleroma.Web.ActivityPub.InternalFetchActor
-  alias Pleroma.Web.FedSockets
-  alias Pleroma.Web.FedSockets.FedRegistry
-  alias Pleroma.Web.FedSockets.FedSocket
-  alias Pleroma.Web.FedSockets.SocketInfo
-
-  def start_link(uri) do
-    GenServer.start_link(__MODULE__, %{uri: uri})
-  end
-
-  def init(%{uri: uri}) do
-    case initiate_connection(uri) do
-      {:ok, ws_origin, conn_pid} ->
-        FedRegistry.add_fed_socket(ws_origin, conn_pid)
-
-      {:error, reason} ->
-        Logger.debug("Outgoing connection failed - #{inspect(reason)}")
-        :ignore
-    end
-  end
-
-  def handle_info({:gun_ws, conn_pid, _ref, {:text, data}}, socket_info) do
-    socket_info = SocketInfo.touch(socket_info)
-
-    case FedSocket.receive_package(socket_info, data) do
-      {:noreply, _} ->
-        {:noreply, socket_info}
-
-      {:reply, reply} ->
-        :gun.ws_send(conn_pid, {:text, Jason.encode!(reply)})
-        {:noreply, socket_info}
-
-      {:error, reason} ->
-        Logger.error("incoming error - receive_package: #{inspect(reason)}")
-        {:noreply, socket_info}
-    end
-  end
-
-  def handle_info(:close, state) do
-    Logger.debug("Sending close frame !!!!!!!")
-    {:close, state}
-  end
-
-  def handle_info({:gun_down, _pid, _prot, :closed, _}, state) do
-    {:stop, :normal, state}
-  end
-
-  def handle_info({:send, data}, %{conn_pid: conn_pid} = socket_info) do
-    socket_info = SocketInfo.touch(socket_info)
-    :gun.ws_send(conn_pid, {:text, data})
-    {:noreply, socket_info}
-  end
-
-  def handle_info({:gun_ws, _, _, :pong}, state) do
-    {:noreply, state, :hibernate}
-  end
-
-  def handle_info(msg, state) do
-    Logger.debug("#{__MODULE__} unhandled event #{inspect(msg)}")
-    {:noreply, state}
-  end
-
-  def terminate(reason, state) do
-    Logger.debug(
-      "#{__MODULE__} terminating outgoing connection for #{inspect(state)} for #{inspect(reason)}"
-    )
-
-    {:ok, state}
-  end
-
-  def initiate_connection(uri) do
-    ws_uri =
-      uri
-      |> SocketInfo.origin()
-      |> FedSockets.uri_for_origin()
-
-    %{host: host, port: port, path: path} = URI.parse(ws_uri)
-
-    with {:ok, conn_pid} <- :gun.open(to_charlist(host), port, %{protocols: [:http]}),
-         {:ok, _} <- :gun.await_up(conn_pid),
-         reference <-
-           :gun.get(conn_pid, to_charlist(path), [
-             {'user-agent', to_charlist(Application.user_agent())}
-           ]),
-         {:response, :fin, 204, _} <- :gun.await(conn_pid, reference),
-         headers <- build_headers(uri),
-         ref <- :gun.ws_upgrade(conn_pid, to_charlist(path), headers, %{silence_pings: false}) do
-      receive do
-        {:gun_upgrade, ^conn_pid, ^ref, [<<"websocket">>], _} ->
-          {:ok, ws_uri, conn_pid}
-      after
-        15_000 ->
-          Logger.debug("Fedsocket timeout connecting to #{inspect(uri)}")
-          {:error, :timeout}
-      end
-    else
-      {:response, :nofin, 404, _} ->
-        {:error, :fedsockets_not_supported}
-
-      e ->
-        Logger.debug("Fedsocket error connecting to #{inspect(uri)}")
-        {:error, e}
-    end
-  end
-
-  defp build_headers(uri) do
-    host_for_sig = uri |> URI.parse() |> host_signature()
-
-    shake = FedSocket.shake()
-    digest = "SHA-256=" <> (:crypto.hash(:sha256, shake) |> Base.encode64())
-    date = Pleroma.Signature.signed_date()
-    shake_size = byte_size(shake)
-
-    signature_opts = %{
-      "(request-target)": shake,
-      "content-length": to_charlist("#{shake_size}"),
-      date: date,
-      digest: digest,
-      host: host_for_sig
-    }
-
-    signature = Pleroma.Signature.sign(InternalFetchActor.get_actor(), signature_opts)
-
-    [
-      {'signature', to_charlist(signature)},
-      {'date', date},
-      {'digest', to_charlist(digest)},
-      {'content-length', to_charlist("#{shake_size}")},
-      {to_charlist("(request-target)"), to_charlist(shake)},
-      {'user-agent', to_charlist(Application.user_agent())}
-    ]
-  end
-
-  defp host_signature(%{host: host, scheme: scheme, port: port}) do
-    if port == URI.default_port(scheme) do
-      host
-    else
-      "#{host}:#{port}"
-    end
-  end
-end

lib/pleroma/web/fed_sockets/socket_info.ex

@@ -1,52 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.FedSockets.SocketInfo do
-  defstruct origin: nil,
-            pid: nil,
-            conn_pid: nil,
-            state: :default,
-            connected_until: nil
-
-  alias Pleroma.Web.FedSockets.SocketInfo
-  @default_connection_duration 15 * 60 * 1000
-
-  def build(uri, conn_pid \\ nil) do
-    uri
-    |> build_origin()
-    |> build_pids(conn_pid)
-    |> touch()
-  end
-
-  def touch(%SocketInfo{} = socket_info),
-    do: %{socket_info | connected_until: new_ttl()}
-
-  def connect(%SocketInfo{} = socket_info),
-    do: %{socket_info | state: :connected}
-
-  def expired?(%{connected_until: connected_until}),
-    do: connected_until < :erlang.monotonic_time(:millisecond)
-
-  def origin(uri),
-    do: build_origin(uri).origin
-
-  defp build_pids(socket_info, conn_pid),
-    do: struct(socket_info, pid: self(), conn_pid: conn_pid)
-
-  defp build_origin(uri) when is_binary(uri),
-    do: uri |> URI.parse() |> build_origin
-
-  defp build_origin(%{host: host, port: nil, scheme: scheme}),
-    do: build_origin(%{host: host, port: URI.default_port(scheme)})
-
-  defp build_origin(%{host: host, port: port}),
-    do: %SocketInfo{origin: "#{host}:#{port}"}
-
-  defp new_ttl do
-    connection_duration =
-      Pleroma.Config.get([:fed_sockets, :connection_duration], @default_connection_duration)
-
-    :erlang.monotonic_time(:millisecond) + connection_duration
-  end
-end

lib/pleroma/web/fed_sockets/supervisor.ex

@@ -1,59 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.FedSockets.Supervisor do
-  use Supervisor
-  import Cachex.Spec
-
-  def start_link(opts) do
-    Supervisor.start_link(__MODULE__, opts, name: __MODULE__)
-  end
-
-  def init(args) do
-    children = [
-      build_cache(:fed_socket_fetches, args),
-      build_cache(:fed_socket_rejections, args),
-      {Registry, keys: :unique, name: FedSockets.Registry, meta: [rejected: %{}]}
-    ]
-
-    opts = [strategy: :one_for_all, name: Pleroma.Web.Streamer.Supervisor]
-    Supervisor.init(children, opts)
-  end
-
-  defp build_cache(name, args) do
-    opts = get_opts(name, args)
-
-    %{
-      id: String.to_atom("#{name}_cache"),
-      start: {Cachex, :start_link, [name, opts]},
-      type: :worker
-    }
-  end
-
-  defp get_opts(cache_name, args)
-       when cache_name in [:fed_socket_fetches, :fed_socket_rejections] do
-    default = get_opts_or_config(args, cache_name, :default, 15_000)
-    interval = get_opts_or_config(args, cache_name, :interval, 3_000)
-    lazy = get_opts_or_config(args, cache_name, :lazy, false)
-
-    [expiration: expiration(default: default, interval: interval, lazy: lazy)]
-  end
-
-  defp get_opts(name, args) do
-    Keyword.get(args, name, [])
-  end
-
-  defp get_opts_or_config(args, name, key, default) do
-    args
-    |> Keyword.get(name, [])
-    |> Keyword.get(key)
-    |> case do
-      nil ->
-        Pleroma.Config.get([:fed_sockets, name, key], default)
-
-      value ->
-        value
-    end
-  end
-end

lib/pleroma/web/feed/feed_view.ex

@@ -83,7 +83,7 @@ def activity_content(%{"content" => content}) do
 
   def activity_content(_), do: ""
 
-  def activity_context(activity), do: activity.data["context"]
+  def activity_context(activity), do: escape(activity.data["context"])
 
   def attachment_href(attachment) do
     attachment["url"]

lib/pleroma/web/mastodon_api/controllers/account_controller.ex

@@ -208,7 +208,9 @@ def update_credentials(%{assigns: %{user: user}, body_params: params} = conn, _p
         if bot, do: {:ok, "Service"}, else: {:ok, "Person"}
       end)
       |> Maps.put_if_present(:actor_type, params[:actor_type])
+      # Note: param name is indeed :locked (not an error)
       |> Maps.put_if_present(:is_locked, params[:locked])
+      # Note: param name is indeed :discoverable (not an error)
       |> Maps.put_if_present(:is_discoverable, params[:discoverable])
 
     # What happens here:
@@ -292,7 +294,8 @@ def statuses(%{assigns: %{user: reading_user}} = conn, params) do
       |> render("index.json",
         activities: activities,
         for: reading_user,
-        as: :activity
+        as: :activity,
+        with_muted: Map.get(params, :with_muted, false)
       )
     else
       error -> user_visibility_error(conn, error)
@@ -394,7 +397,7 @@ def unfollow(%{assigns: %{user: follower, account: followed}} = conn, _params) d
 
   @doc "POST /api/v1/accounts/:id/mute"
   def mute(%{assigns: %{user: muter, account: muted}, body_params: params} = conn, _params) do
-    with {:ok, _user_relationships} <- User.mute(muter, muted, params.notifications) do
+    with {:ok, _user_relationships} <- User.mute(muter, muted, params) do
       render(conn, "relationship.json", user: muter, target: muted)
     else
       {:error, message} -> json_response(conn, :forbidden, %{error: message})

lib/pleroma/web/mastodon_api/controllers/status_controller.ex

@@ -109,7 +109,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
 
   `ids` query param is required
   """
-  def index(%{assigns: %{user: user}} = conn, %{ids: ids} = _params) do
+  def index(%{assigns: %{user: user}} = conn, %{ids: ids} = params) do
     limit = 100
 
     activities =
@@ -121,7 +121,8 @@ def index(%{assigns: %{user: user}} = conn, %{ids: ids} = _params) do
     render(conn, "index.json",
       activities: activities,
       for: user,
-      as: :activity
+      as: :activity,
+      with_muted: Map.get(params, :with_muted, false)
     )
   end
 
@@ -189,13 +190,14 @@ def create(%{assigns: %{user: _user}, body_params: %{media_ids: _} = params} = c
   end
 
   @doc "GET /api/v1/statuses/:id"
-  def show(%{assigns: %{user: user}} = conn, %{id: id}) do
+  def show(%{assigns: %{user: user}} = conn, %{id: id} = params) do
     with %Activity{} = activity <- Activity.get_by_id_with_object(id),
          true <- Visibility.visible_for_user?(activity, user) do
       try_render(conn, "show.json",
         activity: activity,
         for: user,
-        with_direct_conversation_id: true
+        with_direct_conversation_id: true,
+        with_muted: Map.get(params, :with_muted, false)
       )
     else
       _ -> {:error, :not_found}
@@ -284,9 +286,9 @@ def unbookmark(%{assigns: %{user: user}} = conn, %{id: id}) do
   end
 
   @doc "POST /api/v1/statuses/:id/mute"
-  def mute_conversation(%{assigns: %{user: user}} = conn, %{id: id}) do
+  def mute_conversation(%{assigns: %{user: user}, body_params: params} = conn, %{id: id}) do
     with %Activity{} = activity <- Activity.get_by_id(id),
-         {:ok, activity} <- CommonAPI.add_mute(user, activity) do
+         {:ok, activity} <- CommonAPI.add_mute(user, activity, params) do
       try_render(conn, "show.json", activity: activity, for: user, as: :activity)
     end
   end

lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex

@@ -62,7 +62,8 @@ def home(%{assigns: %{user: user}} = conn, params) do
     |> render("index.json",
       activities: activities,
       for: user,
-      as: :activity
+      as: :activity,
+      with_muted: Map.get(params, :with_muted, false)
     )
   end
 
@@ -111,6 +112,7 @@ def public(%{assigns: %{user: user}} = conn, params) do
         |> Map.put(:blocking_user, user)
         |> Map.put(:muting_user, user)
         |> Map.put(:reply_filtering_user, user)
+        |> Map.put(:instance, params[:instance])
         |> ActivityPub.fetch_public_activities()
 
       conn
@@ -118,7 +120,8 @@ def public(%{assigns: %{user: user}} = conn, params) do
       |> render("index.json",
         activities: activities,
         for: user,
-        as: :activity
+        as: :activity,
+        with_muted: Map.get(params, :with_muted, false)
       )
     end
   end
@@ -172,7 +175,8 @@ def hashtag(%{assigns: %{user: user}} = conn, params) do
       |> render("index.json",
         activities: activities,
         for: user,
-        as: :activity
+        as: :activity,
+        with_muted: Map.get(params, :with_muted, false)
       )
     end
   end
@@ -201,7 +205,8 @@ def list(%{assigns: %{user: user}} = conn, %{list_id: id} = params) do
       render(conn, "index.json",
         activities: activities,
         for: user,
-        as: :activity
+        as: :activity,
+        with_muted: Map.get(params, :with_muted, false)
       )
     else
       _e -> render_error(conn, :forbidden, "Error.")

lib/pleroma/web/mastodon_api/views/instance_view.ex

@@ -23,7 +23,7 @@ def render("show.json", _) do
         streaming_api: Pleroma.Web.Endpoint.websocket_url()
       },
       stats: Pleroma.Stats.get_stats(),
-      thumbnail: Keyword.get(instance, :instance_thumbnail),
+      thumbnail: Pleroma.Web.base_url() <> Keyword.get(instance, :instance_thumbnail),
       languages: ["en"],
       registrations: Keyword.get(instance, :registrations_open),
       approval_required: Keyword.get(instance, :account_approval_required),
@@ -34,7 +34,7 @@ def render("show.json", _) do
       avatar_upload_limit: Keyword.get(instance, :avatar_upload_limit),
       background_upload_limit: Keyword.get(instance, :background_upload_limit),
       banner_upload_limit: Keyword.get(instance, :banner_upload_limit),
-      background_image: Keyword.get(instance, :background_image),
+      background_image: Pleroma.Web.base_url() <> Keyword.get(instance, :background_image),
       chat_limit: Keyword.get(instance, :chat_limit),
       description_limit: Keyword.get(instance, :description_limit),
       pleroma: %{

lib/pleroma/web/mastodon_api/views/notification_view.ex

@@ -11,6 +11,8 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.UserRelationship
+  alias Pleroma.Web.AdminAPI.Report
+  alias Pleroma.Web.AdminAPI.ReportView
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.MastodonAPI.NotificationView
@@ -118,11 +120,20 @@ def render(
       "pleroma:chat_mention" ->
         put_chat_message(response, activity, reading_user, status_render_opts)
 
+      "pleroma:report" ->
+        put_report(response, activity)
+
       type when type in ["follow", "follow_request"] ->
         response
     end
   end
 
+  defp put_report(response, activity) do
+    report_render = ReportView.render("show.json", Report.extract_report_info(activity))
+
+    Map.put(response, :report, report_render)
+  end
+
   defp put_emoji(response, activity) do
     Map.put(response, :emoji, activity.data["content"])
   end

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -19,6 +19,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   alias Pleroma.Web.MastodonAPI.PollView
   alias Pleroma.Web.MastodonAPI.StatusView
   alias Pleroma.Web.MediaProxy
+  alias Pleroma.Web.PleromaAPI.EmojiReactionController
 
   import Pleroma.Web.ActivityPub.Visibility, only: [get_visibility: 1, visible_for_user?: 2]
 
@@ -294,21 +295,16 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity}
       end
 
     emoji_reactions =
-      with %{data: %{"reactions" => emoji_reactions}} <- object do
+      object.data
-        Enum.map(emoji_reactions, fn
+      |> Map.get("reactions", [])
-          [emoji, users] when is_list(users) ->
+      |> EmojiReactionController.filter_allowed_users(
-            build_emoji_map(emoji, users, opts[:for])
+        opts[:for],
-
+        Map.get(opts, :with_muted, false)
-          {emoji, users} when is_list(users) ->
+      )
-            build_emoji_map(emoji, users, opts[:for])
+      |> Stream.map(fn {emoji, users} ->
-
+        build_emoji_map(emoji, users, opts[:for])
-          _ ->
+      end)
-            nil
+      |> Enum.to_list()
-        end)
-        |> Enum.reject(&is_nil/1)
-      else
-        _ -> []
-      end
 
     # Status muted state (would do 1 request per status unless user mutes are preloaded)
     muted =
@@ -435,7 +431,8 @@ def render("attachment.json", %{attachment: attachment}) do
       text_url: href,
       type: type,
       description: attachment["name"],
-      pleroma: %{mime_type: media_type}
+      pleroma: %{mime_type: media_type},
+      blurhash: attachment["blurhash"]
     }
   end
 

lib/pleroma/web/metadata/providers/restrict_indexing.ex

@@ -6,7 +6,7 @@ defmodule Pleroma.Web.Metadata.Providers.RestrictIndexing do
   @behaviour Pleroma.Web.Metadata.Providers.Provider
 
   @moduledoc """
-  Restricts indexing of remote users.
+  Restricts indexing of remote and/or non-discoverable users.
   """
 
   @impl true

lib/pleroma/web/pleroma_api/controllers/chat_controller.ex

@@ -140,8 +140,8 @@ def messages(%{assigns: %{user: user}} = conn, %{id: id} = params) do
 
   def index(%{assigns: %{user: %{id: user_id} = user}} = conn, params) do
     exclude_users =
-      User.blocked_users_ap_ids(user) ++
+      User.cached_blocked_users_ap_ids(user) ++
-        if params[:with_muted], do: [], else: User.muted_users_ap_ids(user)
+        if params[:with_muted], do: [], else: User.cached_muted_users_ap_ids(user)
 
     chats =
       user_id

lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiReactionController do
 
   alias Pleroma.Activity
   alias Pleroma.Object
+  alias Pleroma.User
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.StatusView
   alias Pleroma.Web.Plugs.OAuthScopesPlug
@@ -29,13 +30,42 @@ def index(%{assigns: %{user: user}} = conn, %{id: activity_id} = params) do
          %Activity{} = activity <- Activity.get_by_id_with_object(activity_id),
          %Object{data: %{"reactions" => reactions}} when is_list(reactions) <-
            Object.normalize(activity) do
-      reactions = filter(reactions, params)
+      reactions =
+        reactions
+        |> filter(params)
+        |> filter_allowed_users(user, Map.get(params, :with_muted, false))
+
       render(conn, "index.json", emoji_reactions: reactions, user: user)
     else
       _e -> json(conn, [])
     end
   end
 
+  def filter_allowed_users(reactions, user, with_muted) do
+    exclude_ap_ids =
+      if is_nil(user) do
+        []
+      else
+        User.cached_blocked_users_ap_ids(user) ++
+          if not with_muted, do: User.cached_muted_users_ap_ids(user), else: []
+      end
+
+    filter_emoji = fn emoji, users ->
+      case Enum.reject(users, &(&1 in exclude_ap_ids)) do
+        [] -> nil
+        users -> {emoji, users}
+      end
+    end
+
+    reactions
+    |> Stream.map(fn
+      [emoji, users] when is_list(users) -> filter_emoji.(emoji, users)
+      {emoji, users} when is_list(users) -> filter_emoji.(emoji, users)
+      _ -> nil
+    end)
+    |> Stream.reject(&is_nil/1)
+  end
+
   defp filter(reactions, %{emoji: emoji}) when is_binary(emoji) do
     Enum.filter(reactions, fn [e, _] -> e == emoji end)
   end