Merge branch 'develop' of https://git.pleroma.social/pleroma/pleroma into develop

.gitignore

@@ -28,8 +28,6 @@ erl_crash.dump
 # variables.
 /config/*.secret.exs
 /config/generated_config.exs
-/config/*.env
-
 
 # Database setup file, some may forget to delete it
 /config/setup_db.psql

.gitlab-ci.yml

@@ -22,6 +22,7 @@ stages:
   - docker
 
 before_script:
+  - apt-get update && apt-get install -y cmake
   - mix local.hex --force
   - mix local.rebar --force
 
@@ -193,6 +194,7 @@ amd64:
   variables: &release-variables
     MIX_ENV: prod
   before_script: &before-release
+  - apt install cmake -y
   - echo "import Mix.Config" > config/prod.secret.exs
   - mix local.hex --force
   - mix local.rebar --force
@@ -211,7 +213,7 @@ amd64-musl:
   cache: *release-cache
   variables: *release-variables
   before_script: &before-release-musl
-  - apk add git gcc g++ musl-dev make
+  - apk add git gcc g++ musl-dev make cmake
   - echo "import Mix.Config" > config/prod.secret.exs
   - mix local.hex --force
   - mix local.rebar --force

CHANGELOG.md

@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## [unreleased]
 
 ### Changed
+- **Breaking:** Added the ObjectAgePolicy to the default set of MRFs. This will delist and strip the follower collection of any message received that is older than 7 days. This will stop users from seeing very old messages in the timelines. The messages can still be viewed on the user's page and in conversations. They also still trigger notifications.
 - **Breaking:** Elixir >=1.9 is now required (was >= 1.8)
 - **Breaking:** Configuration: `:auto_linker, :opts` moved to `:pleroma, Pleroma.Formatter`. Old config namespace is deprecated.
 - In Conversations, return only direct messages as `last_status`
@@ -15,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Configuration: `:instance, rewrite_policy` moved to `:mrf, policies`, `:instance, :mrf_transparency` moved to `:mrf, :transparency`, `:instance, :mrf_transparency_exclusions` moved to `:mrf, :transparency_exclusions`. Old config namespace is deprecated.
 - Configuration: `:media_proxy, whitelist` format changed to host with scheme (e.g. `http://example.com` instead of `example.com`). Domain format is deprecated.
 - **Breaking:** Configuration: `:instance, welcome_user_nickname` moved to `:welcome, :direct_message, :sender_nickname`, `:instance, :welcome_message` moved to `:welcome, :direct_message, :message`. Old config namespace is deprecated.
+- **Breaking:** LDAP: Fallback to local database authentication has been removed for security reasons and lack of a mechanism to ensure the passwords are synchronized when LDAP passwords are updated.
 
 <details>
   <summary>API Changes</summary>
@@ -49,6 +51,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Added
 
+- Configuration: Added a blacklist for email servers.
 - Chats: Added `accepts_chat_messages` field to user, exposed in APIs and federation.
 - Chats: Added support for federated chats. For details, see the docs.
 - ActivityPub: Added support for existing AP ids for instances migrated from Mastodon.
@@ -69,7 +72,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Support for viewing instances favicons next to posts and accounts
 - Added Pleroma.Upload.Filter.Exiftool as an alternate EXIF stripping mechanism targeting GPS/location metadata.
 - "By approval" registrations mode.
-- Configuration: Added `:welcome` settings for the welcome message to newly registered users.
+- Configuration: Added `:welcome` settings for the welcome message to newly registered users. You can send a welcome message as a direct message, chat or email.
 - Ability to hide favourites and emoji reactions in the API with `[:instance, :show_reactions]` config.
 
 <details>
@@ -101,6 +104,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Fix CSP policy generation to include remote Captcha services
 - Fix edge case where MediaProxy truncates media, usually caused when Caddy is serving content for the other Federated instance.
 - Emoji Packs could not be listed when instance was set to `public: false`
+- Fix whole_word always returning false on filter get requests
 
 ## [Unreleased (patch)]
 

Dockerfile

@@ -4,7 +4,7 @@ COPY . .
 
 ENV MIX_ENV=prod
 
-RUN apk add git gcc g++ musl-dev make &&\
+RUN apk add git gcc g++ musl-dev make cmake &&\
 	echo "import Mix.Config" > config/prod.secret.exs &&\
 	mix local.hex --force &&\
 	mix local.rebar --force &&\

SECURITY.md

@@ -0,0 +1,16 @@
+# Pleroma backend security policy
+
+## Supported versions
+
+Currently, Pleroma offers bugfixes and security patches only for the latest minor release.
+
+| Version | Support 
+|---------| --------
+| 2.0     | Bugfixes and security patches
+
+## Reporting a vulnerability
+
+Please use confidential issues (tick the "This issue is confidential and should only be visible to team members with at least Reporter access." box when submitting) at our [bugtracker](https://git.pleroma.social/pleroma/pleroma/-/issues/new) for reporting vulnerabilities.
+## Announcements
+
+New releases are announced at [pleroma.social](https://pleroma.social/announcements/). All security releases are tagged with ["Security"](https://pleroma.social/announcements/tags/security/). You can be notified of them by subscribing to an Atom feed at <https://pleroma.social/announcements/tags/security/feed.xml>. 

config/config.exs

@@ -264,6 +264,11 @@ config :pleroma, :welcome,
     sender_nickname: nil,
     message: nil
   ],
+  chat_message: [
+    enabled: false,
+    sender_nickname: nil,
+    message: nil
+  ],
   email: [
     enabled: false,
     sender: nil,
@@ -377,6 +382,7 @@ config :pleroma, :mrf_simple,
   federated_timeline_removal: [],
   report_removal: [],
   reject: [],
+  followers_only: [],
   accept: [],
   avatar_removal: [],
   banner_removal: [],
@@ -395,8 +401,9 @@ config :pleroma, :mrf_vocabulary,
   accept: [],
   reject: []
 
+# threshold of 7 days
 config :pleroma, :mrf_object_age,
-  threshold: 172_800,
+  threshold: 604_800,
   actions: [:delist, :strip_followers]
 
 config :pleroma, :rich_media,
@@ -511,8 +518,15 @@ config :pleroma, Pleroma.User,
     "user-search",
     "user_exists",
     "users",
-    "web"
+    "web",
-  ]
+    "verify_credentials",
+    "update_credentials",
+    "relationships",
+    "search",
+    "confirmation_resend",
+    "mfa"
+  ],
+  email_blacklist: []
 
 config :pleroma, Oban,
   repo: Pleroma.Repo,
@@ -722,7 +736,7 @@ config :pleroma, :restrict_unauthenticated,
 config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: false
 
 config :pleroma, :mrf,
-  policies: Pleroma.Web.ActivityPub.MRF.NoOpPolicy,
+  policies: Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy,
   transparency: true,
   transparency_exclusions: []
 
@@ -732,6 +746,10 @@ config :ex_aws, http_client: Pleroma.HTTP.ExAws
 
 config :pleroma, :instances_favicons, enabled: false
 
+config :floki, :html_parser, Floki.HTMLParser.FastHtml
+
+config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/description.exs

@@ -194,7 +194,7 @@ config :pleroma, :config_description, [
         type: [:string, {:list, :string}, {:list, :tuple}],
         description:
           "List of actions for the mogrify command. It's possible to add self-written settings as string. " <>
-            "For example `[\"auto-orient\", \"strip\", {\"resize\", \"3840x1080>\"}]` string will be parsed into list of the settings.",
+            "For example `auto-orient, strip, {\"resize\", \"3840x1080>\"}` value will be parsed into valid list of the settings.",
         suggestions: [
           "strip",
           "auto-orient",
@@ -951,7 +951,7 @@ config :pleroma, :config_description, [
       },
       %{
         key: :instance_thumbnail,
-        type: :string,
+        type: {:string, :image},
         description:
           "The instance thumbnail can be any image that represents your instance and is used by some apps or services when they display information about your instance.",
         suggestions: ["/instance/thumbnail.jpeg"]
@@ -964,25 +964,25 @@ config :pleroma, :config_description, [
     ]
   },
   %{
-    group: :welcome,
+    group: :pleroma,
+    key: :welcome,
     type: :group,
     description: "Welcome messages settings",
     children: [
       %{
-        group: :direct_message,
+        key: :direct_message,
-        type: :group,
+        type: :keyword,
         descpiption: "Direct message settings",
         children: [
           %{
             key: :enabled,
             type: :boolean,
-            description: "Enables sends direct message for new user after registration"
+            description: "Enables sending a direct message to newly registered users"
           },
           %{
             key: :message,
             type: :string,
-            description:
+            description: "A message that will be sent to newly registered users",
-              "A message that will be sent to a newly registered users as a direct message",
             suggestions: [
               "Hi, @username! Welcome on board!"
             ]
@@ -990,7 +990,7 @@ config :pleroma, :config_description, [
           %{
             key: :sender_nickname,
             type: :string,
-            description: "The nickname of the local user that sends the welcome message",
+            description: "The nickname of the local user that sends a welcome message",
             suggestions: [
               "lain"
             ]
@@ -998,20 +998,49 @@ config :pleroma, :config_description, [
         ]
       },
       %{
-        group: :email,
+        key: :chat_message,
-        type: :group,
+        type: :keyword,
+        descpiption: "Chat message settings",
+        children: [
+          %{
+            key: :enabled,
+            type: :boolean,
+            description: "Enables sending a chat message to newly registered users"
+          },
+          %{
+            key: :message,
+            type: :string,
+            description:
+              "A message that will be sent to newly registered users as a chat message",
+            suggestions: [
+              "Hello, welcome on board!"
+            ]
+          },
+          %{
+            key: :sender_nickname,
+            type: :string,
+            description: "The nickname of the local user that sends a welcome chat message",
+            suggestions: [
+              "lain"
+            ]
+          }
+        ]
+      },
+      %{
+        key: :email,
+        type: :keyword,
         descpiption: "Email message settings",
         children: [
           %{
             key: :enabled,
             type: :boolean,
-            description: "Enables sends direct message for new user after registration"
+            description: "Enables sending an email to newly registered users"
           },
           %{
             key: :sender,
             type: [:string, :tuple],
             description:
-              "The email address or tuple with `{nickname, email}` that will use as sender to the welcome email.",
+              "Email address and/or nickname that will be used to send the welcome email.",
             suggestions: [
               {"Pleroma App", "welcome@pleroma.app"}
             ]
@@ -1020,21 +1049,21 @@ config :pleroma, :config_description, [
             key: :subject,
             type: :string,
             description:
-              "The subject of welcome email. Can be use EEX template with `user` and `instance_name` variables.",
+              "Subject of the welcome email. EEX template with user and instance_name variables can be used.",
             suggestions: ["Welcome to <%= instance_name%>"]
           },
           %{
             key: :html,
             type: :string,
             description:
-              "The html content of welcome email. Can be use EEX template with `user` and `instance_name` variables.",
+              "HTML content of the welcome email. EEX template with user and instance_name variables can be used.",
             suggestions: ["<h1>Hello <%= user.name%>. Welcome to <%= instance_name%></h1>"]
           },
           %{
             key: :text,
             type: :string,
             description:
-              "The text content of welcome email. Can be use EEX template with `user` and `instance_name` variables.",
+              "Text content of the welcome email. EEX template with user and instance_name variables can be used.",
             suggestions: ["Hello <%= user.name%>. \n Welcome to <%= instance_name%>\n"]
           }
         ]
@@ -1207,7 +1236,7 @@ config :pleroma, :config_description, [
           },
           %{
             key: :background,
-            type: :string,
+            type: {:string, :image},
             description:
               "URL of the background, unless viewing a user profile with a background that is set",
             suggestions: ["/images/city.jpg"]
@@ -1264,7 +1293,7 @@ config :pleroma, :config_description, [
           },
           %{
             key: :logo,
-            type: :string,
+            type: {:string, :image},
             description: "URL of the logo, defaults to Pleroma's logo",
             suggestions: ["/static/logo.png"]
           },
@@ -1296,7 +1325,7 @@ config :pleroma, :config_description, [
           %{
             key: :nsfwCensorImage,
             label: "NSFW Censor Image",
-            type: :string,
+            type: {:string, :image},
             description:
               "URL of the image to use for hiding NSFW media attachments in the timeline",
             suggestions: ["/static/img/nsfw.74818f9.png"]
@@ -1422,7 +1451,7 @@ config :pleroma, :config_description, [
       },
       %{
         key: :default_user_avatar,
-        type: :string,
+        type: {:string, :image},
         description: "URL of the default user avatar",
         suggestions: ["/images/avi.png"]
       }
@@ -1542,6 +1571,12 @@ config :pleroma, :config_description, [
         description: "List of instances to only accept activities from (except deletes)",
         suggestions: ["example.com", "*.example.com"]
       },
+      %{
+        key: :followers_only,
+        type: {:list, :string},
+        description: "Force posts from the given instances to be visible by followers only",
+        suggestions: ["example.com", "*.example.com"]
+      },
       %{
         key: :report_removal,
         type: {:list, :string},
@@ -2607,7 +2642,7 @@ config :pleroma, :config_description, [
     children: [
       %{
         key: :logo,
-        type: :string,
+        type: {:string, :image},
         description: "A path to a custom logo. Set it to `nil` to use the default Pleroma logo.",
         suggestions: ["some/path/logo.png"]
       },
@@ -3021,6 +3056,7 @@ config :pleroma, :config_description, [
       %{
         key: :restricted_nicknames,
         type: {:list, :string},
+        description: "List of nicknames users may not register with.",
         suggestions: [
           ".well-known",
           "~",
@@ -3053,6 +3089,12 @@ config :pleroma, :config_description, [
           "users",
           "web"
         ]
+      },
+      %{
+        key: :email_blacklist,
+        type: {:list, :string},
+        description: "List of email domains users may not register with.",
+        suggestions: ["mailinator.com", "maildrop.cc"]
       }
     ]
   },
@@ -3257,13 +3299,13 @@ config :pleroma, :config_description, [
     group: :pleroma,
     key: :connections_pool,
     type: :group,
-    description: "Advanced settings for `gun` connections pool",
+    description: "Advanced settings for `Gun` connections pool",
     children: [
       %{
         key: :connection_acquisition_wait,
         type: :integer,
         description:
-          "Timeout to acquire a connection from pool.The total max time is this value multiplied by the number of retries. Default: 250ms.",
+          "Timeout to acquire a connection from pool. The total max time is this value multiplied by the number of retries. Default: 250ms.",
         suggestions: [250]
       },
       %{
@@ -3298,7 +3340,7 @@ config :pleroma, :config_description, [
     group: :pleroma,
     key: :pools,
     type: :group,
-    description: "Advanced settings for `gun` workers pools",
+    description: "Advanced settings for `Gun` workers pools",
     children:
       Enum.map([:federation, :media, :upload, :default], fn pool_name ->
         %{
@@ -3327,7 +3369,7 @@ config :pleroma, :config_description, [
     group: :pleroma,
     key: :hackney_pools,
     type: :group,
-    description: "Advanced settings for `hackney` connections pools",
+    description: "Advanced settings for `Hackney` connections pools",
     children: [
       %{
         key: :federation,
@@ -3391,6 +3433,7 @@ config :pleroma, :config_description, [
   %{
     group: :pleroma,
     key: :restrict_unauthenticated,
+    label: "Restrict Unauthenticated",
     type: :group,
     description:
       "Disallow viewing timelines, user profiles and statuses for unauthenticated users.",
@@ -3513,13 +3556,17 @@ config :pleroma, :config_description, [
         children: [
           %{
             key: "name",
+            label: "Name",
             type: :string,
-            description: "Name of the installed primary frontend"
+            description:
+              "Name of the installed primary frontend. Valid config must include both `Name` and `Reference` values."
           },
           %{
             key: "ref",
+            label: "Reference",
             type: :string,
-            description: "reference of the installed primary frontend to be used"
+            description:
+              "Reference of the installed primary frontend to be used. Valid config must include both `Name` and `Reference` values."
           }
         ]
       }

config/test.exs

@@ -120,6 +120,8 @@ config :pleroma, Pleroma.Uploaders.S3,
 
 config :tzdata, :autoupdate, :disabled
 
+config :pleroma, :mrf, policies: []
+
 if File.exists?("./config/test.secret.exs") do
   import_config "test.secret.exs"
 else

docs/administration/CLI_tasks/database.md

@@ -97,4 +97,14 @@ but should only be run if necessary. **It is safe to cancel this.**
 
 ```sh tab="From Source"
 mix pleroma.database vacuum full
-```
+```
+
+## Add expiration to all local statuses
+
+```sh tab="OTP"
+./bin/pleroma_ctl database ensure_expiration
+```
+
+```sh tab="From Source"
+mix pleroma.database ensure_expiration
+```

docs/administration/CLI_tasks/release_environments.md

@@ -1,9 +0,0 @@
-# Generate release environment file
-
-```sh tab="OTP"
- ./bin/pleroma_ctl release_env gen
-```
-
-```sh tab="From Source"
-mix pleroma.release_env gen
-```

docs/administration/CLI_tasks/robots_txt.md

@@ -0,0 +1,17 @@
+# Managing robots.txt
+
+{! backend/administration/CLI_tasks/general_cli_task_info.include !}
+
+## Generate a new robots.txt file and add it to the static directory
+
+The `robots.txt` that ships by default is permissive. It allows well-behaved search engines to index all of your instance's URIs.
+
+If you want to generate a restrictive `robots.txt`, you can run the following mix task. The generated `robots.txt` will be written in your instance [static directory](../../../configuration/static_dir/).
+
+```elixir tab="OTP"
+./bin/pleroma_ctl robots_txt disallow_all
+```
+
+```elixir tab="From Source"
+mix pleroma.robots_txt disallow_all
+```

docs/clients.md

@@ -75,6 +75,13 @@ Feel free to contact us to be added to this list!
 - Platform: Android, iOS
 - Features: No Streaming
 
+### Indigenous
+- Homepage: <https://indigenous.realize.be/>
+- Source Code: <https://github.com/swentel/indigenous-android/>
+- Contact: [@realize.be@realize.be](@realize.be@realize.be)
+- Platforms: Android
+- Features: No Streaming
+
 ## Alternative Web Interfaces
 ### Brutaldon
 - Homepage: <https://jfm.carcosa.net/projects/software/brutaldon/>

docs/configuration/cheatsheet.md

@@ -69,6 +69,10 @@ To add configuration to your config file, you can copy it from the base config.
   * `enabled`: Enables the send a direct message to a newly registered user. Defaults to `false`.
   * `sender_nickname`: The nickname of the local user that sends the welcome message.
   * `message`: A message that will be send to a newly registered users as a direct message.
+* `chat_message`: - welcome message sent as a chat message.
+  * `enabled`: Enables the send a chat message to a newly registered user. Defaults to `false`.
+  * `sender_nickname`: The nickname of the local user that sends the welcome message.
+  * `message`: A message that will be send to a newly registered users as a chat message.
 * `email`: - welcome message sent as a email.
   * `enabled`: Enables the send a welcome email to a newly registered user. Defaults to `false`.
   * `sender`: The email address or tuple with `{nickname, email}` that will use as sender to the welcome email.
@@ -110,6 +114,7 @@ To add configuration to your config file, you can copy it from the base config.
     * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (See [`:mrf_mention`](#mrf_mention)).
     * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (See [`:mrf_vocabulary`](#mrf_vocabulary)).
     * `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)).
+    * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.ActivityExpiration` to be enabled for processing the scheduled delections.
 * `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo).
 * `transparency_exclusions`: Exclude specific instance names from MRF transparency.  The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.
 
@@ -125,6 +130,7 @@ To add configuration to your config file, you can copy it from the base config.
 * `federated_timeline_removal`: List of instances to remove from Federated (aka The Whole Known Network) Timeline.
 * `reject`: List of instances to reject any activities from.
 * `accept`: List of instances to accept any activities from.
+* `followers_only`: List of instances to decrease post visibility to only the followers, including for DM mentions.
 * `report_removal`: List of instances to reject reports from.
 * `avatar_removal`: List of instances to strip avatars from.
 * `banner_removal`: List of instances to strip banners from.
@@ -202,6 +208,11 @@ config :pleroma, :mrf_user_allowlist, %{
 * `sign_object_fetches`: Sign object fetches with HTTP signatures
 * `authorized_fetch_mode`: Require HTTP signatures for AP fetches
 
+## Pleroma.User
+
+* `restricted_nicknames`: List of nicknames users may not register with.
+* `email_blacklist`: List of email domains users may not register with.
+
 ## Pleroma.ScheduledActivity
 
 * `daily_user_limit`: the number of scheduled activities a user is allowed to create in a single day (Default: `25`)
@@ -210,6 +221,8 @@ config :pleroma, :mrf_user_allowlist, %{
 
 ## Pleroma.ActivityExpiration
 
+Enables the worker which processes posts scheduled for deletion. Pinned posts are exempt from expiration.
+
 * `enabled`: whether expired activities will be sent to the job queue to be deleted
 
 ## Frontends
@@ -848,9 +861,6 @@ Warning: it's discouraged to use this feature because of the associated security
 
 ### :auth
 
-* `Pleroma.Web.Auth.PleromaAuthenticator`: default database authenticator.
-* `Pleroma.Web.Auth.LDAPAuthenticator`: LDAP authentication.
-
 Authentication / authorization settings.
 
 * `auth_template`: authentication form template. By default it's `show.html` which corresponds to `lib/pleroma/web/templates/o_auth/o_auth/show.html.eex`.
@@ -880,6 +890,9 @@ Pleroma account will be created with the same name as the LDAP user name.
 * `base`: LDAP base, e.g. "dc=example,dc=com"
 * `uid`: LDAP attribute name to authenticate the user, e.g. when "cn", the filter will be "cn=username,base"
 
+Note, if your LDAP server is an Active Directory server the correct value is commonly `uid: "cn"`, but if you use an
+OpenLDAP server the value may be `uid: "uid"`.
+
 ### OAuth consumer mode
 
 OAuth consumer mode allows sign in / sign up via external OAuth providers (e.g. Twitter, Facebook, Google, Microsoft, etc.).

docs/configuration/static_dir.md

@@ -1,45 +1,57 @@
 # Static Directory
 
-Static frontend files are shipped in `priv/static/` and tracked by version control in this repository. If you want to overwrite or update these without the possibility of merge conflicts, you can write your custom versions to `instance/static/`.
+Static frontend files are shipped with pleroma. If you want to overwrite or update these without problems during upgrades, you can write your custom versions to the static directory.
 
+You can find the location of the static directory in the [configuration](../cheatsheet/#instance).
+
+```elixir tab="OTP"
+config :pleroma, :instance,
+  static_dir: "/var/lib/pleroma/static/",
 ```
+
+```elixir tab="From Source"
 config :pleroma, :instance,
   static_dir: "instance/static/",
 ```
 
-For example, edit `instance/static/instance/panel.html` .
-
 Alternatively, you can overwrite this value in your configuration to use a different static instance directory.
 
-This document is written assuming `instance/static/`.
+This document is written using `$static_dir` as the value of the `config :pleroma, :instance, static_dir` setting.
 
-Or, if you want to manage your custom file in git repository, basically remove the `instance/` entry from `.gitignore`.
+If you use a From Source installation and want to manage your custom files in the git repository, you can remove the `instance/` entry from `.gitignore`.
 
 ## robots.txt
 
-By default, the `robots.txt` that ships in `priv/static/` is permissive. It allows well-behaved search engines to index all of your instance's URIs.
+There's a mix tasks to [generate a new robot.txt](../../administration/CLI_tasks/robots_txt/).
 
-If you want to generate a restrictive `robots.txt`, you can run the following mix task. The generated `robots.txt` will be written in your instance static directory.
+For more complex things, you can write your own robots.txt to `$static_dir/robots.txt`.
+
+E.g. if you want to block all crawlers except for [fediverse.network](https://fediverse.network/about) you can use
 
 ```
-mix pleroma.robots_txt disallow_all
+User-Agent: *
+Disallow: /
+
+User-Agent: crawler-us-il-1.fediverse.network
+Allow: /
+
+User-Agent: makhnovtchina.random.sh
+Allow: /
 ```
 
 ## Thumbnail
 
-Put on `instance/static/instance/thumbnail.jpeg` with your selfie or other neat picture. It will appear in [Pleroma Instances](http://distsn.org/pleroma-instances.html).
+Add `$static_dir/instance/thumbnail.jpeg` with your selfie or other neat picture. It will be available on `http://your-domain.tld/instance/thumbnail.jpeg` and can be used by external applications.
 
 ## Instance-specific panel
 
-![instance-specific panel demo](/uploads/296b19ec806b130e0b49b16bfe29ce8a/image.png)
+Create and Edit your file at `$static_dir/instance/panel.html`.
-
-Create and Edit your file on `instance/static/instance/panel.html`.
 
 ## Background
 
-You can change the background of your Pleroma instance by uploading it to `instance/static/`, and then changing `background` in `config/prod.secret.exs` accordingly.
+You can change the background of your Pleroma instance by uploading it to `$static_dir/`, and then changing `background` in [your configuration](../cheatsheet/#frontend_configurations) accordingly.
 
-If you put `instance/static/images/background.jpg`
+E.g. if you put `$static_dir/images/background.jpg`
 
 ```
 config :pleroma, :frontend_configurations,
@@ -50,12 +62,14 @@ config :pleroma, :frontend_configurations,
 
 ## Logo
 
-![logo modification demo](/uploads/c70b14de60fa74245e7f0dcfa695ebff/image.png)
+!!! important
+    Note the extra `static` folder for the default logo.png location
 
-If you want to give a brand to your instance, You can change the logo of your instance by uploading it to `instance/static/`.
+If you want to give a brand to your instance, You can change the logo of your instance by uploading it to the static directory `$static_dir/static/logo.png`.
 
-Alternatively, you can specify the path with config.
+Alternatively, you can specify the path to your logo in [your configuration](../cheatsheet/#frontend_configurations).
-If you put `instance/static/static/mylogo-file.png`
+
+E.g. if you put `$static_dir/static/mylogo-file.png`
 
 ```
 config :pleroma, :frontend_configurations,
@@ -66,4 +80,7 @@ config :pleroma, :frontend_configurations,
 
 ## Terms of Service
 
-Terms of Service will be shown to all users on the registration page. It's the best place where to write down the rules for your instance. You can modify the rules by changing `instance/static/static/terms-of-service.html`.
+!!! important
+    Note the extra `static` folder for the terms-of-service.html
+
+Terms of Service will be shown to all users on the registration page. It's the best place where to write down the rules for your instance. You can modify the rules by adding and changing `$static_dir/static/terms-of-service.html`.

docs/installation/alpine_linux_en.md

@@ -14,6 +14,7 @@ It assumes that you have administrative rights, either as root or a user with [s
 * `erlang-xmerl`
 * `git`
 * Development Tools
+* `cmake`
 
 #### Optional packages used in this guide
 
@@ -39,7 +40,7 @@ sudo apk upgrade
 * Install some tools, which are needed later:
 
 ```shell
-sudo apk add git build-base
+sudo apk add git build-base cmake
 ```
 
 ### Install Elixir and Erlang

docs/installation/arch_linux_en.md

@@ -9,6 +9,7 @@ This guide will assume that you have administrative rights, either as root or a
 * `elixir`
 * `git`
 * `base-devel`
+* `cmake`
 
 #### Optional packages used in this guide
 
@@ -26,7 +27,7 @@ sudo pacman -Syu
 * Install some of the above mentioned programs:
 
 ```shell
-sudo pacman -S git base-devel elixir
+sudo pacman -S git base-devel elixir cmake
 ```
 
 ### Install PostgreSQL

docs/installation/debian_based_en.md

@@ -12,6 +12,7 @@ This guide will assume you are on Debian Stretch. This guide should also work wi
 * `erlang-nox`
 * `git`
 * `build-essential`
+* `cmake`
 
 #### Optional packages used in this guide
 
@@ -30,7 +31,7 @@ sudo apt full-upgrade
 * Install some of the above mentioned programs:
 
 ```shell
-sudo apt install git build-essential postgresql postgresql-contrib
+sudo apt install git build-essential postgresql postgresql-contrib cmake
 ```
 
 ### Install Elixir and Erlang

docs/installation/debian_based_jp.md

@@ -16,6 +16,7 @@
 - `erlang-nox`
 - `git`
 - `build-essential`
+- `cmake`
 
 #### このガイドで利用している追加パッケージ
 
@@ -32,7 +33,7 @@ sudo apt full-upgrade
 
 * 上記に挙げたパッケージをインストールしておきます。
 ```
-sudo apt install git build-essential postgresql postgresql-contrib
+sudo apt install git build-essential postgresql postgresql-contrib cmake
 ```
 
 

docs/installation/gentoo_en.md

@@ -28,6 +28,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
 * `dev-db/postgresql`
 * `dev-lang/elixir`
 * `dev-vcs/git`
+* `dev-util/cmake`
 
 #### Optional ebuilds used in this guide
 
@@ -46,7 +47,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
 * Emerge all required the required and suggested software in one go:
 
 ```shell
- # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx
+ # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake
 ```
 
 If you would not like to install the optional packages, remove them from this line. 

docs/installation/netbsd_en.md

@@ -19,6 +19,7 @@ databases/postgresql11-client
 databases/postgresql11-server
 devel/git-base
 devel/git-docs
+devel/cmake
 lang/elixir
 security/acmesh
 security/sudo

docs/installation/openbsd_en.md

@@ -14,11 +14,12 @@ The following packages need to be installed:
   * git
   * postgresql-server
   * postgresql-contrib
+  * cmake
 
 To install them, run the following command (with doas or as root):
 
 ```
-pkg_add elixir gmake ImageMagick git postgresql-server postgresql-contrib
+pkg_add elixir gmake ImageMagick git postgresql-server postgresql-contrib cmake
 ```
 
 Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in this guide) and packages/ports are available for nginx (www/nginx) and apache (www/apache-httpd). Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acme-client) can be used to get a certificate from Let's Encrypt.

docs/installation/openbsd_fi.md

@@ -16,7 +16,7 @@ Matrix-kanava #freenode_#pleroma:matrix.org ovat hyviä paikkoja löytää apua
 
 Asenna tarvittava ohjelmisto:
 
-`# pkg_add git elixir gmake postgresql-server-10.3 postgresql-contrib-10.3`
+`# pkg_add git elixir gmake postgresql-server-10.3 postgresql-contrib-10.3 cmake`
 
 Luo postgresql-tietokanta:
 

docs/installation/otp_en.md

@@ -121,9 +121,6 @@ chown -R pleroma /etc/pleroma
 # Run the config generator
 su pleroma -s $SHELL -lc "./bin/pleroma_ctl instance gen --output /etc/pleroma/config.exs --output-psql /tmp/setup_db.psql"
 
-# Run the environment file generator.
-su pleroma -s $SHELL -lc "./bin/pleroma_ctl release_env gen"
-
 # Create the postgres database
 su postgres -s $SHELL -lc "psql -f /tmp/setup_db.psql"
 
@@ -134,7 +131,7 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
 # su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
 
 # Start the instance to verify that everything is working as expected
-su pleroma -s $SHELL -lc "export $(cat /opt/pleroma/config/pleroma.env); ./bin/pleroma daemon"
+su pleroma -s $SHELL -lc "./bin/pleroma daemon"
 
 # Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly
 sleep 20 && curl http://localhost:4000/api/v1/instance
@@ -203,7 +200,6 @@ rc-update add pleroma
 # Copy the service into a proper directory
 cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service
 
-
 # Start pleroma and enable it on boot
 systemctl start pleroma
 systemctl enable pleroma
@@ -279,3 +275,4 @@ This will create an account withe the username of 'joeuser' with the email addre
 ## Questions
 
 Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
+

installation/init.d/pleroma

@@ -8,7 +8,6 @@ pidfile="/var/run/pleroma.pid"
 directory=/opt/pleroma
 healthcheck_delay=60
 healthcheck_timer=30
-export $(cat /opt/pleroma/config/pleroma.env)
 
 : ${pleroma_port:-4000}
 

installation/pleroma.service

@@ -17,8 +17,6 @@ Environment="MIX_ENV=prod"
 Environment="HOME=/var/lib/pleroma"
 ; Path to the folder containing the Pleroma installation.
 WorkingDirectory=/opt/pleroma
-; Path to the environment file. the file contains RELEASE_COOKIE and etc 
-EnvironmentFile=/opt/pleroma/config/pleroma.env
 ; Path to the Mix binary.
 ExecStart=/usr/bin/mix phx.server
 

lib/mix/tasks/pleroma/database.ex

@@ -10,6 +10,7 @@ defmodule Mix.Tasks.Pleroma.Database do
   alias Pleroma.User
   require Logger
   require Pleroma.Constants
+  import Ecto.Query
   import Mix.Pleroma
   use Mix.Task
 
@@ -53,8 +54,6 @@ defmodule Mix.Tasks.Pleroma.Database do
   end
 
   def run(["prune_objects" | args]) do
-    import Ecto.Query
-
     {options, [], []} =
       OptionParser.parse(
         args,
@@ -94,8 +93,6 @@ defmodule Mix.Tasks.Pleroma.Database do
   end
 
   def run(["fix_likes_collections"]) do
-    import Ecto.Query
-
     start_pleroma()
 
     from(object in Object,
@@ -130,4 +127,33 @@ defmodule Mix.Tasks.Pleroma.Database do
 
     Maintenance.vacuum(args)
   end
+
+  def run(["ensure_expiration"]) do
+    start_pleroma()
+    days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365)
+
+    Pleroma.Activity
+    |> join(:left, [a], u in assoc(a, :expiration))
+    |> join(:inner, [a, _u], o in Object,
+      on:
+        fragment(
+          "(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')",
+          o.data,
+          a.data,
+          a.data
+        )
+    )
+    |> where(local: true)
+    |> where([a, u], is_nil(u))
+    |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
+    |> where([_a, _u, o], fragment("?->>'type' = 'Note'", o.data))
+    |> Pleroma.RepoStreamer.chunk_stream(100)
+    |> Stream.each(fn activities ->
+      Enum.each(activities, fn activity ->
+        expires_at = Timex.shift(activity.inserted_at, days: days)
+        Pleroma.ActivityExpiration.create(activity, expires_at, false)
+      end)
+    end)
+    |> Stream.run()
+  end
 end

lib/mix/tasks/pleroma/release_env.ex

@@ -1,76 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Mix.Tasks.Pleroma.ReleaseEnv do
-  use Mix.Task
-  import Mix.Pleroma
-
-  @shortdoc "Generate Pleroma environment file."
-  @moduledoc File.read!("docs/administration/CLI_tasks/release_environments.md")
-
-  def run(["gen" | rest]) do
-    {options, [], []} =
-      OptionParser.parse(
-        rest,
-        strict: [
-          force: :boolean,
-          path: :string
-        ],
-        aliases: [
-          p: :path,
-          f: :force
-        ]
-      )
-
-    file_path =
-      get_option(
-        options,
-        :path,
-        "Environment file path",
-        "./config/pleroma.env"
-      )
-
-    env_path = Path.expand(file_path)
-
-    proceed? =
-      if File.exists?(env_path) do
-        get_option(
-          options,
-          :force,
-          "Environment file already exists. Do you want to overwrite the #{env_path} file? (y/n)",
-          "n"
-        ) === "y"
-      else
-        true
-      end
-
-    if proceed? do
-      case do_generate(env_path) do
-        {:error, reason} ->
-          shell_error(
-            File.Error.message(%{action: "write to file", reason: reason, path: env_path})
-          )
-
-        _ ->
-          shell_info("\nThe file generated: #{env_path}.\n")
-
-          shell_info("""
-          WARNING: before start pleroma app please make sure to make the file read-only and non-modifiable.
-            Example:
-              chmod 0444 #{file_path}
-              chattr +i #{file_path}
-          """)
-      end
-    else
-      shell_info("\nThe file is exist. #{env_path}.\n")
-    end
-  end
-
-  def do_generate(path) do
-    content = "RELEASE_COOKIE=#{Base.encode32(:crypto.strong_rand_bytes(32))}"
-
-    File.mkdir_p!(Path.dirname(path))
-    File.write(path, content)
-  end
-end

lib/pleroma/activity.ex

@@ -340,4 +340,10 @@ defmodule Pleroma.Activity do
       _ -> nil
     end
   end
+
+  @spec pinned_by_actor?(Activity.t()) :: boolean()
+  def pinned_by_actor?(%Activity{} = activity) do
+    actor = user_actor(activity)
+    activity.id in actor.pinned_activities
+  end
 end

lib/pleroma/activity_expiration.ex

@@ -20,11 +20,11 @@ defmodule Pleroma.ActivityExpiration do
     field(:scheduled_at, :naive_datetime)
   end
 
-  def changeset(%ActivityExpiration{} = expiration, attrs) do
+  def changeset(%ActivityExpiration{} = expiration, attrs, validate_scheduled_at) do
     expiration
     |> cast(attrs, [:scheduled_at])
     |> validate_required([:scheduled_at])
-    |> validate_scheduled_at()
+    |> validate_scheduled_at(validate_scheduled_at)
   end
 
   def get_by_activity_id(activity_id) do
@@ -33,9 +33,9 @@ defmodule Pleroma.ActivityExpiration do
     |> Repo.one()
   end
 
-  def create(%Activity{} = activity, scheduled_at) do
+  def create(%Activity{} = activity, scheduled_at, validate_scheduled_at \\ true) do
     %ActivityExpiration{activity_id: activity.id}
-    |> changeset(%{scheduled_at: scheduled_at})
+    |> changeset(%{scheduled_at: scheduled_at}, validate_scheduled_at)
     |> Repo.insert()
   end
 
@@ -46,10 +46,17 @@ defmodule Pleroma.ActivityExpiration do
 
     ActivityExpiration
     |> where([exp], exp.scheduled_at < ^naive_datetime)
+    |> limit(50)
+    |> preload(:activity)
     |> Repo.all()
+    |> Enum.reject(fn %{activity: activity} ->
+      Activity.pinned_by_actor?(activity)
+    end)
   end
 
-  def validate_scheduled_at(changeset) do
+  def validate_scheduled_at(changeset, false), do: changeset
+
+  def validate_scheduled_at(changeset, true) do
     validate_change(changeset, :scheduled_at, fn _, scheduled_at ->
       if not expires_late_enough?(scheduled_at) do
         [scheduled_at: "an ephemeral activity must live for at least one hour"]

lib/pleroma/application.ex

@@ -47,6 +47,7 @@ defmodule Pleroma.Application do
     Pleroma.ApplicationRequirements.verify!()
     setup_instrumenters()
     load_custom_modules()
+    check_system_commands()
     Pleroma.Docs.JSON.compile()
 
     adapter = Application.get_env(:tesla, :adapter)
@@ -249,4 +250,21 @@ defmodule Pleroma.Application do
   end
 
   defp http_children(_, _), do: []
+
+  defp check_system_commands do
+    filters = Config.get([Pleroma.Upload, :filters])
+
+    check_filter = fn filter, command_required ->
+      with true <- filter in filters,
+           false <- Pleroma.Utils.command_available?(command_required) do
+        Logger.error(
+          "#{filter} is specified in list of Pleroma.Upload filters, but the #{command_required} command is not found"
+        )
+      end
+    end
+
+    check_filter.(Pleroma.Upload.Filters.Exiftool, "exiftool")
+    check_filter.(Pleroma.Upload.Filters.Mogrify, "mogrify")
+    check_filter.(Pleroma.Upload.Filters.Mogrifun, "mogrify")
+  end
 end

lib/pleroma/config.ex

@@ -11,12 +11,10 @@ defmodule Pleroma.Config do
 
   def get([key], default), do: get(key, default)
 
-  def get([parent_key | keys], default) do
+  def get([_ | _] = path, default) do
-    case :pleroma
+    case fetch(path) do
-         |> Application.get_env(parent_key)
+      {:ok, value} -> value
-         |> get_in(keys) do
+      :error -> default
-      nil -> default
-      any -> any
     end
   end
 
@@ -34,6 +32,24 @@ defmodule Pleroma.Config do
     end
   end
 
+  def fetch(key) when is_atom(key), do: fetch([key])
+
+  def fetch([root_key | keys]) do
+    Enum.reduce_while(keys, Application.fetch_env(:pleroma, root_key), fn
+      key, {:ok, config} when is_map(config) or is_list(config) ->
+        case Access.fetch(config, key) do
+          :error ->
+            {:halt, :error}
+
+          value ->
+            {:cont, value}
+        end
+
+      _key, _config ->
+        {:halt, :error}
+    end)
+  end
+
   def put([key], value), do: put(key, value)
 
   def put([parent_key | keys], value) do
@@ -50,12 +66,15 @@ defmodule Pleroma.Config do
 
   def delete([key]), do: delete(key)
 
-  def delete([parent_key | keys]) do
+  def delete([parent_key | keys] = path) do
-    {_, parent} =
+    with {:ok, _} <- fetch(path) do
-      Application.get_env(:pleroma, parent_key)
+      {_, parent} =
-      |> get_and_update_in(keys, fn _ -> :pop end)
+        parent_key
+        |> get()
+        |> get_and_update_in(keys, fn _ -> :pop end)
 
-    Application.put_env(:pleroma, parent_key, parent)
+      Application.put_env(:pleroma, parent_key, parent)
+    end
   end
 
   def delete(key) do

lib/pleroma/following_relationship.ex

@@ -95,7 +95,11 @@ defmodule Pleroma.FollowingRelationship do
     |> where([r], r.state == ^:follow_accept)
   end
 
-  def followers_ap_ids(%User{} = user, from_ap_ids \\ nil) do
+  def followers_ap_ids(user, from_ap_ids \\ nil)
+
+  def followers_ap_ids(_, []), do: []
+
+  def followers_ap_ids(%User{} = user, from_ap_ids) do
     query =
       user
       |> followers_query()

lib/pleroma/job_queue_monitor.ex

@@ -15,8 +15,8 @@ defmodule Pleroma.JobQueueMonitor do
 
   @impl true
   def init(state) do
-    :telemetry.attach("oban-monitor-failure", [:oban, :failure], &handle_event/4, nil)
+    :telemetry.attach("oban-monitor-failure", [:oban, :job, :exception], &handle_event/4, nil)
-    :telemetry.attach("oban-monitor-success", [:oban, :success], &handle_event/4, nil)
+    :telemetry.attach("oban-monitor-success", [:oban, :job, :stop], &handle_event/4, nil)
 
     {:ok, state}
   end
@@ -25,8 +25,11 @@ defmodule Pleroma.JobQueueMonitor do
     GenServer.call(__MODULE__, :stats)
   end
 
-  def handle_event([:oban, status], %{duration: duration}, meta, _) do
+  def handle_event([:oban, :job, event], %{duration: duration}, meta, _) do
-    GenServer.cast(__MODULE__, {:process_event, status, duration, meta})
+    GenServer.cast(
+      __MODULE__,
+      {:process_event, mapping_status(event), duration, meta}
+    )
   end
 
   @impl true
@@ -75,4 +78,7 @@ defmodule Pleroma.JobQueueMonitor do
     |> Map.update!(:processed_jobs, &(&1 + 1))
     |> Map.update!(status, &(&1 + 1))
   end
+
+  defp mapping_status(:stop), do: :success
+  defp mapping_status(:exception), do: :failure
 end

lib/pleroma/object.ex

@@ -255,6 +255,10 @@ defmodule Pleroma.Object do
     end
   end
 
+  defp poll_is_multiple?(%Object{data: %{"anyOf" => [_ | _]}}), do: true
+
+  defp poll_is_multiple?(_), do: false
+
   def decrease_replies_count(ap_id) do
     Object
     |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id)))
@@ -281,10 +285,10 @@ defmodule Pleroma.Object do
   def increase_vote_count(ap_id, name, actor) do
     with %Object{} = object <- Object.normalize(ap_id),
          "Question" <- object.data["type"] do
-      multiple = Map.has_key?(object.data, "anyOf")
+      key = if poll_is_multiple?(object), do: "anyOf", else: "oneOf"
 
       options =
-        (object.data["anyOf"] || object.data["oneOf"] || [])
+        object.data[key]
         |> Enum.map(fn
           %{"name" => ^name} = option ->
             Kernel.update_in(option["replies"]["totalItems"], &(&1 + 1))
@@ -296,11 +300,8 @@ defmodule Pleroma.Object do
       voters = [actor | object.data["voters"] || []] |> Enum.uniq()
 
       data =
-        if multiple do
+        object.data
-          Map.put(object.data, "anyOf", options)
+        |> Map.put(key, options)
-        else
-          Map.put(object.data, "oneOf", options)
-        end
         |> Map.put("voters", voters)
 
       object

lib/pleroma/object/containment.ex

@@ -55,7 +55,7 @@ defmodule Pleroma.Object.Containment do
   defp compare_uris(_id_uri, _other_uri), do: :error
 
   @doc """
-  Checks that an imported AP object's actor matches the domain it came from.
+  Checks that an imported AP object's actor matches the host it came from.
   """
   def contain_origin(_id, %{"actor" => nil}), do: :error
 

lib/pleroma/object/fetcher.ex

@@ -9,6 +9,7 @@ defmodule Pleroma.Object.Fetcher do
   alias Pleroma.Repo
   alias Pleroma.Signature
   alias Pleroma.Web.ActivityPub.InternalFetchActor
+  alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Federator
 
@@ -23,21 +24,39 @@ defmodule Pleroma.Object.Fetcher do
     Ecto.Changeset.put_change(changeset, :updated_at, updated_at)
   end
 
-  defp maybe_reinject_internal_fields(data, %{data: %{} = old_data}) do
+  defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do
     internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields())
 
-    Map.merge(data, internal_fields)
+    Map.merge(new_data, internal_fields)
   end
 
-  defp maybe_reinject_internal_fields(data, _), do: data
+  defp maybe_reinject_internal_fields(_, new_data), do: new_data
 
   @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()}
-  defp reinject_object(struct, data) do
+  defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) do
-    Logger.debug("Reinjecting object #{data["id"]}")
+    Logger.debug("Reinjecting object #{new_data["id"]}")
 
-    with data <- Transmogrifier.fix_object(data),
+    with new_data <- Transmogrifier.fix_object(new_data),
-         data <- maybe_reinject_internal_fields(data, struct),
+         data <- maybe_reinject_internal_fields(object, new_data),
-         changeset <- Object.change(struct, %{data: data}),
+         {:ok, data, _} <- ObjectValidator.validate(data, %{}),
+         changeset <- Object.change(object, %{data: data}),
+         changeset <- touch_changeset(changeset),
+         {:ok, object} <- Repo.insert_or_update(changeset),
+         {:ok, object} <- Object.set_cache(object) do
+      {:ok, object}
+    else
+      e ->
+        Logger.error("Error while processing object: #{inspect(e)}")
+        {:error, e}
+    end
+  end
+
+  defp reinject_object(%Object{} = object, new_data) do
+    Logger.debug("Reinjecting object #{new_data["id"]}")
+
+    with new_data <- Transmogrifier.fix_object(new_data),
+         data <- maybe_reinject_internal_fields(object, new_data),
+         changeset <- Object.change(object, %{data: data}),
          changeset <- touch_changeset(changeset),
          {:ok, object} <- Repo.insert_or_update(changeset),
          {:ok, object} <- Object.set_cache(object) do
@@ -51,8 +70,8 @@ defmodule Pleroma.Object.Fetcher do
 
   def refetch_object(%Object{data: %{"id" => id}} = object) do
     with {:local, false} <- {:local, Object.local?(object)},
-         {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
+         {:ok, new_data} <- fetch_and_contain_remote_object_from_id(id),
-         {:ok, object} <- reinject_object(object, data) do
+         {:ok, object} <- reinject_object(object, new_data) do
       {:ok, object}
     else
       {:local, true} -> {:ok, object}

lib/pleroma/upload/filter/exiftool.ex

@@ -9,9 +9,17 @@ defmodule Pleroma.Upload.Filter.Exiftool do
   """
   @behaviour Pleroma.Upload.Filter
 
+  @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
-    System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true)
+    try do
-    :ok
+      case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do
+        {_response, 0} -> :ok
+        {error, 1} -> {:error, error}
+      end
+    rescue
+      _e in ErlangError ->
+        {:error, "exiftool command not found"}
+    end
   end
 
   def filter(_), do: :ok

lib/pleroma/upload/filter/mogrifun.ex

@@ -34,10 +34,15 @@ defmodule Pleroma.Upload.Filter.Mogrifun do
     [{"fill", "yellow"}, {"tint", "40"}]
   ]
 
+  @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
-    Filter.Mogrify.do_filter(file, [Enum.random(@filters)])
+    try do
-
+      Filter.Mogrify.do_filter(file, [Enum.random(@filters)])
-    :ok
+      :ok
+    rescue
+      _e in ErlangError ->
+        {:error, "mogrify command not found"}
+    end
   end
 
   def filter(_), do: :ok

lib/pleroma/upload/filter/mogrify.ex

@@ -8,11 +8,15 @@ defmodule Pleroma.Upload.Filter.Mogrify do
   @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
   @type conversions :: conversion() | [conversion()]
 
+  @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
-    filters = Pleroma.Config.get!([__MODULE__, :args])
+    try do
-
+      do_filter(file, Pleroma.Config.get!([__MODULE__, :args]))
-    do_filter(file, filters)
+      :ok
-    :ok
+    rescue
+      _e in ErlangError ->
+        {:error, "mogrify command not found"}
+    end
   end
 
   def filter(_), do: :ok

lib/pleroma/user.ex

@@ -638,6 +638,34 @@ defmodule Pleroma.User do
   @spec force_password_reset(User.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
   def force_password_reset(user), do: update_password_reset_pending(user, true)
 
+  # Used to auto-register LDAP accounts which won't have a password hash stored locally
+  def register_changeset_ldap(struct, params = %{password: password})
+      when is_nil(password) do
+    params = Map.put_new(params, :accepts_chat_messages, true)
+
+    params =
+      if Map.has_key?(params, :email) do
+        Map.put_new(params, :email, params[:email])
+      else
+        params
+      end
+
+    struct
+    |> cast(params, [
+      :name,
+      :nickname,
+      :email,
+      :accepts_chat_messages
+    ])
+    |> validate_required([:name, :nickname])
+    |> unique_constraint(:nickname)
+    |> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames]))
+    |> validate_format(:nickname, local_nickname_regex())
+    |> put_ap_id()
+    |> unique_constraint(:ap_id)
+    |> put_following_and_follower_address()
+  end
+
   def register_changeset(struct, params \\ %{}, opts \\ []) do
     bio_limit = Config.get([:instance, :user_bio_length], 5000)
     name_limit = Config.get([:instance, :user_name_length], 100)
@@ -676,10 +704,19 @@ defmodule Pleroma.User do
     |> validate_required([:name, :nickname, :password, :password_confirmation])
     |> validate_confirmation(:password)
     |> unique_constraint(:email)
+    |> validate_format(:email, @email_regex)
+    |> validate_change(:email, fn :email, email ->
+      valid? =
+        Config.get([User, :email_blacklist])
+        |> Enum.all?(fn blacklisted_domain ->
+          !String.ends_with?(email, ["@" <> blacklisted_domain, "." <> blacklisted_domain])
+        end)
+
+      if valid?, do: [], else: [email: "Invalid email"]
+    end)
     |> unique_constraint(:nickname)
     |> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames]))
     |> validate_format(:nickname, local_nickname_regex())
-    |> validate_format(:email, @email_regex)
     |> validate_length(:bio, max: bio_limit)
     |> validate_length(:name, min: 1, max: name_limit)
     |> validate_length(:registration_reason, max: reason_limit)
@@ -734,6 +771,7 @@ defmodule Pleroma.User do
          {:ok, user} <- set_cache(user),
          {:ok, _} <- send_welcome_email(user),
          {:ok, _} <- send_welcome_message(user),
+         {:ok, _} <- send_welcome_chat_message(user),
          {:ok, _} <- try_send_confirmation_email(user) do
       {:ok, user}
     end
@@ -748,6 +786,15 @@ defmodule Pleroma.User do
     end
   end
 
+  def send_welcome_chat_message(user) do
+    if User.WelcomeChatMessage.enabled?() do
+      User.WelcomeChatMessage.post_message(user)
+      {:ok, :enqueued}
+    else
+      {:ok, :noop}
+    end
+  end
+
   def send_welcome_email(%User{email: email} = user) when is_binary(email) do
     if User.WelcomeEmail.enabled?() do
       User.WelcomeEmail.send_email(user)

lib/pleroma/user/query.ex

@@ -130,6 +130,7 @@ defmodule Pleroma.User.Query do
   defp compose_query({:active, _}, query) do
     User.restrict_deactivated(query)
     |> where([u], not is_nil(u.nickname))
+    |> where([u], u.approval_pending == false)
   end
 
   defp compose_query({:legacy_active, _}, query) do

lib/pleroma/user/welcome_chat_message.ex

@@ -0,0 +1,45 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.User.WelcomeChatMessage do
+  alias Pleroma.Config
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
+
+  @spec enabled?() :: boolean()
+  def enabled?, do: Config.get([:welcome, :chat_message, :enabled], false)
+
+  @spec post_message(User.t()) :: {:ok, Pleroma.Activity.t() | nil}
+  def post_message(user) do
+    [:welcome, :chat_message, :sender_nickname]
+    |> Config.get(nil)
+    |> fetch_sender()
+    |> do_post(user, welcome_message())
+  end
+
+  defp do_post(%User{} = sender, recipient, message)
+       when is_binary(message) do
+    CommonAPI.post_chat_message(
+      sender,
+      recipient,
+      message
+    )
+  end
+
+  defp do_post(_sender, _recipient, _message), do: {:ok, nil}
+
+  defp fetch_sender(nickname) when is_binary(nickname) do
+    with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
+      user
+    else
+      _ -> nil
+    end
+  end
+
+  defp fetch_sender(_), do: nil
+
+  defp welcome_message do
+    Config.get([:welcome, :chat_message, :message], nil)
+  end
+end

lib/pleroma/utils.ex

@@ -9,4 +9,19 @@ defmodule Pleroma.Utils do
     |> Enum.map(&Path.join(dir, &1))
     |> Kernel.ParallelCompiler.compile()
   end
+
+  @doc """
+  POSIX-compliant check if command is available in the system
+
+  ## Examples
+      iex> command_available?("git")
+      true
+      iex> command_available?("wrongcmd")
+      false
+
+  """
+  @spec command_available?(String.t()) :: boolean()
+  def command_available?(command) do
+    match?({_output, 0}, System.cmd("sh", ["-c", "command -v #{command}"]))
+  end
 end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -66,7 +66,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp check_remote_limit(_), do: true
 
-  defp increase_note_count_if_public(actor, object) do
+  def increase_note_count_if_public(actor, object) do
     if is_public?(object), do: User.increase_note_count(actor), else: {:ok, actor}
   end
 
@@ -85,17 +85,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp increase_replies_count_if_reply(_create_data), do: :noop
 
-  defp increase_poll_votes_if_vote(%{
+  @object_types ["ChatMessage", "Question", "Answer"]
-         "object" => %{"inReplyTo" => reply_ap_id, "name" => name},
-         "type" => "Create",
-         "actor" => actor
-       }) do
-    Object.increase_vote_count(reply_ap_id, name, actor)
-  end
-
-  defp increase_poll_votes_if_vote(_create_data), do: :noop
-
-  @object_types ["ChatMessage"]
   @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
   def persist(%{"type" => type} = object, meta) when type in @object_types do
     with {:ok, object} <- Object.create(object) do
@@ -258,7 +248,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     with {:ok, activity} <- insert(create_data, local, fake),
          {:fake, false, activity} <- {:fake, fake, activity},
          _ <- increase_replies_count_if_reply(create_data),
-         _ <- increase_poll_votes_if_vote(create_data),
          {:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity},
          {:ok, _actor} <- increase_note_count_if_public(actor, activity),
          _ <- notify_and_stream(activity),
@@ -296,32 +285,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  @spec accept(map()) :: {:ok, Activity.t()} | {:error, any()}
-  def accept(params) do
-    accept_or_reject("Accept", params)
-  end
-
-  @spec reject(map()) :: {:ok, Activity.t()} | {:error, any()}
-  def reject(params) do
-    accept_or_reject("Reject", params)
-  end
-
-  @spec accept_or_reject(String.t(), map()) :: {:ok, Activity.t()} | {:error, any()}
-  defp accept_or_reject(type, %{to: to, actor: actor, object: object} = params) do
-    local = Map.get(params, :local, true)
-    activity_id = Map.get(params, :activity_id, nil)
-
-    data =
-      %{"to" => to, "type" => type, "actor" => actor.ap_id, "object" => object}
-      |> Maps.put_if_present("id", activity_id)
-
-    with {:ok, activity} <- insert(data, local),
-         _ <- notify_and_stream(activity),
-         :ok <- maybe_federate(activity) do
-      {:ok, activity}
-    end
-  end
-
   @spec unfollow(User.t(), User.t(), String.t() | nil, boolean()) ::
           {:ok, Activity.t()} | nil | {:error, any()}
   def unfollow(follower, followed, activity_id \\ nil, local \\ true) do

lib/pleroma/web/activity_pub/builder.ex

@@ -14,6 +14,28 @@ defmodule Pleroma.Web.ActivityPub.Builder do
 
   require Pleroma.Constants
 
+  def accept_or_reject(actor, activity, type) do
+    data = %{
+      "id" => Utils.generate_activity_id(),
+      "actor" => actor.ap_id,
+      "type" => type,
+      "object" => activity.data["id"],
+      "to" => [activity.actor]
+    }
+
+    {:ok, data, []}
+  end
+
+  @spec reject(User.t(), Activity.t()) :: {:ok, map(), keyword()}
+  def reject(actor, rejected_activity) do
+    accept_or_reject(actor, rejected_activity, "Reject")
+  end
+
+  @spec accept(User.t(), Activity.t()) :: {:ok, map(), keyword()}
+  def accept(actor, accepted_activity) do
+    accept_or_reject(actor, accepted_activity, "Accept")
+  end
+
   @spec follow(User.t(), User.t()) :: {:ok, map(), keyword()}
   def follow(follower, followed) do
     data = %{
@@ -80,6 +102,13 @@ defmodule Pleroma.Web.ActivityPub.Builder do
   end
 
   def create(actor, object, recipients) do
+    context =
+      if is_map(object) do
+        object["context"]
+      else
+        nil
+      end
+
     {:ok,
      %{
        "id" => Utils.generate_activity_id(),
@@ -88,7 +117,8 @@ defmodule Pleroma.Web.ActivityPub.Builder do
        "object" => object,
        "type" => "Create",
        "published" => DateTime.utc_now() |> DateTime.to_iso8601()
-     }, []}
+     }
+     |> Pleroma.Maps.put_if_present("context", context), []}
   end
 
   def chat_message(actor, recipient, content, opts \\ []) do
@@ -115,6 +145,22 @@ defmodule Pleroma.Web.ActivityPub.Builder do
     end
   end
 
+  def answer(user, object, name) do
+    {:ok,
+     %{
+       "type" => "Answer",
+       "actor" => user.ap_id,
+       "attributedTo" => user.ap_id,
+       "cc" => [object.data["actor"]],
+       "to" => [],
+       "name" => name,
+       "inReplyTo" => object.data["id"],
+       "context" => object.data["context"],
+       "published" => DateTime.utc_now() |> DateTime.to_iso8601(),
+       "id" => Utils.generate_object_id()
+     }, []}
+  end
+
   @spec tombstone(String.t(), String.t()) :: {:ok, map(), keyword()}
   def tombstone(actor, id) do
     {:ok,

lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex

@@ -21,8 +21,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do
   @impl true
   def describe, do: {:ok, %{}}
 
-  defp local?(%{"id" => id}) do
+  defp local?(%{"actor" => actor}) do
-    String.starts_with?(id, Pleroma.Web.Endpoint.url())
+    String.starts_with?(actor, Pleroma.Web.Endpoint.url())
   end
 
   defp note?(activity) do

lib/pleroma/web/activity_pub/mrf/object_age_policy.ex

@@ -37,8 +37,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
   defp check_delist(message, actions) do
     if :delist in actions do
       with %User{} = user <- User.get_cached_by_ap_id(message["actor"]) do
-        to = List.delete(message["to"], Pleroma.Constants.as_public()) ++ [user.follower_address]
+        to =
-        cc = List.delete(message["cc"], user.follower_address) ++ [Pleroma.Constants.as_public()]
+          List.delete(message["to"] || [], Pleroma.Constants.as_public()) ++
+            [user.follower_address]
+
+        cc =
+          List.delete(message["cc"] || [], user.follower_address) ++
+            [Pleroma.Constants.as_public()]
 
         message =
           message
@@ -58,8 +63,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
   defp check_strip_followers(message, actions) do
     if :strip_followers in actions do
       with %User{} = user <- User.get_cached_by_ap_id(message["actor"]) do
-        to = List.delete(message["to"], user.follower_address)
+        to = List.delete(message["to"] || [], user.follower_address)
-        cc = List.delete(message["cc"], user.follower_address)
+        cc = List.delete(message["cc"] || [], user.follower_address)
 
         message =
           message

lib/pleroma/web/activity_pub/mrf/simple_policy.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
   @behaviour Pleroma.Web.ActivityPub.MRF
 
   alias Pleroma.Config
+  alias Pleroma.FollowingRelationship
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.MRF
 
@@ -108,6 +109,35 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
     {:ok, object}
   end
 
+  defp intersection(list1, list2) do
+    list1 -- list1 -- list2
+  end
+
+  defp check_followers_only(%{host: actor_host} = _actor_info, object) do
+    followers_only =
+      Config.get([:mrf_simple, :followers_only])
+      |> MRF.subdomains_regex()
+
+    object =
+      with true <- MRF.subdomain_match?(followers_only, actor_host),
+           user <- User.get_cached_by_ap_id(object["actor"]) do
+        # Don't use Map.get/3 intentionally, these must not be nil
+        fixed_to = object["to"] || []
+        fixed_cc = object["cc"] || []
+
+        to = FollowingRelationship.followers_ap_ids(user, fixed_to)
+        cc = FollowingRelationship.followers_ap_ids(user, fixed_cc)
+
+        object
+        |> Map.put("to", intersection([user.follower_address | to], fixed_to))
+        |> Map.put("cc", intersection([user.follower_address | cc], fixed_cc))
+      else
+        _ -> object
+      end
+
+    {:ok, object}
+  end
+
   defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do
     report_removal =
       Config.get([:mrf_simple, :report_removal])
@@ -174,6 +204,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
          {:ok, object} <- check_media_removal(actor_info, object),
          {:ok, object} <- check_media_nsfw(actor_info, object),
          {:ok, object} <- check_ftl_removal(actor_info, object),
+         {:ok, object} <- check_followers_only(actor_info, object),
          {:ok, object} <- check_report_removal(actor_info, object) do
       {:ok, object}
     else

lib/pleroma/web/activity_pub/object_validator.ex

@@ -13,20 +13,35 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object
   alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator
 
   @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()}
   def validate(object, meta)
 
+  def validate(%{"type" => type} = object, meta)
+      when type in ~w[Accept Reject] do
+    with {:ok, object} <-
+           object
+           |> AcceptRejectValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object)
+      {:ok, object, meta}
+    end
+  end
+
   def validate(%{"type" => "Follow"} = object, meta) do
     with {:ok, object} <-
            object
@@ -112,17 +127,40 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     end
   end
 
+  def validate(%{"type" => "Question"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> QuestionValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object)
+      {:ok, object, meta}
+    end
+  end
+
+  def validate(%{"type" => "Answer"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> AnswerValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object)
+      {:ok, object, meta}
+    end
+  end
+
   def validate(%{"type" => "EmojiReact"} = object, meta) do
     with {:ok, object} <-
            object
            |> EmojiReactValidator.cast_and_validate()
            |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object |> Map.from_struct())
+      object = stringify_keys(object)
       {:ok, object, meta}
     end
   end
 
-  def validate(%{"type" => "Create", "object" => object} = create_activity, meta) do
+  def validate(
+        %{"type" => "Create", "object" => %{"type" => "ChatMessage"} = object} = create_activity,
+        meta
+      ) do
     with {:ok, object_data} <- cast_and_apply(object),
          meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
          {:ok, create_activity} <-
@@ -134,12 +172,28 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     end
   end
 
+  def validate(
+        %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity,
+        meta
+      )
+      when objtype in ["Question", "Answer"] do
+    with {:ok, object_data} <- cast_and_apply(object),
+         meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
+         {:ok, create_activity} <-
+           create_activity
+           |> CreateGenericValidator.cast_and_validate(meta)
+           |> Ecto.Changeset.apply_action(:insert) do
+      create_activity = stringify_keys(create_activity)
+      {:ok, create_activity, meta}
+    end
+  end
+
   def validate(%{"type" => "Announce"} = object, meta) do
     with {:ok, object} <-
            object
            |> AnnounceValidator.cast_and_validate()
            |> Ecto.Changeset.apply_action(:insert) do
-      object = stringify_keys(object |> Map.from_struct())
+      object = stringify_keys(object)
       {:ok, object, meta}
     end
   end
@@ -148,8 +202,17 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     ChatMessageValidator.cast_and_apply(object)
   end
 
+  def cast_and_apply(%{"type" => "Question"} = object) do
+    QuestionValidator.cast_and_apply(object)
+  end
+
+  def cast_and_apply(%{"type" => "Answer"} = object) do
+    AnswerValidator.cast_and_apply(object)
+  end
+
   def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
 
+  # is_struct/1 isn't present in Elixir 1.8.x
   def stringify_keys(%{__struct__: _} = object) do
     object
     |> Map.from_struct()

lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex

@@ -0,0 +1,56 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator do
+  use Ecto.Schema
+
+  alias Pleroma.Activity
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+
+  import Ecto.Changeset
+  import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+
+  @primary_key false
+
+  embedded_schema do
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:type, :string)
+    field(:object, ObjectValidators.ObjectID)
+    field(:actor, ObjectValidators.ObjectID)
+    field(:to, ObjectValidators.Recipients, default: [])
+    field(:cc, ObjectValidators.Recipients, default: [])
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> cast(data, __schema__(:fields))
+  end
+
+  def validate_data(cng) do
+    cng
+    |> validate_required([:id, :type, :actor, :to, :cc, :object])
+    |> validate_inclusion(:type, ["Accept", "Reject"])
+    |> validate_actor_presence()
+    |> validate_object_presence(allowed_types: ["Follow"])
+    |> validate_accept_reject_rights()
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data
+    |> validate_data
+  end
+
+  def validate_accept_reject_rights(cng) do
+    with object_id when is_binary(object_id) <- get_field(cng, :object),
+         %Activity{data: %{"object" => followed_actor}} <- Activity.get_by_ap_id(object_id),
+         true <- followed_actor == get_field(cng, :actor) do
+      cng
+    else
+      _e ->
+        cng
+        |> add_error(:actor, "can't accept or reject the given activity")
+    end
+  end
+end

lib/pleroma/web/activity_pub/object_validators/answer_validator.ex

@@ -0,0 +1,65 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator do
+  use Ecto.Schema
+
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+
+  import Ecto.Changeset
+
+  @primary_key false
+  @derive Jason.Encoder
+
+  embedded_schema do
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:to, {:array, :string}, default: [])
+    field(:cc, {:array, :string}, default: [])
+
+    # is this actually needed?
+    field(:bto, {:array, :string}, default: [])
+    field(:bcc, {:array, :string}, default: [])
+
+    field(:type, :string)
+    field(:name, :string)
+    field(:inReplyTo, :string)
+    field(:attributedTo, ObjectValidators.ObjectID)
+
+    # TODO: Remove actor on objects
+    field(:actor, ObjectValidators.ObjectID)
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data()
+    |> apply_action(:insert)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+    |> validate_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  def changeset(struct, data) do
+    struct
+    |> cast(data, __schema__(:fields))
+  end
+
+  def validate_data(data_cng) do
+    data_cng
+    |> validate_inclusion(:type, ["Answer"])
+    |> validate_required([:id, :inReplyTo, :name, :attributedTo, :actor])
+    |> CommonValidations.validate_any_presence([:cc, :to])
+    |> CommonValidations.validate_fields_match([:actor, :attributedTo])
+    |> CommonValidations.validate_actor_presence()
+    |> CommonValidations.validate_host_match()
+  end
+end

lib/pleroma/web/activity_pub/object_validators/common_validations.ex

@@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
   alias Pleroma.Object
   alias Pleroma.User
 
-  def validate_recipients_presence(cng, fields \\ [:to, :cc]) do
+  def validate_any_presence(cng, fields) do
     non_empty =
       fields
       |> Enum.map(fn field -> get_field(cng, field) end)
@@ -24,7 +24,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
       fields
       |> Enum.reduce(cng, fn field, cng ->
         cng
-        |> add_error(field, "no recipients in any field")
+        |> add_error(field, "none of #{inspect(fields)} present")
       end)
     end
   end
@@ -34,10 +34,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
 
     cng
     |> validate_change(field_name, fn field_name, actor ->
-      if User.get_cached_by_ap_id(actor) do
+      case User.get_cached_by_ap_id(actor) do
-        []
+        %User{deactivated: true} ->
-      else
+          [{field_name, "user is deactivated"}]
-        [{field_name, "can't find user"}]
+
+        %User{} ->
+          []
+
+        _ ->
+          [{field_name, "can't find user"}]
       end
     end)
   end
@@ -77,4 +82,60 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations do
 
     if actor_cng.valid?, do: actor_cng, else: object_cng
   end
+
+  def validate_host_match(cng, fields \\ [:id, :actor]) do
+    if same_domain?(cng, fields) do
+      cng
+    else
+      fields
+      |> Enum.reduce(cng, fn field, cng ->
+        cng
+        |> add_error(field, "hosts of #{inspect(fields)} aren't matching")
+      end)
+    end
+  end
+
+  def validate_fields_match(cng, fields) do
+    if map_unique?(cng, fields) do
+      cng
+    else
+      fields
+      |> Enum.reduce(cng, fn field, cng ->
+        cng
+        |> add_error(field, "Fields #{inspect(fields)} aren't matching")
+      end)
+    end
+  end
+
+  defp map_unique?(cng, fields, func \\ & &1) do
+    Enum.reduce_while(fields, nil, fn field, acc ->
+      value =
+        cng
+        |> get_field(field)
+        |> func.()
+
+      case {value, acc} do
+        {value, nil} -> {:cont, value}
+        {value, value} -> {:cont, value}
+        _ -> {:halt, false}
+      end
+    end)
+  end
+
+  def same_domain?(cng, fields \\ [:actor, :object]) do
+    map_unique?(cng, fields, fn value -> URI.parse(value).host end)
+  end
+
+  # This figures out if a user is able to create, delete or modify something
+  # based on the domain and superuser status
+  def validate_modification_rights(cng) do
+    actor = User.get_cached_by_ap_id(get_field(cng, :actor))
+
+    if User.superuser?(actor) || same_domain?(cng) do
+      cng
+    else
+      cng
+      |> add_error(:actor, "is not allowed to modify object")
+    end
+  end
 end

lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex

@@ -0,0 +1,133 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+# Code based on CreateChatMessageValidator
+# NOTES
+# - doesn't embed, will only get the object id
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
+  use Ecto.Schema
+
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.Object
+
+  import Ecto.Changeset
+  import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+
+  @primary_key false
+
+  embedded_schema do
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:actor, ObjectValidators.ObjectID)
+    field(:type, :string)
+    field(:to, ObjectValidators.Recipients, default: [])
+    field(:cc, ObjectValidators.Recipients, default: [])
+    field(:object, ObjectValidators.ObjectID)
+    field(:expires_at, ObjectValidators.DateTime)
+
+    # Should be moved to object, done for CommonAPI.Utils.make_context
+    field(:context, :string)
+  end
+
+  def cast_data(data, meta \\ []) do
+    data = fix(data, meta)
+
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data
+    |> apply_action(:insert)
+  end
+
+  def cast_and_validate(data, meta \\ []) do
+    data
+    |> cast_data(meta)
+    |> validate_data(meta)
+  end
+
+  def changeset(struct, data) do
+    struct
+    |> cast(data, __schema__(:fields))
+  end
+
+  defp fix_context(data, meta) do
+    if object = meta[:object_data] do
+      Map.put_new(data, "context", object["context"])
+    else
+      data
+    end
+  end
+
+  defp fix(data, meta) do
+    data
+    |> fix_context(meta)
+  end
+
+  def validate_data(cng, meta \\ []) do
+    cng
+    |> validate_required([:actor, :type, :object])
+    |> validate_inclusion(:type, ["Create"])
+    |> validate_actor_presence()
+    |> validate_any_presence([:to, :cc])
+    |> validate_actors_match(meta)
+    |> validate_context_match(meta)
+    |> validate_object_nonexistence()
+    |> validate_object_containment()
+  end
+
+  def validate_object_containment(cng) do
+    actor = get_field(cng, :actor)
+
+    cng
+    |> validate_change(:object, fn :object, object_id ->
+      %URI{host: object_id_host} = URI.parse(object_id)
+      %URI{host: actor_host} = URI.parse(actor)
+
+      if object_id_host == actor_host do
+        []
+      else
+        [{:object, "The host of the object id doesn't match with the host of the actor"}]
+      end
+    end)
+  end
+
+  def validate_object_nonexistence(cng) do
+    cng
+    |> validate_change(:object, fn :object, object_id ->
+      if Object.get_cached_by_ap_id(object_id) do
+        [{:object, "The object to create already exists"}]
+      else
+        []
+      end
+    end)
+  end
+
+  def validate_actors_match(cng, meta) do
+    attributed_to = meta[:object_data]["attributedTo"] || meta[:object_data]["actor"]
+
+    cng
+    |> validate_change(:actor, fn :actor, actor ->
+      if actor == attributed_to do
+        []
+      else
+        [{:actor, "Actor doesn't match with object attributedTo"}]
+      end
+    end)
+  end
+
+  def validate_context_match(cng, %{object_data: %{"context" => object_context}}) do
+    cng
+    |> validate_change(:context, fn :context, context ->
+      if context == object_context do
+        []
+      else
+        [{:context, "context field not matching between Create and object (#{object_context})"}]
+      end
+    end)
+  end
+
+  def validate_context_match(cng, _), do: cng
+end

lib/pleroma/web/activity_pub/object_validators/delete_validator.ex

@@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do
 
   alias Pleroma.Activity
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
-  alias Pleroma.User
 
   import Ecto.Changeset
   import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
@@ -59,7 +58,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do
     |> validate_required([:id, :type, :actor, :to, :cc, :object])
     |> validate_inclusion(:type, ["Delete"])
     |> validate_actor_presence()
-    |> validate_deletion_rights()
+    |> validate_modification_rights()
     |> validate_object_or_user_presence(allowed_types: @deletable_types)
     |> add_deleted_activity_id()
   end
@@ -68,31 +67,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do
     !same_domain?(cng)
   end
 
-  defp same_domain?(cng) do
-    actor_uri =
-      cng
-      |> get_field(:actor)
-      |> URI.parse()
-
-    object_uri =
-      cng
-      |> get_field(:object)
-      |> URI.parse()
-
-    object_uri.host == actor_uri.host
-  end
-
-  def validate_deletion_rights(cng) do
-    actor = User.get_cached_by_ap_id(get_field(cng, :actor))
-
-    if User.superuser?(actor) || same_domain?(cng) do
-      cng
-    else
-      cng
-      |> add_error(:actor, "is not allowed to delete object")
-    end
-  end
-
   def cast_and_validate(data) do
     data
     |> cast_data

lib/pleroma/web/activity_pub/object_validators/note_validator.ex

@@ -34,7 +34,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
     field(:replies_count, :integer, default: 0)
     field(:like_count, :integer, default: 0)
     field(:announcement_count, :integer, default: 0)
-    field(:inRepyTo, :string)
+    field(:inReplyTo, :string)
     field(:uri, ObjectValidators.Uri)
 
     field(:likes, {:array, :string}, default: [])

lib/pleroma/web/activity_pub/object_validators/question_options_validator.ex

@@ -0,0 +1,37 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+
+  @primary_key false
+
+  embedded_schema do
+    field(:name, :string)
+
+    embeds_one :replies, Replies, primary_key: false do
+      field(:totalItems, :integer)
+      field(:type, :string)
+    end
+
+    field(:type, :string)
+  end
+
+  def changeset(struct, data) do
+    struct
+    |> cast(data, [:name, :type])
+    |> cast_embed(:replies, with: &replies_changeset/2)
+    |> validate_inclusion(:type, ["Note"])
+    |> validate_required([:name, :type])
+  end
+
+  def replies_changeset(struct, data) do
+    struct
+    |> cast(data, [:totalItems, :type])
+    |> validate_inclusion(:type, ["Collection"])
+    |> validate_required([:type])
+  end
+end

lib/pleroma/web/activity_pub/object_validators/question_validator.ex

@@ -0,0 +1,127 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
+  use Ecto.Schema
+
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator
+  alias Pleroma.Web.ActivityPub.Utils
+
+  import Ecto.Changeset
+
+  @primary_key false
+  @derive Jason.Encoder
+
+  # Extends from NoteValidator
+  embedded_schema do
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:to, {:array, :string}, default: [])
+    field(:cc, {:array, :string}, default: [])
+    field(:bto, {:array, :string}, default: [])
+    field(:bcc, {:array, :string}, default: [])
+    # TODO: Write type
+    field(:tag, {:array, :map}, default: [])
+    field(:type, :string)
+    field(:content, :string)
+    field(:context, :string)
+
+    # TODO: Remove actor on objects
+    field(:actor, ObjectValidators.ObjectID)
+
+    field(:attributedTo, ObjectValidators.ObjectID)
+    field(:summary, :string)
+    field(:published, ObjectValidators.DateTime)
+    # TODO: Write type
+    field(:emoji, :map, default: %{})
+    field(:sensitive, :boolean, default: false)
+    embeds_many(:attachment, AttachmentValidator)
+    field(:replies_count, :integer, default: 0)
+    field(:like_count, :integer, default: 0)
+    field(:announcement_count, :integer, default: 0)
+    field(:inReplyTo, :string)
+    field(:uri, ObjectValidators.Uri)
+    # short identifier for PleromaFE to group statuses by context
+    field(:context_id, :integer)
+
+    field(:likes, {:array, :string}, default: [])
+    field(:announcements, {:array, :string}, default: [])
+
+    field(:closed, ObjectValidators.DateTime)
+    field(:voters, {:array, ObjectValidators.ObjectID}, default: [])
+    embeds_many(:anyOf, QuestionOptionsValidator)
+    embeds_many(:oneOf, QuestionOptionsValidator)
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data
+    |> apply_action(:insert)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+    |> validate_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  defp fix_closed(data) do
+    cond do
+      is_binary(data["closed"]) -> data
+      is_binary(data["endTime"]) -> Map.put(data, "closed", data["endTime"])
+      true -> Map.drop(data, ["closed"])
+    end
+  end
+
+  # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults
+  defp fix_defaults(data) do
+    %{data: %{"id" => context}, id: context_id} =
+      Utils.create_context(data["context"] || data["conversation"])
+
+    data
+    |> Map.put_new_lazy("published", &Utils.make_date/0)
+    |> Map.put_new("context", context)
+    |> Map.put_new("context_id", context_id)
+  end
+
+  defp fix_attribution(data) do
+    data
+    |> Map.put_new("actor", data["attributedTo"])
+  end
+
+  defp fix(data) do
+    data
+    |> fix_attribution()
+    |> fix_closed()
+    |> fix_defaults()
+  end
+
+  def changeset(struct, data) do
+    data = fix(data)
+
+    struct
+    |> cast(data, __schema__(:fields) -- [:anyOf, :oneOf, :attachment])
+    |> cast_embed(:attachment)
+    |> cast_embed(:anyOf)
+    |> cast_embed(:oneOf)
+  end
+
+  def validate_data(data_cng) do
+    data_cng
+    |> validate_inclusion(:type, ["Question"])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context])
+    |> CommonValidations.validate_any_presence([:cc, :to])
+    |> CommonValidations.validate_fields_match([:actor, :attributedTo])
+    |> CommonValidations.validate_actor_presence()
+    |> CommonValidations.validate_any_presence([:oneOf, :anyOf])
+    |> CommonValidations.validate_host_match()
+  end
+end

lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex

@@ -13,7 +13,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator do
   embedded_schema do
     field(:type, :string)
     field(:href, ObjectValidators.Uri)
-    field(:mediaType, :string)
+    field(:mediaType, :string, default: "application/octet-stream")
   end
 
   def changeset(struct, data) do

lib/pleroma/web/activity_pub/side_effects.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   """
   alias Pleroma.Activity
   alias Pleroma.Activity.Ir.Topics
+  alias Pleroma.ActivityExpiration
   alias Pleroma.Chat
   alias Pleroma.Chat.MessageReference
   alias Pleroma.FollowingRelationship
@@ -15,13 +16,70 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Builder
   alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Push
   alias Pleroma.Web.Streamer
+  alias Pleroma.Workers.BackgroundWorker
+
+  require Logger
 
   def handle(object, meta \\ [])
 
+  # Task this handles
+  # - Follows
+  # - Sends a notification
+  def handle(
+        %{
+          data: %{
+            "actor" => actor,
+            "type" => "Accept",
+            "object" => follow_activity_id
+          }
+        } = object,
+        meta
+      ) do
+    with %Activity{actor: follower_id} = follow_activity <-
+           Activity.get_by_ap_id(follow_activity_id),
+         %User{} = followed <- User.get_cached_by_ap_id(actor),
+         %User{} = follower <- User.get_cached_by_ap_id(follower_id),
+         {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "accept"),
+         {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_accept) do
+      Notification.update_notification_type(followed, follow_activity)
+      User.update_follower_count(followed)
+      User.update_following_count(follower)
+    end
+
+    {:ok, object, meta}
+  end
+
+  # Task this handles
+  # - Rejects all existing follow activities for this person
+  # - Updates the follow state
+  # - Dismisses notification
+  def handle(
+        %{
+          data: %{
+            "actor" => actor,
+            "type" => "Reject",
+            "object" => follow_activity_id
+          }
+        } = object,
+        meta
+      ) do
+    with %Activity{actor: follower_id} = follow_activity <-
+           Activity.get_by_ap_id(follow_activity_id),
+         %User{} = followed <- User.get_cached_by_ap_id(actor),
+         %User{} = follower <- User.get_cached_by_ap_id(follower_id),
+         {:ok, _follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "reject") do
+      FollowingRelationship.update(follower, followed, :follow_reject)
+      Notification.dismiss(follow_activity)
+    end
+
+    {:ok, object, meta}
+  end
+
   # Tasks this handle
   # - Follows if possible
   # - Sends a notification
@@ -42,33 +100,13 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
          {_, {:ok, _}, _, _} <-
            {:following, User.follow(follower, followed, :follow_pending), follower, followed} do
       if followed.local && !followed.locked do
-        Utils.update_follow_state_for_all(object, "accept")
+        {:ok, accept_data, _} = Builder.accept(followed, object)
-        FollowingRelationship.update(follower, followed, :follow_accept)
+        {:ok, _activity, _} = Pipeline.common_pipeline(accept_data, local: true)
-        User.update_follower_count(followed)
-        User.update_following_count(follower)
-
-        %{
-          to: [following_user],
-          actor: followed,
-          object: follow_id,
-          local: true
-        }
-        |> ActivityPub.accept()
       end
     else
-      {:following, {:error, _}, follower, followed} ->
+      {:following, {:error, _}, _follower, followed} ->
-        Utils.update_follow_state_for_all(object, "reject")
+        {:ok, reject_data, _} = Builder.reject(followed, object)
-        FollowingRelationship.update(follower, followed, :follow_reject)
+        {:ok, _activity, _} = Pipeline.common_pipeline(reject_data, local: true)
-
-        if followed.local do
-          %{
-            to: [follower.ap_id],
-            actor: followed,
-            object: follow_id,
-            local: true
-          }
-          |> ActivityPub.reject()
-        end
 
       _ ->
         nil
@@ -135,10 +173,26 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   # Tasks this handles
   # - Actually create object
   # - Rollback if we couldn't create it
+  # - Increase the user note count
+  # - Increase the reply count
+  # - Increase replies count
+  # - Set up ActivityExpiration
   # - Set up notifications
   def handle(%{data: %{"type" => "Create"}} = activity, meta) do
-    with {:ok, _object, meta} <- handle_object_creation(meta[:object_data], meta) do
+    with {:ok, object, meta} <- handle_object_creation(meta[:object_data], meta),
+         %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do
       {:ok, notifications} = Notification.create_notifications(activity, do_send: false)
+      {:ok, _user} = ActivityPub.increase_note_count_if_public(user, object)
+
+      if in_reply_to = object.data["inReplyTo"] do
+        Object.increase_replies_count(in_reply_to)
+      end
+
+      if expires_at = activity.data["expires_at"] do
+        ActivityExpiration.create(activity, expires_at)
+      end
+
+      BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id})
 
       meta =
         meta
@@ -199,13 +253,15 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   # - Stream out the activity
   def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, meta) do
     deleted_object =
-      Object.normalize(deleted_object, false) || User.get_cached_by_ap_id(deleted_object)
+      Object.normalize(deleted_object, false) ||
+        User.get_cached_by_ap_id(deleted_object)
 
     result =
       case deleted_object do
         %Object{} ->
           with {:ok, deleted_object, activity} <- Object.delete(deleted_object),
-               %User{} = user <- User.get_cached_by_ap_id(deleted_object.data["actor"]) do
+               {_, actor} when is_binary(actor) <- {:actor, deleted_object.data["actor"]},
+               %User{} = user <- User.get_cached_by_ap_id(actor) do
             User.remove_pinnned_activity(user, activity)
 
             {:ok, user} = ActivityPub.decrease_note_count_if_public(user, deleted_object)
@@ -219,6 +275,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
             ActivityPub.stream_out(object)
             ActivityPub.stream_out_participations(deleted_object, user)
             :ok
+          else
+            {:actor, _} ->
+              Logger.error("The object doesn't have an actor: #{inspect(deleted_object)}")
+              :no_object_actor
           end
 
         %User{} ->
@@ -268,9 +328,27 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
     end
   end
 
+  def handle_object_creation(%{"type" => "Answer"} = object_map, meta) do
+    with {:ok, object, meta} <- Pipeline.common_pipeline(object_map, meta) do
+      Object.increase_vote_count(
+        object.data["inReplyTo"],
+        object.data["name"],
+        object.data["actor"]
+      )
+
+      {:ok, object, meta}
+    end
+  end
+
+  def handle_object_creation(%{"type" => "Question"} = object, meta) do
+    with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
+      {:ok, object, meta}
+    end
+  end
+
   # Nothing to do
-  def handle_object_creation(object) do
+  def handle_object_creation(object, meta) do
-    {:ok, object}
+    {:ok, object, meta}
   end
 
   defp undo_like(nil, object), do: delete_object(object)

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -9,9 +9,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   alias Pleroma.Activity
   alias Pleroma.EarmarkRenderer
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
-  alias Pleroma.FollowingRelationship
   alias Pleroma.Maps
-  alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Object.Containment
   alias Pleroma.Repo
@@ -157,7 +155,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def fix_actor(%{"attributedTo" => actor} = object) do
-    Map.put(object, "actor", Containment.get_actor(%{"actor" => actor}))
+    actor = Containment.get_actor(%{"actor" => actor})
+
+    # TODO: Remove actor field for Objects
+    object
+    |> Map.put("actor", actor)
+    |> Map.put("attributedTo", actor)
   end
 
   def fix_in_reply_to(object, options \\ [])
@@ -240,13 +243,17 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
         if href do
           attachment_url =
-            %{"href" => href}
+            %{
+              "href" => href,
+              "type" => Map.get(url || %{}, "type", "Link")
+            }
             |> Maps.put_if_present("mediaType", media_type)
-            |> Maps.put_if_present("type", Map.get(url || %{}, "type"))
 
-          %{"url" => [attachment_url]}
+          %{
+            "url" => [attachment_url],
+            "type" => data["type"] || "Document"
+          }
           |> Maps.put_if_present("mediaType", media_type)
-          |> Maps.put_if_present("type", data["type"])
           |> Maps.put_if_present("name", data["name"])
         else
           nil
@@ -382,32 +389,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   defp fix_content(object), do: object
 
-  defp mastodon_follow_hack(%{"id" => id, "actor" => follower_id}, followed) do
-    with true <- id =~ "follows",
-         %User{local: true} = follower <- User.get_cached_by_ap_id(follower_id),
-         %Activity{} = activity <- Utils.fetch_latest_follow(follower, followed) do
-      {:ok, activity}
-    else
-      _ -> {:error, nil}
-    end
-  end
-
-  defp mastodon_follow_hack(_, _), do: {:error, nil}
-
-  defp get_follow_activity(follow_object, followed) do
-    with object_id when not is_nil(object_id) <- Utils.get_ap_id(follow_object),
-         {_, %Activity{} = activity} <- {:activity, Activity.get_by_ap_id(object_id)} do
-      {:ok, activity}
-    else
-      # Can't find the activity. This might a Mastodon 2.3 "Accept"
-      {:activity, nil} ->
-        mastodon_follow_hack(follow_object, followed)
-
-      _ ->
-        {:error, nil}
-    end
-  end
-
   # Reduce the object list to find the reported user.
   defp get_reported(objects) do
     Enum.reduce_while(objects, nil, fn ap_id, _ ->
@@ -419,6 +400,29 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     end)
   end
 
+  # Compatibility wrapper for Mastodon votes
+  defp handle_create(%{"object" => %{"type" => "Answer"}} = data, _user) do
+    handle_incoming(data)
+  end
+
+  defp handle_create(%{"object" => object} = data, user) do
+    %{
+      to: data["to"],
+      object: object,
+      actor: user,
+      context: object["context"],
+      local: false,
+      published: data["published"],
+      additional:
+        Map.take(data, [
+          "cc",
+          "directMessage",
+          "id"
+        ])
+    }
+    |> ActivityPub.create()
+  end
+
   def handle_incoming(data, options \\ [])
 
   # Flag objects are placed ahead of the ID check because Mastodon 2.8 and earlier send them
@@ -457,30 +461,18 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
         options
       )
-      when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer", "Audio"] do
+      when objtype in ["Article", "Event", "Note", "Video", "Page", "Audio"] do
     actor = Containment.get_actor(data)
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
-         {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(actor),
+         {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(actor) do
-         data <- Map.put(data, "actor", actor) |> fix_addressing() do
+      data =
-      object = fix_object(object, options)
+        data
+        |> Map.put("object", fix_object(object, options))
+        |> Map.put("actor", actor)
+        |> fix_addressing()
 
-      params = %{
+      with {:ok, created_activity} <- handle_create(data, user) do
-        to: data["to"],
-        object: object,
-        actor: user,
-        context: object["context"],
-        local: false,
-        published: data["published"],
-        additional:
-          Map.take(data, [
-            "cc",
-            "directMessage",
-            "id"
-          ])
-      }
-
-      with {:ok, created_activity} <- ActivityPub.create(params) do
         reply_depth = (options[:depth] || 0) + 1
 
         if Federator.allowed_thread_distance?(reply_depth) do
@@ -531,60 +523,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     end
   end
 
-  def handle_incoming(
-        %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => id} = data,
-        _options
-      ) do
-    with actor <- Containment.get_actor(data),
-         {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor),
-         {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
-         {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "accept"),
-         %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
-         {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_accept) do
-      User.update_follower_count(followed)
-      User.update_following_count(follower)
-
-      Notification.update_notification_type(followed, follow_activity)
-
-      ActivityPub.accept(%{
-        to: follow_activity.data["to"],
-        type: "Accept",
-        actor: followed,
-        object: follow_activity.data["id"],
-        local: false,
-        activity_id: id
-      })
-    else
-      _e ->
-        :error
-    end
-  end
-
-  def handle_incoming(
-        %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => id} = data,
-        _options
-      ) do
-    with actor <- Containment.get_actor(data),
-         {:ok, %User{} = followed} <- User.get_or_fetch_by_ap_id(actor),
-         {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
-         {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "reject"),
-         %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
-         {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_reject),
-         {:ok, activity} <-
-           ActivityPub.reject(%{
-             to: follow_activity.data["to"],
-             type: "Reject",
-             actor: followed,
-             object: follow_activity.data["id"],
-             local: false,
-             activity_id: id
-           }) do
-      {:ok, activity}
-    else
-      _e -> :error
-    end
-  end
-
   @misskey_reactions %{
     "like" => "👍",
     "love" => "❤️",
@@ -614,9 +552,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def handle_incoming(
-        %{"type" => "Create", "object" => %{"type" => "ChatMessage"}} = data,
+        %{"type" => "Create", "object" => %{"type" => objtype}} = data,
         _options
-      ) do
+      )
+      when objtype in ["Question", "Answer", "ChatMessage"] do
     with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
          {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}
@@ -638,9 +577,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => type} = data,
         _options
       )
-      when type in ~w{Update Block Follow} do
+      when type in ~w{Update Block Follow Accept Reject} do
     with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
-         {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
+         {:ok, activity, _} <-
+           Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}
     end
   end
@@ -649,7 +589,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Delete"} = data,
         _options
       ) do
-    with {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
+    with {:ok, activity, _} <-
+           Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}
     else
       {:error, {:validate_object, _}} = e ->

lib/pleroma/web/auth/ldap_authenticator.ex

@@ -28,10 +28,6 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
          %User{} = user <- ldap_user(name, password) do
       {:ok, user}
     else
-      {:error, {:ldap_connection_error, _}} ->
-        # When LDAP is unavailable, try default authenticator
-        @base.get_user(conn)
-
       {:ldap, _} ->
         @base.get_user(conn)
 
@@ -92,7 +88,7 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
             user
 
           _ ->
-            register_user(connection, base, uid, name, password)
+            register_user(connection, base, uid, name)
         end
 
       error ->
@@ -100,34 +96,31 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do
     end
   end
 
-  defp register_user(connection, base, uid, name, password) do
+  defp register_user(connection, base, uid, name) do
     case :eldap.search(connection, [
            {:base, to_charlist(base)},
            {:filter, :eldap.equalityMatch(to_charlist(uid), to_charlist(name))},
            {:scope, :eldap.wholeSubtree()},
-           {:attributes, ['mail', 'email']},
            {:timeout, @search_timeout}
          ]) do
       {:ok, {:eldap_search_result, [{:eldap_entry, _, attributes}], _}} ->
-        with {_, [mail]} <- List.keyfind(attributes, 'mail', 0) do
+        params = %{
-          params = %{
+          name: name,
-            email: :erlang.list_to_binary(mail),
+          nickname: name,
-            name: name,
+          password: nil
-            nickname: name,
+        }
-            password: password,
-            password_confirmation: password
-          }
 
-          changeset = User.register_changeset(%User{}, params)
+        params =
-
+          case List.keyfind(attributes, 'mail', 0) do
-          case User.register(changeset) do
+            {_, [mail]} -> Map.put_new(params, :email, :erlang.list_to_binary(mail))
-            {:ok, user} -> user
+            _ -> params
-            error -> error
           end
-        else
+
-          _ ->
+        changeset = User.register_changeset_ldap(%User{}, params)
-            Logger.error("Could not find LDAP attribute mail: #{inspect(attributes)}")
+
-            {:error, :ldap_registration_missing_attributes}
+        case User.register(changeset) do
+          {:ok, user} -> user
+          error -> error
         end
 
       error ->

lib/pleroma/web/common_api/common_api.ex

@@ -6,9 +6,7 @@ defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.Activity
   alias Pleroma.ActivityExpiration
   alias Pleroma.Conversation.Participation
-  alias Pleroma.FollowingRelationship
   alias Pleroma.Formatter
-  alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.ThreadMute
   alias Pleroma.User
@@ -122,33 +120,16 @@ defmodule Pleroma.Web.CommonAPI do
 
   def accept_follow_request(follower, followed) do
     with %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
-         {:ok, follower} <- User.follow(follower, followed),
+         {:ok, accept_data, _} <- Builder.accept(followed, follow_activity),
-         {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "accept"),
+         {:ok, _activity, _} <- Pipeline.common_pipeline(accept_data, local: true) do
-         {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_accept),
-         {:ok, _activity} <-
-           ActivityPub.accept(%{
-             to: [follower.ap_id],
-             actor: followed,
-             object: follow_activity.data["id"],
-             type: "Accept"
-           }) do
-      Notification.update_notification_type(followed, follow_activity)
       {:ok, follower}
     end
   end
 
   def reject_follow_request(follower, followed) do
     with %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
-         {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "reject"),
+         {:ok, reject_data, _} <- Builder.reject(followed, follow_activity),
-         {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_reject),
+         {:ok, _activity, _} <- Pipeline.common_pipeline(reject_data, local: true) do
-         {:ok, _notifications} <- Notification.dismiss(follow_activity),
-         {:ok, _activity} <-
-           ActivityPub.reject(%{
-             to: [follower.ap_id],
-             actor: followed,
-             object: follow_activity.data["id"],
-             type: "Reject"
-           }) do
       {:ok, follower}
     end
   end
@@ -308,18 +289,19 @@ defmodule Pleroma.Web.CommonAPI do
          {:ok, options, choices} <- normalize_and_validate_choices(choices, object) do
       answer_activities =
         Enum.map(choices, fn index ->
-          answer_data = make_answer_data(user, object, Enum.at(options, index)["name"])
+          {:ok, answer_object, _meta} =
+            Builder.answer(user, object, Enum.at(options, index)["name"])
 
-          {:ok, activity} =
+          {:ok, activity_data, _meta} = Builder.create(user, answer_object, [])
-            ActivityPub.create(%{
-              to: answer_data["to"],
-              actor: user,
-              context: object.data["context"],
-              object: answer_data,
-              additional: %{"cc" => answer_data["cc"]}
-            })
 
-          activity
+          {:ok, activity, _meta} =
+            activity_data
+            |> Map.put("cc", answer_object["cc"])
+            |> Map.put("context", answer_object["context"])
+            |> Pipeline.common_pipeline(local: true)
+
+          # TODO: Do preload of Pleroma.Object in Pipeline
+          Activity.normalize(activity.data)
         end)
 
       object = Object.get_cached_by_ap_id(object.data["id"])
@@ -340,8 +322,13 @@ defmodule Pleroma.Web.CommonAPI do
     end
   end
 
-  defp get_options_and_max_count(%{data: %{"anyOf" => any_of}}), do: {any_of, Enum.count(any_of)}
+  defp get_options_and_max_count(%{data: %{"anyOf" => any_of}})
-  defp get_options_and_max_count(%{data: %{"oneOf" => one_of}}), do: {one_of, 1}
+       when is_list(any_of) and any_of != [],
+       do: {any_of, Enum.count(any_of)}
+
+  defp get_options_and_max_count(%{data: %{"oneOf" => one_of}})
+       when is_list(one_of) and one_of != [],
+       do: {one_of, 1}
 
   defp normalize_and_validate_choices(choices, object) do
     choices = Enum.map(choices, fn i -> if is_binary(i), do: String.to_integer(i), else: i end)

lib/pleroma/web/common_api/utils.ex

@@ -548,17 +548,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     end
   end
 
-  def make_answer_data(%User{ap_id: ap_id}, object, name) do
-    %{
-      "type" => "Answer",
-      "actor" => ap_id,
-      "cc" => [object.data["actor"]],
-      "to" => [],
-      "name" => name,
-      "inReplyTo" => object.data["id"]
-    }
-  end
-
   def validate_character_limit("" = _full_payload, [] = _attachments) do
     {:error, dgettext("errors", "Cannot post an empty status without attachments")}
   end

lib/pleroma/web/controller_helper.ex

@@ -18,6 +18,12 @@ defmodule Pleroma.Web.ControllerHelper do
 
   def truthy_param?(value), do: not falsy_param?(value)
 
+  def json_response(conn, status, _) when status in [204, :no_content] do
+    conn
+    |> put_resp_header("content-type", "application/json")
+    |> send_resp(status, "")
+  end
+
   def json_response(conn, status, json) do
     conn
     |> put_status(status)

lib/pleroma/web/mastodon_api/controllers/account_controller.ex

@@ -226,7 +226,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
     with changeset <- User.update_changeset(user, user_params),
          {:ok, unpersisted_user} <- Ecto.Changeset.apply_action(changeset, :update),
          updated_object <-
-           Pleroma.Web.ActivityPub.UserView.render("user.json", user: user)
+           Pleroma.Web.ActivityPub.UserView.render("user.json", user: unpersisted_user)
            |> Map.delete("@context"),
          {:ok, update_data, []} <- Builder.update(user, updated_object),
          {:ok, _update, _} <-

lib/pleroma/web/mastodon_api/views/filter_view.ex

@@ -25,7 +25,7 @@ defmodule Pleroma.Web.MastodonAPI.FilterView do
       context: filter.context,
       expires_at: expires_at,
       irreversible: filter.hide,
-      whole_word: false
+      whole_word: filter.whole_word
     }
   end
 end

lib/pleroma/web/mastodon_api/views/poll_view.ex

@@ -28,10 +28,10 @@ defmodule Pleroma.Web.MastodonAPI.PollView do
 
   def render("show.json", %{object: object} = params) do
     case object.data do
-      %{"anyOf" => options} when is_list(options) ->
+      %{"anyOf" => [_ | _] = options} ->
         render(__MODULE__, "show.json", Map.merge(params, %{multiple: true, options: options}))
 
-      %{"oneOf" => options} when is_list(options) ->
+      %{"oneOf" => [_ | _] = options} ->
         render(__MODULE__, "show.json", Map.merge(params, %{multiple: false, options: options}))
 
       _ ->
@@ -40,15 +40,13 @@ defmodule Pleroma.Web.MastodonAPI.PollView do
   end
 
   defp end_time_and_expired(object) do
-    case object.data["closed"] || object.data["endTime"] do
+    if object.data["closed"] do
-      end_time when is_binary(end_time) ->
+      end_time = NaiveDateTime.from_iso8601!(object.data["closed"])
-        end_time = NaiveDateTime.from_iso8601!(end_time)
+      expired = NaiveDateTime.compare(end_time, NaiveDateTime.utc_now()) == :lt
-        expired = NaiveDateTime.compare(end_time, NaiveDateTime.utc_now()) == :lt
 
-        {Utils.to_masto_date(end_time), expired}
+      {Utils.to_masto_date(end_time), expired}
-
+    else
-      _ ->
+      {nil, false}
-        {nil, false}
     end
   end
 

lib/pleroma/web/oauth/oauth_controller.ex

@@ -76,6 +76,13 @@ defmodule Pleroma.Web.OAuth.OAuthController do
     available_scopes = (app && app.scopes) || []
     scopes = Scopes.fetch_scopes(params, available_scopes)
 
+    scopes =
+      if scopes == [] do
+        available_scopes
+      else
+        scopes
+      end
+
     # Note: `params` might differ from `conn.params`; use `@params` not `@conn.params` in template
     render(conn, Authenticator.auth_template(), %{
       response_type: params["response_type"],

lib/pleroma/web/rich_media/helpers.ex

@@ -9,6 +9,11 @@ defmodule Pleroma.Web.RichMedia.Helpers do
   alias Pleroma.Object
   alias Pleroma.Web.RichMedia.Parser
 
+  @rich_media_options [
+    pool: :media,
+    max_body: 2_000_000
+  ]
+
   @spec validate_page_url(URI.t() | binary()) :: :ok | :error
   defp validate_page_url(page_url) when is_binary(page_url) do
     validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld])
@@ -77,4 +82,20 @@ defmodule Pleroma.Web.RichMedia.Helpers do
     fetch_data_for_activity(activity)
     :ok
   end
+
+  def rich_media_get(url) do
+    headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
+
+    options =
+      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
+        Keyword.merge(@rich_media_options,
+          recv_timeout: 2_000,
+          with_body: true
+        )
+      else
+        @rich_media_options
+      end
+
+    Pleroma.HTTP.get(url, headers, options)
+  end
 end

lib/pleroma/web/rich_media/parser.ex

@@ -3,11 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Parser do
-  @options [
-    pool: :media,
-    max_body: 2_000_000
-  ]
-
   defp parsers do
     Pleroma.Config.get([:rich_media, :parsers])
   end
@@ -75,21 +70,8 @@ defmodule Pleroma.Web.RichMedia.Parser do
   end
 
   defp parse_url(url) do
-    opts =
-      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
-        Keyword.merge(@options,
-          recv_timeout: 2_000,
-          with_body: true
-        )
-      else
-        @options
-      end
-
     try do
-      rich_media_agent = Pleroma.Application.user_agent() <> "; Bot"
+      {:ok, %Tesla.Env{body: html}} = Pleroma.Web.RichMedia.Helpers.rich_media_get(url)
-
-      {:ok, %Tesla.Env{body: html}} =
-        Pleroma.HTTP.get(url, [{"user-agent", rich_media_agent}], adapter: opts)
 
       html
       |> parse_html()

lib/pleroma/web/rich_media/parsers/oembed_parser.ex

@@ -22,7 +22,7 @@ defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do
   end
 
   defp get_oembed_data(url) do
-    with {:ok, %Tesla.Env{body: json}} <- Pleroma.HTTP.get(url, [], adapter: [pool: :media]) do
+    with {:ok, %Tesla.Env{body: json}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url) do
       Jason.decode(json)
     end
   end

lib/pleroma/web/templates/layout/app.html.eex

@@ -37,7 +37,7 @@
       }
 
       a {
-        color: color: #d8a070;
+        color: #d8a070;
         text-decoration: none;
       }
 

mix.exs

@@ -127,7 +127,7 @@ defmodule Pleroma.Mixfile do
       {:pbkdf2_elixir, "~> 1.2"},
       {:bcrypt_elixir, "~> 2.2"},
       {:trailing_format_plug, "~> 0.0.7"},
-      {:fast_sanitize, "~> 0.1"},
+      {:fast_sanitize, "~> 0.2.0"},
       {:html_entities, "~> 0.5", override: true},
       {:phoenix_html, "~> 2.14"},
       {:calendar, "~> 1.0"},
@@ -214,7 +214,8 @@ defmodule Pleroma.Mixfile do
       "ecto.setup": ["ecto.create", "ecto.migrate", "run priv/repo/seeds.exs"],
       "ecto.reset": ["ecto.drop", "ecto.setup"],
       test: ["ecto.create --quiet", "ecto.migrate", "test"],
-      docs: ["pleroma.docs", "docs"]
+      docs: ["pleroma.docs", "docs"],
+      analyze: ["credo --strict --only=warnings,todo,fixme,consistency,readability"]
     ]
   end
 
@@ -228,10 +229,10 @@ defmodule Pleroma.Mixfile do
   defp version(version) do
     identifier_filter = ~r/[^0-9a-z\-]+/i
 
-    {_cmdgit, cmdgit_err} = System.cmd("sh", ["-c", "command -v git"])
+    git_available? = match?({_output, 0}, System.cmd("sh", ["-c", "command -v git"]))
 
     git_pre_release =
-      if cmdgit_err == 0 do
+      if git_available? do
         {tag, tag_err} =
           System.cmd("git", ["describe", "--tags", "--abbrev=0"], stderr_to_stdout: true)
 
@@ -257,7 +258,7 @@ defmodule Pleroma.Mixfile do
 
     # Branch name as pre-release version component, denoted with a dot
     branch_name =
-      with 0 <- cmdgit_err,
+      with true <- git_available?,
            {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]),
            branch_name <- String.trim(branch_name),
            branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name,

mix.lock

@@ -42,8 +42,8 @@
   "ex_machina": {:hex, :ex_machina, "2.4.0", "09a34c5d371bfb5f78399029194a8ff67aff340ebe8ba19040181af35315eabb", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "a20bc9ddc721b33ea913b93666c5d0bdca5cbad7a67540784ae277228832d72c"},
   "ex_syslogger": {:hex, :ex_syslogger, "1.5.2", "72b6aa2d47a236e999171f2e1ec18698740f40af0bd02c8c650bf5f1fd1bac79", [:mix], [{:poison, ">= 1.5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:syslog, "~> 1.1.0", [hex: :syslog, repo: "hexpm", optional: false]}], "hexpm", "ab9fab4136dbc62651ec6f16fa4842f10cf02ab4433fa3d0976c01be99398399"},
   "excoveralls": {:hex, :excoveralls, "0.13.1", "b9f1697f7c9e0cfe15d1a1d737fb169c398803ffcbc57e672aa007e9fd42864c", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "b4bb550e045def1b4d531a37fb766cbbe1307f7628bf8f0414168b3f52021cce"},
-  "fast_html": {:hex, :fast_html, "1.0.3", "2cc0d4b68496266a1530e0c852cafeaede0bd10cfdee26fda50dc696c203162f", [:make, :mix], [], "hexpm", "ab3d782b639d3c4655fbaec0f9d032c91f8cab8dd791ac7469c2381bc7c32f85"},
+  "fast_html": {:hex, :fast_html, "2.0.1", "e126c74d287768ae78c48938da6711164517300d108a78f8a38993df8d588335", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "bdd6f8525c95ad391a4f10d9a1b3da4cea94078ec8638487aa8c24015ad9393a"},
-  "fast_sanitize": {:hex, :fast_sanitize, "0.1.7", "2a7cd8734c88a2de6de55022104f8a3b87f1fdbe8bbf131d9049764b53d50d0d", [:mix], [{:fast_html, "~> 1.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f39fe8ea08fbac17487c30bf09b7d9f3e12472e51fb07a88ffeb8fd17da8ab67"},
+  "fast_sanitize": {:hex, :fast_sanitize, "0.2.0", "004b40d5bbecda182b6fdba762a51fffd3501e689e8eafe196e1a97eb0caf733", [:mix], [{:fast_html, "~> 2.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "11fcb37f26d272a3a2aff861872bf100be4eeacea69505908b8cdbcea5b0813a"},
   "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "31fc8090fde1acd267c07c36ea7365b8604055f897d3a53dd967658c691bd827"},
   "floki": {:hex, :floki, "0.27.0", "6b29a14283f1e2e8fad824bc930eaa9477c462022075df6bea8f0ad811c13599", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "583b8c13697c37179f1f82443bcc7ad2f76fbc0bf4c186606eebd658f7f2631b"},
   "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm", "29bd14a88030980849c7ed2447b8db6d6c9278a28b11a44cafe41b791205440f"},
@@ -76,6 +76,7 @@
   "mox": {:hex, :mox, "0.5.2", "55a0a5ba9ccc671518d068c8dddd20eeb436909ea79d1799e2209df7eaa98b6c", [:mix], [], "hexpm", "df4310628cd628ee181df93f50ddfd07be3e5ecc30232d3b6aadf30bdfe6092b"},
   "myhtmlex": {:git, "https://git.pleroma.social/pleroma/myhtmlex.git", "ad0097e2f61d4953bfef20fb6abddf23b87111e6", [ref: "ad0097e2f61d4953bfef20fb6abddf23b87111e6", submodules: true]},
   "nimble_parsec": {:hex, :nimble_parsec, "0.6.0", "32111b3bf39137144abd7ba1cce0914533b2d16ef35e8abc5ec8be6122944263", [:mix], [], "hexpm", "27eac315a94909d4dc68bc07a4a83e06c8379237c5ea528a9acff4ca1c873c52"},
+  "nimble_pool": {:hex, :nimble_pool, "0.1.0", "ffa9d5be27eee2b00b0c634eb649aa27f97b39186fec3c493716c2a33e784ec6", [:mix], [], "hexpm", "343a1eaa620ddcf3430a83f39f2af499fe2370390d4f785cd475b4df5acaf3f9"},
   "nodex": {:git, "https://git.pleroma.social/pleroma/nodex", "cb6730f943cfc6aad674c92161be23a8411f15d1", [ref: "cb6730f943cfc6aad674c92161be23a8411f15d1"]},
   "oban": {:hex, :oban, "2.0.0", "e6ce70d94dd46815ec0882a1ffb7356df9a9d5b8a40a64ce5c2536617a447379", [:mix], [{:ecto_sql, ">= 3.4.3", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cf574813bd048b98a698aa587c21367d2e06842d4e1b1993dcd6a696e9e633bd"},
   "open_api_spex": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/open_api_spex.git", "f296ac0924ba3cf79c7a588c4c252889df4c2edd", [ref: "f296ac0924ba3cf79c7a588c4c252889df4c2edd"]},

priv/repo/migrations/20200802170532_fix_legacy_tags.exs

@@ -0,0 +1,40 @@
+# Fix legacy tags set by AdminFE that don't align with TagPolicy MRF
+
+defmodule Pleroma.Repo.Migrations.FixLegacyTags do
+  use Ecto.Migration
+  alias Pleroma.Repo
+  alias Pleroma.User
+  import Ecto.Query
+
+  @old_new_map %{
+    "force_nsfw" => "mrf_tag:media-force-nsfw",
+    "strip_media" => "mrf_tag:media-strip",
+    "force_unlisted" => "mrf_tag:force-unlisted",
+    "sandbox" => "mrf_tag:sandbox",
+    "disable_remote_subscription" => "mrf_tag:disable-remote-subscription",
+    "disable_any_subscription" => "mrf_tag:disable-any-subscription"
+  }
+
+  def change do
+    legacy_tags = Map.keys(@old_new_map)
+
+    from(u in User,
+      where: fragment("? && ?", u.tags, ^legacy_tags),
+      select: struct(u, [:tags, :id])
+    )
+    |> Repo.chunk_stream(100)
+    |> Enum.each(fn user ->
+      fix_tags_changeset(user)
+      |> Repo.update()
+    end)
+  end
+
+  defp fix_tags_changeset(%User{tags: tags} = user) do
+    new_tags =
+      Enum.map(tags, fn tag ->
+        Map.get(@old_new_map, tag, tag)
+      end)
+
+    Ecto.Changeset.change(user, tags: new_tags)
+  end
+end

priv/repo/migrations/20200804180322_remove_nonlocal_expirations.exs

@@ -0,0 +1,19 @@
+defmodule Pleroma.Repo.Migrations.RemoveNonlocalExpirations do
+  use Ecto.Migration
+
+  def up do
+    statement = """
+    DELETE FROM
+      activity_expirations A USING activities B
+    WHERE
+      A.activity_id = B.id
+      AND B.local = false;
+    """
+
+    execute(statement)
+  end
+
+  def down do
+    :ok
+  end
+end

priv/repo/migrations/20200804183107_add_unique_index_to_app_client_id.exs

@@ -0,0 +1,7 @@
+defmodule Pleroma.Repo.Migrations.AddUniqueIndexToAppClientId do
+  use Ecto.Migration
+
+  def change do
+    create(unique_index(:apps, [:client_id]))
+  end
+end

priv/repo/migrations/20200808173046_only_expire_creates.exs

@@ -0,0 +1,19 @@
+defmodule Pleroma.Repo.Migrations.OnlyExpireCreates do
+  use Ecto.Migration
+
+  def up do
+    statement = """
+    DELETE FROM
+      activity_expirations a_exp USING activities a, objects o
+    WHERE
+      a_exp.activity_id = a.id AND (o.data->>'id') = COALESCE(a.data->'object'->>'id', a.data->>'object')
+      AND (a.data->>'type' != 'Create' OR o.data->>'type' != 'Note');
+    """
+
+    execute(statement)
+  end
+
+  def down do
+    :ok
+  end
+end

priv/repo/migrations/20200811125613_set_defaults_to_user_approval_pending.exs

@@ -0,0 +1,15 @@
+defmodule Pleroma.Repo.Migrations.SetDefaultsToUserApprovalPending do
+  use Ecto.Migration
+
+  def up do
+    execute("UPDATE users SET approval_pending = false WHERE approval_pending IS NULL")
+
+    alter table(:users) do
+      modify(:approval_pending, :boolean, default: false, null: false)
+    end
+  end
+
+  def down do
+    :ok
+  end
+end

priv/repo/migrations/20200811143147_ap_id_not_null.exs

@@ -0,0 +1,19 @@
+defmodule Pleroma.Repo.Migrations.ApIdNotNull do
+  use Ecto.Migration
+
+  require Logger
+
+  def up do
+    Logger.warn(
+      "If this migration fails please open an issue at https://git.pleroma.social/pleroma/pleroma/-/issues/new \n"
+    )
+
+    alter table(:users) do
+      modify(:ap_id, :string, null: false)
+    end
+  end
+
+  def down do
+    :ok
+  end
+end

priv/static/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,user-scalable=no"><title>Pleroma</title><!--server-generated-meta--><link rel=icon type=image/png href=/favicon.png><link href=/static/css/app.6dbc7dea4fc148c85860.css rel=stylesheet><link href=/static/fontello.1594823398494.css rel=stylesheet></head><body class=hidden><noscript>To use Pleroma, please enable JavaScript.</noscript><div id=app></div><script type=text/javascript src=/static/js/vendors~app.9e24ed238da5a8538f50.js></script><script type=text/javascript src=/static/js/app.31bba9f1e242ff273dcb.js></script></body></html>
+<!DOCTYPE html><html lang=en><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,user-scalable=no"><title>Pleroma</title><!--server-generated-meta--><link rel=icon type=image/png href=/favicon.png><link href=/static/css/app.77b1644622e3bae24b6b.css rel=stylesheet><link href=/static/fontello.1597327457363.css rel=stylesheet></head><body class=hidden><noscript>To use Pleroma, please enable JavaScript.</noscript><div id=app></div><script type=text/javascript src=/static/js/vendors~app.811c8482146cad566f7e.js></script><script type=text/javascript src=/static/js/app.032cb80dafd1f208df1c.js></script></body></html>

priv/static/static/css/app.77b1644622e3bae24b6b.css

@@ -243,4 +243,4 @@
   cursor: pointer;
 }
 
-/*# sourceMappingURL=app.6dbc7dea4fc148c85860.css.map*/
+/*# sourceMappingURL=app.77b1644622e3bae24b6b.css.map*/

priv/static/static/fontello.1597327457363.css

@@ -0,0 +1,158 @@
+@font-face {
+	font-family: "Icons";
+	src: url("./font/fontello.1597327457363.eot");
+	src: url("./font/fontello.1597327457363.eot") format("embedded-opentype"),
+		url("./font/fontello.1597327457363.woff2") format("woff2"),
+		url("./font/fontello.1597327457363.woff") format("woff"),
+		url("./font/fontello.1597327457363.ttf") format("truetype"),
+		url("./font/fontello.1597327457363.svg") format("svg");
+	font-weight: normal;
+	font-style: normal;
+}
+
+[class^="icon-"]::before,
+[class*=" icon-"]::before {
+	font-family: "Icons";
+	font-style: normal;
+	font-weight: normal;
+	speak: none;
+	display: inline-block;
+	text-decoration: inherit;
+	width: 1em;
+	margin-right: .2em;
+	text-align: center;
+	font-variant: normal;
+	text-transform: none;
+	line-height: 1em;
+	margin-left: .2em;
+	-webkit-font-smoothing: antialiased;
+	-moz-osx-font-smoothing: grayscale;
+}
+
+.icon-spin4::before { content: "\e834"; }
+
+.icon-cancel::before { content: "\e800"; }
+
+.icon-upload::before { content: "\e801"; }
+
+.icon-spin3::before { content: "\e832"; }
+
+.icon-reply::before { content: "\f112"; }
+
+.icon-star::before { content: "\e802"; }
+
+.icon-star-empty::before { content: "\e803"; }
+
+.icon-retweet::before { content: "\e804"; }
+
+.icon-eye-off::before { content: "\e805"; }
+
+.icon-binoculars::before { content: "\f1e5"; }
+
+.icon-cog::before { content: "\e807"; }
+
+.icon-user-plus::before { content: "\f234"; }
+
+.icon-menu::before { content: "\f0c9"; }
+
+.icon-logout::before { content: "\e808"; }
+
+.icon-down-open::before { content: "\e809"; }
+
+.icon-attach::before { content: "\e80a"; }
+
+.icon-link-ext::before { content: "\f08e"; }
+
+.icon-link-ext-alt::before { content: "\f08f"; }
+
+.icon-picture::before { content: "\e80b"; }
+
+.icon-video::before { content: "\e80c"; }
+
+.icon-right-open::before { content: "\e80d"; }
+
+.icon-left-open::before { content: "\e80e"; }
+
+.icon-up-open::before { content: "\e80f"; }
+
+.icon-comment-empty::before { content: "\f0e5"; }
+
+.icon-mail-alt::before { content: "\f0e0"; }
+
+.icon-lock::before { content: "\e811"; }
+
+.icon-lock-open-alt::before { content: "\f13e"; }
+
+.icon-globe::before { content: "\e812"; }
+
+.icon-brush::before { content: "\e813"; }
+
+.icon-search::before { content: "\e806"; }
+
+.icon-adjust::before { content: "\e816"; }
+
+.icon-thumbs-up-alt::before { content: "\f164"; }
+
+.icon-attention::before { content: "\e814"; }
+
+.icon-plus-squared::before { content: "\f0fe"; }
+
+.icon-plus::before { content: "\e815"; }
+
+.icon-edit::before { content: "\e817"; }
+
+.icon-play-circled::before { content: "\f144"; }
+
+.icon-pencil::before { content: "\e818"; }
+
+.icon-chart-bar::before { content: "\e81b"; }
+
+.icon-smile::before { content: "\f118"; }
+
+.icon-bell-alt::before { content: "\f0f3"; }
+
+.icon-wrench::before { content: "\e81a"; }
+
+.icon-pin::before { content: "\e819"; }
+
+.icon-ellipsis::before { content: "\f141"; }
+
+.icon-bell-ringing-o::before { content: "\e810"; }
+
+.icon-zoom-in::before { content: "\e81c"; }
+
+.icon-gauge::before { content: "\f0e4"; }
+
+.icon-users::before { content: "\e81d"; }
+
+.icon-info-circled::before { content: "\e81f"; }
+
+.icon-home-2::before { content: "\e821"; }
+
+.icon-chat::before { content: "\e81e"; }
+
+.icon-login::before { content: "\e820"; }
+
+.icon-arrow-curved::before { content: "\e822"; }
+
+.icon-link::before { content: "\e823"; }
+
+.icon-share::before { content: "\f1e0"; }
+
+.icon-user::before { content: "\e824"; }
+
+.icon-ok::before { content: "\e827"; }
+
+.icon-filter::before { content: "\f0b0"; }
+
+.icon-download::before { content: "\e825"; }
+
+.icon-bookmark::before { content: "\e826"; }
+
+.icon-bookmark-empty::before { content: "\f097"; }
+
+.icon-music::before { content: "\e828"; }
+
+.icon-doc::before { content: "\e829"; }
+
+.icon-block::before { content: "\e82a"; }

priv/static/static/js/10.8c5b75840b696a152c7e.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"names":[],"mappings":"","file":"static/js/10.5ef4671883649cf93524.js","sourceRoot":""}
+{"version":3,"sources":[],"names":[],"mappings":"","file":"static/js/10.8c5b75840b696a152c7e.js","sourceRoot":""}

priv/static/static/js/11.bfcde1c26c4d54b84ee4.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"names":[],"mappings":"","file":"static/js/11.c5b938b4349f87567338.js","sourceRoot":""}
+{"version":3,"sources":[],"names":[],"mappings":"","file":"static/js/11.bfcde1c26c4d54b84ee4.js","sourceRoot":""}

priv/static/static/js/12.76095ee23394e0ef65bb.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"names":[],"mappings":"","file":"static/js/12.ab82f9512fa85e78c114.js","sourceRoot":""}
+{"version":3,"sources":[],"names":[],"mappings":"","file":"static/js/12.76095ee23394e0ef65bb.js","sourceRoot":""}

priv/static/static/js/13.957b04ac11d6cde66f5b.js.map

@@ -1 +1 @@
-{"version":3,"sources":[],"names":[],"mappings":"","file":"static/js/13.40e59c5015d3307b94ad.js","sourceRoot":""}
+{"version":3,"sources":[],"names":[],"mappings":"","file":"static/js/13.957b04ac11d6cde66f5b.js","sourceRoot":""}