Merge branch 'develop' of https://git.pleroma.social/pleroma/pleroma into develop

.credo.exs

@@ -25,7 +25,7 @@
       #
       # If you create your own checks, you must specify the source files for
       # them here, so they can be loaded by Credo before running the analysis.
-      requires: [],
+      requires: ["./test/credo/check/consistency/file_location.ex"],
       #
       # Credo automatically checks for updates, like e.g. Hex does.
       # You can disable this behaviour below:
@@ -71,7 +71,6 @@
         # set this value to 0 (zero).
         {Credo.Check.Design.TagTODO, exit_status: 0},
         {Credo.Check.Design.TagFIXME, exit_status: 0},
-
         {Credo.Check.Readability.FunctionNames},
         {Credo.Check.Readability.LargeNumbers},
         {Credo.Check.Readability.MaxLineLength, priority: :low, max_length: 100},
@@ -91,7 +90,6 @@
         {Credo.Check.Readability.VariableNames},
         {Credo.Check.Readability.Semicolons},
         {Credo.Check.Readability.SpaceAfterCommas},
-
         {Credo.Check.Refactor.DoubleBooleanNegation},
         {Credo.Check.Refactor.CondStatements},
         {Credo.Check.Refactor.CyclomaticComplexity},
@@ -102,7 +100,6 @@
         {Credo.Check.Refactor.Nesting},
         {Credo.Check.Refactor.PipeChainStart},
         {Credo.Check.Refactor.UnlessWithElse},
-
         {Credo.Check.Warning.BoolOperationOnSameValues},
         {Credo.Check.Warning.IExPry},
         {Credo.Check.Warning.IoInspect},
@@ -131,6 +128,7 @@
 
         # Custom checks can be created using `mix credo.gen.check`.
         #
+        {Credo.Check.Consistency.FileLocation}
       ]
     }
   ]

.gitignore

@@ -28,6 +28,8 @@ erl_crash.dump
 # variables.
 /config/*.secret.exs
 /config/generated_config.exs
+/config/*.env
+
 
 # Database setup file, some may forget to delete it
 /config/setup_db.psql

.gitlab-ci.yml

@@ -25,6 +25,8 @@ before_script:
   - apt-get update && apt-get install -y cmake
   - mix local.hex --force
   - mix local.rebar --force
+  - apt-get -qq update
+  - apt-get install -y libmagic-dev
 
 build:
   stage: build
@@ -196,7 +198,7 @@ amd64:
   variables: &release-variables
     MIX_ENV: prod
   before_script: &before-release
-  - apt-get update && apt-get install -y cmake
+  - apt-get update && apt-get install -y cmake libmagic-dev
   - echo "import Mix.Config" > config/prod.secret.exs
   - mix local.hex --force
   - mix local.rebar --force
@@ -215,7 +217,7 @@ amd64-musl:
   cache: *release-cache
   variables: *release-variables
   before_script: &before-release-musl
-  - apk add git gcc g++ musl-dev make cmake
+  - apk add git gcc g++ musl-dev make cmake file-dev
   - echo "import Mix.Config" > config/prod.secret.exs
   - mix local.hex --force
   - mix local.rebar --force

CHANGELOG.md

@@ -9,9 +9,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mix tasks for controlling user account confirmation status in bulk (`mix pleroma.user confirm_all` and `mix pleroma.user unconfirm_all`)
 - Mix task for sending confirmation emails to all unconfirmed users (`mix pleroma.email send_confirmation_mails`)
 - Mix task option for force-unfollowing relays
+- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
+- Pleroma API: Importing the mutes users from CSV files.
+- Experimental websocket-based federation between Pleroma instances.
+- Support pagination of blocks and mutes
+- App metrics: ability to restrict access to specified IP whitelist.
+- Account backup
+- Configuration: Add `:instance, autofollowing_nicknames` setting to provide a way to make accounts automatically follow new users that register on the local Pleroma instance.
 
 ### Changed
 
+- **Breaking** Requires `libmagic` (or `file`) to guess file types.
+- **Breaking:** Pleroma Admin API: emoji packs and files routes changed.
+- **Breaking:** Sensitive/NSFW statuses no longer disable link previews.
+- **Breaking:** App metrics endpoint (`/api/pleroma/app_metrics`) is disabled by default, check `docs/API/prometheus.md` on enabling and configuring. 
 - Search: Users are now findable by their urls.
 - Renamed `:await_up_timeout` in `:connections_pool` namespace to `:connect_timeout`, old name is deprecated.
 - Renamed `:timeout` in `pools` namespace to `:recv_timeout`, old name is deprecated.
@@ -20,11 +31,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Minimum lifetime for ephmeral activities changed to 10 minutes and made configurable (`:min_lifetime` option).
 - Introduced optional dependencies on `ffmpeg`, `ImageMagick`, `exiftool` software packages. Please refer to `docs/installation/optional/media_graphics_packages.md`.
 
-### Added
-- Media preview proxy (requires `ffmpeg` and `ImageMagick` to be installed and media proxy to be enabled; see `:media_preview_proxy` config for more details).
+<details>
+  <summary>API Changes</summary>
+
 - Pleroma API: Importing the mutes users from CSV files.
-- Experimental websocket-based federation between Pleroma instances.
 - Admin API: Importing emoji from a zip file
+- Pleroma API: Pagination for remote/local packs and emoji.
+- Admin API: (`GET /api/pleroma/admin/users`) added filters user by `unconfirmed` status
+- Admin API: (`GET /api/pleroma/admin/users`) added filters user by `actor_type`
+- Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending.
+
+</details>
 
 ### Removed
 
@@ -38,6 +55,14 @@ switched to a new configuration mechanism, however it was not officially removed
 
 - Add documented-but-missing chat pagination.
 - Allow sending out emails again.
+- Allow sending chat messages to yourself.
+- Fix remote users with a whitespace name.
+- OStatus / static FE endpoints: fixed inaccessibility for anonymous users on non-federating instances, switched to handling per `:restrict_unauthenticated` setting.
+
+## Unreleased (Patch)
+
+### Changed
+- API: Empty parameter values for integer parameters are now ignored in non-strict validaton mode.
 
 ## [2.1.2] - 2020-09-17
 

config/config.exs

@@ -123,7 +123,6 @@
 
 # Configures the endpoint
 config :pleroma, Pleroma.Web.Endpoint,
-  instrumenters: [Pleroma.Web.Endpoint.Instrumenter],
   url: [host: "localhost"],
   http: [
     ip: {127, 0, 0, 1},
@@ -143,7 +142,7 @@
   secret_key_base: "aK4Abxf29xU9TTDKre9coZPUgevcVCFQJe/5xP/7Lt4BEif6idBIbjupVbOrbKxl",
   signing_salt: "CqaoopA2",
   render_errors: [view: Pleroma.Web.ErrorView, accepts: ~w(json)],
-  pubsub: [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2],
+  pubsub_server: Pleroma.PubSub,
   secure_cookie_flag: true,
   extra_cookie_attrs: [
     "SameSite=Lax"
@@ -238,6 +237,7 @@
   mrf_transparency_exclusions: [],
   staff_transparency: [],
   autofollowed_nicknames: [],
+  autofollowing_nicknames: [],
   max_pinned_statuses: 1,
   attachment_links: false,
   max_report_comment_size: 1000,
@@ -554,6 +554,7 @@
   queues: [
     activity_expiration: 10,
     token_expiration: 5,
+    backup: 1,
     federator_incoming: 50,
     federator_outgoing: 50,
     ingestion_queue: 50,
@@ -639,7 +640,12 @@
 
 config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: false
 
-config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics"
+config :prometheus, Pleroma.Web.Endpoint.MetricsExporter,
+  enabled: false,
+  auth: false,
+  ip_whitelist: [],
+  path: "/api/pleroma/app_metrics",
+  format: :text
 
 config :pleroma, Pleroma.ScheduledActivity,
   daily_user_limit: 25,
@@ -680,7 +686,18 @@
 
 config :pleroma, Pleroma.Workers.PurgeExpiredActivity, enabled: true, min_lifetime: 600
 
-config :pleroma, Pleroma.Plugs.RemoteIp, enabled: true
+config :pleroma, Pleroma.Web.Plugs.RemoteIp,
+  enabled: true,
+  headers: ["x-forwarded-for"],
+  proxies: [],
+  reserved: [
+    "127.0.0.0/8",
+    "::1/128",
+    "fc00::/7",
+    "10.0.0.0/8",
+    "172.16.0.0/12",
+    "192.168.0.0/16"
+  ]
 
 config :pleroma, :static_fe, enabled: false
 
@@ -794,6 +811,8 @@
     timeout: 300_000
   ]
 
+config :pleroma, :majic_pool, size: 2
+
 private_instance? = :if_instance_is_private
 
 config :pleroma, :restrict_unauthenticated,
@@ -820,6 +839,11 @@
 
 config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthenticator
 
+config :pleroma, Pleroma.User.Backup,
+  purge_after_days: 30,
+  limit_days: 7,
+  dir: nil
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/description.exs

@@ -44,11 +44,13 @@
   },
   %{
     key: "git",
+    label: "Git Repository URL",
     type: :string,
     description: "URL of the git repository of the frontend"
   },
   %{
     key: "build_url",
+    label: "Build URL",
     type: :string,
     description:
       "Either an url to a zip file containing the frontend or a template to build it by inserting the `ref`. The string `${ref}` will be replaced by the configured `ref`.",
@@ -56,6 +58,7 @@
   },
   %{
     key: "build_dir",
+    label: "Build directory",
     type: :string,
     description: "The directory inside the zip file "
   }
@@ -826,13 +829,13 @@
         key: :autofollowed_nicknames,
         type: {:list, :string},
         description:
-          "Set to nicknames of (local) users that every new user should automatically follow",
-        suggestions: [
-          "lain",
-          "kaniini",
-          "lanodan",
-          "rinpatch"
-        ]
+          "Set to nicknames of (local) users that every new user should automatically follow"
+      },
+      %{
+        key: :autofollowing_nicknames,
+        type: {:list, :string},
+        description:
+          "Set to nicknames of (local) users that automatically follows every newly registered user"
       },
       %{
         key: :attachment_links,
@@ -1754,28 +1757,37 @@
     related_policy: "Pleroma.Web.ActivityPub.MRF.KeywordPolicy",
     label: "MRF Keyword",
     type: :group,
-    description: "Reject or Word-Replace messages with a keyword or regex",
+    description:
+      "Reject or Word-Replace messages matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).",
     children: [
       %{
         key: :reject,
         type: {:list, :string},
-        description:
-          "A list of patterns which result in message being rejected. Each pattern can be a string or a regular expression.",
+        description: """
+          A list of patterns which result in message being rejected.
+
+          Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+        """,
         suggestions: ["foo", ~r/foo/iu]
       },
       %{
         key: :federated_timeline_removal,
         type: {:list, :string},
-        description:
-          "A list of patterns which result in message being removed from federated timelines (a.k.a unlisted). Each pattern can be a string or a regular expression.",
+        description: """
+          A list of patterns which result in message being removed from federated timelines (a.k.a unlisted).
+
+          Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+        """,
         suggestions: ["foo", ~r/foo/iu]
       },
       %{
         key: :replace,
         type: {:list, :tuple},
-        description:
-          "A list of tuples containing {pattern, replacement}. Each pattern can be a string or a regular expression.",
-        suggestions: [{"foo", "bar"}, {~r/foo/iu, "bar"}]
+        description: """
+          **Pattern**: a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`.
+
+          **Replacement**: a string. Leaving the field empty is permitted.
+        """
       }
     ]
   },
@@ -2285,6 +2297,12 @@
             description: "Activity expiration queue",
             suggestions: [10]
           },
+          %{
+            key: :backup,
+            type: :integer,
+            description: "Backup queue",
+            suggestions: [1]
+          },
           %{
             key: :attachments_cleanup,
             type: :integer,
@@ -3247,10 +3265,10 @@
   },
   %{
     group: :pleroma,
-    key: Pleroma.Plugs.RemoteIp,
+    key: Pleroma.Web.Plugs.RemoteIp,
     type: :group,
     description: """
-    `Pleroma.Plugs.RemoteIp` is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
+    `Pleroma.Web.Plugs.RemoteIp` is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
     **If your instance is not behind at least one reverse proxy, you should not enable this plug.**
     """,
     children: [
@@ -3262,20 +3280,22 @@
       %{
         key: :headers,
         type: {:list, :string},
-        description:
-          "A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Default: `~w[forwarded x-forwarded-for x-client-ip x-real-ip]`."
+        description: """
+          A list of strings naming the HTTP headers to use when deriving the true client IP. Default: `["x-forwarded-for"]`.
+        """
       },
       %{
         key: :proxies,
         type: {:list, :string},
         description:
-          "A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Default: `[]`."
+          "A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128."
       },
       %{
         key: :reserved,
         type: {:list, :string},
-        description:
-          "Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network)."
+        description: """
+          A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`
+        """
       }
     ]
   },
@@ -3681,9 +3701,7 @@
         type: :map,
         description:
           "A map containing available frontends and parameters for their installation.",
-        children: [
-          frontend_options
-        ]
+        children: frontend_options
       }
     ]
   },
@@ -3705,5 +3723,76 @@
         ]
       }
     ]
+  },
+  %{
+    group: :pleroma,
+    key: :majic_pool,
+    type: :group,
+    description: "Majic/libmagic configuration",
+    children: [
+      %{
+        key: :size,
+        type: :integer,
+        description: "Number of majic workers to start.",
+        suggestions: [2]
+      }
+    ]
+  },
+  %{
+    group: :pleroma,
+    key: Pleroma.User.Backup,
+    type: :group,
+    description: "Account Backup",
+    children: [
+      %{
+        key: :purge_after_days,
+        type: :integer,
+        description: "Remove backup achives after N days",
+        suggestions: [30]
+      },
+      %{
+        key: :limit_days,
+        type: :integer,
+        description: "Limit user to export not more often than once per N days",
+        suggestions: [7]
+      }
+    ]
+  },
+  %{
+    group: :prometheus,
+    key: Pleroma.Web.Endpoint.MetricsExporter,
+    type: :group,
+    description: "Prometheus app metrics endpoint configuration",
+    children: [
+      %{
+        key: :enabled,
+        type: :boolean,
+        description: "[Pleroma extension] Enables app metrics endpoint."
+      },
+      %{
+        key: :ip_whitelist,
+        type: [{:list, :string}, {:list, :charlist}, {:list, :tuple}],
+        description:
+          "[Pleroma extension] If non-empty, restricts access to app metrics endpoint to specified IP addresses."
+      },
+      %{
+        key: :auth,
+        type: [:boolean, :tuple],
+        description: "Enables HTTP Basic Auth for app metrics endpoint.",
+        suggestion: [false, {:basic, "myusername", "mypassword"}]
+      },
+      %{
+        key: :path,
+        type: :string,
+        description: "App metrics endpoint URI path.",
+        suggestions: ["/api/pleroma/app_metrics"]
+      },
+      %{
+        key: :format,
+        type: :atom,
+        description: "App metrics endpoint output format.",
+        suggestions: [:text, :protobuf]
+      }
+    ]
   }
 ]

config/test.exs

@@ -113,7 +113,7 @@
 
 config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: true
 
-config :pleroma, Pleroma.Plugs.RemoteIp, enabled: false
+config :pleroma, Pleroma.Web.Plugs.RemoteIp, enabled: false
 
 config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true
 

coveralls.json

@@ -1,6 +1,7 @@
 {
   "skip_files": [
     "test/support",
-    "lib/mix/tasks/pleroma/benchmark.ex"
+    "lib/mix/tasks/pleroma/benchmark.ex",
+    "lib/credo/check/consistency/file_location.ex"
   ]
 }

docs/API/admin_api.md

@@ -20,12 +20,14 @@ Configuration options:
     - `external`: only external users
     - `active`: only active users
     - `need_approval`: only unapproved users
+    - `unconfirmed`: only unconfirmed users
     - `deactivated`: only deactivated users
     - `is_admin`: users with admin role
     - `is_moderator`: users with moderator role
   - *optional* `page`: **integer** page number
   - *optional* `page_size`: **integer** number of users per page (default is `50`)
   - *optional* `tags`: **[string]** tags list
+  - *optional* `actor_types`: **[string]** actor type list (`Person`, `Service`, `Application`)
   - *optional* `name`: **string** user display name
   - *optional* `email`: **string** user email
 - Example: `https://mypleroma.org/api/pleroma/admin/users?query=john&filters=local,active&page=1&page_size=10&tags[]=some_tag&tags[]=another_tag&name=display_name&email=email@example.com`

docs/API/chats.md

@@ -173,11 +173,14 @@ Returned data:
     "created_at": "2020-04-21T15:06:45.000Z",
     "emojis": [],
     "id": "12",
-    "unread": false
+    "unread": false,
+    "idempotency_key": "75442486-0874-440c-9db1-a7006c25a31f"
   }
 ]
 ```
 
+- idempotency_key: The copy of the `idempotency-key` HTTP request header that can be used for optimistic message sending. Included only during the first few minutes after the message creation.
+
 ### Posting a chat message
 
 Posting a chat message for given Chat id works like this:

docs/API/pleroma_api.md

@@ -378,44 +378,43 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Params: None
 * Response: JSON, returns a list of Mastodon Conversation entities that were marked as read (200 - healthy, 503 unhealthy).
 
-## `GET /api/pleroma/emoji/packs/import`
-### Imports packs from filesystem
+## `GET /api/pleroma/emoji/pack?name=:name`
+
+### Get pack.json for the pack
+
 * Method `GET`
-* Authentication: required
-* Params: None
-* Response: JSON, returns a list of imported packs.
-
-## `GET /api/pleroma/emoji/packs/remote`
-### Make request to another instance for packs list
-* Method `GET`
-* Authentication: required
+* Authentication: not required
 * Params:
-  * `url`: url of the instance to get packs from
-* Response: JSON with the pack list, hashmap with pack name and pack contents
+  * `page`: page number for files (default 1)
+  * `page_size`: page size for files (default 30)
+* Response: JSON, pack json with `files`, `files_count` and `pack` keys with 200 status or 404 if the pack does not exist.
 
-## `POST /api/pleroma/emoji/packs/download`
-### Download pack from another instance
-* Method `POST`
-* Authentication: required
-* Params:
-  * `url`: url of the instance to download from
-  * `name`: pack to download from that instance
-  * `as`: (*optional*) name how to save pack
-* Response: JSON, "ok" with 200 status if the pack was downloaded, or 500 if there were
-  errors downloading the pack
+```json
+{
+  "files": {...},
+  "files_count": 0, // emoji count in pack
+  "pack": {...}
+}
+```
+
+## `POST /api/pleroma/emoji/pack?name=:name`
 
-## `POST /api/pleroma/emoji/packs/:name`
 ### Creates an empty pack
+
 * Method `POST`
-* Authentication: required
-* Params: None
+* Authentication: required (admin)
+* Params:
+  * `name`: pack name
 * Response: JSON, "ok" and 200 status or 409 if the pack with that name already exists
 
-## `PATCH /api/pleroma/emoji/packs/:name`
+## `PATCH /api/pleroma/emoji/pack?name=:name`
+
 ### Updates (replaces) pack metadata
+
 * Method `PATCH`
-* Authentication: required
+* Authentication: required (admin)
 * Params:
+  * `name`: pack name
   * `metadata`: metadata to replace the old one
     * `license`: Pack license
     * `homepage`: Pack home page url
@@ -426,39 +425,85 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Response: JSON, updated "metadata" section of the pack and 200 status or 400 if there was a
   problem with the new metadata (the error is specified in the "error" part of the response JSON)
 
-## `DELETE /api/pleroma/emoji/packs/:name`
+## `DELETE /api/pleroma/emoji/pack?name=:name`
+
 ### Delete a custom emoji pack
+
 * Method `DELETE`
-* Authentication: required
-* Params: None
+* Authentication: required (admin)
+* Params:
+  * `name`: pack name
 * Response: JSON, "ok" and 200 status or 500 if there was an error deleting the pack
 
-## `POST /api/pleroma/emoji/packs/:name/files`
-### Add new file to the pack
-* Method `POST`
-* Authentication: required
+## `GET /api/pleroma/emoji/packs/import`
+
+### Imports packs from filesystem
+
+* Method `GET`
+* Authentication: required (admin)
+* Params: None
+* Response: JSON, returns a list of imported packs.
+
+## `GET /api/pleroma/emoji/packs/remote`
+
+### Make request to another instance for packs list
+
+* Method `GET`
+* Authentication: required (admin)
 * Params:
+  * `url`: url of the instance to get packs from
+  * `page`: page number for packs (default 1)
+  * `page_size`: page size for packs (default 50)
+* Response: JSON with the pack list, hashmap with pack name and pack contents
+
+## `POST /api/pleroma/emoji/packs/download`
+
+### Download pack from another instance
+
+* Method `POST`
+* Authentication: required (admin)
+* Params:
+  * `url`: url of the instance to download from
+  * `name`: pack to download from that instance
+  * `as`: (*optional*) name how to save pack
+* Response: JSON, "ok" with 200 status if the pack was downloaded, or 500 if there were
+  errors downloading the pack
+
+## `POST /api/pleroma/emoji/packs/files?name=:name`
+
+### Add new file to the pack
+
+* Method `POST`
+* Authentication: required (admin)
+* Params:
+  * `name`: pack name
   * `file`: file needs to be uploaded with the multipart request or link to remote file.
   * `shortcode`: (*optional*) shortcode for new emoji, must be unique for all emoji. If not sended, shortcode will be taken from original filename.
   * `filename`: (*optional*) new emoji file name. If not specified will be taken from original filename.
 * Response: JSON, list of files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
 
-## `PATCH /api/pleroma/emoji/packs/:name/files`
+## `PATCH /api/pleroma/emoji/packs/files?name=:name`
+
 ### Update emoji file from pack
+
 * Method `PATCH`
-* Authentication: required
+* Authentication: required (admin)
 * Params:
+  * `name`: pack name
   * `shortcode`: emoji file shortcode
   * `new_shortcode`: new emoji file shortcode
   * `new_filename`: new filename for emoji file
   * `force`: (*optional*) with true value to overwrite existing emoji with new shortcode
 * Response: JSON, list with updated files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
 
-## `DELETE /api/pleroma/emoji/packs/:name/files`
+## `DELETE /api/pleroma/emoji/packs/files?name=:name`
+
 ### Delete emoji file from pack
+
 * Method `DELETE`
-* Authentication: required
+* Authentication: required (admin)
 * Params:
+  * `name`: pack name
   * `shortcode`: emoji file shortcode
 * Response: JSON, list with updated files for updated pack (hashmap -> shortcode => filename) with status 200, either error status with error message.
 
@@ -483,30 +528,14 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 }
 ```
 
-## `GET /api/pleroma/emoji/packs/:name`
+## `GET /api/pleroma/emoji/packs/archive?name=:name`
 
-### Get pack.json for the pack
+### Requests a local pack archive from the instance
 
 * Method `GET`
 * Authentication: not required
 * Params:
-  * `page`: page number for files (default 1)
-  * `page_size`: page size for files (default 30)
-* Response: JSON, pack json with `files`, `files_count` and `pack` keys with 200 status or 404 if the pack does not exist.
-
-```json
-{
-  "files": {...},
-  "files_count": 0, // emoji count in pack
-  "pack": {...}
-}
-```
-
-## `GET /api/pleroma/emoji/packs/:name/archive`
-### Requests a local pack archive from the instance
-* Method `GET`
-* Authentication: not required
-* Params: None
+  * `name`: pack name
 * Response: the archive of the pack with a 200 status code, 403 if the pack is not set as shared,
   404 if the pack does not exist
 
@@ -586,3 +615,41 @@ Emoji reactions work a lot like favourites do. They make it possible to react to
   {"name": "😀", "count": 2, "me": true, "accounts": [{"id" => "xyz.."...}, {"id" => "zyx..."}]}
 ]
 ```
+
+## `POST /api/v1/pleroma/backups`
+### Create a user backup archive
+
+* Method: `POST`
+* Authentication: required
+* Params: none
+* Response: JSON
+* Example response:
+
+```json
+[{
+    "content_type": "application/zip",
+    "file_size": 0,
+    "inserted_at": "2020-09-10T16:18:03.000Z",
+    "processed": false,
+    "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip"
+}]
+```
+
+## `GET /api/v1/pleroma/backups`
+### Lists user backups
+
+* Method: `GET`
+* Authentication: not required
+* Params: none
+* Response: JSON
+* Example response:
+
+```json
+[{
+    "content_type": "application/zip",
+    "file_size": 55457,
+    "inserted_at": "2020-09-10T16:18:03.000Z",
+    "processed": true,
+    "url": "https://example.com/media/backups/archive-foobar-20200910T161803-QUhx6VYDRQ2wfV0SdA2Pfj_2CLM_ATUlw-D5l5TJf4Q.zip"
+}]
+```

docs/API/prometheus.md

@@ -2,15 +2,37 @@
 
 Pleroma includes support for exporting metrics via the [prometheus_ex](https://github.com/deadtrickster/prometheus.ex) library.
 
+Config example:
+
+```
+config :prometheus, Pleroma.Web.Endpoint.MetricsExporter,
+  enabled: true,
+  auth: {:basic, "myusername", "mypassword"},
+  ip_whitelist: ["127.0.0.1"],
+  path: "/api/pleroma/app_metrics",
+  format: :text
+```
+
+* `enabled` (Pleroma extension) enables the endpoint
+* `ip_whitelist` (Pleroma extension) could be used to restrict access only to specified IPs
+* `auth` sets the authentication (`false` for no auth; configurable to HTTP Basic Auth, see [prometheus-plugs](https://github.com/deadtrickster/prometheus-plugs#exporting) documentation)
+* `format` sets the output format (`:text` or `:protobuf`)
+* `path` sets the path to app metrics page 
+
+
 ## `/api/pleroma/app_metrics`
+
 ### Exports Prometheus application metrics
+
 * Method: `GET`
-* Authentication: not required
+* Authentication: not required by default (see configuration options above)
 * Params: none
-* Response: JSON
+* Response: text
 
 ## Grafana
+
 ### Config example
+
 The following is a config example to use with [Grafana](https://grafana.com)
 
 ```

docs/administration/CLI_tasks/frontend.md

@@ -1,12 +1,23 @@
 # Managing frontends
 
-`mix pleroma.frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]`
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.frontend install <frontend> [--ref <ref>] [--file <file>] [--build-url <build-url>] [--path <path>] [--build-dir <build-dir>]
+    ```
 
 Frontend can be installed either from local zip file, or automatically downloaded from the web.
 
-You can give all the options directly on the command like, but missing information will be filled out by looking at the data configured under `frontends.available` in the config files.
+You can give all the options directly on the command line, but missing information will be filled out by looking at the data configured under `frontends.available` in the config files.
+
+Currently, known `<frontend>` values are:
 
-Currently known `<frontend>` values are:
 - [admin-fe](https://git.pleroma.social/pleroma/admin-fe)
 - [kenoma](http://git.pleroma.social/lambadalambda/kenoma)
 - [pleroma-fe](http://git.pleroma.social/pleroma/pleroma-fe)
@@ -19,51 +30,67 @@ You can still install frontends that are not configured, see below.
 
 For a frontend configured under the `available` key, it's enough to install it by name.
 
-```sh tab="OTP"
+=== "OTP"
+
+    ```sh
     ./bin/pleroma_ctl frontend install pleroma
     ```
 
-```sh tab="From Source"
+=== "From Source"
+
+    ```sh
     mix pleroma.frontend install pleroma
     ```
 
-This will download the latest build for the the pre-configured `ref` and install it. It can then be configured as the one of the served frontends in the config file (see `primary` or `admin`).
+This will download the latest build for the pre-configured `ref` and install it. It can then be configured as the one of the served frontends in the config file (see `primary` or `admin`).
 
-You can override any of the details. To install a pleroma build from a different url, you could do this:
+You can override any of the details. To install a pleroma build from a different URL, you could do this:
 
-```sh tab="OPT"
+=== "OTP"
+
+    ```sh
     ./bin/pleroma_ctl frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
     ```
 
-```sh tab="From Source"
+=== "From Source"
+
+    ```sh
     mix pleroma.frontend install pleroma --ref 2hu_edition --build-url https://example.org/raymoo.zip
     ```
 
 Similarly, you can also install from a local zip file.
 
-```sh tab="OTP"
+=== "OTP"
+
+    ```sh
     ./bin/pleroma_ctl frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
     ```
 
-```sh tab="From Source"
+=== "From Source"
+
+    ```sh
     mix pleroma.frontend install pleroma --ref mybuild --file ~/Downloads/doomfe.zip
     ```
 
-The resulting frontend will always be installed into a folder of this template: `${instance_static}/frontends/${name}/${ref}`
+The resulting frontend will always be installed into a folder of this template: `${instance_static}/frontends/${name}/${ref}`.
 
-Careful: This folder will be completely replaced on installation
+Careful: This folder will be completely replaced on installation.
 
 ## Example installation for an unknown frontend
 
-The installation process is the same, but you will have to give all the needed options on the commond line. For example:
+The installation process is the same, but you will have to give all the needed options on the command line. For example:
 
-```sh tab="OTP"
+=== "OTP"
+
+    ```sh
     ./bin/pleroma_ctl frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
     ```
 
-```sh tab="From Source"
+=== "From Source"
+
+    ```sh
     mix pleroma.frontend install gensokyo --ref master --build-url https://gensokyo.2hu/builds/marisa.zip
     ```
 
-If you don't have a zip file but just want to install a frontend from a local path, you can simply copy the files over a folder of this template: `${instance_static}/frontends/${name}/${ref}`
+If you don't have a zip file but just want to install a frontend from a local path, you can simply copy the files over a folder of this template: `${instance_static}/frontends/${name}/${ref}`.
 

docs/administration/CLI_tasks/instance.md

@@ -37,3 +37,8 @@ If any of the options are left unspecified, you will be prompted interactively.
 - `--static-dir <path>` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)
 - `--listen-ip <ip>` - the ip the app should listen to, defaults to 127.0.0.1
 - `--listen-port <port>` - the port the app should listen to, defaults to 4000
+- `--strip-uploads <Y|N>` - use ExifTool to strip uploads of sensitive location data
+- `--anonymize-uploads <Y|N>` - randomize uploaded filenames
+- `--dedupe-uploads <Y|N>` - store files based on their hash to reduce data storage requirements if duplicates are uploaded with different filenames
+- `--skip-release-env` - skip generation the release environment file
+- `--release-env-file` - release environment file path

docs/administration/CLI_tasks/release_environments.md

@@ -0,0 +1,9 @@
+# Generate release environment file
+
+```sh tab="OTP"
+ ./bin/pleroma_ctl release_env gen
+```
+
+```sh tab="From Source"
+mix pleroma.release_env gen
+```

docs/administration/backup.md

@@ -5,20 +5,25 @@
 1. Stop the Pleroma service.
 2. Go to the working directory of Pleroma (default is `/opt/pleroma`)
 3. Run `sudo -Hu postgres pg_dump -d <pleroma_db> --format=custom -f </path/to/backup_location/pleroma.pgdump>` (make sure the postgres user has write access to the destination file)
-4. Copy `pleroma.pgdump`, `config/prod.secret.exs` and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too.
+4. Copy `pleroma.pgdump`, `config/prod.secret.exs`, `config/setup_db.psql` (if still available) and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too.
 5. Restart the Pleroma service.
 
 ## Restore/Move
 
-1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers). Try to use the same database name.
+1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers).
 2. Stop the Pleroma service.
 3. Go to the working directory of Pleroma (default is `/opt/pleroma`)
 4. Copy the above mentioned files back to their original position.
-5. Drop the existing database and recreate an empty one `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;';` `sudo -Hu postgres psql -c 'CREATE DATABASE <pleroma_db>;';`
-6. Run `sudo -Hu postgres pg_restore -d <pleroma_db> -v -1 </path/to/backup_location/pleroma.pgdump>`
-7. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
-8. Restart the Pleroma service.
-9. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries.
+5. Drop the existing database and user if restoring in-place. `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;';` `sudo -Hu postgres psql -c 'DROP USER <pleroma_db>;'`
+6. Restore the database schema and pleroma postgres role the with the original `setup_db.psql` if you have it: `sudo -Hu postgres psql -f config/setup_db.psql`. 
+
+  Alternatively, run the `mix pleroma.instance gen` task again. You can ignore most of the questions, but make the database user, name, and password the same as found in your backup of `config/prod.secret.exs`. Then run the restoration of the pleroma role and schema with of the generated `config/setup_db.psql` as instructed above. You may delete the `config/generated_config.exs` file as it is not needed.
+
+7. Now restore the Pleroma instance's data into the empty database schema: `sudo -Hu postgres pg_restore -d <pleroma_db> -v -1 </path/to/backup_location/pleroma.pgdump>`
+8. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
+9. Restart the Pleroma service.
+10. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries.
+11. If setting up on a new server configure Nginx by using the `installation/pleroma.nginx` config sample or reference the Pleroma installation guide for your OS which contains the Nginx configuration instructions.
 
 [^1]: Prefix with `MIX_ENV=prod` to run it using the production config file.
 
@@ -31,6 +36,6 @@
 3. Disable pleroma from systemd `systemctl disable pleroma`
 4. Remove the files and folders you created during installation (see installation guide). This includes the pleroma, nginx and systemd files and folders.
 5. Reload nginx now that the configuration is removed `systemctl reload nginx`
-6. Remove the database and database user `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;';` `sudo -Hu postgres psql -c 'DROP USER <pleroma_db>;';`
+6. Remove the database and database user `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;';` `sudo -Hu postgres psql -c 'DROP USER <pleroma_db>;'`
 7. Remove the system user `userdel pleroma`
 8. Remove the dependencies that you don't need anymore (see installation guide). Make sure you don't remove packages that are still needed for other software that you have running!

docs/ap_extensions.md

@@ -1,11 +1,41 @@
-# ChatMessages
+# AP Extensions
+## Actor endpoints
 
-ChatMessages are the messages sent in 1-on-1 chats. They are similar to
+The following endpoints are additionally present into our actors.
+
+- `oauthRegistrationEndpoint` (`http://litepub.social/ns#oauthRegistrationEndpoint`)
+- `uploadMedia` (`https://www.w3.org/ns/activitystreams#uploadMedia`)
+
+### oauthRegistrationEndpoint
+
+Points to MastodonAPI `/api/v1/apps` for now.
+
+See <https://docs.joinmastodon.org/methods/apps/>
+
+### uploadMedia
+
+Inspired by <https://www.w3.org/wiki/SocialCG/ActivityPub/MediaUpload>, it is part of the ActivityStreams namespace because it used to be part of the ActivityPub specification and got removed from it.
+
+Content-Type: multipart/form-data
+
+Parameters:
+- (required) `file`: The file being uploaded
+- (optionnal) `description`: A plain-text description of the media, for accessibility purposes.
+
+Response: HTTP 201 Created with the object into the body, no `Location` header provided as it doesn't have an `id`
+
+The object given in the reponse should then be inserted into an Object's `attachment` field.
+
+## ChatMessages
+
+`ChatMessage`s are the messages sent in 1-on-1 chats. They are similar to
 `Note`s, but the addresing is done by having a single AP actor in the `to`
 field. Addressing multiple actors is not allowed. These messages are always
 private, there is no public version of them. They are created with a `Create`
 activity.
 
+They are part of the `litepub` namespace as `http://litepub.social/ns#ChatMessage`.
+
 Example:
 
 ```json

docs/clients.md

@@ -7,97 +7,105 @@ Feel free to contact us to be added to this list!
 - Homepage: <https://www.pleroma.com/#desktopApp>
 - Source Code: <https://github.com/roma-apps/roma-desktop>
 - Platforms: Windows, Mac, Linux
-- Features: Streaming Ready
+- Features: MastoAPI, Streaming Ready
 
 ### Social
 - Source Code: <https://gitlab.gnome.org/World/Social>
 - Contact: [@brainblasted@social.libre.fi](https://social.libre.fi/users/brainblasted)
 - Platforms: Linux (GNOME)
 - Note(2019-01-28): Not at a pre-alpha stage yet
+- Features: MastoAPI
 
 ### Whalebird
 - Homepage: <https://whalebird.org/>
 - Source Code: <https://github.com/h3poteto/whalebird-desktop>
 - Contact: [@h3poteto@pleroma.io](https://pleroma.io/users/h3poteto)
 - Platforms: Windows, Mac, Linux
-- Features: Streaming Ready
+- Features: MastoAPI, Streaming Ready
 
 ## Handheld
+### AndStatus
+- Homepage: <http://andstatus.org/>
+- Source Code: <https://github.com/andstatus/andstatus/>
+- Platforms: Android
+- Features: MastoAPI, ActivityPub (Client-to-Server)
+
 ### Amaroq
 - Homepage: <https://itunes.apple.com/us/app/amaroq-for-mastodon/id1214116200>
 - Source Code: <https://github.com/ReticentJohn/Amaroq>
 - Contact: [@eurasierboy@mastodon.social](https://mastodon.social/users/eurasierboy)
 - Platforms: iOS
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Fedilab
 - Homepage: <https://fedilab.app/>
 - Source Code: <https://framagit.org/tom79/fedilab/>
 - Contact: [@fedilab@framapiaf.org](https://framapiaf.org/users/fedilab)
 - Platforms: Android
-- Features: Streaming Ready, Moderation, Text Formatting
+- Features: MastoAPI, Streaming Ready, Moderation, Text Formatting
 
 ### Kyclos
 - Source Code: <https://git.pleroma.social/pleroma/harbour-kyclos>
 - Platforms: SailfishOS
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Husky
 - Source code: <https://git.mentality.rip/FWGS/Husky>
 - Contact: [@Husky@enigmatic.observer](https://enigmatic.observer/users/Husky)
 - Platforms: Android
-- Features: No Streaming, Emoji Reactions, Text Formatting, FE Stickers
+- Features: MastoAPI, No Streaming, Emoji Reactions, Text Formatting, FE Stickers
 
 ### Fedi
 - Homepage: <https://www.fediapp.com/>
 - Source Code: Proprietary, but gratis
 - Platforms: iOS, Android
-- Features: Pleroma-specific features like Reactions
+- Features: MastoAPI, Pleroma-specific features like Reactions
 
 ### Tusky
 - Homepage: <https://tuskyapp.github.io/>
 - Source Code: <https://github.com/tuskyapp/Tusky>
 - Contact: [@ConnyDuck@mastodon.social](https://mastodon.social/users/ConnyDuck)
 - Platforms: Android
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Twidere
 - Homepage: <https://twidere.mariotaku.org/>
 - Source Code: <https://github.com/TwidereProject/Twidere-Android/>
 - Contact: <me@mariotaku.org>
 - Platform: Android
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Indigenous
 - Homepage: <https://indigenous.realize.be/>
 - Source Code: <https://github.com/swentel/indigenous-android/>
-- Contact: [@realize.be@realize.be](@realize.be@realize.be)
+- Contact: [@swentel@realize.be](https://realize.be)
 - Platforms: Android
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ## Alternative Web Interfaces
 ### Brutaldon
 - Homepage: <https://jfm.carcosa.net/projects/software/brutaldon/>
 - Source Code: <https://git.carcosa.net/jmcbray/brutaldon>
 - Contact: [@gcupc@glitch.social](https://glitch.social/users/gcupc)
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Halcyon
 - Source Code: <https://notabug.org/halcyon-suite/halcyon>
 - Contact: [@halcyon@social.csswg.org](https://social.csswg.org/users/halcyon)
-- Features: Streaming Ready
+- Features: MastoAPI, Streaming Ready
 
 ### Pinafore
 - Homepage: <https://pinafore.social/>
 - Source Code: <https://github.com/nolanlawson/pinafore>
 - Contact: [@pinafore@mastodon.technology](https://mastodon.technology/users/pinafore)
 - Note: Pleroma support is a secondary goal
-- Features: No Streaming
+- Features: MastoAPI, No Streaming
 
 ### Sengi
 - Homepage: <https://nicolasconstant.github.io/sengi/>
 - Source Code: <https://github.com/NicolasConstant/sengi>
 - Contact: [@sengi_app@mastodon.social](https://mastodon.social/users/sengi_app)
+- Features: MastoAPI
 
 ### DashFE
 - Source Code: <https://notabug.org/daisuke/DashboardFE>
@@ -107,3 +115,4 @@ Feel free to contact us to be added to this list!
 - Source Code: <https://git.freesoftwareextremist.com/bloat/>
 - Contact: [@r@freesoftwareextremist.com](https://freesoftwareextremist.com/users/r)
 - Features: Does not requires JavaScript
+- Features: MastoAPI

docs/configuration/cheatsheet.md

@@ -45,6 +45,7 @@ To add configuration to your config file, you can copy it from the base config.
     older software for theses nicknames.
 * `max_pinned_statuses`: The maximum number of pinned statuses. `0` will disable the feature.
 * `autofollowed_nicknames`: Set to nicknames of (local) users that every new user should automatically follow.
+* `autofollowing_nicknames`: Set to nicknames of (local) users that automatically follows every newly registered user.
 * `attachment_links`: Set to true to enable automatically adding attachment link text to statuses.
 * `max_report_comment_size`: The maximum size of the report comment (Default: `1000`).
 * `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). Default: `false`.
@@ -113,7 +114,7 @@ To add configuration to your config file, you can copy it from the base config.
     * `Pleroma.Web.ActivityPub.MRF.MentionPolicy`: Drops posts mentioning configurable users. (See [`:mrf_mention`](#mrf_mention)).
     * `Pleroma.Web.ActivityPub.MRF.VocabularyPolicy`: Restricts activities to a configured set of vocabulary. (See [`:mrf_vocabulary`](#mrf_vocabulary)).
     * `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)).
-    * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.ActivityExpiration` to be enabled for processing the scheduled delections.
+    * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.Workers.PurgeExpiredActivity` to be enabled for processing the scheduled delections.
     * `Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy`: Makes all bot posts to disappear from public timelines.
 * `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo).
 * `transparency_exclusions`: Exclude specific instance names from MRF transparency.  The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.
@@ -219,12 +220,6 @@ config :pleroma, :mrf_user_allowlist, %{
 * `total_user_limit`: the number of scheduled activities a user is allowed to create in total (Default: `300`)
 * `enabled`: whether scheduled activities are sent to the job queue to be executed
 
-## Pleroma.ActivityExpiration
-
-Enables the worker which processes posts scheduled for deletion. Pinned posts are exempt from expiration.
-
-* `enabled`: whether expired activities will be sent to the job queue to be deleted
-
 ## FedSockets
 FedSockets is an experimental feature allowing for Pleroma backends to federate using a persistant websocket connection as opposed to making each federation a seperate http connection. This feature is currently off by default. It is configurable throught he following options.
 
@@ -416,25 +411,25 @@ This will make Pleroma listen on `127.0.0.1` port `8080` and generate urls start
 * ``referrer_policy``: The referrer policy to use, either `"same-origin"` or `"no-referrer"`.
 * ``report_uri``: Adds the specified url to `report-uri` and `report-to` group in CSP header.
 
-### Pleroma.Plugs.RemoteIp
+### Pleroma.Web.Plugs.RemoteIp
 
 !!! warning
     If your instance is not behind at least one reverse proxy, you should not enable this plug.
 
-`Pleroma.Plugs.RemoteIp` is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
+`Pleroma.Web.Plugs.RemoteIp` is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
 
 Available options:
 
 * `enabled` - Enable/disable the plug. Defaults to `false`.
-* `headers` - A list of strings naming the `req_headers` to use when deriving the `remote_ip`. Order does not matter. Defaults to `["x-forwarded-for"]`.
-* `proxies` - A list of strings in [CIDR](https://en.wikipedia.org/wiki/CIDR) notation specifying the IPs of known proxies. Defaults to `[]`.
-* `reserved` - Defaults to [localhost](https://en.wikipedia.org/wiki/Localhost) and [private network](https://en.wikipedia.org/wiki/Private_network).
+* `headers` - A list of strings naming the HTTP headers to use when deriving the true client IP address. Defaults to `["x-forwarded-for"]`.
+* `proxies` - A list of upstream proxy IP subnets in CIDR notation from which we will parse the content of `headers`. Defaults to `[]`. IPv4 entries without a bitmask will be assumed to be /32 and IPv6 /128.
+* `reserved` - A list of reserved IP subnets in CIDR notation which should be ignored if found in `headers`. Defaults to `["127.0.0.0/8", "::1/128", "fc00::/7", "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]`.
 
 
 ### :rate_limit
 
 !!! note
-   If your instance is behind a reverse proxy ensure [`Pleroma.Plugs.RemoteIp`](#pleroma-plugs-remoteip) is enabled (it is enabled by default).
+   If your instance is behind a reverse proxy ensure [`Pleroma.Web.Plugs.RemoteIp`](#pleroma-plugs-remoteip) is enabled (it is enabled by default).
 
 A keyword list of rate limiters where a key is a limiter name and value is the limiter configuration. The basic configuration is a tuple where:
 
@@ -1083,6 +1078,20 @@ Control favicons for instances.
 
 * `enabled`: Allow/disallow displaying and getting instances favicons
 
+## Pleroma.User.Backup
+
+!!! note
+    Requires enabled email
+
+* `:purge_after_days` an integer, remove backup achives after N days.
+* `:limit_days` an integer, limit user to export not more often than once per N days.
+* `:dir` a string with a path to backup temporary directory or `nil` to let Pleroma choose temporary directory in the following order:
+    1. the directory named by the TMPDIR environment variable
+    2. the directory named by the TEMP environment variable
+    3. the directory named by the TMP environment variable
+    4. C:\TMP on Windows or /tmp on Unix-like operating systems
+    5. as a last resort, the current working directory
+
 ## Frontend management
 
 Frontends in Pleroma are swappable - you can specify which one to use here.

docs/dev.md

@@ -6,7 +6,7 @@ This document contains notes and guidelines for Pleroma developers.
 
 * Pleroma supports hierarchical OAuth scopes, just like Mastodon but with added granularity of admin scopes. For a reference, see [Mastodon OAuth scopes](https://docs.joinmastodon.org/api/oauth-scopes/).
 
-* It is important to either define OAuth scope restrictions or explicitly mark OAuth scope check as skipped, for every controller action. To define scopes, call `plug(Pleroma.Plugs.OAuthScopesPlug, %{scopes: [...]})`. To explicitly set OAuth scopes check skipped, call `plug(:skip_plug, Pleroma.Plugs.OAuthScopesPlug <when ...>)`.
+* It is important to either define OAuth scope restrictions or explicitly mark OAuth scope check as skipped, for every controller action. To define scopes, call `plug(Pleroma.Web.Plugs.OAuthScopesPlug, %{scopes: [...]})`. To explicitly set OAuth scopes check skipped, call `plug(:skip_plug, Pleroma.Web.Plugs.OAuthScopesPlug <when ...>)`.
 
 * In controllers, `use Pleroma.Web, :controller` will result in `action/2` (see `Pleroma.Web.controller/0` for definition) be called prior to actual controller action, and it'll perform security / privacy checks before passing control to actual controller action.
 
@@ -16,7 +16,7 @@ This document contains notes and guidelines for Pleroma developers.
 
 ## [HTTP Basic Authentication](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Authorization)
 
-* With HTTP Basic Auth, OAuth scopes check is _not_ performed for any action (since password is provided during the auth, requester is able to obtain a token with full permissions anyways). `Pleroma.Plugs.AuthenticationPlug` and `Pleroma.Plugs.LegacyAuthenticationPlug` both call `Pleroma.Plugs.OAuthScopesPlug.skip_plug(conn)` when password is provided.
+* With HTTP Basic Auth, OAuth scopes check is _not_ performed for any action (since password is provided during the auth, requester is able to obtain a token with full permissions anyways). `Pleroma.Web.Plugs.AuthenticationPlug` and `Pleroma.Web.Plugs.LegacyAuthenticationPlug` both call `Pleroma.Web.Plugs.OAuthScopesPlug.skip_plug(conn)` when password is provided.
 
 ## Auth-related configuration, OAuth consumer mode etc.
 

docs/installation/alpine_linux_en.md

@@ -13,6 +13,7 @@ It assumes that you have administrative rights, either as root or a user with [s
 * `erlang-parsetools`
 * `erlang-xmerl`
 * `git`
+* `file-dev`
 * Development Tools
 * `cmake`
 
@@ -42,7 +43,7 @@ sudo apk upgrade
 * Install some tools, which are needed later:
 
 ```shell
-sudo apk add git build-base cmake
+sudo apk add git build-base cmake file-dev
 ```
 
 ### Install Elixir and Erlang

docs/installation/arch_linux_en.md

@@ -10,6 +10,7 @@ This guide will assume that you have administrative rights, either as root or a
 * `git`
 * `base-devel`
 * `cmake`
+* `file`
 
 #### Optional packages used in this guide
 
@@ -30,7 +31,7 @@ sudo pacman -Syu
 * Install some of the above mentioned programs:
 
 ```shell
-sudo pacman -S git base-devel elixir cmake
+sudo pacman -S git base-devel elixir cmake file
 ```
 
 ### Install PostgreSQL

docs/installation/debian_based_en.md

@@ -10,6 +10,7 @@ This guide will assume you are on Debian Stretch. This guide should also work wi
 * `elixir` (1.8+, Follow the guide to install from the Erlang Solutions repo or use [asdf](https://github.com/asdf-vm/asdf) as the pleroma user)
 * `erlang-dev`
 * `erlang-nox`
+* `libmagic-dev`
 * `git`
 * `build-essential`
 * `cmake`
@@ -34,7 +35,7 @@ sudo apt full-upgrade
 * Install some of the above mentioned programs:
 
 ```shell
-sudo apt install git build-essential postgresql postgresql-contrib cmake
+sudo apt install git build-essential postgresql postgresql-contrib cmake libmagic-devel
 ```
 
 ### Install Elixir and Erlang
@@ -100,6 +101,7 @@ sudo -Hu pleroma mix deps.get
 mv config/{generated_config.exs,prod.secret.exs}
 ```
 
+
 * The previous command creates also the file `config/setup_db.psql`, with which you can create the database:
 
 ```shell
@@ -180,6 +182,7 @@ sudo cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.se
 ```
 
 * Edit the service file and make sure that all paths fit your installation
+* Check that `EnvironmentFile` contains the correct path to the env file. Or generate the env file: `sudo -Hu pleroma mix pleroma.release_env gen`
 * Enable and start `pleroma.service`:
 
 ```shell

docs/installation/debian_based_jp.md

@@ -17,6 +17,7 @@
 - `git`
 - `build-essential`
 - `cmake`
+- `libmagic-dev`
 
 #### このガイドで利用している追加パッケージ
 
@@ -36,7 +37,7 @@ sudo apt full-upgrade
 
 * 上記に挙げたパッケージをインストールしておきます。
 ```
-sudo apt install git build-essential postgresql postgresql-contrib cmake ffmpeg imagemagick
+sudo apt install git build-essential postgresql postgresql-contrib cmake ffmpeg imagemagick libmagic-dev
 ```
 
 ### ElixirとErlangをインストールします

docs/installation/gentoo_en.md

@@ -29,6 +29,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
 * `dev-lang/elixir`
 * `dev-vcs/git`
 * `dev-util/cmake`
+* `sys-apps/file`
 
 #### Optional ebuilds used in this guide
 
@@ -50,7 +51,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
 * Emerge all required the required and suggested software in one go:
 
 ```shell
- # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake
+ # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake sys-apps/file
 ```
 
 If you would not like to install the optional packages, remove them from this line. 

docs/installation/otp_en.md

@@ -27,22 +27,23 @@ Other than things bundled in the OTP release Pleroma depends on:
 * PostgreSQL (also utilizes extensions in postgresql-contrib)
 * nginx (could be swapped with another reverse proxy but this guide covers only it)
 * certbot (for Let's Encrypt certificates, could be swapped with another ACME client, but this guide covers only it)
+* libmagic/file
 
 === "Alpine"
     ```
     echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories
     apk update
-    apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot
+    apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot file-dev
     ```
 
 === "Debian/Ubuntu"
     ```
-    apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot
+    apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot libmagic-dev
     ```
 
 ### Installing optional packages
 
-Per [`docs/installation/optional/media_graphics_packages.md`](docs/installation/optional/media_graphics_packages.md):
+Per [`docs/installation/optional/media_graphics_packages.md`](optional/media_graphics_packages.md):
   * ImageMagick
   * ffmpeg
   * exiftool
@@ -101,6 +102,8 @@ It is encouraged to check [Optimizing your PostgreSQL performance](../configurat
 If you are using PostgreSQL 12 or higher, add this to your Ecto database configuration
 
 ```elixir
+#
+config :pleroma, Pleroma.Repo,
 prepare: :named,
 parameters: [
   plan_cache_mode: "force_custom_plan"
@@ -146,6 +149,9 @@ chown -R pleroma /etc/pleroma
 # Run the config generator
 su pleroma -s $SHELL -lc "./bin/pleroma_ctl instance gen --output /etc/pleroma/config.exs --output-psql /tmp/setup_db.psql"
 
+# Run the environment file generator.
+su pleroma -s $SHELL -lc "./bin/pleroma_ctl release_env gen"
+
 # Create the postgres database
 su postgres -s $SHELL -lc "psql -f /tmp/setup_db.psql"
 
@@ -156,7 +162,7 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
 # su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
 
 # Start the instance to verify that everything is working as expected
-su pleroma -s $SHELL -lc "./bin/pleroma daemon"
+su pleroma -s $SHELL -lc "export $(cat /opt/pleroma/config/pleroma.env); ./bin/pleroma daemon"
 
 # Wait for about 20 seconds and query the instance endpoint, if it shows your uri, name and email correctly, you are configured correctly
 sleep 20 && curl http://localhost:4000/api/v1/instance
@@ -308,4 +314,3 @@ This will create an account withe the username of 'joeuser' with the email addre
 ## Questions
 
 Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
-

installation/init.d/pleroma

@@ -8,6 +8,7 @@ pidfile="/var/run/pleroma.pid"
 directory=/opt/pleroma
 healthcheck_delay=60
 healthcheck_timer=30
+export $(cat /opt/pleroma/config/pleroma.env)
 
 : ${pleroma_port:-4000}
 

installation/pleroma.service

@@ -17,6 +17,8 @@ Environment="MIX_ENV=prod"
 Environment="HOME=/var/lib/pleroma"
 ; Path to the folder containing the Pleroma installation.
 WorkingDirectory=/opt/pleroma
+; Path to the environment file. the file contains RELEASE_COOKIE and etc 
+EnvironmentFile=/opt/pleroma/config/pleroma.env
 ; Path to the Mix binary.
 ExecStart=/usr/bin/mix phx.server
 
@@ -29,8 +31,6 @@ ProtectHome=true
 ProtectSystem=full
 ; Sets up a new /dev mount for the process and only adds API pseudo devices like /dev/null, /dev/zero or /dev/random but not physical devices. Disabled by default because it may not work on devices like the Raspberry Pi.
 PrivateDevices=false
-; Ensures that the service process and all its children can never gain new privileges through execve().
-NoNewPrivileges=true
 ; Drops the sysadmin capability from the daemon.
 CapabilityBoundingSet=~CAP_SYS_ADMIN
 

lib/mix/tasks/pleroma/count_statuses.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.CountStatuses do
   @shortdoc "Re-counts statuses for all users"
 

lib/mix/tasks/pleroma/digest.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.Digest do
   use Mix.Task
   import Mix.Pleroma

lib/mix/tasks/pleroma/docs.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.Docs do
   use Mix.Task
   import Mix.Pleroma

lib/mix/tasks/pleroma/email.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.Email do
   use Mix.Task
   import Mix.Pleroma

lib/mix/tasks/pleroma/instance.ex

@@ -33,7 +33,12 @@ def run(["gen" | rest]) do
           uploads_dir: :string,
           static_dir: :string,
           listen_ip: :string,
-          listen_port: :string
+          listen_port: :string,
+          strip_uploads: :string,
+          anonymize_uploads: :string,
+          dedupe_uploads: :string,
+          skip_release_env: :boolean,
+          release_env_file: :string
         ],
         aliases: [
           o: :output,
@@ -158,6 +163,30 @@ def run(["gen" | rest]) do
         )
         |> Path.expand()
 
+      strip_uploads =
+        get_option(
+          options,
+          :strip_uploads,
+          "Do you want to strip location (GPS) data from uploaded images? (y/n)",
+          "y"
+        ) === "y"
+
+      anonymize_uploads =
+        get_option(
+          options,
+          :anonymize_uploads,
+          "Do you want to anonymize the filenames of uploads? (y/n)",
+          "n"
+        ) === "y"
+
+      dedupe_uploads =
+        get_option(
+          options,
+          :dedupe_uploads,
+          "Do you want to deduplicate uploaded files? (y/n)",
+          "n"
+        ) === "y"
+
       Config.put([:instance, :static_dir], static_dir)
 
       secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
@@ -188,7 +217,13 @@ def run(["gen" | rest]) do
           uploads_dir: uploads_dir,
           rum_enabled: rum_enabled,
           listen_ip: listen_ip,
-          listen_port: listen_port
+          listen_port: listen_port,
+          upload_filters:
+            upload_filters(%{
+              strip: strip_uploads,
+              anonymize: anonymize_uploads,
+              dedupe: dedupe_uploads
+            })
         )
 
       result_psql =
@@ -208,6 +243,24 @@ def run(["gen" | rest]) do
 
       write_robots_txt(static_dir, indexable, template_dir)
 
+      if Keyword.get(options, :skip_release_env, false) do
+        shell_info("""
+        Release environment file is skip. Please generate the release env file before start.
+        `MIX_ENV=#{Mix.env()} mix pleroma.release_env gen`
+        """)
+      else
+        shell_info("Generation the environment file:")
+
+        release_env_args =
+          with path when not is_nil(path) <- Keyword.get(options, :release_env_file) do
+            ["gen", "--path", path]
+          else
+            _ -> ["gen"]
+          end
+
+        Mix.Tasks.Pleroma.ReleaseEnv.run(release_env_args)
+      end
+
       shell_info(
         "\n All files successfully written! Refer to the installation instructions for your platform for next steps."
       )
@@ -247,4 +300,31 @@ defp write_robots_txt(static_dir, indexable, template_dir) do
     File.write(robots_txt_path, robots_txt)
     shell_info("Writing #{robots_txt_path}.")
   end
+
+  defp upload_filters(filters) when is_map(filters) do
+    enabled_filters =
+      if filters.strip do
+        [Pleroma.Upload.Filter.ExifTool]
+      else
+        []
+      end
+
+    enabled_filters =
+      if filters.anonymize do
+        enabled_filters ++ [Pleroma.Upload.Filter.AnonymizeFilename]
+      else
+        enabled_filters
+      end
+
+    enabled_filters =
+      if filters.dedupe do
+        enabled_filters ++ [Pleroma.Upload.Filter.Dedupe]
+      else
+        enabled_filters
+      end
+
+    enabled_filters
+  end
+
+  defp upload_filters(_), do: []
 end

lib/mix/tasks/pleroma/notification_settings.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Mix.Tasks.Pleroma.NotificationSettings do
   @shortdoc "Enable&Disable privacy option for push notifications"
   @moduledoc """

lib/mix/tasks/pleroma/release_env.ex

@@ -0,0 +1,76 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.ReleaseEnv do
+  use Mix.Task
+  import Mix.Pleroma
+
+  @shortdoc "Generate Pleroma environment file."
+  @moduledoc File.read!("docs/administration/CLI_tasks/release_environments.md")
+
+  def run(["gen" | rest]) do
+    {options, [], []} =
+      OptionParser.parse(
+        rest,
+        strict: [
+          force: :boolean,
+          path: :string
+        ],
+        aliases: [
+          p: :path,
+          f: :force
+        ]
+      )
+
+    file_path =
+      get_option(
+        options,
+        :path,
+        "Environment file path",
+        "./config/pleroma.env"
+      )
+
+    env_path = Path.expand(file_path)
+
+    proceed? =
+      if File.exists?(env_path) do
+        get_option(
+          options,
+          :force,
+          "Environment file already exists. Do you want to overwrite the #{env_path} file? (y/n)",
+          "n"
+        ) === "y"
+      else
+        true
+      end
+
+    if proceed? do
+      case do_generate(env_path) do
+        {:error, reason} ->
+          shell_error(
+            File.Error.message(%{action: "write to file", reason: reason, path: env_path})
+          )
+
+        _ ->
+          shell_info("\nThe file generated: #{env_path}.\n")
+
+          shell_info("""
+          WARNING: before start pleroma app please make sure to make the file read-only and non-modifiable.
+            Example:
+              chmod 0444 #{file_path}
+              chattr +i #{file_path}
+          """)
+      end
+    else
+      shell_info("\nThe file is exist. #{env_path}.\n")
+    end
+  end
+
+  def do_generate(path) do
+    content = "RELEASE_COOKIE=#{Base.encode32(:crypto.strong_rand_bytes(32))}"
+
+    File.mkdir_p!(Path.dirname(path))
+    File.write(path, content)
+  end
+end

lib/mix/tasks/pleroma/user.ex

@@ -452,7 +452,7 @@ def run(["list"]) do
       |> Enum.each(fn user ->
         shell_info(
           "#{user.nickname} moderator: #{user.is_moderator}, admin: #{user.is_admin}, locked: #{
-            user.locked
+            user.is_locked
           }, deactivated: #{user.deactivated}"
         )
       end)
@@ -480,10 +480,10 @@ defp set_admin(user, value) do
   defp set_locked(user, value) do
     {:ok, user} =
       user
-      |> Changeset.change(%{locked: value})
+      |> Changeset.change(%{is_locked: value})
       |> User.update_and_set_cache()
 
-    shell_info("Locked status of #{user.nickname}: #{user.locked}")
+    shell_info("Locked status of #{user.nickname}: #{user.is_locked}")
     user
   end
 

lib/phoenix/transports/web_socket/raw.ex

@@ -31,7 +31,12 @@ def init(%Plug.Conn{method: "GET"} = conn, {endpoint, handler, transport}) do
 
     case conn do
       %{halted: false} = conn ->
-        case Transport.connect(endpoint, handler, transport, __MODULE__, nil, conn.params) do
+        case handler.connect(%{
+               endpoint: endpoint,
+               transport: transport,
+               options: [serializer: nil],
+               params: conn.params
+             }) do
           {:ok, socket} ->
             {:ok, conn, {__MODULE__, {socket, opts}}}
 

lib/pleroma/application.ex

@@ -52,7 +52,7 @@ def start(_type, _args) do
     Pleroma.HTML.compile_scrubbers()
     Pleroma.Config.Oban.warn()
     Config.DeprecationWarnings.warn()
-    Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
+    Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled()
     Pleroma.ApplicationRequirements.verify!()
     setup_instrumenters()
     load_custom_modules()
@@ -88,18 +88,19 @@ def start(_type, _args) do
         Pleroma.Repo,
         Config.TransferTask,
         Pleroma.Emoji,
-        Pleroma.Plugs.RateLimiter.Supervisor
+        Pleroma.Web.Plugs.RateLimiter.Supervisor
       ] ++
         cachex_children() ++
         http_children(adapter, @env) ++
         [
           Pleroma.Stats,
           Pleroma.JobQueueMonitor,
+          {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Config.get([:majic_pool, :size], 2)]},
           {Oban, Config.get(Oban)}
         ] ++
         task_children(@env) ++
         dont_run_in_test(@env) ++
-        chat_child(@env, chat_enabled?()) ++
+        chat_child(chat_enabled?()) ++
         [
           Pleroma.Web.Endpoint,
           Pleroma.Gopher.Server
@@ -150,7 +151,10 @@ defp setup_instrumenters do
 
     Pleroma.Web.Endpoint.MetricsExporter.setup()
     Pleroma.Web.Endpoint.PipelineInstrumenter.setup()
-    Pleroma.Web.Endpoint.Instrumenter.setup()
+
+    # Note: disabled until prometheus-phx is integrated into prometheus-phoenix:
+    # Pleroma.Web.Endpoint.Instrumenter.setup()
+    PrometheusPhx.setup()
   end
 
   defp cachex_children do
@@ -164,7 +168,11 @@ defp cachex_children do
       build_cachex("web_resp", limit: 2500),
       build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
       build_cachex("failed_proxy_url", limit: 2500),
-      build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000)
+      build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000),
+      build_cachex("chat_message_id_idempotency_key",
+        expiration: chat_message_id_idempotency_key_expiration(),
+        limit: 500_000
+      )
     ]
   end
 
@@ -174,6 +182,9 @@ defp emoji_packs_expiration,
   defp idempotency_expiration,
     do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
 
+  defp chat_message_id_idempotency_key_expiration,
+    do: expiration(default: :timer.minutes(2), interval: :timer.seconds(60))
+
   defp seconds_valid_interval,
     do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
 
@@ -201,11 +212,14 @@ defp dont_run_in_test(_) do
     ]
   end
 
-  defp chat_child(_env, true) do
-    [Pleroma.Web.ChatChannel.ChatChannelState]
+  defp chat_child(true) do
+    [
+      Pleroma.Web.ChatChannel.ChatChannelState,
+      {Phoenix.PubSub, [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]}
+    ]
   end
 
-  defp chat_child(_, _), do: []
+  defp chat_child(_), do: []
 
   defp task_children(:test) do
     [

lib/pleroma/bbs/authenticator.ex

@@ -4,8 +4,8 @@
 
 defmodule Pleroma.BBS.Authenticator do
   use Sshd.PasswordAuthenticator
-  alias Pleroma.Plugs.AuthenticationPlug
   alias Pleroma.User
+  alias Pleroma.Web.Plugs.AuthenticationPlug
 
   def authenticate(username, password) do
     username = to_string(username)

lib/pleroma/config/deprecation_warnings.ex

@@ -39,7 +39,8 @@ def warn do
          :ok <- check_media_proxy_whitelist_config(),
          :ok <- check_welcome_message_config(),
          :ok <- check_gun_pool_options(),
-         :ok <- check_activity_expiration_config() do
+         :ok <- check_activity_expiration_config(),
+         :ok <- check_remote_ip_plug_name() do
       :ok
     else
       _ ->
@@ -176,4 +177,20 @@ def check_activity_expiration_config do
       warning_preface
     )
   end
+
+  @spec check_remote_ip_plug_name() :: :ok | nil
+  def check_remote_ip_plug_name do
+    warning_preface = """
+    !!!DEPRECATION WARNING!!!
+    Your config is using old namespace for RemoteIp Plug. Setting should work for now, but you are advised to change to new namespace to prevent possible issues later:
+    """
+
+    move_namespace_and_warn(
+      [
+        {Pleroma.Plugs.RemoteIp, Pleroma.Web.Plugs.RemoteIp,
+         "\n* `config :pleroma, Pleroma.Plugs.RemoteIp` is now `config :pleroma, Pleroma.Web.Plugs.RemoteIp`"}
+      ],
+      warning_preface
+    )
+  end
 end

lib/pleroma/config/oban.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Config.Oban do
   require Logger
 

lib/pleroma/conversation.ex

@@ -43,7 +43,7 @@ def get_for_ap_id(ap_id) do
   def maybe_create_recipientships(participation, activity) do
     participation = Repo.preload(participation, :recipients)
 
-    if participation.recipients |> Enum.empty?() do
+    if Enum.empty?(participation.recipients) do
       recipients = User.get_all_by_ap_id(activity.recipients)
       RecipientShip.create(recipients, participation)
     end
@@ -69,10 +69,6 @@ def create_or_bump_for(activity, opts \\ []) do
         Enum.map(users, fn user ->
           invisible_conversation = Enum.any?(users, &User.blocks?(user, &1))
 
-          unless invisible_conversation do
-            User.increment_unread_conversation_count(conversation, user)
-          end
-
           opts = Keyword.put(opts, :invisible_conversation, invisible_conversation)
 
           {:ok, participation} =

lib/pleroma/conversation/participation.ex

@@ -63,21 +63,10 @@ def mark_as_read(%User{} = user, %Conversation{} = conversation) do
     end
   end
 
-  def mark_as_read(participation) do
-    __MODULE__
-    |> where(id: ^participation.id)
-    |> update(set: [read: true])
-    |> select([p], p)
-    |> Repo.update_all([])
-    |> case do
-      {1, [participation]} ->
-        participation = Repo.preload(participation, :user)
-        User.set_unread_conversation_count(participation.user)
-        {:ok, participation}
-
-      error ->
-        error
-    end
+  def mark_as_read(%__MODULE__{} = participation) do
+    participation
+    |> change(read: true)
+    |> Repo.update()
   end
 
   def mark_all_as_read(%User{local: true} = user, %User{} = target_user) do
@@ -93,7 +82,6 @@ def mark_all_as_read(%User{local: true} = user, %User{} = target_user) do
     |> update([p], set: [read: true])
     |> Repo.update_all([])
 
-    {:ok, user} = User.set_unread_conversation_count(user)
     {:ok, user, []}
   end
 
@@ -108,7 +96,6 @@ def mark_all_as_read(%User{} = user) do
       |> select([p], p)
       |> Repo.update_all([])
 
-    {:ok, user} = User.set_unread_conversation_count(user)
     {:ok, user, participations}
   end
 
@@ -220,6 +207,12 @@ def set_recipients(participation, user_ids) do
     {:ok, Repo.preload(participation, :recipients, force: true)}
   end
 
+  @spec unread_count(User.t()) :: integer()
+  def unread_count(%User{id: user_id}) do
+    from(q in __MODULE__, where: q.user_id == ^user_id and q.read == false)
+    |> Repo.aggregate(:count, :id)
+  end
+
   def unread_conversation_count_for_user(user) do
     from(p in __MODULE__,
       where: p.user_id == ^user.id,

lib/pleroma/docs/generator.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Docs.Generator do
   @callback process(keyword()) :: {:ok, String.t()}
 

lib/pleroma/docs/json.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Docs.JSON do
   @behaviour Pleroma.Docs.Generator
   @external_resource "config/description.exs"

lib/pleroma/docs/markdown.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Docs.Markdown do
   @behaviour Pleroma.Docs.Generator
 

lib/pleroma/emails/admin_email.ex

@@ -88,7 +88,7 @@ def new_unapproved_registration(to, account) do
     html_body = """
     <p>New account for review: <a href="#{user_url(account)}">@#{account.nickname}</a></p>
     <blockquote>#{HTML.strip_tags(account.registration_reason)}</blockquote>
-    <a href="#{Pleroma.Web.base_url()}/pleroma/admin">Visit AdminFE</a>
+    <a href="#{Pleroma.Web.base_url()}/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a>
     """
 
     new()

lib/pleroma/emails/user_email.ex

@@ -189,4 +189,30 @@ def unsubscribe_url(user, notifications_type) do
 
     Router.Helpers.subscription_url(Endpoint, :unsubscribe, token)
   end
+
+  def backup_is_ready_email(backup, admin_user_id \\ nil) do
+    %{user: user} = Pleroma.Repo.preload(backup, :user)
+    download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup)
+
+    html_body =
+      if is_nil(admin_user_id) do
+        """
+        <p>You requested a full backup of your Pleroma account. It's ready for download:</p>
+        <p><a href="#{download_url}">#{download_url}</a></p>
+        """
+      else
+        admin = Pleroma.Repo.get(User, admin_user_id)
+
+        """
+        <p>Admin @#{admin.nickname} requested a full backup of your Pleroma account. It's ready for download:</p>
+        <p><a href="#{download_url}">#{download_url}</a></p>
+        """
+      end
+
+    new()
+    |> to(recipient(user))
+    |> from(sender())
+    |> subject("Your account archive is ready")
+    |> html_body(html_body)
+  end
 end

lib/pleroma/emoji/pack.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Emoji.Pack do
   @derive {Jason.Encoder, only: [:files, :pack, :files_count]}
   defstruct files: %{},
@@ -198,13 +202,13 @@ def import_from_filesystem do
     end
   end
 
-  @spec list_remote(String.t()) :: {:ok, map()} | {:error, atom()}
-  def list_remote(url) do
-    uri = url |> String.trim() |> URI.parse()
+  @spec list_remote(keyword()) :: {:ok, map()} | {:error, atom()}
+  def list_remote(opts) do
+    uri = opts[:url] |> String.trim() |> URI.parse()
 
     with :ok <- validate_shareable_packs_available(uri) do
       uri
-      |> URI.merge("/api/pleroma/emoji/packs")
+      |> URI.merge("/api/pleroma/emoji/packs?page=#{opts[:page]}&page_size=#{opts[:page_size]}")
       |> http_get()
     end
   end
@@ -244,7 +248,8 @@ def download(name, url, as) do
     uri = url |> String.trim() |> URI.parse()
 
     with :ok <- validate_shareable_packs_available(uri),
-         {:ok, remote_pack} <- uri |> URI.merge("/api/pleroma/emoji/packs/#{name}") |> http_get(),
+         {:ok, remote_pack} <-
+           uri |> URI.merge("/api/pleroma/emoji/pack?name=#{name}") |> http_get(),
          {:ok, %{sha: sha, url: url} = pack_info} <- fetch_pack_info(remote_pack, uri, name),
          {:ok, archive} <- download_archive(url, sha),
          pack <- copy_as(remote_pack, as || name),
@@ -523,7 +528,7 @@ defp get_filename(pack, shortcode) do
   defp http_get(%URI{} = url), do: url |> to_string() |> http_get()
 
   defp http_get(url) do
-    with {:ok, %{body: body}} <- url |> Pleroma.HTTP.get() do
+    with {:ok, %{body: body}} <- Pleroma.HTTP.get(url, [], pool: :default) do
       Jason.decode(body)
     end
   end
@@ -572,7 +577,7 @@ defp fetch_pack_info(remote_pack, uri, name) do
         {:ok,
          %{
            sha: sha,
-           url: URI.merge(uri, "/api/pleroma/emoji/packs/#{name}/archive") |> to_string()
+           url: URI.merge(uri, "/api/pleroma/emoji/packs/archive?name=#{name}") |> to_string()
          }}
 
       %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) ->

lib/pleroma/gun/connection_pool.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Gun.ConnectionPool do
   @registry __MODULE__
 

lib/pleroma/gun/connection_pool/reclaimer.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
   use GenServer, restart: :temporary
 

lib/pleroma/gun/connection_pool/worker.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Gun.ConnectionPool.Worker do
   alias Pleroma.Gun
   use GenServer, restart: :temporary

lib/pleroma/gun/connection_pool/worker_supervisor.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
   @moduledoc "Supervisor for pool workers. Does not do anything except enforce max connection limit"
 

lib/pleroma/helpers/inet_helper.ex

@@ -0,0 +1,19 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Helpers.InetHelper do
+  def parse_address(ip) when is_tuple(ip) do
+    {:ok, ip}
+  end
+
+  def parse_address(ip) when is_binary(ip) do
+    ip
+    |> String.to_charlist()
+    |> parse_address()
+  end
+
+  def parse_address(ip) do
+    :inet.parse_address(ip)
+  end
+end

lib/pleroma/http/adapter_helper/default.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.HTTP.AdapterHelper.Default do
   alias Pleroma.HTTP.AdapterHelper
 

lib/pleroma/http/adapter_helper/hackney.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.HTTP.AdapterHelper.Hackney do
   @behaviour Pleroma.HTTP.AdapterHelper
 

lib/pleroma/jwt.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.JWT do
   use Joken.Config
 

lib/pleroma/mime.ex

@@ -1,120 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.MIME do
-  @moduledoc """
-  Returns the mime-type of a binary and optionally a normalized file-name.
-  """
-  @default "application/octet-stream"
-  @read_bytes 35
-
-  @spec file_mime_type(String.t(), String.t()) ::
-          {:ok, content_type :: String.t(), filename :: String.t()} | {:error, any()} | :error
-  def file_mime_type(path, filename) do
-    with {:ok, content_type} <- file_mime_type(path),
-         filename <- fix_extension(filename, content_type) do
-      {:ok, content_type, filename}
-    end
-  end
-
-  @spec file_mime_type(String.t()) :: {:ok, String.t()} | {:error, any()} | :error
-  def file_mime_type(filename) do
-    File.open(filename, [:read], fn f ->
-      check_mime_type(IO.binread(f, @read_bytes))
-    end)
-  end
-
-  def bin_mime_type(binary, filename) do
-    with {:ok, content_type} <- bin_mime_type(binary),
-         filename <- fix_extension(filename, content_type) do
-      {:ok, content_type, filename}
-    end
-  end
-
-  @spec bin_mime_type(binary()) :: {:ok, String.t()} | :error
-  def bin_mime_type(<<head::binary-size(@read_bytes), _::binary>>) do
-    {:ok, check_mime_type(head)}
-  end
-
-  def bin_mime_type(_), do: :error
-
-  def mime_type(<<_::binary>>), do: {:ok, @default}
-
-  defp fix_extension(filename, content_type) do
-    parts = String.split(filename, ".")
-
-    new_filename =
-      if length(parts) > 1 do
-        Enum.drop(parts, -1) |> Enum.join(".")
-      else
-        Enum.join(parts)
-      end
-
-    cond do
-      content_type == "application/octet-stream" ->
-        filename
-
-      ext = List.first(MIME.extensions(content_type)) ->
-        new_filename <> "." <> ext
-
-      true ->
-        Enum.join([new_filename, String.split(content_type, "/") |> List.last()], ".")
-    end
-  end
-
-  defp check_mime_type(<<0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, _::binary>>) do
-    "image/png"
-  end
-
-  defp check_mime_type(<<0x47, 0x49, 0x46, 0x38, _, 0x61, _::binary>>) do
-    "image/gif"
-  end
-
-  defp check_mime_type(<<0xFF, 0xD8, 0xFF, _::binary>>) do
-    "image/jpeg"
-  end
-
-  defp check_mime_type(<<0x1A, 0x45, 0xDF, 0xA3, _::binary>>) do
-    "video/webm"
-  end
-
-  defp check_mime_type(<<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70, _::binary>>) do
-    "video/mp4"
-  end
-
-  defp check_mime_type(<<0x49, 0x44, 0x33, _::binary>>) do
-    "audio/mpeg"
-  end
-
-  defp check_mime_type(<<255, 251, _, 68, 0, 0, 0, 0, _::binary>>) do
-    "audio/mpeg"
-  end
-
-  defp check_mime_type(
-         <<0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00, _::size(160), 0x80, 0x74, 0x68, 0x65,
-           0x6F, 0x72, 0x61, _::binary>>
-       ) do
-    "video/ogg"
-  end
-
-  defp check_mime_type(<<0x4F, 0x67, 0x67, 0x53, 0x00, 0x02, 0x00, 0x00, _::binary>>) do
-    "audio/ogg"
-  end
-
-  defp check_mime_type(<<"RIFF", _::binary-size(4), "WAVE", _::binary>>) do
-    "audio/wav"
-  end
-
-  defp check_mime_type(<<"RIFF", _::binary-size(4), "WEBP", _::binary>>) do
-    "image/webp"
-  end
-
-  defp check_mime_type(<<"RIFF", _::binary-size(4), "AVI.", _::binary>>) do
-    "video/avi"
-  end
-
-  defp check_mime_type(_) do
-    @default
-  end
-end

lib/pleroma/moderation_log.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.ModerationLog do
   use Ecto.Schema
 
@@ -651,6 +655,16 @@ def get_log_entry_message(%ModerationLog{
     "@#{actor_nickname} deleted chat message ##{subject_id}"
   end
 
+  def get_log_entry_message(%ModerationLog{
+        data: %{
+          "actor" => %{"nickname" => actor_nickname},
+          "action" => "create_backup",
+          "subject" => %{"nickname" => user_nickname}
+        }
+      }) do
+    "@#{actor_nickname} requested account backup for @#{user_nickname}"
+  end
+
   defp nicknames_to_string(nicknames) do
     nicknames
     |> Enum.map(&"@#{&1}")

lib/pleroma/plugs/remote_ip.ex

@@ -1,54 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Plugs.RemoteIp do
-  @moduledoc """
-  This is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
-  """
-
-  import Plug.Conn
-
-  @behaviour Plug
-
-  @headers ~w[
-    x-forwarded-for
-  ]
-
-  # https://en.wikipedia.org/wiki/Localhost
-  # https://en.wikipedia.org/wiki/Private_network
-  @reserved ~w[
-    127.0.0.0/8
-    ::1/128
-    fc00::/7
-    10.0.0.0/8
-    172.16.0.0/12
-    192.168.0.0/16
-  ]
-
-  def init(_), do: nil
-
-  def call(%{remote_ip: original_remote_ip} = conn, _) do
-    config = Pleroma.Config.get(__MODULE__, [])
-
-    if Keyword.get(config, :enabled, false) do
-      %{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts(config))
-      assign(conn, :remote_ip_found, original_remote_ip != new_remote_ip)
-    else
-      conn
-    end
-  end
-
-  defp remote_ip_opts(config) do
-    headers = config |> Keyword.get(:headers, @headers) |> MapSet.new()
-    reserved = Keyword.get(config, :reserved, @reserved)
-
-    proxies =
-      config
-      |> Keyword.get(:proxies, [])
-      |> Enum.concat(reserved)
-      |> Enum.map(&InetCidr.parse/1)
-
-    {headers, proxies}
-  end
-end

lib/pleroma/telemetry/logger.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Telemetry.Logger do
   @moduledoc "Transforms Pleroma telemetry events to logs"
 

lib/pleroma/tests/auth_test_controller.ex

@@ -8,9 +8,9 @@ defmodule Pleroma.Tests.AuthTestController do
 
   use Pleroma.Web, :controller
 
-  alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug
-  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.User
+  alias Pleroma.Web.Plugs.EnsurePublicOrAuthenticatedPlug
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   # Serves only with proper OAuth token (:api and :authenticated_api)
   # Skipping EnsurePublicOrAuthenticatedPlug has no effect in this case

lib/pleroma/upload.ex

@@ -66,6 +66,7 @@ defp get_description(opts, upload) do
   end
 
   @spec store(source, options :: [option()]) :: {:ok, Map.t()} | {:error, any()}
+  @doc "Store a file. If using a `Plug.Upload{}` as the source, be sure to use `Majic.Plug` to ensure its content_type and filename is correct."
   def store(upload, opts \\ []) do
     opts = get_opts(opts)
 
@@ -139,14 +140,13 @@ defp get_opts(opts) do
   end
 
   defp prepare_upload(%Plug.Upload{} = file, opts) do
-    with :ok <- check_file_size(file.path, opts.size_limit),
-         {:ok, content_type, name} <- Pleroma.MIME.file_mime_type(file.path, file.filename) do
+    with :ok <- check_file_size(file.path, opts.size_limit) do
       {:ok,
        %__MODULE__{
          id: UUID.generate(),
-         name: name,
+         name: file.filename,
          tempfile: file.path,
-         content_type: content_type
+         content_type: file.content_type
        }}
     end
   end
@@ -154,16 +154,17 @@ defp prepare_upload(%Plug.Upload{} = file, opts) do
   defp prepare_upload(%{img: "data:image/" <> image_data}, opts) do
     parsed = Regex.named_captures(~r/(?<filetype>jpeg|png|gif);base64,(?<data>.*)/, image_data)
     data = Base.decode64!(parsed["data"], ignore: :whitespace)
-    hash = String.downcase(Base.encode16(:crypto.hash(:sha256, data)))
+    hash = Base.encode16(:crypto.hash(:sha256, data), lower: true)
 
     with :ok <- check_binary_size(data, opts.size_limit),
          tmp_path <- tempfile_for_image(data),
-         {:ok, content_type, name} <-
-           Pleroma.MIME.bin_mime_type(data, hash <> "." <> parsed["filetype"]) do
+         {:ok, %{mime_type: content_type}} <-
+           Majic.perform({:bytes, data}, pool: Pleroma.MajicPool),
+         [ext | _] <- MIME.extensions(content_type) do
       {:ok,
        %__MODULE__{
          id: UUID.generate(),
-         name: name,
+         name: hash <> "." <> ext,
          tempfile: tmp_path,
          content_type: content_type
        }}
@@ -172,7 +173,7 @@ defp prepare_upload(%{img: "data:image/" <> image_data}, opts) do
 
   # For Mix.Tasks.MigrateLocalUploads
   defp prepare_upload(%__MODULE__{tempfile: path} = upload, _opts) do
-    with {:ok, content_type} <- Pleroma.MIME.file_mime_type(path) do
+    with {:ok, %{mime_type: content_type}} <- Majic.perform(path, pool: Pleroma.MajicPool) do
       {:ok, %__MODULE__{upload | content_type: content_type}}
     end
   end

lib/pleroma/uploaders/uploader.ex

@@ -12,7 +12,7 @@ defmodule Pleroma.Uploaders.Uploader do
   @doc """
   Instructs how to get the file from the backend.
 
-  Used by `Pleroma.Plugs.UploadedMedia`.
+  Used by `Pleroma.Web.Plugs.UploadedMedia`.
   """
   @type get_method :: {:static_dir, directory :: String.t()} | {:url, url :: String.t()}
   @callback get_file(file :: String.t()) :: {:ok, get_method()}

lib/pleroma/user.ex

@@ -107,7 +107,7 @@ defmodule Pleroma.User do
     field(:note_count, :integer, default: 0)
     field(:follower_count, :integer, default: 0)
     field(:following_count, :integer, default: 0)
-    field(:locked, :boolean, default: false)
+    field(:is_locked, :boolean, default: false)
     field(:confirmation_pending, :boolean, default: false)
     field(:password_reset_pending, :boolean, default: false)
     field(:approval_pending, :boolean, default: false)
@@ -128,7 +128,6 @@ defmodule Pleroma.User do
     field(:hide_followers, :boolean, default: false)
     field(:hide_follows, :boolean, default: false)
     field(:hide_favorites, :boolean, default: true)
-    field(:unread_conversation_count, :integer, default: 0)
     field(:pinned_activities, {:array, :string}, default: [])
     field(:email_notifications, :map, default: %{"digest" => false})
     field(:mascot, :map, default: nil)
@@ -136,7 +135,7 @@ defmodule Pleroma.User do
     field(:pleroma_settings_store, :map, default: %{})
     field(:fields, {:array, :map}, default: [])
     field(:raw_fields, {:array, :map}, default: [])
-    field(:discoverable, :boolean, default: false)
+    field(:is_discoverable, :boolean, default: false)
     field(:invisible, :boolean, default: false)
     field(:allow_following_move, :boolean, default: true)
     field(:skip_thread_containment, :boolean, default: false)
@@ -426,7 +425,6 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do
       params,
       [
         :bio,
-        :name,
         :emoji,
         :ap_id,
         :inbox,
@@ -436,7 +434,7 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do
         :avatar,
         :ap_enabled,
         :banner,
-        :locked,
+        :is_locked,
         :last_refreshed_at,
         :uri,
         :follower_address,
@@ -448,14 +446,16 @@ def remote_user_changeset(struct \\ %User{local: false}, params) do
         :follower_count,
         :fields,
         :following_count,
-        :discoverable,
+        :is_discoverable,
         :invisible,
         :actor_type,
         :also_known_as,
         :accepts_chat_messages
       ]
     )
-    |> validate_required([:name, :ap_id])
+    |> cast(params, [:name], empty_values: [])
+    |> validate_required([:ap_id])
+    |> validate_required([:name], trim: false)
     |> unique_constraint(:nickname)
     |> validate_format(:nickname, @email_regex)
     |> validate_length(:bio, max: bio_limit)
@@ -479,7 +479,7 @@ def update_changeset(struct, params \\ %{}) do
         :public_key,
         :inbox,
         :shared_inbox,
-        :locked,
+        :is_locked,
         :no_rich_text,
         :default_scope,
         :banner,
@@ -495,7 +495,7 @@ def update_changeset(struct, params \\ %{}) do
         :fields,
         :raw_fields,
         :pleroma_settings_store,
-        :discoverable,
+        :is_discoverable,
         :actor_type,
         :also_known_as,
         :accepts_chat_messages
@@ -765,6 +765,16 @@ defp autofollow_users(user) do
     follow_all(user, autofollowed_users)
   end
 
+  defp autofollowing_users(user) do
+    candidates = Config.get([:instance, :autofollowing_nicknames])
+
+    User.Query.build(%{nickname: candidates, local: true, deactivated: false})
+    |> Repo.all()
+    |> Enum.each(&follow(&1, user, :follow_accept))
+
+    {:ok, :success}
+  end
+
   @doc "Inserts provided changeset, performs post-registration actions (confirmation email sending etc.)"
   def register(%Ecto.Changeset{} = changeset) do
     with {:ok, user} <- Repo.insert(changeset) do
@@ -774,6 +784,7 @@ def register(%Ecto.Changeset{} = changeset) do
 
   def post_register_action(%User{} = user) do
     with {:ok, user} <- autofollow_users(user),
+         {:ok, _} <- autofollowing_users(user),
          {:ok, user} <- set_cache(user),
          {:ok, _} <- send_welcome_email(user),
          {:ok, _} <- send_welcome_message(user),
@@ -847,7 +858,7 @@ def needs_update?(_), do: true
   @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t()} | {:error, String.t()}
 
   # "Locked" (self-locked) users demand explicit authorization of follow requests
-  def maybe_direct_follow(%User{} = follower, %User{local: true, locked: true} = followed) do
+  def maybe_direct_follow(%User{} = follower, %User{local: true, is_locked: true} = followed) do
     follow(follower, followed, :follow_pending)
   end
 
@@ -954,7 +965,7 @@ def get_follow_state(
   end
 
   def locked?(%User{} = user) do
-    user.locked || false
+    user.is_locked || false
   end
 
   def get_by_id(id) do
@@ -1293,47 +1304,6 @@ def update_following_count(%User{local: true} = user) do
     |> update_and_set_cache()
   end
 
-  def set_unread_conversation_count(%User{local: true} = user) do
-    unread_query = Participation.unread_conversation_count_for_user(user)
-
-    User
-    |> join(:inner, [u], p in subquery(unread_query))
-    |> update([u, p],
-      set: [unread_conversation_count: p.count]
-    )
-    |> where([u], u.id == ^user.id)
-    |> select([u], u)
-    |> Repo.update_all([])
-    |> case do
-      {1, [user]} -> set_cache(user)
-      _ -> {:error, user}
-    end
-  end
-
-  def set_unread_conversation_count(user), do: {:ok, user}
-
-  def increment_unread_conversation_count(conversation, %User{local: true} = user) do
-    unread_query =
-      Participation.unread_conversation_count_for_user(user)
-      |> where([p], p.conversation_id == ^conversation.id)
-
-    User
-    |> join(:inner, [u], p in subquery(unread_query))
-    |> update([u, p],
-      inc: [unread_conversation_count: 1]
-    )
-    |> where([u], u.id == ^user.id)
-    |> where([u, p], p.count == 0)
-    |> select([u], u)
-    |> Repo.update_all([])
-    |> case do
-      {1, [user]} -> set_cache(user)
-      _ -> {:error, user}
-    end
-  end
-
-  def increment_unread_conversation_count(_, user), do: {:ok, user}
-
   @spec get_users_from_set([String.t()], keyword()) :: [User.t()]
   def get_users_from_set(ap_ids, opts \\ []) do
     local_only = Keyword.get(opts, :local_only, true)
@@ -1601,7 +1571,7 @@ def purge_user_changeset(user) do
       note_count: 0,
       follower_count: 0,
       following_count: 0,
-      locked: false,
+      is_locked: false,
       confirmation_pending: false,
       password_reset_pending: false,
       approval_pending: false,
@@ -1618,7 +1588,7 @@ def purge_user_changeset(user) do
       pleroma_settings_store: %{},
       fields: [],
       raw_fields: [],
-      discoverable: false,
+      is_discoverable: false,
       also_known_as: []
     })
   end

lib/pleroma/user/backup.ex

@@ -0,0 +1,258 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.User.Backup do
+  use Ecto.Schema
+
+  import Ecto.Changeset
+  import Ecto.Query
+  import Pleroma.Web.Gettext
+
+  require Pleroma.Constants
+
+  alias Pleroma.Activity
+  alias Pleroma.Bookmark
+  alias Pleroma.Repo
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.ActivityPub.UserView
+  alias Pleroma.Workers.BackupWorker
+
+  schema "backups" do
+    field(:content_type, :string)
+    field(:file_name, :string)
+    field(:file_size, :integer, default: 0)
+    field(:processed, :boolean, default: false)
+
+    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
+
+    timestamps()
+  end
+
+  def create(user, admin_id \\ nil) do
+    with :ok <- validate_email_enabled(),
+         :ok <- validate_user_email(user),
+         :ok <- validate_limit(user, admin_id),
+         {:ok, backup} <- user |> new() |> Repo.insert() do
+      BackupWorker.process(backup, admin_id)
+    end
+  end
+
+  def new(user) do
+    rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false)
+    datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now())
+    name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip"
+
+    %__MODULE__{
+      user_id: user.id,
+      content_type: "application/zip",
+      file_name: name
+    }
+  end
+
+  def delete(backup) do
+    uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
+
+    with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do
+      Repo.delete(backup)
+    end
+  end
+
+  defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok
+
+  defp validate_limit(user, nil) do
+    case get_last(user.id) do
+      %__MODULE__{inserted_at: inserted_at} ->
+        days = Pleroma.Config.get([__MODULE__, :limit_days])
+        diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days)
+
+        if diff > days do
+          :ok
+        else
+          {:error,
+           dngettext(
+             "errors",
+             "Last export was less than a day ago",
+             "Last export was less than %{days} days ago",
+             days,
+             days: days
+           )}
+        end
+
+      nil ->
+        :ok
+    end
+  end
+
+  defp validate_email_enabled do
+    if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
+      :ok
+    else
+      {:error, dgettext("errors", "Backups require enabled email")}
+    end
+  end
+
+  defp validate_user_email(%User{email: nil}) do
+    {:error, dgettext("errors", "Email is required")}
+  end
+
+  defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok
+
+  def get_last(user_id) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> order_by(desc: :id)
+    |> limit(1)
+    |> Repo.one()
+  end
+
+  def list(%User{id: user_id}) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> order_by(desc: :id)
+    |> Repo.all()
+  end
+
+  def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do
+    __MODULE__
+    |> where(user_id: ^user_id)
+    |> where([b], b.id != ^latest_id)
+    |> Repo.all()
+    |> Enum.each(&BackupWorker.delete/1)
+  end
+
+  def get(id), do: Repo.get(__MODULE__, id)
+
+  def process(%__MODULE__{} = backup) do
+    with {:ok, zip_file} <- export(backup),
+         {:ok, %{size: size}} <- File.stat(zip_file),
+         {:ok, _upload} <- upload(backup, zip_file) do
+      backup
+      |> cast(%{file_size: size, processed: true}, [:file_size, :processed])
+      |> Repo.update()
+    end
+  end
+
+  @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
+  def export(%__MODULE__{} = backup) do
+    backup = Repo.preload(backup, :user)
+    name = String.trim_trailing(backup.file_name, ".zip")
+    dir = dir(name)
+
+    with :ok <- File.mkdir(dir),
+         :ok <- actor(dir, backup.user),
+         :ok <- statuses(dir, backup.user),
+         :ok <- likes(dir, backup.user),
+         :ok <- bookmarks(dir, backup.user),
+         {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
+         {:ok, _} <- File.rm_rf(dir) do
+      {:ok, to_string(zip_path)}
+    end
+  end
+
+  def dir(name) do
+    dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!()
+    Path.join(dir, name)
+  end
+
+  def upload(%__MODULE__{} = backup, zip_path) do
+    uploader = Pleroma.Config.get([Pleroma.Upload, :uploader])
+
+    upload = %Pleroma.Upload{
+      name: backup.file_name,
+      tempfile: zip_path,
+      content_type: backup.content_type,
+      path: Path.join("backups", backup.file_name)
+    }
+
+    with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload),
+         :ok <- File.rm(zip_path) do
+      {:ok, upload}
+    end
+  end
+
+  defp actor(dir, user) do
+    with {:ok, json} <-
+           UserView.render("user.json", %{user: user})
+           |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
+           |> Jason.encode() do
+      File.write(Path.join(dir, "actor.json"), json)
+    end
+  end
+
+  defp write_header(file, name) do
+    IO.write(
+      file,
+      """
+      {
+        "@context": "https://www.w3.org/ns/activitystreams",
+        "id": "#{name}.json",
+        "type": "OrderedCollection",
+        "orderedItems": [
+
+      """
+    )
+  end
+
+  defp write(query, dir, name, fun) do
+    path = Path.join(dir, "#{name}.json")
+
+    with {:ok, file} <- File.open(path, [:write, :utf8]),
+         :ok <- write_header(file, name) do
+      total =
+        query
+        |> Pleroma.Repo.chunk_stream(100)
+        |> Enum.reduce(0, fn i, acc ->
+          with {:ok, data} <- fun.(i),
+               {:ok, str} <- Jason.encode(data),
+               :ok <- IO.write(file, str <> ",\n") do
+            acc + 1
+          else
+            _ -> acc
+          end
+        end)
+
+      with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n  \"totalItems\": #{total}}") do
+        File.close(file)
+      end
+    end
+  end
+
+  defp bookmarks(dir, %{id: user_id} = _user) do
+    Bookmark
+    |> where(user_id: ^user_id)
+    |> join(:inner, [b], activity in assoc(b, :activity))
+    |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)})
+    |> write(dir, "bookmarks", fn a -> {:ok, a.object} end)
+  end
+
+  defp likes(dir, user) do
+    user.ap_id
+    |> Activity.Queries.by_actor()
+    |> Activity.Queries.by_type("Like")
+    |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)})
+    |> write(dir, "likes", fn a -> {:ok, a.object} end)
+  end
+
+  defp statuses(dir, user) do
+    opts =
+      %{}
+      |> Map.put(:type, ["Create", "Announce"])
+      |> Map.put(:actor_id, user.ap_id)
+
+    [
+      [Pleroma.Constants.as_public(), user.ap_id],
+      User.following(user),
+      Pleroma.List.memberships(user)
+    ]
+    |> Enum.concat()
+    |> ActivityPub.fetch_activities_query(opts)
+    |> write(dir, "outbox", fn a ->
+      with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
+        {:ok, Map.delete(activity, "@context")}
+      end
+    end)
+  end
+end

lib/pleroma/user/query.ex

@@ -43,6 +43,7 @@ defmodule Pleroma.User.Query do
             active: boolean(),
             deactivated: boolean(),
             need_approval: boolean(),
+            unconfirmed: boolean(),
             is_admin: boolean(),
             is_moderator: boolean(),
             super_users: boolean(),
@@ -55,7 +56,8 @@ defmodule Pleroma.User.Query do
             ap_id: [String.t()],
             order_by: term(),
             select: term(),
-            limit: pos_integer()
+            limit: pos_integer(),
+            actor_types: [String.t()]
           }
           | map()
 
@@ -114,6 +116,10 @@ defp compose_query({:is_admin, bool}, query) do
     where(query, [u], u.is_admin == ^bool)
   end
 
+  defp compose_query({:actor_types, actor_types}, query) when is_list(actor_types) do
+    where(query, [u], u.actor_type in ^actor_types)
+  end
+
   defp compose_query({:is_moderator, bool}, query) do
     where(query, [u], u.is_moderator == ^bool)
   end
@@ -156,6 +162,10 @@ defp compose_query({:need_approval, _}, query) do
     where(query, [u], u.approval_pending)
   end
 
+  defp compose_query({:unconfirmed, _}, query) do
+    where(query, [u], u.confirmation_pending)
+  end
+
   defp compose_query({:followers, %User{id: id}}, query) do
     query
     |> where([u], u.id != ^id)

lib/pleroma/user/search.ex

@@ -164,7 +164,7 @@ defp filter_invisible_users(query) do
   end
 
   defp filter_discoverable_users(query) do
-    from(q in query, where: q.discoverable == true)
+    from(q in query, where: q.is_discoverable == true)
   end
 
   defp filter_internal_users(query) do

lib/pleroma/web.ex

@@ -2,11 +2,6 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Web.Plug do
-  # Substitute for `call/2` which is defined with `use Pleroma.Web, :plug`
-  @callback perform(Plug.Conn.t(), Plug.opts()) :: Plug.Conn.t()
-end
-
 defmodule Pleroma.Web do
   @moduledoc """
   A module that keeps using definitions for controllers,
@@ -25,12 +20,12 @@ defmodule Pleroma.Web do
   below.
   """
 
-  alias Pleroma.Plugs.EnsureAuthenticatedPlug
-  alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug
-  alias Pleroma.Plugs.ExpectAuthenticatedCheckPlug
-  alias Pleroma.Plugs.ExpectPublicOrAuthenticatedCheckPlug
-  alias Pleroma.Plugs.OAuthScopesPlug
-  alias Pleroma.Plugs.PlugHelper
+  alias Pleroma.Web.Plugs.EnsureAuthenticatedPlug
+  alias Pleroma.Web.Plugs.EnsurePublicOrAuthenticatedPlug
+  alias Pleroma.Web.Plugs.ExpectAuthenticatedCheckPlug
+  alias Pleroma.Web.Plugs.ExpectPublicOrAuthenticatedCheckPlug
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.Plugs.PlugHelper
 
   def controller do
     quote do
@@ -177,7 +172,7 @@ def router do
   def channel do
     quote do
       # credo:disable-for-next-line Credo.Check.Consistency.MultiAliasImportRequireUse
-      use Phoenix.Channel
+      import Phoenix.Channel
       import Pleroma.Web.Gettext
     end
   end

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -790,7 +790,17 @@ defp restrict_replies(query, %{
       [activity, object] in query,
       where:
         fragment(
-          "?->>'inReplyTo' is null OR ? && array_remove(?, ?) OR ? = ?",
+          """
+          ?->>'type' != 'Create'     -- This isn't a Create      
+          OR ?->>'inReplyTo' is null -- this isn't a reply
+          OR ? && array_remove(?, ?) -- The recipient is us or one of our friends, 
+                                     -- unless they are the author (because authors 
+                                     -- are also part of the recipients). This leads
+                                     -- to a bug that self-replies by friends won't
+                                     -- show up.
+          OR ? = ?                   -- The actor is us
+          """,
+          activity.data,
           object.data,
           ^[user.ap_id | User.get_cached_user_friends_ap_ids(user)],
           activity.recipients,
@@ -817,7 +827,14 @@ defp restrict_muted(query, %{muting_user: %User{} = user} = opts) do
     query =
       from([activity] in query,
         where: fragment("not (? = ANY(?))", activity.actor, ^mutes),
-        where: fragment("not (?->'to' \\?| ?)", activity.data, ^mutes)
+        where:
+          fragment(
+            "not (?->'to' \\?| ?) or ? = ?",
+            activity.data,
+            ^mutes,
+            activity.actor,
+            ^user.ap_id
+          )
       )
 
     unless opts[:skip_preload] do
@@ -1218,11 +1235,11 @@ defp object_to_user_data(data) do
         {String.trim(name, ":"), url}
       end)
 
-    locked = data["manuallyApprovesFollowers"] || false
+    is_locked = data["manuallyApprovesFollowers"] || false
     capabilities = data["capabilities"] || %{}
     accepts_chat_messages = capabilities["acceptsChatMessages"]
     data = Transmogrifier.maybe_fix_user_object(data)
-    discoverable = data["discoverable"] || false
+    is_discoverable = data["discoverable"] || false
     invisible = data["invisible"] || false
     actor_type = data["type"] || "Person"
 
@@ -1247,8 +1264,8 @@ defp object_to_user_data(data) do
       banner: banner,
       fields: fields,
       emoji: emojis,
-      locked: locked,
-      discoverable: discoverable,
+      is_locked: is_locked,
+      is_discoverable: is_discoverable,
       invisible: invisible,
       avatar: avatar,
       name: data["name"],
@@ -1361,6 +1378,7 @@ def fetch_and_prepare_user_from_ap_id(ap_id, opts \\ []) do
          {:ok, data} <- user_data_from_user_object(data) do
       {:ok, maybe_update_follow_information(data)}
     else
+      # If this has been deleted, only log a debug and not an error
       {:error, "Object has been deleted" = e} ->
         Logger.debug("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
         {:error, e}

lib/pleroma/web/activity_pub/activity_pub_controller.ex

@@ -9,7 +9,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   alias Pleroma.Delivery
   alias Pleroma.Object
   alias Pleroma.Object.Fetcher
-  alias Pleroma.Plugs.EnsureAuthenticatedPlug
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Builder
@@ -23,8 +22,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.ControllerHelper
   alias Pleroma.Web.Endpoint
-  alias Pleroma.Web.FederatingPlug
   alias Pleroma.Web.Federator
+  alias Pleroma.Web.Plugs.EnsureAuthenticatedPlug
+  alias Pleroma.Web.Plugs.FederatingPlug
 
   require Logger
 
@@ -45,8 +45,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     when action in [:read_inbox, :update_outbox, :whoami, :upload_media]
   )
 
+  plug(Majic.Plug, [pool: Pleroma.MajicPool] when action in [:upload_media])
+
   plug(
-    Pleroma.Plugs.Cache,
+    Pleroma.Web.Plugs.Cache,
     [query_params: false, tracking_fun: &__MODULE__.track_object_fetch/2]
     when action in [:activity, :object]
   )
@@ -412,7 +414,7 @@ defp handle_user_activity(
       object =
         object
         |> Map.merge(Map.take(params, ["to", "cc"]))
-        |> Map.put("attributedTo", user.ap_id())
+        |> Map.put("attributedTo", user.ap_id)
         |> Transmogrifier.fix_object()
 
       ActivityPub.create(%{
@@ -456,7 +458,7 @@ def update_outbox(
         %{assigns: %{user: %User{nickname: nickname} = user}} = conn,
         %{"nickname" => nickname} = params
       ) do
-    actor = user.ap_id()
+    actor = user.ap_id
 
     params =
       params
@@ -523,19 +525,6 @@ defp ensure_user_keys_present_and_maybe_refresh_for_user(user, for_user) do
     {new_user, for_user}
   end
 
-  @doc """
-  Endpoint based on <https://www.w3.org/wiki/SocialCG/ActivityPub/MediaUpload>
-
-  Parameters:
-  - (required) `file`: data of the media
-  - (optionnal) `description`: description of the media, intended for accessibility
-
-  Response:
-  - HTTP Code: 201 Created
-  - HTTP Body: ActivityPub object to be inserted into another's `attachment` field
-
-  Note: Will not point to a URL with a `Location` header because no standalone Activity has been created.
-  """
   def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do
     with {:ok, object} <-
            ActivityPub.upload(

lib/pleroma/web/activity_pub/builder.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.Builder do
   @moduledoc """
   This module builds the objects. Meant to be used for creating local objects.

lib/pleroma/web/activity_pub/publisher.ex

@@ -242,9 +242,7 @@ def publish(%User{} = actor, %{data: %{"bcc" => bcc}} = activity)
     end)
   end
 
-  @doc """
-  Publishes an activity to all relevant peers.
-  """
+  # Publishes an activity to all relevant peers.
   def publish(%User{} = actor, %Activity{} = activity) do
     public = is_public?(activity)
 

lib/pleroma/web/activity_pub/side_effects.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Web.ActivityPub.SideEffects do
   @moduledoc """
   This module looks at an inserted object and executes the side effects that it
@@ -98,7 +102,7 @@ def handle(
          %User{} = followed <- User.get_cached_by_ap_id(followed_user),
          {_, {:ok, _}, _, _} <-
            {:following, User.follow(follower, followed, :follow_pending), follower, followed} do
-      if followed.local && !followed.locked do
+      if followed.local && !followed.is_locked do
         {:ok, accept_data, _} = Builder.accept(followed, object)
         {:ok, _activity, _} = Pipeline.common_pipeline(accept_data, local: true)
       end
@@ -302,11 +306,18 @@ def handle_object_creation(%{"type" => "ChatMessage"} = object, meta) do
 
       streamables =
         [[actor, recipient], [recipient, actor]]
+        |> Enum.uniq()
         |> Enum.map(fn [user, other_user] ->
           if user.local do
             {:ok, chat} = Chat.bump_or_create(user.id, other_user.ap_id)
             {:ok, cm_ref} = MessageReference.create(chat, object, user.ap_id != actor.ap_id)
 
+            Cachex.put(
+              :chat_message_id_idempotency_key_cache,
+              cm_ref.id,
+              meta[:idempotency_key]
+            )
+
             {
               ["user", "user:pleroma_chat"],
               {user, %{cm_ref | chat: chat, object: object}}

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -40,6 +40,7 @@ def fix_object(object, options \\ []) do
     |> fix_in_reply_to(options)
     |> fix_emoji
     |> fix_tag
+    |> set_sensitive
     |> fix_content_map
     |> fix_addressing
     |> fix_summary
@@ -313,19 +314,21 @@ def fix_tag(%{"tag" => tag} = object) when is_list(tag) do
     tags =
       tag
       |> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end)
-      |> Enum.map(fn data -> String.slice(data["name"], 1..-1) end)
+      |> Enum.map(fn %{"name" => name} ->
+        name
+        |> String.slice(1..-1)
+        |> String.downcase()
+      end)
 
     Map.put(object, "tag", tag ++ tags)
   end
 
-  def fix_tag(%{"tag" => %{"type" => "Hashtag", "name" => hashtag} = tag} = object) do
-    combined = [tag, String.slice(hashtag, 1..-1)]
-
-    Map.put(object, "tag", combined)
+  def fix_tag(%{"tag" => %{} = tag} = object) do
+    object
+    |> Map.put("tag", [tag])
+    |> fix_tag
   end
 
-  def fix_tag(%{"tag" => %{} = tag} = object), do: Map.put(object, "tag", [tag])
-
   def fix_tag(object), do: object
 
   # content map usually only has one language so this will do for now.
@@ -927,7 +930,7 @@ def set_conversation(object) do
     Map.put(object, "conversation", object["context"])
   end
 
-  def set_sensitive(%{"sensitive" => true} = object) do
+  def set_sensitive(%{"sensitive" => _} = object) do
     object
   end
 

lib/pleroma/web/activity_pub/views/user_view.ex

@@ -101,7 +101,7 @@ def render("user.json", %{user: user}) do
       "name" => user.name,
       "summary" => user.bio,
       "url" => user.ap_id,
-      "manuallyApprovesFollowers" => user.locked,
+      "manuallyApprovesFollowers" => user.is_locked,
       "publicKey" => %{
         "id" => "#{user.ap_id}#main-key",
         "owner" => user.ap_id,
@@ -110,7 +110,7 @@ def render("user.json", %{user: user}) do
       "endpoints" => endpoints,
       "attachment" => fields,
       "tag" => emoji_tags,
-      "discoverable" => user.discoverable,
+      "discoverable" => user.is_discoverable,
       "capabilities" => capabilities
     }
     |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user))

lib/pleroma/web/activity_pub/visibility.ex

@@ -44,29 +44,30 @@ def is_direct?(activity) do
   def is_list?(%{data: %{"listMessage" => _}}), do: true
   def is_list?(_), do: false
 
-  @spec visible_for_user?(Activity.t(), User.t() | nil) :: boolean()
-  def visible_for_user?(%{actor: ap_id}, %User{ap_id: ap_id}), do: true
+  @spec visible_for_user?(Activity.t() | nil, User.t() | nil) :: boolean()
+  def visible_for_user?(%Activity{actor: ap_id}, %User{ap_id: ap_id}), do: true
 
   def visible_for_user?(nil, _), do: false
 
-  def visible_for_user?(%{data: %{"listMessage" => _}}, nil), do: false
+  def visible_for_user?(%Activity{data: %{"listMessage" => _}}, nil), do: false
 
-  def visible_for_user?(%{data: %{"listMessage" => list_ap_id}} = activity, %User{} = user) do
+  def visible_for_user?(
+        %Activity{data: %{"listMessage" => list_ap_id}} = activity,
+        %User{} = user
+      ) do
     user.ap_id in activity.data["to"] ||
       list_ap_id
       |> Pleroma.List.get_by_ap_id()
       |> Pleroma.List.member?(user)
   end
 
-  def visible_for_user?(%{local: local} = activity, nil) do
-    cfg_key = if local, do: :local, else: :remote
-
-    if Pleroma.Config.restrict_unauthenticated_access?(:activities, cfg_key),
+  def visible_for_user?(%Activity{} = activity, nil) do
+    if restrict_unauthenticated_access?(activity),
       do: false,
       else: is_public?(activity)
   end
 
-  def visible_for_user?(activity, user) do
+  def visible_for_user?(%Activity{} = activity, user) do
     x = [user.ap_id | User.following(user)]
     y = [activity.actor] ++ activity.data["to"] ++ (activity.data["cc"] || [])
     is_public?(activity) || Enum.any?(x, &(&1 in y))
@@ -82,6 +83,26 @@ def entire_thread_visible_for_user?(%Activity{} = activity, %User{} = user) do
     result
   end
 
+  def restrict_unauthenticated_access?(%Activity{local: local}) do
+    restrict_unauthenticated_access_to_activity?(local)
+  end
+
+  def restrict_unauthenticated_access?(%Object{} = object) do
+    object
+    |> Object.local?()
+    |> restrict_unauthenticated_access_to_activity?()
+  end
+
+  def restrict_unauthenticated_access?(%User{} = user) do
+    User.visible_for(user, _reading_user = nil)
+  end
+
+  defp restrict_unauthenticated_access_to_activity?(local?) when is_boolean(local?) do
+    cfg_key = if local?, do: :local, else: :remote
+
+    Pleroma.Config.restrict_unauthenticated_access?(:activities, cfg_key)
+  end
+
   def get_visibility(object) do
     to = object.data["to"] || []
     cc = object.data["cc"] || []

lib/pleroma/web/admin_api/controllers/admin_api_controller.ex

@@ -5,22 +5,20 @@
 defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   use Pleroma.Web, :controller
 
-  import Pleroma.Web.ControllerHelper, only: [json_response: 3]
+  import Pleroma.Web.ControllerHelper,
+    only: [json_response: 3, fetch_integer_param: 3]
 
   alias Pleroma.Config
   alias Pleroma.MFA
   alias Pleroma.ModerationLog
-  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Stats
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.Builder
-  alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.AdminAPI
   alias Pleroma.Web.AdminAPI.AccountView
   alias Pleroma.Web.AdminAPI.ModerationLogView
-  alias Pleroma.Web.AdminAPI.Search
   alias Pleroma.Web.Endpoint
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
   alias Pleroma.Web.Router
 
   @users_page_size 50
@@ -28,7 +26,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   plug(
     OAuthScopesPlug,
     %{scopes: ["read:accounts"], admin: true}
-    when action in [:list_users, :user_show, :right_get, :show_user_credentials]
+    when action in [:right_get, :show_user_credentials, :create_backup]
   )
 
   plug(
@@ -37,12 +35,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     when action in [
            :get_password_reset,
            :force_password_reset,
-           :user_delete,
-           :users_create,
-           :user_toggle_activation,
-           :user_activate,
-           :user_deactivate,
-           :user_approve,
            :tag_users,
            :untag_users,
            :right_add,
@@ -54,12 +46,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
          ]
   )
 
-  plug(
-    OAuthScopesPlug,
-    %{scopes: ["write:follows"], admin: true}
-    when action in [:user_follow, :user_unfollow]
-  )
-
   plug(
     OAuthScopesPlug,
     %{scopes: ["read:statuses"], admin: true}
@@ -95,132 +81,6 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
 
   action_fallback(AdminAPI.FallbackController)
 
-  def user_delete(conn, %{"nickname" => nickname}) do
-    user_delete(conn, %{"nicknames" => [nickname]})
-  end
-
-  def user_delete(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
-    users =
-      nicknames
-      |> Enum.map(&User.get_cached_by_nickname/1)
-
-    users
-    |> Enum.each(fn user ->
-      {:ok, delete_data, _} = Builder.delete(admin, user.ap_id)
-      Pipeline.common_pipeline(delete_data, local: true)
-    end)
-
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: users,
-      action: "delete"
-    })
-
-    json(conn, nicknames)
-  end
-
-  def user_follow(%{assigns: %{user: admin}} = conn, %{
-        "follower" => follower_nick,
-        "followed" => followed_nick
-      }) do
-    with %User{} = follower <- User.get_cached_by_nickname(follower_nick),
-         %User{} = followed <- User.get_cached_by_nickname(followed_nick) do
-      User.follow(follower, followed)
-
-      ModerationLog.insert_log(%{
-        actor: admin,
-        followed: followed,
-        follower: follower,
-        action: "follow"
-      })
-    end
-
-    json(conn, "ok")
-  end
-
-  def user_unfollow(%{assigns: %{user: admin}} = conn, %{
-        "follower" => follower_nick,
-        "followed" => followed_nick
-      }) do
-    with %User{} = follower <- User.get_cached_by_nickname(follower_nick),
-         %User{} = followed <- User.get_cached_by_nickname(followed_nick) do
-      User.unfollow(follower, followed)
-
-      ModerationLog.insert_log(%{
-        actor: admin,
-        followed: followed,
-        follower: follower,
-        action: "unfollow"
-      })
-    end
-
-    json(conn, "ok")
-  end
-
-  def users_create(%{assigns: %{user: admin}} = conn, %{"users" => users}) do
-    changesets =
-      Enum.map(users, fn %{"nickname" => nickname, "email" => email, "password" => password} ->
-        user_data = %{
-          nickname: nickname,
-          name: nickname,
-          email: email,
-          password: password,
-          password_confirmation: password,
-          bio: "."
-        }
-
-        User.register_changeset(%User{}, user_data, need_confirmation: false)
-      end)
-      |> Enum.reduce(Ecto.Multi.new(), fn changeset, multi ->
-        Ecto.Multi.insert(multi, Ecto.UUID.generate(), changeset)
-      end)
-
-    case Pleroma.Repo.transaction(changesets) do
-      {:ok, users} ->
-        res =
-          users
-          |> Map.values()
-          |> Enum.map(fn user ->
-            {:ok, user} = User.post_register_action(user)
-
-            user
-          end)
-          |> Enum.map(&AccountView.render("created.json", %{user: &1}))
-
-        ModerationLog.insert_log(%{
-          actor: admin,
-          subjects: Map.values(users),
-          action: "create"
-        })
-
-        json(conn, res)
-
-      {:error, id, changeset, _} ->
-        res =
-          Enum.map(changesets.operations, fn
-            {current_id, {:changeset, _current_changeset, _}} when current_id == id ->
-              AccountView.render("create-error.json", %{changeset: changeset})
-
-            {_, {:changeset, current_changeset, _}} ->
-              AccountView.render("create-error.json", %{changeset: current_changeset})
-          end)
-
-        conn
-        |> put_status(:conflict)
-        |> json(res)
-    end
-  end
-
-  def user_show(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
-    with %User{} = user <- User.get_cached_by_nickname_or_id(nickname, for: admin) do
-      conn
-      |> put_view(AccountView)
-      |> render("show.json", %{user: user})
-    else
-      _ -> {:error, :not_found}
-    end
-  end
-
   def list_instance_statuses(conn, %{"instance" => instance} = params) do
     with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true
     {page, page_size} = page_params(params)
@@ -274,69 +134,6 @@ def list_user_chats(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}
     end
   end
 
-  def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
-    user = User.get_cached_by_nickname(nickname)
-
-    {:ok, updated_user} = User.deactivate(user, !user.deactivated)
-
-    action = if user.deactivated, do: "activate", else: "deactivate"
-
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: [user],
-      action: action
-    })
-
-    conn
-    |> put_view(AccountView)
-    |> render("show.json", %{user: updated_user})
-  end
-
-  def user_activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
-    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
-    {:ok, updated_users} = User.deactivate(users, false)
-
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: users,
-      action: "activate"
-    })
-
-    conn
-    |> put_view(AccountView)
-    |> render("index.json", %{users: Keyword.values(updated_users)})
-  end
-
-  def user_deactivate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
-    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
-    {:ok, updated_users} = User.deactivate(users, true)
-
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: users,
-      action: "deactivate"
-    })
-
-    conn
-    |> put_view(AccountView)
-    |> render("index.json", %{users: Keyword.values(updated_users)})
-  end
-
-  def user_approve(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
-    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
-    {:ok, updated_users} = User.approve(users)
-
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: users,
-      action: "approve"
-    })
-
-    conn
-    |> put_view(AccountView)
-    |> render("index.json", %{users: updated_users})
-  end
-
   def tag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "tags" => tags}) do
     with {:ok, _} <- User.tag(nicknames, tags) do
       ModerationLog.insert_log(%{
@@ -363,43 +160,6 @@ def untag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "
     end
   end
 
-  def list_users(conn, params) do
-    {page, page_size} = page_params(params)
-    filters = maybe_parse_filters(params["filters"])
-
-    search_params = %{
-      query: params["query"],
-      page: page,
-      page_size: page_size,
-      tags: params["tags"],
-      name: params["name"],
-      email: params["email"]
-    }
-
-    with {:ok, users, count} <- Search.user(Map.merge(search_params, filters)) do
-      json(
-        conn,
-        AccountView.render("index.json",
-          users: users,
-          count: count,
-          page_size: page_size
-        )
-      )
-    end
-  end
-
-  @filters ~w(local external active deactivated need_approval is_admin is_moderator)
-
-  @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{}
-  defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{}
-
-  defp maybe_parse_filters(filters) do
-    filters
-    |> String.split(",")
-    |> Enum.filter(&Enum.member?(@filters, &1))
-    |> Map.new(&{String.to_existing_atom(&1), true})
-  end
-
   def right_add_multiple(%{assigns: %{user: admin}} = conn, %{
         "permission_group" => permission_group,
         "nicknames" => nicknames
@@ -681,25 +441,19 @@ def stats(conn, params) do
     json(conn, %{"status_visibility" => counters})
   end
 
+  def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
+    with %User{} = user <- User.get_by_nickname(nickname),
+         {:ok, _} <- Pleroma.User.Backup.create(user, admin.id) do
+      ModerationLog.insert_log(%{actor: admin, subject: user, action: "create_backup"})
+
+      json(conn, "")
+    end
+  end
+
   defp page_params(params) do
-    {get_page(params["page"]), get_page_size(params["page_size"])}
-  end
-
-  defp get_page(page_string) when is_nil(page_string), do: 1
-
-  defp get_page(page_string) do
-    case Integer.parse(page_string) do
-      {page, _} -> page
-      :error -> 1
-    end
-  end
-
-  defp get_page_size(page_size_string) when is_nil(page_size_string), do: @users_page_size
-
-  defp get_page_size(page_size_string) do
-    case Integer.parse(page_size_string) do
-      {page_size, _} -> page_size
-      :error -> @users_page_size
-    end
+    {
+      fetch_integer_param(params, "page", 1),
+      fetch_integer_param(params, "page_size", @users_page_size)
+    }
   end
 end

lib/pleroma/web/admin_api/controllers/chat_controller.ex

@@ -10,10 +10,10 @@ defmodule Pleroma.Web.AdminAPI.ChatController do
   alias Pleroma.Chat.MessageReference
   alias Pleroma.ModerationLog
   alias Pleroma.Pagination
-  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Web.AdminAPI
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   require Logger
 

lib/pleroma/web/admin_api/controllers/config_controller.ex

@@ -7,7 +7,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do
 
   alias Pleroma.Config
   alias Pleroma.ConfigDB
-  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   plug(Pleroma.Web.ApiSpec.CastAndValidate)
   plug(OAuthScopesPlug, %{scopes: ["write"], admin: true} when action == :update)

lib/pleroma/web/admin_api/controllers/instance_document_controller.ex

@@ -5,9 +5,9 @@
 defmodule Pleroma.Web.AdminAPI.InstanceDocumentController do
   use Pleroma.Web, :controller
 
-  alias Pleroma.Plugs.InstanceStatic
-  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Web.InstanceDocument
+  alias Pleroma.Web.Plugs.InstanceStatic
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   plug(Pleroma.Web.ApiSpec.CastAndValidate)
 

lib/pleroma/web/admin_api/controllers/invite_controller.ex

@@ -8,8 +8,8 @@ defmodule Pleroma.Web.AdminAPI.InviteController do
   import Pleroma.Web.ControllerHelper, only: [json_response: 3]
 
   alias Pleroma.Config
-  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.UserInviteToken
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   require Logger
 

lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex

@@ -5,9 +5,9 @@
 defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do
   use Pleroma.Web, :controller
 
-  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Web.ApiSpec.Admin, as: Spec
   alias Pleroma.Web.MediaProxy
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   plug(Pleroma.Web.ApiSpec.CastAndValidate)
 

lib/pleroma/web/admin_api/controllers/o_auth_app_controller.ex

@@ -7,8 +7,8 @@ defmodule Pleroma.Web.AdminAPI.OAuthAppController do
 
   import Pleroma.Web.ControllerHelper, only: [json_response: 3]
 
-  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Web.OAuth.App
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   require Logger
 

lib/pleroma/web/admin_api/controllers/relay_controller.ex

@@ -6,8 +6,8 @@ defmodule Pleroma.Web.AdminAPI.RelayController do
   use Pleroma.Web, :controller
 
   alias Pleroma.ModerationLog
-  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Web.ActivityPub.Relay
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   require Logger
 

lib/pleroma/web/admin_api/controllers/report_controller.ex

@@ -9,12 +9,12 @@ defmodule Pleroma.Web.AdminAPI.ReportController do
 
   alias Pleroma.Activity
   alias Pleroma.ModerationLog
-  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.ReportNote
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.AdminAPI
   alias Pleroma.Web.AdminAPI.Report
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   require Logger
 

lib/pleroma/web/admin_api/controllers/status_controller.ex

@@ -7,10 +7,10 @@ defmodule Pleroma.Web.AdminAPI.StatusController do
 
   alias Pleroma.Activity
   alias Pleroma.ModerationLog
-  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
 
   require Logger
 

lib/pleroma/web/admin_api/controllers/user_controller.ex

@@ -0,0 +1,281 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.UserController do
+  use Pleroma.Web, :controller
+
+  import Pleroma.Web.ControllerHelper,
+    only: [fetch_integer_param: 3]
+
+  alias Pleroma.ModerationLog
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.Builder
+  alias Pleroma.Web.ActivityPub.Pipeline
+  alias Pleroma.Web.AdminAPI
+  alias Pleroma.Web.AdminAPI.AccountView
+  alias Pleroma.Web.AdminAPI.Search
+  alias Pleroma.Web.Plugs.OAuthScopesPlug
+
+  @users_page_size 50
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read:accounts"], admin: true}
+    when action in [:list, :show]
+  )
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["write:accounts"], admin: true}
+    when action in [
+           :delete,
+           :create,
+           :toggle_activation,
+           :activate,
+           :deactivate,
+           :approve
+         ]
+  )
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["write:follows"], admin: true}
+    when action in [:follow, :unfollow]
+  )
+
+  action_fallback(AdminAPI.FallbackController)
+
+  def delete(conn, %{"nickname" => nickname}) do
+    delete(conn, %{"nicknames" => [nickname]})
+  end
+
+  def delete(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+
+    Enum.each(users, fn user ->
+      {:ok, delete_data, _} = Builder.delete(admin, user.ap_id)
+      Pipeline.common_pipeline(delete_data, local: true)
+    end)
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "delete"
+    })
+
+    json(conn, nicknames)
+  end
+
+  def follow(%{assigns: %{user: admin}} = conn, %{
+        "follower" => follower_nick,
+        "followed" => followed_nick
+      }) do
+    with %User{} = follower <- User.get_cached_by_nickname(follower_nick),
+         %User{} = followed <- User.get_cached_by_nickname(followed_nick) do
+      User.follow(follower, followed)
+
+      ModerationLog.insert_log(%{
+        actor: admin,
+        followed: followed,
+        follower: follower,
+        action: "follow"
+      })
+    end
+
+    json(conn, "ok")
+  end
+
+  def unfollow(%{assigns: %{user: admin}} = conn, %{
+        "follower" => follower_nick,
+        "followed" => followed_nick
+      }) do
+    with %User{} = follower <- User.get_cached_by_nickname(follower_nick),
+         %User{} = followed <- User.get_cached_by_nickname(followed_nick) do
+      User.unfollow(follower, followed)
+
+      ModerationLog.insert_log(%{
+        actor: admin,
+        followed: followed,
+        follower: follower,
+        action: "unfollow"
+      })
+    end
+
+    json(conn, "ok")
+  end
+
+  def create(%{assigns: %{user: admin}} = conn, %{"users" => users}) do
+    changesets =
+      Enum.map(users, fn %{"nickname" => nickname, "email" => email, "password" => password} ->
+        user_data = %{
+          nickname: nickname,
+          name: nickname,
+          email: email,
+          password: password,
+          password_confirmation: password,
+          bio: "."
+        }
+
+        User.register_changeset(%User{}, user_data, need_confirmation: false)
+      end)
+      |> Enum.reduce(Ecto.Multi.new(), fn changeset, multi ->
+        Ecto.Multi.insert(multi, Ecto.UUID.generate(), changeset)
+      end)
+
+    case Pleroma.Repo.transaction(changesets) do
+      {:ok, users} ->
+        res =
+          users
+          |> Map.values()
+          |> Enum.map(fn user ->
+            {:ok, user} = User.post_register_action(user)
+
+            user
+          end)
+          |> Enum.map(&AccountView.render("created.json", %{user: &1}))
+
+        ModerationLog.insert_log(%{
+          actor: admin,
+          subjects: Map.values(users),
+          action: "create"
+        })
+
+        json(conn, res)
+
+      {:error, id, changeset, _} ->
+        res =
+          Enum.map(changesets.operations, fn
+            {current_id, {:changeset, _current_changeset, _}} when current_id == id ->
+              AccountView.render("create-error.json", %{changeset: changeset})
+
+            {_, {:changeset, current_changeset, _}} ->
+              AccountView.render("create-error.json", %{changeset: current_changeset})
+          end)
+
+        conn
+        |> put_status(:conflict)
+        |> json(res)
+    end
+  end
+
+  def show(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
+    with %User{} = user <- User.get_cached_by_nickname_or_id(nickname, for: admin) do
+      conn
+      |> put_view(AccountView)
+      |> render("show.json", %{user: user})
+    else
+      _ -> {:error, :not_found}
+    end
+  end
+
+  def toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
+    user = User.get_cached_by_nickname(nickname)
+
+    {:ok, updated_user} = User.deactivate(user, !user.deactivated)
+
+    action = if user.deactivated, do: "activate", else: "deactivate"
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: [user],
+      action: action
+    })
+
+    conn
+    |> put_view(AccountView)
+    |> render("show.json", %{user: updated_user})
+  end
+
+  def activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+    {:ok, updated_users} = User.deactivate(users, false)
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "activate"
+    })
+
+    conn
+    |> put_view(AccountView)
+    |> render("index.json", %{users: Keyword.values(updated_users)})
+  end
+
+  def deactivate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+    {:ok, updated_users} = User.deactivate(users, true)
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "deactivate"
+    })
+
+    conn
+    |> put_view(AccountView)
+    |> render("index.json", %{users: Keyword.values(updated_users)})
+  end
+
+  def approve(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+    {:ok, updated_users} = User.approve(users)
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "approve"
+    })
+
+    conn
+    |> put_view(AccountView)
+    |> render("index.json", %{users: updated_users})
+  end
+
+  def list(conn, params) do
+    {page, page_size} = page_params(params)
+    filters = maybe_parse_filters(params["filters"])
+
+    search_params =
+      %{
+        query: params["query"],
+        page: page,
+        page_size: page_size,
+        tags: params["tags"],
+        name: params["name"],
+        email: params["email"],
+        actor_types: params["actor_types"]
+      }
+      |> Map.merge(filters)
+
+    with {:ok, users, count} <- Search.user(search_params) do
+      json(
+        conn,
+        AccountView.render("index.json",
+          users: users,
+          count: count,
+          page_size: page_size
+        )
+      )
+    end
+  end
+
+  @filters ~w(local external active deactivated need_approval unconfirmed is_admin is_moderator)
+
+  @spec maybe_parse_filters(String.t()) :: %{required(String.t()) => true} | %{}
+  defp maybe_parse_filters(filters) when is_nil(filters) or filters == "", do: %{}
+
+  defp maybe_parse_filters(filters) do
+    filters
+    |> String.split(",")
+    |> Enum.filter(&Enum.member?(@filters, &1))
+    |> Map.new(&{String.to_existing_atom(&1), true})
+  end
+
+  defp page_params(params) do
+    {
+      fetch_integer_param(params, "page", 1),
+      fetch_integer_param(params, "page_size", @users_page_size)
+    }
+  end
+end

lib/pleroma/web/admin_api/views/account_view.ex

@@ -39,7 +39,7 @@ def render("credentials.json", %{user: user, for: for_user}) do
       :fields,
       :name,
       :nickname,
-      :locked,
+      :is_locked,
       :no_rich_text,
       :default_scope,
       :hide_follows,
@@ -52,7 +52,7 @@ def render("credentials.json", %{user: user, for: for_user}) do
       :skip_thread_containment,
       :pleroma_settings_store,
       :raw_fields,
-      :discoverable,
+      :is_discoverable,
       :actor_type
     ])
     |> Map.merge(%{

lib/pleroma/web/admin_api/views/report_view.ex

@@ -52,7 +52,7 @@ def render("show.json", %{report: report, user: user, account: account, statuses
   end
 
   def render("index_notes.json", %{notes: notes}) when is_list(notes) do
-    Enum.map(notes, &render(__MODULE__, "show_note.json", &1))
+    Enum.map(notes, &render(__MODULE__, "show_note.json", Map.from_struct(&1)))
   end
 
   def render("index_notes.json", _), do: []

lib/pleroma/web/api_spec/cast_and_validate.ex

@@ -115,6 +115,10 @@ defp cast_and_validate(spec, operation, conn, content_type, false = _strict) do
             %{reason: :unexpected_field, name: name, path: [name]}, params ->
               Map.delete(params, name)
 
+            # Filter out empty params
+            %{reason: :invalid_type, path: [name_atom], value: ""}, params ->
+              Map.delete(params, to_string(name_atom))
+
             %{reason: :invalid_enum, name: nil, path: path, value: value}, params ->
               path = path |> Enum.reverse() |> tl() |> Enum.reverse() |> list_items_to_string()
               update_in(params, path, &List.delete(&1, value))