Merge branch 'release/2.0.0' into 'stable'

Release/2.0.0

See merge request pleroma/pleroma!2273

.dockerignore

@@ -5,7 +5,6 @@ CC-BY-SA-4.0
 COPYING
 *file
 elixir_buildpack.config
-docs/
 test/
 
 # Required to get version

.formatter.exs

@@ -1,3 +1,3 @@
 [
-  inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}"]
+  inputs: ["mix.exs", "{config,lib,test}/**/*.{ex,exs}", "priv/repo/migrations/*.exs", "priv/scrubbers/*.ex"]
 ]

.gitattributes

@@ -0,0 +1,2 @@
+*.ex diff=elixir
+*.exs diff=elixir

.gitignore

@@ -47,3 +47,5 @@ docs/generated_config.md
 .idea
 pleroma.iml
 
+# asdf
+.tool-versions

.gitlab-ci.yml

@@ -1,22 +1,25 @@
 image: elixir:1.8.1
 
-variables:
+variables: &global_variables
   POSTGRES_DB: pleroma_test
   POSTGRES_USER: postgres
   POSTGRES_PASSWORD: postgres
   DB_HOST: postgres
   MIX_ENV: test
 
-cache:
+cache: &global_cache_policy
   key: ${CI_COMMIT_REF_SLUG}
   paths:
-          - deps
+    - deps
-          - _build
+    - _build
+
 stages:
   - build
   - test
+  - benchmark
   - deploy
   - release
+  - docker
 
 before_script:
   - mix local.hex --force
@@ -28,24 +31,27 @@ build:
   - mix deps.get
   - mix compile --force
 
-docs-build:
+benchmark:
-  stage: build
+  stage: benchmark
-  only: &docs-only
+  when: manual
-  - stable@pleroma/pleroma
-  - develop@pleroma/pleroma
   variables:
-    MIX_ENV: dev
+    MIX_ENV: benchmark
+  services:
+  - name: postgres:9.6
+    alias: postgres
+    command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   script:
     - mix deps.get
-    - mix compile
+    - mix ecto.create
-    - mix docs
+    - mix ecto.migrate
-  artifacts:
+    - mix pleroma.load_testing
-    paths:
-      - priv/static/doc
-
 
 unit-testing:
   stage: test
+  cache: &testing_cache_policy
+    <<: *global_cache_policy
+    policy: pull
+
   services:
   - name: postgres:9.6
     alias: postgres
@@ -56,13 +62,29 @@ unit-testing:
     - mix ecto.migrate
     - mix coveralls --preload-modules
 
+federated-testing:
+  stage: test
+  cache: *testing_cache_policy
+  services:
+  - name: minibikini/postgres-with-rum:12
+    alias: postgres
+    command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+  script:
+    - mix deps.get
+    - mix ecto.create
+    - mix ecto.migrate
+    - epmd -daemon
+    - mix test --trace --only federated
+
 unit-testing-rum:
   stage: test
+  cache: *testing_cache_policy
   services:
   - name: minibikini/postgres-with-rum:12
     alias: postgres
     command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   variables:
+    <<: *global_variables
     RUM_ENABLED: "true"
   script:
     - mix deps.get
@@ -73,28 +95,28 @@ unit-testing-rum:
 
 lint:
   stage: test
+  cache: *testing_cache_policy
   script:
     - mix format --check-formatted
 
 analysis:
   stage: test
+  cache: *testing_cache_policy
   script:
     - mix deps.get
     - mix credo --strict --only=warnings,todo,fixme,consistency,readability
 
 docs-deploy:
   stage: deploy
-  image: alpine:3.9
+  cache: *testing_cache_policy
-  only: *docs-only
+  image: alpine:latest
+  only:
+  - stable@pleroma/pleroma
+  - develop@pleroma/pleroma
   before_script:
-    - apk update && apk add openssh-client rsync
+  - apk add curl
   script:
-    - mkdir -p ~/.ssh
+  - curl -X POST -F"token=$DOCS_PIPELINE_TRIGGER" -F'ref=master' -F"variables[BRANCH]=$CI_COMMIT_REF_NAME" https://git.pleroma.social/api/v4/projects/673/trigger/pipeline
-    - echo "${SSH_HOST_KEY}" > ~/.ssh/known_hosts
-    - eval $(ssh-agent -s)
-    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
-    - rsync -hrvz --delete -e "ssh -p ${SSH_PORT}" priv/static/doc/ "${SSH_USER_HOST_LOCATION}/${CI_COMMIT_REF_NAME}"
-
 review_app:
   image: alpine:3.9
   stage: deploy
@@ -117,6 +139,7 @@ review_app:
     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
     - ssh-keyscan -H "pleroma.online" >> ~/.ssh/known_hosts
     - (ssh -t dokku@pleroma.online -- apps:create "$CI_ENVIRONMENT_SLUG") || true
+    - (ssh -t dokku@pleroma.online -- git:set "$CI_ENVIRONMENT_SLUG" keep-git-dir true) || true
     - ssh -t dokku@pleroma.online -- config:set "$CI_ENVIRONMENT_SLUG" APP_NAME="$CI_ENVIRONMENT_SLUG" APP_HOST="$CI_ENVIRONMENT_SLUG.pleroma.online" MIX_ENV=dokku
     - (ssh -t dokku@pleroma.online -- postgres:create $(echo $CI_ENVIRONMENT_SLUG | sed -e 's/-/_/g')_db) || true
     - (ssh -t dokku@pleroma.online -- postgres:link $(echo $CI_ENVIRONMENT_SLUG | sed -e 's/-/_/g')_db "$CI_ENVIRONMENT_SLUG") || true
@@ -142,7 +165,7 @@ stop_review_app:
     - ssh -t dokku@pleroma.online -- --force postgres:destroy $(echo $CI_ENVIRONMENT_SLUG | sed -e 's/-/_/g')_db
 
 amd64:
-  stage: release 
+  stage: release
   # TODO: Replace with upstream image when 1.9.0 comes out
   image: rinpatch/elixir:1.9.0-rc.0
   only: &release-only
@@ -243,3 +266,66 @@ arm64-musl:
   variables: *release-variables
   before_script: *before-release-musl
   script: *release
+
+docker:
+  stage: docker
+  image: docker:latest
+  cache: {}
+  dependencies: []
+  variables: &docker-variables
+    DOCKER_DRIVER: overlay2
+    DOCKER_HOST: unix:///var/run/docker.sock
+    IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
+    IMAGE_TAG_SLUG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG
+    IMAGE_TAG_LATEST: $CI_REGISTRY_IMAGE:latest
+    IMAGE_TAG_LATEST_STABLE: $CI_REGISTRY_IMAGE:latest-stable
+  before_script: &before-docker
+    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - docker pull $IMAGE_TAG_SLUG || true
+    - export CI_JOB_TIMESTAMP=$(date --utc -Iseconds)
+    - export CI_VCS_REF=$CI_COMMIT_SHORT_SHA
+  allow_failure: true
+  script:
+    - docker build --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t $IMAGE_TAG_SLUG -t $IMAGE_TAG_LATEST .
+    - docker push $IMAGE_TAG
+    - docker push $IMAGE_TAG_SLUG
+    - docker push $IMAGE_TAG_LATEST
+  tags:
+    - dind
+  only:
+    - develop@pleroma/pleroma
+
+docker-stable:
+  stage: docker
+  image: docker:latest
+  cache: {}
+  dependencies: []
+  variables: *docker-variables
+  before_script: *before-docker
+  allow_failure: true
+  script:
+    - docker build --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t $IMAGE_TAG_SLUG -t $IMAGE_TAG_LATEST_STABLE .
+    - docker push $IMAGE_TAG
+    - docker push $IMAGE_TAG_SLUG
+    - docker push $IMAGE_TAG_LATEST_STABLE
+  tags:
+    - dind
+  only:
+    - stable@pleroma/pleroma
+
+docker-release:
+  stage: docker
+  image: docker:latest
+  cache: {}
+  dependencies: []
+  variables: *docker-variables
+  before_script: *before-docker
+  allow_failure: true
+  script:
+    - docker build --cache-from $IMAGE_TAG_SLUG --build-arg VCS_REF=$CI_VCS_REF --build-arg BUILD_DATE=$CI_JOB_TIMESTAMP -t $IMAGE_TAG -t       $IMAGE_TAG_SLUG .
+    - docker push $IMAGE_TAG
+    - docker push $IMAGE_TAG_SLUG
+  tags:
+    - dind
+  only:
+    - /^release/.*$/@pleroma/pleroma

CHANGELOG.md

@@ -3,6 +3,153 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [2.0.0] - 2019-03-08
+### Security
+- Mastodon API: Fix being able to request enourmous amount of statuses in timelines leading to DoS. Now limited to 40 per request.
+
+### Removed
+- **Breaking**: Removed 1.0+ deprecated configurations `Pleroma.Upload, :strip_exif` and `:instance, :dedupe_media`
+- **Breaking**: OStatus protocol support
+- **Breaking**: MDII uploader
+- **Breaking**: Using third party engines for user recommendation
+<details>
+  <summary>API Changes</summary>
+- **Breaking**: AdminAPI: migrate_from_db endpoint
+</details>
+
+### Changed
+- **Breaking:** Pleroma won't start if it detects unapplied migrations
+- **Breaking:** Elixir >=1.8 is now required (was >= 1.7)
+- **Breaking:** `Pleroma.Plugs.RemoteIp` and `:rate_limiter` enabled by default. Please ensure your reverse proxy forwards the real IP!
+- **Breaking:** attachment links (`config :pleroma, :instance, no_attachment_links` and `config :pleroma, Pleroma.Upload, link_name`) disabled by default
+- **Breaking:** OAuth: defaulted `[:auth, :enforce_oauth_admin_scope_usage]` setting to `true` which demands `admin` OAuth scope to perform admin actions (in addition to `is_admin` flag on User); make sure to use bundled or newer versions of AdminFE & PleromaFE to access admin / moderator features.
+- **Breaking:** Dynamic configuration has been rearchitected. The `:pleroma, :instance, dynamic_configuration` setting has been replaced with `config :pleroma, configurable_from_database`. Please backup your configuration to a file and run the migration task to ensure consistency with the new schema.
+- **Breaking:** `:instance, no_attachment_links` has been replaced with `:instance, attachment_links` which still takes a boolean value but doesn't use double negative language.
+- Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings)
+- Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler
+- Enabled `:instance, extended_nickname_format` in the default config
+- Add `rel="ugc"` to all links in statuses, to prevent SEO spam
+- Extract RSS functionality from OStatus
+- MRF (Simple Policy): Also use `:accept`/`:reject` on the actors rather than only their activities
+- OStatus: Extract RSS functionality
+- Deprecated `User.Info` embedded schema (fields moved to `User`)
+- Store status data inside Flag activity
+- Deprecated (reorganized as `UserRelationship` entity) User fields with user AP IDs (`blocks`, `mutes`, `muted_reblogs`, `muted_notifications`, `subscribers`).
+- Rate limiter is now disabled for localhost/socket (unless remoteip plug is enabled)
+- Logger: default log level changed from `warn` to `info`.
+- Config mix task `migrate_to_db` truncates `config` table before migrating the config file.
+- Default to `prepare: :unnamed` in the database configuration.
+- Instance stats are now loaded on startup instead of being empty until next hourly job.
+<details>
+  <summary>API Changes</summary>
+
+- **Breaking** EmojiReactions: Change endpoints and responses to align with Mastodon
+- **Breaking** Admin API: `PATCH /api/pleroma/admin/users/:nickname/force_password_reset` is now `PATCH /api/pleroma/admin/users/force_password_reset` (accepts `nicknames` array in the request body)
+- **Breaking:** Admin API: Return link alongside with token on password reset
+- **Breaking:** Admin API: `PUT /api/pleroma/admin/reports/:id` is now `PATCH /api/pleroma/admin/reports`, see admin_api.md for details
+- **Breaking:** `/api/pleroma/admin/users/invite_token` now uses `POST`, changed accepted params and returns full invite in json instead of only token string.
+- **Breaking** replying to reports is now "report notes", enpoint changed from `POST /api/pleroma/admin/reports/:id/respond` to `POST /api/pleroma/admin/reports/:id/notes`
+- Mastodon API: stopped sanitizing display names, field names and subject fields since they are supposed to be treated as plaintext
+- Admin API: Return `total` when querying for reports
+- Mastodon API: Return `pleroma.direct_conversation_id` when creating a direct message (`POST /api/v1/statuses`)
+- Admin API: Return link alongside with token on password reset
+- Admin API: Support authentication via `x-admin-token` HTTP header
+- Mastodon API: Add `pleroma.direct_conversation_id` to the status endpoint (`GET /api/v1/statuses/:id`)
+- Mastodon API: `pleroma.thread_muted` to the Status entity
+- Mastodon API: Mark the direct conversation as read for the author when they send a new direct message
+- Mastodon API, streaming: Add `pleroma.direct_conversation_id` to the `conversation` stream event payload.
+- Admin API: Render whole status in grouped reports
+- Mastodon API: User timelines will now respect blocks, unless you are getting the user timeline of somebody you blocked (which would be empty otherwise).
+- Mastodon API: Favoriting / Repeating a post multiple times will now return the identical response every time. Before, executing that action twice would return an error ("already favorited") on the second try.
+- Mastodon API: Limit timeline requests to 3 per timeline per 500ms per user/ip by default.
+</details>
+
+### Added
+- `:chat_limit` option to limit chat characters.
+- `cleanup_attachments` option to remove attachments along with statuses. Does not affect duplicate files and attachments without status. Enabling this will increase load to database when deleting statuses on larger instances.
+- Refreshing poll results for remote polls
+- Authentication: Added rate limit for password-authorized actions / login existence checks
+- Static Frontend: Add the ability to render user profiles and notices server-side without requiring JS app.
+- Mix task to re-count statuses for all users (`mix pleroma.count_statuses`)
+- Mix task to list all users (`mix pleroma.user list`)
+- Mix task to send a test email (`mix pleroma.email test`)
+- Support for `X-Forwarded-For` and similar HTTP headers which used by reverse proxies to pass a real user IP address to the backend. Must not be enabled unless your instance is behind at least one reverse proxy (such as Nginx, Apache HTTPD or Varnish Cache).
+- MRF: New module which handles incoming posts based on their age. By default, all incoming posts that are older than 2 days will be unlisted and not shown to their followers.
+- User notification settings: Add `privacy_option` option.
+- Support for custom Elixir modules (such as MRF policies)
+- User settings: Add _This account is a_ option.
+- A new users admin digest email
+- OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`).
+- Add an option `authorized_fetch_mode` to require HTTP signatures for AP fetches.
+- ActivityPub: support for `replies` collection (output for outgoing federation & fetching on incoming federation).
+- Mix task to refresh counter cache (`mix pleroma.refresh_counter_cache`)
+<details>
+  <summary>API Changes</summary>
+
+- Job queue stats to the healthcheck page
+- Admin API: Add ability to fetch reports, grouped by status `GET /api/pleroma/admin/grouped_reports`
+- Admin API: Add ability to require password reset
+- Mastodon API: Account entities now include `follow_requests_count` (planned Mastodon 3.x addition)
+- Pleroma API: `GET /api/v1/pleroma/accounts/:id/scrobbles` to get a list of recently scrobbled items
+- Pleroma API: `POST /api/v1/pleroma/scrobble` to scrobble a media item
+- Mastodon API: Add `upload_limit`, `avatar_upload_limit`, `background_upload_limit`, and `banner_upload_limit` to `/api/v1/instance`
+- Mastodon API: Add `pleroma.unread_conversation_count` to the Account entity
+- OAuth: support for hierarchical permissions / [Mastodon 2.4.3 OAuth permissions](https://docs.joinmastodon.org/api/permissions/)
+- Metadata Link: Atom syndication Feed
+- Mix task to re-count statuses for all users (`mix pleroma.count_statuses`)
+- Mastodon API: Add `exclude_visibilities` parameter to the timeline and notification endpoints
+- Admin API: `/users/:nickname/toggle_activation` endpoint is now deprecated in favor of: `/users/activate`, `/users/deactivate`, both accept `nicknames` array
+- Admin API: Multiple endpoints now require `nicknames` array, instead of singe `nickname`:
+  - `POST/DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group` are deprecated in favor of: `POST/DELETE /api/pleroma/admin/users/permission_group/:permission_group`
+  - `DELETE /api/pleroma/admin/users` (`nickname` query param or `nickname` sent in JSON body) is deprecated in favor of: `DELETE /api/pleroma/admin/users` (`nicknames` query array param or `nicknames` sent in JSON body)
+- Admin API: Add `GET /api/pleroma/admin/relay` endpoint - lists all followed relays
+- Pleroma API: `POST /api/v1/pleroma/conversations/read` to mark all conversations as read
+- ActivityPub: Support `Move` activities
+- Mastodon API: Add `/api/v1/markers` for managing timeline read markers
+- Mastodon API: Add the `recipients` parameter to `GET /api/v1/conversations`
+- Configuration: `feed` option for user atom feed.
+- Pleroma API: Add Emoji reactions
+- Admin API: Add `/api/pleroma/admin/instances/:instance/statuses` - lists all statuses from a given instance
+- Admin API: Add `/api/pleroma/admin/users/:nickname/statuses` - lists all statuses from a given user
+- Admin API: `PATCH /api/pleroma/users/confirm_email` to confirm email for multiple users, `PATCH /api/pleroma/users/resend_confirmation_email` to resend confirmation email for multiple users
+- ActivityPub: Configurable `type` field of the actors.
+- Mastodon API: `/api/v1/accounts/:id` has `source/pleroma/actor_type` field.
+- Mastodon API: `/api/v1/update_credentials` accepts `actor_type` field.
+- Captcha: Support native provider
+- Captcha: Enable by default
+- Mastodon API: Add support for `account_id` param to filter notifications by the account
+- Mastodon API: Add `emoji_reactions` property to Statuses
+- Mastodon API: Change emoji reaction reply format
+- Notifications: Added `pleroma:emoji_reaction` notification type
+- Mastodon API: Change emoji reaction reply format once more
+- Configuration: `feed.logo` option for tag feed.
+- Tag feed: `/tags/:tag.rss` - list public statuses by hashtag.
+- Mastodon API: Add `reacted` property to `emoji_reactions`
+- Pleroma API: Add reactions for a single emoji.
+- ActivityPub: `[:activitypub, :note_replies_output_limit]` setting sets the number of note self-replies to output on outgoing federation.
+- Admin API: `GET /api/pleroma/admin/stats` to get status count by visibility scope
+- Admin API: `GET /api/pleroma/admin/statuses` - list all statuses (accepts `godmode` and `local_only`)
+</details>
+
+### Fixed
+- Report emails now include functional links to profiles of remote user accounts
+- Not being able to log in to some third-party apps when logged in to MastoFE
+- MRF: `Delete` activities being exempt from MRF policies
+- OTP releases: Not being able to configure OAuth expired token cleanup interval
+- OTP releases: Not being able to configure HTML sanitization policy
+- OTP releases: Not being able to change upload limit (again)
+- Favorites timeline now ordered by favorite date instead of post date
+- Support for cancellation of a follow request
+<details>
+  <summary>API Changes</summary>
+
+- Mastodon API: Fix private and direct statuses not being filtered out from the public timeline for an authenticated user (`GET /api/v1/timelines/public`)
+- Mastodon API: Inability to get some local users by nickname in `/api/v1/accounts/:id_or_nickname`
+- AdminAPI: If some status received reports both in the "new" format and "old" format it was considered reports on two different statuses (in the context of grouped reports)
+- Admin API: Error when trying to update reports in the "old" format
+- Mastodon API: Marking a conversation as read (`POST /api/v1/conversations/:id/read`) now no longer brings it to the top in the user's direct conversation list
+</details>
+
 ## [1.1.9] - 2020-02-10
 ### Fixed
 - OTP: Inability to set the upload limit (again)
@@ -38,7 +185,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Improved support unauthenticated view of private instances
 
 #### Removed
-- Whitespace hack on empty post content 
+- Whitespace hack on empty post content
 
 ## [1.1.6] - 2019-11-19
 ### Fixed

Dockerfile

@@ -14,6 +14,20 @@ RUN apk add git gcc g++ musl-dev make &&\
 
 FROM alpine:3.9
 
+ARG BUILD_DATE
+ARG VCS_REF
+
+LABEL maintainer="ops@pleroma.social" \
+    org.opencontainers.image.title="pleroma" \
+    org.opencontainers.image.description="Pleroma for Docker" \
+    org.opencontainers.image.authors="ops@pleroma.social" \
+    org.opencontainers.image.vendor="pleroma.social" \
+    org.opencontainers.image.documentation="https://git.pleroma.social/pleroma/pleroma" \
+    org.opencontainers.image.licenses="AGPL-3.0" \
+    org.opencontainers.image.url="https://pleroma.social" \
+    org.opencontainers.image.revision=$VCS_REF \
+    org.opencontainers.image.created=$BUILD_DATE
+
 ARG HOME=/opt/pleroma
 ARG DATA=/var/lib/pleroma
 

README.md

@@ -1,80 +1,43 @@
-# Pleroma
+<img src="https://git.pleroma.social/pleroma/pleroma/uploads/8cec84f5a084d887339f57deeb8a293e/pleroma-banner-vector-nopad-notext.svg" width="300px" />
 
-**Note**: This readme as well as complete documentation is also available at <https://docs-develop.pleroma.social>
+## About 
 
-## About Pleroma
+Pleroma is a microblogging server software that can federate (= exchange messages with) other servers that support ActivityPub. What that means is that you can host a server for yourself or your friends and stay in control of your online identity, but still exchange messages with people on larger servers. Pleroma will federate with all servers that implement ActivityPub, like Friendica, GNU Social, Hubzilla, Mastodon, Misskey, Peertube, and Pixelfed.
 
-Pleroma is a microblogging server software that can federate (= exchange messages with) other servers that support the same federation standards (OStatus and ActivityPub). What that means is that you can host a server for yourself or your friends and stay in control of your online identity, but still exchange messages with people on larger servers. Pleroma will federate with all servers that implement either OStatus or ActivityPub, like Friendica, GNU Social, Hubzilla, Mastodon, Misskey, Peertube, and Pixelfed.
+Pleroma is written in Elixir and uses PostgresSQL for data storage. It's efficient enough to be ran on low-power devices like Raspberry Pi (though we wouldn't recommend storing the database on the internal SD card ;) but can scale well when ran on more powerful hardware (albeit only single-node for now).
 
-Pleroma is written in Elixir, high-performance and can run on small devices like a Raspberry Pi.
+For clients it supports the [Mastodon client API](https://docs.joinmastodon.org/api/guidelines/) with Pleroma extensions (see the API section on <https://docs-develop.pleroma.social>).
 
-For clients it supports the [Mastodon client API](https://docs.joinmastodon.org/api/guidelines/) with Pleroma extensions (see "Pleroma's APIs and Mastodon API extensions" section on <https://docs-develop.pleroma.social>).
+- [Client Applications for Pleroma](https://docs-develop.pleroma.social/backend/clients/)
-
-- [Client Applications for Pleroma](https://docs-develop.pleroma.social/clients.html)
-
-If you want to run your own server, feel free to contact us at @lain@pleroma.soykaf.com or in our dev chat at #pleroma on freenode or via matrix at <https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org>.
 
 ## Installation
-**Note:** The guide below may be outdated and in most cases shouldn't be used. Instead check out our [wiki](https://docs.pleroma.social) for platform-specific installation instructions, most likely [Installing on Linux using OTP releases](https://docs.pleroma.social/otp_en.html) is the guide you need.
+
+### OTP releases (Recommended)
+If you are running Linux (glibc or musl) on x86/arm, the recommended way to install Pleroma is by using OTP releases. OTP releases are as close as you can get to binary releases with Erlang/Elixir. The release is self-contained, and provides everything needed to boot it. The installation instructions are available [here](https://docs-develop.pleroma.social/backend/installation/otp_en/).
+
+### From Source
+If your platform is not supported, or you just want to be able to edit the source code easily, you may install Pleroma from source.
+
+- [Debian-based](https://docs-develop.pleroma.social/backend/installation/debian_based_en/)
+- [Debian-based (jp)](https://docs-develop.pleroma.social/backend/installation/debian_based_jp/)
+- [Alpine Linux](https://docs-develop.pleroma.social/backend/installation/alpine_linux_en/)
+- [Arch Linux](https://docs-develop.pleroma.social/backend/installation/arch_linux_en/)
+- [Gentoo Linux](https://docs-develop.pleroma.social/backend/installation/gentoo_en/)
+- [NetBSD](https://docs-develop.pleroma.social/backend/installation/netbsd_en/)
+- [OpenBSD](https://docs-develop.pleroma.social/backend/installation/openbsd_en/)
+- [OpenBSD (fi)](https://docs-develop.pleroma.social/backend/installation/openbsd_fi/)
+- [CentOS 7](https://docs-develop.pleroma.social/backend/installation/centos7_en/)
 
 ### OS/Distro packages
-Currently Pleroma is not packaged by any OS/Distros, but feel free to reach out to us at [#pleroma-dev on freenode](https://webchat.freenode.net/?channels=%23pleroma-dev) or via matrix at <https://matrix.heldscal.la/#/room/#freenode_#pleroma-dev:matrix.org> for assistance. If you want to change default options in your Pleroma package, please **discuss it with us first**.
+Currently Pleroma is not packaged by any OS/Distros, but if you want to package it for one, we can guide you through the process on our [community channels](#community-channels). If you want to change default options in your Pleroma package, please **discuss it with us first**.
 
 ### Docker
 While we don’t provide docker files, other people have written very good ones. Take a look at <https://github.com/angristan/docker-pleroma> or <https://glitch.sh/sn0w/pleroma-docker>.
 
-### Dependencies
+## Documentation
+- Latest Released revision: <https://docs.pleroma.social>
+- Latest Git revision: <https://docs-develop.pleroma.social>
 
-* Postgresql version 9.6 or newer
+## Community Channels
-* Elixir version 1.7 or newer. If your distribution only has an old version available, check [Elixir’s install page](https://elixir-lang.org/install.html) or use a tool like [asdf](https://github.com/asdf-vm/asdf).
+* IRC: **#pleroma** and **#pleroma-dev** on freenode, webchat is available at <https://irc.pleroma.social>
-* Build-essential tools
+* Matrix: <https://matrix.to/#/#freenode_#pleroma:matrix.org> and <https://matrix.to/#/#freenode_#pleroma-dev:matrix.org>
-
-### Configuration
-
-* Run `mix deps.get` to install elixir dependencies.
-* Run `mix pleroma.instance gen`. This will ask you questions about your instance and generate a configuration file in `config/generated_config.exs`. Check that and copy it to either `config/dev.secret.exs` or `config/prod.secret.exs`. It will also create a `config/setup_db.psql`, which you should run as the PostgreSQL superuser (i.e., `sudo -u postgres psql -f config/setup_db.psql`). It will create the database, user, and password you gave `mix pleroma.gen.instance` earlier, as well as set up the necessary extensions in the database. PostgreSQL superuser privileges are only needed for this step.
-* For these next steps, the default will be to run pleroma using the dev configuration file, `config/dev.secret.exs`. To run them using the prod config file, prefix each command at the shell with `MIX_ENV=prod`. For example: `MIX_ENV=prod mix phx.server`. Documentation for the config can be found at [`docs/config.md`](docs/config.md) in the repository, or at the "Configuration" page on <https://docs-develop.pleroma.social/config.html>
-* Run `mix ecto.migrate` to run the database migrations. You will have to do this again after certain updates.
-* You can check if your instance is configured correctly by running it with `mix phx.server` and checking the instance info endpoint at `/api/v1/instance`. If it shows your uri, name and email correctly, you are configured correctly. If it shows something like `localhost:4000`, your configuration is probably wrong, unless you are running a local development setup.
-* The common and convenient way for adding HTTPS is by using Nginx as a reverse proxy. You can look at example Nginx configuration in `installation/pleroma.nginx`. If you need TLS/SSL certificates for HTTPS, you can look get some for free with letsencrypt: <https://letsencrypt.org/>. The simplest way to obtain and install a certificate is to use [Certbot.](https://certbot.eff.org) Depending on your specific setup, certbot may be able to get a certificate and configure your web server automatically.
-
-## Running
-
-* By default, it listens on port 4000 (TCP), so you can access it on <http://localhost:4000/> (if you are on the same machine). In case of an error it will restart automatically.
-
-### Frontends
-
-Pleroma comes with two frontends. The first one, Pleroma FE, can be reached by normally visiting the site. The other one, based on the Mastodon project, can be found by visiting the /web path of your site.
-
-### As systemd service (with provided .service file)
-
-Example .service file can be found in `installation/pleroma.service`. Copy this to `/etc/systemd/system/`. Running `systemctl enable --now pleroma.service` will run Pleroma and enable startup on boot. Logs can be watched by using `journalctl -fu pleroma.service`.
-
-### As OpenRC service (with provided RC file)
-
-Copy `installation/init.d/pleroma` to `/etc/init.d/pleroma`. You can add it to the services ran by default with: `rc-update add pleroma`
-
-### Standalone/run by other means
-
-Run `mix phx.server` in repository’s root, it will output log into stdout/stderr.
-
-### Using an upstream proxy for federation
-
-Add the following to your `dev.secret.exs` or `prod.secret.exs` if you want to proxify all http requests that Pleroma makes to an upstream proxy server:
-
-```elixir
-config :pleroma, :http,
-  proxy_url: "127.0.0.1:8123"
-```
-
-This is useful for running Pleroma inside Tor or I2P.
-
-## Customization and contribution
-
-The [Pleroma Documentation](https://docs-develop.pleroma.social/readme.html) offers manuals and guides on how to further customize your instance to your liking and how you can contribute to the project.
-
-## Troubleshooting
-
-### No incoming federation
-
-Check that you correctly forward the `host` header to the backend. It is needed to validate signatures.

benchmarks/load_testing/fetcher.ex

@@ -0,0 +1,260 @@
+defmodule Pleroma.LoadTesting.Fetcher do
+  use Pleroma.LoadTesting.Helper
+
+  def fetch_user(user) do
+    Benchee.run(%{
+      "By id" => fn -> Repo.get_by(User, id: user.id) end,
+      "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end,
+      "By email" => fn -> Repo.get_by(User, email: user.email) end,
+      "By nickname" => fn -> Repo.get_by(User, nickname: user.nickname) end
+    })
+  end
+
+  def query_timelines(user) do
+    home_timeline_params = %{
+      "count" => 20,
+      "with_muted" => true,
+      "type" => ["Create", "Announce"],
+      "blocking_user" => user,
+      "muting_user" => user,
+      "user" => user
+    }
+
+    mastodon_public_timeline_params = %{
+      "count" => 20,
+      "local_only" => true,
+      "only_media" => "false",
+      "type" => ["Create", "Announce"],
+      "with_muted" => "true",
+      "blocking_user" => user,
+      "muting_user" => user
+    }
+
+    mastodon_federated_timeline_params = %{
+      "count" => 20,
+      "only_media" => "false",
+      "type" => ["Create", "Announce"],
+      "with_muted" => "true",
+      "blocking_user" => user,
+      "muting_user" => user
+    }
+
+    following = User.following(user)
+
+    Benchee.run(%{
+      "User home timeline" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
+          following,
+          home_timeline_params
+        )
+      end,
+      "User mastodon public timeline" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
+          mastodon_public_timeline_params
+        )
+      end,
+      "User mastodon federated public timeline" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
+          mastodon_federated_timeline_params
+        )
+      end
+    })
+
+    home_activities =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
+        following,
+        home_timeline_params
+      )
+
+    public_activities =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params)
+
+    public_federated_activities =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
+        mastodon_federated_timeline_params
+      )
+
+    Benchee.run(%{
+      "Rendering home timeline" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: home_activities,
+          for: user,
+          as: :activity
+        })
+      end,
+      "Rendering public timeline" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: public_activities,
+          for: user,
+          as: :activity
+        })
+      end,
+      "Rendering public federated timeline" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: public_federated_activities,
+          for: user,
+          as: :activity
+        })
+      end,
+      "Rendering favorites timeline" => fn ->
+        conn = Phoenix.ConnTest.build_conn(:get, "http://localhost:4001/api/v1/favourites", nil)
+        Pleroma.Web.MastodonAPI.StatusController.favourites(
+          %Plug.Conn{conn |
+                     assigns: %{user: user},
+                     query_params:  %{"limit" => "0"},
+                     body_params: %{},
+                     cookies: %{},
+                     params: %{},
+                     path_params: %{},
+                     private: %{
+                       Pleroma.Web.Router => {[], %{}},
+                       phoenix_router: Pleroma.Web.Router,
+                       phoenix_action: :favourites,
+                       phoenix_controller: Pleroma.Web.MastodonAPI.StatusController,
+                       phoenix_endpoint: Pleroma.Web.Endpoint,
+                       phoenix_format: "json",
+                       phoenix_layout: {Pleroma.Web.LayoutView, "app.html"},
+                       phoenix_recycled: true,
+
+                       phoenix_view: Pleroma.Web.MastodonAPI.StatusView,
+                       plug_session: %{"user_id" => user.id},
+                       plug_session_fetch: :done,
+                       plug_session_info: :write,
+                       plug_skip_csrf_protection: true
+                     }
+          },
+          %{})
+      end,
+    })
+  end
+
+  def query_notifications(user) do
+    without_muted_params = %{"count" => "20", "with_muted" => "false"}
+    with_muted_params = %{"count" => "20", "with_muted" => "true"}
+
+    Benchee.run(%{
+      "Notifications without muted" => fn ->
+        Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
+      end,
+      "Notifications with muted" => fn ->
+        Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
+      end
+    })
+
+    without_muted_notifications =
+      Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
+
+    with_muted_notifications =
+      Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
+
+    Benchee.run(%{
+      "Render notifications without muted" => fn ->
+        Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
+          notifications: without_muted_notifications,
+          for: user
+        })
+      end,
+      "Render notifications with muted" => fn ->
+        Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
+          notifications: with_muted_notifications,
+          for: user
+        })
+      end
+    })
+  end
+
+  def query_dms(user) do
+    params = %{
+      "count" => "20",
+      "with_muted" => "true",
+      "type" => "Create",
+      "blocking_user" => user,
+      "user" => user,
+      visibility: "direct"
+    }
+
+    Benchee.run(%{
+      "Direct messages with muted" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+        |> Pleroma.Pagination.fetch_paginated(params)
+      end,
+      "Direct messages without muted" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+        |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
+      end
+    })
+
+    dms_with_muted =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+      |> Pleroma.Pagination.fetch_paginated(params)
+
+    dms_without_muted =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+      |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
+
+    Benchee.run(%{
+      "Rendering dms with muted" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: dms_with_muted,
+          for: user,
+          as: :activity
+        })
+      end,
+      "Rendering dms without muted" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: dms_without_muted,
+          for: user,
+          as: :activity
+        })
+      end
+    })
+  end
+
+  def query_long_thread(user, activity) do
+    Benchee.run(%{
+      "Fetch main post" => fn ->
+        Pleroma.Activity.get_by_id_with_object(activity.id)
+      end,
+      "Fetch context of main post" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
+          activity.data["context"],
+          %{
+            "blocking_user" => user,
+            "user" => user,
+            "exclude_id" => activity.id
+          }
+        )
+      end
+    })
+
+    activity = Pleroma.Activity.get_by_id_with_object(activity.id)
+
+    context =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
+        activity.data["context"],
+        %{
+          "blocking_user" => user,
+          "user" => user,
+          "exclude_id" => activity.id
+        }
+      )
+
+    Benchee.run(%{
+      "Render status" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{
+          activity: activity,
+          for: user
+        })
+      end,
+      "Render context" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render(
+          "index.json",
+          for: user,
+          activities: context,
+          as: :activity
+        )
+        |> Enum.reverse()
+      end
+    })
+  end
+end

benchmarks/load_testing/generator.ex

@@ -0,0 +1,409 @@
+defmodule Pleroma.LoadTesting.Generator do
+  use Pleroma.LoadTesting.Helper
+  alias Pleroma.Web.CommonAPI
+
+  def generate_like_activities(user, posts) do
+    count_likes = Kernel.trunc(length(posts) / 4)
+    IO.puts("Starting generating #{count_likes} like activities...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          Enum.take_random(posts, count_likes),
+          fn post -> {:ok, _, _} = CommonAPI.favorite(post.id, user) end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting like activities take #{to_sec(time)} sec.\n")
+  end
+
+  def generate_users(opts) do
+    IO.puts("Starting generating #{opts[:users_max]} users...")
+    {time, _} = :timer.tc(fn -> do_generate_users(opts) end)
+
+    IO.puts("Inserting users take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_users(opts) do
+    max = Keyword.get(opts, :users_max)
+
+    Task.async_stream(
+      1..max,
+      &generate_user_data(&1),
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Enum.to_list()
+  end
+
+  defp generate_user_data(i) do
+    remote = Enum.random([true, false])
+
+    user = %User{
+      name: "Test テスト User #{i}",
+      email: "user#{i}@example.com",
+      nickname: "nick#{i}",
+      password_hash:
+        "$pbkdf2-sha512$160000$bU.OSFI7H/yqWb5DPEqyjw$uKp/2rmXw12QqnRRTqTtuk2DTwZfF8VR4MYW2xMeIlqPR/UX1nT1CEKVUx2CowFMZ5JON8aDvURrZpJjSgqXrg",
+      bio: "Tester Number #{i}",
+      local: remote
+    }
+
+    user_urls =
+      if remote do
+        base_url =
+          Enum.random(["https://domain1.com", "https://domain2.com", "https://domain3.com"])
+
+        ap_id = "#{base_url}/users/#{user.nickname}"
+
+        %{
+          ap_id: ap_id,
+          follower_address: ap_id <> "/followers",
+          following_address: ap_id <> "/following"
+        }
+      else
+        %{
+          ap_id: User.ap_id(user),
+          follower_address: User.ap_followers(user),
+          following_address: User.ap_following(user)
+        }
+      end
+
+    user = Map.merge(user, user_urls)
+
+    Repo.insert!(user)
+  end
+
+  def generate_activities(user, users) do
+    do_generate_activities(user, users)
+  end
+
+  defp do_generate_activities(user, users) do
+    IO.puts("Starting generating 20000 common activities...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..20_000,
+          fn _ ->
+            do_generate_activity([user | users])
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting common activities take #{to_sec(time)} sec.\n")
+
+    IO.puts("Starting generating 20000 activities with mentions...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..20_000,
+          fn _ ->
+            do_generate_activity_with_mention(user, users)
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting activities with menthions take #{to_sec(time)} sec.\n")
+
+    IO.puts("Starting generating 10000 activities with threads...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..10_000,
+          fn _ ->
+            do_generate_threads([user | users])
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting activities with threads take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_activity(users) do
+    post = %{
+      "status" => "Some status without mention with random user"
+    }
+
+    CommonAPI.post(Enum.random(users), post)
+  end
+
+  def generate_power_intervals(opts \\ []) do
+    count = Keyword.get(opts, :count, 20)
+    power = Keyword.get(opts, :power, 2)
+    IO.puts("Generating #{count} intervals for a power #{power} series...")
+    counts = Enum.map(1..count, fn n -> :math.pow(n, power) end)
+    sum = Enum.sum(counts)
+
+    densities =
+      Enum.map(counts, fn c ->
+        c / sum
+      end)
+
+    densities
+    |> Enum.reduce(0, fn density, acc ->
+      if acc == 0 do
+        [{0, density}]
+      else
+        [{_, lower} | _] = acc
+        [{lower, lower + density} | acc]
+      end
+    end)
+    |> Enum.reverse()
+  end
+
+  def generate_tagged_activities(opts \\ []) do
+    tag_count = Keyword.get(opts, :tag_count, 20)
+    users = Keyword.get(opts, :users, Repo.all(User))
+    activity_count = Keyword.get(opts, :count, 200_000)
+
+    intervals = generate_power_intervals(count: tag_count)
+
+    IO.puts(
+      "Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0"
+    )
+
+    Enum.each(1..activity_count, fn _ ->
+      random = :rand.uniform()
+      i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end)
+      CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"})
+    end)
+  end
+
+  defp do_generate_activity_with_mention(user, users) do
+    mentions_cnt = Enum.random([2, 3, 4, 5])
+    with_user = Enum.random([true, false])
+    users = Enum.shuffle(users)
+    mentions_users = Enum.take(users, mentions_cnt)
+    mentions_users = if with_user, do: [user | mentions_users], else: mentions_users
+
+    mentions_str =
+      Enum.map(mentions_users, fn user -> "@" <> user.nickname end) |> Enum.join(", ")
+
+    post = %{
+      "status" => mentions_str <> "some status with mentions random users"
+    }
+
+    CommonAPI.post(Enum.random(users), post)
+  end
+
+  defp do_generate_threads(users) do
+    thread_length = Enum.random([2, 3, 4, 5])
+    actor = Enum.random(users)
+
+    post = %{
+      "status" => "Start of the thread"
+    }
+
+    {:ok, activity} = CommonAPI.post(actor, post)
+
+    Enum.each(1..thread_length, fn _ ->
+      user = Enum.random(users)
+
+      post = %{
+        "status" => "@#{actor.nickname} reply to thread",
+        "in_reply_to_status_id" => activity.id
+      }
+
+      CommonAPI.post(user, post)
+    end)
+  end
+
+  def generate_remote_activities(user, users) do
+    do_generate_remote_activities(user, users)
+  end
+
+  defp do_generate_remote_activities(user, users) do
+    IO.puts("Starting generating 10000 remote activities...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..10_000,
+          fn i ->
+            do_generate_remote_activity(i, user, users)
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting remote activities take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_remote_activity(i, user, users) do
+    actor = Enum.random(users)
+    %{host: host} = URI.parse(actor.ap_id)
+    date = Date.utc_today()
+    datetime = DateTime.utc_now()
+
+    map = %{
+      "actor" => actor.ap_id,
+      "cc" => [actor.follower_address, user.ap_id],
+      "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
+      "id" => actor.ap_id <> "/statuses/#{i}/activity",
+      "object" => %{
+        "actor" => actor.ap_id,
+        "atomUri" => actor.ap_id <> "/statuses/#{i}",
+        "attachment" => [],
+        "attributedTo" => actor.ap_id,
+        "bcc" => [],
+        "bto" => [],
+        "cc" => [actor.follower_address, user.ap_id],
+        "content" =>
+          "<p><span class=\"h-card\"><a href=\"" <>
+            user.ap_id <>
+            "\" class=\"u-url mention\">@<span>" <> user.nickname <> "</span></a></span></p>",
+        "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
+        "conversation" =>
+          "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
+        "emoji" => %{},
+        "id" => actor.ap_id <> "/statuses/#{i}",
+        "inReplyTo" => nil,
+        "inReplyToAtomUri" => nil,
+        "published" => datetime,
+        "sensitive" => true,
+        "summary" => "cw",
+        "tag" => [
+          %{
+            "href" => user.ap_id,
+            "name" => "@#{user.nickname}@#{host}",
+            "type" => "Mention"
+          }
+        ],
+        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "type" => "Note",
+        "url" => "http://#{host}/@#{actor.nickname}/#{i}"
+      },
+      "published" => datetime,
+      "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+      "type" => "Create"
+    }
+
+    Pleroma.Web.ActivityPub.ActivityPub.insert(map, false)
+  end
+
+  def generate_dms(user, users, opts) do
+    IO.puts("Starting generating #{opts[:dms_max]} DMs")
+    {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end)
+    IO.puts("Inserting dms take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_dms(user, users, opts) do
+    Task.async_stream(
+      1..opts[:dms_max],
+      fn _ ->
+        do_generate_dm(user, users)
+      end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Stream.run()
+  end
+
+  defp do_generate_dm(user, users) do
+    post = %{
+      "status" => "@#{user.nickname} some direct message",
+      "visibility" => "direct"
+    }
+
+    CommonAPI.post(Enum.random(users), post)
+  end
+
+  def generate_long_thread(user, users, opts) do
+    IO.puts("Starting generating long thread with #{opts[:thread_length]} replies")
+    {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end)
+    IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
+    {:ok, activity}
+  end
+
+  defp do_generate_long_thread(user, users, opts) do
+    {:ok, %{id: id} = activity} = CommonAPI.post(user, %{"status" => "Start of long thread"})
+
+    Task.async_stream(
+      1..opts[:thread_length],
+      fn _ -> do_generate_thread(users, id) end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Stream.run()
+
+    activity
+  end
+
+  defp do_generate_thread(users, activity_id) do
+    CommonAPI.post(Enum.random(users), %{
+      "status" => "reply to main post",
+      "in_reply_to_status_id" => activity_id
+    })
+  end
+
+  def generate_non_visible_message(user, users) do
+    IO.puts("Starting generating 1000 non visible posts")
+
+    {time, _} =
+      :timer.tc(fn ->
+        do_generate_non_visible_posts(user, users)
+      end)
+
+    IO.puts("Inserting non visible posts take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_non_visible_posts(user, users) do
+    [not_friend | users] = users
+
+    make_friends(user, users)
+
+    Task.async_stream(1..1000, fn _ -> do_generate_non_visible_post(not_friend, users) end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Stream.run()
+  end
+
+  defp make_friends(_user, []), do: nil
+
+  defp make_friends(user, [friend | users]) do
+    {:ok, _} = User.follow(user, friend)
+    {:ok, _} = User.follow(friend, user)
+    make_friends(user, users)
+  end
+
+  defp do_generate_non_visible_post(not_friend, users) do
+    post = %{
+      "status" => "some non visible post",
+      "visibility" => "private"
+    }
+
+    {:ok, activity} = CommonAPI.post(not_friend, post)
+
+    thread_length = Enum.random([2, 3, 4, 5])
+
+    Enum.each(1..thread_length, fn _ ->
+      user = Enum.random(users)
+
+      post = %{
+        "status" => "@#{not_friend.nickname} reply to non visible post",
+        "in_reply_to_status_id" => activity.id,
+        "visibility" => "private"
+      }
+
+      CommonAPI.post(user, post)
+    end)
+  end
+end

benchmarks/load_testing/helper.ex

@@ -0,0 +1,11 @@
+defmodule Pleroma.LoadTesting.Helper do
+  defmacro __using__(_) do
+    quote do
+      import Ecto.Query
+      alias Pleroma.Repo
+      alias Pleroma.User
+
+      defp to_sec(microseconds), do: microseconds / 1_000_000
+    end
+  end
+end

benchmarks/mix/tasks/pleroma/benchmarks/tags.ex

@@ -0,0 +1,87 @@
+defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
+  use Mix.Task
+  alias Pleroma.Repo
+  alias Pleroma.LoadTesting.Generator
+  import Ecto.Query
+
+  def run(_args) do
+    Mix.Pleroma.start_pleroma()
+    activities_count = Repo.aggregate(from(a in Pleroma.Activity), :count, :id)
+
+    if activities_count == 0 do
+      IO.puts("Did not find any activities, cleaning and generating")
+      clean_tables()
+      Generator.generate_users(users_max: 10)
+      Generator.generate_tagged_activities()
+    else
+      IO.puts("Found #{activities_count} activities, won't generate new ones")
+    end
+
+    tags = Enum.map(0..20, fn i -> {"For #tag_#{i}", "tag_#{i}"} end)
+
+    Enum.each(tags, fn {_, tag} ->
+      query =
+        from(o in Pleroma.Object,
+          where: fragment("(?)->'tag' \\? (?)", o.data, ^tag)
+        )
+
+      count = Repo.aggregate(query, :count, :id)
+      IO.puts("Database contains #{count} posts tagged with #{tag}")
+    end)
+
+    user = Repo.all(Pleroma.User) |> List.first()
+
+    Benchee.run(
+      %{
+        "Hashtag fetching, any" => fn tags ->
+          Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
+            %{
+              "any" => tags
+            },
+            user,
+            false
+          )
+        end,
+        # Will always return zero results because no overlapping hashtags are generated.
+        "Hashtag fetching, all" => fn tags ->
+          Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
+            %{
+              "all" => tags
+            },
+            user,
+            false
+          )
+        end
+      },
+      inputs:
+        tags
+        |> Enum.map(fn {_, v} -> v end)
+        |> Enum.chunk_every(2)
+        |> Enum.map(fn tags -> {"For #{inspect(tags)}", tags} end),
+      time: 5
+    )
+
+    Benchee.run(
+      %{
+        "Hashtag fetching" => fn tag ->
+          Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
+            %{
+              "tag" => tag
+            },
+            user,
+            false
+          )
+        end
+      },
+      inputs: tags,
+      time: 5
+    )
+  end
+
+  defp clean_tables do
+    IO.puts("Deleting old data...\n")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
+  end
+end

benchmarks/mix/tasks/pleroma/load_testing.ex

@@ -0,0 +1,138 @@
+defmodule Mix.Tasks.Pleroma.LoadTesting do
+  use Mix.Task
+  use Pleroma.LoadTesting.Helper
+  import Mix.Pleroma
+  import Pleroma.LoadTesting.Generator
+  import Pleroma.LoadTesting.Fetcher
+
+  @shortdoc "Factory for generation data"
+  @moduledoc """
+  Generates data like:
+  - local/remote users
+  - local/remote activities with notifications
+  - direct messages
+  - long thread
+  - non visible posts
+
+  ## Generate data
+      MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --dms 20000 --thread_length 2000
+      MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -d 20000 -t 2000
+
+  Options:
+  - `--users NUMBER` - number of users to generate. Defaults to: 20000. Alias: `-u`
+  - `--dms NUMBER` - number of direct messages to generate. Defaults to: 20000. Alias `-d`
+  - `--thread_length` - number of messages in thread. Defaults to: 2000. ALias `-t`
+  """
+
+  @aliases [u: :users, d: :dms, t: :thread_length]
+  @switches [
+    users: :integer,
+    dms: :integer,
+    thread_length: :integer
+  ]
+  @users_default 20_000
+  @dms_default 1_000
+  @thread_length_default 2_000
+
+  def run(args) do
+    start_pleroma()
+    Pleroma.Config.put([:instance, :skip_thread_containment], true)
+    {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
+
+    users_max = Keyword.get(opts, :users, @users_default)
+    dms_max = Keyword.get(opts, :dms, @dms_default)
+    thread_length = Keyword.get(opts, :thread_length, @thread_length_default)
+
+    clean_tables()
+
+    opts =
+      Keyword.put(opts, :users_max, users_max)
+      |> Keyword.put(:dms_max, dms_max)
+      |> Keyword.put(:thread_length, thread_length)
+
+    generate_users(opts)
+
+    # main user for queries
+    IO.puts("Fetching local main user...")
+
+    {time, user} =
+      :timer.tc(fn ->
+        Repo.one(
+          from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)
+        )
+      end)
+
+    IO.puts("Fetching main user take #{to_sec(time)} sec.\n")
+
+    IO.puts("Fetching local users...")
+
+    {time, users} =
+      :timer.tc(fn ->
+        Repo.all(
+          from(u in User,
+            where: u.id != ^user.id,
+            where: u.local == true,
+            order_by: fragment("RANDOM()"),
+            limit: 10
+          )
+        )
+      end)
+
+    IO.puts("Fetching local users take #{to_sec(time)} sec.\n")
+
+    IO.puts("Fetching remote users...")
+
+    {time, remote_users} =
+      :timer.tc(fn ->
+        Repo.all(
+          from(u in User,
+            where: u.id != ^user.id,
+            where: u.local == false,
+            order_by: fragment("RANDOM()"),
+            limit: 10
+          )
+        )
+      end)
+
+    IO.puts("Fetching remote users take #{to_sec(time)} sec.\n")
+
+    generate_activities(user, users)
+
+    generate_remote_activities(user, remote_users)
+
+    generate_like_activities(
+      user, Pleroma.Repo.all(Pleroma.Activity.Queries.by_type("Create"))
+    )
+
+    generate_dms(user, users, opts)
+
+    {:ok, activity} = generate_long_thread(user, users, opts)
+
+    generate_non_visible_message(user, users)
+
+    IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}")
+
+    IO.puts("Activities in DB: #{Repo.aggregate(from(a in Pleroma.Activity), :count, :id)}")
+
+    IO.puts("Objects in DB: #{Repo.aggregate(from(o in Pleroma.Object), :count, :id)}")
+
+    IO.puts(
+      "Notifications in DB: #{Repo.aggregate(from(n in Pleroma.Notification), :count, :id)}"
+    )
+
+    fetch_user(user)
+    query_timelines(user)
+    query_notifications(user)
+    query_dms(user)
+    query_long_thread(user, activity)
+    Pleroma.Config.put([:instance, :skip_thread_containment], false)
+    query_timelines(user)
+  end
+
+  defp clean_tables do
+    IO.puts("Deleting old data...\n")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
+  end
+end

config/benchmark.exs

@@ -0,0 +1,92 @@
+use Mix.Config
+
+# We don't run a server during test. If one is required,
+# you can enable the server option below.
+config :pleroma, Pleroma.Web.Endpoint,
+  http: [port: 4001],
+  url: [port: 4001],
+  server: true
+
+# Disable captha for tests
+config :pleroma, Pleroma.Captcha,
+  # It should not be enabled for automatic tests
+  enabled: false,
+  # A fake captcha service for tests
+  method: Pleroma.Captcha.Mock
+
+# Print only warnings and errors during test
+config :logger, level: :warn
+
+config :pleroma, :auth, oauth_consumer_strategies: []
+
+config :pleroma, Pleroma.Upload, filters: [], link_name: false
+
+config :pleroma, Pleroma.Uploaders.Local, uploads: "test/uploads"
+
+config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Test, enabled: true
+
+config :pleroma, :instance,
+  email: "admin@example.com",
+  notify_email: "noreply@example.com",
+  skip_thread_containment: false,
+  federating: false,
+  external_user_synchronization: false
+
+config :pleroma, :activitypub, sign_object_fetches: false
+
+# Configure your database
+config :pleroma, Pleroma.Repo,
+  adapter: Ecto.Adapters.Postgres,
+  username: "postgres",
+  password: "postgres",
+  database: "pleroma_test",
+  hostname: System.get_env("DB_HOST") || "localhost",
+  pool_size: 10
+
+# Reduce hash rounds for testing
+config :pbkdf2_elixir, rounds: 1
+
+config :tesla, adapter: Tesla.Mock
+
+config :pleroma, :rich_media,
+  enabled: false,
+  ignore_hosts: [],
+  ignore_tld: ["local", "localdomain", "lan"]
+
+config :web_push_encryption, :vapid_details,
+  subject: "mailto:administrator@example.com",
+  public_key:
+    "BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4",
+  private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA"
+
+config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock
+
+config :pleroma_job_queue, disabled: true
+
+config :pleroma, Pleroma.ScheduledActivity,
+  daily_user_limit: 2,
+  total_user_limit: 3,
+  enabled: false
+
+config :pleroma, :rate_limit,
+  search: [{1000, 30}, {1000, 30}],
+  app_account_creation: {10_000, 5},
+  password_reset: {1000, 30}
+
+config :pleroma, :http_security, report_uri: "https://endpoint.com"
+
+config :pleroma, :http, send_user_agent: false
+
+rum_enabled = System.get_env("RUM_ENABLED") == "true"
+config :pleroma, :database, rum_enabled: rum_enabled
+IO.puts("RUM enabled: #{rum_enabled}")
+
+config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock
+
+if File.exists?("./config/benchmark.secret.exs") do
+  import_config "benchmark.secret.exs"
+else
+  IO.puts(
+    "You may want to create benchmark.secret.exs to declare custom database connection parameters."
+  )
+end

config/config.exs

@@ -76,7 +76,7 @@
 config :pleroma, Pleroma.Upload,
   uploader: Pleroma.Uploaders.Local,
   filters: [Pleroma.Upload.Filter.Dedupe],
-  link_name: true,
+  link_name: false,
   proxy_remote: false,
   proxy_opts: [
     redirect_on_failure: false,
@@ -91,20 +91,17 @@
 
 config :pleroma, Pleroma.Uploaders.S3,
   bucket: nil,
+  streaming_enabled: true,
   public_endpoint: "https://s3.amazonaws.com"
 
-config :pleroma, Pleroma.Uploaders.MDII,
-  cgi: "https://mdii.sakura.ne.jp/mdii-post.cgi",
-  files: "https://mdii.sakura.ne.jp"
-
 config :pleroma, :emoji,
   shortcode_globs: ["/emoji/custom/**/*.png"],
   pack_extensions: [".png", ".gif"],
   groups: [
-    # Put groups that have higher priority than defaults here. Example in `docs/config/custom_emoji.md`
     Custom: ["/emoji/*.png", "/emoji/**/*.png"]
   ],
-  default_manifest: "https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json"
+  default_manifest: "https://git.pleroma.social/pleroma/emoji-index/raw/master/index.json",
+  shared_pack_cache_seconds_per_file: 60
 
 config :pleroma, :uri_schemes,
   valid_schemes: [
@@ -164,7 +161,8 @@
 
 # Configures Elixir's Logger
 config :logger, :console,
-  format: "$time $metadata[$level] $message\n",
+  level: :debug,
+  format: "\n$time $metadata[$level] $message\n",
   metadata: [:request_id]
 
 config :logger, :ex_syslogger,
@@ -192,6 +190,7 @@
 config :pleroma, :http,
   proxy_url: nil,
   send_user_agent: true,
+  user_agent: :default,
   adapter: [
     ssl_options: [
       # Workaround for remote server certificate chain issues
@@ -207,6 +206,7 @@
   notify_email: "noreply@example.com",
   description: "A Pleroma instance, an alternative fediverse server",
   limit: 5_000,
+  chat_limit: 5_000,
   remote_limit: 100_000,
   upload_limit: 16_000_000,
   avatar_upload_limit: 2_000_000,
@@ -219,13 +219,13 @@
     max_expiration: 365 * 24 * 60 * 60
   },
   registrations_open: true,
+  invites_enabled: false,
+  account_activation_required: false,
   federating: true,
   federation_incoming_replies_max_depth: 100,
   federation_reachability_timeout_days: 7,
   federation_publisher_modules: [
-    Pleroma.Web.ActivityPub.Publisher,
+    Pleroma.Web.ActivityPub.Publisher
-    Pleroma.Web.Websub,
-    Pleroma.Web.Salmon
   ],
   allow_relay: true,
   rewrite_policy: Pleroma.Web.ActivityPub.MRF.NoOpPolicy,
@@ -243,7 +243,7 @@
   mrf_transparency_exclusions: [],
   autofollowed_nicknames: [],
   max_pinned_statuses: 1,
-  no_attachment_links: false,
+  attachment_links: false,
   welcome_user_nickname: nil,
   welcome_message: nil,
   max_report_comment_size: 1000,
@@ -252,14 +252,21 @@
   remote_post_retention_days: 90,
   skip_thread_containment: true,
   limit_to_local_content: :unauthenticated,
-  dynamic_configuration: false,
   user_bio_length: 5000,
   user_name_length: 100,
   max_account_fields: 10,
   max_remote_account_fields: 20,
   account_field_name_length: 512,
-  account_field_value_length: 512,
+  account_field_value_length: 2048,
-  external_user_synchronization: true
+  external_user_synchronization: true,
+  extended_nickname_format: true,
+  cleanup_attachments: false
+
+config :pleroma, :feed,
+  post_title: %{
+    max_length: 100,
+    omission: "..."
+  }
 
 config :pleroma, :markup,
   # XXX - unfortunately, inline images must be enabled by default right now, because
@@ -269,8 +276,8 @@
   allow_tables: false,
   allow_fonts: false,
   scrub_policy: [
-    Pleroma.HTML.Transform.MediaProxy,
+    Pleroma.HTML.Scrubber.Default,
-    Pleroma.HTML.Scrubber.Default
+    Pleroma.HTML.Transform.MediaProxy
   ]
 
 config :pleroma, :frontend_configurations,
@@ -307,11 +314,23 @@
   ],
   default_mascot: :pleroma_fox_tan
 
+config :pleroma, :manifest,
+  icons: [
+    %{
+      src: "/static/logo.png",
+      type: "image/png"
+    }
+  ],
+  theme_color: "#282c37",
+  background_color: "#191b22"
+
 config :pleroma, :activitypub,
   unfollow_blocked: true,
   outgoing_blocks: true,
   follow_handshake_timeout: 500,
-  sign_object_fetches: true
+  note_replies_output_limit: 5,
+  sign_object_fetches: true,
+  authorized_fetch_mode: false
 
 config :pleroma, :streamer,
   workers: 3,
@@ -350,6 +369,10 @@
   accept: [],
   reject: []
 
+config :pleroma, :mrf_object_age,
+  threshold: 172_800,
+  actions: [:delist, :strip_followers]
+
 config :pleroma, :rich_media,
   enabled: true,
   ignore_hosts: [],
@@ -379,6 +402,8 @@
 
 config :phoenix, :json_library, Jason
 
+config :phoenix, :filter_parameters, ["password", "confirm"]
+
 config :pleroma, :gopher,
   enabled: false,
   ip: {0, 0, 0, 0},
@@ -388,18 +413,11 @@
   providers: [
     Pleroma.Web.Metadata.Providers.OpenGraph,
     Pleroma.Web.Metadata.Providers.TwitterCard,
-    Pleroma.Web.Metadata.Providers.RelMe
+    Pleroma.Web.Metadata.Providers.RelMe,
+    Pleroma.Web.Metadata.Providers.Feed
   ],
   unfurl_nsfw: false
 
-config :pleroma, :suggestions,
-  enabled: false,
-  third_party_engine:
-    "http://vinayaka.distsn.org/cgi-bin/vinayaka-user-match-suggestions-api.cgi?{{host}}+{{user}}",
-  timeout: 300_000,
-  limit: 40,
-  web: "https://vinayaka.distsn.org"
-
 config :pleroma, :http_security,
   enabled: true,
   sts: false,
@@ -455,21 +473,36 @@
     "web"
   ]
 
-config :pleroma, Pleroma.Web.Federator.RetryQueue,
+config :pleroma, Oban,
-  enabled: false,
+  repo: Pleroma.Repo,
-  max_jobs: 20,
+  verbose: false,
-  initial_timeout: 30,
+  prune: {:maxlen, 1500},
-  max_retries: 5
+  queues: [
+    activity_expiration: 10,
+    federator_incoming: 50,
+    federator_outgoing: 50,
+    web_push: 50,
+    mailer: 10,
+    transmogrifier: 20,
+    scheduled_activities: 10,
+    background: 5,
+    remote_fetcher: 2,
+    attachments_cleanup: 5,
+    new_users_digest: 1
+  ],
+  crontab: [
+    {"0 0 * * *", Pleroma.Workers.Cron.ClearOauthTokenWorker},
+    {"0 * * * *", Pleroma.Workers.Cron.StatsWorker},
+    {"* * * * *", Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker},
+    {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker},
+    {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker}
+  ]
 
-config :pleroma_job_queue, :queues,
+config :pleroma, :workers,
-  activity_expiration: 10,
+  retries: [
-  federator_incoming: 50,
+    federator_incoming: 5,
-  federator_outgoing: 50,
+    federator_outgoing: 5
-  web_push: 50,
+  ]
-  mailer: 10,
-  transmogrifier: 20,
-  scheduled_activities: 10,
-  background: 5
 
 config :pleroma, :fetch_initial_posts,
   enabled: false,
@@ -477,14 +510,13 @@
 
 config :auto_linker,
   opts: [
-    scheme: true,
     extra: true,
     # TODO: Set to :no_scheme when it works properly
     validate_tld: true,
     class: false,
     strip_prefix: false,
     new_window: false,
-    rel: false
+    rel: "ugc"
   ]
 
 config :pleroma, :ldap,
@@ -519,7 +551,10 @@
        base_path: "/oauth",
        providers: ueberauth_providers
 
-config :pleroma, :auth, oauth_consumer_strategies: oauth_consumer_strategies
+config :pleroma,
+       :auth,
+       enforce_oauth_admin_scope_usage: true,
+       oauth_consumer_strategies: oauth_consumer_strategies
 
 config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Sendmail, enabled: false
 
@@ -534,6 +569,8 @@
     text_muted_color: "#b9b9ba"
   }
 
+config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: false
+
 config :prometheus, Pleroma.Web.Endpoint.MetricsExporter, path: "/api/pleroma/app_metrics"
 
 config :pleroma, Pleroma.ScheduledActivity,
@@ -544,7 +581,6 @@
 config :pleroma, :email_notifications,
   digest: %{
     active: false,
-    schedule: "0 0 * * 0",
     interval: 7,
     inactivity_threshold: 7
   }
@@ -552,8 +588,7 @@
 config :pleroma, :oauth2,
   token_expires_in: 600,
   issue_new_refresh_token: true,
-  clean_expired_tokens: false,
+  clean_expired_tokens: false
-  clean_expired_tokens_interval: 86_400_000
 
 config :pleroma, :database, rum_enabled: false
 
@@ -562,14 +597,37 @@
 config :http_signatures,
   adapter: Pleroma.Signature
 
-config :pleroma, :rate_limit, nil
+config :pleroma, :rate_limit,
+  authentication: {60_000, 15},
+  timeline: {500, 3},
+  search: [{1000, 10}, {1000, 30}],
+  app_account_creation: {1_800_000, 25},
+  relations_actions: {10_000, 10},
+  relation_id_action: {60_000, 2},
+  statuses_actions: {10_000, 15},
+  status_id_action: {60_000, 3},
+  password_reset: {1_800_000, 5},
+  account_confirmation_resend: {8_640_000, 5},
+  ap_routes: {60_000, 15}
 
 config :pleroma, Pleroma.ActivityExpiration, enabled: true
 
+config :pleroma, Pleroma.Plugs.RemoteIp, enabled: true
+
+config :pleroma, :static_fe, enabled: false
+
 config :pleroma, :web_cache_ttl,
   activity_pub: nil,
   activity_pub_question: 30_000
 
+config :pleroma, :modules, runtime_dir: "instance/modules"
+
+config :pleroma, configurable_from_database: false
+
+config :pleroma, Pleroma.Repo,
+  parameters: [gin_fuzzy_search_limit: "500"],
+  prepare: :unnamed
+
 # Import environment specific config. This must remain at the bottom
 # of this file so it overrides the configuration defined above.
 import_config "#{Mix.env()}.exs"

config/prod.exs

@@ -20,7 +20,8 @@
 config :phoenix, serve_endpoints: true
 
 # Do not print debug messages in production
-config :logger, level: :warn
+config :logger, :console, level: :info
+config :logger, :ex_syslogger, level: :info
 
 # ## SSL Support
 #

config/releases.exs

@@ -2,9 +2,12 @@
 
 config :pleroma, :instance, static_dir: "/var/lib/pleroma/static"
 config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads"
+config :pleroma, :modules, runtime_dir: "/var/lib/pleroma/modules"
 
 config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"
 
+config :pleroma, release: true, config_path: config_path
+
 if File.exists?(config_path) do
   import_config config_path
 else
@@ -17,3 +20,12 @@
 
   IO.puts(warning)
 end
+
+exported_config =
+  config_path
+  |> Path.dirname()
+  |> Path.join("prod.exported_from_db.secret.exs")
+
+if File.exists?(exported_config) do
+  import_config exported_config
+end

config/test.exs

@@ -15,7 +15,9 @@
   method: Pleroma.Captcha.Mock
 
 # Print only warnings and errors during test
-config :logger, level: :warn
+config :logger, :console,
+  level: :warn,
+  format: "\n[$level] $message\n"
 
 config :pleroma, :auth, oauth_consumer_strategies: []
 
@@ -30,7 +32,8 @@
   notify_email: "noreply@example.com",
   skip_thread_containment: false,
   federating: false,
-  external_user_synchronization: false
+  external_user_synchronization: false,
+  static_dir: "test/instance_static/"
 
 config :pleroma, :activitypub, sign_object_fetches: false
 
@@ -61,18 +64,17 @@
 
 config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock
 
-config :pleroma_job_queue, disabled: true
+config :pleroma, Oban,
+  queues: false,
+  prune: :disabled,
+  crontab: false
 
 config :pleroma, Pleroma.ScheduledActivity,
   daily_user_limit: 2,
   total_user_limit: 3,
   enabled: false
 
-config :pleroma, :rate_limit,
+config :pleroma, :rate_limit, %{}
-  search: [{1000, 30}, {1000, 30}],
-  app_account_creation: {10_000, 5},
-  password_reset: {1000, 30},
-  ap_routes: nil
 
 config :pleroma, :http_security, report_uri: "https://endpoint.com"
 
@@ -86,6 +88,10 @@
 
 config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock
 
+config :pleroma, :modules, runtime_dir: "test/fixtures/modules"
+
+config :pleroma, Pleroma.Emails.NewUsersDigestEmail, enabled: true
+
 if File.exists?("./config/test.secret.exs") do
   import_config "test.secret.exs"
 else
@@ -93,5 +99,3 @@
     "You may want to create test.secret.exs to declare custom database connection parameters."
   )
 end
-
-config :pleroma, Pleroma.Captcha.Kocaptcha, endpoint: "https://captcha.kotobank.ch"

docs/API/admin_api.md

@@ -0,0 +1,985 @@
+# Admin API
+
+Authentication is required and the user must be an admin.
+
+Configuration options:
+
+* `[:auth, :enforce_oauth_admin_scope_usage]` — OAuth admin scope requirement toggle.
+    If `true`, admin actions explicitly demand admin OAuth scope(s) presence in OAuth token (client app must support admin scopes).
+    If `false` and token doesn't have admin scope(s), `is_admin` user flag grants access to admin-specific actions.
+    Note that client app needs to explicitly support admin scopes and request them when obtaining auth token.
+
+## `GET /api/pleroma/admin/users`
+
+### List users
+
+- Query Params:
+  - *optional* `query`: **string** search term (e.g. nickname, domain, nickname@domain)
+  - *optional* `filters`: **string** comma-separated string of filters:
+    - `local`: only local users
+    - `external`: only external users
+    - `active`: only active users
+    - `deactivated`: only deactivated users
+    - `is_admin`: users with admin role
+    - `is_moderator`: users with moderator role
+  - *optional* `page`: **integer** page number
+  - *optional* `page_size`: **integer** number of users per page (default is `50`)
+  - *optional* `tags`: **[string]** tags list
+  - *optional* `name`: **string** user display name
+  - *optional* `email`: **string** user email
+- Example: `https://mypleroma.org/api/pleroma/admin/users?query=john&filters=local,active&page=1&page_size=10&tags[]=some_tag&tags[]=another_tag&name=display_name&email=email@example.com`
+- Response:
+
+```json
+{
+  "page_size": integer,
+  "count": integer,
+  "users": [
+    {
+      "deactivated": bool,
+      "id": integer,
+      "nickname": string,
+      "roles": {
+        "admin": bool,
+        "moderator": bool
+      },
+      "local": bool,
+      "tags": array,
+      "avatar": string,
+      "display_name": string
+    },
+    ...
+  ]
+}
+```
+
+## DEPRECATED `DELETE /api/pleroma/admin/users`
+
+### Remove a user
+
+- Params:
+  - `nickname`
+- Response: User’s nickname
+
+## `DELETE /api/pleroma/admin/users`
+
+### Remove a user
+
+- Params:
+  - `nicknames`
+- Response: Array of user nicknames
+
+### Create a user
+
+- Method: `POST`
+- Params:
+  `users`: [
+    {
+      `nickname`,
+      `email`,
+      `password`
+    }
+  ]
+- Response: User’s nickname
+
+## `POST /api/pleroma/admin/users/follow`
+
+### Make a user follow another user
+
+- Params:
+  - `follower`: The nickname of the follower
+  - `followed`: The nickname of the followed
+- Response:
+  - "ok"
+
+## `POST /api/pleroma/admin/users/unfollow`
+
+### Make a user unfollow another user
+
+- Params:
+  - `follower`: The nickname of the follower
+  - `followed`: The nickname of the followed
+- Response:
+  - "ok"
+
+## `PATCH /api/pleroma/admin/users/:nickname/toggle_activation`
+
+### Toggle user activation
+
+- Params:
+  - `nickname`
+- Response: User’s object
+
+```json
+{
+  "deactivated": bool,
+  "id": integer,
+  "nickname": string
+}
+```
+
+## `PUT /api/pleroma/admin/users/tag`
+
+### Tag a list of users
+
+- Params:
+  - `nicknames` (array)
+  - `tags` (array)
+
+## `DELETE /api/pleroma/admin/users/tag`
+
+### Untag a list of users
+
+- Params:
+  - `nicknames` (array)
+  - `tags` (array)
+
+## `GET /api/pleroma/admin/users/:nickname/permission_group`
+
+### Get user user permission groups membership
+
+- Params: none
+- Response:
+
+```json
+{
+  "is_moderator": bool,
+  "is_admin": bool
+}
+```
+
+## `GET /api/pleroma/admin/users/:nickname/permission_group/:permission_group`
+
+Note: Available `:permission_group` is currently moderator and admin. 404 is returned when the permission group doesn’t exist.
+
+### Get user user permission groups membership per permission group
+
+- Params: none
+- Response:
+
+```json
+{
+  "is_moderator": bool,
+  "is_admin": bool
+}
+```
+
+## DEPRECATED `POST /api/pleroma/admin/users/:nickname/permission_group/:permission_group`
+
+### Add user to permission group
+
+- Params: none
+- Response:
+  - On failure: `{"error": "…"}`
+  - On success: JSON of the user
+
+## `POST /api/pleroma/admin/users/permission_group/:permission_group`
+
+### Add users to permission group
+
+- Params:
+  - `nicknames`: nicknames array
+- Response:
+  - On failure: `{"error": "…"}`
+  - On success: JSON of the user
+
+## DEPRECATED `DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group`
+
+## `DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group`
+
+### Remove user from permission group
+
+- Params: none
+- Response:
+  - On failure: `{"error": "…"}`
+  - On success: JSON of the user
+- Note: An admin cannot revoke their own admin status.
+
+## `DELETE /api/pleroma/admin/users/permission_group/:permission_group`
+
+### Remove users from permission group
+
+- Params:
+  - `nicknames`: nicknames array
+- Response:
+  - On failure: `{"error": "…"}`
+  - On success: JSON of the user
+- Note: An admin cannot revoke their own admin status.
+
+## `PATCH /api/pleroma/admin/users/activate`
+
+### Activate user
+
+- Params:
+  - `nicknames`: nicknames array
+- Response:
+
+```json
+{
+  users: [
+    {
+      // user object
+    }
+  ]
+}
+```
+
+## `PATCH /api/pleroma/admin/users/deactivate`
+
+### Deactivate user
+
+- Params:
+  - `nicknames`: nicknames array
+- Response:
+
+```json
+{
+  users: [
+    {
+      // user object
+    }
+  ]
+}
+```
+
+## `GET /api/pleroma/admin/users/:nickname_or_id`
+
+### Retrive the details of a user
+
+- Params:
+  - `nickname` or `id`
+- Response:
+  - On failure: `Not found`
+  - On success: JSON of the user
+
+## `GET /api/pleroma/admin/users/:nickname_or_id/statuses`
+
+### Retrive user's latest statuses
+
+- Params:
+  - `nickname` or `id`
+  - *optional* `page_size`: number of statuses to return (default is `20`)
+  - *optional* `godmode`: `true`/`false` – allows to see private statuses
+  - *optional* `with_reblogs`: `true`/`false` – allows to see reblogs (default is false)
+- Response:
+  - On failure: `Not found`
+  - On success: JSON array of user's latest statuses
+
+## `GET /api/pleroma/admin/instances/:instance/statuses`
+
+### Retrive instance's latest statuses
+
+- Params:
+  - `instance`: instance name
+  - *optional* `page_size`: number of statuses to return (default is `20`)
+  - *optional* `godmode`: `true`/`false` – allows to see private statuses
+  - *optional* `with_reblogs`: `true`/`false` – allows to see reblogs (default is false)
+- Response:
+  - On failure: `Not found`
+  - On success: JSON array of instance's latest statuses
+
+## `GET /api/pleroma/admin/statuses`
+
+### Retrives all latest statuses
+
+- Params:
+  - *optional* `page_size`: number of statuses to return (default is `20`)
+  - *optional* `local_only`: excludes remote statuses
+  - *optional* `godmode`: `true`/`false` – allows to see private statuses
+  - *optional* `with_reblogs`: `true`/`false` – allows to see reblogs (default is false)
+- Response:
+  - On failure: `Not found`
+  - On success: JSON array of user's latest statuses
+
+## `POST /api/pleroma/admin/relay`
+
+### Follow a Relay
+
+- Params:
+  - `relay_url`
+- Response:
+  - On success: URL of the followed relay
+
+## `DELETE /api/pleroma/admin/relay`
+
+### Unfollow a Relay
+
+- Params:
+  - `relay_url`
+- Response:
+  - On success: URL of the unfollowed relay
+
+## `GET /api/pleroma/admin/relay`
+
+### List Relays
+
+- Params: none
+- Response:
+  - On success: JSON array of relays
+
+## `POST /api/pleroma/admin/users/invite_token`
+
+### Create an account registration invite token
+
+- Params:
+  - *optional* `max_use` (integer)
+  - *optional* `expires_at` (date string e.g. "2019-04-07")
+- Response:
+
+```json
+{
+  "id": integer,
+  "token": string,
+  "used": boolean,
+  "expires_at": date,
+  "uses": integer,
+  "max_use": integer,
+  "invite_type": string (possible values: `one_time`, `reusable`, `date_limited`, `reusable_date_limited`)
+}
+```
+
+## `GET /api/pleroma/admin/users/invites`
+
+### Get a list of generated invites
+
+- Params: none
+- Response:
+
+```json
+{
+
+  "invites": [
+    {
+      "id": integer,
+      "token": string,
+      "used": boolean,
+      "expires_at": date,
+      "uses": integer,
+      "max_use": integer,
+      "invite_type": string (possible values: `one_time`, `reusable`, `date_limited`, `reusable_date_limited`)
+    },
+    ...
+  ]
+}
+```
+
+## `POST /api/pleroma/admin/users/revoke_invite`
+
+### Revoke invite by token
+
+- Params:
+  - `token`
+- Response:
+
+```json
+{
+  "id": integer,
+  "token": string,
+  "used": boolean,
+  "expires_at": date,
+  "uses": integer,
+  "max_use": integer,
+  "invite_type": string (possible values: `one_time`, `reusable`, `date_limited`, `reusable_date_limited`)
+
+}
+```
+
+## `POST /api/pleroma/admin/users/email_invite`
+
+### Sends registration invite via email
+
+- Params:
+  - `email`
+  - `name`, optional
+
+## `GET /api/pleroma/admin/users/:nickname/password_reset`
+
+### Get a password reset token for a given nickname
+
+- Params: none
+- Response:
+
+```json
+{
+  "token": "base64 reset token",
+  "link": "https://pleroma.social/api/pleroma/password_reset/url-encoded-base64-token"
+}
+```
+
+## `PATCH /api/pleroma/admin/users/force_password_reset`
+
+### Force passord reset for a user with a given nickname
+
+- Params:
+  - `nicknames`
+- Response: none (code `204`)
+
+## `GET /api/pleroma/admin/reports`
+
+### Get a list of reports
+
+- Params:
+  - *optional* `state`: **string** the state of reports. Valid values are `open`, `closed` and `resolved`
+  - *optional* `limit`: **integer** the number of records to retrieve
+  - *optional* `page`: **integer** page number
+  - *optional* `page_size`: **integer** number of log entries per page (default is `50`)
+- Response:
+  - On failure: 403 Forbidden error `{"error": "error_msg"}` when requested by anonymous or non-admin
+  - On success: JSON, returns a list of reports, where:
+    - `account`: the user who has been reported
+    - `actor`: the user who has sent the report
+    - `statuses`: list of statuses that have been included to the report
+
+```json
+{
+  "totalReports" : 1,
+  "reports": [
+    {
+      "account": {
+        "acct": "user",
+        "avatar": "https://pleroma.example.org/images/avi.png",
+        "avatar_static": "https://pleroma.example.org/images/avi.png",
+        "bot": false,
+        "created_at": "2019-04-23T17:32:04.000Z",
+        "display_name": "User",
+        "emojis": [],
+        "fields": [],
+        "followers_count": 1,
+        "following_count": 1,
+        "header": "https://pleroma.example.org/images/banner.png",
+        "header_static": "https://pleroma.example.org/images/banner.png",
+        "id": "9i6dAJqSGSKMzLG2Lo",
+        "locked": false,
+        "note": "",
+        "pleroma": {
+          "confirmation_pending": false,
+          "hide_favorites": true,
+          "hide_followers": false,
+          "hide_follows": false,
+          "is_admin": false,
+          "is_moderator": false,
+          "relationship": {},
+          "tags": []
+        },
+        "source": {
+          "note": "",
+          "pleroma": {},
+          "sensitive": false
+        },
+        "tags": ["force_unlisted"],
+        "statuses_count": 3,
+        "url": "https://pleroma.example.org/users/user",
+        "username": "user"
+      },
+      "actor": {
+        "acct": "lain",
+        "avatar": "https://pleroma.example.org/images/avi.png",
+        "avatar_static": "https://pleroma.example.org/images/avi.png",
+        "bot": false,
+        "created_at": "2019-03-28T17:36:03.000Z",
+        "display_name": "Roger Braun",
+        "emojis": [],
+        "fields": [],
+        "followers_count": 1,
+        "following_count": 1,
+        "header": "https://pleroma.example.org/images/banner.png",
+        "header_static": "https://pleroma.example.org/images/banner.png",
+        "id": "9hEkA5JsvAdlSrocam",
+        "locked": false,
+        "note": "",
+        "pleroma": {
+          "confirmation_pending": false,
+          "hide_favorites": false,
+          "hide_followers": false,
+          "hide_follows": false,
+          "is_admin": false,
+          "is_moderator": false,
+          "relationship": {},
+          "tags": []
+        },
+        "source": {
+          "note": "",
+          "pleroma": {},
+          "sensitive": false
+        },
+        "tags": ["force_unlisted"],
+        "statuses_count": 1,
+        "url": "https://pleroma.example.org/users/lain",
+        "username": "lain"
+      },
+      "content": "Please delete it",
+      "created_at": "2019-04-29T19:48:15.000Z",
+      "id": "9iJGOv1j8hxuw19bcm",
+      "state": "open",
+      "statuses": [
+        {
+          "account": { ... },
+          "application": {
+            "name": "Web",
+            "website": null
+          },
+          "bookmarked": false,
+          "card": null,
+          "content": "<span class=\"h-card\"><a data-user=\"9hEkA5JsvAdlSrocam\" class=\"u-url mention\" href=\"https://pleroma.example.org/users/lain\">@<span>lain</span></a></span> click on my link <a href=\"https://www.google.com/\">https://www.google.com/</a>",
+          "created_at": "2019-04-23T19:15:47.000Z",
+          "emojis": [],
+          "favourited": false,
+          "favourites_count": 0,
+          "id": "9i6mQ9uVrrOmOime8m",
+          "in_reply_to_account_id": null,
+          "in_reply_to_id": null,
+          "language": null,
+          "media_attachments": [],
+          "mentions": [
+            {
+              "acct": "lain",
+              "id": "9hEkA5JsvAdlSrocam",
+              "url": "https://pleroma.example.org/users/lain",
+              "username": "lain"
+            },
+            {
+              "acct": "user",
+              "id": "9i6dAJqSGSKMzLG2Lo",
+              "url": "https://pleroma.example.org/users/user",
+              "username": "user"
+            }
+          ],
+          "muted": false,
+          "pinned": false,
+          "pleroma": {
+            "content": {
+              "text/plain": "@lain click on my link https://www.google.com/"
+            },
+            "conversation_id": 28,
+            "in_reply_to_account_acct": null,
+            "local": true,
+            "spoiler_text": {
+              "text/plain": ""
+            }
+          },
+          "reblog": null,
+          "reblogged": false,
+          "reblogs_count": 0,
+          "replies_count": 0,
+          "sensitive": false,
+          "spoiler_text": "",
+          "tags": [],
+          "uri": "https://pleroma.example.org/objects/8717b90f-8e09-4b58-97b0-e3305472b396",
+          "url": "https://pleroma.example.org/notice/9i6mQ9uVrrOmOime8m",
+          "visibility": "direct"
+        }
+      ]
+    }
+  ]
+}
+```
+
+## `GET /api/pleroma/admin/grouped_reports`
+
+### Get a list of reports, grouped by status
+
+- Params: none
+- On success: JSON, returns a list of reports, where:
+  - `date`: date of the latest report
+  - `account`: the user who has been reported (see `/api/pleroma/admin/reports` for reference)
+  - `status`: reported status (see `/api/pleroma/admin/reports` for reference)
+  - `actors`: users who had reported this status (see `/api/pleroma/admin/reports` for reference)
+  - `reports`: reports (see `/api/pleroma/admin/reports` for reference)
+
+```json
+  "reports": [
+    {
+      "date": "2019-10-07T12:31:39.615149Z",
+      "account": { ... },
+      "status": { ... },
+      "actors": [{ ... }, { ... }],
+      "reports": [{ ... }]
+    }
+  ]
+```
+
+## `GET /api/pleroma/admin/reports/:id`
+
+### Get an individual report
+
+- Params:
+  - `id`
+- Response:
+  - On failure:
+    - 403 Forbidden `{"error": "error_msg"}`
+    - 404 Not Found `"Not found"`
+  - On success: JSON, Report object (see above)
+
+## `PATCH /api/pleroma/admin/reports`
+
+### Change the state of one or multiple reports
+
+- Params:
+
+```json
+  `reports`: [
+    {
+      `id`, // required, report id
+      `state` // required, the new state. Valid values are `open`, `closed` and `resolved`
+    },
+    ...
+  ]
+```
+
+- Response:
+  - On failure:
+    - 400 Bad Request, JSON:
+
+    ```json
+      [
+        {
+          `id`, // report id
+          `error` // error message
+        }
+      ]
+    ```
+
+  - On success: `204`, empty response
+
+## `POST /api/pleroma/admin/reports/:id/notes`
+
+### Create report note
+
+- Params:
+  - `id`: required, report id
+  - `content`: required, the message
+- Response:
+  - On failure:
+    - 400 Bad Request `"Invalid parameters"` when `status` is missing
+  - On success: `204`, empty response
+
+## `POST /api/pleroma/admin/reports/:report_id/notes/:id`
+
+### Delete report note
+
+- Params:
+  - `report_id`: required, report id
+  - `id`: required, note id
+- Response:
+  - On failure:
+    - 400 Bad Request `"Invalid parameters"` when `status` is missing
+  - On success: `204`, empty response
+
+## `PUT /api/pleroma/admin/statuses/:id`
+
+### Change the scope of an individual reported status
+
+- Params:
+  - `id`
+  - `sensitive`: optional, valid values are `true` or `false`
+  - `visibility`: optional, valid values are `public`, `private` and `unlisted`
+- Response:
+  - On failure:
+    - 400 Bad Request `"Unsupported visibility"`
+    - 403 Forbidden `{"error": "error_msg"}`
+    - 404 Not Found `"Not found"`
+  - On success: JSON, Mastodon Status entity
+
+## `DELETE /api/pleroma/admin/statuses/:id`
+
+### Delete an individual reported status
+
+- Params:
+  - `id`
+- Response:
+  - On failure:
+    - 403 Forbidden `{"error": "error_msg"}`
+    - 404 Not Found `"Not found"`
+  - On success: 200 OK `{}`
+
+## `GET /api/pleroma/admin/restart`
+
+### Restarts pleroma application
+
+- Params: none
+- Response:
+  - On failure:
+    - 400 Bad Request `"To use this endpoint you need to enable configuration from database."`
+
+```json
+{}
+```
+
+## `GET /api/pleroma/admin/config`
+
+### Get list of merged default settings with saved in database.
+
+*If `need_reboot` flag exists in response, instance must be restarted, so reboot time settings can take effect.*
+
+**Only works when configuration from database is enabled.**
+
+- Params:
+  - `only_db`: true (*optional*, get only saved in database settings)
+- Response:
+  - On failure:
+    - 400 Bad Request `"To use this endpoint you need to enable configuration from database."`
+
+```json
+{
+  "configs": [
+    {
+      "group": ":pleroma",
+      "key": "Pleroma.Upload",
+      "value": []
+     }
+  ],
+  "need_reboot": true
+}
+```
+ need_reboot - *optional*, if were changed reboot time settings.
+
+## `POST /api/pleroma/admin/config`
+
+### Update config settings
+
+*If `need_reboot` flag exists in response, instance must be restarted, so reboot time settings can take effect.*
+
+**Only works when configuration from database is enabled.**
+
+Some modifications are necessary to save the config settings correctly:
+
+- strings which start with `Pleroma.`, `Phoenix.`, `Tesla.` or strings like `Oban`, `Ueberauth` will be converted to modules;
+```
+"Pleroma.Upload" -> Pleroma.Upload
+"Oban" -> Oban
+```
+- strings starting with `:` will be converted to atoms;
+```
+":pleroma" -> :pleroma
+```
+- objects with `tuple` key and array value will be converted to tuples;
+```
+{"tuple": ["string", "Pleroma.Upload", []]} -> {"string", Pleroma.Upload, []}
+```
+- arrays with *tuple objects* will be converted to keywords;
+```
+[{"tuple": [":key1", "value"]}, {"tuple": [":key2", "value"]}] -> [key1: "value", key2: "value"]
+```
+
+Most of the settings will be applied in `runtime`, this means that you don't need to restart the instance. But some settings are applied in `compile time` and require a reboot of the instance, such as:
+- all settings inside these keys:
+  - `:hackney_pools`
+  - `:chat`
+- partially settings inside these keys:
+  - `:seconds_valid` in `Pleroma.Captcha`
+  - `:proxy_remote` in `Pleroma.Upload`
+  - `:upload_limit` in `:instance`
+
+- Params:
+  - `configs` - array of config objects
+  - config object params:
+    - `group` - string (**required**)
+    - `key` - string (**required**)
+    - `value` - string, [], {} or {"tuple": []} (**required**)
+    - `delete` - true (*optional*, if setting must be deleted)
+    - `subkeys` - array of strings (*optional*, only works when `delete=true` parameter is passed, otherwise will be ignored)
+
+*When a value have several nested settings, you can delete only some nested settings by passing a parameter `subkeys`, without deleting all settings by key.*
+```
+[subkey: val1, subkey2: val2, subkey3: val3] \\ initial value
+{"group": ":pleroma", "key": "some_key", "delete": true, "subkeys": [":subkey", ":subkey3"]} \\ passing json for deletion
+[subkey2: val2] \\ value after deletion
+```
+
+*Most of the settings can be partially updated through merge old values with new values, except settings value of which is list or is not keyword.*
+
+Example of setting without keyword in value:
+```elixir
+config :tesla, :adapter, Tesla.Adapter.Hackney
+```
+
+List of settings which support only full update by key:
+```elixir
+@full_key_update [
+    {:pleroma, :ecto_repos},
+    {:quack, :meta},
+    {:mime, :types},
+    {:cors_plug, [:max_age, :methods, :expose, :headers]},
+    {:auto_linker, :opts},
+    {:swarm, :node_blacklist},
+    {:logger, :backends}
+  ]
+```
+
+List of settings which support only full update by subkey:
+```elixir
+@full_subkey_update [
+    {:pleroma, :assets, :mascots},
+    {:pleroma, :emoji, :groups},
+    {:pleroma, :workers, :retries},
+    {:pleroma, :mrf_subchain, :match_actor},
+    {:pleroma, :mrf_keyword, :replace}
+  ]
+```
+
+*Settings without explicit key must be sended in separate config object params.*
+```elixir
+config :quack,
+  level: :debug,
+  meta: [:all],
+  ...
+```
+```json
+{
+  "configs": [
+    {"group": ":quack", "key": ":level", "value": ":debug"},
+    {"group": ":quack", "key": ":meta", "value": [":all"]},
+    ...
+  ]
+}
+```
+- Request:
+
+```json
+{
+  "configs": [
+    {
+      "group": ":pleroma",
+      "key": "Pleroma.Upload",
+      "value": [
+        {"tuple": [":uploader", "Pleroma.Uploaders.Local"]},
+        {"tuple": [":filters", ["Pleroma.Upload.Filter.Dedupe"]]},
+        {"tuple": [":link_name", true]},
+        {"tuple": [":proxy_remote", false]},
+        {"tuple": [":proxy_opts", [
+          {"tuple": [":redirect_on_failure", false]},
+          {"tuple": [":max_body_length", 1048576]},
+          {"tuple": [":http", [
+            {"tuple": [":follow_redirect", true]},
+            {"tuple": [":pool", ":upload"]},
+          ]]}
+        ]
+        ]},
+        {"tuple": [":dispatch", {
+          "tuple": ["/api/v1/streaming", "Pleroma.Web.MastodonAPI.WebsocketHandler", []]
+        }]}
+      ]
+    }
+  ]
+}
+```
+
+- Response:
+  - On failure:
+    - 400 Bad Request `"To use this endpoint you need to enable configuration from database."`
+```json
+{
+  "configs": [
+    {
+      "group": ":pleroma",
+      "key": "Pleroma.Upload",
+      "value": [...]
+     }
+  ],
+  "need_reboot": true
+}
+```
+need_reboot - *optional*, if were changed reboot time settings.
+
+## ` GET /api/pleroma/admin/config/descriptions`
+
+### Get JSON with config descriptions.
+Loads json generated from `config/descriptions.exs`.
+
+- Params: none
+- Response:
+
+```json
+[{
+    "group": ":pleroma", // string
+    "key": "ModuleName", // string
+    "type": "group", // string or list with possible values,
+    "description": "Upload general settings", // string
+    "children": [
+      {
+        "key": ":uploader", // string or module name `Pleroma.Upload`
+        "type": "module",
+        "description": "Module which will be used for uploads",
+        "suggestions": ["module1", "module2"]
+      },
+      {
+        "key": ":filters",
+        "type": ["list", "module"],
+        "description": "List of filter modules for uploads",
+        "suggestions": [
+          "module1", "module2", "module3"
+        ]
+      }
+    ]
+}]
+```
+
+## `GET /api/pleroma/admin/moderation_log`
+
+### Get moderation log
+
+- Params:
+  - *optional* `page`: **integer** page number
+  - *optional* `page_size`: **integer** number of log entries per page (default is `50`)
+  - *optional* `start_date`: **datetime (ISO 8601)** filter logs by creation date, start from `start_date`. Accepts datetime in ISO 8601 format (YYYY-MM-DDThh:mm:ss), e.g. `2005-08-09T18:31:42`
+  - *optional* `end_date`: **datetime (ISO 8601)** filter logs by creation date, end by from `end_date`. Accepts datetime in ISO 8601 format (YYYY-MM-DDThh:mm:ss), e.g. 2005-08-09T18:31:42
+  - *optional* `user_id`: **integer** filter logs by actor's id
+  - *optional* `search`: **string** search logs by the log message
+- Response:
+
+```json
+[
+  {
+    "data": {
+      "actor": {
+        "id": 1,
+        "nickname": "lain"
+      },
+      "action": "relay_follow"
+    },
+    "time": 1502812026, // timestamp
+    "message": "[2017-08-15 15:47:06] @nick0 followed relay: https://example.org/relay" // log message
+  }
+]
+```
+
+## `POST /api/pleroma/admin/reload_emoji`
+
+### Reload the instance's custom emoji
+
+- Authentication: required
+- Params: None
+- Response: JSON, "ok" and 200 status
+
+## `PATCH /api/pleroma/admin/users/confirm_email`
+
+### Confirm users' emails
+
+- Params:
+  - `nicknames`
+- Response: Array of user nicknames
+
+## `PATCH /api/pleroma/admin/users/resend_confirmation_email`
+
+### Resend confirmation email
+
+- Params:
+  - `nicknames`
+- Response: Array of user nicknames
+
+## `GET /api/pleroma/admin/stats`
+
+### Stats
+
+- Response:
+
+```json
+{
+  "status_visibility": {
+    "direct": 739,
+    "private": 9,
+    "public": 17,
+    "unlisted": 14
+  }
+}
+```

docs/API/differences_in_mastoapi_responses.md

@@ -13,6 +13,7 @@ Some apps operate under the assumption that no more than 4 attachments can be re
 ## Timelines
 
 Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users.
+Adding the parameter `exclude_visibilities` to the timeline queries will exclude the statuses with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`), e.g., `exclude_visibilities[]=direct&exclude_visibilities[]=private`.
 
 ## Statuses
 
@@ -21,12 +22,14 @@ Adding the parameter `with_muted=true` to the timeline queries will also return
 Has these additional fields under the `pleroma` object:
 
 - `local`: true if the post was made on the local instance
-- `conversation_id`: the ID of the conversation the status is associated with (if any)
+- `conversation_id`: the ID of the AP context the status is associated with (if any)
+- `direct_conversation_id`: the ID of the Mastodon direct message conversation the status is associated with (if any)
 - `in_reply_to_account_acct`: the `acct` property of User entity for replied user (if any)
 - `content`: a map consisting of alternate representations of the `content` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`
 - `spoiler_text`: a map consisting of alternate representations of the `spoiler_text` property with the key being it's mimetype. Currently the only alternate representation supported is `text/plain`
 - `expires_at`: a datetime (iso8601) that states when the post will expire (be deleted automatically), or empty if the post won't expire
 - `thread_muted`: true if the thread the post belongs to is muted
+- `emoji_reactions`: A list with emoji / reaction maps. The format is `{name: "☕", count: 1, me: true}`. Contains no information about the reacting users, for that use the `/statuses/:id/reactions` endpoint.
 
 ## Attachments
 
@@ -44,15 +47,19 @@ The `id` parameter can also be the `nickname` of the user. This only works in th
 Has these additional fields under the `pleroma` object:
 
 - `tags`: Lists an array of tags for the user
-- `relationship{}`: Includes fields as documented for Mastodon API https://docs.joinmastodon.org/api/entities/#relationship
+- `relationship{}`: Includes fields as documented for Mastodon API https://docs.joinmastodon.org/entities/relationship/
 - `is_moderator`: boolean, nullable,  true if user is a moderator
 - `is_admin`: boolean, nullable, true if user is an admin
 - `confirmation_pending`: boolean, true if a new user account is waiting on email confirmation to be activated
 - `hide_followers`: boolean, true when the user has follower hiding enabled
 - `hide_follows`: boolean, true when the user has follow hiding enabled
+- `hide_followers_count`: boolean, true when the user has follower stat hiding enabled
+- `hide_follows_count`: boolean, true when the user has follow stat hiding enabled
 - `settings_store`: A generic map of settings for frontends. Opaque to the backend. Only returned in `verify_credentials` and `update_credentials`
 - `chat_token`: The token needed for Pleroma chat. Only returned in `verify_credentials`
 - `deactivated`: boolean, true when the user is deactivated
+- `allow_following_move`: boolean, true when the user allows automatically follow moved following accounts
+- `unread_conversation_count`: The count of unread conversations. Only returned to the account owner.
 
 ### Source
 
@@ -60,6 +67,8 @@ Has these additional fields under the `pleroma` object:
 
 - `show_role`: boolean, nullable, true when the user wants his role (e.g admin, moderator) to be shown
 - `no_rich_text` - boolean, nullable, true when html tags are stripped from all statuses requested from the API
+- `discoverable`: boolean, true when the user allows discovery of the account in search results and other services.
+- `actor_type`: string, the type of this account.
 
 ## Conversations
 
@@ -67,12 +76,21 @@ Has an additional field under the `pleroma` object:
 
 - `recipients`: The list of the recipients of this Conversation. These will be addressed when replying to this conversation.
 
+## GET `/api/v1/conversations`
+
+Accepts additional parameters:
+
+- `recipients`: Only return conversations with the given recipients (a list of user ids). Usage example: `GET /api/v1/conversations?recipients[]=1&recipients[]=2`
+
 ## Account Search
 
 Behavior has changed:
 
 - `/api/v1/accounts/search`: Does not require authentication
 
+## Search (global)
+
+Unlisted posts are available in search results, they are considered to be public posts that shouldn't be shown in local/federated timeline.
 
 ## Notifications
 
@@ -80,6 +98,27 @@ Has these additional fields under the `pleroma` object:
 
 - `is_seen`: true if the notification was read by the user
 
+### Move Notification
+
+The `type` value is `move`. Has an additional field:
+
+- `target`: new account
+
+### EmojiReact Notification
+
+The `type` value is `pleroma:emoji_reaction`. Has these fields:
+
+- `emoji`: The used emoji
+- `account`: The account of the user who reacted
+- `status`: The status that was reacted on
+
+## GET `/api/v1/notifications`
+
+Accepts additional parameters:
+
+- `exclude_visibilities`: will exclude the notifications for activities with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`). Usage example: `GET /api/v1/notifications?exclude_visibilities[]=direct&exclude_visibilities[]=private`.
+- `with_move`: boolean, when set to `true` will include Move notifications. `false` by default.
+
 ## POST `/api/v1/statuses`
 
 Additional parameters can be added to the JSON body/Form data:
@@ -112,12 +151,17 @@ Additional parameters can be added to the JSON body/Form data:
 - `no_rich_text` - if true, html tags are stripped from all statuses requested from the API
 - `hide_followers` - if true, user's followers will be hidden
 - `hide_follows` - if true, user's follows will be hidden
+- `hide_followers_count` - if true, user's follower count will be hidden
+- `hide_follows_count` - if true, user's follow count will be hidden
 - `hide_favorites` - if true, user's favorites timeline will be hidden
 - `show_role` - if true, user's role (e.g admin, moderator) will be exposed to anyone in the API
 - `default_scope` - the scope returned under `privacy` key in Source subentity
 - `pleroma_settings_store` - Opaque user settings to be saved on the backend.
 - `skip_thread_containment` - if true, skip filtering out broken threads
+- `allow_following_move` - if true, allows automatically follow moved following accounts
 - `pleroma_background_image` - sets the background image of the user.
+- `discoverable` - if true, discovery of this account in search results and other services is allowed.
+- `actor_type` - the type of this account.
 
 ### Pleroma Settings Store
 Pleroma has mechanism that allows frontends to save blobs of json for each user on the backend. This can be used to save frontend-specific settings for a user that the backend does not need to know about.

docs/API/pleroma_api.md

@@ -70,61 +70,8 @@ Request parameters can be passed via [query strings](https://en.wikipedia.org/wi
 * Response: JSON. Returns `{"status": "success"}` if the account was successfully disabled, `{"error": "[error message]"}` otherwise
 * Example response: `{"error": "Invalid password."}`
 
-## `/api/account/register`
-### Register a new user
-* Method `POST`
-* Authentication: not required
-* Params:
-    * `nickname`
-    * `fullname`
-    * `bio`
-    * `email`
-    * `password`
-    * `confirm`
-    * `captcha_solution`: optional, contains provider-specific captcha solution,
-    * `captcha_token`: optional, contains provider-specific captcha token
-    * `token`: invite token required when the registrations aren't public.
-* Response: JSON. Returns a user object on success, otherwise returns `{"error": "error_msg"}`
-* Example response:
-```json
-{
-	"background_image": null,
-	"cover_photo": "https://pleroma.soykaf.com/images/banner.png",
-	"created_at": "Tue Dec 18 16:55:56 +0000 2018",
-	"default_scope": "public",
-	"description": "blushy-crushy fediverse idol + pleroma dev\nlet's be friends \nぷれろまの生徒会長。謎の外人。日本語OK. \n公主病.",
-	"description_html": "blushy-crushy fediverse idol + pleroma dev.<br />let's be friends <br />ぷれろまの生徒会長。謎の外人。日本語OK. <br />公主病.",
-	"favourites_count": 0,
-	"fields": [],
-	"followers_count": 0,
-	"following": false,
-	"follows_you": false,
-	"friends_count": 0,
-	"id": 6,
-	"is_local": true,
-	"locked": false,
-	"name": "lain",
-	"name_html": "lain",
-	"no_rich_text": false,
-	"pleroma": {
-		"tags": []
-	},
-	"profile_image_url": "https://pleroma.soykaf.com/images/avi.png",
-	"profile_image_url_https": "https://pleroma.soykaf.com/images/avi.png",
-	"profile_image_url_original": "https://pleroma.soykaf.com/images/avi.png",
-	"profile_image_url_profile_size": "https://pleroma.soykaf.com/images/avi.png",
-	"rights": {
-		"delete_others_notice": false
-	},
-	"screen_name": "lain",
-	"statuses_count": 0,
-	"statusnet_blocking": false,
-	"statusnet_profile_url": "https://pleroma.soykaf.com/users/lain"
-}
-```
-
 ## `/api/pleroma/admin/`…
-See [Admin-API](Admin-API.md)
+See [Admin-API](admin_api.md)
 
 ## `/api/v1/pleroma/notifications/read`
 ### Mark notifications as read
@@ -302,6 +249,7 @@ See [Admin-API](Admin-API.md)
     * `follows`: BOOLEAN field, receives notifications from people the user follows
     * `remote`: BOOLEAN field, receives notifications from people on remote instances
     * `local`: BOOLEAN field, receives notifications from people on the local instance
+    * `privacy_option`: BOOLEAN field. When set to true, it removes the contents of a message from the push notification.
 * Response: JSON. Returns `{"status": "success"}` if the update was successful, otherwise returns `{"error": "error_msg"}`
 
 ## `/api/pleroma/healthcheck`
@@ -317,7 +265,8 @@ See [Admin-API](Admin-API.md)
   "active": 0, # active processes
   "idle": 0, # idle processes
   "memory_used": 0.00, # Memory used
-  "healthy": true # Instance state
+  "healthy": true, # Instance state
+  "job_queue_stats": {} # Job queue stats
 }
 ```
 
@@ -365,3 +314,161 @@ The status posting endpoint takes an additional parameter, `in_reply_to_conversa
 * Params:
     * `recipients`: A list of ids of users that should receive posts to this conversation. This will replace the current list of recipients, so submit the full list. The owner of owner of the conversation will always be part of the set of recipients, though.
 * Response: JSON, statuses (200 - healthy, 503 unhealthy)
+
+## `GET /api/v1/pleroma/conversations/read`
+### Marks all user's conversations as read.
+* Method `POST`
+* Authentication: required
+* Params: None
+* Response: JSON, returns a list of Mastodon Conversation entities that were marked as read (200 - healthy, 503 unhealthy).
+
+## `GET /api/pleroma/emoji/packs`
+### Lists the custom emoji packs on the server
+* Method `GET`
+* Authentication: not required
+* Params: None
+* Response: JSON, "ok" and 200 status and the JSON hashmap of "pack name" to "pack contents"
+
+## `PUT /api/pleroma/emoji/packs/:name`
+### Creates an empty custom emoji pack
+* Method `PUT`
+* Authentication: required
+* Params: None
+* Response: JSON, "ok" and 200 status or 409 if the pack with that name already exists
+
+## `DELETE /api/pleroma/emoji/packs/:name`
+### Delete a custom emoji pack
+* Method `DELETE`
+* Authentication: required
+* Params: None
+* Response: JSON, "ok" and 200 status or 500 if there was an error deleting the pack
+
+## `POST /api/pleroma/emoji/packs/:name/update_file`
+### Update a file in a custom emoji pack
+* Method `POST`
+* Authentication: required
+* Params:
+    * if the `action` is `add`, adds an emoji named `shortcode` to the pack `pack_name`,
+      that means that the emoji file needs to be uploaded with the request
+      (thus requiring it to be a multipart request) and be named `file`.
+      There can also be an optional `filename` that will be the new emoji file name
+      (if it's not there, the name will be taken from the uploaded file).
+    * if the `action` is `update`, changes emoji shortcode
+      (from `shortcode` to `new_shortcode` or moves the file (from the current filename to `new_filename`)
+    * if the `action` is `remove`, removes the emoji named `shortcode` and it's associated file
+* Response: JSON, updated "files" section of the pack and 200 status, 409 if the trying to use a shortcode
+  that is already taken, 400 if there was an error with the shortcode, filename or file (additional info
+  in the "error" part of the response JSON)
+
+## `POST /api/pleroma/emoji/packs/:name/update_metadata`
+### Updates (replaces) pack metadata
+* Method `POST`
+* Authentication: required
+* Params:
+  * `new_data`: new metadata to replace the old one
+* Response: JSON, updated "metadata" section of the pack and 200 status or 400 if there was a
+  problem with the new metadata (the error is specified in the "error" part of the response JSON)
+
+## `POST /api/pleroma/emoji/packs/download_from`
+### Requests the instance to download the pack from another instance
+* Method `POST`
+* Authentication: required
+* Params:
+  * `instance_address`: the address of the instance to download from
+  * `pack_name`: the pack to download from that instance
+* Response: JSON, "ok" and 200 status if the pack was downloaded, or 500 if there were
+  errors downloading the pack
+
+## `POST /api/pleroma/emoji/packs/list_from`
+### Requests the instance to list the packs from another instance
+* Method `POST`
+* Authentication: required
+* Params:
+  * `instance_address`: the address of the instance to download from
+* Response: JSON with the pack list, same as if the request was made to that instance's
+  list endpoint directly + 200 status
+
+## `GET /api/pleroma/emoji/packs/:name/download_shared`
+### Requests a local pack from the instance
+* Method `GET`
+* Authentication: not required
+* Params: None
+* Response: the archive of the pack with a 200 status code, 403 if the pack is not set as shared,
+  404 if the pack does not exist
+
+## `GET /api/v1/pleroma/accounts/:id/scrobbles`
+### Requests a list of current and recent Listen activities for an account
+* Method `GET`
+* Authentication: not required
+* Params: None
+* Response: An array of media metadata entities.
+* Example response:
+```json
+[
+   {
+       "account": {...},
+       "id": "1234",
+       "title": "Some Title",
+       "artist": "Some Artist",
+       "album": "Some Album",
+       "length": 180000,
+       "created_at": "2019-09-28T12:40:45.000Z"
+   }
+]
+```
+
+## `POST /api/v1/pleroma/scrobble`
+### Creates a new Listen activity for an account
+* Method `POST`
+* Authentication: required
+* Params:
+  * `title`: the title of the media playing
+  * `album`: the album of the media playing [optional]
+  * `artist`: the artist of the media playing [optional]
+  * `length`: the length of the media playing [optional]
+* Response: the newly created media metadata entity representing the Listen activity
+
+# Emoji Reactions
+
+Emoji reactions work a lot like favourites do. They make it possible to react to a post with a single emoji character.
+
+## `PUT /api/v1/pleroma/statuses/:id/reactions/:emoji`
+### React to a post with a unicode emoji
+* Method: `PUT`
+* Authentication: required
+* Params: `emoji`: A single character unicode emoji
+* Response: JSON, the status.
+
+## `DELETE /api/v1/pleroma/statuses/:id/reactions/:emoji`
+### Remove a reaction to a post with a unicode emoji
+* Method: `DELETE`
+* Authentication: required
+* Params: `emoji`: A single character unicode emoji
+* Response: JSON, the status.
+
+## `GET /api/v1/pleroma/statuses/:id/reactions`
+### Get an object of emoji to account mappings with accounts that reacted to the post
+* Method: `GET`
+* Authentication: optional
+* Params: None
+* Response: JSON, a list of emoji/account list tuples, sorted by emoji insertion date, in ascending order, e.g, the first emoji in the list is the oldest.
+* Example Response:
+```json
+[
+  {"name": "😀", "count": 2, "me": true, "accounts": [{"id" => "xyz.."...}, {"id" => "zyx..."}]},
+  {"name": "☕", "count": 1, "me": false, "accounts": [{"id" => "abc..."}]}
+]
+```
+
+## `GET /api/v1/pleroma/statuses/:id/reactions/:emoji`
+### Get an object of emoji to account mappings with accounts that reacted to the post for a specific emoji`
+* Method: `GET`
+* Authentication: optional
+* Params: None
+* Response: JSON, a list of emoji/account list tuples
+* Example Response:
+```json
+[
+  {"name": "😀", "count": 2, "me": true, "accounts": [{"id" => "xyz.."...}, {"id" => "zyx..."}]}
+]
+```

docs/admin/backup.md

@@ -1,17 +0,0 @@
-# Backup/Restore your instance
-
-## Backup
-
-1. Stop the Pleroma service.
-2. Go to the working directory of Pleroma (default is `/opt/pleroma`)
-3. Run `sudo -Hu postgres pg_dump -d <pleroma_db> --format=custom -f </path/to/backup_location/pleroma.pgdump>`
-4. Copy `pleroma.pgdump`, `config/prod.secret.exs` and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too.
-5. Restart the Pleroma service.
-
-## Restore
-
-1. Stop the Pleroma service.
-2. Go to the working directory of Pleroma (default is `/opt/pleroma`)
-3. Copy the above mentioned files back to their original position.
-4. Run `sudo -Hu postgres pg_restore -d <pleroma_db> -v -1 </path/to/backup_location/pleroma.pgdump>`
-5. Restart the Pleroma service.

docs/admin/updating.md

@@ -1,9 +0,0 @@
-# Updating your instance
-1. Go to the working directory of Pleroma (default is `/opt/pleroma`)
-2. Run `git pull`. This pulls the latest changes from upstream.
-3. Run `mix deps.get`. This pulls in any new dependencies.
-4. Stop the Pleroma service.
-5. Run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
-6. Start the Pleroma service.
-
-[^1]: Prefix with `MIX_ENV=prod` to run it using the production config file.

docs/administration/CLI_tasks/config.md

@@ -0,0 +1,40 @@
+# Transfering the config to/from the database
+
+{! backend/administration/CLI_tasks/general_cli_task_info.include !}
+
+## Transfer config from file to DB.
+
+!!! note
+    You need to add the following to your config before executing this command:
+
+    ```elixir
+    config :pleroma, configurable_from_database: true
+    ```
+
+```sh tab="OTP"
+ ./bin/pleroma_ctl config migrate_to_db
+```
+
+```sh tab="From Source"
+mix pleroma.config migrate_to_db
+```
+
+
+## Transfer config from DB to `config/env.exported_from_db.secret.exs`
+
+!!! note
+    In-Database configuration will still be applied after executing this command unless you set the following in your config:
+
+    ```elixir
+    config :pleroma, configurable_from_database: false
+    ```
+
+To delete transfered settings from database optional flag `-d` can be used. `<env>` is `prod` by default.
+
+```sh tab="OTP"
+ ./bin/pleroma_ctl config migrate_from_db [--env=<env>] [-d]
+```
+
+```sh tab="From Source"
+mix pleroma.config migrate_from_db [--env=<env>] [-d]
+```

docs/administration/CLI_tasks/database.md

@@ -0,0 +1,71 @@
+# Database maintenance tasks
+
+{! backend/administration/CLI_tasks/general_cli_task_info.include !}
+
+!!! danger
+    These mix tasks can take a long time to complete. Many of them were written to address specific database issues that happened because of bugs in migrations or other specific scenarios. Do not run these tasks "just in case" if everything is fine your instance.
+
+## Replace embedded objects with their references
+
+Replaces embedded objects with references to them in the `objects` table. Only needs to be ran once if the instance was created before Pleroma 1.0.5. The reason why this is not a migration is because it could significantly increase the database size after being ran, however after this `VACUUM FULL` will be able to reclaim about 20% (really depends on what is in the database, your mileage may vary) of the db size before the migration.
+
+```sh tab="OTP"
+./bin/pleroma_ctl database remove_embedded_objects [<options>]
+```
+
+```sh tab="From Source"
+mix pleroma.database remove_embedded_objects [<options>]
+```
+
+### Options
+- `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
+
+## Prune old remote posts from the database
+
+This will prune remote posts older than 90 days (configurable with [`config :pleroma, :instance, remote_post_retention_days`](../../configuration/cheatsheet.md#instance)) from the database, they will be refetched from source when accessed.
+
+!!! danger
+    The disk space will only be reclaimed after `VACUUM FULL`. You may run out of disk space during the execution of the task or vacuuming if you don't have about 1/3rds of the database size free.
+
+```sh tab="OTP"
+./bin/pleroma_ctl database prune_objects [<options>]
+```
+
+```sh tab="From Source"
+mix pleroma.database prune_objects [<options>]
+```
+
+### Options
+- `--vacuum` - run `VACUUM FULL` after the objects are pruned
+
+## Create a conversation for all existing DMs
+
+Can be safely re-run
+
+```sh tab="OTP"
+./bin/pleroma_ctl database bump_all_conversations
+```
+
+```sh tab="From Source"
+mix pleroma.database bump_all_conversations
+```
+
+## Remove duplicated items from following and update followers count for all users
+
+```sh tab="OTP"
+./bin/pleroma_ctl database update_users_following_followers_counts
+```
+
+```sh tab="From Source"
+mix pleroma.database update_users_following_followers_counts
+```
+
+## Fix the pre-existing "likes" collections for all objects
+
+```sh tab="OTP"
+./bin/pleroma_ctl database fix_likes_collections
+```
+
+```sh tab="From Source"
+mix pleroma.database fix_likes_collections
+```

docs/administration/CLI_tasks/digest.md

@@ -0,0 +1,25 @@
+# Managing digest emails
+
+{! backend/administration/CLI_tasks/general_cli_task_info.include !}
+
+## Send digest email since given date (user registration date by default) ignoring user activity status.
+
+```sh tab="OTP"
+ ./bin/pleroma_ctl digest test <nickname> [<since_date>]
+```
+
+```sh tab="From Source"
+mix pleroma.digest test <nickname> [<since_date>]
+```
+
+
+Example: 
+
+```sh tab="OTP"
+./bin/pleroma_ctl digest test donaldtheduck 2019-05-20
+```
+
+```sh tab="From Source"
+mix pleroma.digest test donaldtheduck 2019-05-20
+```
+

docs/administration/CLI_tasks/email.md

@@ -0,0 +1,24 @@
+# Managing emails
+
+{! backend/administration/CLI_tasks/general_cli_task_info.include !}
+
+## Send test email (instance email by default)
+
+```sh tab="OTP"
+ ./bin/pleroma_ctl email test [--to <destination email address>]
+```
+
+```sh tab="From Source"
+mix pleroma.email test [--to <destination email address>]
+```
+
+
+Example: 
+
+```sh tab="OTP"
+./bin/pleroma_ctl email test --to root@example.org
+```
+
+```sh tab="From Source"
+mix pleroma.email test --to root@example.org
+```

docs/administration/CLI_tasks/emoji.md

@@ -0,0 +1,46 @@
+# Managing emoji packs
+
+{! backend/administration/CLI_tasks/general_cli_task_info.include !}
+
+## Lists emoji packs and metadata specified in the manifest
+
+```sh tab="OTP"
+./bin/pleroma_ctl emoji ls-packs [<options>]
+```
+
+```sh tab="From Source"
+mix pleroma.emoji ls-packs [<options>]
+```
+
+
+### Options
+- `-m, --manifest PATH/URL` - path to a custom manifest, it can either be an URL starting with `http`, in that case the manifest will be fetched from that address, or a local path
+
+## Fetch, verify and install the specified packs from the manifest into `STATIC-DIR/emoji/PACK-NAME`
+
+```sh tab="OTP"
+./bin/pleroma_ctl emoji get-packs [<options>] <packs>
+```
+
+```sh tab="From Source"
+mix pleroma.emoji get-packs [<options>] <packs>
+```
+
+### Options
+- `-m, --manifest PATH/URL` - same as [`ls-packs`](#ls-packs)
+
+## Create a new manifest entry and a file list from the specified remote pack file
+
+```sh tab="OTP"
+./bin/pleroma_ctl emoji gen-pack PACK-URL
+```
+
+```sh tab="From Source"
+mix pleroma.emoji gen-pack PACK-URL
+```
+
+Currently, only .zip archives are recognized as remote pack files and packs are therefore assumed to be zip archives. This command is intended to run interactively and will first ask you some basic questions about the pack, then download the remote file and generate an SHA256 checksum for it, then generate an emoji file list for you. 
+
+  The manifest entry will either be written to a newly created `index.json` file or appended to the existing one, *replacing* the old pack with the same name if it was in the file previously.
+
+  The file list will be written to the file specified previously, *replacing* that file. You _should_ check that the file list doesn't contain anything you don't need in the pack, that is, anything that is not an emoji (the whole pack is downloaded, but only emoji files are extracted).

docs/administration/CLI_tasks/general_cli_task_info.include

@@ -0,0 +1,5 @@
+Every command should be ran as the `pleroma` user from it's home directory. For example if you are superuser, you would have to wrap the command in `su pleroma -s $SHELL -lc "$COMMAND"`.
+
+??? note "From source note about `MIX_ENV`"
+
+     The `mix` command should be prefixed with the name of environment your Pleroma server is running in, usually it's `MIX_ENV=prod`

docs/administration/CLI_tasks/instance.md

@@ -0,0 +1,35 @@
+# Managing instance configuration
+
+{! backend/administration/CLI_tasks/general_cli_task_info.include !}
+
+## Generate a new configuration file
+```sh tab="OTP"
+ ./bin/pleroma_ctl instance gen [<options>]
+```
+
+```sh tab="From Source"
+mix pleroma.instance gen [<options>]
+```
+
+
+If any of the options are left unspecified, you will be prompted interactively.
+
+### Options
+- `-f`, `--force` - overwrite any output files
+- `-o <path>`, `--output <path>` - the output file for the generated configuration
+- `--output-psql <path>` - the output file for the generated PostgreSQL setup
+- `--domain <domain>` - the domain of your instance
+- `--instance-name <instance_name>` - the name of your instance
+- `--admin-email <email>` - the email address of the instance admin
+- `--notify-email <email>` - email address for notifications
+- `--dbhost <hostname>` - the hostname of the PostgreSQL database to use
+- `--dbname <database_name>` - the name of the database to use
+- `--dbuser <username>` - the user (aka role) to use for the database connection
+- `--dbpass <password>` - the password to use for the database connection
+- `--rum <Y|N>` - Whether to enable RUM indexes
+- `--indexable <Y|N>` - Allow/disallow indexing site by search engines
+- `--db-configurable <Y|N>` - Allow/disallow configuring instance from admin part
+- `--uploads-dir <path>` - the directory uploads go in when using a local uploader
+- `--static-dir <path>` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)
+- `--listen-ip <ip>` - the ip the app should listen to, defaults to 127.0.0.1
+- `--listen-port <port>` - the port the app should listen to, defaults to 4000

docs/administration/CLI_tasks/relay.md

@@ -0,0 +1,33 @@
+# Managing relays
+
+{! backend/administration/CLI_tasks/general_cli_task_info.include !}
+
+## Follow a relay
+
+```sh tab="OTP"
+./bin/pleroma_ctl relay follow <relay_url>
+```
+
+```sh tab="From Source"
+mix pleroma.relay follow <relay_url>
+```
+
+## Unfollow a remote relay
+
+```sh tab="OTP"
+./bin/pleroma_ctl relay unfollow <relay_url>
+```
+
+```sh tab="From Source"
+mix pleroma.relay unfollow <relay_url>
+```
+
+## List relay subscriptions
+
+```sh tab="OTP"
+./bin/pleroma_ctl relay list
+```
+
+```sh tab="From Source"
+mix pleroma.relay list
+```

docs/administration/CLI_tasks/uploads.md

@@ -0,0 +1,17 @@
+# Managing uploads
+
+{! backend/administration/CLI_tasks/general_cli_task_info.include !}
+
+## Migrate uploads from local to remote storage
+```sh tab="OTP"
+ ./bin/pleroma_ctl uploads migrate_local <target_uploader> [<options>]
+```
+
+```sh tab="From Source"
+mix pleroma.uploads migrate_local <target_uploader> [<options>]
+```
+
+### Options
+- `--delete` - delete local uploads after migrating them to the target uploader
+
+A list of available uploaders can be seen in [Configuration Cheat Sheet](../../configuration/cheatsheet.md#pleromaupload)

docs/administration/CLI_tasks/user.md

@@ -0,0 +1,180 @@
+# Managing users
+
+{! backend/administration/CLI_tasks/general_cli_task_info.include !}
+
+## Create a user
+
+```sh tab="OTP"
+./bin/pleroma_ctl user new <email> [<options>]
+```
+
+```sh tab="From Source"
+mix pleroma.user new <email> [<options>]
+```
+
+
+### Options
+- `--name <name>` - the user's display name
+- `--bio <bio>` - the user's bio
+- `--password <password>` - the user's password
+- `--moderator`/`--no-moderator` - whether the user should be a moderator
+- `--admin`/`--no-admin` - whether the user should be an admin
+- `-y`, `--assume-yes`/`--no-assume-yes` - whether to assume yes to all questions
+
+## List local users
+```sh tab="OTP"
+ ./bin/pleroma_ctl user list
+```
+
+```sh tab="From Source"
+mix pleroma.user list
+```
+
+
+## Generate an invite link
+```sh tab="OTP"
+ ./bin/pleroma_ctl user invite [<options>]
+```
+
+```sh tab="From Source"
+mix pleroma.user invite [<options>]
+```
+
+
+### Options
+- `--expires-at DATE` - last day on which token is active (e.g. "2019-04-05")
+- `--max-use NUMBER` - maximum numbers of token uses
+
+## List generated invites
+```sh tab="OTP"
+ ./bin/pleroma_ctl user invites
+```
+
+```sh tab="From Source"
+mix pleroma.user invites
+```
+
+
+## Revoke invite
+```sh tab="OTP"
+ ./bin/pleroma_ctl user revoke_invite <token_or_id>
+```
+
+```sh tab="From Source"
+mix pleroma.user revoke_invite <token_or_id>
+```
+
+
+## Delete a user
+```sh tab="OTP"
+ ./bin/pleroma_ctl user rm <nickname>
+```
+
+```sh tab="From Source"
+mix pleroma.user rm <nickname>
+```
+
+
+## Delete user's posts and interactions
+```sh tab="OTP"
+ ./bin/pleroma_ctl user delete_activities <nickname>
+```
+
+```sh tab="From Source"
+mix pleroma.user delete_activities <nickname>
+```
+
+
+## Sign user out from all applications (delete user's OAuth tokens and authorizations)
+```sh tab="OTP"
+ ./bin/pleroma_ctl user sign_out <nickname>
+```
+
+```sh tab="From Source"
+mix pleroma.user sign_out <nickname>
+```
+
+
+## Deactivate or activate a user 
+```sh tab="OTP"
+ ./bin/pleroma_ctl user toggle_activated <nickname> 
+```
+
+```sh tab="From Source"
+mix pleroma.user toggle_activated <nickname> 
+```
+
+
+## Unsubscribe local users from a user and deactivate the user
+```sh tab="OTP"
+ ./bin/pleroma_ctl user unsubscribe NICKNAME
+```
+
+```sh tab="From Source"
+mix pleroma.user unsubscribe NICKNAME
+```
+
+
+## Unsubscribe local users from an instance and deactivate all accounts on it
+```sh tab="OTP"
+ ./bin/pleroma_ctl user unsubscribe_all_from_instance <instance>
+```
+
+```sh tab="From Source"
+mix pleroma.user unsubscribe_all_from_instance <instance>
+```
+
+
+## Create a password reset link for user
+```sh tab="OTP"
+ ./bin/pleroma_ctl user reset_password <nickname>
+```
+
+```sh tab="From Source"
+mix pleroma.user reset_password <nickname>
+```
+
+
+## Set the value of the given user's settings
+```sh tab="OTP"
+ ./bin/pleroma_ctl user set <nickname> [<options>]
+```
+
+```sh tab="From Source"
+mix pleroma.user set <nickname> [<options>]
+```
+
+### Options
+- `--locked`/`--no-locked` - whether the user should be locked
+- `--moderator`/`--no-moderator` - whether the user should be a moderator
+- `--admin`/`--no-admin` - whether the user should be an admin
+
+## Add tags to a user
+```sh tab="OTP"
+ ./bin/pleroma_ctl user tag <nickname> <tags>
+```
+
+```sh tab="From Source"
+mix pleroma.user tag <nickname> <tags>
+```
+
+
+## Delete tags from a user
+```sh tab="OTP"
+ ./bin/pleroma_ctl user untag <nickname> <tags>
+```
+
+```sh tab="From Source"
+mix pleroma.user untag <nickname> <tags>
+```
+
+
+## Toggle confirmation status of the user
+```sh tab="OTP"
+ ./bin/pleroma_ctl user toggle_confirmed <nickname>
+```
+
+```sh tab="From Source"
+mix pleroma.user toggle_confirmed <nickname>
+```
+

docs/administration/backup.md

@@ -0,0 +1,37 @@
+# Backup/Restore/Move/Remove your instance
+
+## Backup
+
+1. Stop the Pleroma service.
+2. Go to the working directory of Pleroma (default is `/opt/pleroma`)
+3. Run `sudo -Hu postgres pg_dump -d <pleroma_db> --format=custom -f </path/to/backup_location/pleroma.pgdump>` (make sure the postgres user has write access to the destination file)
+4. Copy `pleroma.pgdump`, `config/prod.secret.exs` and the `uploads` folder to your backup destination. If you have other modifications, copy those changes too.
+5. Restart the Pleroma service.
+
+## Restore/Move
+
+1. Optionally reinstall Pleroma (either on the same server or on another server if you want to move servers). Try to use the same database name.
+2. Stop the Pleroma service.
+3. Go to the working directory of Pleroma (default is `/opt/pleroma`)
+4. Copy the above mentioned files back to their original position.
+5. Drop the existing database and recreate an empty one `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;';` `sudo -Hu postgres psql -c 'CREATE DATABASE <pleroma_db>;';`
+6. Run `sudo -Hu postgres pg_restore -d <pleroma_db> -v -1 </path/to/backup_location/pleroma.pgdump>`
+7. If you installed a newer Pleroma version, you should run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
+8. Restart the Pleroma service.
+9. After you've restarted Pleroma, you will notice that postgres will take up more cpu resources than usual. A lot in fact. To fix this you must do a VACUUM ANLAYZE. This can also be done while the instance is still running like so: 
+   $ sudo -u postgres psql pleroma_database_name
+   pleroma=# VACUUM ANALYZE;
+[^1]: Prefix with `MIX_ENV=prod` to run it using the production config file.
+
+## Remove
+
+1. Optionally you can remove the users of your instance. This will trigger delete requests for their accounts and posts. Note that this is 'best effort' and doesn't mean that all traces of your instance will be gone from the fediverse.
+    * You can do this from the admin-FE where you can select all local users and delete the accounts using the *Moderate multiple users* dropdown.
+    * You can also list local users and delete them individualy using the CLI tasks for [Managing users](./CLI_tasks/user.md).
+2. Stop the Pleroma service `systemctl stop pleroma`
+3. Disable pleroma from systemd `systemctl disable pleroma`
+4. Remove the files and folders you created during installation (see installation guide). This includes the pleroma, nginx and systemd files and folders.
+5. Reload nginx now that the configuration is removed `systemctl reload nginx`
+6. Remove the database and database user `sudo -Hu postgres psql -c 'DROP DATABASE <pleroma_db>;';` `sudo -Hu postgres psql -c 'DROP USER <pleroma_db>;';`
+7. Remove the system user `userdel pleroma`
+8. Remove the dependencies that you don't need anymore (see installation guide). Make sure you don't remove packages that are still needed for other software that you have running!

docs/administration/updating.md

@@ -0,0 +1,26 @@
+# Updating your instance
+
+You should **always check the release notes/changelog** in case there are config deprecations, special update special update steps, etc.
+
+Besides that, doing the following is generally enough:
+
+## For OTP installations
+
+```sh
+# Download the new release
+su pleroma -s $SHELL -lc "./bin/pleroma_ctl update"
+
+# Migrate the database, you are advised to stop the instance before doing that
+su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
+```
+
+## For from source installations (using git)
+
+1. Go to the working directory of Pleroma (default is `/opt/pleroma`)
+2. Run `git pull`. This pulls the latest changes from upstream.
+3. Run `mix deps.get`. This pulls in any new dependencies.
+4. Stop the Pleroma service.
+5. Run `mix ecto.migrate`[^1]. This task performs database migrations, if there were any.
+6. Start the Pleroma service.
+
+[^1]: Prefix with `MIX_ENV=prod` to run it using the production config file.

docs/api/admin_api.md

@@ -1,724 +0,0 @@
-# Admin API
-
-Authentication is required and the user must be an admin.
-
-## `/api/pleroma/admin/users`
-
-### List users
-
-- Method `GET`
-- Query Params:
-  - *optional* `query`: **string** search term (e.g. nickname, domain, nickname@domain)
-  - *optional* `filters`: **string** comma-separated string of filters:
-    - `local`: only local users
-    - `external`: only external users
-    - `active`: only active users
-    - `deactivated`: only deactivated users
-    - `is_admin`: users with admin role
-    - `is_moderator`: users with moderator role
-  - *optional* `page`: **integer** page number
-  - *optional* `page_size`: **integer** number of users per page (default is `50`)
-  - *optional* `tags`: **[string]** tags list
-  - *optional* `name`: **string** user display name
-  - *optional* `email`: **string** user email
-- Example: `https://mypleroma.org/api/pleroma/admin/users?query=john&filters=local,active&page=1&page_size=10&tags[]=some_tag&tags[]=another_tag&name=display_name&email=email@example.com`
-- Response:
-
-```json
-{
-  "page_size": integer,
-  "count": integer,
-  "users": [
-    {
-      "deactivated": bool,
-      "id": integer,
-      "nickname": string,
-      "roles": {
-        "admin": bool,
-        "moderator": bool
-      },
-      "local": bool,
-      "tags": array,
-      "avatar": string,
-      "display_name": string
-    },
-    ...
-  ]
-}
-```
-
-## `/api/pleroma/admin/users`
-
-### Remove a user
-
-- Method `DELETE`
-- Params:
-  - `nickname`
-- Response: User’s nickname
-
-### Create a user
-
-- Method: `POST`
-- Params:
-  `users`: [
-    {
-      `nickname`,
-      `email`,
-      `password`
-    }
-  ]
-- Response: User’s nickname
-
-## `/api/pleroma/admin/users/follow`
-### Make a user follow another user
-
-- Methods: `POST`
-- Params:
- - `follower`: The nickname of the follower
- - `followed`: The nickname of the followed
-- Response:
- - "ok"
-
-## `/api/pleroma/admin/users/unfollow`
-### Make a user unfollow another user
-
-- Methods: `POST`
-- Params:
- - `follower`: The nickname of the follower
- - `followed`: The nickname of the followed
-- Response:
- - "ok"
-
-## `/api/pleroma/admin/users/:nickname/toggle_activation`
-
-### Toggle user activation
-
-- Method: `PATCH`
-- Params:
-  - `nickname`
-- Response: User’s object
-
-```json
-{
-  "deactivated": bool,
-  "id": integer,
-  "nickname": string
-}
-```
-
-## `/api/pleroma/admin/users/tag`
-
-### Tag a list of users
-
-- Method: `PUT`
-- Params:
-  - `nicknames` (array)
-  - `tags` (array)
-
-### Untag a list of users
-
-- Method: `DELETE`
-- Params:
-  - `nicknames` (array)
-  - `tags` (array)
-
-## `/api/pleroma/admin/users/:nickname/permission_group`
-
-### Get user user permission groups membership
-
-- Method: `GET`
-- Params: none
-- Response:
-
-```json
-{
-  "is_moderator": bool,
-  "is_admin": bool
-}
-```
-
-## `/api/pleroma/admin/users/:nickname/permission_group/:permission_group`
-
-Note: Available `:permission_group` is currently moderator and admin. 404 is returned when the permission group doesn’t exist.
-
-### Get user user permission groups membership per permission group
-
-- Method: `GET`
-- Params: none
-- Response:
-
-```json
-{
-  "is_moderator": bool,
-  "is_admin": bool
-}
-```
-
-### Add user in permission group
-
-- Method: `POST`
-- Params: none
-- Response:
-  - On failure: `{"error": "…"}`
-  - On success: JSON of the `user.info`
-
-### Remove user from permission group
-
-- Method: `DELETE`
-- Params: none
-- Response:
-  - On failure: `{"error": "…"}`
-  - On success: JSON of the `user.info`
-- Note: An admin cannot revoke their own admin status.
-
-## `/api/pleroma/admin/users/:nickname/activation_status`
-
-### Active or deactivate a user
-
-- Method: `PUT`
-- Params:
-  - `nickname`
-  - `status` BOOLEAN field, false value means deactivation.
-
-## `/api/pleroma/admin/users/:nickname_or_id`
-
-### Retrive the details of a user
-
-- Method: `GET`
-- Params:
-  - `nickname` or `id`
-- Response:
-  - On failure: `Not found`
-  - On success: JSON of the user
-
-## `/api/pleroma/admin/users/:nickname_or_id/statuses`
-
-### Retrive user's latest statuses
-
-- Method: `GET`
-- Params:
-  - `nickname` or `id`
-  - *optional* `page_size`: number of statuses to return (default is `20`)
-  - *optional* `godmode`: `true`/`false` – allows to see private statuses
-- Response:
-  - On failure: `Not found`
-  - On success: JSON array of user's latest statuses
-
-## `/api/pleroma/admin/relay`
-
-### Follow a Relay
-
-- Methods: `POST`
-- Params:
-  - `relay_url`
-- Response:
-  - On success: URL of the followed relay
-
-### Unfollow a Relay
-
-- Methods: `DELETE`
-- Params:
-  - `relay_url`
-- Response:
-  - On success: URL of the unfollowed relay
-
-## `/api/pleroma/admin/users/invite_token`
-
-### Get an account registration invite token
-
-- Methods: `GET`
-- Params:
-  - *optional* `invite` => [
-    - *optional* `max_use` (integer)
-    - *optional* `expires_at` (date string e.g. "2019-04-07")
-  ]
-- Response: invite token (base64 string)
-
-## `/api/pleroma/admin/users/invites`
-
-### Get a list of generated invites
-
-- Methods: `GET`
-- Params: none
-- Response:
-
-```json
-{
-
-  "invites": [
-    {
-      "id": integer,
-      "token": string,
-      "used": boolean,
-      "expires_at": date,
-      "uses": integer,
-      "max_use": integer,
-      "invite_type": string (possible values: `one_time`, `reusable`, `date_limited`, `reusable_date_limited`)
-    },
-    ...
-  ]
-}
-```
-
-## `/api/pleroma/admin/users/revoke_invite`
-
-### Revoke invite by token
-
-- Methods: `POST`
-- Params:
-  - `token`
-- Response:
-
-```json
-{
-  "id": integer,
-  "token": string,
-  "used": boolean,
-  "expires_at": date,
-  "uses": integer,
-  "max_use": integer,
-  "invite_type": string (possible values: `one_time`, `reusable`, `date_limited`, `reusable_date_limited`)
-
-}
-```
-
-
-## `/api/pleroma/admin/users/email_invite`
-
-### Sends registration invite via email
-
-- Methods: `POST`
-- Params:
-  - `email`
-  - `name`, optional
-
-## `/api/pleroma/admin/users/:nickname/password_reset`
-
-### Get a password reset token for a given nickname
-
-- Methods: `GET`
-- Params: none
-- Response: password reset token (base64 string)
-
-## `/api/pleroma/admin/reports`
-### Get a list of reports
-- Method `GET`
-- Params:
-  - `state`: optional, the state of reports. Valid values are `open`, `closed` and `resolved`
-  - `limit`: optional, the number of records to retrieve
-  - `since_id`: optional, returns results that are more recent than the specified id
-  - `max_id`: optional, returns results that are older than the specified id
-- Response:
-  - On failure: 403 Forbidden error `{"error": "error_msg"}` when requested by anonymous or non-admin
-  - On success: JSON, returns a list of reports, where:
-    - `account`: the user who has been reported
-    - `actor`: the user who has sent the report
-    - `statuses`: list of statuses that have been included to the report
-
-```json
-{
-  "reports": [
-    {
-      "account": {
-        "acct": "user",
-        "avatar": "https://pleroma.example.org/images/avi.png",
-        "avatar_static": "https://pleroma.example.org/images/avi.png",
-        "bot": false,
-        "created_at": "2019-04-23T17:32:04.000Z",
-        "display_name": "User",
-        "emojis": [],
-        "fields": [],
-        "followers_count": 1,
-        "following_count": 1,
-        "header": "https://pleroma.example.org/images/banner.png",
-        "header_static": "https://pleroma.example.org/images/banner.png",
-        "id": "9i6dAJqSGSKMzLG2Lo",
-        "locked": false,
-        "note": "",
-        "pleroma": {
-          "confirmation_pending": false,
-          "hide_favorites": true,
-          "hide_followers": false,
-          "hide_follows": false,
-          "is_admin": false,
-          "is_moderator": false,
-          "relationship": {},
-          "tags": []
-        },
-        "source": {
-          "note": "",
-          "pleroma": {},
-          "sensitive": false
-        },
-        "tags": ["force_unlisted"],
-        "statuses_count": 3,
-        "url": "https://pleroma.example.org/users/user",
-        "username": "user"
-      },
-      "actor": {
-        "acct": "lain",
-        "avatar": "https://pleroma.example.org/images/avi.png",
-        "avatar_static": "https://pleroma.example.org/images/avi.png",
-        "bot": false,
-        "created_at": "2019-03-28T17:36:03.000Z",
-        "display_name": "Roger Braun",
-        "emojis": [],
-        "fields": [],
-        "followers_count": 1,
-        "following_count": 1,
-        "header": "https://pleroma.example.org/images/banner.png",
-        "header_static": "https://pleroma.example.org/images/banner.png",
-        "id": "9hEkA5JsvAdlSrocam",
-        "locked": false,
-        "note": "",
-        "pleroma": {
-          "confirmation_pending": false,
-          "hide_favorites": false,
-          "hide_followers": false,
-          "hide_follows": false,
-          "is_admin": false,
-          "is_moderator": false,
-          "relationship": {},
-          "tags": []
-        },
-        "source": {
-          "note": "",
-          "pleroma": {},
-          "sensitive": false
-        },
-        "tags": ["force_unlisted"],
-        "statuses_count": 1,
-        "url": "https://pleroma.example.org/users/lain",
-        "username": "lain"
-      },
-      "content": "Please delete it",
-      "created_at": "2019-04-29T19:48:15.000Z",
-      "id": "9iJGOv1j8hxuw19bcm",
-      "state": "open",
-      "statuses": [
-        {
-          "account": { ... },
-          "application": {
-            "name": "Web",
-            "website": null
-          },
-          "bookmarked": false,
-          "card": null,
-          "content": "<span class=\"h-card\"><a data-user=\"9hEkA5JsvAdlSrocam\" class=\"u-url mention\" href=\"https://pleroma.example.org/users/lain\">@<span>lain</span></a></span> click on my link <a href=\"https://www.google.com/\">https://www.google.com/</a>",
-          "created_at": "2019-04-23T19:15:47.000Z",
-          "emojis": [],
-          "favourited": false,
-          "favourites_count": 0,
-          "id": "9i6mQ9uVrrOmOime8m",
-          "in_reply_to_account_id": null,
-          "in_reply_to_id": null,
-          "language": null,
-          "media_attachments": [],
-          "mentions": [
-            {
-              "acct": "lain",
-              "id": "9hEkA5JsvAdlSrocam",
-              "url": "https://pleroma.example.org/users/lain",
-              "username": "lain"
-            },
-            {
-              "acct": "user",
-              "id": "9i6dAJqSGSKMzLG2Lo",
-              "url": "https://pleroma.example.org/users/user",
-              "username": "user"
-            }
-          ],
-          "muted": false,
-          "pinned": false,
-          "pleroma": {
-            "content": {
-              "text/plain": "@lain click on my link https://www.google.com/"
-            },
-            "conversation_id": 28,
-            "in_reply_to_account_acct": null,
-            "local": true,
-            "spoiler_text": {
-              "text/plain": ""
-            }
-          },
-          "reblog": null,
-          "reblogged": false,
-          "reblogs_count": 0,
-          "replies_count": 0,
-          "sensitive": false,
-          "spoiler_text": "",
-          "tags": [],
-          "uri": "https://pleroma.example.org/objects/8717b90f-8e09-4b58-97b0-e3305472b396",
-          "url": "https://pleroma.example.org/notice/9i6mQ9uVrrOmOime8m",
-          "visibility": "direct"
-        }
-      ]
-    }
-  ]
-}
-```
-
-## `/api/pleroma/admin/reports/:id`
-### Get an individual report
-- Method `GET`
-- Params:
-  - `id`
-- Response:
-  - On failure:
-    - 403 Forbidden `{"error": "error_msg"}`
-    - 404 Not Found `"Not found"`
-  - On success: JSON, Report object (see above)
-
-## `/api/pleroma/admin/reports/:id`
-### Change the state of the report
-- Method `PUT`
-- Params:
-  - `id`
-  - `state`: required, the new state. Valid values are `open`, `closed` and `resolved`
-- Response:
-  - On failure:
-    - 400 Bad Request `"Unsupported state"`
-    - 403 Forbidden `{"error": "error_msg"}`
-    - 404 Not Found `"Not found"`
-  - On success: JSON, Report object (see above)
-
-## `/api/pleroma/admin/reports/:id/respond`
-### Respond to a report
-- Method `POST`
-- Params:
-  - `id`
-  - `status`: required, the message
-- Response:
-  - On failure:
-    - 400 Bad Request `"Invalid parameters"` when `status` is missing
-    - 403 Forbidden `{"error": "error_msg"}`
-    - 404 Not Found `"Not found"`
-  - On success: JSON, created Mastodon Status entity
-
-```json
-{
-  "account": { ... },
-  "application": {
-    "name": "Web",
-    "website": null
-  },
-  "bookmarked": false,
-  "card": null,
-  "content": "Your claim is going to be closed",
-  "created_at": "2019-05-11T17:13:03.000Z",
-  "emojis": [],
-  "favourited": false,
-  "favourites_count": 0,
-  "id": "9ihuiSL1405I65TmEq",
-  "in_reply_to_account_id": null,
-  "in_reply_to_id": null,
-  "language": null,
-  "media_attachments": [],
-  "mentions": [
-    {
-      "acct": "user",
-      "id": "9i6dAJqSGSKMzLG2Lo",
-      "url": "https://pleroma.example.org/users/user",
-      "username": "user"
-    },
-    {
-      "acct": "admin",
-      "id": "9hEkA5JsvAdlSrocam",
-      "url": "https://pleroma.example.org/users/admin",
-      "username": "admin"
-    }
-  ],
-  "muted": false,
-  "pinned": false,
-  "pleroma": {
-    "content": {
-      "text/plain": "Your claim is going to be closed"
-    },
-    "conversation_id": 35,
-    "in_reply_to_account_acct": null,
-    "local": true,
-    "spoiler_text": {
-      "text/plain": ""
-    }
-  },
-  "reblog": null,
-  "reblogged": false,
-  "reblogs_count": 0,
-  "replies_count": 0,
-  "sensitive": false,
-  "spoiler_text": "",
-  "tags": [],
-  "uri": "https://pleroma.example.org/objects/cab0836d-9814-46cd-a0ea-529da9db5fcb",
-  "url": "https://pleroma.example.org/notice/9ihuiSL1405I65TmEq",
-  "visibility": "direct"
-}
-```
-
-## `/api/pleroma/admin/statuses/:id`
-### Change the scope of an individual reported status
-- Method `PUT`
-- Params:
-  - `id`
-  - `sensitive`: optional, valid values are `true` or `false`
-  - `visibility`: optional, valid values are `public`, `private` and `unlisted`
-- Response:
-  - On failure:
-    - 400 Bad Request `"Unsupported visibility"`
-    - 403 Forbidden `{"error": "error_msg"}`
-    - 404 Not Found `"Not found"`
-  - On success: JSON, Mastodon Status entity
-
-## `/api/pleroma/admin/statuses/:id`
-### Delete an individual reported status
-- Method `DELETE`
-- Params:
-  - `id`
-- Response:
-  - On failure:
-    - 403 Forbidden `{"error": "error_msg"}`
-    - 404 Not Found `"Not found"`
-  - On success: 200 OK `{}`
-
-
-## `/api/pleroma/admin/config/migrate_to_db`
-### Run mix task pleroma.config migrate_to_db
-Copy settings on key `:pleroma` to DB.
-- Method `GET`
-- Params: none
-- Response:
-
-```json
-{}
-```
-
-## `/api/pleroma/admin/config/migrate_from_db`
-### Run mix task pleroma.config migrate_from_db
-Copy all settings from DB to `config/prod.exported_from_db.secret.exs` with deletion from DB.
-- Method `GET`
-- Params: none
-- Response:
-
-```json
-{}
-```
-
-## `/api/pleroma/admin/config`
-### List config settings
-List config settings only works with `:pleroma => :instance => :dynamic_configuration` setting to `true`.
-- Method `GET`
-- Params: none
-- Response:
-
-```json
-{
-  configs: [
-    {
-      "group": string,
-      "key": string or string with leading `:` for atoms,
-      "value": string or {} or [] or {"tuple": []}
-     }
-  ]
-}
-```
-
-## `/api/pleroma/admin/config`
-### Update config settings
-Updating config settings only works with `:pleroma => :instance => :dynamic_configuration` setting to `true`.
-Module name can be passed as string, which starts with `Pleroma`, e.g. `"Pleroma.Upload"`.
-Atom keys and values can be passed with `:` in the beginning, e.g. `":upload"`.
-Tuples can be passed as `{"tuple": ["first_val", Pleroma.Module, []]}`.
-`{"tuple": ["some_string", "Pleroma.Some.Module", []]}` will be converted to `{"some_string", Pleroma.Some.Module, []}`.
-Keywords can be passed as lists with 2 child tuples, e.g.
-`[{"tuple": ["first_val", Pleroma.Module]}, {"tuple": ["second_val", true]}]`.
-
-If value contains list of settings `[subkey: val1, subkey2: val2, subkey3: val3]`, it's possible to remove only subkeys instead of all settings passing `subkeys` parameter. E.g.:
-{"group": "pleroma", "key": "some_key", "delete": "true", "subkeys": [":subkey", ":subkey3"]}.
-
-Compile time settings (need instance reboot):
-- all settings by this keys:
-  - `:hackney_pools`
-  - `:chat`
-  - `Pleroma.Web.Endpoint`
-  - `Pleroma.Repo`
-- part settings:
-  - `Pleroma.Captcha` -> `:seconds_valid`
-  - `Pleroma.Upload` -> `:proxy_remote`
-  - `:instance` -> `:upload_limit`
-
-- Method `POST`
-- Params:
-  - `configs` => [
-    - `group` (string)
-    - `key` (string or string with leading `:` for atoms)
-    - `value` (string, [], {} or {"tuple": []})
-    - `delete` = true (optional, if parameter must be deleted)
-    - `subkeys` [(string with leading `:` for atoms)] (optional, works only if `delete=true` parameter is passed, otherwise will be ignored)
-  ]
-
-- Request (example):
-
-```json
-{
-  configs: [
-    {
-      "group": "pleroma",
-      "key": "Pleroma.Upload",
-      "value": [
-        {"tuple": [":uploader", "Pleroma.Uploaders.Local"]},
-        {"tuple": [":filters", ["Pleroma.Upload.Filter.Dedupe"]]},
-        {"tuple": [":link_name", true]},
-        {"tuple": [":proxy_remote", false]},
-        {"tuple": [":proxy_opts", [
-          {"tuple": [":redirect_on_failure", false]},
-          {"tuple": [":max_body_length", 1048576]},
-          {"tuple": [":http": [
-            {"tuple": [":follow_redirect", true]},
-            {"tuple": [":pool", ":upload"]},
-          ]]}
-        ]
-        ]},
-        {"tuple": [":dispatch", {
-          "tuple": ["/api/v1/streaming", "Pleroma.Web.MastodonAPI.WebsocketHandler", []]
-        }]}
-      ]
-    }
-  ]
-}
-
-- Response:
-
-```json
-{
-  configs: [
-    {
-      "group": string,
-      "key": string or string with leading `:` for atoms,
-      "value": string or {} or [] or {"tuple": []}
-     }
-  ]
-}
-```
-
-## `/api/pleroma/admin/moderation_log`
-### Get moderation log
-- Method `GET`
-- Params:
-  - *optional* `page`: **integer** page number
-  - *optional* `page_size`: **integer** number of users per page (default is `50`)
-- Response:
-
-```json
-[
-  {
-    "data": {
-      "actor": {
-        "id": 1,
-        "nickname": "lain"
-      },
-      "action": "relay_follow"
-    },
-    "time": 1502812026, // timestamp
-    "message": "[2017-08-15 15:47:06] @nick0 followed relay: https://example.org/relay" // log message
-  }
-]
-```

docs/clients.md

@@ -37,9 +37,14 @@ Feel free to contact us to be added to this list!
 - Platforms: Android
 - Features: Streaming Ready, Moderation, Text Formatting 
 
+### Kyclos
+- Source Code: <https://git.pleroma.social/pleroma/harbour-kyclos>
+- Platforms: SailfishOS
+- Features: No Streaming
+
 ### Nekonium
 - Homepage: [F-Droid Repository](https://repo.gdgd.jp.net/), [Google Play](https://play.google.com/store/apps/details?id=com.apps.nekonium), [Amazon](https://www.amazon.co.jp/dp/B076FXPRBC/)
-- Source: <https://git.gdgd.jp.net/lin/nekonium/>
+- Source: <https://gogs.gdgd.jp.net/lin/nekonium>
 - Contact: [@lin@pleroma.gdgd.jp.net](https://pleroma.gdgd.jp.net/users/lin)
 - Platforms: Android
 - Features: Streaming Ready
@@ -67,7 +72,7 @@ Feel free to contact us to be added to this list!
 ## Alternative Web Interfaces
 ### Brutaldon
 - Homepage: <https://jfm.carcosa.net/projects/software/brutaldon/>
-- Source Code: <https://github.com/jfmcbrayer/brutaldon>
+- Source Code: <https://git.carcosa.net/jmcbray/brutaldon>
 - Contact: [@gcupc@glitch.social](https://glitch.social/users/gcupc)
 - Features: No Streaming
 

docs/config/General-tips-for-customizing-Pleroma-FE.md

@@ -1,17 +0,0 @@
-# General tips for customizing Pleroma FE
-There are some configuration scripts for Pleroma BE and FE:
-
-1. `config/prod.secret.exs`
-1. `config/config.exs`
-1. `priv/static/static/config.json`
-
-The `prod.secret.exs` affects first. `config.exs` is for fallback or default. `config.json` is for GNU-social-BE-Pleroma-FE instances.
-
-Usually all you have to do is:
-
-1. Copy the section in the `config/config.exs` which you want to activate.
-1. Paste into `config/prod.secret.exs`.
-1. Edit `config/prod.secret.exs`.
-1. Restart the Pleroma daemon.
-
-`prod.secret.exs` is for the `MIX_ENV=prod` environment. `dev.secret.exs` is for the `MIX_ENV=dev` environment respectively.

docs/config/howto_user_recomendation.md

@@ -1,31 +0,0 @@
-# How to activate user recommendation (Who to follow panel)
-![who-to-follow-panel-small](/uploads/9de1b1300436c32461d272945f1bc23e/who-to-follow-panel-small.png)
-
-To show the *who to follow* panel, edit `config/prod.secret.exs` in the Pleroma backend. Following code activates the *who to follow* panel:
-
-```elixir
-config :pleroma, :suggestions,
-  enabled: true,
-  third_party_engine:
-    "http://vinayaka.distsn.org/cgi-bin/vinayaka-user-match-suggestions-api.cgi?{{host}}+{{user}}",
-  timeout: 300_000,
-  limit: 40,
-  web: "https://vinayaka.distsn.org"
-
-```
-
-`config/config.exs` already includes this code, but `enabled:` is `false`.
-
-`/api/v1/suggestions` is also provided when *who to follow* panel is enabled.
-
-For advanced customization, following code shows the newcomers of the fediverse at the *who to follow* panel:
-
-```elixir
-config :pleroma, :suggestions,
-  enabled: true,
-  third_party_engine:
-    "http://vinayaka.distsn.org/cgi-bin/vinayaka-user-new-suggestions-api.cgi?{{host}}+{{user}}",
-  timeout: 60_000,
-  limit: 40,
-  web: "https://vinayaka.distsn.org/user-new.html"
-```

docs/config/small_customizations.md

@@ -1,12 +0,0 @@
-# Small customizations
-
-See also static_dir.md for visual settings.
-
-## Theme
-
-All users of your instance will be able to change the theme they use by going to the settings (the cog in the top-right hand corner). However, if you wish to change the default theme, you can do so by editing `theme` in `config/dev.secret.exs` accordingly.
-
-## Message Visibility
-
-To enable message visibility options when posting like in the Mastodon frontend, set
-`scope_options_enabled` to `true` in `config/dev.secret.exs`.

docs/configuration/custom_emoji.md

@@ -4,6 +4,7 @@ Before you add your own custom emoji, check if they are available in an existing
 See `Mix.Tasks.Pleroma.Emoji` for information about emoji packs.
 
 To add custom emoji:
+
 * Create the `STATIC-DIR/emoji/` directory if it doesn't exist
   (`STATIC-DIR` is configurable, `instance/static/` by default)
 * Create a directory with whatever name you want (custom is a good name to show the purpose of it).

docs/configuration/howto_theming_your_instance.md

@@ -0,0 +1,74 @@
+# Theming your instance
+
+To add a custom theme to your instance, you'll first need to get a custom theme, upload it to the server, make it available to the instance and eventually you can set it as default.
+
+## Getting a custom theme
+
+### Create your own theme
+
+* You can create your own theme using the Pleroma FE by going to settings (gear on the top right) and choose the Theme tab. Here you have the options to create a personal theme.
+* To download your theme, you can do Save preset
+* If you want to upload a theme to customise it further, you can upload it using Load preset
+
+This will only save the theme for you personally. To make it available to the whole instance, you'll need to upload it to the server.
+
+### Get an existing theme
+
+* You can download a theme from another instance by going to that instance, go to settings and make sure you have the theme selected that you want. Then you can do Save preset to download it.
+* You can also find and download custom themes at <https://plthemes.vulpes.one/>
+
+## Adding the custom theme to the instance
+
+### Upload the theme to the server
+
+Themes can be found in the [static directory](static_dir.md). Create `STATIC-DIR/static/themes/` if needed and copy your theme there. Next you need to add an entry for your theme to `STATIC-DIR/static/styles.json`. If you use a from source installation, you'll first need to copy the file from `priv/static/static/styles.json`.
+
+Example of `styles.json` where we add our own `my-awesome-theme.json`
+```json
+{
+  "pleroma-dark": [ "Pleroma Dark", "#121a24", "#182230", "#b9b9ba", "#d8a070", "#d31014", "#0fa00f", "#0095ff", "#ffa500" ],
+  "pleroma-light": [ "Pleroma Light", "#f2f4f6", "#dbe0e8", "#304055", "#f86f0f", "#d31014", "#0fa00f", "#0095ff", "#ffa500" ],
+  "classic-dark": [ "Classic Dark", "#161c20", "#282e32", "#b9b9b9", "#baaa9c", "#d31014", "#0fa00f", "#0095ff", "#ffa500" ],
+  "bird": [ "Bird", "#f8fafd", "#e6ecf0", "#14171a", "#0084b8", "#e0245e", "#17bf63", "#1b95e0", "#fab81e"],
+  "ir-black": [ "Ir Black", "#000000", "#242422", "#b5b3aa", "#ff6c60", "#FF6C60", "#A8FF60", "#96CBFE", "#FFFFB6" ],
+  "monokai": [ "Monokai", "#272822", "#383830", "#f8f8f2", "#f92672", "#F92672", "#a6e22e", "#66d9ef", "#f4bf75" ],
+
+  "redmond-xx": "/static/themes/redmond-xx.json",
+  "redmond-xx-se": "/static/themes/redmond-xx-se.json",
+  "redmond-xxi": "/static/themes/redmond-xxi.json",
+  "breezy-dark": "/static/themes/breezy-dark.json",
+  "breezy-light": "/static/themes/breezy-light.json",
+  "mammal": "/static/themes/mammal.json",
+  "my-awesome-theme": "/static/themes/my-awesome-theme.json"
+}
+```
+
+Now you'll already be able to select the theme in Pleroma FE from the drop-down. You don't need to restart Pleroma because we only changed static served files. You may need to refresh the page in your browser. You'll notice however that the theme doesn't have a name, it's just an empty entry in the drop-down.
+
+### Give the theme a name
+
+When you open one of the themes that ship with Pleroma, you'll notice that the json has a `"name"` key. Add a key-value pair to your theme where the key name is `"name"` and the value the name you want to give your theme. After this you can refresh te page in your browser and the name should be visible in the drop-down.
+
+Example of `my-awesome-theme.json` where we add the name "My Awesome Theme"
+```json
+{
+  "_pleroma_theme_version": 2,
+  "name": "My Awesome Theme",
+  "theme": {}
+}
+```
+
+### Set as default theme
+
+Now we can set the new theme as default in the [Pleroma FE configuration](General-tips-for-customizing-Pleroma-FE.md).
+
+Example of adding the new theme in the back-end config files
+```elixir
+config :pleroma, :frontend_configurations,
+  pleroma_fe: %{
+    theme: "my-awesome-theme"
+  }
+```
+
+If you added it in the back-end configuration file, you'll need to restart your instance for the changes to take effect. If you don't see the changes, it's probably because the browser has cached the previous theme. In that case you'll want to clear browser caches. Alternatively you can use a private/incognito window just to see the changes.
+

docs/configuration/i2p.md

@@ -123,7 +123,7 @@ In addition to that, replace the existing nginx config's contents with the examp
 
 If not an I2P-only instance, add the nginx config below to your existing config at `/etc/nginx/sites-enabled/pleroma.nginx`.
 
-And for both cases, disable CSP in Pleroma's config (STS is disabled by default) so you can define those yourself seperately from the clearnet (if your instance is also on the clearnet).
+And for both cases, disable CSP in Pleroma's config (STS is disabled by default) so you can define those yourself separately from the clearnet (if your instance is also on the clearnet).
 Copy the following into the `config/prod.secret.exs` in your Pleroma folder (/home/pleroma/pleroma/):
 ```
 config :pleroma, :http_security,

docs/configuration/mrf.md

@@ -1,4 +1,5 @@
 # Message Rewrite Facility
+
 The Message Rewrite Facility (MRF) is a subsystem that is implemented as a series of hooks that allows the administrator to rewrite or discard messages.
 
 Possible uses include:
@@ -10,7 +11,8 @@ Possible uses include:
 * removing media from messages
 * sending only public messages to a specific instance
 
-The MRF provides user-configurable policies.  The default policy is `NoOpPolicy`, which disables the MRF functionality.  Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module.
+The MRF provides user-configurable policies. The default policy is `NoOpPolicy`, which disables the MRF functionality. Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module.
+
 It is possible to use multiple, active MRF policies at the same time.
 
 ## Quarantine Instances
@@ -18,7 +20,8 @@ It is possible to use multiple, active MRF policies at the same time.
 You have the ability to prevent from private / followers-only messages from federating with specific instances. Which means they will only get the public or unlisted messages from your instance.
 
 If, for example, you're using `MIX_ENV=prod` aka using production mode, you would open your configuration file located in `config/prod.secret.exs` and edit or add the option under your `:instance` config object. Then you would specify the instance within quotes.
-```
+
+```elixir
 config :pleroma, :instance,
   [...]
   quarantined_instances: ["instance.example", "other.example"]
@@ -28,15 +31,15 @@ config :pleroma, :instance,
 
 `SimplePolicy` is capable of handling most common admin tasks.
 
-To use `SimplePolicy`, you must enable it.  Do so by adding the following to your `:instance` config object, so that it looks like this:
+To use `SimplePolicy`, you must enable it. Do so by adding the following to your `:instance` config object, so that it looks like this:
 
-```
+```elixir
 config :pleroma, :instance,
   [...]
   rewrite_policy: Pleroma.Web.ActivityPub.MRF.SimplePolicy
 ```
 
-Once `SimplePolicy` is enabled, you can configure various groups in the `:mrf_simple` config object.  These groups are:
+Once `SimplePolicy` is enabled, you can configure various groups in the `:mrf_simple` config object. These groups are:
 
 * `media_removal`: Servers in this group will have media stripped from incoming messages.
 * `media_nsfw`: Servers in this group will have the #nsfw tag and sensitive setting injected into incoming messages which contain media.
@@ -50,7 +53,7 @@ Servers should be configured as lists.
 
 This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com`, remove messages from `spam.university` from the federated timeline and block reports (flags) from `whiny.whiner`:
 
-```
+```elixir
 config :pleroma, :instance,
   rewrite_policy: [Pleroma.Web.ActivityPub.MRF.SimplePolicy]
 
@@ -60,30 +63,31 @@ config :pleroma, :mrf_simple,
   reject: ["spam.com"],
   federated_timeline_removal: ["spam.university"],
   report_removal: ["whiny.whiner"]
-
 ```
 
 ### Use with Care
 
-The effects of MRF policies can be very drastic.  It is important to use this functionality carefully.  Always try to talk to an admin before writing an MRF policy concerning their instance.
+The effects of MRF policies can be very drastic. It is important to use this functionality carefully. Always try to talk to an admin before writing an MRF policy concerning their instance.
 
 ## Writing your own MRF Policy
 
-As discussed above, the MRF system is a modular system that supports pluggable policies.  This means that an admin may write a custom MRF policy in Elixir or any other language that runs on the Erlang VM, by specifying the module name in the `rewrite_policy` config setting.
+As discussed above, the MRF system is a modular system that supports pluggable policies. This means that an admin may write a custom MRF policy in Elixir or any other language that runs on the Erlang VM, by specifying the module name in the `rewrite_policy` config setting.
 
 For example, here is a sample policy module which rewrites all messages to "new message content":
 
 ```elixir
-# This is a sample MRF policy which rewrites all Notes to have "new message
+defmodule Pleroma.Web.ActivityPub.MRF.RewritePolicy do
-# content."
+  @moduledoc "MRF policy which rewrites all Notes to have 'new message content'."
-defmodule Site.RewritePolicy do
+  @behaviour Pleroma.Web.ActivityPub.MRF
-  @behavior Pleroma.Web.ActivityPub.MRF
 
   # Catch messages which contain Note objects with actual data to filter.
   # Capture the object as `object`, the message content as `content` and the
   # message itself as `message`.
   @impl true
-  def filter(%{"type" => Create", "object" => {"type" => "Note", "content" => content} = object} = message)
+  def filter(
+        %{"type" => "Create", "object" => %{"type" => "Note", "content" => content} = object} =
+          message
+      )
       when is_binary(content) do
     # Subject / CW is stored as summary instead of `name` like other AS2 objects
     # because of Mastodon doing it that way.
@@ -106,17 +110,22 @@ defmodule Site.RewritePolicy do
   # Let all other messages through without modifying them.
   @impl true
   def filter(message), do: {:ok, message}
+
+  @impl true
+  def describe do
+    {:ok, %{mrf_sample: %{content: "new message content"}}}`
+  end
 end
 ```
 
-If you save this file as `lib/site/mrf/rewrite_policy.ex`, it will be included when you next rebuild Pleroma.  You can enable it in the configuration like so:
+If you save this file as `lib/pleroma/web/activity_pub/mrf/rewrite_policy.ex`, it will be included when you next rebuild Pleroma.  You can enable it in the configuration like so:
 
-```
+```elixir
 config :pleroma, :instance,
   rewrite_policy: [
     Pleroma.Web.ActivityPub.MRF.SimplePolicy,
-    Site.RewritePolicy
+    Pleroma.Web.ActivityPub.MRF.RewritePolicy
   ]
 ```
 
-Please note that the Pleroma developers consider custom MRF policy modules to fall under the purview of the AGPL.  As such, you are obligated to release the sources to your custom MRF policy modules upon request.
+Please note that the Pleroma developers consider custom MRF policy modules to fall under the purview of the AGPL. As such, you are obligated to release the sources to your custom MRF policy modules upon request.

docs/configuration/onion_federation.md

@@ -75,7 +75,7 @@ If not a Tor-only instance,
 add the nginx config below to your existing config at `/etc/nginx/sites-enabled/pleroma.nginx`.
 
 ---
-For both cases, disable CSP in Pleroma's config (STS is disabled by default) so you can define those yourself seperately from the clearnet (if your instance is also on the clearnet).
+For both cases, disable CSP in Pleroma's config (STS is disabled by default) so you can define those yourself separately from the clearnet (if your instance is also on the clearnet).
 Copy the following into the `config/prod.secret.exs` in your Pleroma folder (/home/pleroma/pleroma/):
 ```
 config :pleroma, :http_security,

docs/installation/alpine_linux_en.md

@@ -1,7 +1,9 @@
 # Installing on Alpine Linux
 ## Installation
 
-This guide is a step-by-step installation guide for Alpine Linux. It also assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.linode.com/docs/tools-reference/custom-kernels-distros/install-alpine-linux-on-your-linode/#configuration). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su -l <username> -s $SHELL -c 'command'` instead.
+This guide is a step-by-step installation guide for Alpine Linux. The instructions were verified against Alpine v3.10 standard image. You might miss additional dependencies if you use `netboot` instead.
+
+It assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.linode.com/docs/tools-reference/custom-kernels-distros/install-alpine-linux-on-your-linode/#configuration). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su -l <username> -s $SHELL -c 'command'` instead.
 
 ### Required packages
 
@@ -20,12 +22,13 @@ This guide is a step-by-step installation guide for Alpine Linux. It also assume
 
 ### Prepare the system
 
-* First make sure to have the community repository enabled:
+* The community repository must be enabled in `/etc/apk/repositories`. Depending on which version and mirror you use this looks like `http://alpine.42.fr/v3.10/community`. If you autogenerated the mirror during installation:
 
 ```shell
-echo "https://nl.alpinelinux.org/alpine/latest-stable/community" | sudo tee -a /etc/apk/repository
+awk 'NR==2' /etc/apk/repositories | sed 's/main/community/' | tee -a /etc/apk/repositories
 ```
 
+
 * Then update the system, if not already done:
 
 ```shell
@@ -77,7 +80,8 @@ sudo rc-update add postgresql
 * Add a new system user for the Pleroma service:
 
 ```shell
-sudo adduser -S -s /bin/false -h /opt/pleroma -H pleroma
+sudo addgroup pleroma
+sudo adduser -S -s /bin/false -h /opt/pleroma -H -G pleroma pleroma
 ```
 
 **Note**: To execute a single command as the Pleroma system user, use `sudo -Hu pleroma command`. You can also switch to a shell by using `sudo -Hu pleroma $SHELL`. If you don’t have and want `sudo` on your system, you can use `su` as root user (UID 0) for a single command by using `su -l pleroma -s $SHELL -c 'command'` and `su -l pleroma -s $SHELL` for starting a shell.
@@ -164,7 +168,26 @@ If that doesn’t work, make sure, that nginx is not already running. If it stil
 sudo cp /opt/pleroma/installation/pleroma.nginx /etc/nginx/conf.d/pleroma.conf
 ```
 
-* Before starting nginx edit the configuration and change it to your needs (e.g. change servername, change cert paths)
+* Before starting nginx edit the configuration and change it to your needs. You must change change `server_name` and the paths to the certificates. You can use `nano` (install with `apk add nano` if missing).
+
+```
+server {
+    server_name    your.domain;
+    listen         80;
+    ...
+}
+
+server {
+    server_name your.domain;
+    listen 443 ssl http2;
+    ...
+    ssl_trusted_certificate   /etc/letsencrypt/live/your.domain/chain.pem;
+    ssl_certificate           /etc/letsencrypt/live/your.domain/fullchain.pem;
+    ssl_certificate_key       /etc/letsencrypt/live/your.domain/privkey.pem;
+    ...
+}
+```
+
 * Enable and start nginx:
 
 ```shell
@@ -202,12 +225,10 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress
 
 #### Further reading
 
-* [Backup your instance](backup.html)
+* [Backup your instance](../administration/backup.md)
-* [Configuration tips](general-tips-for-customizing-pleroma-fe.html)
+* [Hardening your instance](../configuration/hardening.md)
-* [Hardening your instance](hardening.html)
+* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [How to activate mediaproxy](howto_mediaproxy.html)
+* [Updating your instance](../administration/updating.md)
-* [Small Pleroma-FE customizations](small_customizations.html)
-* [Updating your instance](updating.html)
 
 ## Questions
 

docs/installation/arch_linux_en.md

@@ -200,12 +200,10 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress
 
 #### Further reading
 
-* [Backup your instance](backup.html)
+* [Backup your instance](../administration/backup.md)
-* [Configuration tips](general-tips-for-customizing-pleroma-fe.html)
+* [Hardening your instance](../configuration/hardening.md)
-* [Hardening your instance](hardening.html)
+* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [How to activate mediaproxy](howto_mediaproxy.html)
+* [Updating your instance](../administration/updating.md)
-* [Small Pleroma-FE customizations](small_customizations.html)
-* [Updating your instance](updating.html)
 
 ## Questions
 

docs/installation/centos7_en.md

@@ -1,276 +0,0 @@
-# Installing on CentOS 7
-## Installation
-
-This guide is a step-by-step installation guide for CentOS 7. It also assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.digitalocean.com/community/tutorials/how-to-create-a-sudo-user-on-centos-quickstart). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su <username> -s $SHELL -c 'command'` instead.
-
-### Required packages
-
-* `postgresql` (9,6+, CentOS 7 comes with 9.2, we will install version 11 in this guide)
-* `elixir` (1.5+)
-* `erlang`
-* `erlang-parsetools`
-* `erlang-xmerl`
-* `git`
-* Development Tools
-
-#### Optional packages used in this guide
-
-* `nginx` (preferred, example configs for other reverse proxies can be found in the repo)
-* `certbot` (or any other ACME client for Let’s Encrypt certificates)
-
-### Prepare the system
-
-* First update the system, if not already done:
-
-```shell
-sudo yum update
-```
-
-* Install some of the above mentioned programs:
-
-```shell
-sudo yum install wget git unzip
-```
-
-* Install development tools:
-
-```shell
-sudo yum group install "Development Tools"
-```
-
-### Install Elixir and Erlang
-
-* Add the EPEL repo:
-
-```shell
-sudo yum install epel-release
-sudo yum -y update
-```
-
-* Install Erlang repository:
-
-```shell
-wget -P /tmp/ https://packages.erlang-solutions.com/erlang-solutions-1.0-1.noarch.rpm
-sudo rpm -Uvh erlang-solutions-1.0-1.noarch.rpm
-```
-
-* Install Erlang:
-
-```shell
-sudo yum install erlang erlang-parsetools erlang-xmerl
-```
-
-* Download [latest Elixir release from Github](https://github.com/elixir-lang/elixir/releases/tag/v1.8.1) (Example for the newest version at the time when this manual was written)
-
-```shell
-wget -P /tmp/ https://github.com/elixir-lang/elixir/releases/download/v1.8.1/Precompiled.zip
-```
-
-* Create folder where you want to install Elixir, we’ll use:
-
-```shell
-sudo mkdir -p /opt/elixir
-```
-
-* Unzip downloaded file there:
-
-```shell
-sudo unzip /tmp/Precompiled.zip -d /opt/elixir
-```
-
-* Create symlinks for the pre-compiled binaries:
-
-```shell
-for e in elixir elixirc iex mix; do sudo ln -s /opt/elixir/bin/${e} /usr/local/bin/${e}; done
-```
-
-### Install PostgreSQL
-
-* Add the Postgresql repository:
-
-```shell
-sudo yum install https://download.postgresql.org/pub/repos/yum/11/redhat/rhel-7-x86_64/pgdg-centos11-11-2.noarch.rpm
-```
-
-* Install the Postgresql server:
-
-```shell
-sudo yum install postgresql11-server postgresql11-contrib
-```
-
-* Initialize database:
-
-```shell
-sudo /usr/pgsql-11/bin/postgresql-11-setup initdb
-```
-
-* Open configuration file `/var/lib/pgsql/11/data/pg_hba.conf` and change the following lines from:
-
-```plain
-# IPv4 local connections:
-host    all             all             127.0.0.1/32            ident
-# IPv6 local connections:
-host    all             all             ::1/128                 ident
-```
-
-to
-
-```plain
-# IPv4 local connections:
-host    all             all             127.0.0.1/32            md5
-# IPv6 local connections:
-host    all             all             ::1/128                 md5
-```
-
-* Enable and start postgresql server:
-
-```shell
-sudo systemctl enable --now postgresql-11.service
-```
-
-### Install PleromaBE
-
-* Add a new system user for the Pleroma service:
-
-```shell
-sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma
-```
-
-**Note**: To execute a single command as the Pleroma system user, use `sudo -Hu pleroma command`. You can also switch to a shell by using `sudo -Hu pleroma $SHELL`. If you don’t have and want `sudo` on your system, you can use `su` as root user (UID 0) for a single command by using `su -l pleroma -s $SHELL -c 'command'` and `su -l pleroma -s $SHELL` for starting a shell.
-
-* Git clone the PleromaBE repository and make the Pleroma user the owner of the directory:
-
-```shell
-sudo mkdir -p /opt/pleroma
-sudo chown -R pleroma:pleroma /opt/pleroma
-sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma
-```
-
-* Change to the new directory:
-
-```shell
-cd /opt/pleroma
-```
-
-* Install the dependencies for Pleroma and answer with `yes` if it asks you to install `Hex`:
-
-```shell
-sudo -Hu pleroma mix deps.get
-```
-
-* Generate the configuration: `sudo -Hu pleroma mix pleroma.instance gen`
-  * Answer with `yes` if it asks you to install `rebar3`.
-  * This may take some time, because parts of pleroma get compiled first.
-  * After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`.
-
-* Check the configuration and if all looks right, rename it, so Pleroma will load it (`prod.secret.exs` for productive instance, `dev.secret.exs` for development instances):
-
-```shell
-mv config/{generated_config.exs,prod.secret.exs}
-```
-
-* The previous command creates also the file `config/setup_db.psql`, with which you can create the database:
-
-```shell
-sudo -Hu postgres psql -f config/setup_db.psql
-```
-
-* Now run the database migration:
-
-```shell
-sudo -Hu pleroma MIX_ENV=prod mix ecto.migrate
-```
-
-* Now you can start Pleroma already
-
-```shell
-sudo -Hu pleroma MIX_ENV=prod mix phx.server
-```
-
-### Finalize installation
-
-If you want to open your newly installed instance to the world, you should run nginx or some other webserver/proxy in front of Pleroma and you should consider to create a systemd service file for Pleroma.
-
-#### Nginx
-
-* Install nginx, if not already done:
-
-```shell
-sudo yum install nginx
-```
-
-* Setup your SSL cert, using your method of choice or certbot. If using certbot, first install it:
-
-```shell
-sudo yum install certbot-nginx
-```
-
-and then set it up:
-
-```shell
-sudo mkdir -p /var/lib/letsencrypt/
-sudo certbot certonly --email <your@emailaddress> -d <yourdomain> --standalone
-```
-
-If that doesn’t work, make sure, that nginx is not already running. If it still doesn’t work, try setting up nginx first (change ssl “on” to “off” and try again).
-
----
-
-* Copy the example nginx configuration to the nginx folder
-
-```shell
-sudo cp /opt/pleroma/installation/pleroma.nginx /etc/nginx/conf.d/pleroma.conf
-```
-
-* Before starting nginx edit the configuration and change it to your needs (e.g. change servername, change cert paths)
-* Enable and start nginx:
-
-```shell
-sudo systemctl enable --now nginx
-```
-
-If you need to renew the certificate in the future, uncomment the relevant location block in the nginx config and run:
-
-```shell
-sudo certbot certonly --email <your@emailaddress> -d <yourdomain> --webroot -w /var/lib/letsencrypt/
-```
-
-#### Other webserver/proxies
-
-You can find example configurations for them in `/opt/pleroma/installation/`.
-
-#### Systemd service
-
-* Copy example service file
-
-```shell
-sudo cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service
-```
-
-* Edit the service file and make sure that all paths fit your installation
-* Enable and start `pleroma.service`:
-
-```shell
-sudo systemctl enable --now pleroma.service
-```
-
-#### Create your first user
-
-If your instance is up and running, you can create your first user with administrative rights with the following task:
-
-```shell
-sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress> --admin
-```
-
-#### Further reading
-
-* [Backup your instance](backup.html)
-* [Configuration tips](general-tips-for-customizing-pleroma-fe.html)
-* [Hardening your instance](hardening.html)
-* [How to activate mediaproxy](howto_mediaproxy.html)
-* [Small Pleroma-FE customizations](small_customizations.html)
-* [Updating your instance](updating.html)
-
-## Questions
-
-Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.

docs/installation/debian_based_en.md

@@ -190,12 +190,10 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress
 
 #### Further reading
 
-* [Backup your instance](backup.html)
+* [Backup your instance](../administration/backup.md)
-* [Configuration tips](general-tips-for-customizing-pleroma-fe.html)
+* [Hardening your instance](../configuration/hardening.md)
-* [Hardening your instance](hardening.html)
+* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [How to activate mediaproxy](howto_mediaproxy.html)
+* [Updating your instance](../administration/updating.md)
-* [Small Pleroma-FE customizations](small_customizations.html)
-* [Updating your instance](updating.html)
 
 ## Questions
 

docs/installation/debian_based_jp.md

@@ -5,67 +5,64 @@
 
 ## インストール
 
-このガイドはDebian Stretchを仮定しています。Ubuntu 16.04でも可能です。
+このガイドはDebian Stretchを利用することを想定しています。Ubuntu 16.04や18.04でもおそらく動作します。また、ユーザはrootもしくはsudoにより管理者権限を持っていることを前提とします。もし、以下の操作をrootユーザで行う場合は、 `sudo` を無視してください。ただし、`sudo -Hu pleroma` のようにユーザを指定している場合には `su <username> -s $SHELL -c 'command'` を代わりに使ってください。
 
 ### 必要なソフトウェア
 
-- PostgreSQL 9.6+ (postgresql-contrib-9.6 または他のバージョンの PSQL をインストールしてください)
+- PostgreSQL 9.6以上 (Ubuntu16.04では9.5しか提供されていないので，[](https://www.postgresql.org/download/linux/ubuntu/)こちらから新しいバージョンを入手してください)
-- Elixir 1.5 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like))。または [asdf](https://github.com/asdf-vm/asdf) を pleroma ユーザーでインストール。
+- postgresql-contrib 9.6以上 (同上)
-- erlang-dev
+- Elixir 1.5 以上 ([Debianのリポジトリからインストールしないこと！！！ ここからインストールすること！](https://elixir-lang.org/install.html#unix-and-unix-like)。または [asdf](https://github.com/asdf-vm/asdf) をpleromaユーザーでインストールしてください)
+  - erlang-dev
 - erlang-tools
 - erlang-parsetools
+- erlang-eldap (LDAP認証を有効化するときのみ必要)
 - erlang-ssh
-- erlang-xmerl (Jessieではバックポートからインストールすること！)
+- erlang-xmerl
 - git
 - build-essential
-- openssh
+
-- openssl
+#### このガイドで利用している追加パッケージ
-- nginx prefered (Apacheも動くかもしれませんが、誰もテストしていません！)
+
-- certbot (または何らかのACME Let's encryptクライアント)
+- nginx (おすすめです。他のリバースプロキシを使う場合は、参考となる設定をこのリポジトリから探してください)
+- certbot (または何らかのLet's Encrypt向けACMEクライアント)
 
 ### システムを準備する
 
 * まずシステムをアップデートしてください。
 ```
-apt update && apt dist-upgrade
+sudo apt update
+sudo apt full-upgrade
 ```
 
-* 複数のツールとpostgresqlをインストールします。あとで必要になるので。
+* 上記に挙げたパッケージをインストールしておきます。
 ```
-apt install git build-essential openssl ssh sudo postgresql-9.6 postgresql-contrib-9.6
+sudo apt install git build-essential postgresql postgresql-contrib
 ```
-(postgresqlのバージョンは、あなたのディストロにあわせて変えてください。または、バージョン番号がいらないかもしれません。)
+
 
 ### ElixirとErlangをインストールします
 
 * Erlangのリポジトリをダウンロードおよびインストールします。
 ```
-wget -P /tmp/ https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb && sudo dpkg -i /tmp/erlang-solutions_1.0_all.deb
+wget -P /tmp/ https://packages.erlang-solutions.com/erlang-solutions_1.0_all.deb
+sudo dpkg -i /tmp/erlang-solutions_1.0_all.deb
 ```
 
 * ElixirとErlangをインストールします、
 ```
-apt update && apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools erlang-ssh
+sudo apt update
+sudo apt install elixir erlang-dev erlang-parsetools erlang-xmerl erlang-tools erlang-ssh
 ```
 
 ### Pleroma BE (バックエンド) をインストールします
 
-*  新しいユーザーを作ります。
+*  Pleroma用に新しいユーザーを作ります。
-```
-adduser pleroma
-``` 
-(Give it any password you want, make it STRONG)
 
-*  新しいユーザーをsudoグループに入れます。
 ```
-usermod -aG sudo pleroma
+sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma
 ```
 
-*  新しいユーザーに変身し、ホームディレクトリに移動します。
+**注意**: Pleromaユーザとして単発のコマンドを実行したい場合はは、`sudo -Hu pleroma command` を使ってください。シェルを使いたい場合は `sudo -Hu pleroma $SHELL`です。もし `sudo` を使わない場合は、rootユーザで `su -l pleroma -s $SHELL -c 'command'` とすることでコマンドを、`su -l pleroma -s $SHELL` とすることでシェルを開始できます。
-```
-su pleroma
-cd ~
-```
 
 *  Gitリポジトリをクローンします。
 ```
@@ -76,118 +73,116 @@ sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma
 
 *  新しいディレクトリに移動します。
 ```
-cd pleroma/
+cd /opt/pleroma
 ```
 
 * Pleromaが依存するパッケージをインストールします。Hexをインストールしてもよいか聞かれたら、yesを入力してください。
 ```
-mix deps.get
+sudo -Hu pleroma mix deps.get
 ```
 
 * コンフィギュレーションを生成します。
 ```
-mix pleroma.instance gen
+sudo -Hu pleroma mix pleroma.instance gen
 ```
     * rebar3をインストールしてもよいか聞かれたら、yesを入力してください。
-    * この処理には時間がかかります。私もよく分かりませんが、何らかのコンパイルが行われているようです。
+    * このときにpleromaの一部がコンパイルされるため、この処理には時間がかかります。
-    * あなたのインスタンスについて、いくつかの質問があります。その回答は `config/generated_config.exs` というコンフィギュレーションファイルに保存されます。
+    * あなたのインスタンスについて、いくつかの質問されます。この質問により `config/generated_config.exs` という設定ファイルが生成されます。
 
-**注意**: メディアプロクシを有効にすると回答して、なおかつ、キャッシュのURLは空欄のままにしている場合は、`generated_config.exs` を編集して、`base_url` で始まる行をコメントアウトまたは削除してください。そして、上にある行の `true` の後にあるコンマを消してください。
 
 * コンフィギュレーションを確認して、もし問題なければ、ファイル名を変更してください。
 ```
 mv config/{generated_config.exs,prod.secret.exs}
 ```
 
-* これまでのコマンドで、すでに `config/setup_db.psql` というファイルが作られています。このファイルをもとに、データベースを作成します。
+* 先程のコマンドで、すでに `config/setup_db.psql` というファイルが作られています。このファイルをもとに、データベースを作成します。
 ```
-sudo su postgres -c 'psql -f config/setup_db.psql'
+sudo -Hu pleroma mix pleroma.instance gen
 ```
 
-* そして、データベースのミグレーションを実行します。
+* そして、データベースのマイグレーションを実行します。
 ```
-MIX_ENV=prod mix ecto.migrate
+sudo -Hu pleroma MIX_ENV=prod mix ecto.migrate
 ```
 
-* Pleromaを起動できるようになりました。
+* これでPleromaを起動できるようになりました。
 ```
-MIX_ENV=prod mix phx.server
+sudo -Hu pleroma MIX_ENV=prod mix phx.server
 ```
 
-### インストールを終わらせる
+### インストールの最終段階
 
-あなたの新しいインスタンスを世界に向けて公開するには、nginxまたは何らかのウェブサーバー (プロクシ) を使用する必要があります。また、Pleroma のためにシステムサービスファイルを作成する必要があります。
+あなたの新しいインスタンスを世界に向けて公開するには、nginx等のWebサーバやプロキシサーバをPleromaの前段に使用する必要があります。また、Pleroma のためにシステムサービスファイルを作成する必要があります。
 
 #### Nginx
 
 * まだインストールしていないなら、nginxをインストールします。
 ```
-apt install nginx
+sudo apt install nginx
 ```
 
 * SSLをセットアップします。他の方法でもよいですが、ここではcertbotを説明します。
 certbotを使うならば、まずそれをインストールします。
 ```
-apt install certbot
+sudo apt install certbot
 ```
 そしてセットアップします。
 ```
-mkdir -p /var/lib/letsencrypt/.well-known
+sudo mkdir -p /var/lib/letsencrypt/
-% certbot certonly --email your@emailaddress --webroot -w /var/lib/letsencrypt/ -d yourdomain
+sudo certbot certonly --email <your@emailaddress> -d <yourdomain> --standalone
 ```
-もしうまくいかないときは、先にnginxを設定してください。ssl "on" を "off" に変えてから再試行してください。
+もしうまくいかないときは、nginxが正しく動いていない可能性があります。先にnginxを設定してください。ssl "on" を "off" に変えてから再試行してください。
 
 ---
 
-* nginxコンフィギュレーションの例をnginxフォルダーにコピーします。
+* nginxの設定ファイルサンプルをnginxフォルダーにコピーします。
 ```
-cp /home/pleroma/pleroma/installation/pleroma.nginx /etc/nginx/sites-enabled/pleroma.nginx
+sudo cp /opt/pleroma/installation/pleroma.nginx /etc/nginx/sites-available/pleroma.nginx
+sudo ln -s /etc/nginx/sites-available/pleroma.nginx /etc/nginx/sites-enabled/pleroma.nginx
 ```
 
-* nginxを起動する前に、コンフィギュレーションを編集してください。例えば、サーバー名、証明書のパスなどを変更する必要があります。
+* nginxを起動する前に、設定ファイルを編集してください。例えば、サーバー名、証明書のパスなどを変更する必要があります。
 * nginxを再起動します。
 ```
-systemctl reload nginx.service
+sudo systemctl enable --now nginx.service
 ```
 
+もし証明書を更新する必要が出てきた場合には、nginxの関連するlocationブロックのコメントアウトを外し、以下のコマンドを動かします。
+
+```
+sudo certbot certonly --email <your@emailaddress> -d <yourdomain> --webroot -w /var/lib/letsencrypt/
+```
+
+#### 他のWebサーバやプロキシ
+これに関してはサンプルが `/opt/pleroma/installation/` にあるので、探してみてください。
+
 #### Systemd サービス
 
-* サービスファイルの例をコピーします。
+* サービスファイルのサンプルをコピーします。
 ```
-cp /home/pleroma/pleroma/installation/pleroma.service /usr/lib/systemd/system/pleroma.service
+sudo cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service
 ```
 
-* サービスファイルを変更します。すべてのパスが正しいことを確認してください。また、`[Service]` セクションに以下の行があることを確認してください。
+* サービスファイルを変更します。すべてのパスが正しいことを確認してください
+* サービスを有効化し `pleroma.service` を開始してください
 ```
-Environment="MIX_ENV=prod"
+sudo systemctl enable --now pleroma.service
 ```
 
-* `pleroma.service` を enable および start してください。
+#### 初期ユーザの作成
+
+新たにインスタンスを作成したら、以下のコマンドにより管理者権限を持った初期ユーザを作成できます。
+
 ```
-systemctl enable --now pleroma.service
+sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress> --admin
 ```
 
-#### モデレーターを作る
+#### その他の設定とカスタマイズ
 
-新たにユーザーを作ったら、モデレーター権限を与えたいかもしれません。以下のタスクで可能です。
+* [Backup your instance](../administration/backup.md)
-```
+* [Hardening your instance](../configuration/hardening.md)
-mix set_moderator username [true|false]
+* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-```
+* [Updating your instance](../administration/updating.md)
-
-モデレーターはすべてのポストを消すことができます。将来的には他のことも可能になるかもしれません。
-
-#### メディアプロクシを有効にする
-
-`generate_config` でメディアプロクシを有効にしているなら、すでにメディアプロクシが動作しています。あとから設定を変更したいなら、[How to activate mediaproxy](How-to-activate-mediaproxy) を見てください。
-
-#### コンフィギュレーションとカスタマイズ
-
-* [Backup your instance](backup.html)
-* [Configuration tips](general-tips-for-customizing-pleroma-fe.html)
-* [Hardening your instance](hardening.html)
-* [How to activate mediaproxy](howto_mediaproxy.html)
-* [Small Pleroma-FE customizations](small_customizations.html)
-* [Updating your instance](updating.html)
 
 ## 質問ある？
 

docs/installation/gentoo_en.md

@@ -283,12 +283,10 @@ If you opted to allow sudo for the `pleroma` user but would like to remove the a
 
 #### Further reading
 
-* [Backup your instance](backup.html)
+* [Backup your instance](../administration/backup.md)
-* [Configuration tips](general-tips-for-customizing-pleroma-fe.html)
+* [Hardening your instance](../configuration/hardening.md)
-* [Hardening your instance](hardening.html)
+* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [How to activate mediaproxy](howto_mediaproxy.html)
+* [Updating your instance](../administration/updating.md)
-* [Small Pleroma-FE customizations](small_customizations.html)
-* [Updating your instance](updating.html)
 
 ## Questions
 

docs/installation/migrating_from_source_otp_en.md

@@ -1,42 +1,28 @@
 # Switching a from-source install to OTP releases
+
 ## What are OTP releases?
 OTP releases are as close as you can get to binary releases with Erlang/Elixir. The release is self-contained, and provides everything needed to boot it, it is easily administered via the provided shell script to open up a remote console, start/stop/restart the release, start in the background, send remote commands, and more.
-### Can I still run the develop branch if I decide to use them?
-Yes, we produce builds for every commit in `develop`. However `develop` is considered unstable, please don't use it in production because of faster access to new features, unless you need them as an app developer.
-## Why would one want to switch?
-Benefits of OTP releases over from-source installs include:
-* **Less space used.** OTP releases come without source code, build tools, have docs and debug symbols stripped from the compiled bytecode and do not cointain tests, docs, revision history.
-* **Minimal system dependencies.** Excluding the database and reverse proxy, only `curl`, `unzip` and `ncurses` are needed to download and run the release. Because Erlang runtime and Elixir are shipped with Pleroma, one can use the latest BEAM optimizations and Pleroma features, without having to worry about outdated system repos or a missing `erlang-*` package.
-* **Potentially less bugs and better performance.** This extends on the previous point, because we have control over exactly what gets shipped, we can tweak the VM arguments and forget about weird bugs due to Erlang/Elixir version mismatches.
-* **Faster and less bug-prone mix tasks.** On a from-source install one has to wait untill a new Pleroma node is started for each mix task and they execute outside of the instance context (for example if a user was deleted via a mix task, the instance will have no knowledge of that and continue to display status count and follows before the cache expires). Mix tasks in OTP releases are executed by calling into a running instance via RPC, which solves both of these problems.
 
-### Sounds great, how do I switch?
-Currently we support Linux machines with GNU (e.g. Debian, Ubuntu) or musl (e.g. Alpine) libc and `x86_64`, `aarch64` or `armv7l` CPUs. If you are unsure, check the [Detecting flavour](otp_en.html#detecting-flavour) section in OTP install guide. If your platform is supported, proceed with the guide, if not check the [My platform is not supported](#my-platform-is-not-supported) section.
-### I don't think it is worth the effort, can I stay on a from-source install?
-Yes, currently there are no plans to deprecate them.
-
-### My platform is not supported
-If you think your platform is a popular choice for running Pleroma instances, or has the potential to become one, you can [file an issue on our Gitlab](https://git.pleroma.social/pleroma/pleroma/issues/new). If not, guides on how to build and update releases by yourself will be available soon.
 ## Pre-requisites
 You will be running commands as root. If you aren't root already, please elevate your priviledges by executing `sudo su`/`su`.
 
 The system needs to have `curl` and `unzip` installed for downloading and unpacking release builds.
 
-Debian/Ubuntu:
+```sh tab="Alpine"
-```sh
+apk add curl unzip
+```
+
+```sh tab="Debian/Ubuntu"
 apt install curl unzip
 ```
-Alpine:
-```
-apk add curl unzip
 
-```
 ## Moving content out of the application directory
 When using OTP releases the application directory changes with every version so it would be a bother to keep content there (and also dangerous unless `--no-rm` option is used when updating). Fortunately almost all paths in Pleroma are configurable, so it is possible to move them out of there.
 
 Pleroma should be stopped before proceeding.
 
 ### Moving uploads/custom public files directory
+
 ```sh
 # Create uploads directory and set proper permissions (skip if using a remote uploader)
 # Note: It does not have to be `/var/lib/pleroma/uploads`, you can configure it to be something else later
@@ -70,7 +56,7 @@ and then copy custom emojis to `/var/lib/pleroma/static/emoji/custom`.
 
 This is needed because storing custom emojis in the root directory is deprecated, but if you just move them to `/var/lib/pleroma/static/emoji/custom` it will break emoji urls on old posts.
 
-Note that globs have been replaced with `pack_extensions`, so if your emojis are not in png/gif you should [modify the default value](config.html#emoji).
+Note that globs have been replaced with `pack_extensions`, so if your emojis are not in png/gif you should [modify the default value](../configuration/cheatsheet.md#emoji).
 
 ### Moving the config
 ```sh
@@ -86,14 +72,14 @@ mv ~pleroma/config/prod.secret.exs /etc/pleroma/config.exs
 $EDITOR /etc/pleroma/config.exs
 ```
 ## Installing the release
-Before proceeding, get the flavour from [Detecting flavour](otp_en.html#detecting-flavour) section in OTP installation guide.
+Before proceeding, get the flavour from [Detecting flavour](otp_en.md#detecting-flavour) section in OTP installation guide.
 ```sh
 # Delete all files in pleroma user's directory
 rm -r ~pleroma/*
 
 # Set the flavour environment variable to the string you got in Detecting flavour section.
-# For example if the flavour is `arm64-musl` the command will be
+# For example if the flavour is `amd64-musl` the command will be
-export FLAVOUR="arm64-musl"
+export FLAVOUR="amd64-musl"
 
 # Clone the release build into a temporary directory and unpack it
 # Replace `stable` with `unstable` if you want to run the unstable branch
@@ -124,8 +110,15 @@ OTP releases have different service files than from-source installs so they need
 
 **Warning:** The service files assume pleroma user's home directory is `/opt/pleroma`, please make sure all paths fit your installation.
 
-Debian/Ubuntu:
+```sh tab="Alpine"
-```sh
+# Copy the service into a proper directory
+cp -f ~pleroma/installation/init.d/pleroma /etc/init.d/pleroma
+
+# Start pleroma
+rc-service pleroma start
+```
+
+```sh tab="Debian/Ubuntu"
 # Copy the service into a proper directory
 cp ~pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service
 
@@ -139,15 +132,7 @@ systemctl reenable pleroma
 systemctl start pleroma
 ```
 
-Alpine:
-```sh
-# Copy the service into a proper directory
-cp -f ~pleroma/installation/init.d/pleroma /etc/init.d/pleroma
-
-# Start pleroma
-rc-service pleroma start
-```
 ## Running mix tasks
-Refer to [Running mix tasks](otp_en.html#running-mix-tasks) section from OTP release installation guide.
+Refer to [Running mix tasks](otp_en.md#running-mix-tasks) section from OTP release installation guide.
 ## Updating
-Refer to [Updating](otp_en.html#updating) section from OTP release installation guide.
+Refer to [Updating](otp_en.md#updating) section from OTP release installation guide.

docs/installation/openbsd_en.md

@@ -1,9 +1,13 @@
 # Installing on OpenBSD
-This guide describes the installation and configuration of pleroma (and the required software to run it) on a single OpenBSD 6.4 server.
+
+This guide describes the installation and configuration of pleroma (and the required software to run it) on a single OpenBSD 6.6 server.
+
 For any additional information regarding commands and configuration files mentioned here, check the man pages [online](https://man.openbsd.org/) or directly on your server with the man command.
 
 #### Required software
+
 The following packages need to be installed:
+
   * elixir
   * gmake
   * ImageMagick
@@ -11,8 +15,11 @@ The following packages need to be installed:
   * postgresql-server
   * postgresql-contrib
 
-To install them, run the following command (with doas or as root):  
+To install them, run the following command (with doas or as root):
-`pkg_add elixir gmake ImageMagick git postgresql-server postgresql-contrib`
+
+```
+pkg_add elixir gmake ImageMagick git postgresql-server postgresql-contrib
+```
 
 Pleroma requires a reverse proxy, OpenBSD has relayd in base (and is used in this guide) and packages/ports are available for nginx (www/nginx) and apache (www/apache-httpd). Independently of the reverse proxy, [acme-client(1)](https://man.openbsd.org/acme-client) can be used to get a certificate from Let's Encrypt.
 
@@ -31,9 +38,14 @@ Create the \_pleroma user, assign it the pleroma login class and create its home
 #### Clone pleroma's directory
 Enter a shell as the \_pleroma user. As root, run `su _pleroma -;cd`. Then clone the repository with `git clone -b stable https://git.pleroma.social/pleroma/pleroma.git`. Pleroma is now installed in /home/\_pleroma/pleroma/, it will be configured and started at the end of this guide.
 
-#### Postgresql
+#### PostgreSQL
-Start a shell as the \_postgresql user (as root run `su _postgresql -` then run the `initdb` command to initialize postgresql:  
+Start a shell as the \_postgresql user (as root run `su _postgresql -` then run the `initdb` command to initialize postgresql:
-If you wish to not use the default location for postgresql's data (/var/postgresql/data), add the following switch at the end of the command: `-D <path>` and modify the `datadir` variable in the /etc/rc.d/postgresql script.
+You will need to specify pgdata directory to the default (/var/postgresql/data) with the `-D <path>` and set the user to postgres with the `-U <username>` flag. This can be done as follows:
+
+```
+initdb -D /var/postgresql/data -U postgres
+```
+If you are not using the default directory, you will have to update the `datadir` variable in the /etc/rc.d/postgresql script.
 
 When this is done, enable postgresql so that it starts on boot and start it. As root, run:
 ```
@@ -44,6 +56,7 @@ To check that it started properly and didn't fail right after starting, you can
 
 #### httpd
 httpd will have three fuctions:
+
   * redirect requests trying to reach the instance over http to the https URL
   * serve a robots.txt file
   * get Let's Encrypt certificates, with acme-client
@@ -73,12 +86,11 @@ server "default" {
 }
 
 types {
-	include "/usr/share/misc/mime.types"
 }
 ```
-Do not forget to change *\<IPv4/6 address\>* to your server's address(es). If httpd should only listen on one protocol family, comment one of the two first *listen* options.
+Do not forget to change *<IPv4/6 address\>* to your server's address(es). If httpd should only listen on one protocol family, comment one of the two first *listen* options.
 
-Create the /var/www/htdocs/local/ folder and write the content of your robots.txt in /var/www/htdocs/local/robots.txt.  
+Create the /var/www/htdocs/local/ folder and write the content of your robots.txt in /var/www/htdocs/local/robots.txt.
 Check the configuration with `httpd -n`, if it is OK enable and start httpd (as root):
 ```
 rcctl enable httpd
@@ -86,7 +98,7 @@ rcctl start httpd
 ```
 
 #### acme-client
-acme-client is used to get SSL/TLS certificates from Let's Encrypt. 
+acme-client is used to get SSL/TLS certificates from Let's Encrypt.
 Insert the following configuration in /etc/acme-client.conf:
 ```
 #
@@ -95,7 +107,7 @@ Insert the following configuration in /etc/acme-client.conf:
 
 authority letsencrypt-<domain name> {
 	#agreement url "https://letsencrypt.org/documents/LE-SA-v1.2-November-15-2017.pdf"
-	api url "https://acme-v01.api.letsencrypt.org/directory"
+	api url "https://acme-v02.api.letsencrypt.org/directory"
 	account key "/etc/acme/letsencrypt-privkey-<domain name>.pem"
 }
 
@@ -107,7 +119,7 @@ domain <domain name> {
 	challengedir "/var/www/acme/"
 }
 ```
-Replace *\<domain name\>* by the domain name you'll use for your instance. As root, run `acme-client -n` to check the config, then `acme-client -ADv <domain name>` to create account and domain keys, and request a certificate for the first time.  
+Replace *<domain name\>* by the domain name you'll use for your instance. As root, run `acme-client -n` to check the config, then `acme-client -ADv <domain name>` to create account and domain keys, and request a certificate for the first time.
 Make acme-client run everyday by adding it in /etc/daily.local. As root, run the following command: `echo "acme-client <domain name>" >> /etc/daily.local`.
 
 Relayd will look for certificates and keys based on the address it listens on (see next part), the easiest way to make them available to relayd is to create a link, as root run:
@@ -118,7 +130,7 @@ ln -s /etc/ssl/private/<domain name>.key /etc/ssl/private/<IP address>.key
 This will have to be done for each IPv4 and IPv6 address relayd listens on.
 
 #### relayd
-relayd will be used as the reverse proxy sitting in front of pleroma. 
+relayd will be used as the reverse proxy sitting in front of pleroma.
 Insert the following configuration in /etc/relayd.conf:
 ```
 # $OpenBSD: relayd.conf,v 1.4 2018/03/23 09:55:06 claudio Exp $
@@ -169,7 +181,7 @@ relay wwwtls {
 	forward to <httpd_server> port 80 check http "/robots.txt" code 200
 }
 ```
-Again, change *\<IPv4/6 address\>* to your server's address(es) and comment one of the two *listen* options if needed. Also change *wss://CHANGEME.tld* to *wss://\<your instance's domain name\>*.  
+Again, change *<IPv4/6 address\>* to your server's address(es) and comment one of the two *listen* options if needed. Also change *wss://CHANGEME.tld* to *wss://<your instance's domain name\>*.
 Check the configuration with `relayd -n`, if it is OK enable and start relayd (as root):
 ```
 rcctl enable relayd
@@ -177,7 +189,7 @@ rcctl start relayd
 ```
 
 #### pf
-Enabling and configuring pf is highly recommended.  
+Enabling and configuring pf is highly recommended.
 In /etc/pf.conf, insert the following configuration:
 ```
 # Macros
@@ -202,21 +214,31 @@ pass in quick on $if inet6 proto icmp6 to ($if) icmp6-type { echoreq unreach par
 pass in quick on $if proto tcp to ($if) port { http https } # relayd/httpd
 pass in quick on $if proto tcp from $authorized_ssh_clients to ($if) port ssh
 ```
-Replace *\<network interface\>* by your server's network interface name (which you can get with ifconfig). Consider replacing the content of the authorized\_ssh\_clients macro by, for exemple, your home IP address, to avoid SSH connection attempts from bots.
+Replace *<network interface\>* by your server's network interface name (which you can get with ifconfig). Consider replacing the content of the authorized\_ssh\_clients macro by, for exemple, your home IP address, to avoid SSH connection attempts from bots.
 
 Check pf's configuration by running `pfctl -nf /etc/pf.conf`, load it with `pfctl -f /etc/pf.conf` and enable pf at boot with `rcctl enable pf`.
 
 #### Configure and start pleroma
-Enter a shell as \_pleroma (as root `su _pleroma -`) and enter pleroma's installation directory (`cd ~/pleroma/`).  
+Enter a shell as \_pleroma (as root `su _pleroma -`) and enter pleroma's installation directory (`cd ~/pleroma/`).
+
 Then follow the main installation guide:
+
   * run `mix deps.get`
   * run `mix pleroma.instance gen` and enter your instance's information when asked
   * copy config/generated\_config.exs to config/prod.secret.exs. The default values should be sufficient but you should edit it and check that everything seems OK.
-  * exit your current shell back to a root one and run `psql -U postgres -f /home/_pleroma/config/setup_db.psql` to setup the database.
+  * exit your current shell back to a root one and run `psql -U postgres -f /home/_pleroma/pleroma/config/setup_db.psql` to setup the database.
   * return to a \_pleroma shell into pleroma's installation directory (`su _pleroma -;cd ~/pleroma`) and run `MIX_ENV=prod mix ecto.migrate`
 
-As \_pleroma in /home/\_pleroma/pleroma, you can now run `LC_ALL=en_US.UTF-8 MIX_ENV=prod mix phx.server` to start your instance.  
+As \_pleroma in /home/\_pleroma/pleroma, you can now run `LC_ALL=en_US.UTF-8 MIX_ENV=prod mix phx.server` to start your instance.
 In another SSH session/tmux window, check that it is working properly by running `ftp -MVo - http://127.0.0.1:4000/api/v1/instance`, you should get json output. Double-check that *uri*'s value is your instance's domain name.
 
 ##### Starting pleroma at boot
 An rc script to automatically start pleroma at boot hasn't been written yet, it can be run in a tmux session (tmux is in base).
+
+
+#### Create administrative user
+
+If your instance is up and running, you can create your first user with administrative rights with the following command as the \_pleroma user.
+```
+LC_ALL=en_US.UTF-8 MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress> --admin
+```

docs/installation/otp_en.md

@@ -6,7 +6,7 @@
 
 You will be running commands as root. If you aren't root already, please elevate your priviledges by executing `sudo su`/`su`.
 
-While in theory OTP releases are possbile to install on any compatible machine, for the sake of simplicity this guide focuses only on Debian/Ubuntu/Alpine.
+While in theory OTP releases are possbile to install on any compatible machine, for the sake of simplicity this guide focuses only on Debian/Ubuntu and Alpine.
 
 ### Detecting flavour
 
@@ -20,6 +20,7 @@ If your platform is supported the output will contain the flavour string, you wi
 ### Installing the required packages
 
 Other than things bundled in the OTP release Pleroma depends on:
+
 * curl (to download the release build)
 * unzip (needed to unpack release builds)
 * ncurses (ERTS won't run without it)
@@ -27,29 +28,26 @@ Other than things bundled in the OTP release Pleroma depends on:
 * nginx (could be swapped with another reverse proxy but this guide covers only it)
 * certbot (for Let's Encrypt certificates, could be swapped with another ACME client, but this guide covers only it)
 
-Debian/Ubuntu:
+```sh tab="Alpine"
-```sh
-apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot
-```
-Alpine:
-
-```sh
 echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories
 apk update
 apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot
 ```
 
+```sh tab="Debian/Ubuntu"
+apt install curl unzip libncurses5 postgresql postgresql-contrib nginx certbot
+```
+
 ## Setup
 ### Configuring PostgreSQL
 #### (Optional) Installing RUM indexes
-RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. You can read more about them on the [Configuration page](config.html#rum-indexing-for-full-text-search). They are completely optional and most of the time are not worth it, especially if you are running a single user instance (unless you absolutely need ordered search results).
 
-Debian/Ubuntu (available only on Buster/19.04):
+!!! warning
-```sh
+    It is recommended to use PostgreSQL v11 or newer. We have seen some minor issues with lower PostgreSQL versions.
-apt install postgresql-11-rum
+
-```
+RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. You can read more about them on the [Configuration page](../configuration/cheatsheet.md#rum-indexing-for-full-text-search). They are completely optional and most of the time are not worth it, especially if you are running a single user instance (unless you absolutely need ordered search results).
-Alpine:
+
-```sh
+```sh tab="Alpine"
 apk add git build-base postgresql-dev
 git clone https://github.com/postgrespro/rum /tmp/rum
 cd /tmp/rum
@@ -58,25 +56,40 @@ make USE_PGXS=1 install
 cd
 rm -r /tmp/rum
 ```
+
+```sh tab="Debian/Ubuntu"
+# Available only on Buster/19.04
+apt install postgresql-11-rum
+```
+
 #### (Optional) Performance configuration
 For optimal performance, you may use [PGTune](https://pgtune.leopard.in.ua), don't forget to restart postgresql after editing the configuration
 
-Debian/Ubuntu:
+```sh tab="Alpine"
-```sh
-systemctl restart postgresql
-```
-Alpine:
-```sh
 rc-service postgresql restart
 ```
+
+```sh tab="Debian/Ubuntu"
+systemctl restart postgresql
+```
+
+If you are using PostgreSQL 12 or higher, add this to your Ecto database configuration
+
+```elixir
+prepare: :named,
+parameters: [
+  plan_cache_mode: "force_custom_plan"
+]
+```
+
 ### Installing Pleroma
 ```sh
-# Create the Pleroma user
+# Create a Pleroma user
 adduser --system --shell  /bin/false --home /opt/pleroma pleroma
 
-# Set the flavour environment variable to the string you got in Detecting flavour section. 
+# Set the flavour environment variable to the string you got in Detecting flavour section.
-# For example if the flavour is `arm64-musl` the command will be
+# For example if the flavour is `amd64-musl` the command will be
-export FLAVOUR="arm64-musl"
+export FLAVOUR="amd64-musl"
 
 # Clone the release build into a temporary directory and unpack it
 su pleroma -s $SHELL -lc "
@@ -129,49 +142,52 @@ su pleroma -s $SHELL -lc "./bin/pleroma stop"
 
 ### Setting up nginx and getting Let's Encrypt SSL certificaties
 
+#### Get a Let's Encrypt certificate
 ```sh
-# Get a Let's Encrypt certificate
 certbot certonly --standalone --preferred-challenges http -d yourinstance.tld
+```
 
-# Copy the Pleroma nginx configuration to the nginx folder
+#### Copy Pleroma nginx configuration to the nginx folder
-# The location of nginx configs is dependent on the distro
 
-# For Debian/Ubuntu:
+The location of nginx configs is dependent on the distro
+
+```sh tab="Alpine"
+cp /opt/pleroma/installation/pleroma.nginx /etc/nginx/conf.d/pleroma.conf
+```
+
+```sh tab="Debian/Ubuntu"
 cp /opt/pleroma/installation/pleroma.nginx /etc/nginx/sites-available/pleroma.nginx
 ln -s /etc/nginx/sites-available/pleroma.nginx /etc/nginx/sites-enabled/pleroma.nginx
-# For Alpine:
+```
-cp /opt/pleroma/installation/pleroma.nginx /etc/nginx/conf.d/pleroma.conf
-# If your distro does not have either of those you can append
-# `include /etc/nginx/pleroma.conf` to the end of the http section in /etc/nginx/nginx.conf and
-cp /opt/pleroma/installation/pleroma.nginx /etc/nginx/pleroma.conf
 
-# Edit the nginx config replacing example.tld with your (sub)domain
+If your distro does not have either of those you can append `include /etc/nginx/pleroma.conf` to the end of the http section in /etc/nginx/nginx.conf and
+```sh
+cp /opt/pleroma/installation/pleroma.nginx /etc/nginx/pleroma.conf
+```
+
+#### Edit the nginx config
+```sh
+# Replace example.tld with your (sub)domain
 $EDITOR path-to-nginx-config
 
 # Verify that the config is valid
 nginx -t
+```
+#### Start nginx
 
-# Start nginx
+```sh tab="Alpine"
-# For Debian/Ubuntu:
-systemctl start nginx
-# For Alpine:
 rc-service nginx start
 ```
 
-At this point if you open your (sub)domain in a browser you should see a 502 error, that's because pleroma is not started yet.
+```sh tab="Debian/Ubuntu"
+systemctl start nginx
+```
+
+At this point if you open your (sub)domain in a browser you should see a 502 error, that's because Pleroma is not started yet.
 
 ### Setting up a system service
-Debian/Ubuntu:
-```sh
-# Copy the service into a proper directory
-cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service
 
-# Start pleroma and enable it on boot
+```sh tab="Alpine"
-systemctl start pleroma
-systemctl enable pleroma
-```
-Alpine:
-```sh
 # Copy the service into a proper directory
 cp /opt/pleroma/installation/init.d/pleroma /etc/init.d/pleroma
 
@@ -180,13 +196,22 @@ rc-service pleroma start
 rc-update add pleroma
 ```
 
-If everything worked, you should see Pleroma-FE when visiting your domain. If that didn't happen, try reviewing the installation steps, starting Pleroma in the foreground and seeing if there are any errrors. 
+```sh tab="Debian/Ubuntu"
+# Copy the service into a proper directory
+cp /opt/pleroma/installation/pleroma.service /etc/systemd/system/pleroma.service
 
-Still doesn't work? Feel free to contact us on [#pleroma on freenode](https://webchat.freenode.net/?channels=%23pleroma) or via matrix at <https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org>, you can also [file an issue on our Gitlab](https://git.pleroma.social/pleroma/pleroma/issues/new)
+# Start pleroma and enable it on boot
+systemctl start pleroma
+systemctl enable pleroma
+```
+
+If everything worked, you should see Pleroma-FE when visiting your domain. If that didn't happen, try reviewing the installation steps, starting Pleroma in the foreground and seeing if there are any errrors.
+
+Still doesn't work? Feel free to contact us on [#pleroma on freenode](https://irc.pleroma.social) or via matrix at <https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org>, you can also [file an issue on our Gitlab](https://git.pleroma.social/pleroma/pleroma-support/issues/new)
 
 ## Post installation
 
-### Setting up auto-renew Let's Encrypt certificate
+### Setting up auto-renew of the Let's Encrypt certificate
 ```sh
 # Create the directory for webroot challenges
 mkdir -p /var/lib/letsencrypt
@@ -197,25 +222,8 @@ $EDITOR path-to-nginx-config
 # Verify that the config is valid
 nginx -t
 ```
-Debian/Ubuntu:
-```sh
-# Restart nginx
-systemctl restart nginx
 
-# Ensure the webroot menthod and post hook is working
+```sh tab="Alpine"
-certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --dry-run --post-hook 'systemctl nginx reload'
-
-# Add it to the daily cron
-echo '#!/bin/sh
-certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --post-hook "systemctl reload nginx"
-' > /etc/cron.daily/renew-pleroma-cert
-chmod +x /etc/cron.daily/renew-pleroma-cert
-
-# If everything worked the output should contain /etc/cron.daily/renew-pleroma-cert
-run-parts --test /etc/cron.daily
-```
-Alpine:
-```sh
 # Restart nginx
 rc-service nginx restart
 
@@ -232,15 +240,25 @@ certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --
 ' > /etc/periodic/daily/renew-pleroma-cert
 chmod +x /etc/periodic/daily/renew-pleroma-cert
 
-# If everything worked this should output /etc/periodic/daily/renew-pleroma-cert
+# If everything worked the output should contain /etc/cron.daily/renew-pleroma-cert
 run-parts --test /etc/periodic/daily
 ```
-### Running mix tasks
-Throughout the wiki and guides there is a lot of references to mix tasks. Since `mix` is a build tool, you can't just call `mix pleroma.task`, instead you should call `pleroma_ctl` stripping pleroma/ecto namespace.
 
-So for example, if the task is `mix pleroma.user set admin --admin`, you should run it like this:
+```sh tab="Debian/Ubuntu"
-```sh
+# Restart nginx
-su pleroma -s $SHELL -lc "./bin/pleroma_ctl user set admin --admin"
+systemctl restart nginx
+
+# Ensure the webroot menthod and post hook is working
+certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --dry-run --post-hook 'systemctl reload nginx'
+
+# Add it to the daily cron
+echo '#!/bin/sh
+certbot renew --cert-name yourinstance.tld --webroot -w /var/lib/letsencrypt/ --post-hook "systemctl reload nginx"
+' > /etc/cron.daily/renew-pleroma-cert
+chmod +x /etc/cron.daily/renew-pleroma-cert
+
+# If everything worked the output should contain /etc/cron.daily/renew-pleroma-cert
+run-parts --test /etc/cron.daily
 ```
 
 ## Create your first user and set as admin
@@ -250,20 +268,14 @@ su pleroma -s $SHELL -lc "./bin/pleroma_ctl user new joeuser joeuser@sld.tld --a
 ```
 This will create an account withe the username of 'joeuser' with the email address of joeuser@sld.tld, and set that user's account as an admin. This will result in a link that you can paste into the browser, which logs you in and enables you to set the password.
 
-### Updating
-Generally, doing the following is enough:
-```sh
-# Download the new release
-su pleroma -s $SHELL -lc "./bin/pleroma_ctl update"
-
-# Migrate the database, you are advised to stop the instance before doing that
-su pleroma -s $SHELL -lc "./bin/pleroma_ctl migrate"
-```
-But you should **always check the release notes/changelog** in case there are config deprecations, special update steps, etc.
-
 ## Further reading
-* [Configuration](config.html)
+
-* [Pleroma's base config.exs](https://git.pleroma.social/pleroma/pleroma/blob/master/config/config.exs)
+* [Backup your instance](../administration/backup.md)
-* [Hardening your instance](hardening.html)
+* [Hardening your instance](../configuration/hardening.md)
-* [Pleroma Clients](clients.html)
+* [How to activate mediaproxy](../configuration/howto_mediaproxy.md)
-* [Emoji pack manager](Mix.Tasks.Pleroma.Emoji.html)
+* [Updating your instance](../administration/updating.md)
+
+## Questions
+
+Questions about the installation or didn’t it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**.
+

docs/introduction.md

@@ -3,53 +3,63 @@
 Pleroma is a federated social networking platform, compatible with GNU social, Mastodon and other OStatus and ActivityPub implementations. It is free software licensed under the AGPLv3.
 It actually consists of two components: a backend, named simply Pleroma, and a user-facing frontend, named Pleroma-FE. It also includes the Mastodon frontend, if that's your thing.
 It's part of what we call the fediverse, a federated network of instances which speak common protocols and can communicate with each other.
-One account on a instance is enough to talk to the entire fediverse!
+One account on an instance is enough to talk to the entire fediverse!
 
 ## How can I use it?
 
-Pleroma instances are already widely deployed, a list can be found here:
+Pleroma instances are already widely deployed, a list can be found at <http://distsn.org/pleroma-instances.html>. Information on all existing fediverse instances can be found at <https://fediverse.network/>.
-http://distsn.org/pleroma-instances.html
 
 If you don't feel like joining an existing instance, but instead prefer to deploy your own instance, that's easy too!
-Installation instructions can be found here:
+Installation instructions can be found in the installation section of these docs.
-[main Pleroma wiki](/)
 
 ## I got an account, now what?
-Great! Now you can explore the fediverse!
+Great! Now you can explore the fediverse! Open the login page for your Pleroma instance (e.g. <https://pleroma.soykaf.com>) and login with your username and password. (If you don't have an account yet, click on Register)
-- Open the login page for your Pleroma instance (for ex. https://pleroma.soykaf.com) and login with your username and password.
-(If you don't have one yet, click on Register) :slightly_smiling_face:
 
 At this point you will have two columns in front of you.
 
 ### Left column
-- first block: here you can see your avatar, your nickname a bio, and statistics (Statuses, Following, Followers).
+
-Under that you have a text form which allows you to post new statuses. The icon on the left is for uploading media files and attach them to your post. The number under the text form is a character counter, every instance can have a different character limit (the default is 5000).
+- first block: here you can see your avatar, your nickname and statistics (Statuses, Following, Followers). Clicking your profile pic will open your profile.
-If you want to mention someone, type @ + name of the person. A drop-down menu will help you in finding the right person. :slight_smile:
+Under that you have a text form which allows you to post new statuses. The number on the bottom of the text form is a character counter, every instance can have a different character limit (the default is 5000).
+If you want to mention someone, type @ + name of the person. A drop-down menu will help you in finding the right person.
+Under the text form there are also several visibility options and there is the option to use rich text.
+Under that the icon on the left is for uploading media files and attach them to your post. There is also an emoji-picker and an option to post a poll.
 To post your status, simply press Submit.
+On the top right you will also see a wrench icon. This opens your personal settings.
 
 - second block: Here you can switch between the different timelines:
-  - Timeline: all the people that you follow
+   - Timeline: all the people that you follow
-  - Mentions: all the statutes where you are mentioned
+   - Interactions: here you can switch between different timelines where there was interaction with your account. There is Mentions, Repeats and Favorites, and New follows
-  - Public Timeline: all the statutes from the local instance
+   - Direct Messages: these are the Direct Messages sent to you
-  - The Whole Known Network: everything, local and remote!
+   - Public Timeline: all the statutes from the local instance
-
+   - The Whole Known Network: all public posts the instance knows about, both local and remote!
-- third block: this is the Chat block, where you communicate with people on the same instance in realtime. It is local-only, for now, but we're planning to make it extendable to the entire fediverse! :sweat_smile:
+   - About: This isn't a Timeline but shows relevant info about the instance. You can find a list of the moderators and admins, Terms of Service, MRF policies and enabled features.
-
+- Optional third block: This is the Instance panel that can be activated, but is deactivated by default. It's fully customisable and by default has links to the pleroma-fe and Mastodon-fe.
 - fourth block: This is the Notifications block, here you will get notified whenever somebody mentions you, follows you, repeats or favorites one of your statuses.
 
 ### Right column
-This is where the interesting stuff happens! :slight_smile:
+This is where the interesting stuff happens!
 Depending on the timeline you will see different statuses, but each status has a standard structure:
-- Icon + name + link to profile. An optional left-arrow if it's a reply to another status (hovering will reveal the replied-to status).
-- A + button on the right allows you to Expand/Collapse an entire discussion thread. It also updates in realtime!
-- A binocular icon allows you to open the status on the instance where it's originating from.
-- The text of the status, including mentions. If you click on a mention, it will automatically open the profile page of that person.
-- Four buttons (left to right): Reply, Repeat, Favorite, Delete.
 
-## Mastodon interface
+- Profile pic, name and link to profile. An optional left-arrow if it's a reply to another status (hovering will reveal the reply-to status). Clicking on the profile pic will uncollapse the user's profile.
-If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too! :smile:
+- A `+` button on the right allows you to Expand/Collapse an entire discussion thread. It also updates in realtime!
-Just add a "/web" after your instance url (for ex. https://pleroma.soycaf.com/web) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC! :fireworks:
+- An arrow icon allows you to open the status on the instance where it's originating from.
-For more information on the Mastodon interface, please look here:
+- The text of the status, including mentions and attachements. If you click on a mention, it will automatically open the profile page of that person.
-https://github.com/tootsuite/documentation/blob/master/Using-Mastodon/User-guide.md
+- Three buttons (left to right): Reply, Repeat, Favorite. There is also a forth button, this is a dropdown menu for simple moderation like muting the conversation or, if you have moderation rights, delete the status from the server.
+
+### Top right
+
+- The magnifier icon opens the search screen where you can search for statuses, people and hashtags. It's also possible to import statusses from remote servers by pasting the url to the post in the search field.
+- The gear icon gives you general settings
+- If you have admin rights, you'll see an icon that opens the admin interface
+- The last icon is to log out
+
+### Bottom right
+On the bottom right you have a chatbox. Here you can communicate with people on the same instance in realtime. It is local-only, for now, but there are plans to make it extendable to the entire fediverse!
+
+### Mastodon interface
+If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too!
+Just add a "/web" after your instance url (e.g. <https://pleroma.soycaf.com/web>) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC!
+The Mastodon interface is from the Glitch-soc fork. For more information on the Mastodon interface you can check the [Mastodon](https://docs.joinmastodon.org/) and [Glitch-soc](https://glitch-soc.github.io/docs/) documentation.
 
 Remember, what you see is only the frontend part of Mastodon, the backend is still Pleroma.

installation/download-mastofe-build.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 project_id="74"
 project_branch="rebase/glitch-soc"

installation/pleroma-mongooseim.cfg

@@ -215,7 +215,9 @@
       ]}
   ]},
 
-  { 5222, ejabberd_c2s, [
+  %% If you want dual stack, you have to clone this entire config stanza
+  %% and change the bind to "::"
+  { {5222, "0.0.0.0"}, ejabberd_c2s, [
 
 			%%
 			%% If TLS is compiled in and you installed a SSL
@@ -246,7 +248,9 @@
   %%			{max_stanza_size, 65536}
   %%		       ]},
 
-  { 5269, ejabberd_s2s_in, [
+  %% If you want dual stack, you have to clone this entire config stanza
+  %% and change the bind to "::"
+  { {5269, "0.0.0.0"}, ejabberd_s2s_in, [
 			   {shaper, s2s_shaper},
 			   {max_stanza_size, 131072},
 			   {protocol_options, ["no_sslv3"]}

installation/pleroma.nginx

@@ -70,6 +70,7 @@ server {
         proxy_set_header Upgrade $http_upgrade;
         proxy_set_header Connection "upgrade";
         proxy_set_header Host $http_host;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
 
         # this is explicitly IPv4 since Pleroma.Web.Endpoint binds on IPv4 only
         # and `localhost.` resolves to [::0] on some systems: see issue #930

lib/jason_types.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 Postgrex.Types.define(

lib/mix/pleroma.ex

@@ -1,12 +1,30 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Pleroma do
   @doc "Common functions to be reused in mix tasks"
   def start_pleroma do
     Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
+
+    if Pleroma.Config.get(:env) != :test do
+      Application.put_env(:logger, :console, level: :debug)
+    end
+
     {:ok, _} = Application.ensure_all_started(:pleroma)
+
+    if Pleroma.Config.get(:env) not in [:test, :benchmark] do
+      pleroma_rebooted?()
+    end
+  end
+
+  defp pleroma_rebooted? do
+    if Restarter.Pleroma.rebooted?() do
+      :ok
+    else
+      Process.sleep(10)
+      pleroma_rebooted?()
+    end
   end
 
   def load_pleroma do

lib/mix/tasks/pleroma/benchmark.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Benchmark do

lib/mix/tasks/pleroma/config.ex

@@ -1,72 +1,150 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Config do
-  @doc false
   use Mix.Task
+
   import Mix.Pleroma
+
+  alias Pleroma.ConfigDB
   alias Pleroma.Repo
-  alias Pleroma.Web.AdminAPI.Config
+
+  @shortdoc "Manages the location of the config"
+  @moduledoc File.read!("docs/administration/CLI_tasks/config.md")
 
   def run(["migrate_to_db"]) do
     start_pleroma()
-
+    migrate_to_db()
-    if Pleroma.Config.get([:instance, :dynamic_configuration]) do
-      Application.get_all_env(:pleroma)
-      |> Enum.reject(fn {k, _v} -> k in [Pleroma.Repo, :env] end)
-      |> Enum.each(fn {k, v} ->
-        key = to_string(k) |> String.replace("Elixir.", "")
-
-        key =
-          if String.starts_with?(key, "Pleroma.") do
-            key
-          else
-            ":" <> key
-          end
-
-        {:ok, _} = Config.update_or_create(%{group: "pleroma", key: key, value: v})
-        Mix.shell().info("#{key} is migrated.")
-      end)
-
-      Mix.shell().info("Settings migrated.")
-    else
-      Mix.shell().info(
-        "Migration is not allowed by config. You can change this behavior in instance settings."
-      )
-    end
   end
 
-  def run(["migrate_from_db", env, delete?]) do
+  def run(["migrate_from_db" | options]) do
     start_pleroma()
 
-    delete? = if delete? == "true", do: true, else: false
+    {opts, _} =
-
+      OptionParser.parse!(options,
-    if Pleroma.Config.get([:instance, :dynamic_configuration]) do
+        strict: [env: :string, delete: :boolean],
-      config_path = "config/#{env}.exported_from_db.secret.exs"
+        aliases: [d: :delete]
-
-      {:ok, file} = File.open(config_path, [:write])
-      IO.write(file, "use Mix.Config\r\n")
-
-      Repo.all(Config)
-      |> Enum.each(fn config ->
-        IO.write(
-          file,
-          "config :#{config.group}, #{config.key}, #{inspect(Config.from_binary(config.value))}\r\n\r\n"
-        )
-
-        if delete? do
-          {:ok, _} = Repo.delete(config)
-          Mix.shell().info("#{config.key} deleted from DB.")
-        end
-      end)
-
-      File.close(file)
-      System.cmd("mix", ["format", config_path])
-    else
-      Mix.shell().info(
-        "Migration is not allowed by config. You can change this behavior in instance settings."
       )
+
+    migrate_from_db(opts)
+  end
+
+  @spec migrate_to_db(Path.t() | nil) :: any()
+  def migrate_to_db(file_path \\ nil) do
+    if Pleroma.Config.get([:configurable_from_database]) do
+      config_file =
+        if file_path do
+          file_path
+        else
+          if Pleroma.Config.get(:release) do
+            Pleroma.Config.get(:config_path)
+          else
+            "config/#{Pleroma.Config.get(:env)}.secret.exs"
+          end
+        end
+
+      do_migrate_to_db(config_file)
+    else
+      migration_error()
     end
   end
+
+  defp do_migrate_to_db(config_file) do
+    if File.exists?(config_file) do
+      Ecto.Adapters.SQL.query!(Repo, "TRUNCATE config;")
+      Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;")
+
+      custom_config =
+        config_file
+        |> read_file()
+        |> elem(0)
+
+      custom_config
+      |> Keyword.keys()
+      |> Enum.each(&create(&1, custom_config))
+    else
+      shell_info("To migrate settings, you must define custom settings in #{config_file}.")
+    end
+  end
+
+  defp create(group, settings) do
+    group
+    |> Pleroma.Config.Loader.filter_group(settings)
+    |> Enum.each(fn {key, value} ->
+      key = inspect(key)
+      {:ok, _} = ConfigDB.update_or_create(%{group: inspect(group), key: key, value: value})
+
+      shell_info("Settings for key #{key} migrated.")
+    end)
+
+    shell_info("Settings for group :#{group} migrated.")
+  end
+
+  defp migrate_from_db(opts) do
+    if Pleroma.Config.get([:configurable_from_database]) do
+      env = opts[:env] || "prod"
+
+      config_path =
+        if Pleroma.Config.get(:release) do
+          :config_path
+          |> Pleroma.Config.get()
+          |> Path.dirname()
+        else
+          "config"
+        end
+        |> Path.join("#{env}.exported_from_db.secret.exs")
+
+      file = File.open!(config_path, [:write, :utf8])
+
+      IO.write(file, config_header())
+
+      ConfigDB
+      |> Repo.all()
+      |> Enum.each(&write_and_delete(&1, file, opts[:delete]))
+
+      :ok = File.close(file)
+      System.cmd("mix", ["format", config_path])
+    else
+      migration_error()
+    end
+  end
+
+  defp migration_error do
+    shell_error(
+      "Migration is not allowed in config. You can change this behavior by setting `configurable_from_database` to true."
+    )
+  end
+
+  if Code.ensure_loaded?(Config.Reader) do
+    defp config_header, do: "import Config\r\n\r\n"
+    defp read_file(config_file), do: Config.Reader.read_imports!(config_file)
+  else
+    defp config_header, do: "use Mix.Config\r\n\r\n"
+    defp read_file(config_file), do: Mix.Config.eval!(config_file)
+  end
+
+  defp write_and_delete(config, file, delete?) do
+    config
+    |> write(file)
+    |> delete(delete?)
+  end
+
+  defp write(config, file) do
+    value =
+      config.value
+      |> ConfigDB.from_binary()
+      |> inspect(limit: :infinity)
+
+    IO.write(file, "config #{config.group}, #{config.key}, #{value}\r\n\r\n")
+
+    config
+  end
+
+  defp delete(config, true) do
+    {:ok, _} = Repo.delete(config)
+    shell_info("#{config.key} deleted from DB.")
+  end
+
+  defp delete(_config, _), do: :ok
 end

lib/mix/tasks/pleroma/count_statuses.ex

@@ -0,0 +1,22 @@
+defmodule Mix.Tasks.Pleroma.CountStatuses do
+  @shortdoc "Re-counts statuses for all users"
+
+  use Mix.Task
+  alias Pleroma.User
+  import Ecto.Query
+
+  def run([]) do
+    Mix.Pleroma.start_pleroma()
+
+    stream =
+      User
+      |> where(local: true)
+      |> Pleroma.Repo.stream()
+
+    Pleroma.Repo.transaction(fn ->
+      Enum.each(stream, &User.update_note_count/1)
+    end)
+
+    Mix.Pleroma.shell_info("Done")
+  end
+end

lib/mix/tasks/pleroma/database.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Database do
@@ -13,34 +13,8 @@ defmodule Mix.Tasks.Pleroma.Database do
   use Mix.Task
 
   @shortdoc "A collection of database related tasks"
-  @moduledoc """
+  @moduledoc File.read!("docs/administration/CLI_tasks/database.md")
-   A collection of database related tasks
 
-   ## Replace embedded objects with their references
-
-   Replaces embedded objects with references to them in the `objects` table. Only needs to be ran once. The reason why this is not a migration is because it could significantly increase the database size after being ran, however after this `VACUUM FULL` will be able to reclaim about 20% (really depends on what is in the database, your mileage may vary) of the db size before the migration.
-
-       mix pleroma.database remove_embedded_objects
-
-    Options:
-    - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
-
-  ## Prune old objects from the database
-
-      mix pleroma.database prune_objects
-
-  ## Create a conversation for all existing DMs. Can be safely re-run.
-
-      mix pleroma.database bump_all_conversations
-
-  ## Remove duplicated items from following and update followers count for all users
-
-      mix pleroma.database update_users_following_followers_counts
-
-  ## Fix the pre-existing "likes" collections for all objects
-
-      mix pleroma.database fix_likes_collections
-  """
   def run(["remove_embedded_objects" | args]) do
     {options, [], []} =
       OptionParser.parse(
@@ -78,9 +52,9 @@ def run(["bump_all_conversations"]) do
   def run(["update_users_following_followers_counts"]) do
     start_pleroma()
 
-    users = Repo.all(User)
+    User
-    Enum.each(users, &User.remove_duplicated_following/1)
+    |> Repo.all()
-    Enum.each(users, &User.update_follower_count/1)
+    |> Enum.each(&User.update_follower_count/1)
   end
 
   def run(["prune_objects" | args]) do

lib/mix/tasks/pleroma/digest.ex

@@ -2,16 +2,8 @@ defmodule Mix.Tasks.Pleroma.Digest do
   use Mix.Task
 
   @shortdoc "Manages digest emails"
-  @moduledoc """
+  @moduledoc File.read!("docs/administration/CLI_tasks/digest.md")
-  Manages digest emails
 
-  ## Send digest email since given date (user registration date by default)
-  ignoring user activity status.
-
-  ``mix pleroma.digest test <nickname> <since_date>``
-
-  Example: ``mix pleroma.digest test donaldtheduck 2019-05-20``
-  """
   def run(["test", nickname | opts]) do
     Mix.Pleroma.start_pleroma()
 

lib/mix/tasks/pleroma/docs.ex

@@ -28,7 +28,7 @@ def run(_) do
   defp do_run(implementation) do
     start_pleroma()
 
-    with {descriptions, _paths} <- Mix.Config.eval!("config/description.exs"),
+    with descriptions <- Pleroma.Config.Loader.load("config/description.exs"),
          {:ok, file_path} <-
            Pleroma.Docs.Generator.process(
              implementation,

lib/mix/tasks/pleroma/ecto/ecto.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-onl
 
 defmodule Mix.Tasks.Pleroma.Ecto do

lib/mix/tasks/pleroma/ecto/migrate.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-onl
 
 defmodule Mix.Tasks.Pleroma.Ecto.Migrate do

lib/mix/tasks/pleroma/ecto/rollback.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-onl
 
 defmodule Mix.Tasks.Pleroma.Ecto.Rollback do

lib/mix/tasks/pleroma/email.ex

@@ -0,0 +1,24 @@
+defmodule Mix.Tasks.Pleroma.Email do
+  use Mix.Task
+  import Mix.Pleroma
+
+  @shortdoc "Simple Email test"
+  @moduledoc File.read!("docs/administration/CLI_tasks/email.md")
+
+  def run(["test" | args]) do
+    Mix.Pleroma.start_pleroma()
+
+    {options, [], []} =
+      OptionParser.parse(
+        args,
+        strict: [
+          to: :string
+        ]
+      )
+
+    email = Pleroma.Emails.AdminEmail.test_email(options[:to])
+    {:ok, _} = Pleroma.Emails.Mailer.deliver(email)
+
+    shell_info("Test email has been sent to #{inspect(email.to)} from #{inspect(email.from)}")
+  end
+end

lib/mix/tasks/pleroma/emoji.ex

@@ -1,61 +1,15 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Emoji do
   use Mix.Task
 
   @shortdoc "Manages emoji packs"
-  @moduledoc """
+  @moduledoc File.read!("docs/administration/CLI_tasks/emoji.md")
-  Manages emoji packs
-
-  ## ls-packs
-
-      mix pleroma.emoji ls-packs [OPTION...]
-
-  Lists the emoji packs and metadata specified in the manifest.
-
-  ### Options
-
-  - `-m, --manifest PATH/URL` - path to a custom manifest, it can
-    either be an URL starting with `http`, in that case the
-    manifest will be fetched from that address, or a local path
-
-  ## get-packs
-
-      mix pleroma.emoji get-packs [OPTION...] PACKS
-
-  Fetches, verifies and installs the specified PACKS from the
-  manifest into the `STATIC-DIR/emoji/PACK-NAME`
-
-  ### Options
-
-  - `-m, --manifest PATH/URL` - same as ls-packs
-
-  ## gen-pack
-
-      mix pleroma.emoji gen-pack PACK-URL
-
-  Creates a new manifest entry and a file list from the specified
-  remote pack file. Currently, only .zip archives are recognized
-  as remote pack files and packs are therefore assumed to be zip
-  archives. This command is intended to run interactively and will
-  first ask you some basic questions about the pack, then download
-  the remote file and generate an SHA256 checksum for it, then
-  generate an emoji file list for you.
-
-  The manifest entry will either be written to a newly created
-  `index.json` file or appended to the existing one, *replacing*
-  the old pack with the same name if it was in the file previously.
-
-  The file list will be written to the file specified previously,
-  *replacing* that file. You _should_ check that the file list doesn't
-  contain anything you don't need in the pack, that is, anything that is
-  not an emoji (the whole pack is downloaded, but only emoji files
-  are extracted).
-  """
 
   def run(["ls-packs" | args]) do
+    Mix.Pleroma.start_pleroma()
     Application.ensure_all_started(:hackney)
 
     {options, [], []} = parse_global_opts(args)
@@ -82,6 +36,7 @@ def run(["ls-packs" | args]) do
   end
 
   def run(["get-packs" | args]) do
+    Mix.Pleroma.start_pleroma()
     Application.ensure_all_started(:hackney)
 
     {options, pack_names, []} = parse_global_opts(args)
@@ -158,19 +113,21 @@ def run(["get-packs" | args]) do
             file_list: files_to_unzip
           )
 
-        IO.puts(IO.ANSI.format(["Writing emoji.txt for ", :bright, pack_name]))
+        IO.puts(IO.ANSI.format(["Writing pack.json for ", :bright, pack_name]))
 
-        emoji_txt_str =
+        pack_json = %{
-          Enum.map(
+          pack: %{
-            files,
+            "license" => pack["license"],
-            fn {shortcode, path} ->
+            "homepage" => pack["homepage"],
-              emojo_path = Path.join("/emoji/#{pack_name}", path)
+            "description" => pack["description"],
-              "#{shortcode}, #{emojo_path}"
+            "fallback-src" => pack["src"],
-            end
+            "fallback-src-sha256" => pack["src_sha256"],
-          )
+            "share-files" => true
-          |> Enum.join("\n")
+          },
+          files: files
+        }
 
-        File.write!(Path.join(pack_path, "emoji.txt"), emoji_txt_str)
+        File.write!(Path.join(pack_path, "pack.json"), Jason.encode!(pack_json, pretty: true))
       else
         IO.puts(IO.ANSI.format([:bright, :red, "No pack named \"#{pack_name}\" found"]))
       end
@@ -229,13 +186,9 @@ def run(["gen-pack", src]) do
 
     tmp_pack_dir = Path.join(System.tmp_dir!(), "emoji-pack-#{name}")
 
-    {:ok, _} =
+    {:ok, _} = :zip.unzip(binary_archive, cwd: String.to_charlist(tmp_pack_dir))
-      :zip.unzip(
-        binary_archive,
-        cwd: tmp_pack_dir
-      )
 
-    emoji_map = Pleroma.Emoji.make_shortcode_to_file_map(tmp_pack_dir, exts)
+    emoji_map = Pleroma.Emoji.Loader.make_shortcode_to_file_map(tmp_pack_dir, exts)
 
     File.write!(files_name, Jason.encode!(emoji_map, pretty: true))
 

lib/mix/tasks/pleroma/instance.ex

@@ -1,41 +1,15 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Instance do
   use Mix.Task
   import Mix.Pleroma
 
+  alias Pleroma.Config
+
   @shortdoc "Manages Pleroma instance"
-  @moduledoc """
+  @moduledoc File.read!("docs/administration/CLI_tasks/instance.md")
-  Manages Pleroma instance.
-
-  ## Generate a new instance config.
-
-    mix pleroma.instance gen [OPTION...]
-
-  If any options are left unspecified, you will be prompted interactively
-
-  ## Options
-
-  - `-f`, `--force` - overwrite any output files
-  - `-o PATH`, `--output PATH` - the output file for the generated configuration
-  - `--output-psql PATH` - the output file for the generated PostgreSQL setup
-  - `--domain DOMAIN` - the domain of your instance
-  - `--instance-name INSTANCE_NAME` - the name of your instance
-  - `--admin-email ADMIN_EMAIL` - the email address of the instance admin
-  - `--notify-email NOTIFY_EMAIL` - email address for notifications
-  - `--dbhost HOSTNAME` - the hostname of the PostgreSQL database to use
-  - `--dbname DBNAME` - the name of the database to use
-  - `--dbuser DBUSER` - the user (aka role) to use for the database connection
-  - `--dbpass DBPASS` - the password to use for the database connection
-  - `--rum Y/N` - Whether to enable RUM indexes
-  - `--indexable Y/N` - Allow/disallow indexing site by search engines
-  - `--uploads-dir` - the directory uploads go in when using a local uploader
-  - `--static-dir` - the directory custom public files should be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)
-  - `--listen-ip` - the ip the app should listen to, defaults to 127.0.0.1
-  - `--listen-port` - the port the app should listen to, defaults to 4000
-  """
 
   def run(["gen" | rest]) do
     {options, [], []} =
@@ -55,6 +29,7 @@ def run(["gen" | rest]) do
           dbpass: :string,
           rum: :string,
           indexable: :string,
+          db_configurable: :string,
           uploads_dir: :string,
           static_dir: :string,
           listen_ip: :string,
@@ -90,7 +65,8 @@ def run(["gen" | rest]) do
         get_option(
           options,
           :instance_name,
-          "What is the name of your instance? (e.g. Pleroma/Soykaf)"
+          "What is the name of your instance? (e.g. The Corndog Emporium)",
+          domain
         )
 
       email = get_option(options, :admin_email, "What is your admin email address?")
@@ -111,6 +87,14 @@ def run(["gen" | rest]) do
           "y"
         ) === "y"
 
+      db_configurable? =
+        get_option(
+          options,
+          :db_configurable,
+          "Do you want to store the configuration in the database (allows controlling it from admin-fe)? (y/n)",
+          "n"
+        ) === "y"
+
       dbhost = get_option(options, :dbhost, "What is the hostname of your database?", "localhost")
 
       dbname = get_option(options, :dbname, "What is the name of your database?", "pleroma")
@@ -172,6 +156,8 @@ def run(["gen" | rest]) do
           Pleroma.Config.get([:instance, :static_dir])
         )
 
+      Config.put([:instance, :static_dir], static_dir)
+
       secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
       jwt_secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64)
       signing_salt = :crypto.strong_rand_bytes(8) |> Base.encode64() |> binary_part(0, 8)
@@ -195,7 +181,7 @@ def run(["gen" | rest]) do
           signing_salt: signing_salt,
           web_push_public_key: Base.url_encode64(web_push_public_key, padding: false),
           web_push_private_key: Base.url_encode64(web_push_private_key, padding: false),
-          db_configurable?: false,
+          db_configurable?: db_configurable?,
           static_dir: static_dir,
           uploads_dir: uploads_dir,
           rum_enabled: rum_enabled,
@@ -221,8 +207,14 @@ def run(["gen" | rest]) do
       write_robots_txt(indexable, template_dir)
 
       shell_info(
-        "\n All files successfully written! Refer to the installation instructions for your platform for next steps"
+        "\n All files successfully written! Refer to the installation instructions for your platform for next steps."
       )
+
+      if db_configurable? do
+        shell_info(
+          " Please transfer your config to the database after running database migrations. Refer to \"Transfering the config to/from the database\" section of the docs for more information."
+        )
+      end
     else
       shell_error(
         "The task would have overwritten the following files:\n" <>

lib/mix/tasks/pleroma/notification_settings.ex

@@ -0,0 +1,83 @@
+defmodule Mix.Tasks.Pleroma.NotificationSettings do
+  @shortdoc "Enable&Disable privacy option for push notifications"
+  @moduledoc """
+  Example:
+
+  > mix pleroma.notification_settings --privacy-option=false --nickname-users="parallel588"  # set false only for parallel588 user
+  > mix pleroma.notification_settings --privacy-option=true # set true for all users
+
+  """
+
+  use Mix.Task
+  import Mix.Pleroma
+  import Ecto.Query
+
+  def run(args) do
+    start_pleroma()
+
+    {options, _, _} =
+      OptionParser.parse(
+        args,
+        strict: [
+          privacy_option: :boolean,
+          email_users: :string,
+          nickname_users: :string
+        ]
+      )
+
+    privacy_option = Keyword.get(options, :privacy_option)
+
+    if not is_nil(privacy_option) do
+      privacy_option
+      |> build_query(options)
+      |> Pleroma.Repo.update_all([])
+    end
+
+    shell_info("Done")
+  end
+
+  defp build_query(privacy_option, options) do
+    query =
+      from(u in Pleroma.User,
+        update: [
+          set: [
+            notification_settings:
+              fragment(
+                "jsonb_set(notification_settings, '{privacy_option}', ?)",
+                ^privacy_option
+              )
+          ]
+        ]
+      )
+
+    user_emails =
+      options
+      |> Keyword.get(:email_users, "")
+      |> String.split(",")
+      |> Enum.map(&String.trim(&1))
+      |> Enum.reject(&(&1 == ""))
+
+    query =
+      if length(user_emails) > 0 do
+        where(query, [u], u.email in ^user_emails)
+      else
+        query
+      end
+
+    user_nicknames =
+      options
+      |> Keyword.get(:nickname_users, "")
+      |> String.split(",")
+      |> Enum.map(&String.trim(&1))
+      |> Enum.reject(&(&1 == ""))
+
+    query =
+      if length(user_nicknames) > 0 do
+        where(query, [u], u.nickname in ^user_nicknames)
+      else
+        query
+      end
+
+    query
+  end
+end

lib/mix/tasks/pleroma/refresh_counter_cache.ex

@@ -0,0 +1,46 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.RefreshCounterCache do
+  @shortdoc "Refreshes counter cache"
+
+  use Mix.Task
+
+  alias Pleroma.Activity
+  alias Pleroma.CounterCache
+  alias Pleroma.Repo
+
+  require Logger
+  import Ecto.Query
+
+  def run([]) do
+    Mix.Pleroma.start_pleroma()
+
+    ["public", "unlisted", "private", "direct"]
+    |> Enum.each(fn visibility ->
+      count = status_visibility_count_query(visibility)
+      name = "status_visibility_#{visibility}"
+      CounterCache.set(name, count)
+      Mix.Pleroma.shell_info("Set #{name} to #{count}")
+    end)
+
+    Mix.Pleroma.shell_info("Done")
+  end
+
+  defp status_visibility_count_query(visibility) do
+    Activity
+    |> where(
+      [a],
+      fragment(
+        "activity_visibility(?, ?, ?) = ?",
+        a.actor,
+        a.recipients,
+        a.data,
+        ^visibility
+      )
+    )
+    |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
+    |> Repo.aggregate(:count, :id, timeout: :timer.minutes(30))
+  end
+end

lib/mix/tasks/pleroma/relay.ex

@@ -1,33 +1,15 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Relay do
   use Mix.Task
   import Mix.Pleroma
-  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Relay
 
   @shortdoc "Manages remote relays"
-  @moduledoc """
+  @moduledoc File.read!("docs/administration/CLI_tasks/relay.md")
-  Manages remote relays
 
-  ## Follow a remote relay
-
-  ``mix pleroma.relay follow <relay_url>``
-
-  Example: ``mix pleroma.relay follow  https://example.org/relay``
-
-  ## Unfollow a remote relay
-
-  ``mix pleroma.relay unfollow <relay_url>``
-
-  Example: ``mix pleroma.relay unfollow https://example.org/relay``
-
-  ## List relay subscriptions
-
-  ``mix pleroma.relay list``
-  """
   def run(["follow", target]) do
     start_pleroma()
 
@@ -53,13 +35,10 @@ def run(["unfollow", target]) do
   def run(["list"]) do
     start_pleroma()
 
-    with %User{following: following} = _user <- Relay.get_actor() do
+    with {:ok, list} <- Relay.list() do
-      following
+      list |> Enum.each(&shell_info(&1))
-      |> Enum.map(fn entry -> URI.parse(entry).host end)
-      |> Enum.uniq()
-      |> Enum.each(&shell_info(&1))
     else
-      e -> shell_error("Error while fetching relay subscription list: #{inspect(e)}")
+      {:error, e} -> shell_error("Error while fetching relay subscription list: #{inspect(e)}")
     end
   end
 end

lib/mix/tasks/pleroma/robotstxt.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.RobotsTxt do
@@ -18,6 +18,7 @@ defmodule Mix.Tasks.Pleroma.RobotsTxt do
 
   """
   def run(["disallow_all"]) do
+    Mix.Pleroma.start_pleroma()
     static_dir = Pleroma.Config.get([:instance, :static_dir], "instance/static/")
 
     if !File.exists?(static_dir) do

lib/mix/tasks/pleroma/uploads.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Uploads do
@@ -12,16 +12,8 @@ defmodule Mix.Tasks.Pleroma.Uploads do
   @log_every 50
 
   @shortdoc "Migrates uploads from local to remote storage"
-  @moduledoc """
+  @moduledoc File.read!("docs/administration/CLI_tasks/uploads.md")
-   Manages uploads
 
-   ## Migrate uploads from local to remote storage
-       mix pleroma.uploads migrate_local TARGET_UPLOADER [OPTIONS...]
-   Options:
-   - `--delete` - delete local uploads after migrating them to the target uploader
-
-   A list of available uploaders can be seen in config.exs
-  """
   def run(["migrate_local", target_uploader | args]) do
     delete? = Enum.member?(args, "--delete")
     start_pleroma()

lib/mix/tasks/pleroma/user.ex

@@ -1,96 +1,17 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.User do
   use Mix.Task
-  import Ecto.Changeset
   import Mix.Pleroma
+  alias Ecto.Changeset
   alias Pleroma.User
   alias Pleroma.UserInviteToken
-  alias Pleroma.Web.OAuth
 
   @shortdoc "Manages Pleroma users"
-  @moduledoc """
+  @moduledoc File.read!("docs/administration/CLI_tasks/user.md")
-  Manages Pleroma users.
 
-  ## Create a new user.
-
-      mix pleroma.user new NICKNAME EMAIL [OPTION...]
-
-  Options:
-  - `--name NAME` - the user's name (i.e., "Lain Iwakura")
-  - `--bio BIO` - the user's bio
-  - `--password PASSWORD` - the user's password
-  - `--moderator`/`--no-moderator` - whether the user is a moderator
-  - `--admin`/`--no-admin` - whether the user is an admin
-  - `-y`, `--assume-yes`/`--no-assume-yes` - whether to assume yes to all questions
-
-  ## Generate an invite link.
-
-      mix pleroma.user invite [OPTION...]
-
-    Options:
-    - `--expires-at DATE` - last day on which token is active (e.g. "2019-04-05")
-    - `--max-use NUMBER` - maximum numbers of token uses
-
-  ## List generated invites
-
-      mix pleroma.user invites
-
-  ## Revoke invite
-
-      mix pleroma.user revoke_invite TOKEN OR TOKEN_ID
-
-  ## Delete the user's account.
-
-      mix pleroma.user rm NICKNAME
-
-  ## Delete the user's activities.
-
-      mix pleroma.user delete_activities NICKNAME
-
-  ## Sign user out from all applications (delete user's OAuth tokens and authorizations).
-
-      mix pleroma.user sign_out NICKNAME
-
-  ## Deactivate or activate the user's account.
-
-      mix pleroma.user toggle_activated NICKNAME
-
-  ## Unsubscribe local users from user's account and deactivate it
-
-      mix pleroma.user unsubscribe NICKNAME
-
-  ## Unsubscribe local users from an entire instance and deactivate all accounts
-
-      mix pleroma.user unsubscribe_all_from_instance INSTANCE
-
-  ## Create a password reset link.
-
-      mix pleroma.user reset_password NICKNAME
-
-  ## Set the value of the given user's settings.
-
-      mix pleroma.user set NICKNAME [OPTION...]
-
-  Options:
-  - `--locked`/`--no-locked` - whether the user's account is locked
-  - `--moderator`/`--no-moderator` - whether the user is a moderator
-  - `--admin`/`--no-admin` - whether the user is an admin
-
-  ## Add tags to a user.
-
-      mix pleroma.user tag NICKNAME TAGS
-
-  ## Delete tags from a user.
-
-      mix pleroma.user untag NICKNAME TAGS
-
-  ## Toggle confirmation of the user's account.
-
-      mix pleroma.user toggle_confirmed NICKNAME
-  """
   def run(["new", nickname, email | rest]) do
     {options, [], []} =
       OptionParser.parse(
@@ -179,8 +100,7 @@ def run(["rm", nickname]) do
       User.perform(:delete, user)
       shell_info("User #{nickname} deleted.")
     else
-      _ ->
+      _ -> shell_error("No local user #{nickname}")
-        shell_error("No local user #{nickname}")
     end
   end
 
@@ -188,10 +108,10 @@ def run(["toggle_activated", nickname]) do
     start_pleroma()
 
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
-      {:ok, user} = User.deactivate(user, !user.info.deactivated)
+      {:ok, user} = User.deactivate(user, !user.deactivated)
 
       shell_info(
-        "Activation status of #{nickname}: #{if(user.info.deactivated, do: "de", else: "")}activated"
+        "Activation status of #{nickname}: #{if(user.deactivated, do: "de", else: "")}activated"
       )
     else
       _ ->
@@ -228,9 +148,9 @@ def run(["unsubscribe", nickname]) do
       shell_info("Deactivating #{user.nickname}")
       User.deactivate(user)
 
-      {:ok, friends} = User.get_friends(user)
+      user
-
+      |> User.get_friends()
-      Enum.each(friends, fn friend ->
+      |> Enum.each(fn friend ->
         user = User.get_cached_by_id(user.id)
 
         shell_info("Unsubscribing #{friend.nickname} from #{user.nickname}")
@@ -241,7 +161,7 @@ def run(["unsubscribe", nickname]) do
 
       user = User.get_cached_by_id(user.id)
 
-      if Enum.empty?(user.following) do
+      if Enum.empty?(User.get_friends(user)) do
         shell_info("Successfully unsubscribed all followers from #{user.nickname}")
       end
     else
@@ -405,7 +325,7 @@ def run(["delete_activities", nickname]) do
     start_pleroma()
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
-      {:ok, _} = User.delete_user_activities(user)
+      User.delete_user_activities(user)
       shell_info("User #{nickname} statuses deleted.")
     else
       _ ->
@@ -419,7 +339,7 @@ def run(["toggle_confirmed", nickname]) do
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
       {:ok, user} = User.toggle_confirmation(user)
 
-      message = if user.info.confirmation_pending, do: "needs", else: "doesn't need"
+      message = if user.confirmation_pending, do: "needs", else: "doesn't need"
 
       shell_info("#{nickname} #{message} confirmation.")
     else
@@ -432,8 +352,7 @@ def run(["sign_out", nickname]) do
     start_pleroma()
 
     with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do
-      OAuth.Token.delete_user_tokens(user)
+      User.global_sign_out(user)
-      OAuth.Authorization.delete_user_authorizations(user)
 
       shell_info("#{nickname} signed out from all apps.")
     else
@@ -442,42 +361,48 @@ def run(["sign_out", nickname]) do
     end
   end
 
+  def run(["list"]) do
+    start_pleroma()
+
+    Pleroma.User.Query.build(%{local: true})
+    |> Pleroma.RepoStreamer.chunk_stream(500)
+    |> Stream.each(fn users ->
+      users
+      |> Enum.each(fn user ->
+        shell_info(
+          "#{user.nickname} moderator: #{user.is_moderator}, admin: #{user.is_admin}, locked: #{
+            user.locked
+          }, deactivated: #{user.deactivated}"
+        )
+      end)
+    end)
+    |> Stream.run()
+  end
+
   defp set_moderator(user, value) do
-    info_cng = User.Info.admin_api_update(user.info, %{is_moderator: value})
+    {:ok, user} =
+      user
+      |> Changeset.change(%{is_moderator: value})
+      |> User.update_and_set_cache()
 
-    user_cng =
+    shell_info("Moderator status of #{user.nickname}: #{user.is_moderator}")
-      Ecto.Changeset.change(user)
-      |> put_embed(:info, info_cng)
-
-    {:ok, user} = User.update_and_set_cache(user_cng)
-
-    shell_info("Moderator status of #{user.nickname}: #{user.info.is_moderator}")
     user
   end
 
   defp set_admin(user, value) do
-    info_cng = User.Info.admin_api_update(user.info, %{is_admin: value})
+    {:ok, user} = User.admin_api_update(user, %{is_admin: value})
 
-    user_cng =
+    shell_info("Admin status of #{user.nickname}: #{user.is_admin}")
-      Ecto.Changeset.change(user)
-      |> put_embed(:info, info_cng)
-
-    {:ok, user} = User.update_and_set_cache(user_cng)
-
-    shell_info("Admin status of #{user.nickname}: #{user.info.is_admin}")
     user
   end
 
   defp set_locked(user, value) do
-    info_cng = User.Info.user_upgrade(user.info, %{locked: value})
+    {:ok, user} =
+      user
+      |> Changeset.change(%{locked: value})
+      |> User.update_and_set_cache()
 
-    user_cng =
+    shell_info("Locked status of #{user.nickname}: #{user.locked}")
-      Ecto.Changeset.change(user)
-      |> put_embed(:info, info_cng)
-
-    {:ok, user} = User.update_and_set_cache(user_cng)
-
-    shell_info("Locked status of #{user.nickname}: #{user.info.locked}")
     user
   end
 end

lib/pleroma/activity.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Activity do
@@ -12,6 +12,7 @@ defmodule Pleroma.Activity do
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
+  alias Pleroma.ReportNote
   alias Pleroma.ThreadMute
   alias Pleroma.User
 
@@ -28,7 +29,9 @@ defmodule Pleroma.Activity do
     "Create" => "mention",
     "Follow" => "follow",
     "Announce" => "reblog",
-    "Like" => "favourite"
+    "Like" => "favourite",
+    "Move" => "move",
+    "EmojiReact" => "pleroma:emoji_reaction"
   }
 
   @mastodon_to_ap_notification_types for {k, v} <- @mastodon_notification_types,
@@ -41,8 +44,14 @@ defmodule Pleroma.Activity do
     field(:actor, :string)
     field(:recipients, {:array, :string}, default: [])
     field(:thread_muted?, :boolean, virtual: true)
+
+    # This is a fake relation,
+    # do not use outside of with_preloaded_user_actor/with_joined_user_actor
+    has_one(:user_actor, User, on_delete: :nothing, foreign_key: :id)
     # This is a fake relation, do not use outside of with_preloaded_bookmark/get_bookmark
     has_one(:bookmark, Bookmark)
+    # This is a fake relation, do not use outside of with_preloaded_report_notes
+    has_many(:report_notes, ReportNote)
     has_many(:notifications, Notification, on_delete: :delete_all)
 
     # Attention: this is a fake relation, don't try to preload it blindly and expect it to work!
@@ -86,6 +95,19 @@ def with_preloaded_object(query, join_type \\ :inner) do
     |> preload([activity, object: object], object: object)
   end
 
+  def with_joined_user_actor(query, join_type \\ :inner) do
+    join(query, join_type, [activity], u in User,
+      on: u.ap_id == activity.actor,
+      as: :user_actor
+    )
+  end
+
+  def with_preloaded_user_actor(query, join_type \\ :inner) do
+    query
+    |> with_joined_user_actor(join_type)
+    |> preload([activity, user_actor: user_actor], user_actor: user_actor)
+  end
+
   def with_preloaded_bookmark(query, %User{} = user) do
     from([a] in query,
       left_join: b in Bookmark,
@@ -96,6 +118,16 @@ def with_preloaded_bookmark(query, %User{} = user) do
 
   def with_preloaded_bookmark(query, _), do: query
 
+  def with_preloaded_report_notes(query) do
+    from([a] in query,
+      left_join: r in ReportNote,
+      on: a.id == r.activity_id,
+      preload: [report_notes: r]
+    )
+  end
+
+  def with_preloaded_report_notes(query, _), do: query
+
   def with_set_thread_muted_field(query, %User{} = user) do
     from([a] in query,
       left_join: tm in ThreadMute,
@@ -137,11 +169,18 @@ def get_by_ap_id_with_object(ap_id) do
     |> Repo.one()
   end
 
+  @spec get_by_id(String.t()) :: Activity.t() | nil
   def get_by_id(id) do
-    Activity
+    case FlakeId.flake_id?(id) do
-    |> where([a], a.id == ^id)
+      true ->
-    |> restrict_deactivated_users()
+        Activity
-    |> Repo.one()
+        |> where([a], a.id == ^id)
+        |> restrict_deactivated_users()
+        |> Repo.one()
+
+      _ ->
+        nil
+    end
   end
 
   def get_by_id_with_object(id) do
@@ -216,9 +255,10 @@ def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"])
   def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id_with_object(ap_id)
   def normalize(_), do: nil
 
-  def delete_by_ap_id(id) when is_binary(id) do
+  def delete_all_by_object_ap_id(id) when is_binary(id) do
     id
     |> Queries.by_object_id()
+    |> Queries.exclude_type("Delete")
     |> select([u], u)
     |> Repo.delete_all()
     |> elem(1)
@@ -230,7 +270,7 @@ def delete_by_ap_id(id) when is_binary(id) do
     |> purge_web_resp_cache()
   end
 
-  def delete_by_ap_id(_), do: nil
+  def delete_all_by_object_ap_id(_), do: nil
 
   defp purge_web_resp_cache(%Activity{} = activity) do
     %{path: path} = URI.parse(activity.data["id"])
@@ -270,13 +310,24 @@ def follow_requests_for_actor(%Pleroma.User{ap_id: ap_id}) do
 
   def restrict_deactivated_users(query) do
     deactivated_users =
-      from(u in User.Query.build(deactivated: true), select: u.ap_id)
+      from(u in User.Query.build(%{deactivated: true}), select: u.ap_id)
       |> Repo.all()
 
-    from(activity in query,
+    Activity.Queries.exclude_authors(query, deactivated_users)
-      where: activity.actor not in ^deactivated_users
-    )
   end
 
   defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search
+
+  def direct_conversation_id(activity, for_user) do
+    alias Pleroma.Conversation.Participation
+
+    with %{data: %{"context" => context}} when is_binary(context) <- activity,
+         %Pleroma.Conversation{} = conversation <- Pleroma.Conversation.get_for_ap_id(context),
+         %Participation{id: participation_id} <-
+           Participation.for_user_and_conversation(for_user, conversation) do
+      participation_id
+    else
+      _ -> nil
+    end
+  end
 end

lib/pleroma/activity/ir/topics.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Activity.Ir.Topics do

lib/pleroma/activity/queries.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Activity.Queries do
@@ -7,11 +7,12 @@ defmodule Pleroma.Activity.Queries do
   Contains queries for Activity.
   """
 
-  import Ecto.Query, only: [from: 2]
+  import Ecto.Query, only: [from: 2, where: 3]
 
   @type query :: Ecto.Queryable.t() | Activity.t()
 
   alias Pleroma.Activity
+  alias Pleroma.User
 
   @spec by_ap_id(query, String.t()) :: query
   def by_ap_id(query \\ Activity, ap_id) do
@@ -29,6 +30,11 @@ def by_actor(query \\ Activity, actor) do
     )
   end
 
+  @spec by_author(query, User.t()) :: query
+  def by_author(query \\ Activity, %User{ap_id: ap_id}) do
+    from(a in query, where: a.actor == ^ap_id)
+  end
+
   @spec by_object_id(query, String.t() | [String.t()]) :: query
   def by_object_id(query \\ Activity, object_id)
 
@@ -57,6 +63,22 @@ def by_object_id(query, object_id) when is_binary(object_id) do
     )
   end
 
+  @spec by_object_in_reply_to_id(query, String.t(), keyword()) :: query
+  def by_object_in_reply_to_id(query, in_reply_to_id, opts \\ []) do
+    query =
+      if opts[:skip_preloading] do
+        Activity.with_joined_object(query)
+      else
+        Activity.with_preloaded_object(query)
+      end
+
+    where(
+      query,
+      [activity, object: o],
+      fragment("(?)->>'inReplyTo' = ?", o.data, ^to_string(in_reply_to_id))
+    )
+  end
+
   @spec by_type(query, String.t()) :: query
   def by_type(query \\ Activity, activity_type) do
     from(
@@ -64,4 +86,16 @@ def by_type(query \\ Activity, activity_type) do
       where: fragment("(?)->>'type' = ?", activity.data, ^activity_type)
     )
   end
+
+  @spec exclude_type(query, String.t()) :: query
+  def exclude_type(query \\ Activity, activity_type) do
+    from(
+      activity in query,
+      where: fragment("(?)->>'type' != ?", activity.data, ^activity_type)
+    )
+  end
+
+  def exclude_authors(query \\ Activity, actors) do
+    from(activity in query, where: activity.actor not in ^actors)
+  end
 end

lib/pleroma/activity/search.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Activity.Search do
@@ -26,18 +26,23 @@ def search(user, search_query, options \\ []) do
     |> query_with(index_type, search_query)
     |> maybe_restrict_local(user)
     |> maybe_restrict_author(author)
+    |> maybe_restrict_blocked(user)
     |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => limit}, :offset)
     |> maybe_fetch(user, search_query)
   end
 
   def maybe_restrict_author(query, %User{} = author) do
-    from([a, o] in query,
+    Activity.Queries.by_author(query, author)
-      where: a.actor == ^author.ap_id
-    )
   end
 
   def maybe_restrict_author(query, _), do: query
 
+  def maybe_restrict_blocked(query, %User{} = user) do
+    Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user))
+  end
+
+  def maybe_restrict_blocked(query, _), do: query
+
   defp restrict_public(q) do
     from([a, o] in q,
       where: fragment("?->>'type' = 'Create'", a.data),
@@ -86,7 +91,7 @@ defp maybe_fetch(activities, user, search_query) do
          {:ok, object} <- Fetcher.fetch_object_from_id(search_query),
          %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
          true <- Visibility.visible_for_user?(activity, user) do
-      activities ++ [activity]
+      [activity | activities]
     else
       _ -> activities
     end

lib/pleroma/activity_expiration.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ActivityExpiration do
@@ -62,6 +62,6 @@ def validate_scheduled_at(changeset) do
   def expires_late_enough?(scheduled_at) do
     now = NaiveDateTime.utc_now()
     diff = NaiveDateTime.diff(scheduled_at, now, :millisecond)
-    diff >= @min_activity_lifetime
+    diff > @min_activity_lifetime
   end
 end

lib/pleroma/activity_expiration_worker.ex

@@ -1,62 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.ActivityExpirationWorker do
-  alias Pleroma.Activity
-  alias Pleroma.ActivityExpiration
-  alias Pleroma.Config
-  alias Pleroma.Repo
-  alias Pleroma.User
-  alias Pleroma.Web.CommonAPI
-  require Logger
-  use GenServer
-  import Ecto.Query
-
-  @schedule_interval :timer.minutes(1)
-
-  def start_link(_) do
-    GenServer.start_link(__MODULE__, nil)
-  end
-
-  @impl true
-  def init(_) do
-    if Config.get([ActivityExpiration, :enabled]) do
-      schedule_next()
-      {:ok, nil}
-    else
-      :ignore
-    end
-  end
-
-  def perform(:execute, expiration_id) do
-    try do
-      expiration =
-        ActivityExpiration
-        |> where([e], e.id == ^expiration_id)
-        |> Repo.one!()
-
-      activity = Activity.get_by_id_with_object(expiration.activity_id)
-      user = User.get_by_ap_id(activity.object.data["actor"])
-      CommonAPI.delete(activity.id, user)
-    rescue
-      error ->
-        Logger.error("#{__MODULE__} Couldn't delete expired activity: #{inspect(error)}")
-    end
-  end
-
-  @impl true
-  def handle_info(:perform, state) do
-    ActivityExpiration.due_expirations(@schedule_interval)
-    |> Enum.each(fn expiration ->
-      PleromaJobQueue.enqueue(:activity_expiration, __MODULE__, [:execute, expiration.id])
-    end)
-
-    schedule_next()
-    {:noreply, state}
-  end
-
-  defp schedule_next do
-    Process.send_after(self(), :perform, @schedule_interval)
-  end
-end

lib/pleroma/application.ex

@@ -1,10 +1,11 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Application do
   import Cachex.Spec
   use Application
+  require Logger
 
   @name Mix.Project.config()[:name]
   @version Mix.Project.config()[:version]
@@ -17,15 +18,25 @@ def named_version, do: @name <> " " <> @version
   def repository, do: @repository
 
   def user_agent do
-    info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"
+    case Pleroma.Config.get([:http, :user_agent], :default) do
-    named_version() <> "; " <> info
+      :default ->
+        info = "#{Pleroma.Web.base_url()} <#{Pleroma.Config.get([:instance, :email], "")}>"
+        named_version() <> "; " <> info
+
+      custom ->
+        custom
+    end
   end
 
   # See http://elixir-lang.org/docs/stable/elixir/Application.html
   # for more information on OTP Applications
   def start(_type, _args) do
+    Pleroma.HTML.compile_scrubbers()
     Pleroma.Config.DeprecationWarnings.warn()
+    Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
+    Pleroma.Repo.check_migrations_applied!()
     setup_instrumenters()
+    load_custom_modules()
 
     # Define workers and child supervisors to be supervised
     children =
@@ -34,17 +45,16 @@ def start(_type, _args) do
         Pleroma.Config.TransferTask,
         Pleroma.Emoji,
         Pleroma.Captcha,
-        Pleroma.ScheduledActivityWorker,
+        Pleroma.Plugs.RateLimiter.Supervisor
-        Pleroma.ActivityExpirationWorker
       ] ++
         cachex_children() ++
         hackney_pool_children() ++
         [
-          Pleroma.Web.Federator.RetryQueue,
+          Pleroma.Stats,
-          Pleroma.Stats
+          Pleroma.JobQueueMonitor,
+          {Oban, Pleroma.Config.get(Oban)}
         ] ++
         task_children(@env) ++
-        oauth_cleanup_child(oauth_cleanup_enabled?()) ++
         streamer_child(@env) ++
         chat_child(@env, chat_enabled?()) ++
         [
@@ -58,6 +68,28 @@ def start(_type, _args) do
     Supervisor.start_link(children, opts)
   end
 
+  def load_custom_modules do
+    dir = Pleroma.Config.get([:modules, :runtime_dir])
+
+    if dir && File.exists?(dir) do
+      dir
+      |> Pleroma.Utils.compile_dir()
+      |> case do
+        {:error, _errors, _warnings} ->
+          raise "Invalid custom modules"
+
+        {:ok, modules, _warnings} ->
+          if @env != :test do
+            Enum.each(modules, fn mod ->
+              Logger.info("Custom module loaded: #{inspect(mod)}")
+            end)
+          end
+
+          :ok
+      end
+    end
+  end
+
   defp setup_instrumenters do
     require Prometheus.Registry
 
@@ -101,10 +133,14 @@ defp cachex_children do
       build_cachex("scrubber", limit: 2500),
       build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
       build_cachex("web_resp", limit: 2500),
+      build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
       build_cachex("failed_proxy_url", limit: 2500)
     ]
   end
 
+  defp emoji_packs_expiration,
+    do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60))
+
   defp idempotency_expiration,
     do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
 
@@ -120,22 +156,12 @@ defp build_cachex(type, opts),
 
   defp chat_enabled?, do: Pleroma.Config.get([:chat, :enabled])
 
-  defp oauth_cleanup_enabled?,
-    do: Pleroma.Config.get([:oauth2, :clean_expired_tokens], false)
-
   defp streamer_child(:test), do: []
 
   defp streamer_child(_) do
     [Pleroma.Web.Streamer.supervisor()]
   end
 
-  defp oauth_cleanup_child(true),
-    do: [Pleroma.Web.OAuth.Token.CleanWorker]
-
-  defp oauth_cleanup_child(_), do: []
-
-  defp chat_child(:test, _), do: []
-
   defp chat_child(_env, true) do
     [Pleroma.Web.ChatChannel.ChatChannelState]
   end
@@ -155,11 +181,6 @@ defp task_children(:test) do
         id: :web_push_init,
         start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
         restart: :temporary
-      },
-      %{
-        id: :federator_init,
-        start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]},
-        restart: :temporary
       }
     ]
   end
@@ -171,11 +192,6 @@ defp task_children(_) do
         start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
         restart: :temporary
       },
-      %{
-        id: :federator_init,
-        start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]},
-        restart: :temporary
-      },
       %{
         id: :internal_fetch_init,
         start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]},

lib/pleroma/bbs/authenticator.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.BBS.Authenticator do

lib/pleroma/bbs/handler.ex

@@ -1,10 +1,11 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.BBS.Handler do
   use Sshd.ShellHandler
   alias Pleroma.Activity
+  alias Pleroma.HTML
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.CommonAPI
 
@@ -42,9 +43,9 @@ defp loop(state) do
   end
 
   def puts_activity(activity) do
-    status = Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{activity: activity})
+    status = Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{activity: activity})
     IO.puts("-- #{status.id} by #{status.account.display_name} (#{status.account.acct})")
-    IO.puts(HtmlSanitizeEx.strip_tags(status.content))
+    IO.puts(HTML.strip_tags(status.content))
     IO.puts("")
   end
 
@@ -97,7 +98,7 @@ def handle_command(state, "home") do
       |> Map.put("user", user)
 
     activities =
-      [user.ap_id | user.following]
+      [user.ap_id | Pleroma.User.following(user)]
       |> ActivityPub.fetch_activities(params)
 
     Enum.each(activities, fn activity ->

lib/pleroma/bookmark.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Bookmark do

lib/pleroma/captcha/captcha.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha do
@@ -50,7 +50,7 @@ def handle_call(:new, _from, state) do
       token = new_captcha[:token]
       secret = KeyGenerator.generate(secret_key_base, token <> "_encrypt")
       sign_secret = KeyGenerator.generate(secret_key_base, token <> "_sign")
-      # Basicallty copy what Phoenix.Token does here, add the time to
+      # Basically copy what Phoenix.Token does here, add the time to
       # the actual data and make it a binary to then encrypt it
       encrypted_captcha_answer =
         %{
@@ -62,7 +62,7 @@ def handle_call(:new, _from, state) do
 
       {
         :reply,
-        # Repalce the answer with the encrypted answer
+        # Replace the answer with the encrypted answer
         %{new_captcha | answer_data: encrypted_captcha_answer},
         state
       }
@@ -82,7 +82,8 @@ def handle_call({:validate, token, captcha, answer_data}, _from, state) do
     valid_if_after = DateTime.subtract!(DateTime.now_utc(), seconds_valid)
 
     result =
-      with {:ok, data} <- MessageEncryptor.decrypt(answer_data, secret, sign_secret),
+      with false <- is_nil(answer_data),
+           {:ok, data} <- MessageEncryptor.decrypt(answer_data, secret, sign_secret),
            %{at: at, answer_data: answer_md5} <- :erlang.binary_to_term(data) do
         try do
           if DateTime.before?(at, valid_if_after),

lib/pleroma/captcha/captcha_service.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha.Service do

lib/pleroma/captcha/kocaptcha.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Captcha.Kocaptcha do