.woodpecker/build-amd64.yml

@@ -0,0 +1,110 @@
+labels:
+  platform: linux/amd64
+
+when:
+  event:
+    - push
+    - tag
+  branch:
+    - develop
+    - stable
+
+variables:
+  - &scw-secrets
+    SCW_ACCESS_KEY:
+      from_secret: SCW_ACCESS_KEY
+    SCW_SECRET_KEY:
+      from_secret: SCW_SECRET_KEY
+    SCW_DEFAULT_ORGANIZATION_ID:
+      from_secret: SCW_DEFAULT_ORGANIZATION_ID
+  - &setup-hex "mix local.hex --force && mix local.rebar --force"
+  - &on-stable
+    when:
+      event:
+        - push
+        - tag
+      branch:
+        - stable
+
+  - &tag-build "export BUILD_TAG=$${CI_COMMIT_TAG:-\"$CI_COMMIT_BRANCH\"} && export PLEROMA_BUILD_BRANCH=$BUILD_TAG"
+
+  - &clean "(rm -rf release || true) && (rm -rf _build || true) && (rm -rf /root/.mix)"
+  - &mix-clean "mix deps.clean --all && mix clean"
+
+steps:
+  # Canonical amd64
+  debian-bookworm:
+    image: hexpm/elixir:1.15.4-erlang-26.0.2-debian-bookworm-20230612
+    environment:
+      MIX_ENV: prod
+      DEBIAN_FRONTEND: noninteractive
+    commands:
+      - apt-get update && apt-get install -y cmake libmagic-dev rclone zip imagemagick libmagic-dev git build-essential g++ wget
+      - *clean
+      - echo "import Config" > config/prod.secret.exs
+      - *setup-hex
+      - *tag-build
+      - mix deps.get --only prod
+      - mix release --path release
+      - zip akkoma-amd64.zip -r release
+
+  release-debian-bookworm:
+    image: akkoma/releaser
+    environment: *scw-secrets
+    commands:
+      - export SOURCE=akkoma-amd64.zip
+      # AMD64
+      - export DEST=scaleway:akkoma-updates/$${CI_COMMIT_TAG:-"$CI_COMMIT_BRANCH"}/akkoma-amd64.zip
+      - /bin/sh /entrypoint.sh
+      # Ubuntu jammy (currently compatible)
+      - export DEST=scaleway:akkoma-updates/$${CI_COMMIT_TAG:-"$CI_COMMIT_BRANCH"}/akkoma-amd64-ubuntu-jammy.zip
+      - /bin/sh /entrypoint.sh
+
+  debian-bullseye:
+    image: hexpm/elixir:1.15.4-erlang-26.0.2-debian-bullseye-20230612
+    environment:
+      MIX_ENV: prod
+      DEBIAN_FRONTEND: noninteractive
+    commands:
+      - apt-get update && apt-get install -y cmake libmagic-dev rclone zip imagemagick libmagic-dev git build-essential g++ wget
+      - *clean
+      - echo "import Config" > config/prod.secret.exs
+      - *setup-hex
+      - *tag-build
+      - mix deps.get --only prod
+      - mix release --path release
+      - zip akkoma-amd64-debian-bullseye.zip -r release
+
+  release-debian-bullseye:
+    image: akkoma/releaser
+    environment: *scw-secrets
+    commands:
+      - export SOURCE=akkoma-amd64-debian-bullseye.zip
+      # AMD64
+      - export DEST=scaleway:akkoma-updates/$${CI_COMMIT_TAG:-"$CI_COMMIT_BRANCH"}/akkoma-amd64-debian-bullseye.zip
+      - /bin/sh /entrypoint.sh
+
+  # Canonical amd64-musl
+  musl:
+    image: hexpm/elixir:1.15.4-erlang-26.0.2-alpine-3.18.2
+    <<: *on-stable
+    environment:
+      MIX_ENV: prod
+    commands:
+      - apk add git gcc g++ musl-dev make cmake file-dev rclone wget zip imagemagick
+      - *clean
+      - *setup-hex
+      - *mix-clean
+      - *tag-build
+      - mix deps.get --only prod
+      - mix release --path release
+      - zip akkoma-amd64-musl.zip -r release
+
+  release-musl:
+    image: akkoma/releaser
+    <<: *on-stable
+    environment: *scw-secrets
+    commands:
+      - export SOURCE=akkoma-amd64-musl.zip
+      - export DEST=scaleway:akkoma-updates/$${CI_COMMIT_TAG:-"$CI_COMMIT_BRANCH"}/akkoma-amd64-musl.zip
+      - /bin/sh /entrypoint.sh

.woodpecker/build-arm64.yml

@@ -0,0 +1,84 @@
+labels:
+  platform: linux/arm64
+
+when:
+  event:
+    - push
+    - tag
+  branch:
+    - develop
+    - stable
+
+variables:
+  - &scw-secrets
+    SCW_ACCESS_KEY:
+      from_secret: SCW_ACCESS_KEY
+    SCW_SECRET_KEY:
+      from_secret: SCW_SECRET_KEY
+    SCW_DEFAULT_ORGANIZATION_ID:
+      from_secret: SCW_DEFAULT_ORGANIZATION_ID
+  - &setup-hex "mix local.hex --force && mix local.rebar --force"
+  - &on-stable
+    when:
+      event:
+        - push
+        - tag
+      branch:
+        - stable
+
+  - &tag-build "export BUILD_TAG=$${CI_COMMIT_TAG:-\"$CI_COMMIT_BRANCH\"} && export PLEROMA_BUILD_BRANCH=$BUILD_TAG"
+
+  - &clean "(rm -rf release || true) && (rm -rf _build || true) && (rm -rf /root/.mix)"
+  - &mix-clean "mix deps.clean --all && mix clean"
+
+steps:
+  # Canonical arm64
+  debian-bookworm:
+    image: hexpm/elixir:1.15.4-erlang-26.0.2-debian-bookworm-20230612
+    environment:
+      MIX_ENV: prod
+      DEBIAN_FRONTEND: noninteractive
+    commands:
+      - apt-get update && apt-get install -y cmake libmagic-dev rclone zip imagemagick libmagic-dev git build-essential g++ wget
+      - *clean
+      - echo "import Config" > config/prod.secret.exs
+      - *setup-hex
+      - *tag-build
+      - mix deps.get --only prod
+      - mix release --path release
+      - zip akkoma-arm64.zip -r release
+
+  release-debian-bookworm:
+    image: akkoma/releaser:arm64
+    environment: *scw-secrets
+    commands:
+      - export SOURCE=akkoma-arm64.zip
+      - export DEST=scaleway:akkoma-updates/$${CI_COMMIT_TAG:-"$CI_COMMIT_BRANCH"}/akkoma-arm64-ubuntu-jammy.zip
+      - /bin/sh /entrypoint.sh
+      - export DEST=scaleway:akkoma-updates/$${CI_COMMIT_TAG:-"$CI_COMMIT_BRANCH"}/akkoma-arm64.zip
+      - /bin/sh /entrypoint.sh
+
+  # Canonical arm64-musl
+  musl:
+    image: hexpm/elixir:1.15.4-erlang-26.0.2-alpine-3.18.2
+    <<: *on-stable
+    environment:
+      MIX_ENV: prod
+    commands:
+      - apk add git gcc g++ musl-dev make cmake file-dev rclone wget zip imagemagick
+      - *clean
+      - *setup-hex
+      - *mix-clean
+      - *tag-build
+      - mix deps.get --only prod
+      - mix release --path release
+      - zip akkoma-arm64-musl.zip -r release
+
+  release-musl:
+    image: akkoma/releaser:arm64
+    <<: *on-stable
+    environment: *scw-secrets
+    commands:
+      - export SOURCE=akkoma-arm64-musl.zip
+      - export DEST=scaleway:akkoma-updates/$${CI_COMMIT_TAG:-"$CI_COMMIT_BRANCH"}/akkoma-arm64-musl.zip
+      - /bin/sh /entrypoint.sh

.woodpecker/docs.yml

@@ -1,5 +1,8 @@
 labels:
-  platform: linux/arm64
+  platform: linux/amd64
+
+depends_on:
+  - build-amd64
 
 when:
   event:
@@ -21,8 +24,8 @@ steps:
     image: python:3.10-slim
     commands:
       - apt-get update && apt-get install -y rclone wget git zip
-      - wget https://github.com/scaleway/scaleway-cli/releases/download/v2.5.1/scaleway-cli_2.5.1_linux_arm64
-      - mv scaleway-cli_2.5.1_linux_arm64 scaleway-cli
+      - wget https://github.com/scaleway/scaleway-cli/releases/download/v2.5.1/scaleway-cli_2.5.1_linux_amd64
+      - mv scaleway-cli_2.5.1_linux_amd64 scaleway-cli
       - chmod +x scaleway-cli
       - ./scaleway-cli object config install type=rclone
       - cd docs

.woodpecker/publish.yml

@@ -1,104 +0,0 @@
-when:
-  event:
-    - push
-    - tag
-  branch:
-    - develop
-    - stable
-  evaluate: 'SKIP_DEVELOP != "YES" || CI_COMMIT_BRANCH != "develop"'
-
-matrix:
-  include:
-    # Canonical amd64
-    - ARCH: amd64
-      SUFFIX:
-      IMG_VAR: debian-bookworm-20230612
-      UBUNTU_EXPORT: YES
-
-    # old debian variant of amd64
-    - ARCH: amd64
-      SUFFIX: -debian-bullseye
-      IMG_VAR: debian-bullseye-20230612
-
-    # Canonical amd64-musl
-    - ARCH: amd64
-      SUFFIX: -musl
-      IMG_VAR: alpine-3.18.2
-      SKIP_DEVELOP: YES
-
-    # Canonical arm64
-    - ARCH: arm64
-      SUFFIX:
-      RELEASER_TAG: :arm64
-      IMG_VAR: debian-bookworm-20230612
-      UBUNTU_EXPORT: YES
-
-    # Canonical arm64-musl
-    - ARCH: arm64
-      SUFFIX: -musl
-      RELEASER_TAG: :arm64
-      IMG_VAR: alpine-3.18.2
-      SKIP_DEVELOP: YES
-
-labels:
-  platform: linux/${ARCH}
-
-steps:
-  # Canonical amd64
-  build:
-    image: hexpm/elixir:1.15.4-erlang-26.0.2-${IMG_VAR}
-    environment:
-      MIX_ENV: prod
-      DEBIAN_FRONTEND: noninteractive
-    commands: |
-       # install deps
-       case "${IMG_VAR}" in
-        debian*)
-          apt-get update && apt-get install -y \
-            cmake libmagic-dev rclone zip git wget \
-            build-essential g++  imagemagick libmagic-dev
-          ;;
-        alpine*)
-          apk add git gcc g++ musl-dev make cmake file-dev rclone wget zip imagemagick
-          ;;
-        *)
-          echo "No package manager defined for ${BASE_IMG}!"
-          exit 1
-       esac
-
-       # clean leftovers
-       rm -rf release
-       rm -rf _build
-       rm -rf /root/.mix
-
-       # setup
-       echo "import Config" > config/prod.secret.exs
-       mix local.hex --force
-       mix local.rebar --force
-       export BUILD_TAG=$${CI_COMMIT_TAG:-\"$CI_COMMIT_BRANCH\"}
-       export PLEROMA_BUILD_BRANCH=$BUILD_TAG
-
-       # actually build and zip up
-       mix deps.get --only prod
-       mix release --path release
-       zip akkoma-${ARCH}${SUFFIX}.zip -r release
-
-  release:
-    image: akkoma/releaser${RELEASER_TAG}
-    environment:
-      SCW_ACCESS_KEY:
-        from_secret: SCW_ACCESS_KEY
-      SCW_SECRET_KEY:
-        from_secret: SCW_SECRET_KEY
-      SCW_DEFAULT_ORGANIZATION_ID:
-        from_secret: SCW_DEFAULT_ORGANIZATION_ID
-    commands: |
-      export SOURCE=akkoma-${ARCH}${SUFFIX}.zip
-      export DEST=scaleway:akkoma-updates/$${CI_COMMIT_TAG:-"$CI_COMMIT_BRANCH"}/$${SOURCE}
-      /bin/sh /entrypoint.sh
-
-      if [ "${UBUNTU_EXPORT}" = "YES" ] ; then
-        # Ubuntu jammy (currently compatible with our default debian builds)
-        export DEST=scaleway:akkoma-updates/$${CI_COMMIT_TAG:-"$CI_COMMIT_BRANCH"}/akkoma-${ARCH}-ubuntu-jammy.zip
-        /bin/sh /entrypoint.sh
-      fi

.woodpecker/test.yml

@@ -1,20 +1,18 @@
+labels:
+  platform: linux/amd64
+
 when:
   - event: pull_request
 
 matrix:
   # test the lowest and highest versions
   include:
-    - ELIXIR_VERSION: 1.15
+    - ELIXIR_VERSION: 1.14
       OTP_VERSION: 25
       LINT: NO
-      PLATFORM: linux/amd64
-    - ELIXIR_VERSION: 1.19
-      OTP_VERSION: 28
+    - ELIXIR_VERSION: 1.18
+      OTP_VERSION: 27
       LINT: YES
-      PLATFORM: linux/arm64
-
-labels:
-  platform: ${PLATFORM}
 
 services:
   postgres:
@@ -35,7 +33,6 @@ steps:
       DB_HOST: postgres
       LINT: ${LINT}
     commands:
-      - sh -c 'uname -a && cat /etc/os-release || :'
       - mix local.hex --force
       - mix local.rebar --force
       - mix deps.get

CHANGELOG.md

@@ -6,225 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## Unreleased
 
-### Update note
-- If you are using database search with a non-default RUM index,
-  you _MUST_ apply the new optional RUM migration before upgrading.  
-  Then after upgrading you wil need to refresh your RUM index setup
-  to also get the new search behaviour. This can be done by "changing"
-  your text search config to your current value (or something else, like `simple` if you so wish)
-  via the `database set_text_search_config <value>` mix task
-
-### Removed
-- vestige C2S access to follow* collections was dropped
-
-### Added
-- federated voter count of polls is now parsed and federated out too;
-    this fixes vote percetanges for new and refreshed remote multi-selection polls
-- new config options to restrict unauthenticated search API access under `:pleroma, :restrict_unauthenticated, :search`
-- extended MFM support further
-
-### Fixed
-- fixed status search not respecting `resolve=false`
-- fixed later search result pages again fetching remote content
-- handle reports referring to a single plain id as their object;
-    affected e.g. JSON-LD compacted reports without status references from Iceshrimp.NET
-- fixed several issues parsing remote Question objects
-- fixed signatures from blocked or deleted actors still being accepted
-- fixed follow* collections being readable without signature even if authorized_fetch mode is enabled
-- fixed one’s own follow* counts sometimes being redacted in API if hiding count for others
-- fixed delete&redraft still deleting attachment files most oft the time once the initial redraft delay elapsed
-- fixed fetched objects not normalising various valid forms of the public addressing URI
-- fixed searches for an already known statuses with resolve=true sometimes failing
-    if searching by a display URL instead of canonical AP ID.
-    With resolve=false no display URL lookups are possible.
-- fixed single-selection poll states not updating when receiving an Update activity for the status
-- fixed fetched updates of statuses with a poll or rediscovered pruned objects not being passed through MRFs
-- fixed potential data inconsistencies and API ordering for rediscovered partially pruned objects
-- fixed tagged (mentioned) but not addressed users receiving notifications about
-    statuses they are not actually allowed to access
-- fixed tranlator service being queried for supported languages even if not enabled
-- fixed explicitly static media proxy previews pointing to a broken redirect for non-animated files
-
-### Changed
-- New installations (not existing instances) now default to the `simple` full-text-search config
-- Unauthenticated search requests now by default force-disable remote fetches and pagination
-- Post search can now match text in the content warning with the database provider
-- prefixing a user search query with `@` will limit results to matching nicknames only, if the query contains no enclosed whitespace
-
-
-## 2026.05 (3.19.0)
-
-### General note
-- backup restore instructions very slightly changed but in an important way.  
-    It is no longer necessary to force sequential, single-transaction mode.
-    Now indexes can and are recommended to be restored in parallel significantly speeding up the overall process.
-   *(If ignoring instructions and using parallel mode before we got rid of index interdependence, `pg_restore` started restoring some indexes before other indexes they heavily depend on were done. Ironically leading to **much** worse restore times than pure sequential mode)*
-
-### Removed
-- as announced in 2025.12 (3.17) and since no complaints were raised, the semi-broken, seemingly unused and improvement-blocking thread containment feature is now removed. This entails the following API changes:
-    - dropped `PATCH /api/v1/accounts/update_credentials` input parameter `skip_thread_containment`
-    - dropped `GET /api/v1/accounts/:id` response key `pleroma.skip_thread_containment`
-    - dropped `GET /api/v1/pleroma/admin/users/:nickname/credentials` response key `skip_thread_containment`
-    - dropped the `skipThreadContainment` key from nodeinfo’s `metadata` object
-- for the same reason `GET /api/v1/timelines/direct` was removed too
-
-### Added
-- `{POST,PUT} api/v1/lists` now accepts the `exclusive` parameter from Mastodon allowing followed users in the list to be removed from the home timeline
-- User profile media now (can) have federated alt text; to this end:
-  - Mastodon-compatible `avatar_description` and `header_description` parameters are added to account API responses and as input for `PATCH /api/v1/accounts/update_credentials`
-  - `pleroma.background_image_description` is added to account API responses
-  - `pleroma_background_image_description` is added as a new parameter to `PATCH /api/v1/accounts/update_credentials`
-- `GET /api/v1/statuses/:id` contains the new `poll.akkoma.anonymous` parameter if `poll` is non-null.  
-    It relays if and whether the source instance promised to keep votes anonymous or disclose votes with voter identity.
-    There are no plans to enable creating non-anonymous polls in Akkoma, but some implementations do.
-
-### Fixed
-- fix date-time format in `* /api/v1/markers` to strictly conform to Mastodon’s ISO 8061 subset
-- fix response content-type and styling for the `/embed` endpoint
-- do not crash handler when attempting to refresh remote follow stats for users without follow* addresses
-- list timelines now include reblogs of users in the list matching Mastodon
-- non-federating instances now return a 405 response on inbox `POST`s, matching AP spec
-- fixed `GET /api/v1/statuses/:id/context` omitting most local-only posts for authenticated users
-- fixed nondeterministic API results in endpoints using GIN indexes; e.g. full-text search
-- enforced the host header being present on signatures, and matching our server
-
-### Changed
-- our Docker container now sets a default `nofile` `ulimit` to avoid issues on some systems.
-    Methods to customise this are documented under Configuration - General Optimisation.
-- Add reasonable defaults for `:database_config_whitelist`
-
-
-## 2026.03.1 (3.18.1)
-
-### Update notes
-- If you experience degraded performance of database queries after upgrading,
-    try running `VACUUM ANALYZE;` either manually with `psql` or via the
-    [database vacuum analyze mix task](https://docs.akkoma.dev/develop/administration/CLI_tasks/database/#analyze).
-    This will force the planner to pick up index changes if it didn’t do so on its own.
-
-### Fixed
-- fix WebFinger validation even more.  
-    While actor consent was now properly taken into account after the previous release, some scenarios still allowed bypassing domain consent
-- fix posts being federated to us with explicit zero extents crashing the status renderer
-- fix pagination parameters being ignored on hashtag timelines
-
-## 2026.03 (3.18.0)
-
-### BREAKING
-- Elixir 1.14 is no longer suported, and it's EOL! Upgrade to Elixir 1.15+
-- `account` entities in API responses now only contain a cut down version of their servers nodeinfo.
-  TEMPORARILY a config option is provided to serve the full nodeinfo data again.
-  HOWEVER this option WILL be removed soon. If you encounter any issues with third-party clients fixed
-  by using this setting, tell us so we can include all actually needed keys by default.
-
-### REMOVED
-
-### Added
-- Mastodon-compatible translation endpoints are now supported too;
-    the older Akkoma endpoints are deprecated but no immediate plans for removal
-- `GET pleroma/conversation/:id/statuses` now supports `with_muted`
-- `POST /api/v1/statuses` accepts and now prefers the Mastodon-compatible `quoted_status_id` parameter for quoting a post
-- `status` API entities now expose non-shallow quotes in a manner also compatible with Mastodon clients
-- support for WebFinger backlinks in ActivityPub actors (FEP-2c59)
-
-### Fixed
-- pinning, muting or unmuting a status one is not allowed to access no longer leaks its content
-- revoking a favourite on a post one lost access to no longer leaks its content
-- user info updates again are actively federated to other servers;
-  this was accidentally broken in the previous release
-- it is no longer possible to reference posts one cannot access when reporting another user
-- streamed relationship updates no longer leak follow* counts for users who chose to hide their counts
-- WebFinger data and user nicknames no longer allow non-consential associations
-- Correctly setup custom WebFinger domains work again
-- fix paths of emojis added or updated at runtime and remove emoji from runtime when deleting an entire pack without requiring a full emoji reload
-- fix retraction of remote emoji reaction when id is not present or its domain differs from image host
-- fix AP ids declared with the canonical type being ignored in XML WebFinger responses
-- fix many, many bugs in the conversations API family
-- notifications about muted entities are no longer streamed out
-- non-UTF-8 usernames no longer lead to internal server errors in API endpoints
-- when SimplePolicy rules are configured but the MRF not enabled, it’s rules no longer interfere with fetching
-- fixed remote follow counter refresh on user (re)fetch
-- remote users whose follow* counts are private are now actually shown as such in API instead of represeneting them with public zero counters
-- fix local follow* collections counting and including AP IDs of deleted users
-
-### Changed
-- `PATCH /api/v1/pleroma/conversations/:id` now accepts update parameters via JSON body too
-- it is now possible to quote local and one’s own private posts provided a compatible scope is used
-- on final activity failures the error log now includes the afected activity
-- improved performance of `GET api/v1/custom_emoji`
-- outgoing HTTP requests now accept compressed responses
-- the system CA certificate store is now used by default
-- when refreshing remote follow* stats all fetch-related erros are now treated as stats being private;
-    this avoids spurious error logs and better matches the intent of implementations serving fallback HTML responses on the AP collection endpoints
-
-
-## 2025.12 (3.17.0)
-
-### REMOVED
-- DEPRECATE `/api/v1/timelines/direct`.  
-    Technically this was already deprecated, given we extend mastodon 2.7.2 API
-    and Mastodon already deprecated it in 2.6.0 before removing it in 3.0.0.
-    But now we have concrete plans to remove this endpoint in a coming release.
-    The few remaining useres should switch to the conversations API.
-- DEPRECATE `config :pleroma, :instance, skip_thread_containment: false`.
-  It is due to be removed in one of the next releases if no strong arguments for keeping it are brought up.
-  It is already semi-broken for large threads and conflicts with pending optimisation and cleanup work.
-- support for `exclude_visibilities` in timeline and notification endpoints has been dropped
-- support for list visibility / list addressing has been dropped due to lack of usage, maintenance burden and redundancy with the still supported explicit-addressing feature
-- support for conversations addressing has been dropped due to lack of usage, maintenance burden and being mostly redundant with explicit addressing
-- per-visibility status counters have been dropped from `/api/v1/pleroma/admin/stats`
-  due to unreasonably perf costs added on most database operations.
-  For now, the response still contains the fields, but with stubbed-out values.
-
-### Added
-- status responses include two new fields for ActivityPub cross-referencing: `akkoma.quote_apid` and `akkoma.in_reply_to_apid`
-- attempting to reply to an already deleted post will return an error
-  (in akkoma-fe the error will be shown and your draft message retained so you can decide
-   for yourself whether to discard it or copy and repost as a, now intentional, new thread)
-- the notification endpoint now supports the `types` parameter for filtering added in vanilla Mastodon
-- the mute endpoint now supports the `duration` parameter added in vanilla Mastodon
-  (fixes temporary mutes created via e.g. Husky)
-
-### Fixed
-- replies and quotes to unresolvable posts now fill out IDs for replied to
-  status, user or quoted status with a 404-ing ID to make them recognisable as
-  replies/quotes instead of pretending they’re root posts
-- querying a status using the ID of a non-post AP activity no longer displays
-  a duplicate of the post referenced by said activity with mangled author information
-- fix users being able to interact (like, emoji react, ...) with posts they cannot access
-- fix AP fetches of local non-Create, non-Undo activities exposing the raw, unsanitised content of the referenced object
-- the above two combined allowed local users to gain access to private posts
-  of user they do not follow, but follow a follower of the author.
-  (remote users and other scenarios were to our knowledge not able to achieve this due to other restrictions)
-- fix RSS and Atom feeds of hashtag timelines potentially exposing more information than Mastodon API when restricting unauthenticated API access
-- fix mentioning and sending DMs to users with non-ASCII-alphanumerical usernames
-- correctly hide and show inlined fallback links for quotes from Mastodon instances
-- API requests with multiple unsupported parameters now will ignore all of them up to a certain limit.
-  If there are too many unsupported parameters this is indicated in the returned error message.
-- expose generic type of attachment via Masto API if remote did not send a full MIME type but indicated a generic one
-  (the \*oma-specific full mime type field in the API response remains generic however, since we don't have this info)
-- add back the default banner image we advertise in Masto API
-- correctly redirect `/users/:nickname.rss` to the RSS instead of Atom feed
-
-### Changed
-- depreacted the `included_types` parameter in the notification endpoint; replaced by `types`
-- depreacted the `expires_in` parameter in the mute endpoint; replaced by `duration`
-- optimised emoji addition and removal
-- emoji reloading now happens asynchronously so you won't run into timeout issues with many emoji and/or a slow disk
-- upgraded all of our dependencies; this should reduce issues when running akoma with OTP28
-- prefer "summary" over "name" for the attachment alt text of incoming ActivityPub documents;
-  this fixes alt text federation from GtS and Honk
-- slightly improve index overhead for the users table
-
-
-## 2025.10 (3.16.0)
-
 ### REMOVED
 - Dropped `accepts_chat_messages` column from users table in database;
   it has been unused for almost 3 years
-- Healthcheck responses no longer contain job queue data;
-  it was useless anyway due to lacking any temporal information about failures
-  and more complete data can be obtained from Prometheus metrics.
 
 ### Added
 - We mark our MFM posts as FEP-c16b compliant, and retain remote HTML representations for incoming posts marked as FEP-c16b-compliant. (Safety scrubbers are still applied)
@@ -243,9 +27,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Added a reference Grafana dashboard and improved documentation for Prometheus metrics
 - New mix task `clean_inlined_replies` to delete some unused data from objects
 - New mix task `resync_inlined_caches` to retroactively fix various issues with e.g. boosts, emoji reacts and likes
-- It is now possible to allow outgoing requests to use HTTP2 via config option,
-  but due to bugs in the relevant backend this is not the default nor recommended.
-- Prometheus metrics now expose count of scheduled and pending jobs per queue
 
 ### Fixed
 - Internal actors no longer pretend to have unresolvable follow(er|ing) collections
@@ -266,10 +47,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - fixed handling of inlined "featured" collections
 - fixed user endpoint serving invalid ActivityPub for minimal, authfetch-fallback responses
 - remote emoji reacts from IceShrimp.NET instances are now handled consistently and always merged with identical other emoji reactions
-- ActivityPub requests signatures are now renewed when following redirects making sure path and host actually match the final URL
-- private replies no longer increase the publicly visible reply counter
-- unblock activities are no longer federated when block federation is disabled (the default)
-- fix like activity database IDs rendering as misattributed posts
 
 ### Changed
 - Internal and relay actors are now again represented with type "Application"
@@ -287,13 +64,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - `/api/v1/statuses/:id/reblog` now honours all possible visibilities except `list` and `conversation`
   instead of mapping them down to a boolean private/public
 - we no longer repeatedly try to deliver to explicitly deleted inboxes
-- Config option `Pleroma.Web.MediaProxy.Invalidation.Http, :options` and
-  the `:http` subkey of `:media_proxy, :proxy_opts` now only accept
-  adapter-related settings inside the `:adapter` subkey, no longer on the top-level
-- follow requests are now ordered reverse chronologically
+- outgoing requests may now use HTTP2 by default
 
 
-## 2025.03 (3.15.0, 3.15.1, 3.15.2)
+## 2025.03
 
 ### Added
 - Oban (worker) dashboard at `/akkoma/oban`
@@ -312,11 +86,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - The HTML content for new posts (both Client-to-Server as well as Server-to-Server communication) will now use a different formatting to represent MFM. See [FEP-c16b](https://codeberg.org/fediverse/fep/src/branch/main/fep/c16b/fep-c16b.md) for more details.
 - HTTP signatures now test the most likely request-target alias first cutting down on overhead
 
-## 2025.01.01 (3.14.1)
+## 2025.01.01
 
 Hotfix: Federation could break if a null value found its way into `should_federate?\1`
 
-## 2025.01 (3.14.0)
+## 2025.01
 
 ### Added
 - New config option `:instance, :cleanup_attachments_delay`
@@ -337,7 +111,7 @@ Hotfix: Federation could break if a null value found its way into `should_federa
   adopting a proposed AP spec errata and restoring federation
   with e.g. IceShrimp.NET and fedify-based implementations
 
-## 2024.11 (3.13.3)
+## 3.13.3 
 
 ### BREAKING
 - Minimum PostgreSQL version is raised to 12
@@ -360,14 +134,14 @@ Hotfix: Federation could break if a null value found its way into `should_federa
 ### Changed
 - Refactored Rich Media to cache the content in the database. Fetching operations that could block status rendering have been eliminated.
 
-## 2024.04.1 (Security) (3.13.2)
+## 2024.04.1 (Security)
 
 ### Fixed
 - Issue allowing non-owners to use media objects in posts
 - Issue allowing use of non-media objects as attachments and crashing timeline rendering
 - Issue allowing webfinger spoofing in certain situations
 
-## 2024.04 (3.13.0, 3.13.1)
+## 2024.04
 
 ### Added
 - Support for [FEP-fffd](https://codeberg.org/fediverse/fep/src/branch/main/fep/fffd/fep-fffd.md) (proxy objects)
@@ -404,7 +178,7 @@ Hotfix: Federation could break if a null value found its way into `should_federa
 - ActivityPub Client-To-Server write API endpoints have been disabled;
   read endpoints are planned to be removed next release unless a clear need is demonstrated
 
-## 2024.03 (3.12.0, 3.12.1, 3.12.3)
+## 2024.03
 
 ### Added
 - CLI tasks best-effort checking for past abuse of the recent spoofing exploit
@@ -445,7 +219,7 @@ Hotfix: Federation could break if a null value found its way into `should_federa
   - Akkoma will refuse to start if this is not set.
 - Same with media proxy.
 
-## 2024.02 (3.11.0)
+## 2024.02
 
 ### Added
 - Full compatibility with Erlang OTP26
@@ -462,15 +236,11 @@ Hotfix: Federation could break if a null value found its way into `should_federa
 
 ### Fixed
 - Documentation issue in which a non-existing nginx file was referenced
+- Issue where a bad inbox URL could break federation
 - Issue where hashtag rel values would be scrubbed
 - Issue where short domains listed in `transparency_obfuscate_domains` were not actually obfuscated
 
-## 2023.08.1 (3.10.4)
-
-### Fixed
-- Issue where a bad inbox URL could break federation
-
-## 2023.08 (3.10.0, 3.10.1, 3.10.3)
+## 2023.08
 
 ### Added
 
@@ -514,7 +284,7 @@ Hotfix: Federation could break if a null value found its way into `should_federa
   - If you are on oldstable you should NOT attempt to update OTP builds without
     first updating your machine.
 
-## 2023.05 (3.9.0, 3.9.1, 3.9.2, 3.9.3)
+## 2023.05
 
 ### Added
 - Custom options for users to accept/reject private messages
@@ -531,7 +301,7 @@ Hotfix: Federation could break if a null value found its way into `should_federa
 ### Security
 - Fixed mediaproxy being a bit of a silly billy
 
-## 2023.04 (3.8.0)
+## 2023.04
 
 ### Added
 - Nodeinfo keys for unauthenticated timeline visibility
@@ -545,7 +315,7 @@ Hotfix: Federation could break if a null value found its way into `should_federa
   use [asdf](https://asdf-vm.com/). At time of writing, elixir 1.14.3 / erlang 25.3
   is confirmed to work.
 
-## 2023.03 (3.7.0, 3.7.1)
+## 2023.03
 
 ### Fixed
 - Allowed contentMap to be updated on edit
@@ -562,7 +332,7 @@ Hotfix: Federation could break if a null value found its way into `should_federa
 - Possibility of using the `style` parameter on `span` elements. This will break certain MFM parameters.
 - Option for "default" image description.
 
-## 2023.02 (3.6.0)
+## 2023.02
 
 ### Added
 - Prometheus metrics exporting from `/api/v1/akkoma/metrics`
@@ -605,7 +375,7 @@ Hotfix: Federation could break if a null value found its way into `should_federa
 - Ensure `config :tesla, :adapter` is either unset, or set to `{Tesla.Adapter.Finch, name: MyFinch}` in your .exs config
 - Pleroma-FE will need to be updated to handle the new /api/v1/pleroma endpoints for  custom emoji
 
-## 2022.12 (3.5.0)
+## 2022.12
 
 ### Added
 - Config: HTTP timeout options, :pool\_timeout and :receive\_timeout
@@ -633,7 +403,7 @@ Hotfix: Federation could break if a null value found its way into `should_federa
 ### Upgrade Notes
 - If you have an old instance, you will probably want to run `mix pleroma.database prune_task` in the foreground to catch it up with the history of your instance.
 
-## 2022.11 (3.4.0)
+## 2022.11
 
 ### Added
 - Officially supported docker release
@@ -642,6 +412,8 @@ Hotfix: Federation could break if a null value found its way into `should_federa
 - `requested_by` in relationships when the user has requested to follow you
 
 ### Changed
+- Follows no longer override domain blocks, a domain block is final
+- Deletes are now the lowest priority to publish and will be handled after creates
 - Domain blocks are now subdomain-matches by default
 
 ### Fixed
@@ -652,14 +424,7 @@ Hotfix: Federation could break if a null value found its way into `should_federa
   to the latest. The changes in OTP24.3 are breaking.
 - You can now remove the leading `*.` from domain blocks, but you do not have to.
 
-## 2022.10.1 (3.3.1)
-
-### Changed
-- Follows no longer override domain blocks, a domain block is final
-- Deletes are now the lowest priority to publish and will be handled after creates
-- Verify that the signature on posts is not domain blocked, and belongs to the correct user
-
-## 2022.10 (3.3.0)
+## 2022.10
 
 ### Added
 - Ability to sync frontend profiles between clients, with a name attached
@@ -668,13 +433,14 @@ Hotfix: Federation could break if a null value found its way into `should_federa
 ### Changed
 - Emoji updated to latest 15.0 draft
 - **Breaking**: `/api/v1/pleroma/backups` endpoints now requires `read:backups` scope instead of `read:accounts`
+- Verify that the signature on posts is not domain blocked, and belongs to the correct user
 
 ### Fixed
 - OAuthPlug no longer joins with the database every call and uses the user cache
 - Undo activities no longer try to look up by ID, and render correctly
 - prevent false-errors from meilisearch
 
-## 2022.09 (3.2.0)
+## 2022.09
 
 ### Added
 - support for fedibird-fe, and non-breaking API parity for it to function
@@ -701,7 +467,7 @@ Hotfix: Federation could break if a null value found its way into `should_federa
 ### Removed
 - Non-finch HTTP adapters. `:tesla, :adapter` is now highly recommended to be set to the default.
 
-## 2022.08 (3.1.0)
+## 2022.08
 
 ### Added
 - extended runtime module support, see config cheatsheet
@@ -731,7 +497,7 @@ Hotfix: Federation could break if a null value found its way into `should_federa
 - Chats, they were half-baked. Just use PMs.
 - Prometheus, it causes massive slowdown
 
-## 2022.07 (3.0.0)
+## 2022.07
 
 ### Added
 - Added move account API

FEDERATION.md

@@ -10,7 +10,6 @@
 ## Supported FEPs
 
 - [FEP-67ff: FEDERATION](https://codeberg.org/fediverse/fep/src/branch/main/fep/67ff/fep-67ff.md)
-- [FEP-2c59: Discovery of a Webfinger address from an ActivityPub actor](https://codeberg.org/fediverse/fep/src/branch/main/fep/2c59/fep-2c59.md)
 - [FEP-dc88: Formatting Mathematics](https://codeberg.org/fediverse/fep/src/branch/main/fep/dc88/fep-dc88.md)
 - [FEP-f1d5: NodeInfo in Fediverse Software](https://codeberg.org/fediverse/fep/src/branch/main/fep/f1d5/fep-f1d5.md)
 - [FEP-fffd: Proxy Objects](https://codeberg.org/fediverse/fep/src/branch/main/fep/fffd/fep-fffd.md)
@@ -38,22 +37,6 @@ Depending on instance configuration the same may be true for GET requests.
 We set the optional extension term `htmlMfm: true` when using content type "text/x.misskeymarkdown".
 Incoming messages containing `htmlMfm: true` will not have their content re-parsed.
 
-## WebFinger
-
-Akkoma requires WebFinger implmentations to respond to queries about a given user both when
-`acct:user@domain` or the canonical ActivityPub id of the actor is passed as the `resource`.
-
-Akkoma strongly encourages ActivityPub implementations to include
-a FEP-2c59-compliant WebFinger backlink in their actor documents.
-
-Without FEP-2c59 and if different domains are used for ActivityPub and the Webfinger subject,
-Akkoma relies on either the presence of an host-meta LRDD template on the ActivityPub domain
-or a working WebFinger endpoint on the ActivityPub domain. Additionally all WebFinger endpoints
-related to the ActivityPub and canonical WebFinger domain SHOULD also respond to queries about
-an alternative acct URI constructed with the WebFinger domain passed as the resource.  
-Without FEP-2c59 Akkoma may not become aware of changes to the
-preferred WebFinger `subject` domain for already discovered users.
-
 ## Nodeinfo
 
 Akkoma provides many additional entries in its nodeinfo response,

benchmarks/load_testing/fetcher.ex

@@ -53,7 +53,7 @@ defmodule Pleroma.LoadTesting.Fetcher do
     fetch_public_timeline(user, :local)
     fetch_public_timeline(user, :tag)
     fetch_notifications(user)
-    fetch_favourited_with_fav_id(user)
+    fetch_favourites(user)
     fetch_long_thread(user)
     fetch_timelines_with_reply_filtering(user)
   end
@@ -378,21 +378,21 @@ defmodule Pleroma.LoadTesting.Fetcher do
   end
 
   defp fetch_favourites(user) do
-    first_page_last = ActivityPub.fetch_favourited_with_fav_id(user) |> List.last()
+    first_page_last = ActivityPub.fetch_favourites(user) |> List.last()
 
     second_page_last =
-      ActivityPub.fetch_favourited_with_fav_id(user, %{:max_id => first_page_last.id}) |> List.last()
+      ActivityPub.fetch_favourites(user, %{:max_id => first_page_last.id}) |> List.last()
 
     third_page_last =
-      ActivityPub.fetch_favourited_with_fav_id(user, %{:max_id => second_page_last.id}) |> List.last()
+      ActivityPub.fetch_favourites(user, %{:max_id => second_page_last.id}) |> List.last()
 
     forth_page_last =
-      ActivityPub.fetch_favourited_with_fav_id(user, %{:max_id => third_page_last.id}) |> List.last()
+      ActivityPub.fetch_favourites(user, %{:max_id => third_page_last.id}) |> List.last()
 
     Benchee.run(
       %{
         "Favourites" => fn opts ->
-          ActivityPub.fetch_favourited_with_fav_id(user, opts)
+          ActivityPub.fetch_favourites(user, opts)
         end
       },
       inputs: %{
@@ -465,8 +465,7 @@ defmodule Pleroma.LoadTesting.Fetcher do
 
     notifications = MastodonAPI.get_notifications(user, opts)
 
-    favourites_keyed = ActivityPub.fetch_favourited_with_fav_id(user)
-    favourites = Pagiation.unwrap(favourites_keyed)
+    favourites = ActivityPub.fetch_favourites(user)
 
     Benchee.run(
       %{

ci/build-all.sh

@@ -1,2 +1,3 @@
-./build.sh 1.15-otp25 1.15.8-erlang-25.3.2.18-alpine-3.22.2
-./build.sh 1.19-otp28 1.19-erlang-28.0-alpine-3.23.2
+./build.sh 1.14-otp25 1.14.3-erlang-25.3.2-alpine-3.18.0
+./build.sh 1.15-otp25 1.15.8-erlang-25.3.2.18-alpine-3.19.7
+./build.sh 1.18-otp27 1.18.2-erlang-27.2.4-alpine-3.19.7

config/benchmark.exs

@@ -27,6 +27,7 @@ config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Test, enabled:
 config :pleroma, :instance,
   email: "admin@example.com",
   notify_email: "noreply@example.com",
+  skip_thread_containment: false,
   federating: false,
   external_user_synchronization: false
 
@@ -73,6 +74,8 @@ rum_enabled = System.get_env("RUM_ENABLED") == "true"
 config :pleroma, :database, rum_enabled: rum_enabled
 IO.puts("RUM enabled: #{rum_enabled}")
 
+config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock
+
 if File.exists?("./config/benchmark.secret.exs") do
   import_config "benchmark.secret.exs"
 else

config/config.exs

@@ -51,11 +51,6 @@ config :pleroma, Pleroma.Repo,
   queue_target: 20_000,
   migration_lock: nil
 
-# password hash strength
-config :argon2_elixir,
-  t_cost: 8,
-  parallelism: 2
-
 config :pleroma, Pleroma.Captcha,
   enabled: true,
   seconds_valid: 300,
@@ -189,7 +184,7 @@ config :pleroma, :http,
   pool_timeout: :timer.seconds(60),
   receive_timeout: :timer.seconds(15),
   proxy_url: nil,
-  protocols: [:http1],
+  protocols: [:http2, :http1],
   user_agent: :default,
   pool_size: 10,
   adapter: []
@@ -247,8 +242,8 @@ config :pleroma, :instance,
   safe_dm_mentions: false,
   healthcheck: false,
   remote_post_retention_days: 90,
+  skip_thread_containment: true,
   limit_to_local_content: :unauthenticated,
-  filter_embedded_nodeinfo: true,
   user_bio_length: 5000,
   user_name_length: 100,
   max_account_fields: 10,
@@ -359,6 +354,16 @@ config :pleroma, :assets,
   ],
   default_mascot: :pleroma_fox_tan
 
+config :pleroma, :manifest,
+  icons: [
+    %{
+      src: "/static/logo.svg",
+      type: "image/svg+xml"
+    }
+  ],
+  theme_color: "#282c37",
+  background_color: "#191b22"
+
 config :pleroma, :activitypub,
   unfollow_blocked: true,
   outgoing_blocks: false,
@@ -555,15 +560,6 @@ config :pleroma, Pleroma.User,
   ],
   email_blacklist: []
 
-config :pleroma, :database_config_whitelist, [
-  {:pleroma},
-  {:cors_plug},
-  {:ex_aws, :s3},
-  {:mime},
-  {:web_push_encryption, :vapid_details},
-  {:logger}
-]
-
 config :pleroma, Oban,
   repo: Pleroma.Repo,
   log: false,
@@ -780,9 +776,7 @@ config :pleroma, :frontends,
   available: %{
     "pleroma-fe" => %{
       "name" => "pleroma-fe",
-      "blind_trust" => true,
-      "git" => "https://akkoma.dev/AkkomaGang/akkoma-fe",
-      "bugtracker" => "https://akkoma.dev/AkkomaGang/akkoma-fe/issues",
+      "git" => "https://akkoma.dev/AkkomaGang/pleroma-fe",
       "build_url" =>
         "https://akkoma-updates.s3-website.fr-par.scw.cloud/frontend/${ref}/akkoma-fe.zip",
       "ref" => "stable",
@@ -791,9 +785,7 @@ config :pleroma, :frontends,
     # Mastodon-Fe cannot be set as a primary - this is only here so we can update this seperately
     "mastodon-fe" => %{
       "name" => "mastodon-fe",
-      "blind_trust" => true,
       "git" => "https://akkoma.dev/AkkomaGang/masto-fe",
-      "bugtracker" => "https://akkoma.dev/AkkomaGang/masto-fe/issues",
       "build_url" =>
         "https://akkoma-updates.s3-website.fr-par.scw.cloud/frontend/${ref}/masto-fe.zip",
       "build_dir" => "distribution",
@@ -801,9 +793,7 @@ config :pleroma, :frontends,
     },
     "fedibird-fe" => %{
       "name" => "fedibird-fe",
-      "blind_trust" => true,
       "git" => "https://akkoma.dev/AkkomaGang/fedibird-fe",
-      "bugtracker" => "https://akkoma.dev/AkkomaGang/fedibird-fe/issues",
       "build_url" =>
         "https://akkoma-updates.s3-website.fr-par.scw.cloud/frontend/${ref}/fedibird-fe.zip",
       "build_dir" => "distribution",
@@ -811,9 +801,7 @@ config :pleroma, :frontends,
     },
     "admin-fe" => %{
       "name" => "admin-fe",
-      "blind_trust" => true,
       "git" => "https://akkoma.dev/AkkomaGang/admin-fe",
-      "bugtracker" => "https://akkoma.dev/AkkomaGang/admin-fe/issues",
       "build_url" =>
         "https://akkoma-updates.s3-website.fr-par.scw.cloud/frontend/${ref}/admin-fe.zip",
       "ref" => "stable"
@@ -821,31 +809,10 @@ config :pleroma, :frontends,
     # For developers - enables a swagger frontend to view the openapi spec
     "swagger-ui" => %{
       "name" => "swagger-ui",
-      "blind_trust" => true,
       "git" => "https://github.com/swagger-api/swagger-ui",
-      # API spec definitions are part of the backend (and the swagger-ui build outdated)
-      "bugtracker" => "https://akkoma.dev/AkkomaGang/akkoma/issues",
       "build_url" => "https://akkoma-updates.s3-website.fr-par.scw.cloud/frontend/swagger-ui.zip",
       "build_dir" => "dist",
       "ref" => "stable"
-    },
-    # Third-party frontends
-    "pleroma-fe-vanilla" => %{
-      "name" => "pleroma-fe-vanilla",
-      "git" => "https://git.pleroma.social/pleroma/pleroma-fe/",
-      "build_url" =>
-        "https://git.pleroma.social/api/packages/pleroma/generic/pleroma-fe-builds/${ref}/latest.zip",
-      "ref" => "develop",
-      "build_dir" => "dist",
-      "bugtracker" => "https://git.pleroma.social/pleroma/pleroma-fe/issues"
-    },
-    "pl-fe" => %{
-      "name" => "pl-fe",
-      "git" => "https://codeberg.org/mkljczk/pl-fe",
-      "build_url" => "https://pl.mkljczk.pl/pl-fe.zip",
-      "ref" => "develop",
-      "build_dir" => ".",
-      "bugtracker" => "https://codeberg.org/mkljczk/pl-fe/issues"
     }
   }
 
@@ -859,6 +826,7 @@ config :pleroma, configurable_from_database: false
 
 config :pleroma, Pleroma.Repo,
   parameters: [
+    gin_fuzzy_search_limit: "500",
     plan_cache_mode: "force_custom_plan"
   ]
 
@@ -869,8 +837,7 @@ private_instance? = :if_instance_is_private
 config :pleroma, :restrict_unauthenticated,
   timelines: %{local: private_instance?, federated: private_instance?, bubble: true},
   profiles: %{local: private_instance?, remote: private_instance?},
-  activities: %{local: private_instance?, remote: private_instance?},
-  search: %{all: private_instance?, resolve: true, paginate: true}
+  activities: %{local: private_instance?, remote: private_instance?}
 
 config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: false
 
@@ -902,18 +869,9 @@ config :pleroma, ConcurrentLimiter, [
   {Pleroma.Search, [max_running: 30, max_waiting: 50]}
 ]
 
-config :pleroma, Pleroma.Web.WebFinger,
-  domain: nil,
-  # this _forces_ a nickname rediscovery and validation, otherwise only updates when detecting a change
-  # TODO: default this to false after the fallout from recent WebFinger bugs is healed
-  update_nickname_on_user_fetch: true
+config :pleroma, Pleroma.Web.WebFinger, domain: nil, update_nickname_on_user_fetch: true
 
-config :pleroma, Pleroma.Search,
-  module: Pleroma.Search.DatabaseSearch,
-  # note this + pre- & postprocessing needs to fit into Phoenix/Cowboy’s timeout too (default: 60s)
-  task_timeout: 45_000
-
-config :pleroma, Pleroma.Search.DatabaseSearch, gin_fuzzy_search_limit: nil
+config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
 
 config :pleroma, Pleroma.Search.Meilisearch,
   url: "http://127.0.0.1:7700/",

config/description.exs

@@ -61,18 +61,6 @@ frontend_options = [
     type: :string,
     description: "The directory inside the zip file "
   },
-  %{
-    key: "blind_trust",
-    label: "Blindly trust frontend devs?",
-    type: :boolean,
-    description: "Do NOT change this unless you’re really sure"
-  },
-  %{
-    key: "bugtracker",
-    label: "Bug tracker",
-    type: :string,
-    description: "Where to report bugs (for third-party FEs)"
-  },
   %{
     key: "custom-http-headers",
     label: "Custom HTTP headers",
@@ -863,6 +851,11 @@ config :pleroma, :config_description, [
           100
         ]
       },
+      %{
+        key: :skip_thread_containment,
+        type: :boolean,
+        description: "Skip filtering out broken threads. Default: enabled."
+      },
       %{
         key: :limit_to_local_content,
         type: {:dropdown, :atom},
@@ -1496,7 +1489,8 @@ config :pleroma, :config_description, [
     group: :pleroma,
     key: :manifest,
     type: :group,
-    description: "This section describe PWA manifest instance-specific values.",
+    description:
+      "This section describe PWA manifest instance-specific values. Currently this option relate only for MastoFE.",
     children: [
       %{
         key: :icons,
@@ -3115,29 +3109,6 @@ config :pleroma, :config_description, [
             description: "Disallow viewing remote posts."
           }
         ]
-      },
-      %{
-        key: :search,
-        type: :map,
-        description: "Settings for search endpoints.",
-        children: [
-          %{
-            key: :all,
-            type: :boolean,
-            description: "Disallow search access entirely."
-          },
-          %{
-            key: :resolve,
-            type: :boolean,
-            description: "Disallow fetching not-yet-known remote content via search."
-          },
-          %{
-            key: :paginate,
-            type: :boolean,
-            description:
-              "Disallow traversing past the first page of results (search pagination can be inefficient)."
-          }
-        ]
       }
     ]
   },
@@ -3279,7 +3250,8 @@ config :pleroma, :config_description, [
         suggestions: [
           Pleroma.Web.Preload.Providers.Instance,
           Pleroma.Web.Preload.Providers.User,
-          Pleroma.Web.Preload.Providers.Timelines
+          Pleroma.Web.Preload.Providers.Timelines,
+          Pleroma.Web.Preload.Providers.StatusNet
         ]
       }
     ]
@@ -3378,12 +3350,6 @@ config :pleroma, :config_description, [
         type: :module,
         description: "Selected search module.",
         suggestions: {:list_behaviour_implementations, Pleroma.Search.SearchBackend}
-      },
-      %{
-        key: :task_timeout,
-        type: :integer,
-        description: "Timeout for individual search tasks.",
-        suggestions: [45_000]
       }
     ]
   },
@@ -3518,7 +3484,7 @@ config :pleroma, :config_description, [
         key: :module,
         type: :module,
         description: "Translation module.",
-        suggestions: {:list_behaviour_implementations, Pleroma.Akkoma.Translator.Provider}
+        suggestions: {:list_behaviour_implementations, Pleroma.Akkoma.Translator}
       }
     ]
   },

config/test.exs

@@ -35,6 +35,7 @@ config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Test, enabled:
 config :pleroma, :instance,
   email: "admin@example.com",
   notify_email: "noreply@example.com",
+  skip_thread_containment: false,
   federating: false,
   external_user_synchronization: false,
   static_dir: "test/instance_static/"
@@ -128,7 +129,9 @@ config :pleroma, :cachex, provider: Pleroma.CachexMock
 
 config :pleroma, Pleroma.Web.WebFinger, update_nickname_on_user_fetch: false
 
-config :pleroma, :side_effects, ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock
+config :pleroma, :side_effects,
+  ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock,
+  logger: Pleroma.LoggerMock
 
 config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
 

docker-compose.yml

@@ -45,10 +45,6 @@ services:
     ]
     volumes:
       - .:/opt/akkoma
-    ulimits:
-      # Maximum number of open file descriptors. The default should work well for most,
-      # but can be bumped for very large instances or if seeing FD-related errors.
-      nofile: 524288
 
   # Copy this into docker-compose.override.yml and uncomment there if you want to use a reverse proxy
   #proxy:

docs/docs/administration/CLI_tasks/config.md

@@ -205,17 +205,3 @@ Once you have modified the JSON file, you can load it back into the database.
     An instance reboot is needed for many changes to take effect,
     you may want to visit `/api/v1/pleroma/admin/restart` on your instance
     to soft-restart the instance.
-
-## Remove non-whitelisted configs from the database
-
-This removes any configuration value that is not explicitly whitelisted by `:pleroma, :database_config_whitelist`. Might be useful after updating the whitelist.
-
-=== "OTP"
-    ```sh
-    ./bin/pleroma_ctl config filter_whitelisted
-    ```
-
-=== "From Source"
-    ```sh
-    mix pleroma.config filter_whitelisted
-    ```

docs/docs/administration/CLI_tasks/database.md

@@ -120,6 +120,22 @@ when all orphaned activities have been deleted.
 - `--no-singles` - Do not delete activites referencing single objects
 - `--no-arrays` - Do not delete activites referencing an array of objects
 
+## Create a conversation for all existing DMs
+
+Can be safely re-run
+
+=== "OTP"
+
+    ```sh
+    ./bin/pleroma_ctl database bump_all_conversations
+    ```
+
+=== "From Source"
+
+    ```sh
+    mix pleroma.database bump_all_conversations
+    ```
+
 ## Remove duplicated items from following and update followers count for all users
 
 === "OTP"

docs/docs/administration/CLI_tasks/uploads.md

@@ -19,26 +19,3 @@
 - `--delete` - delete local uploads after migrating them to the target uploader
 
 A list of available uploaders can be seen in [Configuration Cheat Sheet](../../configuration/cheatsheet.md#pleromaupload)
-
-## Rewriting old media URLs
-
-After a migration has taken place, old URLs in your database will not have been changed. You
-will want to run this task to update these URLs.
-
-Use the full URL here. So if you moved from `media.example.com/media` to `media.another.com/data`, you'd run with arguments
-`old_url = https://media.example.com/media` and `new_url = https://media.another.com/data`.
-
-=== "OTP"
-
-    ```sh
-     ./bin/pleroma_ctl uploads rewrite_media_domain <old_url> <new_url>
-    ```
-
-=== "From Source"
-
-    ```sh
-    mix pleroma.uploads rewrite_media_domain <old_url> <new_url>
-    ```
-
-### Options
-- `--dry-run` - Do not action any update and simply print what _would_ happen

docs/docs/administration/backup.md

@@ -18,8 +18,8 @@
 3. Go to the working directory of Akkoma (default is `/opt/akkoma`)
 4. Copy the above-mentioned files back to their original position.
 5. Drop the existing database and user[¹]. `sudo -Hu postgres psql -c 'DROP DATABASE akkoma;';` `sudo -Hu postgres psql -c 'DROP USER akkoma;'`
-6. Restore the database schema and akkoma role[¹] (replace the password with the one you find in the configuration file), `sudo -Hu postgres psql -c "CREATE USER akkoma WITH ENCRYPTED PASSWORD '<database-password-wich-you-can-find-in-your-configuration-file>';";` `sudo -Hu postgres psql -c "CREATE DATABASE akkoma OWNER akkoma;"`.
-7. Now restore the Akkoma instance's data into the empty database schema[¹]: `sudo -Hu postgres pg_restore -j "$(nproc)" -d akkoma -v </path/to/backup_location/akkoma.pgdump>`
+6. Restore the database schema and akkoma role[¹] (replace the password with the one you find in the configuration file), `sudo -Hu postgres psql -c "CREATE USER akkoma WITH ENCRYPTED PASSWORD '<database-password-wich-you-can-find-in-your-configuration-file>';"` `sudo -Hu postgres psql -c "CREATE DATABASE akkoma OWNER akkoma;"`.
+7. Now restore the Akkoma instance's data into the empty database schema[¹]: `sudo -Hu postgres pg_restore -d akkoma -v -1 </path/to/backup_location/akkoma.pgdump>`
 8. If you installed a newer Akkoma version, you should run the database migrations `./bin/pleroma_ctl migrate`[²].
 9. Restart the Akkoma service.
 10. Run `sudo -Hu postgres vacuumdb --all --analyze-in-stages`. This will quickly generate the statistics so that postgres can properly plan queries.

docs/docs/administration/monitoring.md

@@ -53,10 +53,6 @@ curl -i -H 'Authorization: Bearer $ACCESS_TOKEN' https://myinstance.example/api/
 You may use the eponymous [Prometheus](https://prometheus.io/)
 or anything compatible with it like e.g. [VictoriaMetrics](https://victoriametrics.com/).
 The latter claims better performance and storage efficiency.
-However, at the moment our reference dashboard only works with VictoriaMetrics,
-thus if you wish to use use the reference as an easy dropin template you must
-use VictoriaMetrics.
-Patches to allow the dashboard to work with plain Prometheus are welcome though.
 
 Both of them can usually be easily installed via distro-packages or docker.
 Depending on your distro or installation method the preferred way to change the CLI arguments and the location of config files may differ; consult the documentation of your chosen method to find out.  
@@ -123,8 +119,7 @@ That’s it, you’ve got a fancy dashboard with long-term, 24/7 metrics now!
 Updating the dashboard can be done by just repeating the import process.
 
 Here’s an example taken from a healthy, small instance where
-nobody was logged in except for a few minutes
-resulting in an (expected) spike in incoming requests:  
+nobody was logged in for about the first half of the displayed time span:  
 ![Full view of the reference dashboard as it looked at the time of writing](img/grafana_dashboard.webp)
 
 !!! note
@@ -241,7 +236,7 @@ If this makes too much noise, consider filtering out telltale delivery failures.
 
 On the opposite side of things, a `http_401` error for example is always worth looking into!
 
-## Built-in Dashboard (Phoenix)
+## Built-in Dashboard
 
 Administrators can access a live dashboard under `/phoenix/live_dashboard`
 giving an overview of uptime, software versions, database stats and more.
@@ -258,56 +253,18 @@ as well as database diagnostics.
 BEAM VM stats include detailed memory consumption breakdowns
 and a full list of running processes for example.
 
-### Postgres Statements Statistics
-
-The built-in dashboard can list the queries your instances spends the 
-most accumulative time on giving insight into potential bottlenecks
-and what might be worth optimising.
-This is the “Outliers” tab in “Ecto Stats”.  
-However for this to work you first need to enable a PostgreSQL extension
-as follows:
-
-Add the following two lines two your `postgresql.conf` (typically placed in your data dir):
-
-```
-shared_preload_libraries = 'pg_stat_statements'
-pg_stat_statements.track = all
-```
-
-Now restart PostgreSQL. Then connect to your akkoma database using `psql` and run:
-
-```sql
-CREATE EXTENSION IF NOT EXISTS pg_stat_statements;
-```
-
-Execution time statistics will now start to be gathered.
-To get a representative sample of your instances workload you should wait a week or at least a day.
-
-These statistics are never reset automatically, but with new Akkoma releases and
-changes in the servers your instance federates with the workload will evolve.
-Thus it’s a good idea to reset this occasionally using:
-
-```sql
--- get user oid:  SELECT oid FROM pg_roles WHERE rolname = 'akkoma';
--- get db oid: SELECT oid FROM pg_database WHERE datname = 'akkoma';
-SELECT pg_stat_statements_reset('<akkoma user oid>'::regclass::oid, '<akkoma database oid>'::regclass::oid);
-
--- or alternatively, to just reset stats for all users and databases:
---   SELECT pg_stat_statements_reset();
-```
-
 ## Oban Web
 
 This too requires administrator rights to access and can be found under `/akkoma/oban` if enabled.
 The exposed aggregate info is mostly redundant with job statistics already tracked in Prometheus,
 but it additionally also:
 
+ - shows not-yet executed jobs in the backlog of queues
  - shows full argument and meta details for each job
  - allows interactively deleting or manually retrying jobs  
   *(keep this in mind when granting people administrator rights!)*
 
 However, there are two caveats:
-
 1. Just as with the other built-in dashboard, data is not kept around
     (although here a **short** backlog actually exists);
     when you notice an issue during use and go here to check it likely is already too late.
@@ -315,4 +272,4 @@ However, there are two caveats:
     by default failed and succeeded jobs will disappear after about a minute.
 2. This dashboard comes with some seemingly constant-ish overhead.
     For large instances this appears to be negligible, but small instances on weaker hardware might suffer.
-    Thus this dashboard can be disabled in the [config](../configuration/cheatsheet.md#oban-web).
+    Thus this dashboard can be disabled in the [config](../cheatsheet.md#oban-web).

docs/docs/administration/updating.md

@@ -31,10 +31,6 @@ su -s "$SHELL" akkoma
 
 # Update frontend(s). See Frontend Configuration doc for more information.
 ./bin/pleroma_ctl frontend install pleroma-fe --ref stable
-
-# If there were migrations and you now experience degraded db perf
-# force the planner to pick up the changes to resolve it:
-./bin/pleroma_ctl database vacuum analyze
 ```
 
 If you selected an alternate flavour on installation,
@@ -68,8 +64,4 @@ sudo systemctl start akkoma
 
 # Update akkoma-fe frontend to latest stable. For other Frontends see Frontend Configuration doc for more information.
 mix pleroma.frontend install pleroma-fe --ref stable
-
-# If there were migrations and you now experience degraded db perf
-# force the planner to pick up the changes to resolve it:
-mix pleroma.database vacuum analyze
 ```

docs/docs/configuration/cheatsheet.md

@@ -54,8 +54,7 @@ To add configuration to your config file, you can copy it from the base config.
 * `remote_post_retention_days`: The default number of days to retain remote posts when pruning the database.
 * `user_bio_length`: A user bio maximum length (default: `5000`).
 * `user_name_length`: A user name maximum length (default: `100`).
-* `skip_thread_containment`: **DEPRECATED**, DO NOT CHANGE THE DEFAULT!  
-    Skip filter out broken threads. The default is `false`.
+* `skip_thread_containment`: Skip filter out broken threads. The default is `false`.
 * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`.
 * `max_account_fields`: The maximum number of custom fields in the user profile (default: `10`).
 * `max_remote_account_fields`: The maximum number of custom fields in the remote user profile (default: `20`).
@@ -307,11 +306,8 @@ To add your own configuration for akkoma-fe, use it like this:
 config :pleroma, :frontend_configurations,
   pleroma_fe: %{
     theme: "pleroma-dark",
-    backendCommitUrl: 'https://akkoma.dev/AkkomaGang/akkoma/commit/',
-    frontendCommitUrl: 'https://akkoma.dev/AkkomaGang/akkoma-fe/commit/',
-    # ... see all available keys at Akkoma-FE’s /static/config.json and
-    # https://akkoma.dev/AkkomaGang/akkoma-fe/src/commit/develop/src/boot/after_store.js :: setSettings
-  },
+    # ... see /priv/static/static/config.json for the available keys.
+},
   masto_fe: %{
     showInstanceSpecificPanel: true
   }
@@ -371,8 +367,7 @@ Currently, the only option relates to mascots on masto-fe
 
 ### :manifest
 
-This section describes PWA manifest instance-specific values.
-Almost all PWA manifest keys can be overriden or added here, below some particularly relevant examples:
+This section describes PWA manifest instance-specific values. Currently this option relate only for masto-fe.
 
 * `icons`: Describe the icons of the app, this is a list of maps describing icons as
   detailed in its [spec](https://www.w3.org/TR/appmanifest/#imageresource-and-its-members).
@@ -859,6 +854,25 @@ config :logger, :ex_syslogger,
   format: "$metadata[$level] $message"
 ```
 
+## Database options
+
+### RUM indexing for full text search
+
+!!! warning
+    It is recommended to use PostgreSQL v11 or newer. We have seen some minor issues with lower PostgreSQL versions.
+
+* `rum_enabled`: If RUM indexes should be used. Defaults to `false`.
+
+RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. While they may eventually be mainlined, for now they have to be installed as a PostgreSQL extension from https://github.com/postgrespro/rum.
+
+Their advantage over the standard GIN indexes is that they allow efficient ordering of search results by timestamp, which makes search queries a lot faster on larger servers, by one or two orders of magnitude. They take up around 3 times as much space as GIN indexes.
+
+To enable them, both the `rum_enabled` flag has to be set and the following special migration has to be run:
+
+`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/`
+
+This will probably take a long time.
+
 ## Authentication
 
 ### :admin_token
@@ -1072,9 +1086,8 @@ Boolean, enables/disables in-database configuration. Read [transferring the conf
 
 List of valid configuration sections which are allowed to be configured from the
 database. Settings stored in the database before the whitelist is configured are
-still applied. Consider running the `mix pleroma.config filter_whitelisted` task
-after updating the whitelist. Read [Remove non-whitelisted configs from the database](../administration/CLI_tasks/config.md#remove-non-whitelisted-configs-from-the-database)
-for more information.
+still applied, so it is suggested to only use the whitelist on instances that
+have not migrated the config to the database.
 
 Example:
 ```elixir

docs/docs/configuration/howto_database_config.md

@@ -7,72 +7,72 @@ The configuration of Akkoma (and Pleroma) has traditionally been managed with a
 
 1. Run the mix task to migrate to the database.
 
-    **Source:**
+  **Source:**
 
-    ```
-    $ mix pleroma.config migrate_to_db
-    ```
+  ```
+  $ mix pleroma.config migrate_to_db
+  ```
 
-    or
+  or
 
-    **OTP:**
+  **OTP:**
 
-    *Note: OTP users need Akkoma to be running for `pleroma_ctl` commands to work*
+  *Note: OTP users need Akkoma to be running for `pleroma_ctl` commands to work*
 
-    ```
-    $ ./bin/pleroma_ctl config migrate_to_db
-    ```
+  ```
+  $ ./bin/pleroma_ctl config migrate_to_db
+  ```
 
-    ```
-      Migrating settings from file: /home/pleroma/config/dev.secret.exs
+  ```
+    Migrating settings from file: /home/pleroma/config/dev.secret.exs
 
-     Settings for key instance migrated.
-     Settings for group :pleroma migrated.
-    ```
+   Settings for key instance migrated.
+   Settings for group :pleroma migrated.
+  ```
 
 2. It is recommended to backup your config file now.
 
-    ```
-    cp config/dev.secret.exs config/dev.secret.exs.orig
-    ```
+  ```
+  cp config/dev.secret.exs config/dev.secret.exs.orig
+  ```
 
 3. Edit your Akkoma config to enable database configuration:
 
-    ```
-    config :pleroma, configurable_from_database: true
-    ```
+  ```
+  config :pleroma, configurable_from_database: true
+  ```
 
 4. ⚠️ **THIS IS NOT REQUIRED** ⚠️
 
-    Now you can edit your config file and strip it down to the only settings which are not possible to control in the database. e.g., the Postgres (Repo) and webserver (Endpoint) settings cannot be controlled in the database because the application needs the settings to start up and access the database.
+  Now you can edit your config file and strip it down to the only settings which are not possible to control in the database. e.g., the Postgres (Repo) and webserver (Endpoint) settings cannot be controlled in the database because the application needs the settings to start up and access the database.
 
-    Any settings in the database will override those in the config file, but you may find it less confusing if the setting is only declared in one place.
+  Any settings in the database will override those in the config file, but you may find it less confusing if the setting is only declared in one place.
 
-    A non-exhaustive list of settings that are only possible in the config file include the following:
+  A non-exhaustive list of settings that are only possible in the config file include the following:
 
-    * config :pleroma, Pleroma.Web.Endpoint
-    * config :pleroma, Pleroma.Repo
-    * config :pleroma, configurable\_from\_database
-    * config :pleroma, :database, rum_enabled
-    * config :pleroma, :connections_pool
+  * config :pleroma, Pleroma.Web.Endpoint
+  * config :pleroma, Pleroma.Repo
+  * config :pleroma, configurable\_from\_database
+  * config :pleroma, :database, rum_enabled
+  * config :pleroma, :connections_pool
 
-    Here is an example of a server config stripped down after migration:
+  Here is an example of a server config stripped down after migration:
 
-    ```
-    use Mix.Config
+  ```
+  use Mix.Config
 
-    config :pleroma, Pleroma.Web.Endpoint,
-      url: [host: "cool.pleroma.site", scheme: "https", port: 443]
+  config :pleroma, Pleroma.Web.Endpoint,
+    url: [host: "cool.pleroma.site", scheme: "https", port: 443]
 
-    config :pleroma, Pleroma.Repo,
-      adapter: Ecto.Adapters.Postgres,
-      username: "akkoma",
-      password: "MySecretPassword",
-      database: "akkoma_prod",
-      hostname: "localhost"
+  config :pleroma, Pleroma.Repo,
+    adapter: Ecto.Adapters.Postgres,
+    username: "akkoma",
+    password: "MySecretPassword",
+    database: "akkoma_prod",
+    hostname: "localhost"
 
-    config :pleroma, configurable_from_database: true
-    ```
+  config :pleroma, configurable_from_database: true
+  ```
 
 5. Restart your instance and you can now access the Settings tab in admin-fe.
 
@@ -81,28 +81,28 @@ The configuration of Akkoma (and Pleroma) has traditionally been managed with a
 
 1. Run the mix task to migrate back from the database. You'll receive some debugging output and a few messages informing you of what happened.
 
-    **Source:**
+  **Source:**
 
-    ```
-    $ mix pleroma.config migrate_from_db
-    ```
+  ```
+  $ mix pleroma.config migrate_from_db
+  ```
 
-    or
+  or
 
-    **OTP:**
+  **OTP:**
 
-    ```
-    $ ./bin/pleroma_ctl config migrate_from_db
-    ```
+  ```
+  $ ./bin/pleroma_ctl config migrate_from_db
+  ```
 
-    ```
-    10:26:30.593 [debug] QUERY OK source="config" db=9.8ms decode=1.2ms queue=26.0ms idle=0.0ms
-    SELECT c0."id", c0."key", c0."group", c0."value", c0."inserted_at", c0."updated_at" FROM "config" AS c0 []
+  ```
+  10:26:30.593 [debug] QUERY OK source="config" db=9.8ms decode=1.2ms queue=26.0ms idle=0.0ms
+  SELECT c0."id", c0."key", c0."group", c0."value", c0."inserted_at", c0."updated_at" FROM "config" AS c0 []
 
-    10:26:30.659 [debug] QUERY OK source="config" db=1.1ms idle=80.7ms
-    SELECT c0."id", c0."key", c0."group", c0."value", c0."inserted_at", c0."updated_at" FROM "config" AS c0 []
-    Database configuration settings have been saved to config/dev.exported_from_db.secret.exs
-    ```
+  10:26:30.659 [debug] QUERY OK source="config" db=1.1ms idle=80.7ms
+  SELECT c0."id", c0."key", c0."group", c0."value", c0."inserted_at", c0."updated_at" FROM "config" AS c0 []
+  Database configuration settings have been saved to config/dev.exported_from_db.secret.exs
+  ```
 
 2. Remove `config :pleroma, configurable_from_database: true` from your config. The in-database configuration still exists, but it will not be used. Future migrations will erase the database config before importing your config file again.
 

docs/docs/configuration/optimisation/general.md

@@ -46,38 +46,3 @@ might help alleviate the impact.
 
 If this condition does **not** hold though,
 setting up such a cache likely only worsens latency and wastes memory.
-
-# Ulimits
-
-Large instances may run into issues with too restrictive OS limits,
-such as `nofile` (the maximum number of open file descriptors).
-For `nofile` specifically, excessively large values can cause issues too however,
-since BEAM will allocate a port management structure whose size is based
-on this maximum leading to just as excessive RAM usage.
-
-On many distros the default limits per user can be configured in a file
-like `/etc/security/limits.conf` (see `man 5 limits.conf`) or an equivalent.
-Remember to change both soft and hard limits, such that the hard limit is
-always greater or equal to the soft limit.  
-You can always lower (but not raise) the limits dynamically for initial testing
-using e.g. `ulimit -Sn 65536 && /usr/bin/mix phx.server` *(this changes just the soft limit)*.
-
-Our Docker setup initialises the `nofile` limit to a reasonable default.
-If needed it can be changed via the `nofile` key in `ulimits` section
-of the `akkoma` service; either in `docker-compose.yml` directly,
-using the `docker-compose.override.yml` file or without recompiling the
-container image via `docker run --ulimit …`.
-
-# DB Search Fuzziness
-
-If using the default database search with a GIN (not RUM!) index
-and specifically search database queries hog up unacceptably
-much database time, you may cap this impact in exchange for
-worse and unstable search results.  
-See [Search Configuraton](../search.md).
-
-This should not be necessary on all but the largest or oldest unpruned servers.
-Also when evaluating search performance keep in mind the time the API endpoint
-takes to respond is not the same as the database time. In particular when
-searching for URLs they may be fetched via the network taking orders of magnitudes
-longer than any normal database query.

docs/docs/configuration/search.md

@@ -2,153 +2,48 @@
 
 {! administration/CLI_tasks/general_cli_task_info.include !}
 
-## General
-
-Processing of each search request is capped.
-If a search exceeds this limit, the API will return an empty result for the affected category.
-Too large values might end up capped by Cowboy’s HTTP request idle timeout, killing the whole response.
-The value can be configured with:
-
-```elixir
-config :pleroma, Pleroma.Search, task_timeout: 51_610
-```
-
-## Search providers
-### Built-in search
+## Built-in search
 
 To use built-in search that has no external dependencies, set the search module to `Pleroma.Activity`:
 
-```elixir
-config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
-```
+> config :pleroma, Pleroma.Search, module: Pleroma.Search.DatabaseSearch
 
-It has no external dependencies and requires the least amount of disk usage.
-However, it is slower than external providers and for performance reasons
-is limited to sorting results by recency alone instead of match quality.
-Result quality may depend on how well your FTS config (typically a language)
-matches the posts on your server.
+While it has no external dependencies, it has problems with performance and relevancy.
 
-Also keep in mind to make use of PostgreSQL’s websearch syntax. Full documentation can be found 
-[here](https://www.postgresql.org/docs/current/textsearch-controls.html#TEXTSEARCH-PARSING-QUERIES).
+## Meilisearch
 
-In short, searching for `apple pie` will match everything containing those two words in any order or place of the text.
-`"apple pie"` only matches if both words appear and `pie` immediately follows `apple`.
-`apple -orange` matches everything referencing apple(s) but not orange(s).
-And `or` acts as an operator to be used for alternatives; for a literal word it must be quoted `"or"`.
+Note that it's quite a bit more memory hungry than PostgreSQL (around 4-5G for ~1.2 million
+posts while idle and up to 7G while indexing initially). The disk usage for this additional index is also
+around 4 gigabytes. Like [RUM](./cheatsheet.md#rum-indexing-for-full-text-search) indexes, it offers considerably
+higher performance and ordering by timestamp in a reasonable amount of time.
+Additionally, the search results seem to be more accurate.
 
-#### Change FTS config
-By default Akkoma uses the `simple` config which performs almost no normalisation (except casing)
-nor removes any stop words, making it rather strict but independent of language.
-
-You can change your config with the `database set_text_search_config` task.
-See [docs on CJK search](./howto_search_cjk.md) for advanced examples.
-For many languages, Postgres already has a preset built-in you can use as is; e.g.
-
-```sh
-...database set_text_search_config english
-```
-
-#### Fuzzy search
-You may choose to limit the set of posts considered during a FTS search for better performance
-in exchange for potentially non-deterministic and less relevant results. See documentation of
-`gin_fuzzy_search_limit` in [PostgreSQL’s docs](https://www.postgresql.org/docs/17/gin.html#GIN-TIPS).
-By default FTS search is exact and considers everything. To enforce a limit only for FTS queries set e.g.:
-
-```elixir
-config :pleroma, Pleroma.Search.DatabaseSearch, gin_fuzzy_search_limit: 10_000
-```
-
-#### RUM
-
-Instead of a GIN index, the built-in database search can also use a RUM index.
-The hope was to improve performance at the cost of higher disk-storage (around 3× more)
-by leveraging RUM’s capability to include extra data (here timestamps for sorting)
-in the index thus avoiding additional heap lookups.
-
-However, since search queries still need to filter results down to respect visibility etc
-heap lookups are still needed anyway and in practice this probably doesn’t actually help much if at all
-while taking up more disk space and needing extra setup work.  
-You probably don’t want to use this and if you already do, consider migrating back to GIN.
-
-##### Enable
-
-!!! warning
-    It is recommended to use PostgreSQL v11 or newer. We have seen some minor issues with lower PostgreSQL versions.
-
-RUM indexes are a third-party PostgreSQL extension.
-First install it via your distro if available or manually from source: https://github.com/postgrespro/rum.
-
-Then change your Akkoma config to set:
-```elixir
-config :pleroma, :database, rum_enabled: true
-```
-
-To enable them, both the `rum_enabled` flag has to be set and the following special migration has to be run:
-
-Finally run the special RUM migrations:
-
-```sh
-mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/
-```
-
-This will probably take a long time.
-
-##### Disable
-Just delete the config setting again and revert RUM-specific migrations with
-
-```sh
-mix ecto.rollback --all --migrations-path priv/repo/optional_migrations/rum_indexing/
-```
-
-Then reapply any potential regular GIN versions of the reverted migrations:
-
-```sh
-mix ecto.migrate
-```
-
-You can now remove the extension again.
-
-
-### Meilisearch
-
-Note that it's quite a bit more memory hungry than PostgreSQL (around 4-5GB for ~1.2 million
-posts while idle and up to 7GB while indexing initially). It also requires significantly more
-disk space (around 4GB for the previous example setup).  
-This however allows it to process individually queries faster while also sorting results
-by match quality ("relevancy") rather than just by recency.
-Additionally, the search profits from Meilisearch’s typo tolerance etc;
-see: [Meilisearch’s documentation](https://www.meilisearch.com/docs/capabilities/full_text_search/overview).
-
-Due to high memory usage, it may be best to set it up on a different machine, if running Akkoma on a low-resource
+Due to high memory usage, it may be best to set it up on a different machine, if running akkoma on a low-resource
 computer, and use private key authentication to secure the remote search instance.
 
 To use [meilisearch](https://www.meilisearch.com/), set the search module to `Pleroma.Search.Meilisearch`:
 
-```elixir
-config :pleroma, Pleroma.Search, module: Pleroma.Search.Meilisearch
-```
+> config :pleroma, Pleroma.Search, module: Pleroma.Search.Meilisearch
 
 You then need to set the address of the meilisearch instance, and optionally the private key for authentication. You might
 also want to change the `initial_indexing_chunk_size` to be smaller if your server is not very powerful, but not higher than `100_000`,
 because Meilisearch will refuse to process it if it's too big. However, in general you want this to be as big as possible, because Meilisearch
 indexes faster when it can process many posts in a single batch.
 
-```elixir
-config :pleroma, Pleroma.Search.Meilisearch,
-  url: "http://127.0.0.1:7700/",
-  private_key: "private key",
-  search_key: "search key",
-  initial_indexing_chunk_size: 100_000
-```
+> config :pleroma, Pleroma.Search.Meilisearch,
+>    url: "http://127.0.0.1:7700/",
+>    private_key: "private key",
+>    search_key: "search key",
+>    initial_indexing_chunk_size: 100_000
 
 Information about setting up Meilisearch can be found in the
 [official documentation](https://docs.meilisearch.com/learn/getting_started/installation.html).
 You probably want to start it with `MEILI_NO_ANALYTICS=true` environment variable to disable analytics.
-At least version 0.25.0 is required, but you are strongly advised to use at least 0.26.0, as it introduces
+At least version 0.25.0 is required, but you are strongly adviced to use at least 0.26.0, as it introduces
 the `--enable-auto-batching` option which drastically improves performance. Without this option, the search
 is hardly usable on a somewhat big instance.
 
-#### Private key authentication (optional)
+### Private key authentication (optional)
 
 To set the private key, use the `MEILI_MASTER_KEY` environment variable when starting. After setting the _master key_,
 you have to get the _private key_ and possibly _search key_, which are actually used for authentication.
@@ -169,9 +64,9 @@ your configuration file as `private_key`. You should also see a
 If your version of Meilisearch only showed the former,
 just leave `search_key` completely unset in Akkoma's config.
 
-#### Initial indexing
+### Initial indexing
 
-After setting up the configuration, you'll want to index all of your already existing posts. Only public posts are indexed.  You'll only
+After setting up the configuration, you'll want to index all of your already existsing posts. Only public posts are indexed.  You'll only
 have to do it one time, but it might take a while, depending on the amount of posts your instance has seen. This is also a fairly RAM
 consuming process for `meilisearch`, and it will take a lot of RAM when running if you have a lot of posts (seems to be around 5G for ~1.2
 million posts while idle and up to 7G while indexing initially, but your experience may be different).
@@ -212,7 +107,7 @@ of indexing and how many posts have actually been indexed, use the `stats` comma
     mix pleroma.search.meilisearch stats
     ```
 
-#### Clearing the index
+### Clearing the index
 
 In case you need to clear the index (for example, to re-index from scratch, if that needs to happen for some reason), you can
 use the `clear` command:
@@ -232,7 +127,7 @@ there is no need to actually clear the whole index, unless you want **all** of i
 that cannot be re-created from the database, it should also generally be a lot smaller than the size of your database. Still, the size
 depends on the amount of text in posts.
 
-### Elasticsearch
+## Elasticsearch
 
 **Note: This requires at least ElasticSearch 7**
 
@@ -240,22 +135,18 @@ As with Meilisearch, this can be rather memory-hungry, but it is very good at wh
 
 To use [Elasticsearch](https://www.elastic.co/), set the search module to `Pleroma.Search.Elasticsearch`:
 
-```elixir
-config :pleroma, Pleroma.Search, module: Pleroma.Search.Elasticsearch
-```
+> config :pleroma, Pleroma.Search, module: Pleroma.Search.Elasticsearch
 
 You then need to set the URL and authentication credentials if relevant.
 
-```elixir
-config :pleroma, Pleroma.Search.Elasticsearch.Cluster,
-  url: "http://127.0.0.1:9200/",
-  username: "elastic",
-  password: "changeme"
-```
+> config :pleroma, Pleroma.Search.Elasticsearch.Cluster,
+>    url: "http://127.0.0.1:9200/",
+>    username: "elastic",
+>    password: "changeme",
 
-#### Initial indexing
+### Initial indexing
 
-After setting up the configuration, you'll want to index all of your already existing posts. You'll only have to do it one time, but it might take a while, depending on the amount of posts your instance has seen.
+After setting up the configuration, you'll want to index all of your already existsing posts. You'll only have to do it one time, but it might take a while, depending on the amount of posts your instance has seen.
 
 The sequence of actions is as follows:
 

docs/docs/development/API/admin_api.md

@@ -34,7 +34,7 @@ Backwards-compatibility for admin API endpoints without version prefixes (`/api/
   "count": integer,
   "users": [
     {
-      "is_active": bool,
+      "deactivated": bool,
       "id": integer,
       "nickname": string,
       "roles": {
@@ -45,8 +45,8 @@ Backwards-compatibility for admin API endpoints without version prefixes (`/api/
       "tags": array,
       "avatar": string,
       "display_name": string,
-      "is_confirmed": bool,
-      "is_approved": bool,
+      "confirmation_pending": bool,
+      "approval_pending": bool,
       "registration_reason": string,
     },
     ...
@@ -433,7 +433,7 @@ Response:
 * On success: URL of the unfollowed relay
 
 ```json
-"https://example.com/relay"
+{"https://example.com/relay"}
 ```
 
 ## `POST /api/v1/pleroma/admin/users/invite_token`
@@ -1173,23 +1173,20 @@ Loads JSON generated from `config/descriptions.exs`.
 - Response:
 
 ```json
-{
-  "items": [
-    {
-      "id": 1234,
-      "data": {
-        "actor": {
-          "id": 1,
-          "nickname": "lain"
-        },
-        "action": "relay_follow"
+[
+  {
+    "id": 1234,
+    "data": {
+      "actor": {
+        "id": 1,
+        "nickname": "lain"
       },
-      "time": 1502812026, // timestamp
-      "message": "[2017-08-15 15:47:06] @nick0 followed relay: https://example.org/relay" // log message
-    }
-  ],
-  "total": 1
-}
+      "action": "relay_follow"
+    },
+    "time": 1502812026, // timestamp
+    "message": "[2017-08-15 15:47:06] @nick0 followed relay: https://example.org/relay" // log message
+  }
+]
 ```
 
 ## `POST /api/v1/pleroma/admin/reload_emoji`
@@ -1218,10 +1215,24 @@ Loads JSON generated from `config/descriptions.exs`.
 
 ## `GET /api/v1/pleroma/admin/stats`
 
-**DEPRECATED; DO NOT USE**!!
+### Stats
 
-Returned information is only stubbed out.
-The endpoint will be removed entirely in an upcoming release.
+- Query Params:
+  - *optional* `instance`: **string** instance hostname (without protocol) to get stats for
+- Example: `https://mypleroma.org/api/v1/pleroma/admin/stats?instance=lain.com`
+
+- Response:
+
+```json
+{
+  "status_visibility": {
+    "direct": 739,
+    "private": 9,
+    "public": 17,
+    "unlisted": 14
+  }
+}
+```
 
 ## `GET /api/v1/pleroma/admin/oauth_app`
 

docs/docs/development/API/differences_in_mastoapi_responses.md

@@ -14,6 +14,8 @@ by the administrator. It is available under `/api/v1/timelines/bubble`.
 
 Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users.
 
+Adding the parameter `exclude_visibilities` to the timeline queries will exclude the statuses with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`), e.g., `exclude_visibilities[]=direct&exclude_visibilities[]=private`.
+
 Adding the parameter `reply_visibility` to the public, bubble or home timelines queries will filter replies. Possible values: without parameter (default) shows all replies, `following` - replies directed to you or users you follow, `self` - replies directed to you.
 
 Adding the parameter `instance=lain.com` to the public timeline will show only statuses originating from `lain.com` (or any remote instance).
@@ -30,7 +32,7 @@ Home, public, hashtag & list timelines further accept:
 
 ## Statuses
 
-- `visibility`: has additional possible value `local` (for local-only statuses)
+- `visibility`: has additional possible values `list` and `local` (for local-only statuses)
 - `emoji_reactions`: additional field since Akkoma 3.2.0; identical to `pleroma/emoji_reactions`
 
 Has these additional fields under the `pleroma` object:
@@ -49,15 +51,6 @@ Has these additional fields under the `pleroma` object:
 - `parent_visible`: If the parent of this post is visible to the user or not.
 - `pinned_at`: a datetime (iso8601) when status was pinned, `null` otherwise.
 
-Furthermore, has the following additional attributes under the `poll.akkoma` object *(if `poll` is non-null)*:
-- `anonymous`: relays whether the poll creator promised to process votes anonymously *(`true`)*,
-    publishes votes with voter identity to some parties or the public *(`false`)*
-    or did not indicate how votes are processed (`null`).   
-    Note this assumes any remote’s claim about its process are true and
-    even if this is truthfully set to `true`, while no regular users ought to have access to voter identities,
-    the server operator of the poll’s home instance may in principle still be able to
-    extract voter identites via the database or side channels.
-
 The `GET /api/v1/statuses/:id/source` endpoint additionally has the following attributes:
 
 - `content_type`: The content type of the status source.
@@ -67,7 +60,6 @@ The `GET /api/v1/statuses/:id/source` endpoint additionally has the following at
 Has these additional fields in `params`:
 
 - `expires_in`: the number of seconds the posted activity should expire in.
-    **Deprecated**; replaced by Mastodon-compatible `duration`
 
 ## Media Attachments
 
@@ -98,6 +90,7 @@ The `id` parameter can also be the `nickname` of the user. This only works in th
 - `with_muted`: include statuses/reactions from muted accounts
 - `exclude_reblogs`: exclude reblogs
 - `exclude_replies`: exclude replies
+- `exclude_visibilities`: exclude visibilities
 
 Endpoints which accept `with_relationships` parameter:
 
@@ -198,8 +191,8 @@ The `type` value is `pleroma:report`
 
 Accepts additional parameters:
 
+- `exclude_visibilities`: will exclude the notifications for activities with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`). Usage example: `GET /api/v1/notifications?exclude_visibilities[]=direct&exclude_visibilities[]=private`.
 - `include_types`: will include the notifications for activities with the given types. The parameter accepts an array of types (`mention`, `follow`, `reblog`, `favourite`, `move`, `pleroma:emoji_reaction`, `pleroma:report`). Usage example: `GET /api/v1/notifications?include_types[]=mention&include_types[]=reblog`.
-    **Deprecated:** replaced by `types` which is equivalent but (by now) also supported by vanilla Mastodon.
 
 ## DELETE `/api/v1/notifications/destroy_multiple`
 
@@ -221,8 +214,8 @@ Additional parameters can be added to the JSON body/Form data:
 - `content_type`: string, contain the MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint.
 - `to`: A list of nicknames (like `admin@otp.akkoma.dev` or `admin` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for post visibility are not affected by this and will still apply.
 - `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted`, `local` or `public`) it can be used to address a List by setting it to `list:LIST_ID`.
-- `expires_in`: **Deprecated**; replaced by `duration`.  
-    The number of seconds the posted activity should expire in. When a posted activity expires it will be deleted from the server, and a delete request for it will be federated. This needs to be longer than an hour.
+- `expires_in`: The number of seconds the posted activity should expire in. When a posted activity expires it will be deleted from the server, and a delete request for it will be federated. This needs to be longer than an hour.
+- `in_reply_to_conversation_id`: Will reply to a given conversation, addressing only the people who are part of the recipient set of that conversation. Sets the visibility to `direct`.
 
 ## GET `/api/v1/statuses`
 
@@ -260,7 +253,6 @@ Additional parameters can be added to the JSON body/Form data:
 - `allow_following_move` - if true, allows automatically follow moved following accounts
 - `also_known_as` - array of ActivityPub IDs, needed for following move
 - `pleroma_background_image` - sets the background image of the user. Can be set to "" (an empty string) to reset.
-- `pleroma_background_image_description` - sets plaintext alt text for the background image of the user. Can be set to "" (an empty string) to delete.
 - `discoverable` - if true, external services (search bots) etc. are allowed to index / list the account (regardless of this setting, user will still appear in regular search results).
 - `actor_type` - the type of this account.
 - `language` - user's preferred language for receiving emails (digest, confirmation, etc.)
@@ -369,6 +361,10 @@ The message payload consists of:
   - `follower_count`: follower count
   - `following_count`: following count
 
+## User muting and thread muting
+
+Both user muting and thread muting can be done for only a certain time by adding an `expires_in` parameter to the API calls and giving the expiration time in seconds.
+
 ## Not implemented
 
 Akkoma is generally compatible with the Mastodon 2.7.2 API, but some newer features and non-essential features are omitted. These features usually return an HTTP 200 status code, but with an empty response. While they may be added in the future, they are considered low priority.

docs/docs/development/API/pleroma_api.md

@@ -376,8 +376,13 @@ See [Admin-API](admin_api.md)
 
 Pleroma Conversations have the same general structure that Mastodon Conversations have. The behavior differs in the following ways when using these endpoints:
 
-1. Pleroma Conversations never add or remove recipients (`accounts` key), unless explicitly changed by the user.
+1. Pleroma Conversations never add or remove recipients, unless explicitly changed by the user.
 2. Pleroma Conversations statuses can be requested by Conversation id.
+3. Pleroma Conversations can be replied to.
+
+Conversations have the additional field `recipients` under the `pleroma` key. This holds a list of all the accounts that will receive a message in this conversation.
+
+The status posting endpoint takes an additional parameter, `in_reply_to_conversation_id`, which, when set, will set the visiblity to direct and address only the people who are the recipients of that Conversation.
 
 ⚠ Conversation IDs can be found in direct messages with the `pleroma.direct_conversation_id` key, do not confuse it with `pleroma.conversation_id`.
 

docs/docs/development/ap_extensions.md

@@ -244,7 +244,7 @@ Post using this scope will never federate to other servers
 but for the sake of completeness it is listed here.
 
 In addition to the usual scopes *(public, unlisted, followers-only, direct)*
-Akkoma supports a “local” post scope. Such posts will not federate to
+Akkoma supports an “unlisted” post scope. Such posts will not federate to
 other instances and only be shown to logged-in users on the same instance.
 It is included into the local timeline.  
 This may be useful to discuss or announce instance-specific policies and topics.
@@ -267,32 +267,16 @@ special meaning to the potential local-scope identifier.
     however those are also shown publicly on the local web interface
     and are thus visible to non-members.
 
-# Deprecated and Removed Extensions
-
-The following extensions were used in the past but have been dropped.
-Documentation is retained here as a reference and since old objects might
-still contains related fields.
-
 ## List post scope
 
 Messages originally addressed to a custom list will contain
 a `listMessage` field with an unresolvable pseudo ActivityPub id.
 
-!!! note
-    The concept did not work out too well in practice with even remote servers
-    recognising the `listMessage` extension being unaware of the state of the
-    list and resulting weird desyncs in thread display and handling between
-    servers.  
-    As it was it also never found its way in any known clients or frontends.
+# Deprecated and Removed Extensions
 
-    A more consistent superset of what this was able to actually do
-    can be achieved without ActivityPub extensions by explicitly addressing
-    all intended participants without inline mentions.
-    While true federated and moderated "lists" or "groups"
-    will need more work and a different approach.
-
-    Thus suport for it was removed and it is recommended
-    to not create any new implementation of it.
+The following extensions were used in the past but have been dropped.
+Documentation is retained here as a reference and since old objects might
+still contains related fields.
 
 ## Actor endpoints
 

docs/docs/installation/frontends.include

@@ -8,7 +8,6 @@ command to install.
 
 You **must** run frontend management tasks as the akkoma user,
 the same way you downloaded the build or cloned the git repo before.
-This includes adding the same `MIX_ENV=…` prefix as used in prior commands if any!
 But otherwise, for most installations, the following will suffice:
 
 === "OTP" 

docs/docs/installation/generic_dependencies.include

@@ -1,8 +1,8 @@
 ## Required dependencies
 
 * PostgreSQL 12+
-* Elixir 1.15+ (currently tested up to 1.19)
-* Erlang OTP 25+ (currently tested up to OTP28)
+* Elixir 1.14.1+ (currently tested up to 1.18)
+* Erlang OTP 25+ (currently tested up to OTP27)
 * git
 * file / libmagic
 * gcc (clang might also work)

installation/nginx/akkoma.nginx

@@ -22,7 +22,7 @@ upstream phoenix {
 #     listen         [::]:80;
 #
 #     location / {
-#         return         301 https://$host$request_uri;
+#         return         301 https://$server_name$request_uri;
 #     }
 # }
 

lib/mix/tasks/pleroma/benchmark.ex

@@ -11,7 +11,7 @@ defmodule Mix.Tasks.Pleroma.Benchmark do
 
     Benchee.run(%{
       "search" => fn ->
-        Pleroma.Search.DatabaseSearch.search(nil, "cofe", resolve: false)
+        Pleroma.Activity.search(nil, "cofe")
       end
     })
   end

lib/mix/tasks/pleroma/config.ex

@@ -244,61 +244,6 @@ defmodule Mix.Tasks.Pleroma.Config do
     end
   end
 
-  # Removes non-whitelisted configuration sections
-  def run(["filter_whitelisted" | rest]) do
-    {options, [], []} =
-      OptionParser.parse(
-        rest,
-        strict: [force: :boolean],
-        aliases: [f: :force]
-      )
-
-    force = Keyword.get(options, :force, false)
-
-    start_pleroma()
-
-    whitelisted_configs = Pleroma.Config.get(:database_config_whitelist)
-
-    if whitelisted_configs in [nil, false] do
-      shell_info("No unwanted settings in ConfigDB. No changes made.")
-    else
-      whitelisted_groups =
-        whitelisted_configs
-        |> Enum.filter(fn
-          {_group} -> true
-          _ -> false
-        end)
-        |> Enum.map(fn {group} -> group end)
-
-      whitelisted_keys =
-        whitelisted_configs
-        |> Enum.filter(fn
-          {_group, _key} -> true
-          _ -> false
-        end)
-
-      filtered =
-        from(c in ConfigDB)
-        |> Repo.all()
-        |> Enum.filter(&not_whitelisted?(&1, whitelisted_groups, whitelisted_keys))
-
-      if not Enum.empty?(filtered) do
-        shell_info("The following settings will be removed from ConfigDB:\n")
-        Enum.each(filtered, &dump(&1))
-
-        if force or shell_prompt("Are you sure you want to continue?", "n") in ~w(Yn Y y) do
-          filtered_ids = Enum.map(filtered, fn %{id: id} -> id end)
-
-          Repo.delete_all(from(c in ConfigDB, where: c.id in ^filtered_ids))
-        else
-          shell_error("No changes made.")
-        end
-      else
-        shell_info("No unwanted settings in ConfigDB. No changes made.")
-      end
-    end
-  end
-
   @spec migrate_to_db(Path.t() | nil) :: any()
   def migrate_to_db(file_path \\ nil) do
     with :ok <- Pleroma.Config.DeprecationWarnings.warn() do
@@ -498,9 +443,4 @@ defmodule Mix.Tasks.Pleroma.Config do
     Ecto.Adapters.SQL.query!(Repo, "TRUNCATE config;")
     Ecto.Adapters.SQL.query!(Repo, "ALTER SEQUENCE config_id_seq RESTART;")
   end
-
-  defp not_whitelisted?(%{group: group, key: key}, whitelisted_groups, whitelisted_keys) do
-    not Enum.member?(whitelisted_groups, group) and
-      not Enum.member?(whitelisted_keys, {group, key})
-  end
 end

lib/mix/tasks/pleroma/database.ex

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Mix.Tasks.Pleroma.Database do
+  alias Pleroma.Conversation
   alias Pleroma.Maintenance
   alias Pleroma.Object
   alias Pleroma.Repo
@@ -285,12 +286,17 @@ defmodule Mix.Tasks.Pleroma.Database do
     end
   end
 
+  def run(["bump_all_conversations"]) do
+    start_pleroma()
+    Conversation.bump_for_all_activities()
+  end
+
   def run(["update_users_following_followers_counts"]) do
     start_pleroma()
 
     Repo.transaction(
       fn ->
-        from(u in User, select: u, where: u.local or u.is_active)
+        from(u in User, select: u)
         |> Repo.stream()
         |> Stream.each(&User.update_follower_count/1)
         |> Stream.run()
@@ -597,7 +603,7 @@ defmodule Mix.Tasks.Pleroma.Database do
         Ecto.Adapters.SQL.query!(
           Pleroma.Repo,
           "CREATE OR REPLACE FUNCTION objects_fts_update() RETURNS trigger AS $$ BEGIN
-          new.fts_content := to_tsvector(COALESCE(new.data->>'summary', '') || ' ' || (new.data->>'content'));
+          new.fts_content := to_tsvector(new.data->>'content');
           RETURN new;
           END
           $$ LANGUAGE plpgsql",
@@ -612,14 +618,7 @@ defmodule Mix.Tasks.Pleroma.Database do
 
         Ecto.Adapters.SQL.query!(
           Pleroma.Repo,
-          """
-          CREATE INDEX CONCURRENTLY objects_fts ON objects USING gin(
-            to_tsvector(
-              '#{tsconfig}',
-              COALESCE(data->>'summary', '') || ' ' || (data->>'content')
-            )
-          );
-          """,
+          "CREATE INDEX CONCURRENTLY objects_fts ON objects USING gin(to_tsvector('#{tsconfig}', data->>'content')); ",
           [],
           timeout: :infinity
         )

lib/mix/tasks/pleroma/diagnostics.ex

@@ -44,11 +44,12 @@ defmodule Mix.Tasks.Pleroma.Diagnostics do
       |> Map.put(:followed_hashtags, followed_hashtags)
       |> Map.delete(:local)
 
+    list_memberships = Pleroma.List.memberships(user)
     recipients = [user.ap_id | User.following(user)]
 
     query =
       Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query(
-        recipients,
+        recipients ++ list_memberships,
         params
       )
       |> limit(20)
@@ -70,6 +71,8 @@ defmodule Mix.Tasks.Pleroma.Diagnostics do
       |> Map.put(:actor_id, user.ap_id)
       |> Map.put(:pinned_object_ids, Map.keys(user.pinned_objects))
 
+    list_memberships = Pleroma.List.memberships(user)
+
     recipients =
       %{
         godmode: params[:godmode],
@@ -78,7 +81,7 @@ defmodule Mix.Tasks.Pleroma.Diagnostics do
       |> Pleroma.Web.ActivityPub.ActivityPub.user_activities_recipients()
 
     query =
-      recipients
+      (recipients ++ list_memberships)
       |> Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query(params)
       |> limit(20)
 

lib/mix/tasks/pleroma/email.ex

@@ -33,7 +33,7 @@ defmodule Mix.Tasks.Pleroma.Email do
 
     Pleroma.User.Query.build(%{
       local: true,
-      deactivated: false,
+      is_active: true,
       is_confirmed: false,
       invisible: false
     })

lib/mix/tasks/pleroma/notification_settings.ex

@@ -41,7 +41,18 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do
   end
 
   defp build_query(hide_notification_contents, options) do
-    criteria = %{internal: :allowed, local: true}
+    query =
+      from(u in Pleroma.User,
+        update: [
+          set: [
+            notification_settings:
+              fragment(
+                "jsonb_set(notification_settings, '{hide_notification_contents}', ?)",
+                ^hide_notification_contents
+              )
+          ]
+        ]
+      )
 
     user_emails =
       options
@@ -50,11 +61,11 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do
       |> Enum.map(&String.trim(&1))
       |> Enum.reject(&(&1 == ""))
 
-    criteria =
+    query =
       if length(user_emails) > 0 do
-        Map.put(criteria, :email, user_emails)
+        where(query, [u], u.email in ^user_emails)
       else
-        criteria
+        query
       end
 
     user_nicknames =
@@ -64,23 +75,13 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do
       |> Enum.map(&String.trim(&1))
       |> Enum.reject(&(&1 == ""))
 
-    criteria =
+    query =
       if length(user_nicknames) > 0 do
-        Map.put(criteria, :nickname, user_nicknames)
+        where(query, [u], u.nickname in ^user_nicknames)
       else
-        criteria
+        query
       end
 
-    criteria
-    |> Pleroma.User.Query.build(criteria)
-    |> update([u],
-      set: [
-        notification_settings:
-          fragment(
-            "jsonb_set(notification_settings, '{hide_notification_contents}', ?)",
-            ^hide_notification_contents
-          )
-      ]
-    )
+    query
   end
 end

lib/mix/tasks/pleroma/refresh_counter_cache.ex

@@ -0,0 +1,68 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.RefreshCounterCache do
+  @shortdoc "Refreshes counter cache"
+
+  use Mix.Task
+
+  alias Pleroma.Activity
+  alias Pleroma.CounterCache
+  alias Pleroma.Repo
+
+  import Ecto.Query
+
+  def run([]) do
+    Mix.Pleroma.start_pleroma()
+
+    instances =
+      Activity
+      |> distinct([a], true)
+      |> select([a], fragment("split_part(?, '/', 3)", a.actor))
+      |> Repo.all()
+
+    instances
+    |> Enum.with_index(1)
+    |> Enum.each(fn {instance, i} ->
+      counters = instance_counters(instance)
+      CounterCache.set(instance, counters)
+
+      Mix.Pleroma.shell_info(
+        "[#{i}/#{length(instances)}] Setting #{instance} counters: #{inspect(counters)}"
+      )
+    end)
+
+    Mix.Pleroma.shell_info("Done")
+  end
+
+  defp instance_counters(instance) do
+    counters = %{"public" => 0, "unlisted" => 0, "private" => 0, "direct" => 0}
+
+    Activity
+    |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
+    |> where([a], fragment("split_part(?, '/', 3) = ?", a.actor, ^instance))
+    |> select(
+      [a],
+      {fragment(
+         "activity_visibility(?, ?, ?)",
+         a.actor,
+         a.recipients,
+         a.data
+       ), count(a.id)}
+    )
+    |> group_by(
+      [a],
+      fragment(
+        "activity_visibility(?, ?, ?)",
+        a.actor,
+        a.recipients,
+        a.data
+      )
+    )
+    |> Repo.all(timeout: :timer.minutes(30))
+    |> Enum.reduce(counters, fn {visibility, count}, acc ->
+      Map.put(acc, visibility, count)
+    end)
+  end
+end

lib/mix/tasks/pleroma/uploads.ex

@@ -5,7 +5,6 @@
 defmodule Mix.Tasks.Pleroma.Uploads do
   use Mix.Task
   import Mix.Pleroma
-  import Ecto.Query
   alias Pleroma.Upload
   alias Pleroma.Uploaders.Local
   require Logger
@@ -98,106 +97,4 @@ defmodule Mix.Tasks.Pleroma.Uploads do
 
     shell_info("Done!")
   end
-
-  @doc """
-  Rewrite media domains to somewhere new
-  """
-  def run(["rewrite_media_domain", from_url, to_url | args]) do
-    dry_run = Enum.member?(args, "--dry-run")
-    start_pleroma()
-    shell_info("Rewriting media domain from #{from_url} to #{to_url}")
-    shell_info("Dry run: #{dry_run}")
-    # actually selecting based on the attachment URL is stupidly difficult due to it being
-    # stored as a JSONB array in the `data` field... the easier way to do this is just to iterate though
-    # local posts
-    from(o in Pleroma.Object)
-    |> where(
-      [o],
-      fragment(
-        "?->'url'->0->>'href' LIKE ?
-      OR
-      ?->'attachment'->0->'url'->0->>'href' LIKE ?",
-        o.data,
-        ^"#{from_url}%",
-        o.data,
-        ^"#{from_url}%"
-      )
-    )
-    |> Pleroma.Repo.chunk_stream(100, :batches, timeout: :infinity)
-    |> Stream.each(fn chunk ->
-      # now we just rewrite it and save it back, ezpz
-      chunk
-      |> Enum.each(fn object ->
-        new_data =
-          rewrite_url_object(Map.get(object, :id), Map.get(object, :data), from_url, to_url)
-
-        if dry_run do
-          shell_info(
-            "Dry run: would update object #{object.id} to new media domain (#{inspect(new_data)})"
-          )
-        else
-          Pleroma.Repo.update!(Ecto.Changeset.change(object, data: new_data))
-          shell_info("Updated object #{object.id} to new media domain")
-        end
-      end)
-    end)
-    |> Stream.run()
-  end
-
-  defp rewrite_url(id, url, from_url, to_url) do
-    new_uri = String.replace(url, from_url, to_url)
-    check = URI.parse(new_uri)
-
-    case check do
-      %URI{scheme: nil, host: nil} ->
-        raise("Invalid URL after rewriting: #{new_uri} (object ID: #{id})")
-
-      _ ->
-        new_uri
-    end
-  end
-
-  # The base object - we're looking for this, it has the actual url
-  defp rewrite_url_object(id, %{"type" => "Link", "href" => href} = link, from_url, to_url) do
-    Map.put(link, "href", rewrite_url(id, href, from_url, to_url))
-  end
-
-  defp rewrite_url_object(id, %{"type" => type, "url" => urls} = object, from_url, to_url)
-       when type in ["Document", "Image"] do
-    # Document and Image contain url field, which will always be an array of links
-    Map.put(
-      object,
-      "url",
-      Enum.map(
-        urls,
-        fn url -> rewrite_url_object(id, url, from_url, to_url) end
-      )
-    )
-  end
-
-  defp rewrite_url_object(
-         id,
-         %{"type" => _type, "attachment" => attachments} = object,
-         from_url,
-         to_url
-       ) do
-    # Note will contain an attachment field, which is an array of documents
-    Map.put(
-      object,
-      "attachment",
-      Enum.map(attachments, fn attachment ->
-        rewrite_url_object(id, attachment, from_url, to_url)
-      end)
-    )
-  end
-
-  defp rewrite_url_object(
-         _id,
-         object,
-         _,
-         _
-       ) do
-    shell_info(inspect(object))
-    raise("Unhandled object format!")
-  end
 end

lib/mix/tasks/pleroma/user.ex

@@ -176,7 +176,7 @@ defmodule Mix.Tasks.Pleroma.User do
   def run(["deactivate_all_from_instance", instance]) do
     start_pleroma()
 
-    Pleroma.User.Query.build(%{nickname_suffix: "@#{instance}"})
+    Pleroma.User.Query.build(%{nickname: "@#{instance}"})
     |> Pleroma.Repo.chunk_stream(500, :batches)
     |> Stream.each(fn users ->
       users
@@ -262,7 +262,7 @@ defmodule Mix.Tasks.Pleroma.User do
 
     Pleroma.User.Query.build(%{
       external: true,
-      deactivated: false
+      is_active: true
     })
     |> refetch_public_keys()
   end
@@ -408,7 +408,7 @@ defmodule Mix.Tasks.Pleroma.User do
 
     Pleroma.User.Query.build(%{
       local: true,
-      deactivated: false,
+      is_active: true,
       is_moderator: false,
       is_admin: false,
       invisible: false
@@ -426,7 +426,7 @@ defmodule Mix.Tasks.Pleroma.User do
 
     Pleroma.User.Query.build(%{
       local: true,
-      deactivated: false,
+      is_active: true,
       is_moderator: false,
       is_admin: false,
       invisible: false

lib/pleroma/activity.ex

@@ -33,6 +33,10 @@ defmodule Pleroma.Activity do
     field(:recipients, {:array, :string}, default: [])
     field(:thread_muted?, :boolean, virtual: true)
 
+    # A field that can be used if you need to join some kind of other
+    # id to order / paginate this field by
+    field(:pagination_id, :string, virtual: true)
+
     # This is a fake relation,
     # do not use outside of with_preloaded_user_actor/with_joined_user_actor
     has_one(:user_actor, User, on_delete: :nothing, foreign_key: :id)
@@ -412,6 +416,8 @@ defmodule Pleroma.Activity do
     )
   end
 
+  defdelegate search(user, query, options \\ []), to: Pleroma.Search.DatabaseSearch
+
   def direct_conversation_id(activity, for_user) do
     alias Pleroma.Conversation.Participation
 

lib/pleroma/activity/html.ex

@@ -59,8 +59,6 @@ defmodule Pleroma.Activity.HTML do
       object = Object.normalize(activity, fetch: false)
 
       add_cache_key_for(activity.id, key)
-
-      # callback already produces :commit or :ignore tuples
       HTML.ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
     end)
   end

lib/pleroma/akkoma/translator.ex

@@ -1,17 +1,8 @@
 defmodule Pleroma.Akkoma.Translator do
-  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
-
-  def languages do
-    module = Pleroma.Config.get([:translator, :module])
-
-    @cachex.fetch!(:translations_cache, "languages:#{module}}", fn _ ->
-      with {_, true} <- {:enabled, Pleroma.Config.get([:translator, :enabled])},
-           {:ok, source_languages, dest_languages} <- module.languages() do
-        {:commit, {:ok, source_languages, dest_languages}}
-      else
-        {:enabled, _} -> {:commit, {:enabled, false}}
-        {:error, err} -> {:ignore, {:error, err}}
-      end
-    end)
-  end
+  @callback translate(String.t(), String.t() | nil, String.t()) ::
+              {:ok, String.t(), String.t()} | {:error, any()}
+  @callback languages() ::
+              {:ok, [%{name: String.t(), code: String.t()}],
+               [%{name: String.t(), code: String.t()}]}
+              | {:error, any()}
 end

lib/pleroma/akkoma/translators/argos_translate.ex

@@ -1,5 +1,5 @@
 defmodule Pleroma.Akkoma.Translators.ArgosTranslate do
-  @behaviour Pleroma.Akkoma.Translator.Provider
+  @behaviour Pleroma.Akkoma.Translator
 
   alias Pleroma.Config
 
@@ -23,7 +23,7 @@ defmodule Pleroma.Akkoma.Translators.ArgosTranslate do
     end
   end
 
-  @impl Pleroma.Akkoma.Translator.Provider
+  @impl Pleroma.Akkoma.Translator
   def languages do
     with {response, 0} <- safe_languages() do
       langs =
@@ -83,7 +83,7 @@ defmodule Pleroma.Akkoma.Translators.ArgosTranslate do
 
   defp htmlify_response(string, _), do: string
 
-  @impl Pleroma.Akkoma.Translator.Provider
+  @impl Pleroma.Akkoma.Translator
   def translate(string, nil, to_language) do
     # Akkoma's Pleroma-fe expects us to detect the source language automatically.
     # Argos-translate doesn't have that option (yet?)
@@ -106,7 +106,4 @@ defmodule Pleroma.Akkoma.Translators.ArgosTranslate do
       {response, _} -> {:error, "ArgosTranslate failed to translate (#{response})"}
     end
   end
-
-  @impl Pleroma.Akkoma.Translator.Provider
-  def name, do: "Argos Translate"
 end

lib/pleroma/akkoma/translators/deepl.ex

@@ -1,5 +1,5 @@
 defmodule Pleroma.Akkoma.Translators.DeepL do
-  @behaviour Pleroma.Akkoma.Translator.Provider
+  @behaviour Pleroma.Akkoma.Translator
 
   alias Pleroma.HTTP
   alias Pleroma.Config
@@ -21,7 +21,7 @@ defmodule Pleroma.Akkoma.Translators.DeepL do
     Config.get([:deepl, :tier])
   end
 
-  @impl Pleroma.Akkoma.Translator.Provider
+  @impl Pleroma.Akkoma.Translator
   def languages do
     with {:ok, %{status: 200} = source_response} <- do_languages("source"),
          {:ok, %{status: 200} = dest_response} <- do_languages("target"),
@@ -48,7 +48,7 @@ defmodule Pleroma.Akkoma.Translators.DeepL do
     end
   end
 
-  @impl Pleroma.Akkoma.Translator.Provider
+  @impl Pleroma.Akkoma.Translator
   def translate(string, from_language, to_language) do
     with {:ok, %{status: 200} = response} <-
            do_request(api_key(), tier(), string, from_language, to_language),
@@ -97,7 +97,4 @@ defmodule Pleroma.Akkoma.Translators.DeepL do
       ]
     )
   end
-
-  @impl Pleroma.Akkoma.Translator.Provider
-  def name, do: "DeepL"
 end

lib/pleroma/akkoma/translators/libre_translate.ex

@@ -1,5 +1,5 @@
 defmodule Pleroma.Akkoma.Translators.LibreTranslate do
-  @behaviour Pleroma.Akkoma.Translator.Provider
+  @behaviour Pleroma.Akkoma.Translator
 
   alias Pleroma.Config
   alias Pleroma.HTTP
@@ -13,7 +13,7 @@ defmodule Pleroma.Akkoma.Translators.LibreTranslate do
     Config.get([:libre_translate, :url])
   end
 
-  @impl Pleroma.Akkoma.Translator.Provider
+  @impl Pleroma.Akkoma.Translator
   def languages do
     with {:ok, %{status: 200} = response} <- do_languages(),
          {:ok, body} <- Jason.decode(response.body) do
@@ -30,7 +30,7 @@ defmodule Pleroma.Akkoma.Translators.LibreTranslate do
     end
   end
 
-  @impl Pleroma.Akkoma.Translator.Provider
+  @impl Pleroma.Akkoma.Translator
   def translate(string, from_language, to_language) do
     with {:ok, %{status: 200} = response} <- do_request(string, from_language, to_language),
          {:ok, body} <- Jason.decode(response.body) do
@@ -79,7 +79,4 @@ defmodule Pleroma.Akkoma.Translators.LibreTranslate do
 
     HTTP.get(to_string(url))
   end
-
-  @impl Pleroma.Akkoma.Translator.Provider
-  def name, do: "LibreTranslate"
 end

lib/pleroma/akkoma/translators/provider.ex

@@ -1,9 +0,0 @@
-defmodule Pleroma.Akkoma.Translator.Provider do
-  @callback translate(String.t(), String.t() | nil, String.t()) ::
-              {:ok, String.t(), String.t()} | {:error, any()}
-  @callback languages() ::
-              {:ok, [%{name: String.t(), code: String.t()}],
-               [%{name: String.t(), code: String.t()}]}
-              | {:error, any()}
-  @callback name() :: String.t()
-end

lib/pleroma/application.ex

@@ -68,13 +68,14 @@ defmodule Pleroma.Application do
         http_children() ++
         [
           Pleroma.Stats,
+          Pleroma.JobQueueMonitor,
           {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Config.get([:majic_pool, :size], 2)]},
           {Oban, Config.get(Oban)},
           Pleroma.Web.Endpoint,
           Pleroma.Web.Telemetry
         ] ++
         elasticsearch_children() ++
-        task_children() ++
+        task_children(@mix_env) ++
         dont_run_in_test(@mix_env)
 
     # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
@@ -144,90 +145,34 @@ defmodule Pleroma.Application do
 
   defp cachex_children do
     [
-      build_cachex(
-        "used_captcha",
-        expiration: expiration(interval: seconds_valid_interval())
-      ),
-      build_cachex(
-        "user",
-        expiration: expiration(default: 3_000, interval: 1_000),
-        hooks: [cachex_sched_limit(2500)]
-      ),
-      build_cachex(
-        "object",
-        expiration: expiration(default: 3_000, interval: 1_000),
-        hooks: [cachex_sched_limit(2500)]
-      ),
-      build_cachex(
-        "rich_media",
-        expiration: expiration(default: :timer.hours(2)),
-        hooks: [cachex_sched_limit(5000)]
-      ),
-      build_cachex(
-        "scrubber",
-        hooks: [cachex_sched_limit(2500)]
-      ),
-      build_cachex(
-        "scrubber_management",
-        hooks: [cachex_sched_limit(2500)]
-      ),
-      build_cachex(
-        "idempotency",
-        expiration: expiration(default: :timer.hours(6), interval: :timer.minutes(1)),
-        hooks: [cachex_sched_limit(2500, [], frequency: :timer.minutes(1))]
-      ),
-      build_cachex(
-        "web_resp",
-        hooks: [cachex_sched_limit(2500)]
-      ),
-      build_cachex(
-        "emoji_packs",
-        expiration: expiration(default: :timer.minutes(5), interval: :timer.minutes(1)),
-        hooks: [cachex_sched_limit(10)]
-      ),
-      build_cachex(
-        "failed_proxy_url",
-        hooks: [cachex_sched_limit(2500)]
-      ),
-      build_cachex(
-        "banned_urls",
-        expiration: expiration(default: :timer.hours(24 * 30)),
-        hooks: [cachex_sched_limit(5_000, [], frequency: :timer.minutes(5))]
-      ),
-      build_cachex(
-        "translations",
-        expiration: expiration(default: :timer.hours(24 * 30)),
-        hooks: [cachex_sched_limit(2500)]
-      ),
-      build_cachex(
-        "instances",
-        expiration: expiration(default: :timer.hours(24), interval: 1000),
-        hooks: [cachex_sched_limit(2500)]
-      ),
-      build_cachex(
-        "rel_me",
-        expiration: expiration(default: :timer.hours(24 * 30)),
-        hooks: [cachex_sched_limit(300, [], frequency: :timer.minutes(1))]
-      ),
-      build_cachex(
-        "host_meta",
-        expiration: expiration(default: :timer.minutes(120)),
-        hooks: [cachex_sched_limit(5000, [], frequency: :timer.minutes(1))]
-      ),
-      build_cachex(
-        "http_backoff",
-        expiration: expiration(default: :timer.hours(24 * 30)),
-        hooks: [cachex_sched_limit(10_000, [], frequency: :timer.minutes(5))]
-      )
+      build_cachex("used_captcha", ttl_interval: seconds_valid_interval()),
+      build_cachex("user", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
+      build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500),
+      build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000),
+      build_cachex("scrubber", limit: 2500),
+      build_cachex("scrubber_management", limit: 2500),
+      build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500),
+      build_cachex("web_resp", limit: 2500),
+      build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
+      build_cachex("failed_proxy_url", limit: 2500),
+      build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000),
+      build_cachex("translations", default_ttl: :timer.hours(24 * 30), limit: 2500),
+      build_cachex("instances", default_ttl: :timer.hours(24), ttl_interval: 1000, limit: 2500),
+      build_cachex("rel_me", default_ttl: :timer.hours(24 * 30), limit: 300),
+      build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000),
+      build_cachex("http_backoff", default_ttl: :timer.hours(24 * 30), limit: 10000)
     ]
   end
 
+  defp emoji_packs_expiration,
+    do: expiration(default: :timer.seconds(5 * 60), interval: :timer.seconds(60))
+
+  defp idempotency_expiration,
+    do: expiration(default: :timer.seconds(6 * 60 * 60), interval: :timer.seconds(60))
+
   defp seconds_valid_interval,
     do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid]))
 
-  defp cachex_sched_limit(limit, prune_opts \\ [], sched_opts \\ []),
-    do: hook(module: Cachex.Limit.Scheduled, args: {limit, prune_opts, sched_opts})
-
   @spec build_cachex(String.t(), keyword()) :: map()
   def build_cachex(type, opts),
     do: %{
@@ -255,29 +200,31 @@ defmodule Pleroma.Application do
     ]
   end
 
-  @spec task_children() :: [map()]
-  defp task_children() do
-    always =
-      [
-        %{
-          id: :web_push_init,
-          start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
-          restart: :temporary
-        }
-      ]
+  @spec task_children(atom()) :: [map()]
 
-    if @mix_env == :test do
-      always
-    else
-      [
-        %{
-          id: :internal_fetch_init,
-          start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]},
-          restart: :temporary
-        }
-        | always
-      ]
-    end
+  defp task_children(:test) do
+    [
+      %{
+        id: :web_push_init,
+        start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
+        restart: :temporary
+      }
+    ]
+  end
+
+  defp task_children(_) do
+    [
+      %{
+        id: :web_push_init,
+        start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
+        restart: :temporary
+      },
+      %{
+        id: :internal_fetch_init,
+        start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]},
+        restart: :temporary
+      }
+    ]
   end
 
   @spec elasticsearch_children :: [Pleroma.Search.Elasticsearch.Cluster]

lib/pleroma/application_requirements.ex

@@ -164,13 +164,13 @@ defmodule Pleroma.ApplicationRequirements do
 
   defp check_system_commands!(:ok) do
     filter_commands_statuses = [
-      check_filter(Pleroma.Upload.Filter.Exiftool.StripMetadata, ["exiftool"]),
-      check_filter(Pleroma.Upload.Filter.Exiftool.ReadDescription, ["exiftool"]),
-      check_filter(Pleroma.Upload.Filter.Mogrify, ["mogrify"]),
-      check_filter(Pleroma.Upload.Filter.Mogrifun, ["mogrify"]),
-      check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, ["mogrify"]),
-      check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, ["magick", "convert"]),
-      check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, ["ffprobe"])
+      check_filter(Pleroma.Upload.Filter.Exiftool.StripMetadata, "exiftool"),
+      check_filter(Pleroma.Upload.Filter.Exiftool.ReadDescription, "exiftool"),
+      check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"),
+      check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"),
+      check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "mogrify"),
+      check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "convert"),
+      check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "ffprobe")
     ]
 
     preview_proxy_commands_status =
@@ -219,13 +219,13 @@ defmodule Pleroma.ApplicationRequirements do
 
   defp check_repo_pool_size!(result), do: result
 
-  defp check_filter(filter, commands_required) do
+  defp check_filter(filter, command_required) do
     filters = Config.get([Pleroma.Upload, :filters])
 
-    if filter in filters and not Enum.any?(commands_required, &Pleroma.Utils.command_available?/1) do
+    if filter in filters and not Pleroma.Utils.command_available?(command_required) do
       Logger.error(
-        "#{filter} is specified in list of Pleroma.Upload filters, but none of the commands in " <>
-          "[#{Enum.join(commands_required, ", ")}] are found"
+        "#{filter} is specified in list of Pleroma.Upload filters, but the " <>
+          "#{command_required} command is not found"
       )
 
       false

lib/pleroma/bookmark.ex

@@ -53,15 +53,13 @@ defmodule Pleroma.Bookmark do
   end
 
   @spec destroy(FlakeId.Ecto.CompatType.t(), FlakeId.Ecto.CompatType.t()) ::
-          :ok | {:error, any()}
+          {:ok, Bookmark.t()} | {:error, Changeset.t()}
   def destroy(user_id, activity_id) do
-    {cnt, _} =
-      from(b in Bookmark,
-        where: b.user_id == ^user_id,
-        where: b.activity_id == ^activity_id
-      )
-      |> Repo.delete_all()
-
-    if cnt >= 1, do: :ok, else: {:error, :not_found}
+    from(b in Bookmark,
+      where: b.user_id == ^user_id,
+      where: b.activity_id == ^activity_id
+    )
+    |> Repo.one()
+    |> Repo.delete()
   end
 end

lib/pleroma/captcha.ex

@@ -97,7 +97,7 @@ defmodule Pleroma.Captcha do
 
   defp mark_captcha_as_used(token) do
     ttl = seconds_valid() |> :timer.seconds()
-    @cachex.put(:used_captcha_cache, token, true, expire: ttl)
+    @cachex.put(:used_captcha_cache, token, true, ttl: ttl)
   end
 
   defp method, do: Pleroma.Config.get!([__MODULE__, :method])

lib/pleroma/config/deprecation_warnings.ex

@@ -22,41 +22,6 @@ defmodule Pleroma.Config.DeprecationWarnings do
      "\n* `config :pleroma, :instance, :quarantined_instances` is now covered by `:pleroma, :mrf_simple, :reject`"}
   ]
 
-  def check_skip_thread_containment do
-    # The default in config/config.exs is "true" since 593b8b1e6a8502cca9bf5559b8bec86f172bbecb
-    # but when the default is retrieved in code the fallback is still "false"
-    uses_thread_visibility_filtering = Config.get([:instance, :skip_thread_containment]) != nil
-
-    if uses_thread_visibility_filtering do
-      Logger.warning("""
-      !!!CONFIG ERROR!!!
-      config :pleroma, :instance, skip_thread_containment: false
-      was removed after a deprecation window during which no complaints were raised.
-      You should drop this setting from your config.
-      """)
-
-      :error
-    else
-      :ok
-    end
-  end
-
-  def check_truncated_nodeinfo_in_accounts do
-    if !Config.get!([:instance, :filter_embedded_nodeinfo]) do
-      Logger.warning("""
-      !!!BUG WORKAROUND DETECTED!!!
-      Your config is explicitly disabling filtering of nodeinfo data embedded in other Masto API responses
-
-        config :pleroma, :instance, filter_embedded_nodeinfo: false
-
-      This setting will soon be removed. Any usage of it merely serves as a temporary workaround.
-      Make sure to file a bug telling us which problems you encountered and circumvented by setting this!
-         https://akkoma.dev/AkkomaGang/akkoma/issues
-      We can’t fix bugs we don’t know about.
-      """)
-    end
-  end
-
   def check_exiftool_filter do
     filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, [])
 
@@ -257,8 +222,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
       check_http_adapter(),
       check_uploader_base_url_set(),
       check_uploader_base_url_is_not_base_domain(),
-      check_exiftool_filter(),
-      check_skip_thread_containment()
+      check_exiftool_filter()
     ]
     |> Enum.reduce(:ok, fn
       :ok, :ok -> :ok

lib/pleroma/config_db.ex

@@ -7,9 +7,7 @@ defmodule Pleroma.ConfigDB do
 
   import Ecto.Changeset
   import Ecto.Query, only: [select: 3, from: 2]
-
-  use Gettext,
-    backend: Pleroma.Web.Gettext
+  import Pleroma.Web.Gettext
 
   alias __MODULE__
   alias Pleroma.Repo

lib/pleroma/constants.ex

@@ -19,8 +19,7 @@ defmodule Pleroma.Constants do
       "context_id",
       "deleted_activity_id",
       "pleroma_internal",
-      "generator",
-      "voters"
+      "generator"
     ]
   )
 
@@ -29,10 +28,6 @@ defmodule Pleroma.Constants do
       ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance embed sw.js sw-pleroma.js favicon.png schemas doc)
   )
 
-  # XXX: should we start allowing addressing/visibility to change via updates,
-  # we must also make sure to sync this new data to the Create activity and its recipients field
-  # as well as adding more safeguards to inlined statuses in Notifications, since access to
-  # a previously accessible and notified-about post may be lost.
   const(status_updatable_fields,
     do: [
       "source",

lib/pleroma/conversation.ex

@@ -15,6 +15,7 @@ defmodule Pleroma.Conversation do
     # This is the context ap id.
     field(:ap_id, :string)
     has_many(:participations, Participation)
+    has_many(:users, through: [:participations, :user])
 
     timestamps()
   end
@@ -44,11 +45,7 @@ defmodule Pleroma.Conversation do
     participation = Repo.preload(participation, :recipients)
 
     if Enum.empty?(participation.recipients) do
-      recipients =
-        [activity.actor | activity.recipients]
-        |> Enum.uniq()
-        |> User.get_all_by_ap_id()
-
+      recipients = User.get_all_by_ap_id(activity.recipients)
       RecipientShip.create(recipients, participation)
     end
   end
@@ -67,16 +64,15 @@ defmodule Pleroma.Conversation do
          ap_id when is_binary(ap_id) and byte_size(ap_id) > 0 <- object.data["context"],
          {:ok, conversation} <- create_for_ap_id(ap_id) do
       users = User.get_users_from_set(activity.recipients, local_only: false)
-      local_users = Enum.filter(users, & &1.local)
 
       participations =
-        Enum.map(local_users, fn user ->
+        Enum.map(users, fn user ->
           invisible_conversation = Enum.any?(users, &User.blocks?(user, &1))
 
           opts = Keyword.put(opts, :invisible_conversation, invisible_conversation)
 
           {:ok, participation} =
-            Participation.create_or_bump(user, conversation, activity.id, opts)
+            Participation.create_for_user_and_conversation(user, conversation, opts)
 
           maybe_create_recipientships(participation, activity)
           participation
@@ -91,4 +87,21 @@ defmodule Pleroma.Conversation do
       e -> {:error, e}
     end
   end
+
+  @doc """
+  This is only meant to be run by a mix task. It creates conversations/participations for all direct messages in the database.
+  """
+  def bump_for_all_activities do
+    stream =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_direct_messages_query()
+      |> Repo.stream()
+
+    Repo.transaction(
+      fn ->
+        stream
+        |> Enum.each(fn a -> create_or_bump_for(a, read: true) end)
+      end,
+      timeout: :infinity
+    )
+  end
 end

lib/pleroma/conversation/participation.ex

@@ -12,12 +12,9 @@ defmodule Pleroma.Conversation.Participation do
   import Ecto.Changeset
   import Ecto.Query
 
-  @type t() :: %__MODULE__{}
-
   schema "conversation_participations" do
     belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
     belongs_to(:conversation, Conversation)
-    field(:last_bump, FlakeId.Ecto.CompatType)
     field(:read, :boolean, default: false)
     field(:last_activity_id, FlakeId.Ecto.CompatType, virtual: true)
 
@@ -27,26 +24,24 @@ defmodule Pleroma.Conversation.Participation do
     timestamps()
   end
 
-  defp creation_cng(struct, params) do
+  def creation_cng(struct, params) do
     struct
-    |> cast(params, [:user_id, :conversation_id, :last_bump, :read])
-    |> validate_required([:user_id, :conversation_id, :last_bump])
+    |> cast(params, [:user_id, :conversation_id, :read])
+    |> validate_required([:user_id, :conversation_id])
   end
 
-  def create_or_bump(user, conversation, status_id, opts \\ []) do
+  def create_for_user_and_conversation(user, conversation, opts \\ []) do
     read = !!opts[:read]
     invisible_conversation = !!opts[:invisible_conversation]
 
     update_on_conflict =
       if(invisible_conversation, do: [], else: [read: read])
       |> Keyword.put(:updated_at, NaiveDateTime.utc_now())
-      |> Keyword.put(:last_bump, status_id)
 
     %__MODULE__{}
     |> creation_cng(%{
       user_id: user.id,
       conversation_id: conversation.id,
-      last_bump: status_id,
       read: invisible_conversation || read
     })
     |> Repo.insert(
@@ -56,7 +51,7 @@ defmodule Pleroma.Conversation.Participation do
     )
   end
 
-  defp read_cng(struct, params) do
+  def read_cng(struct, params) do
     struct
     |> cast(params, [:read])
     |> validate_required([:read])
@@ -104,87 +99,43 @@ defmodule Pleroma.Conversation.Participation do
     {:ok, user, participations}
   end
 
-  # used for tests
   def mark_as_unread(participation) do
     participation
     |> read_cng(%{read: false})
     |> Repo.update()
   end
 
-  def for_user_with_pagination(user, params \\ %{}) do
+  def for_user(user, params \\ %{}) do
     from(p in __MODULE__,
       where: p.user_id == ^user.id,
-      preload: [:conversation]
+      order_by: [desc: p.updated_at],
+      preload: [conversation: [:users]]
     )
     |> restrict_recipients(user, params)
-    |> select([p], %{id: p.last_bump, entry: p})
-    |> Pleroma.Pagination.fetch_paginated(Map.put(params, :pagination_field, :last_bump))
+    |> Pleroma.Pagination.fetch_paginated(params)
   end
 
-  def preload_last_activity_id_and_filter(participations) when is_list(participations) do
-    participations
-    |> Enum.map(fn p -> load_last_activity_id(p) end)
-    |> Enum.filter(fn p -> p.last_activity_id end)
-  end
-
-  defp load_last_activity_id(%__MODULE__{} = participation) do
-    %{
-      participation
-      | last_activity_id: last_activity_id(participation)
-    }
-  end
-
-  @spec last_activity_id(t(), User.t() | nil) :: Flake.t()
-  def last_activity_id(participation, user \\ nil)
-
-  def last_activity_id(
-        %__MODULE__{conversation: %Conversation{}} = participation,
-        user
-      ) do
-    user =
-      if user && user.id == participation.user_id do
-        user
-      else
-        case participation.user do
-          %User{} -> participation.user
-          _ -> User.get_cached_by_id(participation.user_id)
-        end
-      end
-
-    ActivityPub.fetch_latest_direct_activity_id_for_context(
-      participation.conversation.ap_id,
-      user
-    )
-  end
-
-  def last_activity_id(%__MODULE__{} = participation, user) do
-    case Repo.preload(participation, :conversation) do
-      %{conversation: %Conversation{}} = p -> last_activity_id(p, user)
-      _ -> nil
-    end
-  end
-
-  defp restrict_recipients(query, user, %{recipients: user_ids}) do
+  def restrict_recipients(query, user, %{recipients: user_ids}) do
     user_binary_ids =
       [user.id | user_ids]
       |> Enum.uniq()
       |> User.binary_id()
 
-    recipient_subquery =
-      RecipientShip
-      |> group_by([r], r.participation_id)
+    conversation_subquery =
+      __MODULE__
+      |> group_by([p], p.conversation_id)
       |> having(
-        [r],
-        count(r.user_id) == ^length(user_binary_ids) and
-          fragment("array_agg(?) @> ?", r.user_id, ^user_binary_ids)
+        [p],
+        count(p.user_id) == ^length(user_binary_ids) and
+          fragment("array_agg(?) @> ?", p.user_id, ^user_binary_ids)
       )
-      |> select([r], %{pid: r.participation_id})
+      |> select([p], %{id: p.conversation_id})
 
     query
-    |> join(:inner, [p], r in subquery(recipient_subquery), on: p.id == r.pid)
+    |> join(:inner, [p], c in subquery(conversation_subquery), on: p.conversation_id == c.id)
   end
 
-  defp restrict_recipients(query, _, _), do: query
+  def restrict_recipients(query, _, _), do: query
 
   def for_user_and_conversation(user, conversation) do
     from(p in __MODULE__,
@@ -194,6 +145,26 @@ defmodule Pleroma.Conversation.Participation do
     |> Repo.one()
   end
 
+  def for_user_with_last_activity_id(user, params \\ %{}) do
+    for_user(user, params)
+    |> Enum.map(fn participation ->
+      activity_id =
+        ActivityPub.fetch_latest_direct_activity_id_for_context(
+          participation.conversation.ap_id,
+          %{
+            user: user,
+            blocking_user: user
+          }
+        )
+
+      %{
+        participation
+        | last_activity_id: activity_id
+      }
+    end)
+    |> Enum.reject(&is_nil(&1.last_activity_id))
+  end
+
   def get(_, _ \\ [])
   def get(nil, _), do: nil
 
@@ -242,6 +213,14 @@ defmodule Pleroma.Conversation.Participation do
     |> Repo.aggregate(:count, :id)
   end
 
+  def unread_conversation_count_for_user(user) do
+    from(p in __MODULE__,
+      where: p.user_id == ^user.id,
+      where: not p.read,
+      select: %{count: count(p.id)}
+    )
+  end
+
   def delete(%__MODULE__{} = participation) do
     Repo.delete(participation)
   end

lib/pleroma/counter_cache.ex

@@ -0,0 +1,79 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.CounterCache do
+  alias Pleroma.CounterCache
+  alias Pleroma.Repo
+  use Ecto.Schema
+  import Ecto.Changeset
+  import Ecto.Query
+
+  schema "counter_cache" do
+    field(:instance, :string)
+    field(:public, :integer)
+    field(:unlisted, :integer)
+    field(:private, :integer)
+    field(:direct, :integer)
+  end
+
+  def changeset(struct, params) do
+    struct
+    |> cast(params, [:instance, :public, :unlisted, :private, :direct])
+    |> validate_required([:instance])
+    |> unique_constraint(:instance)
+  end
+
+  def get_by_instance(instance) do
+    CounterCache
+    |> select([c], %{
+      "public" => c.public,
+      "unlisted" => c.unlisted,
+      "private" => c.private,
+      "direct" => c.direct
+    })
+    |> where([c], c.instance == ^instance)
+    |> Repo.one()
+    |> case do
+      nil -> %{"public" => 0, "unlisted" => 0, "private" => 0, "direct" => 0}
+      val -> val
+    end
+  end
+
+  def get_sum do
+    CounterCache
+    |> select([c], %{
+      "public" => type(sum(c.public), :integer),
+      "unlisted" => type(sum(c.unlisted), :integer),
+      "private" => type(sum(c.private), :integer),
+      "direct" => type(sum(c.direct), :integer)
+    })
+    |> Repo.one()
+  end
+
+  def set(instance, values) do
+    params =
+      Enum.reduce(
+        ["public", "private", "unlisted", "direct"],
+        %{"instance" => instance},
+        fn param, acc ->
+          Map.put_new(acc, param, Map.get(values, param, 0))
+        end
+      )
+
+    %CounterCache{}
+    |> changeset(params)
+    |> Repo.insert(
+      on_conflict: [
+        set: [
+          public: params["public"],
+          private: params["private"],
+          unlisted: params["unlisted"],
+          direct: params["direct"]
+        ]
+      ],
+      returning: true,
+      conflict_target: :instance
+    )
+  end
+end

lib/pleroma/docs/translator.ex

@@ -4,7 +4,7 @@
 
 defmodule Pleroma.Docs.Translator do
   require Pleroma.Docs.Translator.Compiler
-  use Gettext, backend: Pleroma.Web.Gettext
+  require Pleroma.Web.Gettext
 
   @before_compile Pleroma.Docs.Translator.Compiler
 end

lib/pleroma/docs/translator/compiler.ex

@@ -7,8 +7,6 @@ defmodule Pleroma.Docs.Translator.Compiler do
   @raw_config Pleroma.Config.Loader.read("config/description.exs")
   @raw_descriptions @raw_config[:pleroma][:config_description]
 
-  require Gettext.Macros
-
   defmacro __before_compile__(_env) do
     strings =
       __MODULE__.descriptions()
@@ -23,8 +21,7 @@ defmodule Pleroma.Docs.Translator.Compiler do
               ctxt = msgctxt_for(path, type)
 
               quote do
-                Gettext.Macros.dpgettext_noop_with_backend(
-                  Pleroma.Web.Gettext,
+                Pleroma.Web.Gettext.dpgettext_noop(
                   "config_descriptions",
                   unquote(ctxt),
                   unquote(string)

lib/pleroma/emails/user_email.ex

@@ -5,13 +5,12 @@
 defmodule Pleroma.Emails.UserEmail do
   @moduledoc "User emails"
 
-  require Pleroma.Web.GettextCompanion
-  use Gettext, backend: Pleroma.Web.Gettext
+  require Pleroma.Web.Gettext
   use Pleroma.Web, :mailer
 
   alias Pleroma.Config
   alias Pleroma.User
-  alias Pleroma.Web.GettextCompanion
+  alias Pleroma.Web.Gettext
 
   import Swoosh.Email
   import Phoenix.Swoosh, except: [render_body: 3]
@@ -30,7 +29,7 @@ defmodule Pleroma.Emails.UserEmail do
 
   @spec welcome(User.t(), map()) :: Swoosh.Email.t()
   def welcome(user, opts \\ %{}) do
-    GettextCompanion.with_locale_or_default user.language do
+    Gettext.with_locale_or_default user.language do
       new()
       |> to(recipient(user))
       |> from(Map.get(opts, :sender, sender()))
@@ -38,7 +37,7 @@ defmodule Pleroma.Emails.UserEmail do
         Map.get(
           opts,
           :subject,
-          dpgettext(
+          Gettext.dpgettext(
             "static_pages",
             "welcome email subject",
             "Welcome to %{instance_name}!",
@@ -50,7 +49,7 @@ defmodule Pleroma.Emails.UserEmail do
         Map.get(
           opts,
           :html,
-          dpgettext(
+          Gettext.dpgettext(
             "static_pages",
             "welcome email html body",
             "Welcome to %{instance_name}!",
@@ -62,7 +61,7 @@ defmodule Pleroma.Emails.UserEmail do
         Map.get(
           opts,
           :text,
-          dpgettext(
+          Gettext.dpgettext(
             "static_pages",
             "welcome email text body",
             "Welcome to %{instance_name}!",
@@ -74,11 +73,11 @@ defmodule Pleroma.Emails.UserEmail do
   end
 
   def password_reset_email(user, token) when is_binary(token) do
-    GettextCompanion.with_locale_or_default user.language do
+    Gettext.with_locale_or_default user.language do
       password_reset_url = url(~p[/api/v1/pleroma/password_reset/#{token}])
 
       html_body =
-        dpgettext(
+        Gettext.dpgettext(
           "static_pages",
           "password reset email body",
           """
@@ -94,7 +93,9 @@ defmodule Pleroma.Emails.UserEmail do
       new()
       |> to(recipient(user))
       |> from(sender())
-      |> subject(dpgettext("static_pages", "password reset email subject", "Password reset"))
+      |> subject(
+        Gettext.dpgettext("static_pages", "password reset email subject", "Password reset")
+      )
       |> html_body(html_body)
     end
   end
@@ -105,11 +106,11 @@ defmodule Pleroma.Emails.UserEmail do
         to_email,
         to_name \\ nil
       ) do
-    GettextCompanion.with_locale_or_default user.language do
+    Gettext.with_locale_or_default user.language do
       registration_url = url(~p[/registration/#{user_invite_token.token}])
 
       html_body =
-        dpgettext(
+        Gettext.dpgettext(
           "static_pages",
           "user invitation email body",
           """
@@ -126,7 +127,7 @@ defmodule Pleroma.Emails.UserEmail do
       |> to(recipient(to_email, to_name))
       |> from(sender())
       |> subject(
-        dpgettext(
+        Gettext.dpgettext(
           "static_pages",
           "user invitation email subject",
           "Invitation to %{instance_name}",
@@ -138,11 +139,11 @@ defmodule Pleroma.Emails.UserEmail do
   end
 
   def account_confirmation_email(user) do
-    GettextCompanion.with_locale_or_default user.language do
+    Gettext.with_locale_or_default user.language do
       confirmation_url = url(~p[/api/account/confirm_email/#{user.id}/#{user.confirmation_token}])
 
       html_body =
-        dpgettext(
+        Gettext.dpgettext(
           "static_pages",
           "confirmation email body",
           """
@@ -158,7 +159,7 @@ defmodule Pleroma.Emails.UserEmail do
       |> to(recipient(user))
       |> from(sender())
       |> subject(
-        dpgettext(
+        Gettext.dpgettext(
           "static_pages",
           "confirmation email subject",
           "%{instance_name} account confirmation",
@@ -170,9 +171,9 @@ defmodule Pleroma.Emails.UserEmail do
   end
 
   def approval_pending_email(user) do
-    GettextCompanion.with_locale_or_default user.language do
+    Gettext.with_locale_or_default user.language do
       html_body =
-        dpgettext(
+        Gettext.dpgettext(
           "static_pages",
           "approval pending email body",
           """
@@ -186,7 +187,7 @@ defmodule Pleroma.Emails.UserEmail do
       |> to(recipient(user))
       |> from(sender())
       |> subject(
-        dpgettext(
+        Gettext.dpgettext(
           "static_pages",
           "approval pending email subject",
           "Your account is awaiting approval"
@@ -197,9 +198,9 @@ defmodule Pleroma.Emails.UserEmail do
   end
 
   def successful_registration_email(user) do
-    GettextCompanion.with_locale_or_default user.language do
+    Gettext.with_locale_or_default user.language do
       html_body =
-        dpgettext(
+        Gettext.dpgettext(
           "static_pages",
           "successful registration email body",
           """
@@ -215,7 +216,7 @@ defmodule Pleroma.Emails.UserEmail do
       |> to(recipient(user))
       |> from(sender())
       |> subject(
-        dpgettext(
+        Gettext.dpgettext(
           "static_pages",
           "successful registration email subject",
           "Account registered on %{instance_name}",
@@ -233,7 +234,7 @@ defmodule Pleroma.Emails.UserEmail do
   """
   @spec digest_email(User.t()) :: Swoosh.Email.t() | nil
   def digest_email(user) do
-    GettextCompanion.with_locale_or_default user.language do
+    Gettext.with_locale_or_default user.language do
       notifications = Pleroma.Notification.for_user_since(user, user.last_digest_emailed_at)
 
       mentions =
@@ -294,7 +295,7 @@ defmodule Pleroma.Emails.UserEmail do
         |> to(recipient(user))
         |> from(sender())
         |> subject(
-          dpgettext(
+          Gettext.dpgettext(
             "static_pages",
             "digest email subject",
             "Your digest from %{instance_name}",
@@ -335,12 +336,12 @@ defmodule Pleroma.Emails.UserEmail do
   def backup_is_ready_email(backup, admin_user_id \\ nil) do
     %{user: user} = Pleroma.Repo.preload(backup, :user)
 
-    GettextCompanion.with_locale_or_default user.language do
+    Gettext.with_locale_or_default user.language do
       download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup)
 
       html_body =
         if is_nil(admin_user_id) do
-          dpgettext(
+          Gettext.dpgettext(
             "static_pages",
             "account archive email body - self-requested",
             """
@@ -352,7 +353,7 @@ defmodule Pleroma.Emails.UserEmail do
         else
           admin = Pleroma.Repo.get(User, admin_user_id)
 
-          dpgettext(
+          Gettext.dpgettext(
             "static_pages",
             "account archive email body - admin requested",
             """
@@ -368,7 +369,7 @@ defmodule Pleroma.Emails.UserEmail do
       |> to(recipient(user))
       |> from(sender())
       |> subject(
-        dpgettext(
+        Gettext.dpgettext(
           "static_pages",
           "account archive email subject",
           "Your account archive is ready"

lib/pleroma/emoji.ex

@@ -16,7 +16,7 @@ defmodule Pleroma.Emoji do
 
   @ets __MODULE__.Ets
   @ets_options [
-    :set,
+    :ordered_set,
     :protected,
     :named_table,
     {:read_concurrency, true}
@@ -25,8 +25,6 @@ defmodule Pleroma.Emoji do
 
   defstruct [:code, :file, :tags, :safe_code, :safe_file]
 
-  @type t :: %__MODULE__{}
-
   @doc "Build emoji struct"
   def build({code, file, tags}) do
     %__MODULE__{
@@ -45,14 +43,14 @@ defmodule Pleroma.Emoji do
     GenServer.start_link(__MODULE__, [], name: __MODULE__)
   end
 
-  @doc "Reloads the emojis from disk (asynchronous)"
+  @doc "Reloads the emojis from disk."
   @spec reload() :: :ok
   def reload do
-    GenServer.cast(__MODULE__, :reload)
+    GenServer.call(__MODULE__, :reload)
   end
 
-  @doc "Returns the emoji struct of the given `name` if it exists."
-  @spec get(String.t()) :: t() | nil
+  @doc "Returns the path of the emoji `name`."
+  @spec get(String.t()) :: String.t() | nil
   def get(name) do
     name =
       if String.starts_with?(name, ":") do
@@ -64,23 +62,11 @@ defmodule Pleroma.Emoji do
       end
 
     case :ets.lookup(@ets, name) do
-      [{_, emoji}] -> emoji
+      [{_, path}] -> path
       _ -> nil
     end
   end
 
-  @doc "Updates or inserts new emoji (asynchronous)"
-  @spec add_or_update(t()) :: :ok
-  def add_or_update(%__MODULE__{} = emoji) do
-    GenServer.cast(__MODULE__, {:add, emoji})
-  end
-
-  @doc "Delete emoji with given shortcode if it exists (asynchronous)"
-  @spec delete(String.t()) :: :ok
-  def delete(code) do
-    GenServer.cast(__MODULE__, {:delete, code})
-  end
-
   @spec exist?(String.t()) :: boolean()
   def exist?(name), do: not is_nil(get(name))
 
@@ -103,14 +89,10 @@ defmodule Pleroma.Emoji do
     {:noreply, state}
   end
 
-  def handle_cast({:add, %__MODULE__{} = emoji}, state) do
-    :ets.insert(@ets, {emoji.code, emoji})
-    {:noreply, state}
-  end
-
-  def handle_cast({:delete, code}, state) do
-    :ets.delete(@ets, code)
-    {:noreply, state}
+  @doc false
+  def handle_call(:reload, _from, state) do
+    update_emojis(Loader.load())
+    {:reply, :ok, state}
   end
 
   @doc false

lib/pleroma/emoji/pack.ex

@@ -49,15 +49,12 @@ defmodule Pleroma.Emoji.Pack do
     Path.join(dir, safe_path)
   end
 
-  defp tags(%__MODULE__{} = pack), do: ["pack:" <> pack.name]
-
   @spec create(String.t()) :: {:ok, t()} | {:error, File.posix()} | {:error, :empty_values}
   def create(name) do
     with :ok <- validate_not_empty([name]),
          dir <- path_join_name_safe(emoji_path(), name),
          :ok <- File.mkdir(dir) do
       save_pack(%__MODULE__{
-        name: name,
         path: dir,
         pack_file: Path.join(dir, "pack.json")
       })
@@ -93,13 +90,9 @@ defmodule Pleroma.Emoji.Pack do
   @spec delete(String.t()) ::
           {:ok, [binary()]} | {:error, File.posix(), binary()} | {:error, :empty_values}
   def delete(name) do
-    with {_, :ok} <- {:empty, validate_not_empty([name])},
-         {:ok, pack} <- load_pack(name) do
-      Enum.each(pack.files, fn {shortcode, _} -> Emoji.delete(shortcode) end)
-      File.rm_rf(pack.path)
-    else
-      {:empty, error} -> error
-      _ -> {:ok, []}
+    with :ok <- validate_not_empty([name]),
+         pack_path <- path_join_name_safe(emoji_path(), name) do
+      File.rm_rf(pack_path)
     end
   end
 
@@ -149,6 +142,8 @@ defmodule Pleroma.Emoji.Pack do
             {item, updated_pack}
           end)
 
+        Emoji.reload()
+
         {:ok, updated_pack}
       after
         File.rm_rf(tmp_dir)
@@ -174,14 +169,16 @@ defmodule Pleroma.Emoji.Pack do
     with :ok <- validate_not_empty([shortcode, filename]),
          :ok <- validate_emoji_not_exists(shortcode),
          {:ok, updated_pack} <- do_add_file(pack, shortcode, filename, file) do
+      Emoji.reload()
       {:ok, updated_pack}
     end
   end
 
   defp do_add_file(pack, shortcode, filename, file) do
-    with :ok <- save_file(file, pack, filename),
-         {:ok, pack} <- put_emoji(pack, shortcode, filename) do
-      {:ok, pack}
+    with :ok <- save_file(file, pack, filename) do
+      pack
+      |> put_emoji(shortcode, filename)
+      |> save_pack()
     end
   end
 
@@ -191,7 +188,7 @@ defmodule Pleroma.Emoji.Pack do
     with :ok <- validate_not_empty([shortcode]),
          :ok <- remove_file(pack, shortcode),
          {:ok, updated_pack} <- pack |> delete_emoji(shortcode) |> save_pack() do
-      Emoji.delete(shortcode)
+      Emoji.reload()
       {:ok, updated_pack}
     end
   end
@@ -206,8 +203,9 @@ defmodule Pleroma.Emoji.Pack do
          {:ok, updated_pack} <-
            pack
            |> delete_emoji(shortcode)
-           |> put_emoji(new_shortcode, new_filename) do
-      if shortcode != new_shortcode, do: Emoji.delete(shortcode)
+           |> put_emoji(new_shortcode, new_filename)
+           |> save_pack() do
+      Emoji.reload()
       {:ok, updated_pack}
     end
   end
@@ -457,7 +455,7 @@ defmodule Pleroma.Emoji.Pack do
       # if pack.json MD5 changes, the cache is not valid anymore
       %{hash: hash, pack_data: result},
       # Add a minute to cache time for every file in the pack
-      expire: overall_ttl
+      ttl: overall_ttl
     )
 
     result
@@ -521,17 +519,7 @@ defmodule Pleroma.Emoji.Pack do
 
   defp put_emoji(pack, shortcode, filename) do
     files = Map.put(pack.files, shortcode, filename)
-    pack = %{pack | files: files, files_count: length(Map.keys(files))}
-
-    url_path = path_join_name_safe("/emoji/", pack.name) |> path_join_safe(filename)
-
-    with {:ok, pack} <- save_pack(pack) do
-      {shortcode, url_path, tags(pack)}
-      |> Emoji.build()
-      |> Emoji.add_or_update()
-
-      {:ok, pack}
-    end
+    %{pack | files: files, files_count: length(Map.keys(files))}
   end
 
   defp delete_emoji(pack, shortcode) do
@@ -592,7 +580,7 @@ defmodule Pleroma.Emoji.Pack do
   defp http_get(%URI{} = url), do: url |> to_string() |> http_get()
 
   defp http_get(url) do
-    with {:ok, %{body: body}} <- Pleroma.HTTP.get(url) do
+    with {:ok, %{body: body}} <- Pleroma.HTTP.get(url, [], []) do
       Jason.decode(body)
     end
   end

lib/pleroma/filter.ex

@@ -193,12 +193,6 @@ defmodule Pleroma.Filter do
     end
   end
 
-  defp escape_for_regex(plain_phrase) do
-    # Escape all active characters:
-    #   .^$*+?()[{\|
-    Regex.replace(~r/\.\^\$\*\+\?\(\)\[\{\\\|/, plain_phrase, fn m -> "\\" <> m end)
-  end
-
   @spec compose_regex(User.t() | [t()], format()) :: String.t() | Regex.t() | nil
   def compose_regex(user_or_filters, format \\ :postgres)
 
@@ -213,7 +207,7 @@ defmodule Pleroma.Filter do
   def compose_regex([_ | _] = filters, format) do
     phrases =
       filters
-      |> Enum.map(&escape_for_regex(&1.phrase))
+      |> Enum.map(& &1.phrase)
       |> Enum.join("|")
 
     case format do

lib/pleroma/following_relationship.ex

@@ -161,11 +161,7 @@ defmodule Pleroma.FollowingRelationship do
     |> where([r], r.state == ^:follow_pending)
     |> where([r], r.following_id == ^id)
     |> where([r, follower: f], f.is_active == true)
-  end
-
-  def get_follow_requesting_users_with_request_id(%User{} = user) do
-    get_follow_requests_query(user)
-    |> select([r, follower: f], %{id: r.id, entry: f})
+    |> select([r, follower: f], f)
   end
 
   def following?(%User{id: follower_id}, %User{id: followed_id}) do

lib/pleroma/formatter.ex

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Formatter do
-  alias PhoenixHTMLHelpers.Tag
   alias Pleroma.HTML
   alias Pleroma.User
 
@@ -38,10 +37,10 @@ defmodule Pleroma.Formatter do
     nickname_text = get_nickname_text(nickname, opts)
 
     :span
-    |> Tag.content_tag(
-      Tag.content_tag(
+    |> Phoenix.HTML.Tag.content_tag(
+      Phoenix.HTML.Tag.content_tag(
         :a,
-        ["@", Tag.content_tag(:span, nickname_text)],
+        ["@", Phoenix.HTML.Tag.content_tag(:span, nickname_text)],
         "data-user": id,
         class: "u-url mention",
         href: user_url,
@@ -69,7 +68,7 @@ defmodule Pleroma.Formatter do
     url = "#{Pleroma.Web.Endpoint.url()}/tag/#{tag}"
 
     link =
-      Tag.content_tag(:a, tag_text,
+      Phoenix.HTML.Tag.content_tag(:a, tag_text,
         class: "hashtag",
         "data-tag": tag,
         href: url,

lib/pleroma/frontend.ex

@@ -14,8 +14,6 @@ defmodule Pleroma.Frontend do
       "build_dir" => opts[:build_dir]
     }
 
-    explicit_source = !!(opts[:file] || opts[:build_dir] || opts[:build_url])
-
     frontend_info =
       [:frontends, :available, name]
       |> Config.get(%{})
@@ -30,25 +28,6 @@ defmodule Pleroma.Frontend do
       raise "No ref given or configured"
     end
 
-    if Map.get(frontend_info, "blind_trust", false) !== true do
-      bugtracker = frontend_info["bugtracker"]
-
-      unless bugtracker || explicit_source do
-        raise "Configured third-party frontend without a bugtracker; refusing install."
-      end
-
-      bugtracker = bugtracker || "the external frontend developers"
-
-      Logger.warning("""
-        !!!!!!!!
-        You are installing a third-party frontend not vetted by the Akkoma team.
-        THERE ARE NO GUARANTTES ABOUT SAFETY AND FUNCTIONALITY!
-        Do NOT report problems to Akkoma, instead
-        all bugs must be reported to #{bugtracker}
-        !!!!!!!!
-      """)
-    end
-
     dest = Path.join([dir(), name, ref])
 
     label = "#{name} (#{ref})"
@@ -90,7 +69,7 @@ defmodule Pleroma.Frontend do
     end
   end
 
-  defp unzip(zip, dest) do
+  def unzip(zip, dest) do
     File.rm_rf!(dest)
     File.mkdir_p!(dest)
 
@@ -105,7 +84,7 @@ defmodule Pleroma.Frontend do
     url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
 
     with {:ok, %{status: 200, body: zip_body}} <-
-           Pleroma.HTTP.get(url, [], adapter: [receive_timeout: 120_000]) do
+           Pleroma.HTTP.get(url, [], receive_timeout: 120_000) do
       unzip(zip_body, dest)
     else
       {:error, e} -> {:error, e}

lib/pleroma/healthcheck.ex

@@ -14,6 +14,7 @@ defmodule Pleroma.Healthcheck do
             active: 0,
             idle: 0,
             memory_used: 0,
+            job_queue_stats: nil,
             healthy: true
 
   @type t :: %__MODULE__{
@@ -21,6 +22,7 @@ defmodule Pleroma.Healthcheck do
           active: non_neg_integer(),
           idle: non_neg_integer(),
           memory_used: number(),
+          job_queue_stats: map(),
           healthy: boolean()
         }
 
@@ -30,6 +32,7 @@ defmodule Pleroma.Healthcheck do
       memory_used: Float.round(:recon_alloc.memory(:allocated) / 1024 / 1024, 2)
     }
     |> assign_db_info()
+    |> assign_job_queue_stats()
     |> check_health()
   end
 
@@ -55,6 +58,11 @@ defmodule Pleroma.Healthcheck do
     Map.merge(healthcheck, db_info)
   end
 
+  defp assign_job_queue_stats(healthcheck) do
+    stats = Pleroma.JobQueueMonitor.stats()
+    Map.put(healthcheck, :job_queue_stats, stats)
+  end
+
   @spec check_health(Healthcheck.t()) :: Healthcheck.t()
   def check_health(%{pool_size: pool_size, active: active} = check)
       when active >= pool_size do

lib/pleroma/helpers/media_helper.ex

@@ -12,10 +12,8 @@ defmodule Pleroma.Helpers.MediaHelper do
   require Logger
 
   def missing_dependencies do
-    Enum.reduce([imagemagick: ["magick", "convert"], ffmpeg: ["ffmpeg"]], [], fn {sym,
-                                                                                  executables},
-                                                                                 acc ->
-      if Enum.any?(executables, &Pleroma.Utils.command_available?/1) do
+    Enum.reduce([imagemagick: "convert", ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc ->
+      if Pleroma.Utils.command_available?(executable) do
         acc
       else
         [sym | acc]
@@ -24,10 +22,9 @@ defmodule Pleroma.Helpers.MediaHelper do
   end
 
   def image_resize(url, options) do
-    with executable when is_binary(executable) <-
-           Enum.find_value(["magick", "convert"], &System.find_executable/1),
+    with executable when is_binary(executable) <- System.find_executable("convert"),
          {:ok, args} <- prepare_image_resize_args(options),
-         {:ok, env} <- HTTP.get(url),
+         {:ok, env} <- HTTP.get(url, [], []),
          {:ok, fifo_path} <- mkfifo() do
       args = List.flatten([fifo_path, args])
       run_fifo(fifo_path, env, executable, args)
@@ -76,7 +73,7 @@ defmodule Pleroma.Helpers.MediaHelper do
   # Note: video thumbnail is intentionally not resized (always has original dimensions)
   def video_framegrab(url) do
     with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
-         {:ok, env} <- HTTP.get(url),
+         {:ok, env} <- HTTP.get(url, [], []),
          {:ok, fifo_path} <- mkfifo(),
          args = [
            "-y",

lib/pleroma/http.ex

@@ -7,6 +7,10 @@ defmodule Pleroma.HTTP do
     Wrapper for `Tesla.request/2`.
   """
 
+  alias Pleroma.HTTP.AdapterHelper
+  alias Pleroma.HTTP.Request
+  alias Pleroma.HTTP.RequestBuilder, as: Builder
+  alias Tesla.Client
   alias Tesla.Env
 
   require Logger
@@ -14,8 +18,6 @@ defmodule Pleroma.HTTP do
   @type t :: __MODULE__
   @type method() :: :get | :post | :put | :delete | :head
 
-  @mix_env Mix.env()
-
   @doc """
   Performs GET request.
 
@@ -57,70 +59,40 @@ defmodule Pleroma.HTTP do
   @spec request(method(), Request.url(), String.t(), Request.headers(), keyword()) ::
           {:ok, Env.t()} | {:error, any()}
   def request(method, url, body, headers, options) when is_binary(url) do
-    params = options[:params] || []
-    options = options |> Keyword.delete(:params)
-    headers = maybe_add_user_agent(headers)
+    uri = URI.parse(url)
+    adapter_opts = AdapterHelper.options(options || [])
 
-    client = build_client(method)
+    adapter_opts =
+      if uri.scheme == :https do
+        AdapterHelper.maybe_add_cacerts(adapter_opts, :public_key.cacerts_get())
+      else
+        adapter_opts
+      end
+
+    options = put_in(options[:adapter], adapter_opts)
+    params = options[:params] || []
+    request = build_request(method, headers, options, url, body, params)
+    client = Tesla.client([Tesla.Middleware.FollowRedirects, Tesla.Middleware.Telemetry])
 
     Logger.debug("Outbound: #{method} #{url}")
-
-    Tesla.request(client,
-      method: method,
-      url: url,
-      query: params,
-      headers: headers,
-      body: body,
-      opts: options
-    )
+    request(client, request)
   rescue
     e ->
       Logger.error("Failed to fetch #{url}: #{Exception.format(:error, e, __STACKTRACE__)}")
       {:error, :fetch_error}
   end
 
-  defp build_client(method) do
-    # Orders of middlewares matters!
-    # We start construction with the middlewares _last_ to run
-    # on outgoing requests (and first on incoming responses).
-    # This allows using more efficient list prepending.
-    middlewares = [Tesla.Middleware.Telemetry]
+  @spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()}
+  def request(client, request), do: Tesla.request(client, request)
 
-    # XXX: just like the user-agent header below, our current mocks can't handle extra headers
-    #      and would break if we used the decompression middleware during tests.
-    #      The :test condition can and should be removed once mocks are fixed.
-    #
-    # HEAD responses won't contain a body to compress anyway and we sometimes use
-    # HEAD requests to determine whether a remote resource is within size limits before fetching it.
-    # If the server would send a compressed response however, Content-Length will be the size of
-    # the _compressed_ response body skewing results.
-    middlewares =
-      if method != :head and @mix_env != :test do
-        [Tesla.Middleware.DecompressResponse | middlewares]
-      else
-        middlewares
-      end
-
-    middlewares = [
-      Tesla.Middleware.FollowRedirects,
-      Pleroma.HTTP.Middleware.HTTPSignature | middlewares
-    ]
-
-    Tesla.client(middlewares)
-  end
-
-  # XXX: our test mocks are (too) strict about headers and cannot handle user-agent atm
-  if @mix_env == :test do
-    defp maybe_add_user_agent(headers) do
-      with true <- Pleroma.Config.get([:http, :send_user_agent]) do
-        [{"user-agent", Pleroma.Application.user_agent()} | headers]
-      else
-        _ ->
-          headers
-      end
-    end
-  else
-    defp maybe_add_user_agent(headers),
-      do: [{"user-agent", Pleroma.Application.user_agent()} | headers]
+  defp build_request(method, headers, options, url, body, params) do
+    Builder.new()
+    |> Builder.method(method)
+    |> Builder.headers(headers)
+    |> Builder.opts(options)
+    |> Builder.url(url)
+    |> Builder.add_param(:body, :body, body)
+    |> Builder.add_param(:query, :query, params)
+    |> Builder.convert_to_keyword()
   end
 end

lib/pleroma/http/adapter_helper.ex

@@ -15,6 +15,12 @@ defmodule Pleroma.HTTP.AdapterHelper do
 
   @type proxy :: {Connection.proxy_type(), Connection.host(), pos_integer(), list()}
 
+  def maybe_add_cacerts(opts, nil), do: opts
+
+  def maybe_add_cacerts(opts, cacerts) do
+    put_in(opts, [:pools, :default, :conn_opts, :transport_opts, :cacerts], cacerts)
+  end
+
   @doc """
   Merge default connection & adapter options with received ones.
   """
@@ -29,11 +35,13 @@ defmodule Pleroma.HTTP.AdapterHelper do
           conn_max_idle_time: Config.get!([:http, :receive_timeout]),
           protocols: Config.get!([:http, :protocols]),
           conn_opts: [
-            transport_opts: [
-              inet6: true,
-              inet4: true,
-              cacerts: :public_key.cacerts_get()
-            ]
+            # Do NOT add cacerts here as this will cause issues for plain HTTP connections!
+            # (when we upgrade our deps to Mint >= 1.6.0 we can also explicitly enable "inet4: true")
+            transport_opts: [inet6: true],
+            # up to at least version 0.20.0, Finch leaves server_push enabled by default for HTTP2,
+            # but will actually raise an exception when receiving such a response. Tell servers we don't want it.
+            # see: https://github.com/sneako/finch/issues/325
+            client_settings: [enable_push: false]
           ]
         ]
       }

lib/pleroma/http/backoff.ex

@@ -94,7 +94,7 @@ defmodule Pleroma.HTTP.Backoff do
         log_ratelimit(status, host, timestamp)
         ttl = Timex.diff(timestamp, DateTime.utc_now(), :seconds)
         # we will cache the host for 5 minutes
-        @cachex.put(@backoff_cache, host, true, expire: ttl)
+        @cachex.put(@backoff_cache, host, true, ttl: ttl)
         {:error, :ratelimit}
 
       _ ->

lib/pleroma/http/middleware/httpsignature.ex

@@ -1,121 +0,0 @@
-# Akkoma: Magically expressive social media
-# Copyright © 2025 Akkoma Authors <https://akkoma.dev/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.HTTP.Middleware.HTTPSignature do
-  alias Pleroma.User.SigningKey
-  alias Pleroma.Signature
-
-  require Logger
-
-  @behaviour Tesla.Middleware
-
-  @moduledoc """
-  Adds a HTTP signature and related headers to requests, if a signing key is set in the request env.
-  If any other middleware can update the target location (e.g. redirects) this MUST be placed after all of them!
-
-  (Note: the third argument holds static middleware options from client creation)
-  """
-  @impl true
-  def call(env, next, _options) do
-    env = maybe_sign(env)
-    Tesla.run(env, next)
-  end
-
-  defp maybe_sign(env) do
-    case Keyword.get(env.opts, :httpsig) do
-      %{signing_key: %SigningKey{} = key} ->
-        set_signature_headers(env, key)
-
-      _ ->
-        env
-    end
-  end
-
-  defp set_signature_headers(env, key) do
-    Logger.debug("Signing request to: #{env.url}")
-    {http_headers, signing_headers} = collect_headers_for_signature(env)
-    signature = Signature.sign(key, signing_headers, has_body: has_body(env))
-    set_headers(env, [{"signature", signature} | http_headers])
-  end
-
-  defp has_body(%{body: body}) when body in [nil, ""], do: false
-  defp has_body(_), do: true
-
-  defp set_headers(env, []), do: env
-
-  defp set_headers(env, [{key, val} | rest]) do
-    headers = :proplists.delete(key, env.headers)
-    headers = [{key, val} | headers]
-    set_headers(%{env | headers: headers}, rest)
-  end
-
-  # Returns tuple.
-  # First element is headers+values which need to be added to the HTTP request.
-  # Second element are all headers to be used for signing, including already existing and pseudo headers.
-  defp collect_headers_for_signature(env) do
-    {request_target, host} = get_request_target_and_host(env)
-    date = http_date()
-
-    # content-length is always automatically set later on
-    # since they are needed to establish working connection.
-    # Similarly host will always be set for HTTP/1, and technically may be omitted for HTTP/2+
-    # but Tesla doesn’t handle it well if we preset it ourselves (and seems to set it even for HTTP/2 anyway)
-    http_headers = [{"date", date}]
-
-    signing_headers = %{
-      "(request-target)" => request_target,
-      "host" => host,
-      "date" => date
-    }
-
-    if has_body(env) do
-      append_body_headers(env, http_headers, signing_headers)
-    else
-      {http_headers, signing_headers}
-    end
-  end
-
-  defp append_body_headers(env, http_headers, signing_headers) do
-    content_length = byte_size(env.body)
-    digest = digest_value(env)
-
-    http_headers = [{"digest", digest} | http_headers]
-
-    signing_headers =
-      Map.merge(signing_headers, %{
-        "digest" => digest,
-        "content-length" => content_length
-      })
-
-    {http_headers, signing_headers}
-  end
-
-  defp get_request_target_and_host(env) do
-    uri = URI.parse(env.url)
-    rt = "#{env.method} #{uri.path}"
-    host = host_from_uri(uri)
-    {rt, host}
-  end
-
-  defp digest_value(env) do
-    # case Tesla.get_header(env, "digest")
-    encoded_hash = :crypto.hash(:sha256, env.body) |> Base.encode64()
-    "SHA-256=" <> encoded_hash
-  end
-
-  defp host_from_uri(%URI{port: port, scheme: scheme, host: host}) do
-    # https://httpwg.org/specs/rfc9110.html#field.host
-    # https://www.rfc-editor.org/rfc/rfc3986.html#section-3.2.3
-    if port == URI.default_port(scheme) do
-      host
-    else
-      "#{host}:#{port}"
-    end
-  end
-
-  defp http_date() do
-    now = NaiveDateTime.utc_now()
-    Timex.lformat!(now, "{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT", "en")
-  end
-end

lib/pleroma/http/request.ex

@@ -0,0 +1,23 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.Request do
+  @moduledoc """
+  Request struct.
+  """
+  defstruct method: :get, url: "", query: [], headers: [], body: "", opts: []
+
+  @type method :: :head | :get | :delete | :trace | :options | :post | :put | :patch
+  @type url :: String.t()
+  @type headers :: [{String.t(), String.t()}]
+
+  @type t :: %__MODULE__{
+          method: method(),
+          url: url(),
+          query: keyword(),
+          headers: headers(),
+          body: String.t(),
+          opts: keyword()
+        }
+end

lib/pleroma/http/request_builder.ex

@@ -0,0 +1,102 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.RequestBuilder do
+  @moduledoc """
+  Helper functions for building Tesla requests
+  """
+
+  alias Pleroma.HTTP.Request
+  alias Tesla.Multipart
+
+  @mix_env Mix.env()
+
+  @doc """
+  Creates new request
+  """
+  @spec new(Request.t()) :: Request.t()
+  def new(%Request{} = request \\ %Request{}), do: request
+
+  @doc """
+  Specify the request method when building a request
+  """
+  @spec method(Request.t(), Request.method()) :: Request.t()
+  def method(request, m), do: %{request | method: m}
+
+  @doc """
+  Specify the request method when building a request
+  """
+  @spec url(Request.t(), Request.url()) :: Request.t()
+  def url(request, u), do: %{request | url: u}
+
+  @doc """
+  Add headers to the request
+  """
+  @spec headers(Request.t(), Request.headers()) :: Request.t()
+  def headers(request, headers) do
+    headers_list = maybe_add_user_agent(headers, @mix_env)
+
+    %{request | headers: headers_list}
+  end
+
+  @doc """
+  Add custom, per-request middleware or adapter options to the request
+  """
+  @spec opts(Request.t(), keyword()) :: Request.t()
+  def opts(request, options), do: %{request | opts: options}
+
+  @doc """
+  Add optional parameters to the request
+  """
+  @spec add_param(Request.t(), atom(), atom(), any()) :: Request.t()
+  def add_param(request, :query, :query, values), do: %{request | query: values}
+
+  def add_param(request, :body, :body, value), do: %{request | body: value}
+
+  def add_param(request, :body, key, value) do
+    request
+    |> Map.put(:body, Multipart.new())
+    |> Map.update!(
+      :body,
+      &Multipart.add_field(
+        &1,
+        key,
+        Jason.encode!(value),
+        headers: [{"content-type", "application/json"}]
+      )
+    )
+  end
+
+  def add_param(request, :file, name, path) do
+    request
+    |> Map.put(:body, Multipart.new())
+    |> Map.update!(:body, &Multipart.add_file(&1, path, name: name))
+  end
+
+  def add_param(request, :form, name, value) do
+    Map.update(request, :body, %{name => value}, &Map.put(&1, name, value))
+  end
+
+  def add_param(request, location, key, value) do
+    Map.update(request, location, [{key, value}], &(&1 ++ [{key, value}]))
+  end
+
+  def convert_to_keyword(request) do
+    request
+    |> Map.from_struct()
+    |> Enum.into([])
+  end
+
+  defp maybe_add_user_agent(headers, :test) do
+    with true <- Pleroma.Config.get([:http, :send_user_agent]) do
+      [{"user-agent", Pleroma.Application.user_agent()} | headers]
+    else
+      _ ->
+        headers
+    end
+  end
+
+  defp maybe_add_user_agent(headers, _),
+    do: [{"user-agent", Pleroma.Application.user_agent()} | headers]
+end

lib/pleroma/http/tzdata.ex

@@ -10,15 +10,15 @@ defmodule Pleroma.HTTP.Tzdata do
   alias Pleroma.HTTP
 
   @impl true
-  def get(url, headers, _options) do
-    with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers) do
+  def get(url, headers, options) do
+    with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers, options) do
       {:ok, {env.status, env.headers, env.body}}
     end
   end
 
   @impl true
-  def head(url, headers, _options) do
-    with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers) do
+  def head(url, headers, options) do
+    with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers, options) do
       {:ok, {env.status, env.headers}}
     end
   end

lib/pleroma/http/web_push.ex

@@ -5,8 +5,8 @@
 defmodule Pleroma.HTTP.WebPush do
   @moduledoc false
 
-  def post(url, payload, headers, _options) do
+  def post(url, payload, headers, options \\ []) do
     list_headers = Map.to_list(headers)
-    Pleroma.HTTP.post(url, payload, list_headers)
+    Pleroma.HTTP.post(url, payload, list_headers, options)
   end
 end

lib/pleroma/instances/instance.ex

@@ -242,7 +242,7 @@ defmodule Pleroma.Instances.Instance do
            {:ok,
             Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0"))},
          {:ok, %Tesla.Env{body: data}} <-
-           Pleroma.HTTP.get(href, [{"accept", "application/json"}]),
+           Pleroma.HTTP.get(href, [{"accept", "application/json"}], []),
          {:length, true} <- {:length, String.length(data) < 50_000},
          {:ok, nodeinfo} <- Jason.decode(data) do
       nodeinfo
@@ -270,7 +270,7 @@ defmodule Pleroma.Instances.Instance do
     with true <- Pleroma.Config.get([:instances_favicons, :enabled]),
          {_, true} <- {:reachable, reachable?(instance_uri.host)},
          {:ok, %Tesla.Env{body: html}} <-
-           Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}]),
+           Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], []),
          {_, [favicon_rel | _]} when is_binary(favicon_rel) <-
            {:parse, html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")},
          {_, favicon} when is_binary(favicon) <-
@@ -299,7 +299,7 @@ defmodule Pleroma.Instances.Instance do
   end
 
   def perform(:delete_instance, host) when is_binary(host) do
-    User.Query.build(%{nickname_suffix: "@#{host}"})
+    User.Query.build(%{nickname: "@#{host}"})
     |> Repo.chunk_stream(100, :batches)
     |> Stream.each(fn users ->
       users

lib/pleroma/job_queue_monitor.ex

@@ -0,0 +1,95 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.JobQueueMonitor do
+  use GenServer
+
+  @initial_state %{workers: %{}, queues: %{}, processed_jobs: 0}
+  @queue %{processed_jobs: 0, success: 0, failure: 0}
+  @operation %{processed_jobs: 0, success: 0, failure: 0}
+
+  def start_link(_) do
+    GenServer.start_link(__MODULE__, @initial_state, name: __MODULE__)
+  end
+
+  @impl true
+  def init(state) do
+    :telemetry.attach(
+      "oban-monitor-failure",
+      [:oban, :job, :exception],
+      &Pleroma.JobQueueMonitor.handle_event/4,
+      nil
+    )
+
+    :telemetry.attach(
+      "oban-monitor-success",
+      [:oban, :job, :stop],
+      &Pleroma.JobQueueMonitor.handle_event/4,
+      nil
+    )
+
+    {:ok, state}
+  end
+
+  def stats do
+    GenServer.call(__MODULE__, :stats)
+  end
+
+  def handle_event([:oban, :job, event], %{duration: duration}, meta, _) do
+    GenServer.cast(
+      __MODULE__,
+      {:process_event, mapping_status(event), duration, meta}
+    )
+  end
+
+  @impl true
+  def handle_call(:stats, _from, state) do
+    {:reply, state, state}
+  end
+
+  @impl true
+  def handle_cast({:process_event, status, duration, meta}, state) do
+    state =
+      state
+      |> Map.update!(:workers, fn workers ->
+        workers
+        |> Map.put_new(meta.worker, %{})
+        |> Map.update!(meta.worker, &update_worker(&1, status, meta, duration))
+      end)
+      |> Map.update!(:queues, fn workers ->
+        workers
+        |> Map.put_new(meta.queue, @queue)
+        |> Map.update!(meta.queue, &update_queue(&1, status, meta, duration))
+      end)
+      |> Map.update!(:processed_jobs, &(&1 + 1))
+
+    {:noreply, state}
+  end
+
+  defp update_worker(worker, status, meta, duration) do
+    worker
+    |> Map.put_new(meta.args["op"], @operation)
+    |> Map.update!(meta.args["op"], &update_op(&1, status, meta, duration))
+  end
+
+  defp update_op(op, :enqueue, _meta, _duration) do
+    op
+    |> Map.update!(:enqueued, &(&1 + 1))
+  end
+
+  defp update_op(op, status, _meta, _duration) do
+    op
+    |> Map.update!(:processed_jobs, &(&1 + 1))
+    |> Map.update!(status, &(&1 + 1))
+  end
+
+  defp update_queue(queue, status, _meta, _duration) do
+    queue
+    |> Map.update!(:processed_jobs, &(&1 + 1))
+    |> Map.update!(status, &(&1 + 1))
+  end
+
+  defp mapping_status(:stop), do: :success
+  defp mapping_status(:exception), do: :failure
+end

lib/pleroma/list.ex

@@ -16,14 +16,14 @@ defmodule Pleroma.List do
     belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
     field(:title, :string)
     field(:following, {:array, :string}, default: [])
-    field(:exclusive, :boolean, default: false)
+    field(:ap_id, :string)
 
     timestamps()
   end
 
-  def update_changeset(list, attrs \\ %{}) do
+  def title_changeset(list, attrs \\ %{}) do
     list
-    |> cast(attrs, [:title, :exclusive])
+    |> cast(attrs, [:title])
     |> validate_required([:title])
   end
 
@@ -56,6 +56,10 @@ defmodule Pleroma.List do
     Repo.one(query)
   end
 
+  def get_by_ap_id(ap_id) do
+    Repo.get_by(__MODULE__, ap_id: ap_id)
+  end
+
   def get_following(%Pleroma.List{following: following} = _list) do
     q =
       from(
@@ -87,17 +91,23 @@ defmodule Pleroma.List do
     |> Repo.all()
   end
 
-  def update(%Pleroma.List{} = list, params) do
+  def rename(%Pleroma.List{} = list, title) do
     list
-    |> update_changeset(params)
+    |> title_changeset(%{title: title})
     |> Repo.update()
   end
 
-  def create(params, %User{} = creator) do
-    changeset = update_changeset(%Pleroma.List{user_id: creator.id}, params)
+  def create(title, %User{} = creator) do
+    changeset = title_changeset(%Pleroma.List{user_id: creator.id}, %{title: title})
 
     if changeset.valid? do
-      Repo.insert(changeset)
+      Repo.transaction(fn ->
+        list = Repo.insert!(changeset)
+
+        list
+        |> change(ap_id: "#{creator.ap_id}/lists/#{list.id}")
+        |> Repo.update!()
+      end)
     else
       {:error, changeset}
     end
@@ -125,19 +135,18 @@ defmodule Pleroma.List do
     |> Repo.update()
   end
 
+  def memberships(%User{follower_address: follower_address}) do
+    Pleroma.List
+    |> where([l], ^follower_address in l.following)
+    |> select([l], l.ap_id)
+    |> Repo.all()
+  end
+
+  def memberships(_), do: []
+
   def member?(%Pleroma.List{following: following}, %User{follower_address: follower_address}) do
     Enum.member?(following, follower_address)
   end
 
   def member?(_, _), do: false
-
-  def get_exclusive_list_members(%User{id: user_id}) do
-    Pleroma.List
-    |> where([l], l.user_id == ^user_id)
-    |> where([l], l.exclusive == true)
-    |> join(:cross_lateral, [l], fragment("UNNEST(?)", l.following))
-    |> select([_l, r], fragment("?", r))
-    |> distinct(true)
-    |> Repo.all()
-  end
 end

lib/pleroma/logging.ex

@@ -0,0 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Logging do
+  @callback error(String.t()) :: any()
+end

lib/pleroma/marker.ex

@@ -78,7 +78,7 @@ defmodule Pleroma.Marker do
 
   defp get_marker(user, timeline) do
     case Repo.find_resource(get_query(user, timeline)) do
-      {:ok, %__MODULE__{} = marker} -> %__MODULE__{marker | user: user}
+      {:ok, marker} -> %__MODULE__{marker | user: user}
       _ -> %__MODULE__{timeline: timeline, user_id: user.id}
     end
   end

lib/pleroma/mfa.ex

@@ -54,7 +54,6 @@ defmodule Pleroma.MFA do
   end
 
   @doc false
-  @spec fetch_settings(User.t()) :: Settings.t()
   def fetch_settings(%User{} = user) do
     user.multi_factor_authentication_settings || %Settings{}
   end

lib/pleroma/mfa/changeset.ex

@@ -8,8 +8,7 @@ defmodule Pleroma.MFA.Changeset do
   alias Pleroma.User
 
   def disable(%Ecto.Changeset{} = changeset, force \\ false) do
-    %Settings{} =
-      settings =
+    settings =
       changeset
       |> Ecto.Changeset.apply_changes()
       |> MFA.fetch_settings()
@@ -23,18 +22,18 @@ defmodule Pleroma.MFA.Changeset do
 
   def disable_totp(%User{multi_factor_authentication_settings: settings} = user) do
     user
-    |> put_change(%{settings | totp: %Settings.TOTP{}})
+    |> put_change(%Settings{settings | totp: %Settings.TOTP{}})
   end
 
   def confirm_totp(%User{multi_factor_authentication_settings: settings} = user) do
-    totp_settings = %{settings.totp | confirmed: true}
+    totp_settings = %Settings.TOTP{settings.totp | confirmed: true}
 
     user
-    |> put_change(%{settings | totp: totp_settings, enabled: true})
+    |> put_change(%Settings{settings | totp: totp_settings, enabled: true})
   end
 
   def setup_totp(%User{} = user, attrs) do
-    %Settings{} = mfa_settings = MFA.fetch_settings(user)
+    mfa_settings = MFA.fetch_settings(user)
 
     totp_settings =
       %Settings.TOTP{}
@@ -46,7 +45,7 @@ defmodule Pleroma.MFA.Changeset do
 
   def cast_backup_codes(%User{} = user, codes) do
     user
-    |> put_change(%{
+    |> put_change(%Settings{
       user.multi_factor_authentication_settings
       | backup_codes: codes
     })

lib/pleroma/notification.ex

@@ -15,6 +15,7 @@ defmodule Pleroma.Notification do
   alias Pleroma.Repo
   alias Pleroma.ThreadMute
   alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.Push
   alias Pleroma.Web.Streamer
@@ -128,6 +129,7 @@ defmodule Pleroma.Notification do
     |> exclude_blocked(user, exclude_blocked_opts)
     |> exclude_blockers(user)
     |> exclude_filtered(user)
+    |> exclude_visibility(opts)
   end
 
   # Excludes blocked users and non-followed domain-blocked users
@@ -146,7 +148,9 @@ defmodule Pleroma.Notification do
       fragment(
         # "(actor's domain NOT in domain_blocks)"
         """
-        split_part(?, '/', 3) <> ALL(?)
+        NOT (
+          substring(? from '.*://([^/]*)') = ANY(?)
+        )
         """,
         activity.actor,
         ^user.domain_blocks
@@ -197,6 +201,67 @@ defmodule Pleroma.Notification do
     end
   end
 
+  @valid_visibilities ~w[direct unlisted public private]
+
+  defp exclude_visibility(query, %{exclude_visibilities: visibility})
+       when is_list(visibility) do
+    if Enum.all?(visibility, &(&1 in @valid_visibilities)) do
+      query
+      |> join(:left, [n, a], mutated_activity in Pleroma.Activity,
+        on:
+          fragment(
+            "COALESCE((?->'object')->>'id', ?->>'object')",
+            a.data,
+            a.data
+          ) ==
+            fragment(
+              "COALESCE((?->'object')->>'id', ?->>'object')",
+              mutated_activity.data,
+              mutated_activity.data
+            ) and
+            fragment("(?->>'type' = 'Like' or ?->>'type' = 'Announce')", a.data, a.data) and
+            fragment("?->>'type'", mutated_activity.data) == "Create",
+        as: :mutated_activity
+      )
+      |> where(
+        [n, a, mutated_activity: mutated_activity],
+        not fragment(
+          """
+          CASE WHEN (?->>'type') = 'Like' or (?->>'type') = 'Announce'
+            THEN (activity_visibility(?, ?, ?) = ANY (?))
+            ELSE (activity_visibility(?, ?, ?) = ANY (?)) END
+          """,
+          a.data,
+          a.data,
+          mutated_activity.actor,
+          mutated_activity.recipients,
+          mutated_activity.data,
+          ^visibility,
+          a.actor,
+          a.recipients,
+          a.data,
+          ^visibility
+        )
+      )
+    else
+      Logger.error("Could not exclude visibility to #{visibility}")
+      query
+    end
+  end
+
+  defp exclude_visibility(query, %{exclude_visibilities: visibility})
+       when visibility in @valid_visibilities do
+    exclude_visibility(query, [visibility])
+  end
+
+  defp exclude_visibility(query, %{exclude_visibilities: visibility})
+       when visibility not in @valid_visibilities do
+    Logger.error("Could not exclude visibility to #{visibility}")
+    query
+  end
+
+  defp exclude_visibility(query, _visibility), do: query
+
   def for_user(user, opts \\ %{}) do
     user
     |> for_user_query(opts)
@@ -323,46 +388,40 @@ defmodule Pleroma.Notification do
     end
   end
 
-  @doc """
-  Create notifications for given Activity in database, but does NOT send them to streams and webpush.
-  On success returns :ok triple with non-muted notifications in the second position and
-  muted (i.e. likely not supposed to be pro-actively sent) notifications in the third position.
-  """
-  @spec create_notifications(Activity.t()) ::
-          {:ok, [Notification.t()] | [], [Notification.t()] | []}
-  def create_notifications(activity)
+  @spec create_notifications(Activity.t(), keyword()) :: {:ok, [Notification.t()] | []}
+  def create_notifications(activity, options \\ [])
 
-  def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do
+  def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity, options) do
     object = Object.normalize(activity, fetch: false)
 
     if object && object.data["type"] == "Answer" do
-      {:ok, [], []}
+      {:ok, []}
     else
-      do_create_notifications(activity)
+      do_create_notifications(activity, options)
     end
   end
 
-  def create_notifications(%Activity{data: %{"type" => type}} = activity)
+  def create_notifications(%Activity{data: %{"type" => type}} = activity, options)
       when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag", "Update"] do
-    do_create_notifications(activity)
+    do_create_notifications(activity, options)
   end
 
-  def create_notifications(_), do: {:ok, [], []}
+  def create_notifications(_, _), do: {:ok, []}
+
+  defp do_create_notifications(%Activity{} = activity, options) do
+    do_send = Keyword.get(options, :do_send, true)
 
-  defp do_create_notifications(%Activity{} = activity) do
     {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity)
+    potential_receivers = enabled_receivers ++ disabled_receivers
 
-    notifications_active =
-      enabled_receivers
-      |> Enum.map(&create_notification(activity, &1))
+    notifications =
+      Enum.map(potential_receivers, fn user ->
+        do_send = do_send && user in enabled_receivers
+        create_notification(activity, user, do_send: do_send)
+      end)
       |> Enum.reject(&is_nil/1)
 
-    notifications_silent =
-      disabled_receivers
-      |> Enum.map(&create_notification(activity, &1, seen: true))
-      |> Enum.reject(&is_nil/1)
-
-    {:ok, notifications_active, notifications_silent}
+    {:ok, notifications}
   end
 
   defp type_from_activity(%{data: %{"type" => type}} = activity) do
@@ -408,9 +467,9 @@ defmodule Pleroma.Notification do
   defp type_from_activity_object(%{data: %{"type" => "Create"}}), do: "mention"
 
   # TODO move to sql, too.
-  defp create_notification(%Activity{} = activity, %User{} = user, opts \\ []) do
+  def create_notification(%Activity{} = activity, %User{} = user, opts \\ []) do
+    do_send = Keyword.get(opts, :do_send, true)
     type = Keyword.get(opts, :type, type_from_activity(activity))
-    seen = Keyword.get(opts, :seen, false)
 
     unless skip?(activity, user, opts) do
       {:ok, %{notification: notification}} =
@@ -418,12 +477,17 @@ defmodule Pleroma.Notification do
         |> Multi.insert(:notification, %Notification{
           user_id: user.id,
           activity: activity,
-          seen: seen,
+          seen: mark_as_read?(activity, user),
           type: type
         })
         |> Marker.multi_set_last_read_id(user, "notifications")
         |> Repo.transaction()
 
+      if do_send do
+        Streamer.stream(["user", "user:notification"], notification)
+        Push.send(notification)
+      end
+
       notification
     end
   end
@@ -614,12 +678,6 @@ defmodule Pleroma.Notification do
     end
   end
 
-  def skip?(:internal, %Activity{} = activity, _user, _opts) do
-    actor = activity.data["actor"]
-    user = User.get_cached_by_ap_id(actor)
-    User.is_internal_user?(user)
-  end
-
   def skip?(:invisible, %Activity{} = activity, _user, _opts) do
     actor = activity.data["actor"]
     user = User.get_cached_by_ap_id(actor)
@@ -682,6 +740,11 @@ defmodule Pleroma.Notification do
 
   def skip?(_type, _activity, _user, _opts), do: false
 
+  def mark_as_read?(activity, target_user) do
+    user = Activity.user_actor(activity)
+    User.mutes_user?(target_user, user) || CommonAPI.thread_muted?(target_user, activity)
+  end
+
   def for_user_and_activity(user, activity) do
     from(n in __MODULE__,
       where: n.user_id == ^user.id,
@@ -701,12 +764,4 @@ defmodule Pleroma.Notification do
     )
     |> Repo.update_all(set: [seen: true])
   end
-
-  @spec send(list(Notification.t())) :: :ok
-  def send(notifications) do
-    Enum.each(notifications, fn notification ->
-      Streamer.stream(["user", "user:notification"], notification)
-      Push.send(notification)
-    end)
-  end
 end

lib/pleroma/object.ex

@@ -144,7 +144,7 @@ defmodule Pleroma.Object do
     Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}")
   end
 
-  def normalize(_, options \\ [fetch: false])
+  def normalize(_, options \\ [fetch: false, id_only: false])
 
   # If we pass an Activity to Object.normalize(), we can try to use the preloaded object.
   # Use this whenever possible, especially when walking graphs in an O(N) loop!
@@ -173,6 +173,9 @@ defmodule Pleroma.Object do
 
   def normalize(ap_id, options) when is_binary(ap_id) do
     cond do
+      Keyword.get(options, :id_only) ->
+        ap_id
+
       Keyword.get(options, :fetch) ->
         case Fetcher.fetch_object_from_id(ap_id, options) do
           {:ok, object} -> object
@@ -336,21 +339,12 @@ defmodule Pleroma.Object do
             option
         end)
 
-      old_voters = object.data["voters"] || []
-      old_voters_count = object.data["votersCount"] || length(old_voters)
-
-      {voters, voters_count} =
-        if actor in old_voters do
-          {old_voters, old_voters_count}
-        else
-          {[actor | old_voters], old_voters_count + 1}
-        end
+      voters = [actor | object.data["voters"] || []] |> Enum.uniq()
 
       data =
         object.data
         |> Map.put(key, options)
         |> Map.put("voters", voters)
-        |> Map.put("votersCount", voters_count)
 
       object
       |> Object.change(%{data: data})

lib/pleroma/object/fetcher.ex

@@ -3,17 +3,14 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Object.Fetcher do
-  alias Pleroma.Activity
   alias Pleroma.HTTP
   alias Pleroma.Instances
   alias Pleroma.Maps
   alias Pleroma.Object
   alias Pleroma.Object.Containment
-  alias Pleroma.Object.Updater
   alias Pleroma.Repo
-  alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Signature
   alias Pleroma.Web.ActivityPub.InternalFetchActor
-  alias Pleroma.Web.ActivityPub.MRF
   alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Federator
@@ -39,85 +36,79 @@ defmodule Pleroma.Object.Fetcher do
     Ecto.Changeset.put_change(changeset, :updated_at, updated_at)
   end
 
-  defp persist_revived_data(old_activity, activity_data, object_changeset) do
-    Repo.transact(fn ->
-      with {:ok, _} <- Repo.delete(old_activity),
-           {:ok, _, _} <- ActivityPub.persist(activity_data, local: false) do
-        Repo.insert_or_update(object_changeset)
+  defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do
+    has_history? = fn
+      %{"formerRepresentations" => %{"orderedItems" => list}} when is_list(list) -> true
+      _ -> false
+    end
+
+    internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields())
+
+    remote_history_exists? = has_history?.(new_data)
+
+    # If the remote history exists, we treat that as the only source of truth.
+    new_data =
+      if has_history?.(old_data) and not remote_history_exists? do
+        Map.put(new_data, "formerRepresentations", old_data["formerRepresentations"])
+      else
+        new_data
       end
-    end)
+
+    # If the remote does not have history information, we need to manage it ourselves
+    new_data =
+      if not remote_history_exists? do
+        changed? =
+          Pleroma.Constants.status_updatable_fields()
+          |> Enum.any?(fn field -> Map.get(old_data, field) != Map.get(new_data, field) end)
+
+        %{updated_object: updated_object} =
+          new_data
+          |> Object.Updater.maybe_update_history(old_data,
+            updated: changed?,
+            use_history_in_new_object?: false
+          )
+
+        updated_object
+      else
+        new_data
+      end
+
+    Map.merge(new_data, internal_fields)
   end
 
-  defp revive_pruned_object(activity, new_data) do
-    object_changeset = fn data ->
-      %Object{}
-      |> Object.change(%{data: data})
-      |> touch_changeset()
-    end
-
-    # We don’t store the inlined object inside Create in db
-    activity_data_db = fn data ->
-      data
-      |> Pleroma.Maps.put_if_present("object", is_map(data["object"]) && data["object"]["id"])
-    end
-
-    # Before revivial pass through MRFs again
-    # and recreate activity in db to ensure data consistency (e.g. changed addressing)
-    fake_create =
-      prepare_create_activity_params(new_data)
-      |> Pleroma.Maps.put_if_present("id", activity.data && activity.data["id"])
-
-    with {_, {:ok, %{"object" => new_data_processed} = activity_data}} <-
-           {:mrf_check, MRF.filter(fake_create)},
-         object_changeset <- object_changeset.(new_data_processed),
-         activity_data_db <- activity_data_db.(activity_data),
-         {:ok, object} <- persist_revived_data(activity, activity_data_db, object_changeset),
-         {:ok, object} <- Object.set_cache(object) do
-      {:ok, object}
-    else
-      {:mrf_check, error} ->
-        Logger.debug("Rejected fetched object revival due to MRF: #{inspect(error)}")
-        {:reject, :mrf}
-
-      e ->
-        Logger.error(
-          "Error while processing object for reinjection after deletion: #{inspect(e)}"
-        )
-
-        {:error, e}
-    end
-  end
+  defp maybe_reinject_internal_fields(_, new_data), do: new_data
 
   @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()}
-  defp reinject_object(old_ref, new_data) do
+  defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) do
     Logger.debug("Reinjecting object #{new_data["id"]}")
 
-    # AP ID is checked early on in transmogrifier and assumed to hold later
-    with ap_id when is_binary(ap_id) <- new_data["id"],
-         new_data <- Transmogrifier.fix_object(new_data),
-         {:ok, new_data, _} <- ObjectValidator.validate(new_data, %{}),
-         {_, object = %Object{}, _} <- {:pruned_old, old_ref, new_data},
-         fake_update <- prepare_update_activity_params(new_data),
-         {_, {:ok, %{"object" => new_data}}} <- {:mrf_check, MRF.filter(fake_update)},
-         %{updated_data: new_data} <-
-           Updater.make_new_object_data_from_update_object(object.data, new_data, true),
-         changeset <- Object.change(object, %{data: new_data}),
+    with data <- maybe_reinject_internal_fields(object, new_data),
+         {:ok, data, _} <- ObjectValidator.validate(data, %{}),
+         changeset <- Object.change(object, %{data: data}),
          changeset <- touch_changeset(changeset),
          {:ok, object} <- Repo.insert_or_update(changeset),
          {:ok, object} <- Object.set_cache(object) do
       {:ok, object}
     else
-      {:pruned_old, %Activity{} = activity, new_data_fixed} ->
-        # we don't know the object anymore, but there’s already an orphaned Create pointing at it
-        # (can happen when running prune_objects without prune_orphaned_activities)
-        revive_pruned_object(activity, new_data_fixed)
-
-      {:mrf_check, error} ->
-        Logger.debug("Rejected fetched object update due to MRF: #{inspect(error)}")
-        {:reject, :mrf}
-
       e ->
-        Logger.error("Error while processing object for reinjection: #{inspect(e)}")
+        Logger.error("Error while processing object: #{inspect(e)}")
+        {:error, e}
+    end
+  end
+
+  defp reinject_object(%Object{} = object, new_data) do
+    Logger.debug("Reinjecting object #{new_data["id"]}")
+
+    with new_data <- Transmogrifier.fix_object(new_data),
+         data <- maybe_reinject_internal_fields(object, new_data),
+         changeset <- Object.change(object, %{data: data}),
+         changeset <- touch_changeset(changeset),
+         {:ok, object} <- Repo.insert_or_update(changeset),
+         {:ok, object} <- Object.set_cache(object) do
+      {:ok, object}
+    else
+      e ->
+        Logger.error("Error while processing object: #{inspect(e)}")
         {:error, e}
     end
   end
@@ -148,18 +139,19 @@ defmodule Pleroma.Object.Fetcher do
          {:valid_uri_scheme, true} <-
            {:valid_uri_scheme, uri.scheme == "http" or uri.scheme == "https"},
          # If we have instance restrictions, apply them here to prevent fetching from unwanted instances
-         {_, {:ok, _}} <- {:mrf_check, maybe_restrict_uri_mrf(uri)},
+         {:mrf_reject_check, {:ok, nil}} <-
+           {:mrf_reject_check, Pleroma.Web.ActivityPub.MRF.SimplePolicy.check_reject(uri)},
+         {:mrf_accept_check, {:ok, _}} <-
+           {:mrf_accept_check, Pleroma.Web.ActivityPub.MRF.SimplePolicy.check_accept(uri)},
          {_, nil} <- {:fetch_object, Object.get_cached_by_ap_id(id)},
          {_, true} <- {:allowed_depth, Federator.allowed_thread_distance?(options[:depth])},
          {_, {:ok, data}} <- {:fetch, fetch_and_contain_remote_object_from_id(id)},
-         {_, unknown, _} when unknown in [false, nil] <-
-           {:cached_new_id, data["id"] != id && Object.get_cached_by_ap_id(data["id"]), data},
          {_, nil} <- {:normalize, Object.normalize(data, fetch: false)},
-         params <- prepare_create_activity_params(data),
+         params <- prepare_activity_params(data),
          {_, {:ok, activity}} <-
            {:transmogrifier, Transmogrifier.handle_incoming(params, options)},
-         {_, _data, _activity, %Object{} = object} <-
-           {:object, data, activity, Object.normalize(activity, fetch: false)} do
+         {_, _data, %Object{} = object} <-
+           {:object, data, Object.normalize(activity, fetch: false)} do
       {:ok, object}
     else
       {:allowed_depth, false} = e ->
@@ -170,7 +162,11 @@ defmodule Pleroma.Object.Fetcher do
         log_fetch_error(id, e)
         {:error, :invalid_uri_scheme}
 
-      {:mrf_check, _} = e ->
+      {:mrf_reject_check, _} = e ->
+        log_fetch_error(id, e)
+        {:reject, :mrf}
+
+      {:mrf_accept_check, _} = e ->
         log_fetch_error(id, e)
         {:reject, :mrf}
 
@@ -194,8 +190,8 @@ defmodule Pleroma.Object.Fetcher do
         log_fetch_error(id, e)
         {:error, reason}
 
-      {:object, data, %Activity{} = activity, nil} ->
-        reinject_object(activity, data)
+      {:object, data, nil} ->
+        reinject_object(%Object{}, data)
 
       {:normalize, object = %Object{}} ->
         {:ok, object}
@@ -203,13 +199,6 @@ defmodule Pleroma.Object.Fetcher do
       {:fetch_object, %Object{} = object} ->
         {:ok, object}
 
-      {:cached_new_id, %Object{} = object, data} ->
-        # might as well make use of the fresh canonical source we now already fetched
-        case reinject_object(object, data) do
-          {:ok, _} = ok -> ok
-          _ -> {:ok, object}
-        end
-
       {:fetch, {:error, reason}} = e ->
         log_fetch_error(id, e)
         {:error, reason}
@@ -225,20 +214,9 @@ defmodule Pleroma.Object.Fetcher do
     Logger.error("Object rejected while fetching #{id} #{inspect(error)}")
   end
 
-  defp maybe_restrict_uri_mrf(uri) do
-    with {:enabled, true} <- {:enabled, MRF.SimplePolicy in MRF.get_policies()},
-         {:ok, _} <- MRF.SimplePolicy.check_reject(uri),
-         {:ok, _} <- MRF.SimplePolicy.check_accept(uri) do
-      {:ok, nil}
-    else
-      {:enabled, false} -> {:ok, nil}
-      {:reject, reason} -> {:reject, reason}
-    end
-  end
-
-  defp prepare_activity_params(data, type) do
+  defp prepare_activity_params(data) do
     %{
-      "type" => type,
+      "type" => "Create",
       # Should we seriously keep this attributedTo thing?
       "actor" => data["actor"] || data["attributedTo"],
       "object" => data
@@ -249,11 +227,35 @@ defmodule Pleroma.Object.Fetcher do
     |> Maps.put_if_present("bcc", data["bcc"])
   end
 
-  # We also need a Create activity for posts, but typically fetch Note objects directly
-  defp prepare_create_activity_params(data), do: prepare_activity_params(data, "Create")
+  defp make_signature(id, date) do
+    uri = URI.parse(id)
 
-  # Only used to pass to MRFs when refetching an object
-  defp prepare_update_activity_params(data), do: prepare_activity_params(data, "Update")
+    signature =
+      InternalFetchActor.get_actor()
+      |> Signature.sign(%{
+        "(request-target)" => "get #{uri.path}",
+        "host" => uri.host,
+        "date" => date
+      })
+
+    {"signature", signature}
+  end
+
+  defp sign_fetch(headers, id, date) do
+    if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
+      [make_signature(id, date) | headers]
+    else
+      headers
+    end
+  end
+
+  defp maybe_date_fetch(headers, date) do
+    if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
+      [{"date", date} | headers]
+    else
+      headers
+    end
+  end
 
   @doc """
   Fetches arbitrary remote object and performs basic safety and authenticity checks.
@@ -327,7 +329,10 @@ defmodule Pleroma.Object.Fetcher do
 
     with {:valid_uri_scheme, true} <- {:valid_uri_scheme, String.starts_with?(id, "http")},
          %URI{} = uri <- URI.parse(id),
-         {_, {:ok, _}} <- {:mrf_check, maybe_restrict_uri_mrf(uri)},
+         {:mrf_reject_check, {:ok, nil}} <-
+           {:mrf_reject_check, Pleroma.Web.ActivityPub.MRF.SimplePolicy.check_reject(uri)},
+         {:mrf_accept_check, {:ok, _}} <-
+           {:mrf_accept_check, Pleroma.Web.ActivityPub.MRF.SimplePolicy.check_accept(uri)},
          {:local_fetch, :ok} <- {:local_fetch, Containment.contain_local_fetch(id)},
          {:ok, final_id, body} <- get_object(id),
          # a canonical ID shouldn't be a redirect
@@ -354,7 +359,11 @@ defmodule Pleroma.Object.Fetcher do
           {:error, :id_mismatch}
         end
 
-      {:mrf_check, _} = e ->
+      {:mrf_reject_check, _} = e ->
+        log_fetch_error(id, e)
+        {:reject, :mrf}
+
+      {:mrf_accept_check, _} = e ->
         log_fetch_error(id, e)
         {:reject, :mrf}
 
@@ -393,25 +402,20 @@ defmodule Pleroma.Object.Fetcher do
 
   @doc "Do NOT use; only public for use in tests"
   def get_object(id) do
+    date = Pleroma.Signature.signed_date()
+
     headers =
       [
         # The first is required by spec, the second provided as a fallback for buggy implementations
         {"accept", "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\""},
         {"accept", "application/activity+json"}
       ]
-
-    http_opts =
-      if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
-        signing_actor = InternalFetchActor.get_actor() |> Pleroma.User.SigningKey.load_key()
-        signing_key = signing_actor.signing_key
-        [httpsig: %{signing_key: signing_key}]
-      else
-        []
-      end
+      |> maybe_date_fetch(date)
+      |> sign_fetch(id, date)
 
     with {:ok, %{body: body, status: code, headers: headers, url: final_url}}
          when code in 200..299 <-
-           HTTP.Backoff.get(id, headers, http_opts),
+           HTTP.Backoff.get(id, headers),
          {:has_content_type, {_, content_type}} <-
            {:has_content_type, List.keyfind(headers, "content-type", 0)},
          {:parse_content_type, {:ok, "application", subtype, type_params}} <-
@@ -439,13 +443,6 @@ defmodule Pleroma.Object.Fetcher do
       {:ok, %{status: code}} when code in [404, 410] ->
         {:error, :not_found}
 
-      {:ok, %{status: code, headers: headers}} ->
-        {:error, {:http_error, code, headers}}
-
-      # connection/protocol-related error
-      {:ok, %Tesla.Env{} = env} ->
-        {:error, {:http_error, :connect, env}}
-
       {:error, e} ->
         {:error, e}
 

lib/pleroma/object/updater.ex

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Object.Updater do
-  alias Pleroma.Maps
   require Pleroma.Constants
 
   def update_content_fields(orig_object_data, updated_object) do
@@ -99,8 +98,7 @@ defmodule Pleroma.Object.Updater do
 
   defp maybe_update_poll(to_be_updated, updated_object) do
     choice_key = fn data ->
-      # Our normalisation fills out the not-used-type with empty array
-      if data["anyOf"] not in [nil, []], do: "anyOf", else: "oneOf"
+      if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf"
     end
 
     with true <- to_be_updated["type"] == "Question",
@@ -112,11 +110,9 @@ defmodule Pleroma.Object.Updater do
       # Choices are the same, but counts are different
       to_be_updated
       |> Map.put(key, updated_object[key])
-      |> Maps.put_if_present("votersCount", updated_object["votersCount"])
     else
       # Choices (or vote type) have changed, do not allow this
-      _ ->
-        to_be_updated
+      _ -> to_be_updated
     end
   end
 
@@ -145,7 +141,7 @@ defmodule Pleroma.Object.Updater do
 
   # This calculates the data of the new Object from an Update.
   # new_data's formerRepresentations is considered.
-  def make_new_object_data_from_update_object(original_data, new_data, force_update \\ false) do
+  def make_new_object_data_from_update_object(original_data, new_data) do
     update_is_reasonable =
       with {_, updated} when not is_nil(updated) <- {:cur_updated, new_data["updated"]},
            {_, {:ok, updated_time, _}} <- {:cur_updated, DateTime.from_iso8601(updated)},
@@ -165,8 +161,6 @@ defmodule Pleroma.Object.Updater do
         _ -> false
       end
 
-    update_is_reasonable = if force_update, do: :update_everything, else: update_is_reasonable
-
     %{
       updated_object: updated_data,
       used_history_in_new_object?: used_history_in_new_object?,

lib/pleroma/pagination.ex

@@ -86,20 +86,7 @@ defmodule Pleroma.Pagination do
     |> restrict(:limit, options, table_binding)
   end
 
-  @doc """
-  Unwraps a result list for a query paginated by a foreign id.
-  Usually you want to keep those foreign ids around until after pagination Link headers got generated.
-  """
-  @spec unwrap([%{id: any(), entry: any()}]) :: [any()]
-  def unwrap(list) when is_list(list), do: do_unwrap(list, [])
-
-  defp do_unwrap([%{entry: entry} | rest], acc), do: do_unwrap(rest, [entry | acc])
-  defp do_unwrap([], acc), do: Enum.reverse(acc)
-
   defp cast_params(params) do
-    # Ecto doesn’t support atom types
-    pfield = params[:pagination_field] || :id
-
     param_types = %{
       min_id: params[:id_type] || :string,
       since_id: params[:id_type] || :string,
@@ -111,54 +98,54 @@ defmodule Pleroma.Pagination do
       order_asc: :boolean
     }
 
-    params = Map.drop(params, [:id_type, :pagination_field])
+    params = Map.delete(params, :id_type)
     changeset = cast({%{}, param_types}, params, Map.keys(param_types))
-    Map.put(changeset.changes, :pagination_field, pfield)
+    changeset.changes
   end
 
-  defp order_statement(query, table_binding, :asc, %{pagination_field: fname}) do
+  defp order_statement(query, table_binding, :asc) do
     order_by(
       query,
       [{u, table_position(query, table_binding)}],
-      fragment("? asc nulls last", field(u, ^fname))
+      fragment("? asc nulls last", u.id)
     )
   end
 
-  defp order_statement(query, table_binding, :desc, %{pagination_field: fname}) do
+  defp order_statement(query, table_binding, :desc) do
     order_by(
       query,
       [{u, table_position(query, table_binding)}],
-      fragment("? desc nulls last", field(u, ^fname))
+      fragment("? desc nulls last", u.id)
     )
   end
 
-  defp restrict(query, :min_id, %{min_id: min_id, pagination_field: fname}, table_binding) do
-    where(query, [{q, table_position(query, table_binding)}], field(q, ^fname) > ^min_id)
+  defp restrict(query, :min_id, %{min_id: min_id}, table_binding) do
+    where(query, [{q, table_position(query, table_binding)}], q.id > ^min_id)
   end
 
-  defp restrict(query, :since_id, %{since_id: since_id, pagination_field: fname}, table_binding) do
-    where(query, [{q, table_position(query, table_binding)}], field(q, ^fname) > ^since_id)
+  defp restrict(query, :since_id, %{since_id: since_id}, table_binding) do
+    where(query, [{q, table_position(query, table_binding)}], q.id > ^since_id)
   end
 
-  defp restrict(query, :max_id, %{max_id: max_id, pagination_field: fname}, table_binding) do
-    where(query, [{q, table_position(query, table_binding)}], field(q, ^fname) < ^max_id)
+  defp restrict(query, :max_id, %{max_id: max_id}, table_binding) do
+    where(query, [{q, table_position(query, table_binding)}], q.id < ^max_id)
   end
 
   defp restrict(query, :order, %{skip_order: true}, _), do: query
 
   defp restrict(%{order_bys: [_ | _]} = query, :order, %{skip_extra_order: true}, _), do: query
 
-  defp restrict(query, :order, %{min_id: _} = options, table_binding) do
-    order_statement(query, table_binding, :asc, options)
+  defp restrict(query, :order, %{min_id: _}, table_binding) do
+    order_statement(query, table_binding, :asc)
   end
 
-  defp restrict(query, :order, %{max_id: _} = options, table_binding) do
-    order_statement(query, table_binding, :desc, options)
+  defp restrict(query, :order, %{max_id: _}, table_binding) do
+    order_statement(query, table_binding, :desc)
   end
 
   defp restrict(query, :order, options, table_binding) do
     dir = if options[:order_asc], do: :asc, else: :desc
-    order_statement(query, table_binding, dir, options)
+    order_statement(query, table_binding, dir)
   end
 
   defp restrict(query, :offset, %{offset: offset}, _table_binding) do

lib/pleroma/reverse_proxy.ex

@@ -10,7 +10,7 @@ defmodule Pleroma.ReverseProxy do
   @keep_resp_headers @resp_cache_headers ++
                        ~w(content-length content-type content-disposition content-encoding) ++
                        ~w(content-range accept-ranges vary expires)
-  @default_cache_control_header "public, max-age=1209600, immutable"
+  @default_cache_control_header "public, max-age=1209600"
   @valid_resp_codes [200, 206, 304]
   @max_read_duration :timer.seconds(30)
   @max_body_length :infinity
@@ -109,9 +109,7 @@ defmodule Pleroma.ReverseProxy do
     with {:ok, nil} <- @cachex.get(:failed_proxy_url_cache, url),
          {:ok, status, headers, body} <- request(method, url, req_headers, client_opts),
          :ok <-
-           check_length_constraint(
-             method,
-             body,
+           header_length_constraint(
              headers,
              Keyword.get(opts, :max_body_length, @max_body_length)
            ) do
@@ -344,9 +342,7 @@ defmodule Pleroma.ReverseProxy do
     List.keystore(headers, "content-security-policy", 0, {"content-security-policy", "sandbox"})
   end
 
-  defp check_length_constraint(_, _, _, limit) when not is_integer(limit) or limit <= 0, do: :ok
-
-  defp check_length_constraint(:head, _, headers, limit) do
+  defp header_length_constraint(headers, limit) when is_integer(limit) and limit > 0 do
     with {_, size} <- List.keyfind(headers, "content-length", 0),
          {size, _} <- Integer.parse(size),
          true <- size <= limit do
@@ -360,15 +356,7 @@ defmodule Pleroma.ReverseProxy do
     end
   end
 
-  defp check_length_constraint(_, body, _, limit) when is_binary(body) do
-    if byte_size(body) <= limit do
-      :ok
-    else
-      {:error, :body_too_large}
-    end
-  end
-
-  defp check_length_constraint(_, _, _, _), do: :ok
+  defp header_length_constraint(_, _), do: :ok
 
   defp track_failed_url(url, error, opts) do
     ttl =
@@ -378,6 +366,6 @@ defmodule Pleroma.ReverseProxy do
         nil
       end
 
-    @cachex.put(:failed_proxy_url_cache, url, true, expire: ttl)
+    @cachex.put(:failed_proxy_url_cache, url, true, ttl: ttl)
   end
 end

lib/pleroma/reverse_proxy/client.ex

@@ -0,0 +1,20 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ReverseProxy.Client do
+  @type status :: pos_integer()
+  @type header_name :: String.t()
+  @type header_value :: String.t()
+  @type headers :: [{header_name(), header_value()}]
+
+  @callback request(atom(), String.t(), headers(), String.t(), list()) ::
+              {:ok, status(), headers(), reference() | map()}
+              | {:ok, status(), headers()}
+              | {:ok, reference()}
+              | {:error, term()}
+
+  @callback stream_body(map()) :: {:ok, binary(), map()} | :done | {:error, atom() | String.t()}
+
+  @callback close(reference() | pid() | map()) :: :ok
+end

lib/pleroma/reverse_proxy/client/tesla.ex

@@ -0,0 +1,77 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ReverseProxy.Client.Tesla do
+  @behaviour Pleroma.ReverseProxy.Client
+
+  @type headers() :: [{String.t(), String.t()}]
+  @type status() :: pos_integer()
+
+  @spec request(atom(), String.t(), headers(), String.t(), keyword()) ::
+          {:ok, status(), headers}
+          | {:ok, status(), headers, map()}
+          | {:error, atom() | String.t()}
+          | no_return()
+
+  @impl true
+  def request(method, url, headers, body, opts \\ []) do
+    check_adapter()
+
+    opts = Keyword.put(opts, :body_as, :chunks)
+
+    with {:ok, response} <-
+           Pleroma.HTTP.request(
+             method,
+             url,
+             body,
+             headers,
+             opts
+           ) do
+      if is_map(response.body) and method != :head do
+        {:ok, response.status, response.headers, response.body}
+      else
+        {:ok, response.status, response.headers}
+      end
+    else
+      {:error, error} -> {:error, error}
+    end
+  end
+
+  @impl true
+  @spec stream_body(map()) ::
+          {:ok, binary(), map()} | {:error, atom() | String.t()} | :done | no_return()
+  def stream_body(%{pid: _pid, fin: true}) do
+    :done
+  end
+
+  def stream_body(client) do
+    case read_chunk!(client) do
+      {:fin, body} ->
+        {:ok, body, Map.put(client, :fin, true)}
+
+      {:nofin, part} ->
+        {:ok, part, client}
+
+      {:error, error} ->
+        {:error, error}
+    end
+  end
+
+  defp read_chunk!(%{pid: pid, stream: stream, opts: opts}) do
+    adapter = check_adapter()
+    adapter.read_chunk(pid, stream, opts)
+  end
+
+  @impl true
+  @spec close(map) :: :ok | no_return()
+  def close(%{pid: _pid}) do
+    :ok
+  end
+
+  defp check_adapter do
+    adapter = Application.get_env(:tesla, :adapter)
+
+    adapter
+  end
+end

lib/pleroma/reverse_proxy/client/wrapper.ex

@@ -0,0 +1,28 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.ReverseProxy.Client.Wrapper do
+  @moduledoc "Meta-client that calls the appropriate client from the config."
+  @behaviour Pleroma.ReverseProxy.Client
+
+  @impl true
+  def request(method, url, headers, body \\ "", opts \\ []) do
+    client().request(method, url, headers, body, opts)
+  end
+
+  @impl true
+  def stream_body(ref), do: client().stream_body(ref)
+
+  @impl true
+  def close(ref), do: client().close(ref)
+
+  defp client do
+    :tesla
+    |> Application.get_env(:adapter)
+    |> client()
+  end
+
+  defp client({Tesla.Adapter.Finch, _}), do: Pleroma.ReverseProxy.Client.Tesla
+  defp client(_), do: Pleroma.Config.get!(Pleroma.ReverseProxy.Client)
+end

lib/pleroma/search/database_search.ex

@@ -4,10 +4,8 @@
 
 defmodule Pleroma.Search.DatabaseSearch do
   alias Pleroma.Activity
-  alias Pleroma.Object
   alias Pleroma.Object.Fetcher
   alias Pleroma.Pagination
-  alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Visibility
 
@@ -18,99 +16,28 @@ defmodule Pleroma.Search.DatabaseSearch do
   @behaviour Pleroma.Search.SearchBackend
 
   def search(user, search_query, options \\ []) do
-    maybe_network = should_resolve_remote(options) && is_uri(search_query)
-    action = fn -> do_search(user, search_query, options) end
-
-    # We want to checkout a connection to not be stuck repeatedly in queue
-    # but not needlessly stall other queries while we wait for a network response
-    if !maybe_network do
-      Repo.checkout(action)
-    else
-      action.()
-    end
-  end
-
-  defp is_uri("https://" <> _), do: true
-  defp is_uri("http://" <> _), do: true
-  defp is_uri(_), do: false
-
-  defp should_resolve_remote(options) do
-    options[:resolve] && Keyword.get(options, :offset, 0) == 0
-  end
-
-  defp do_search(user, search_query, options) do
-    apid_match = is_uri(search_query) && maybe_locate_apid(search_query, user, options)
-
-    if apid_match do
-      [apid_match]
-    else
-      # Now, we can definitely checkout a conn
-      Repo.checkout(fn ->
-        fts_search(user, search_query, options)
-      end)
-    end
-  end
-
-  def maybe_locate_apid(apid, user, options) do
-    known = Object.get_by_ap_id(apid)
-
-    object_res =
-      cond do
-        known != nil -> {:ok, known}
-        should_resolve_remote(options) -> Fetcher.fetch_object_from_id(apid)
-        true -> nil
-      end
-
-    with {:ok, %Object{} = object} <- object_res,
-         %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
-         true <- activity.local || !should_restrict_local(user),
-         true <- Visibility.visible_for_user?(activity, user) do
-      activity
-    else
-      _ -> nil
-    end
-  end
-
-  defp fts_search(user, search_query, options) do
-    gin_limit = Pleroma.Config.get([__MODULE__, :gin_fuzzy_search_limit])
-
-    try do
-      if is_integer(gin_limit) do
-        Repo.transact(fn ->
-          # SET LOCAL statement cannot be parametrised it seems; safe since integer
-          Repo.query!("SET LOCAL gin_fuzzy_search_limit TO #{gin_limit}", [])
-          {:ok, do_fts_query(user, search_query, options)}
-        end)
-        |> then(fn
-          {:ok, result} -> result
-          error -> raise "#{__MODULE__}: db search transaction failed: #{inspect(error)}"
-        end)
-      else
-        do_fts_query(user, search_query, options)
-      end
-    rescue
-      _ -> []
-    end
-  end
-
-  defp do_fts_query(user, search_query, options) do
     index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin
     limit = Enum.min([Keyword.get(options, :limit), 40])
     offset = Keyword.get(options, :offset, 0)
     author = Keyword.get(options, :author)
 
-    Activity
-    |> Activity.with_preloaded_object()
-    |> Activity.restrict_deactivated_users()
-    |> restrict_public()
-    |> query_with(index_type, search_query)
-    |> maybe_restrict_local(user)
-    |> maybe_restrict_author(author)
-    |> maybe_restrict_blocked(user)
-    |> Pagination.fetch_paginated(
-      %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
-      :offset
-    )
+    try do
+      Activity
+      |> Activity.with_preloaded_object()
+      |> Activity.restrict_deactivated_users()
+      |> restrict_public()
+      |> query_with(index_type, search_query)
+      |> maybe_restrict_local(user)
+      |> maybe_restrict_author(author)
+      |> maybe_restrict_blocked(user)
+      |> Pagination.fetch_paginated(
+        %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum},
+        :offset
+      )
+      |> maybe_fetch(user, search_query)
+    rescue
+      _ -> maybe_fetch([], user, search_query)
+    end
   end
 
   def maybe_restrict_author(query, %User{} = author) do
@@ -132,46 +59,30 @@ defmodule Pleroma.Search.DatabaseSearch do
     )
   end
 
-  defp get_text_search_config() do
+  defp query_with(q, :gin, search_query) do
     %{rows: [[tsc]]} =
       Ecto.Adapters.SQL.query!(
         Pleroma.Repo,
         "select current_setting('default_text_search_config')::regconfig::oid;"
       )
 
-    tsc
-  end
-
-  defp query_with(q, :gin, search_query) do
-    tsc = get_text_search_config()
-
     from([a, o] in q,
       where:
         fragment(
-          """
-          to_tsvector(
-            ?::oid::regconfig,
-            COALESCE(?->>'summary', '') || ' ' || (?->>'content')
-          ) @@ websearch_to_tsquery(?::oid::regconfig, ?)
-          """,
+          "to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)",
           ^tsc,
           o.data,
-          o.data,
-          ^tsc,
           ^search_query
         )
     )
   end
 
   defp query_with(q, :rum, search_query) do
-    tsc = get_text_search_config()
-
     from([a, o] in q,
       where:
         fragment(
-          "? @@ websearch_to_tsquery(?::oid::regconfig, ?)",
+          "? @@ websearch_to_tsquery(?)",
           o.fts_content,
-          ^tsc,
           ^search_query
         ),
       order_by: [fragment("? <=> now()::date", o.inserted_at)]
@@ -197,4 +108,16 @@ defmodule Pleroma.Search.DatabaseSearch do
   end
 
   defp restrict_local(q), do: where(q, local: true)
+
+  def maybe_fetch(activities, user, search_query) do
+    with false <- should_restrict_local(user),
+         true <- Regex.match?(~r/https?:/, search_query),
+         {:ok, object} <- Fetcher.fetch_object_from_id(search_query),
+         %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
+         true <- Visibility.visible_for_user?(activity, user) do
+      [activity | activities]
+    else
+      _ -> activities
+    end
+  end
 end

lib/pleroma/search/elasticsearch.ex

@@ -30,10 +30,8 @@ defmodule Pleroma.Search.Elasticsearch do
     }
   end
 
-  defp maybe_fetch(:activity, search_query, options) do
-    with true <- options[:resolve],
-         0 <- Keyword.get(options, :offset, 0),
-         true <- Regex.match?(~r/https?:/, search_query),
+  defp maybe_fetch(:activity, search_query) do
+    with true <- Regex.match?(~r/https?:/, search_query),
          {:ok, object} <- Fetcher.fetch_object_from_id(search_query),
          %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]) do
       activity
@@ -53,7 +51,7 @@ defmodule Pleroma.Search.Elasticsearch do
 
     activity_fetch_task =
       Task.async(fn ->
-        maybe_fetch(:activity, String.trim(query), options)
+        maybe_fetch(:activity, String.trim(query))
       end)
 
     activity_task =

lib/pleroma/search/meilisearch.ex

@@ -87,16 +87,6 @@ defmodule Pleroma.Search.Meilisearch do
       )
   end
 
-  defp maybe_fetch(activities, user, search_query, options) do
-    located = maybe_locate_apid(user, search_query, options)
-
-    if located != nil do
-      [located | activities]
-    else
-      activities
-    end
-  end
-
   def search(user, query, options \\ []) do
     limit = Enum.min([Keyword.get(options, :limit), 40])
     offset = Keyword.get(options, :offset, 0)
@@ -116,10 +106,10 @@ defmodule Pleroma.Search.Meilisearch do
         |> maybe_restrict_local(user)
         |> maybe_restrict_author(author)
         |> maybe_restrict_blocked(user)
-        |> maybe_fetch(user, query, options)
+        |> maybe_fetch(user, query)
         |> Pleroma.Repo.all()
       rescue
-        _ -> maybe_fetch([], user, query, options)
+        _ -> maybe_fetch([], user, query)
       end
     end
   end

lib/pleroma/signature.ex

@@ -62,16 +62,20 @@ defmodule Pleroma.Signature do
     end
   end
 
-  def sign(%SigningKey{} = key, headers, opts \\ []) do
-    with {:ok, private_key_binary} <- SigningKey.private_key_binary(key) do
+  def sign(%User{} = user, headers, opts \\ []) do
+    with {:ok, private_key} <- SigningKey.private_key(user) do
       HTTPSignatures.sign(
-        %HTTPKey{key: private_key_binary},
-        key.key_id,
+        %HTTPKey{key: private_key},
+        SigningKey.local_key_id(user.ap_id),
         headers,
         opts
       )
-    else
-      _ -> raise "Tried to sign with #{key.key_id} but it has no private key!"
     end
   end
+
+  def signed_date, do: signed_date(NaiveDateTime.utc_now())
+
+  def signed_date(%NaiveDateTime{} = date) do
+    Timex.lformat!(date, "{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT", "en")
+  end
 end

lib/pleroma/stats.ex

@@ -7,6 +7,7 @@ defmodule Pleroma.Stats do
 
   import Ecto.Query
 
+  alias Pleroma.CounterCache
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Instances.Instance
@@ -106,6 +107,15 @@ defmodule Pleroma.Stats do
     }
   end
 
+  @spec get_status_visibility_count(String.t() | nil) :: map()
+  def get_status_visibility_count(instance \\ nil) do
+    if is_nil(instance) do
+      CounterCache.get_sum()
+    else
+      CounterCache.get_by_instance(instance)
+    end
+  end
+
   @impl true
   def handle_continue(:calculate_stats, _) do
     stats = calculate_stat_data()

lib/pleroma/upload.ex

@@ -82,7 +82,7 @@ defmodule Pleroma.Upload do
   def store(upload, opts \\ []) do
     opts = get_opts(opts)
 
-    with {:ok, %__MODULE__{} = upload} <- prepare_upload(upload, opts),
+    with {:ok, upload} <- prepare_upload(upload, opts),
          upload = %__MODULE__{upload | path: upload.path || "#{upload.id}/#{upload.name}"},
          {:ok, upload} <- Pleroma.Upload.Filter.filter(opts.filters, upload),
          description = Map.get(upload, :description) || "",