Merge remote-tracking branch 'pleroma/develop' into poll-notification-fixes

This commit is contained in:
Alex Gleason 2021-07-18 11:36:41 -05:00
commit 62bf6d67e3
No known key found for this signature in database
GPG key ID: 7211D1F99744FBB7
361 changed files with 8361 additions and 2945 deletions

3
.gitignore vendored
View file

@ -28,6 +28,7 @@ erl_crash.dump
# variables. # variables.
/config/*.secret.exs /config/*.secret.exs
/config/generated_config.exs /config/generated_config.exs
/config/runtime.exs
/config/*.env /config/*.env
@ -56,4 +57,4 @@ pleroma.iml
# Editor temp files # Editor temp files
/*~ /*~
/*# /*#

View file

@ -8,7 +8,9 @@ variables: &global_variables
MIX_ENV: test MIX_ENV: test
cache: &global_cache_policy cache: &global_cache_policy
key: ${CI_COMMIT_REF_SLUG} key:
files:
- mix.lock
paths: paths:
- deps - deps
- _build - _build
@ -22,20 +24,34 @@ stages:
- docker - docker
before_script: before_script:
- echo $MIX_ENV
- rm -rf _build/*/lib/pleroma
- apt-get update && apt-get install -y cmake - apt-get update && apt-get install -y cmake
- mix local.hex --force - mix local.hex --force
- mix local.rebar --force - mix local.rebar --force
- mix deps.get
- apt-get -qq update - apt-get -qq update
- apt-get install -y libmagic-dev - apt-get install -y libmagic-dev
after_script:
- rm -rf _build/*/lib/pleroma
build: build:
stage: build stage: build
only:
changes:
- "**/*.ex"
- "**/*.exs"
- "mix.lock"
script: script:
- mix deps.get
- mix compile --force - mix compile --force
spec-build: spec-build:
stage: test stage: test
only:
changes:
- "lib/pleroma/web/api_spec/**/*.ex"
- "lib/pleroma/web/api_spec.ex"
artifacts: artifacts:
paths: paths:
- spec.json - spec.json
@ -52,13 +68,17 @@ benchmark:
alias: postgres alias: postgres
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
script: script:
- mix deps.get
- mix ecto.create - mix ecto.create
- mix ecto.migrate - mix ecto.migrate
- mix pleroma.load_testing - mix pleroma.load_testing
unit-testing: unit-testing:
stage: test stage: test
only:
changes:
- "**/*.ex"
- "**/*.exs"
- "mix.lock"
retry: 2 retry: 2
cache: &testing_cache_policy cache: &testing_cache_policy
<<: *global_cache_policy <<: *global_cache_policy
@ -70,7 +90,6 @@ unit-testing:
command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
script: script:
- apt-get update && apt-get install -y libimage-exiftool-perl ffmpeg - apt-get update && apt-get install -y libimage-exiftool-perl ffmpeg
- mix deps.get
- mix ecto.create - mix ecto.create
- mix ecto.migrate - mix ecto.migrate
- mix coveralls --preload-modules - mix coveralls --preload-modules
@ -93,6 +112,11 @@ unit-testing:
unit-testing-rum: unit-testing-rum:
stage: test stage: test
only:
changes:
- "**/*.ex"
- "**/*.exs"
- "mix.lock"
retry: 2 retry: 2
cache: *testing_cache_policy cache: *testing_cache_policy
services: services:
@ -104,7 +128,6 @@ unit-testing-rum:
RUM_ENABLED: "true" RUM_ENABLED: "true"
script: script:
- apt-get update && apt-get install -y libimage-exiftool-perl ffmpeg - apt-get update && apt-get install -y libimage-exiftool-perl ffmpeg
- mix deps.get
- mix ecto.create - mix ecto.create
- mix ecto.migrate - mix ecto.migrate
- "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/" - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/"
@ -112,17 +135,40 @@ unit-testing-rum:
lint: lint:
stage: test stage: test
only:
changes:
- "**/*.ex"
- "**/*.exs"
- "mix.lock"
cache: *testing_cache_policy cache: *testing_cache_policy
script: script:
- mix format --check-formatted - mix format --check-formatted
analysis: analysis:
stage: test stage: test
only:
changes:
- "**/*.ex"
- "**/*.exs"
- "mix.lock"
cache: *testing_cache_policy cache: *testing_cache_policy
script: script:
- mix deps.get
- mix credo --strict --only=warnings,todo,fixme,consistency,readability - mix credo --strict --only=warnings,todo,fixme,consistency,readability
cycles:
stage: test
image: elixir:1.11
only:
changes:
- "**/*.ex"
- "**/*.exs"
- "mix.lock"
cache: {}
script:
- mix deps.get
- mix compile
- mix xref graph --format cycles --label compile | awk '{print $0} END{exit ($0 != "No cycles found")}'
docs-deploy: docs-deploy:
stage: deploy stage: deploy
cache: *testing_cache_policy cache: *testing_cache_policy
@ -175,8 +221,8 @@ spec-deploy:
- apk add curl - apk add curl
script: script:
- curl -X POST -F"token=$API_DOCS_PIPELINE_TRIGGER" -F'ref=master' -F"variables[BRANCH]=$CI_COMMIT_REF_NAME" -F"variables[JOB_REF]=$CI_JOB_ID" https://git.pleroma.social/api/v4/projects/1130/trigger/pipeline - curl -X POST -F"token=$API_DOCS_PIPELINE_TRIGGER" -F'ref=master' -F"variables[BRANCH]=$CI_COMMIT_REF_NAME" -F"variables[JOB_REF]=$CI_JOB_ID" https://git.pleroma.social/api/v4/projects/1130/trigger/pipeline
stop_review_app: stop_review_app:
image: alpine:3.9 image: alpine:3.9
stage: deploy stage: deploy
@ -235,7 +281,7 @@ amd64-musl:
stage: release stage: release
artifacts: *release-artifacts artifacts: *release-artifacts
only: *release-only only: *release-only
image: elixir:1.10.3-alpine image: elixir:1.10.3-alpine
cache: *release-cache cache: *release-cache
variables: *release-variables variables: *release-variables
before_script: &before-release-musl before_script: &before-release-musl
@ -393,4 +439,4 @@ docker-adhoc:
tags: tags:
- dind - dind
only: only:
- /^build-docker/.*$/@pleroma/pleroma - /^build-docker/.*$/@pleroma/pleroma

View file

@ -4,6 +4,51 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
## Unreleased
### Changed
- **Breaking:** Configuration: `:chat, enabled` moved to `:shout, enabled` and `:instance, chat_limit` moved to `:shout, limit`
- Support for Erlang/OTP 24
- The `application` metadata returned with statuses is no longer hardcoded. Apps that want to display these details will now have valid data for new posts after this change.
- HTTPSecurityPlug now sends a response header to opt out of Google's FLoC (Federated Learning of Cohorts) targeted advertising.
- Email address is now returned if requesting user is the owner of the user account so it can be exposed in client and FE user settings UIs.
- Improved Twittercard and OpenGraph meta tag generation including thumbnails and image dimension metadata when available.
- AdminAPI: sort users so the newest are at the top.
- ActivityPub Client-to-Server(C2S): Limitation on the type of Activity/Object are lifted as they are now passed through ObjectValidators
### Added
- MRF (`FollowBotPolicy`): New MRF Policy which makes a designated local Bot account attempt to follow all users in public Notes received by your instance. Users who require approving follower requests or have #nobot in their profile are excluded.
- Return OAuth token `id` (primary key) in POST `/oauth/token`.
- AdminAPI: return `created_at` date with users.
- `AnalyzeMetadata` upload filter for extracting image/video attachment dimensions and generating blurhashes for images. Blurhashes for videos are not generated at this time.
- Attachment dimensions and blurhashes are federated when available.
- Pinned posts federation
### Fixed
- Don't crash so hard when email settings are invalid.
- Checking activated Upload Filters for required commands.
- Remote users can no longer reappear after being deleted.
- Deactivated users may now be deleted.
- Mix task `pleroma.database prune_objects`
- Linkify: Parsing crash with URLs ending in unbalanced closed paren, no path separator, and no query parameters
### Removed
- **Breaking**: Remove deprecated `/api/qvitter/statuses/notifications/read` (replaced by `/api/v1/pleroma/notifications/read`)
## Unreleased (Patch)
### Fixed
- Try to save exported ConfigDB settings (migrate_from_db) in the system temp directory if default location is not writable.
- Uploading custom instance thumbnail via AdminAPI/AdminFE generated invalid URL to the image
- Applying ConcurrentLimiter settings via AdminAPI
- User login failures if their `notification_settings` were in a NULL state.
- Mix task `pleroma.user delete_activities` query transaction timeout is now :infinity
- MRF (`SimplePolicy`): Embedded objects are now checked. If any embedded object would be rejected, its parent is rejected. This fixes Announces leaking posts from blocked domains.
- Fixed some Markdown issues, including trailing slash in links.
## [2.3.0] - 2020-03-01 ## [2.3.0] - 2020-03-01
### Security ### Security
@ -18,6 +63,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- **Breaking**: Changed `mix pleroma.user toggle_confirmed` to `mix pleroma.user confirm` - **Breaking**: Changed `mix pleroma.user toggle_confirmed` to `mix pleroma.user confirm`
- **Breaking**: Changed `mix pleroma.user toggle_activated` to `mix pleroma.user activate/deactivate` - **Breaking**: Changed `mix pleroma.user toggle_activated` to `mix pleroma.user activate/deactivate`
- **Breaking:** NSFW hashtag is no longer added on sensitive posts
- Polls now always return a `voters_count`, even if they are single-choice. - Polls now always return a `voters_count`, even if they are single-choice.
- Admin Emails: The ap id is used as the user link in emails now. - Admin Emails: The ap id is used as the user link in emails now.
- Improved registration workflow for email confirmation and account approval modes. - Improved registration workflow for email confirmation and account approval modes.
@ -44,6 +90,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Pleroma API: Reroute `/api/pleroma/*` to `/api/v1/pleroma/*` - Pleroma API: Reroute `/api/pleroma/*` to `/api/v1/pleroma/*`
</details> </details>
- Improved hashtag timeline performance (requires a background migration).
### Added ### Added
@ -67,6 +114,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
<details> <details>
<summary>API Changes</summary> <summary>API Changes</summary>
- Admin API: (`GET /api/pleroma/admin/users`) filter users by `unconfirmed` status and `actor_type`. - Admin API: (`GET /api/pleroma/admin/users`) filter users by `unconfirmed` status and `actor_type`.
- Admin API: OpenAPI spec for the user-related operations
- Pleroma API: `GET /api/v2/pleroma/chats` added. It is exactly like `GET /api/v1/pleroma/chats` except supports pagination. - Pleroma API: `GET /api/v2/pleroma/chats` added. It is exactly like `GET /api/v1/pleroma/chats` except supports pagination.
- Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending. - Pleroma API: Add `idempotency_key` to the chat message entity that can be used for optimistic message sending.
- Pleroma API: (`GET /api/v1/pleroma/federation_status`) Add a way to get a list of unreachable instances. - Pleroma API: (`GET /api/v1/pleroma/federation_status`) Add a way to get a list of unreachable instances.
@ -498,7 +546,6 @@ switched to a new configuration mechanism, however it was not officially removed
- Static-FE: Fix remote posts not being sanitized - Static-FE: Fix remote posts not being sanitized
### Fixed ### Fixed
=======
- Rate limiter crashes when there is no explicitly specified ip in the config - Rate limiter crashes when there is no explicitly specified ip in the config
- 500 errors when no `Accept` header is present if Static-FE is enabled - 500 errors when no `Accept` header is present if Static-FE is enabled
- Instance panel not being updated immediately due to wrong `Cache-Control` headers - Instance panel not being updated immediately due to wrong `Cache-Control` headers

View file

@ -33,7 +33,7 @@ ARG DATA=/var/lib/pleroma
RUN echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\ RUN echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories &&\
apk update &&\ apk update &&\
apk add exiftool imagemagick libmagic ncurses postgresql-client &&\ apk add exiftool ffmpeg imagemagick libmagic ncurses postgresql-client &&\
adduser --system --shell /bin/false --home ${HOME} pleroma &&\ adduser --system --shell /bin/false --home ${HOME} pleroma &&\
mkdir -p ${DATA}/uploads &&\ mkdir -p ${DATA}/uploads &&\
mkdir -p ${DATA}/static &&\ mkdir -p ${DATA}/static &&\

View file

@ -35,6 +35,9 @@ Currently Pleroma is not packaged by any OS/Distros, but if you want to package
### Docker ### Docker
While we dont provide docker files, other people have written very good ones. Take a look at <https://github.com/angristan/docker-pleroma> or <https://glitch.sh/sn0w/pleroma-docker>. While we dont provide docker files, other people have written very good ones. Take a look at <https://github.com/angristan/docker-pleroma> or <https://glitch.sh/sn0w/pleroma-docker>.
### Raspberry Pi
Community maintained Raspberry Pi image that you can flash and run Pleroma on your Raspberry Pi. Available here <https://github.com/guysoft/PleromaPi>.
### Compilation Troubleshooting ### Compilation Troubleshooting
If you ever encounter compilation issues during the updating of Pleroma, you can try these commands and see if they fix things: If you ever encounter compilation issues during the updating of Pleroma, you can try these commands and see if they fix things:
@ -50,5 +53,5 @@ If you are not developing Pleroma, it is better to use the OTP release, which co
- Latest Git revision: <https://docs-develop.pleroma.social> - Latest Git revision: <https://docs-develop.pleroma.social>
## Community Channels ## Community Channels
* IRC: **#pleroma** and **#pleroma-dev** on freenode, webchat is available at <https://irc.pleroma.social> * IRC: **#pleroma** and **#pleroma-dev** on libera.chat, webchat is available at <https://irc.pleroma.social>
* Matrix: <https://matrix.to/#/#freenode_#pleroma:matrix.org> and <https://matrix.to/#/#freenode_#pleroma-dev:matrix.org> * Matrix: [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) and [#pleroma-dev:libera.chat](https://matrix.to/#/#pleroma-dev:libera.chat)

View file

@ -299,7 +299,7 @@ defmodule Pleroma.LoadTesting.Activities do
"url" => [ "url" => [
%{ %{
"href" => "href" =>
"#{Pleroma.Web.base_url()}/media/b1b873552422a07bf53af01f3c231c841db4dfc42c35efde681abaf0f2a4eab7.jpg", "#{Pleroma.Web.Endpoint.url()}/media/b1b873552422a07bf53af01f3c231c841db4dfc42c35efde681abaf0f2a4eab7.jpg",
"mediaType" => "image/jpeg", "mediaType" => "image/jpeg",
"type" => "Link" "type" => "Link"
} }

View file

@ -1,4 +1,4 @@
use Mix.Config import Config
# We don't run a server during test. If one is required, # We don't run a server during test. If one is required,
# you can enable the server option below. # you can enable the server option below.

View file

@ -41,7 +41,7 @@
# #
# This configuration file is loaded before any dependency and # This configuration file is loaded before any dependency and
# is restricted to this project. # is restricted to this project.
use Mix.Config import Config
# General application configuration # General application configuration
config :pleroma, ecto_repos: [Pleroma.Repo] config :pleroma, ecto_repos: [Pleroma.Repo]
@ -190,7 +190,6 @@ config :pleroma, :instance,
instance_thumbnail: "/instance/thumbnail.jpeg", instance_thumbnail: "/instance/thumbnail.jpeg",
limit: 5_000, limit: 5_000,
description_limit: 5_000, description_limit: 5_000,
chat_limit: 5_000,
remote_limit: 100_000, remote_limit: 100_000,
upload_limit: 16_000_000, upload_limit: 16_000_000,
avatar_upload_limit: 2_000_000, avatar_upload_limit: 2_000_000,
@ -391,6 +390,11 @@ config :pleroma, :mrf_keyword,
federated_timeline_removal: [], federated_timeline_removal: [],
replace: [] replace: []
config :pleroma, :mrf_hashtag,
sensitive: ["nsfw"],
reject: [],
federated_timeline_removal: []
config :pleroma, :mrf_subchain, match_actor: %{} config :pleroma, :mrf_subchain, match_actor: %{}
config :pleroma, :mrf_activity_expiration, days: 365 config :pleroma, :mrf_activity_expiration, days: 365
@ -404,6 +408,8 @@ config :pleroma, :mrf_object_age,
threshold: 604_800, threshold: 604_800,
actions: [:delist, :strip_followers] actions: [:delist, :strip_followers]
config :pleroma, :mrf_follow_bot, follower_nickname: nil
config :pleroma, :rich_media, config :pleroma, :rich_media,
enabled: true, enabled: true,
ignore_hosts: [], ignore_hosts: [],
@ -450,7 +456,9 @@ config :pleroma, :media_preview_proxy,
image_quality: 85, image_quality: 85,
min_content_length: 100 * 1024 min_content_length: 100 * 1024
config :pleroma, :chat, enabled: true config :pleroma, :shout,
enabled: true,
limit: 5_000
config :phoenix, :format_encoders, json: Jason config :phoenix, :format_encoders, json: Jason
@ -655,6 +663,10 @@ config :pleroma, :oauth2,
config :pleroma, :database, rum_enabled: false config :pleroma, :database, rum_enabled: false
config :pleroma, :features, improved_hashtag_timeline: :auto
config :pleroma, :populate_hashtags_table, fault_rate_allowance: 0.01
config :pleroma, :env, Mix.env() config :pleroma, :env, Mix.env()
config :http_signatures, config :http_signatures,

View file

@ -1,4 +1,4 @@
use Mix.Config import Config
websocket_config = [ websocket_config = [
path: "/websocket", path: "/websocket",
@ -459,6 +459,42 @@ config :pleroma, :config_description, [
} }
] ]
}, },
%{
group: :pleroma,
key: :features,
type: :group,
description: "Customizable features",
children: [
%{
key: :improved_hashtag_timeline,
type: {:dropdown, :atom},
description:
"Setting to force toggle / force disable improved hashtags timeline. `:enabled` forces hashtags to be fetched from `hashtags` table for hashtags timeline. `:disabled` forces object-embedded hashtags to be used (slower). Keep it `:auto` for automatic behaviour (it is auto-set to `:enabled` [unless overridden] when HashtagsTableMigrator completes).",
suggestions: [:auto, :enabled, :disabled]
}
]
},
%{
group: :pleroma,
key: :populate_hashtags_table,
type: :group,
description: "`populate_hashtags_table` background migration settings",
children: [
%{
key: :fault_rate_allowance,
type: :float,
description:
"Max accepted rate of objects that failed in the migration. Any value from 0.0 which tolerates no errors to 1.0 which will enable the feature even if hashtags transfer failed for all records.",
suggestions: [0.01]
},
%{
key: :sleep_interval_ms,
type: :integer,
description:
"Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances)."
}
]
},
%{ %{
group: :pleroma, group: :pleroma,
key: :instance, key: :instance,
@ -508,14 +544,6 @@ config :pleroma, :config_description, [
5_000 5_000
] ]
}, },
%{
key: :chat_limit,
type: :integer,
description: "Character limit of the instance chat messages",
suggestions: [
5_000
]
},
%{ %{
key: :remote_limit, key: :remote_limit,
type: :integer, type: :integer,
@ -646,7 +674,8 @@ config :pleroma, :config_description, [
%{ %{
key: :allow_relay, key: :allow_relay,
type: :boolean, type: :boolean,
description: "Enable Pleroma's Relay, which makes it possible to follow a whole instance" description:
"Permits remote instances to subscribe to all public posts of your instance. (Important!) This may increase the visibility of your instance."
}, },
%{ %{
key: :public, key: :public,
@ -1146,7 +1175,6 @@ config :pleroma, :config_description, [
alwaysShowSubjectInput: true, alwaysShowSubjectInput: true,
background: "/static/aurora_borealis.jpg", background: "/static/aurora_borealis.jpg",
collapseMessageWithSubject: false, collapseMessageWithSubject: false,
disableChat: false,
greentext: false, greentext: false,
hideFilteredStatuses: false, hideFilteredStatuses: false,
hideMutedPosts: false, hideMutedPosts: false,
@ -1193,12 +1221,6 @@ config :pleroma, :config_description, [
description: description:
"When a message has a subject (aka Content Warning), collapse it by default" "When a message has a subject (aka Content Warning), collapse it by default"
}, },
%{
key: :disableChat,
label: "PleromaFE Chat",
type: :boolean,
description: "Disables PleromaFE Chat component"
},
%{ %{
key: :greentext, key: :greentext,
label: "Greentext", label: "Greentext",
@ -2616,13 +2638,22 @@ config :pleroma, :config_description, [
}, },
%{ %{
group: :pleroma, group: :pleroma,
key: :chat, key: :shout,
type: :group, type: :group,
description: "Pleroma chat settings", description: "Pleroma shout settings",
children: [ children: [
%{ %{
key: :enabled, key: :enabled,
type: :boolean type: :boolean,
description: "Enables the backend Shoutbox chat feature."
},
%{
key: :limit,
type: :integer,
description: "Shout message character limit.",
suggestions: [
5_000
]
} }
] ]
}, },
@ -2906,6 +2937,23 @@ config :pleroma, :config_description, [
} }
] ]
}, },
%{
group: :pleroma,
key: :mrf_follow_bot,
tab: :mrf,
related_policy: "Pleroma.Web.ActivityPub.MRF.FollowBotPolicy",
label: "MRF FollowBot Policy",
type: :group,
description: "Automatically follows newly discovered accounts.",
children: [
%{
key: :follower_nickname,
type: :string,
description: "The name of the bot account to use for following newly discovered users.",
suggestions: ["followbot"]
}
]
},
%{ %{
group: :pleroma, group: :pleroma,
key: :modules, key: :modules,

View file

@ -1,4 +1,4 @@
use Mix.Config import Config
# For development, we disable any cache and enable # For development, we disable any cache and enable
# debugging and code reloading. # debugging and code reloading.
@ -54,6 +54,10 @@ config :pleroma, Pleroma.Repo,
config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true config :pleroma, Pleroma.Web.ApiSpec.CastAndValidate, strict: true
# Reduce recompilation time
# https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects
config :phoenix, :plug_init_mode, :runtime
if File.exists?("./config/dev.secret.exs") do if File.exists?("./config/dev.secret.exs") do
import_config "dev.secret.exs" import_config "dev.secret.exs"
else else

View file

@ -1,4 +1,4 @@
use Mix.Config import Config
config :pleroma, Pleroma.Web.Endpoint, config :pleroma, Pleroma.Web.Endpoint,
http: [ http: [

View file

@ -1,4 +1,4 @@
use Mix.Config import Config
# For production, we often load configuration from external # For production, we often load configuration from external
# sources, such as your system environment. For this reason, # sources, such as your system environment. For this reason,
@ -63,7 +63,12 @@ config :logger, :ex_syslogger, level: :info
# Finally import the config/prod.secret.exs # Finally import the config/prod.secret.exs
# which should be versioned separately. # which should be versioned separately.
import_config "prod.secret.exs" if File.exists?("./config/prod.secret.exs") do
import_config "prod.secret.exs"
else
"`config/prod.secret.exs` not found. You may want to create one by running `mix pleroma.instance gen`"
|> IO.warn([])
end
if File.exists?("./config/prod.exported_from_db.secret.exs"), if File.exists?("./config/prod.exported_from_db.secret.exs"),
do: import_config("prod.exported_from_db.secret.exs") do: import_config("prod.exported_from_db.secret.exs")

View file

@ -1,4 +1,4 @@
use Mix.Config import Config
# We don't run a server during test. If one is required, # We don't run a server during test. If one is required,
# you can enable the server option below. # you can enable the server option below.
@ -133,6 +133,10 @@ config :pleroma, :side_effects,
ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock, ap_streamer: Pleroma.Web.ActivityPub.ActivityPubMock,
logger: Pleroma.LoggerMock logger: Pleroma.LoggerMock
# Reduce recompilation time
# https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects
config :phoenix, :plug_init_mode, :runtime
if File.exists?("./config/test.secret.exs") do if File.exists?("./config/test.secret.exs") do
import_config "test.secret.exs" import_config "test.secret.exs"
else else

View file

@ -32,16 +32,20 @@
config :pleroma, configurable_from_database: false config :pleroma, configurable_from_database: false
``` ```
To delete transferred settings from database optional flag `-d` can be used. `<env>` is `prod` by default. Options:
- `<path>` - where to save migrated config. E.g. `--path=/tmp`. If file saved into non standart folder, you must manually copy file into directory where Pleroma can read it. For OTP install path will be `PLEROMA_CONFIG_PATH` or `/etc/pleroma`. For installation from source - `config` directory in the pleroma folder.
- `<env>` - environment, for which is migrated config. By default is `prod`.
- To delete transferred settings from database optional flag `-d` can be used
=== "OTP" === "OTP"
```sh ```sh
./bin/pleroma_ctl config migrate_from_db [--env=<env>] [-d] ./bin/pleroma_ctl config migrate_from_db [--env=<env>] [-d] [--path=<path>]
``` ```
=== "From Source" === "From Source"
```sh ```sh
mix pleroma.config migrate_from_db [--env=<env>] [-d] mix pleroma.config migrate_from_db [--env=<env>] [-d] [--path=<path>]
``` ```
## Dump all of the config settings defined in the database ## Dump all of the config settings defined in the database

View file

@ -8,9 +8,10 @@ For from source installations Pleroma configuration works by first importing the
To add configuration to your config file, you can copy it from the base config. The latest version of it can be viewed [here](https://git.pleroma.social/pleroma/pleroma/blob/develop/config/config.exs). You can also use this file if you don't know how an option is supposed to be formatted. To add configuration to your config file, you can copy it from the base config. The latest version of it can be viewed [here](https://git.pleroma.social/pleroma/pleroma/blob/develop/config/config.exs). You can also use this file if you don't know how an option is supposed to be formatted.
## :chat ## :shout
* `enabled` - Enables the backend chat. Defaults to `true`. * `enabled` - Enables the backend Shoutbox chat feature. Defaults to `true`.
* `limit` - Shout character limit. Defaults to `5_000`
## :instance ## :instance
* `name`: The instances name. * `name`: The instances name.
@ -19,7 +20,6 @@ To add configuration to your config file, you can copy it from the base config.
* `description`: The instances description, can be seen in nodeinfo and ``/api/v1/instance``. * `description`: The instances description, can be seen in nodeinfo and ``/api/v1/instance``.
* `limit`: Posts character limit (CW/Subject included in the counter). * `limit`: Posts character limit (CW/Subject included in the counter).
* `description_limit`: The character limit for image descriptions. * `description_limit`: The character limit for image descriptions.
* `chat_limit`: Character limit of the instance chat messages.
* `remote_limit`: Hard character limit beyond which remote posts will be dropped. * `remote_limit`: Hard character limit beyond which remote posts will be dropped.
* `upload_limit`: File size limit of uploads (except for avatar, background, banner). * `upload_limit`: File size limit of uploads (except for avatar, background, banner).
* `avatar_upload_limit`: File size limit of users profile avatars. * `avatar_upload_limit`: File size limit of users profile avatars.
@ -37,7 +37,7 @@ To add configuration to your config file, you can copy it from the base config.
* `federating`: Enable federation with other instances. * `federating`: Enable federation with other instances.
* `federation_incoming_replies_max_depth`: Max. depth of reply-to activities fetching on incoming federation, to prevent out-of-memory situations while fetching very long threads. If set to `nil`, threads of any depth will be fetched. Lower this value if you experience out-of-memory crashes. * `federation_incoming_replies_max_depth`: Max. depth of reply-to activities fetching on incoming federation, to prevent out-of-memory situations while fetching very long threads. If set to `nil`, threads of any depth will be fetched. Lower this value if you experience out-of-memory crashes.
* `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it. * `federation_reachability_timeout_days`: Timeout (in days) of each external federation target being unreachable prior to pausing federating to it.
* `allow_relay`: Enable Pleromas Relay, which makes it possible to follow a whole instance. * `allow_relay`: Permits remote instances to subscribe to all public posts of your instance. This may increase the visibility of your instance.
* `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. Note that there is a dependent setting restricting or allowing unauthenticated access to specific resources, see `restrict_unauthenticated` for more details. * `public`: Makes the client API in authenticated mode-only except for user-profiles. Useful for disabling the Local Timeline and The Whole Known Network. Note that there is a dependent setting restricting or allowing unauthenticated access to specific resources, see `restrict_unauthenticated` for more details.
* `quarantined_instances`: List of ActivityPub instances where private (DMs, followers-only) activities will not be send. * `quarantined_instances`: List of ActivityPub instances where private (DMs, followers-only) activities will not be send.
* `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML). * `allowed_post_formats`: MIME-type list of formats allowed to be posted (transformed into HTML).
@ -65,6 +65,13 @@ To add configuration to your config file, you can copy it from the base config.
* `show_reactions`: Let favourites and emoji reactions be viewed through the API (default: `true`). * `show_reactions`: Let favourites and emoji reactions be viewed through the API (default: `true`).
* `password_reset_token_validity`: The time after which reset tokens aren't accepted anymore, in seconds (default: one day). * `password_reset_token_validity`: The time after which reset tokens aren't accepted anymore, in seconds (default: one day).
## :database
* `improved_hashtag_timeline`: Setting to force toggle / force disable improved hashtags timeline. `:enabled` forces hashtags to be fetched from `hashtags` table for hashtags timeline. `:disabled` forces object-embedded hashtags to be used (slower). Keep it `:auto` for automatic behaviour (it is auto-set to `:enabled` [unless overridden] when HashtagsTableMigrator completes).
## Background migrations
* `populate_hashtags_table/sleep_interval_ms`: Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances).
* `populate_hashtags_table/fault_rate_allowance`: Max rate of failed objects to actually processed objects in order to enable the feature (any value from 0.0 which tolerates no errors to 1.0 which will enable the feature even if hashtags transfer failed for all records).
## Welcome ## Welcome
* `direct_message`: - welcome message sent as a direct message. * `direct_message`: - welcome message sent as a direct message.
* `enabled`: Enables the send a direct message to a newly registered user. Defaults to `false`. * `enabled`: Enables the send a direct message to a newly registered user. Defaults to `false`.
@ -117,6 +124,7 @@ To add configuration to your config file, you can copy it from the base config.
* `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)). * `Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy`: Rejects or delists posts based on their age when received. (See [`:mrf_object_age`](#mrf_object_age)).
* `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.Workers.PurgeExpiredActivity` to be enabled for processing the scheduled delections. * `Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy`: Sets a default expiration on all posts made by users of the local instance. Requires `Pleroma.Workers.PurgeExpiredActivity` to be enabled for processing the scheduled delections.
* `Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy`: Makes all bot posts to disappear from public timelines. * `Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy`: Makes all bot posts to disappear from public timelines.
* `Pleroma.Web.ActivityPub.MRF.FollowBotPolicy`: Automatically follows newly discovered users from the specified bot account. Local accounts, locked accounts, and users with "#nobot" in their bio are respected and excluded from being followed.
* `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo). * `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo).
* `transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value. * `transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value.
@ -203,6 +211,21 @@ config :pleroma, :mrf_user_allowlist, %{
* `days`: Default global expiration time for all local Create activities (in days) * `days`: Default global expiration time for all local Create activities (in days)
#### :mrf_hashtag
* `sensitive`: List of hashtags to mark activities as sensitive (default: `nsfw`)
* `federated_timeline_removal`: List of hashtags to remove activities from the federated timeline (aka TWNK)
* `reject`: List of hashtags to reject activities from
Notes:
- The hashtags in the configuration do not have a leading `#`.
- This MRF Policy is always enabled, if you want to disable it you have to set empty lists
#### :mrf_follow_bot
* `follower_nickname`: The name of the bot account to use for following newly discovered users. Using `followbot` or similar is strongly suggested.
### :activitypub ### :activitypub
* `unfollow_blocked`: Whether blocks result in people getting unfollowed * `unfollow_blocked`: Whether blocks result in people getting unfollowed
* `outgoing_blocks`: Whether to federate blocks to other instances * `outgoing_blocks`: Whether to federate blocks to other instances

View file

@ -82,7 +82,7 @@ For example, here is a sample policy module which rewrites all messages to "new
```elixir ```elixir
defmodule Pleroma.Web.ActivityPub.MRF.RewritePolicy do defmodule Pleroma.Web.ActivityPub.MRF.RewritePolicy do
@moduledoc "MRF policy which rewrites all Notes to have 'new message content'." @moduledoc "MRF policy which rewrites all Notes to have 'new message content'."
@behaviour Pleroma.Web.ActivityPub.MRF @behaviour Pleroma.Web.ActivityPub.MRF.Policy
# Catch messages which contain Note objects with actual data to filter. # Catch messages which contain Note objects with actual data to filter.
# Capture the object as `object`, the message content as `content` and the # Capture the object as `object`, the message content as `content` and the

View file

@ -38,6 +38,7 @@ Has these additional fields under the `pleroma` object:
- `thread_muted`: true if the thread the post belongs to is muted - `thread_muted`: true if the thread the post belongs to is muted
- `emoji_reactions`: A list with emoji / reaction maps. The format is `{name: "☕", count: 1, me: true}`. Contains no information about the reacting users, for that use the `/statuses/:id/reactions` endpoint. - `emoji_reactions`: A list with emoji / reaction maps. The format is `{name: "☕", count: 1, me: true}`. Contains no information about the reacting users, for that use the `/statuses/:id/reactions` endpoint.
- `parent_visible`: If the parent of this post is visible to the user or not. - `parent_visible`: If the parent of this post is visible to the user or not.
- `pinned_at`: a datetime (iso8601) when status was pinned, `null` otherwise.
## Scheduled statuses ## Scheduled statuses
@ -255,9 +256,29 @@ This information is returned in the `/api/v1/accounts/verify_credentials` endpoi
*Pleroma supports refreshing tokens.* *Pleroma supports refreshing tokens.*
`POST /oauth/token` ### POST `/oauth/token`
Post here request with `grant_type=refresh_token` to obtain new access token. Returns an access token. You can obtain access tokens for a user in a few additional ways.
#### Refreshing a token
To obtain a new access token from a refresh token, pass `grant_type=refresh_token` with the following extra parameters:
- `refresh_token`: The refresh token.
#### Getting a token with a password
To obtain a token from a user's password, pass `grant_type=password` with the following extra parameters:
- `username`: Username to authenticate.
- `password`: The user's password.
#### Response body
Additional fields are returned in the response:
- `id`: The primary key of this token in Pleroma's database.
- `me` (user tokens only): The ActivityPub ID of the user who owns the token.
## Account Registration ## Account Registration

View file

@ -300,7 +300,7 @@ See [Admin-API](admin_api.md)
* Note: Behaves exactly the same as `POST /api/v1/upload`. * Note: Behaves exactly the same as `POST /api/v1/upload`.
Can only accept images - any attempt to upload non-image files will be met with `HTTP 415 Unsupported Media Type`. Can only accept images - any attempt to upload non-image files will be met with `HTTP 415 Unsupported Media Type`.
## `/api/v1/pleroma/notification_settings` ## `/api/pleroma/notification_settings`
### Updates user notification settings ### Updates user notification settings
* Method `PUT` * Method `PUT`
* Authentication: required * Authentication: required

View file

@ -20,7 +20,7 @@ The default front-end used by Pleroma is Pleroma-FE. You can find more informati
### Mastodon interface ### Mastodon interface
If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too! If the Pleroma interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too!
Just add a "/web" after your instance url (e.g. <https://pleroma.soycaf.com/web>) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC! Just add a "/web" after your instance url (e.g. <https://pleroma.soykaf.com/web>) and you'll end on the Mastodon web interface, but with a Pleroma backend! MAGIC!
The Mastodon interface is from the Glitch-soc fork. For more information on the Mastodon interface you can check the [Mastodon](https://docs.joinmastodon.org/) and [Glitch-soc](https://glitch-soc.github.io/docs/) documentation. The Mastodon interface is from the Glitch-soc fork. For more information on the Mastodon interface you can check the [Mastodon](https://docs.joinmastodon.org/) and [Glitch-soc](https://glitch-soc.github.io/docs/) documentation.
Remember, what you see is only the frontend part of Mastodon, the backend is still Pleroma. Remember, what you see is only the frontend part of Mastodon, the backend is still Pleroma.

View file

@ -5,25 +5,7 @@ This guide is a step-by-step installation guide for Alpine Linux. The instructio
It assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.linode.com/docs/tools-reference/custom-kernels-distros/install-alpine-linux-on-your-linode/#configuration). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su -l <username> -s $SHELL -c 'command'` instead. It assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.linode.com/docs/tools-reference/custom-kernels-distros/install-alpine-linux-on-your-linode/#configuration). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su -l <username> -s $SHELL -c 'command'` instead.
### Required packages {! backend/installation/generic_dependencies.include !}
* `postgresql`
* `elixir`
* `erlang`
* `erlang-parsetools`
* `erlang-xmerl`
* `git`
* `file-dev`
* Development Tools
* `cmake`
#### Optional packages used in this guide
* `nginx` (preferred, example configs for other reverse proxies can be found in the repo)
* `certbot` (or any other ACME client for Lets Encrypt certificates)
* `ImageMagick`
* `ffmpeg`
* `exiftool`
### Prepare the system ### Prepare the system
@ -117,7 +99,7 @@ cd /opt/pleroma
sudo -Hu pleroma mix deps.get sudo -Hu pleroma mix deps.get
``` ```
* Generate the configuration: `sudo -Hu pleroma mix pleroma.instance gen` * Generate the configuration: `sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen`
* Answer with `yes` if it asks you to install `rebar3`. * Answer with `yes` if it asks you to install `rebar3`.
* This may take some time, because parts of pleroma get compiled first. * This may take some time, because parts of pleroma get compiled first.
* After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`. * After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`.
@ -240,4 +222,4 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress
## Questions ## Questions
Questions about the installation or didnt it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**. Questions about the installation or didnt it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC.

View file

@ -92,7 +92,7 @@ cd /opt/pleroma
sudo -Hu pleroma mix deps.get sudo -Hu pleroma mix deps.get
``` ```
* Generate the configuration: `sudo -Hu pleroma mix pleroma.instance gen` * Generate the configuration: `sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen`
* Answer with `yes` if it asks you to install `rebar3`. * Answer with `yes` if it asks you to install `rebar3`.
* This may take some time, because parts of pleroma get compiled first. * This may take some time, because parts of pleroma get compiled first.
* After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`. * After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`.
@ -215,4 +215,4 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress
## Questions ## Questions
Questions about the installation or didnt it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**. Questions about the installation or didnt it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC.

View file

@ -1,27 +1,9 @@
# Installing on Debian Based Distributions # Installing on Debian Based Distributions
## Installation ## Installation
This guide will assume you are on Debian Stretch. This guide should also work with Ubuntu 16.04 and 18.04. It also assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.digitalocean.com/community/tutorials/how-to-add-delete-and-grant-sudo-privileges-to-users-on-a-debian-vps). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su <username> -s $SHELL -c 'command'` instead. This guide will assume you are on Debian 11 (“bullseye”) or later. This guide should also work with Ubuntu 18.04 (“Bionic Beaver”) and later. It also assumes that you have administrative rights, either as root or a user with [sudo permissions](https://www.digitalocean.com/community/tutorials/how-to-add-delete-and-grant-sudo-privileges-to-users-on-a-debian-vps). If you want to run this guide with root, ignore the `sudo` at the beginning of the lines, unless it calls a user like `sudo -Hu pleroma`; in this case, use `su <username> -s $SHELL -c 'command'` instead.
### Required packages {! backend/installation/generic_dependencies.include !}
* `postgresql` (9.6+, Ubuntu 16.04 comes with 9.5, you can get a newer version from [here](https://www.postgresql.org/download/linux/ubuntu/))
* `postgresql-contrib` (9.6+, same situtation as above)
* `elixir` (1.8+, Follow the guide to install from the Erlang Solutions repo or use [asdf](https://github.com/asdf-vm/asdf) as the pleroma user)
* `erlang-dev`
* `erlang-nox`
* `libmagic-dev`
* `git`
* `build-essential`
* `cmake`
#### Optional packages used in this guide
* `nginx` (preferred, example configs for other reverse proxies can be found in the repo)
* `certbot` (or any other ACME client for Lets Encrypt certificates)
* `ImageMagick`
* `ffmpeg`
* `exiftool`
### Prepare the system ### Prepare the system
@ -40,20 +22,14 @@ sudo apt install git build-essential postgresql postgresql-contrib cmake libmagi
### Install Elixir and Erlang ### Install Elixir and Erlang
* Download and add the Erlang repository: * Install Elixir and Erlang (you might need to use backports or [asdf](https://github.com/asdf-vm/asdf) on old systems):
```shell
wget -P /tmp/ https://packages.erlang-solutions.com/erlang-solutions_2.0_all.deb
sudo dpkg -i /tmp/erlang-solutions_2.0_all.deb
```
* Install Elixir and Erlang:
```shell ```shell
sudo apt update sudo apt update
sudo apt install elixir erlang-dev erlang-nox sudo apt install elixir erlang-dev erlang-nox
``` ```
### Optional packages: [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md) ### Optional packages: [`docs/installation/optional/media_graphics_packages.md`](../installation/optional/media_graphics_packages.md)
```shell ```shell
@ -90,7 +66,7 @@ cd /opt/pleroma
sudo -Hu pleroma mix deps.get sudo -Hu pleroma mix deps.get
``` ```
* Generate the configuration: `sudo -Hu pleroma mix pleroma.instance gen` * Generate the configuration: `sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen`
* Answer with `yes` if it asks you to install `rebar3`. * Answer with `yes` if it asks you to install `rebar3`.
* This may take some time, because parts of pleroma get compiled first. * This may take some time, because parts of pleroma get compiled first.
* After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`. * After that it will ask you a few questions about your instance and generates a configuration file in `config/generated_config.exs`.
@ -202,4 +178,4 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress
## Questions ## Questions
Questions about the installation or didnt it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**. Questions about the installation or didnt it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC.

View file

@ -89,7 +89,7 @@ sudo -Hu pleroma mix deps.get
* コンフィギュレーションを生成します。 * コンフィギュレーションを生成します。
``` ```
sudo -Hu pleroma mix pleroma.instance gen sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen
``` ```
* rebar3をインストールしてもよいか聞かれたら、yesを入力してください。 * rebar3をインストールしてもよいか聞かれたら、yesを入力してください。
* このときにpleromaの一部がコンパイルされるため、この処理には時間がかかります。 * このときにpleromaの一部がコンパイルされるため、この処理には時間がかかります。
@ -103,7 +103,7 @@ sudo -Hu pleroma mv config/{generated_config.exs,prod.secret.exs}
* 先程のコマンドで、すでに `config/setup_db.psql` というファイルが作られています。このファイルをもとに、データベースを作成します。 * 先程のコマンドで、すでに `config/setup_db.psql` というファイルが作られています。このファイルをもとに、データベースを作成します。
``` ```
sudo -Hu pleroma mix pleroma.instance gen sudo -Hu pleroma MIX_ENV=prod mix pleroma.instance gen
``` ```
* そして、データベースのマイグレーションを実行します。 * そして、データベースのマイグレーションを実行します。
@ -191,5 +191,5 @@ sudo -Hu pleroma MIX_ENV=prod mix pleroma.user new <username> <your@emailaddress
インストールについて質問がある、もしくは、うまくいかないときは、以下のところで質問できます。 インストールについて質問がある、もしくは、うまくいかないときは、以下のところで質問できます。
* [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) * [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat)
* **Freenode** の **#pleroma** IRCチャンネル * **libera.chat** の **#pleroma** IRCチャンネル

View file

@ -1,8 +1,10 @@
# Installing on FreeBSD # Installing on FreeBSD
This document was written for FreeBSD 12.1, but should be work on future releases. This document was written for FreeBSD 12.1, but should be work on future releases.
## Required software {! backend/installation/generic_dependencies.include !}
## Installing software used in this guide
This assumes the target system has `pkg(8)`. This assumes the target system has `pkg(8)`.
@ -54,7 +56,7 @@ Configure Pleroma. Note that you need a domain name at this point:
``` ```
$ cd /home/pleroma/pleroma $ cd /home/pleroma/pleroma
$ mix deps.get # Enter "y" when asked to install Hex $ mix deps.get # Enter "y" when asked to install Hex
$ mix pleroma.instance gen # You will be asked a few questions here. $ MIX_ENV=prod mix pleroma.instance gen # You will be asked a few questions here.
$ cp config/generated_config.exs config/prod.secret.exs $ cp config/generated_config.exs config/prod.secret.exs
``` ```
@ -213,4 +215,4 @@ incorrect timestamps. You should have ntpd running.
## Questions ## Questions
Questions about the installation or didnt it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**. Questions about the installation or didnt it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC.

View file

@ -0,0 +1,16 @@
## Required dependencies
* PostgreSQL 9.6+
* Elixir 1.9+
* Erlang OTP 22.2+
* git
* file / libmagic
* gcc (clang might also work)
* GNU make
* CMake
## Optionnal dependencies
* ImageMagick
* FFmpeg
* exiftool

View file

@ -3,9 +3,7 @@
This guide will assume that you have administrative rights, either as root or a user with [sudo permissions](https://wiki.gentoo.org/wiki/Sudo). Lines that begin with `#` indicate that they should be run as the superuser. Lines using `$` should be run as the indicated user, e.g. `pleroma$` should be run as the `pleroma` user. This guide will assume that you have administrative rights, either as root or a user with [sudo permissions](https://wiki.gentoo.org/wiki/Sudo). Lines that begin with `#` indicate that they should be run as the superuser. Lines using `$` should be run as the indicated user, e.g. `pleroma$` should be run as the `pleroma` user.
### Configuring your hostname (optional) {! backend/installation/generic_dependencies.include !}
If you would like your prompt to permanently include your host/domain, change `/etc/conf.d/hostname` to your hostname. You can reboot or use the `hostname` command to make immediate changes.
### Your make.conf, package.use, and USE flags ### Your make.conf, package.use, and USE flags
@ -54,7 +52,7 @@ Gentoo quite pointedly does not come with a cron daemon installed, and as such i
# emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake sys-apps/file # emerge --ask dev-db/postgresql dev-lang/elixir dev-vcs/git www-servers/nginx app-crypt/certbot app-crypt/certbot-nginx dev-util/cmake sys-apps/file
``` ```
If you would not like to install the optional packages, remove them from this line. If you would not like to install the optional packages, remove them from this line.
If you're running this from a low-powered virtual machine, it should work though it will take some time. There were no issues on a VPS with a single core and 1GB of RAM; if you are using an even more limited device and run into issues, you can try creating a swapfile or use a more powerful machine running Gentoo to [cross build](https://wiki.gentoo.org/wiki/Cross_build_environment). If you have a wait ahead of you, now would be a good time to take a break, strech a bit, refresh your beverage of choice and/or get a snack, and reply to Arch users' posts with "I use Gentoo btw" as we do. If you're running this from a low-powered virtual machine, it should work though it will take some time. There were no issues on a VPS with a single core and 1GB of RAM; if you are using an even more limited device and run into issues, you can try creating a swapfile or use a more powerful machine running Gentoo to [cross build](https://wiki.gentoo.org/wiki/Cross_build_environment). If you have a wait ahead of you, now would be a good time to take a break, strech a bit, refresh your beverage of choice and/or get a snack, and reply to Arch users' posts with "I use Gentoo btw" as we do.
@ -79,12 +77,12 @@ The output from emerging postgresql should give you a command for initializing t
``` ```
* Start postgres and enable the system service * Start postgres and enable the system service
```shell ```shell
# /etc/init.d/postgresql-11 start # /etc/init.d/postgresql-11 start
# rc-update add postgresql-11 default # rc-update add postgresql-11 default
``` ```
### A note on licenses, the AGPL, and deployment procedures ### A note on licenses, the AGPL, and deployment procedures
If you do not plan to make any modifications to your Pleroma instance, cloning directly from the main repo will get you what you need. However, if you plan on doing any contributions to upstream development, making changes or modifications to your instance, making custom themes, or want to play around--and let's be honest here, if you're using Gentoo that is most likely you--you will save yourself a lot of headache later if you take the time right now to fork the Pleroma repo and use that in the following section. If you do not plan to make any modifications to your Pleroma instance, cloning directly from the main repo will get you what you need. However, if you plan on doing any contributions to upstream development, making changes or modifications to your instance, making custom themes, or want to play around--and let's be honest here, if you're using Gentoo that is most likely you--you will save yourself a lot of headache later if you take the time right now to fork the Pleroma repo and use that in the following section.
@ -135,7 +133,7 @@ pleroma$ mix deps.get
* Generate the configuration: * Generate the configuration:
```shell ```shell
pleroma$ mix pleroma.instance gen pleroma$ MIX_ENV=prod mix pleroma.instance gen
``` ```
* Answer with `yes` if it asks you to install `rebar3`. * Answer with `yes` if it asks you to install `rebar3`.
@ -241,7 +239,7 @@ First, ensure that the command you will be installing into your crontab works.
# /usr/bin/certbot renew --nginx # /usr/bin/certbot renew --nginx
``` ```
Assuming not much time has passed since you got certbot working a few steps ago, you should get a message for all domains you installed certificates for saying `Cert not yet due for renewal`. Assuming not much time has passed since you got certbot working a few steps ago, you should get a message for all domains you installed certificates for saying `Cert not yet due for renewal`.
Now, run crontab as a superuser with `crontab -e` or `sudo crontab -e` as appropriate, and add the following line to your cron: Now, run crontab as a superuser with `crontab -e` or `sudo crontab -e` as appropriate, and add the following line to your cron:
@ -298,4 +296,4 @@ If you opted to allow sudo for the `pleroma` user but would like to remove the a
## Questions ## Questions
Questions about the installation or didnt it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**. Questions about the installation or didnt it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC.

View file

@ -1,6 +1,8 @@
# Installing on NetBSD # Installing on NetBSD
## Required software {! backend/installation/generic_dependencies.include !}
## Installing software used in this guide
pkgin should have been installed by the NetBSD installer if you selected pkgin should have been installed by the NetBSD installer if you selected
the right options. If it isn't installed, install it using pkg_add. the right options. If it isn't installed, install it using pkg_add.
@ -71,7 +73,7 @@ Configure Pleroma. Note that you need a domain name at this point:
``` ```
$ cd /home/pleroma/pleroma $ cd /home/pleroma/pleroma
$ mix deps.get $ mix deps.get
$ mix pleroma.instance gen # You will be asked a few questions here. $ MIX_ENV=prod mix pleroma.instance gen # You will be asked a few questions here.
``` ```
Since Postgres is configured, we can now initialize the database. There should Since Postgres is configured, we can now initialize the database. There should
@ -193,8 +195,6 @@ Run `# /etc/rc.d/pleroma start` to start Pleroma.
Restart nginx with `# /etc/rc.d/nginx restart` and you should be up and running. Restart nginx with `# /etc/rc.d/nginx restart` and you should be up and running.
If you need further help, contact niaa on freenode.
Make sure your time is in sync, or other instances will receive your posts with Make sure your time is in sync, or other instances will receive your posts with
incorrect timestamps. You should have ntpd running. incorrect timestamps. You should have ntpd running.
@ -208,4 +208,4 @@ incorrect timestamps. You should have ntpd running.
## Questions ## Questions
Questions about the installation or didnt it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**. Questions about the installation or didnt it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC.

View file

@ -4,19 +4,11 @@ This guide describes the installation and configuration of pleroma (and the requ
For any additional information regarding commands and configuration files mentioned here, check the man pages [online](https://man.openbsd.org/) or directly on your server with the man command. For any additional information regarding commands and configuration files mentioned here, check the man pages [online](https://man.openbsd.org/) or directly on your server with the man command.
{! backend/installation/generic_dependencies.include !}
### Preparing the system
#### Required software #### Required software
The following packages need to be installed:
* elixir
* gmake
* git
* postgresql-server
* postgresql-contrib
* cmake
* ffmpeg
* ImageMagick
To install them, run the following command (with doas or as root): To install them, run the following command (with doas or as root):
``` ```
@ -239,7 +231,7 @@ Enter a shell as \_pleroma (as root `su _pleroma -`) and enter pleroma's install
Then follow the main installation guide: Then follow the main installation guide:
* run `mix deps.get` * run `mix deps.get`
* run `mix pleroma.instance gen` and enter your instance's information when asked * run `MIX_ENV=prod mix pleroma.instance gen` and enter your instance's information when asked
* copy config/generated\_config.exs to config/prod.secret.exs. The default values should be sufficient but you should edit it and check that everything seems OK. * copy config/generated\_config.exs to config/prod.secret.exs. The default values should be sufficient but you should edit it and check that everything seems OK.
* exit your current shell back to a root one and run `psql -U postgres -f /home/_pleroma/pleroma/config/setup_db.psql` to setup the database. * exit your current shell back to a root one and run `psql -U postgres -f /home/_pleroma/pleroma/config/setup_db.psql` to setup the database.
* return to a \_pleroma shell into pleroma's installation directory (`su _pleroma -;cd ~/pleroma`) and run `MIX_ENV=prod mix ecto.migrate` * return to a \_pleroma shell into pleroma's installation directory (`su _pleroma -;cd ~/pleroma`) and run `MIX_ENV=prod mix ecto.migrate`
@ -264,4 +256,4 @@ LC_ALL=en_US.UTF-8 MIX_ENV=prod mix pleroma.user new <username> <your@emailaddre
## Questions ## Questions
Questions about the installation or didnt it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**. Questions about the installation or didnt it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC.

View file

@ -10,8 +10,8 @@ suositeltavaa tehdä komennon `doas` avulla, katso `doas (1)` ja `doas.conf (5)`
Tästä eteenpäin oletuksena on, että domain "esimerkki.com" osoittaa Tästä eteenpäin oletuksena on, että domain "esimerkki.com" osoittaa
serverin IP-osoitteeseen. serverin IP-osoitteeseen.
Jos asennuksen kanssa on ongelmia, IRC-kanava #pleroma Freenodessa tai Jos asennuksen kanssa on ongelmia, IRC-kanava #pleroma Libera.chat tai
Matrix-kanava #freenode_#pleroma:matrix.org ovat hyviä paikkoja löytää apua Matrix-kanava #pleroma:libera.chat ovat hyviä paikkoja löytää apua
(englanniksi), `/msg eal kukkuu` jos haluat välttämättä puhua härmää. (englanniksi), `/msg eal kukkuu` jos haluat välttämättä puhua härmää.
Asenna tarvittava ohjelmisto: Asenna tarvittava ohjelmisto:

View file

@ -31,7 +31,7 @@ Other than things bundled in the OTP release Pleroma depends on:
=== "Alpine" === "Alpine"
``` ```
echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories awk 'NR==2' /etc/apk/repositories | sed 's/main/community/' | tee -a /etc/apk/repositories
apk update apk update
apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot file-dev apk add curl unzip ncurses postgresql postgresql-contrib nginx certbot file-dev
``` ```
@ -50,7 +50,6 @@ Per [`docs/installation/optional/media_graphics_packages.md`](optional/media_gra
=== "Alpine" === "Alpine"
``` ```
echo "http://nl.alpinelinux.org/alpine/latest-stable/community" >> /etc/apk/repositories
apk update apk update
apk add imagemagick ffmpeg exiftool apk add imagemagick ffmpeg exiftool
``` ```
@ -232,7 +231,7 @@ At this point if you open your (sub)domain in a browser you should see a 502 err
If everything worked, you should see Pleroma-FE when visiting your domain. If that didn't happen, try reviewing the installation steps, starting Pleroma in the foreground and seeing if there are any errrors. If everything worked, you should see Pleroma-FE when visiting your domain. If that didn't happen, try reviewing the installation steps, starting Pleroma in the foreground and seeing if there are any errrors.
Still doesn't work? Feel free to contact us on [#pleroma on freenode](https://irc.pleroma.social) or via matrix at <https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org>, you can also [file an issue on our Gitlab](https://git.pleroma.social/pleroma/pleroma-support/issues/new) Questions about the installation or didnt it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC, you can also [file an issue on our Gitlab](https://git.pleroma.social/pleroma/pleroma-support/issues/new).
## Post installation ## Post installation
@ -290,7 +289,7 @@ nginx -t
## Create your first user and set as admin ## Create your first user and set as admin
```sh ```sh
cd /opt/pleroma/bin cd /opt/pleroma
su pleroma -s $SHELL -lc "./bin/pleroma_ctl user new joeuser joeuser@sld.tld --admin" su pleroma -s $SHELL -lc "./bin/pleroma_ctl user new joeuser joeuser@sld.tld --admin"
``` ```
This will create an account withe the username of 'joeuser' with the email address of joeuser@sld.tld, and set that user's account as an admin. This will result in a link that you can paste into the browser, which logs you in and enables you to set the password. This will create an account withe the username of 'joeuser' with the email address of joeuser@sld.tld, and set that user's account as an admin. This will result in a link that you can paste into the browser, which logs you in and enables you to set the password.
@ -301,4 +300,4 @@ This will create an account withe the username of 'joeuser' with the email addre
## Questions ## Questions
Questions about the installation or didnt it work as it should be, ask in [#pleroma:matrix.org](https://matrix.heldscal.la/#/room/#freenode_#pleroma:matrix.org) or IRC Channel **#pleroma** on **Freenode**. Questions about the installation or didnt it work as it should be, ask in [#pleroma:libera.chat](https://matrix.to/#/#pleroma:libera.chat) via Matrix or **#pleroma** on **libera.chat** via IRC, you can also [file an issue on our Gitlab](https://git.pleroma.social/pleroma/pleroma-support/issues/new).

View file

@ -27,7 +27,7 @@ defmodule Mix.Tasks.Pleroma.Config do
{opts, _} = {opts, _} =
OptionParser.parse!(options, OptionParser.parse!(options,
strict: [env: :string, delete: :boolean], strict: [env: :string, delete: :boolean, path: :string],
aliases: [d: :delete] aliases: [d: :delete]
) )
@ -259,18 +259,43 @@ defmodule Mix.Tasks.Pleroma.Config do
defp migrate_from_db(opts) do defp migrate_from_db(opts) do
env = opts[:env] || Pleroma.Config.get(:env) env = opts[:env] || Pleroma.Config.get(:env)
filename = "#{env}.exported_from_db.secret.exs"
config_path = config_path =
if Pleroma.Config.get(:release) do cond do
:config_path opts[:path] ->
|> Pleroma.Config.get() opts[:path]
|> Path.dirname()
else Pleroma.Config.get(:release) ->
"config" :config_path
|> Pleroma.Config.get()
|> Path.dirname()
true ->
"config"
end end
|> Path.join("#{env}.exported_from_db.secret.exs") |> Path.join(filename)
file = File.open!(config_path, [:write, :utf8]) with {:ok, file} <- File.open(config_path, [:write, :utf8]) do
write_config(file, config_path, opts)
shell_info("Database configuration settings have been exported to #{config_path}")
else
_ ->
shell_error("Impossible to save settings to this directory #{Path.dirname(config_path)}")
tmp_config_path = Path.join(System.tmp_dir!(), filename)
file = File.open!(tmp_config_path)
shell_info(
"Saving database configuration settings to #{tmp_config_path}. Copy it to the #{
Path.dirname(config_path)
} manually."
)
write_config(file, tmp_config_path, opts)
end
end
defp write_config(file, path, opts) do
IO.write(file, config_header()) IO.write(file, config_header())
ConfigDB ConfigDB
@ -278,11 +303,7 @@ defmodule Mix.Tasks.Pleroma.Config do
|> Enum.each(&write_and_delete(&1, file, opts[:delete])) |> Enum.each(&write_and_delete(&1, file, opts[:delete]))
:ok = File.close(file) :ok = File.close(file)
System.cmd("mix", ["format", config_path]) System.cmd("mix", ["format", path])
shell_info(
"Database configuration settings have been exported to config/#{env}.exported_from_db.secret.exs"
)
end end
if Code.ensure_loaded?(Config.Reader) do if Code.ensure_loaded?(Config.Reader) do

View file

@ -8,10 +8,13 @@ defmodule Mix.Tasks.Pleroma.Database do
alias Pleroma.Object alias Pleroma.Object
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.User alias Pleroma.User
require Logger require Logger
require Pleroma.Constants require Pleroma.Constants
import Ecto.Query import Ecto.Query
import Mix.Pleroma import Mix.Pleroma
use Mix.Task use Mix.Task
@shortdoc "A collection of database related tasks" @shortdoc "A collection of database related tasks"
@ -93,6 +96,15 @@ defmodule Mix.Tasks.Pleroma.Database do
) )
|> Repo.delete_all(timeout: :infinity) |> Repo.delete_all(timeout: :infinity)
prune_hashtags_query = """
DELETE FROM hashtags AS ht
WHERE NOT EXISTS (
SELECT 1 FROM hashtags_objects hto
WHERE ht.id = hto.hashtag_id)
"""
Repo.query(prune_hashtags_query)
if Keyword.get(options, :vacuum) do if Keyword.get(options, :vacuum) do
Maintenance.vacuum("full") Maintenance.vacuum("full")
end end
@ -214,4 +226,32 @@ defmodule Mix.Tasks.Pleroma.Database do
shell_info('Done.') shell_info('Done.')
end end
end end
# Rolls back a specific migration (leaving subsequent migrations applied).
# WARNING: imposes a risk of unrecoverable data loss — proceed at your own responsibility.
# Based on https://stackoverflow.com/a/53825840
def run(["rollback", version]) do
prompt = "SEVERE WARNING: this operation may result in unrecoverable data loss. Continue?"
if shell_prompt(prompt, "n") in ~w(Yn Y y) do
{_, result, _} =
Ecto.Migrator.with_repo(Pleroma.Repo, fn repo ->
version = String.to_integer(version)
re = ~r/^#{version}_.*\.exs/
path = Ecto.Migrator.migrations_path(repo)
with {_, "" <> file} <- {:find, Enum.find(File.ls!(path), &String.match?(&1, re))},
{_, [{mod, _} | _]} <- {:compile, Code.compile_file(Path.join(path, file))},
{_, :ok} <- {:rollback, Ecto.Migrator.down(repo, version, mod)} do
{:ok, "Reversed migration: #{file}"}
else
{:find, _} -> {:error, "No migration found with version prefix: #{version}"}
{:compile, e} -> {:error, "Problem compiling migration module: #{inspect(e)}"}
{:rollback, e} -> {:error, "Problem reversing migration: #{inspect(e)}"}
end
end)
shell_info(inspect(result))
end
end
end end

View file

@ -113,6 +113,7 @@ defmodule Pleroma.Activity do
from([a] in query, from([a] in query,
left_join: b in Bookmark, left_join: b in Bookmark,
on: b.user_id == ^user.id and b.activity_id == a.id, on: b.user_id == ^user.id and b.activity_id == a.id,
as: :bookmark,
preload: [bookmark: b] preload: [bookmark: b]
) )
end end
@ -123,6 +124,7 @@ defmodule Pleroma.Activity do
from([a] in query, from([a] in query,
left_join: r in ReportNote, left_join: r in ReportNote,
on: a.id == r.activity_id, on: a.id == r.activity_id,
as: :report_note,
preload: [report_notes: r] preload: [report_notes: r]
) )
end end
@ -182,40 +184,48 @@ defmodule Pleroma.Activity do
|> Repo.one() |> Repo.one()
end end
@spec get_by_id(String.t()) :: Activity.t() | nil @doc """
def get_by_id(id) do Gets activity by ID, doesn't load activities from deactivated actors by default.
case FlakeId.flake_id?(id) do """
true -> @spec get_by_id(String.t(), keyword()) :: t() | nil
Activity def get_by_id(id, opts \\ [filter: [:restrict_deactivated]]), do: get_by_id_with_opts(id, opts)
|> where([a], a.id == ^id)
|> restrict_deactivated_users()
|> Repo.one()
_ -> @spec get_by_id_with_user_actor(String.t()) :: t() | nil
nil def get_by_id_with_user_actor(id), do: get_by_id_with_opts(id, preload: [:user_actor])
@spec get_by_id_with_object(String.t()) :: t() | nil
def get_by_id_with_object(id), do: get_by_id_with_opts(id, preload: [:object])
defp get_by_id_with_opts(id, opts) do
if FlakeId.flake_id?(id) do
query = Queries.by_id(id)
with_filters_query =
if is_list(opts[:filter]) do
Enum.reduce(opts[:filter], query, fn
{:type, type}, acc -> Queries.by_type(acc, type)
:restrict_deactivated, acc -> restrict_deactivated_users(acc)
_, acc -> acc
end)
else
query
end
with_preloads_query =
if is_list(opts[:preload]) do
Enum.reduce(opts[:preload], with_filters_query, fn
:user_actor, acc -> with_preloaded_user_actor(acc)
:object, acc -> with_preloaded_object(acc)
_, acc -> acc
end)
else
with_filters_query
end
Repo.one(with_preloads_query)
end end
end end
def get_by_id_with_user_actor(id) do
case FlakeId.flake_id?(id) do
true ->
Activity
|> where([a], a.id == ^id)
|> with_preloaded_user_actor()
|> Repo.one()
_ ->
nil
end
end
def get_by_id_with_object(id) do
Activity
|> where(id: ^id)
|> with_preloaded_object()
|> Repo.one()
end
def all_by_ids_with_object(ids) do def all_by_ids_with_object(ids) do
Activity Activity
|> where([a], a.id in ^ids) |> where([a], a.id in ^ids)
@ -267,6 +277,11 @@ defmodule Pleroma.Activity do
def get_create_by_object_ap_id_with_object(_), do: nil def get_create_by_object_ap_id_with_object(_), do: nil
@spec create_by_id_with_object(String.t()) :: t() | nil
def create_by_id_with_object(id) do
get_by_id_with_opts(id, preload: [:object], filter: [type: "Create"])
end
defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do
get_create_by_object_ap_id_with_object(ap_id) get_create_by_object_ap_id_with_object(ap_id)
end end
@ -277,7 +292,8 @@ defmodule Pleroma.Activity do
get_in_reply_to_activity_from_object(Object.normalize(activity, fetch: false)) get_in_reply_to_activity_from_object(Object.normalize(activity, fetch: false))
end end
def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"]) def normalize(%Activity{data: %{"id" => ap_id}}), do: get_by_ap_id_with_object(ap_id)
def normalize(%{"id" => ap_id}), do: get_by_ap_id_with_object(ap_id)
def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id_with_object(ap_id) def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id_with_object(ap_id)
def normalize(_), do: nil def normalize(_), do: nil
@ -298,13 +314,15 @@ defmodule Pleroma.Activity do
def delete_all_by_object_ap_id(_), do: nil def delete_all_by_object_ap_id(_), do: nil
defp purge_web_resp_cache(%Activity{} = activity) do defp purge_web_resp_cache(%Activity{data: %{"id" => id}} = activity) when is_binary(id) do
%{path: path} = URI.parse(activity.data["id"]) with %{path: path} <- URI.parse(id) do
@cachex.del(:web_resp_cache, path) @cachex.del(:web_resp_cache, path)
end
activity activity
end end
defp purge_web_resp_cache(nil), do: nil defp purge_web_resp_cache(activity), do: activity
def follow_accepted?( def follow_accepted?(
%Activity{data: %{"type" => "Follow", "object" => followed_ap_id}} = activity %Activity{data: %{"type" => "Follow", "object" => followed_ap_id}} = activity
@ -366,12 +384,6 @@ defmodule Pleroma.Activity do
end end
end end
@spec pinned_by_actor?(Activity.t()) :: boolean()
def pinned_by_actor?(%Activity{} = activity) do
actor = user_actor(activity)
activity.id in actor.pinned_activities
end
@spec get_by_object_ap_id_with_object(String.t()) :: t() | nil @spec get_by_object_ap_id_with_object(String.t()) :: t() | nil
def get_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do def get_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
ap_id ap_id
@ -382,4 +394,13 @@ defmodule Pleroma.Activity do
end end
def get_by_object_ap_id_with_object(_), do: nil def get_by_object_ap_id_with_object(_), do: nil
@spec add_by_params_query(String.t(), String.t(), String.t()) :: Ecto.Query.t()
def add_by_params_query(object_id, actor, target) do
object_id
|> Queries.by_object_id()
|> Queries.by_type("Add")
|> Queries.by_actor(actor)
|> where([a], fragment("?->>'target' = ?", a.data, ^target))
end
end end

View file

@ -0,0 +1,45 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Activity.HTML do
alias Pleroma.HTML
alias Pleroma.Object
@cachex Pleroma.Config.get([:cachex, :provider], Cachex)
def get_cached_scrubbed_html_for_activity(
content,
scrubbers,
activity,
key \\ "",
callback \\ fn x -> x end
) do
key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}"
@cachex.fetch!(:scrubber_cache, key, fn _key ->
object = Object.normalize(activity, fetch: false)
HTML.ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
end)
end
def get_cached_stripped_html_for_activity(content, activity, key) do
get_cached_scrubbed_html_for_activity(
content,
FastSanitize.Sanitizer.StripTags,
activity,
key,
&HtmlEntities.decode/1
)
end
defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do
generate_scrubber_signature([scrubber])
end
defp generate_scrubber_signature(scrubbers) do
Enum.reduce(scrubbers, "", fn scrubber, signature ->
"#{signature}#{to_string(scrubber)}"
end)
end
end

View file

@ -48,14 +48,12 @@ defmodule Pleroma.Activity.Ir.Topics do
tags tags
end end
defp hashtags_to_topics(%{data: %{"tag" => tags}}) do defp hashtags_to_topics(object) do
tags object
|> Enum.filter(&is_bitstring(&1)) |> Object.hashtags()
|> Enum.map(fn tag -> "hashtag:" <> tag end) |> Enum.map(fn hashtag -> "hashtag:" <> hashtag end)
end end
defp hashtags_to_topics(_), do: []
defp remote_topics(%{local: true}), do: [] defp remote_topics(%{local: true}), do: []
defp remote_topics(%{actor: actor}) when is_binary(actor), defp remote_topics(%{actor: actor}) when is_binary(actor),

View file

@ -14,6 +14,11 @@ defmodule Pleroma.Activity.Queries do
alias Pleroma.Activity alias Pleroma.Activity
alias Pleroma.User alias Pleroma.User
@spec by_id(query(), String.t()) :: query()
def by_id(query \\ Activity, id) do
from(a in query, where: a.id == ^id)
end
@spec by_ap_id(query, String.t()) :: query @spec by_ap_id(query, String.t()) :: query
def by_ap_id(query \\ Activity, ap_id) do def by_ap_id(query \\ Activity, ap_id) do
from( from(

View file

@ -25,7 +25,7 @@ defmodule Pleroma.Application do
if Process.whereis(Pleroma.Web.Endpoint) do if Process.whereis(Pleroma.Web.Endpoint) do
case Config.get([:http, :user_agent], :default) do case Config.get([:http, :user_agent], :default) do
:default -> :default ->
info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>" info = "#{Pleroma.Web.Endpoint.url()} <#{Config.get([:instance, :email], "")}>"
named_version() <> "; " <> info named_version() <> "; " <> info
custom -> custom ->
@ -102,10 +102,8 @@ defmodule Pleroma.Application do
] ++ ] ++
task_children(@mix_env) ++ task_children(@mix_env) ++
dont_run_in_test(@mix_env) ++ dont_run_in_test(@mix_env) ++
chat_child(chat_enabled?()) ++ shout_child(shout_enabled?()) ++
[ [Pleroma.Gopher.Server]
Pleroma.Gopher.Server
]
# See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html
# for other strategies and supported options # for other strategies and supported options
@ -218,7 +216,7 @@ defmodule Pleroma.Application do
type: :worker type: :worker
} }
defp chat_enabled?, do: Config.get([:chat, :enabled]) defp shout_enabled?, do: Config.get([:shout, :enabled])
defp dont_run_in_test(env) when env in [:test, :benchmark], do: [] defp dont_run_in_test(env) when env in [:test, :benchmark], do: []
@ -230,17 +228,23 @@ defmodule Pleroma.Application do
keys: :duplicate, keys: :duplicate,
partitions: System.schedulers_online() partitions: System.schedulers_online()
]} ]}
] ++ background_migrators()
end
defp background_migrators do
[
Pleroma.Migrators.HashtagsTableMigrator
] ]
end end
defp chat_child(true) do defp shout_child(true) do
[ [
Pleroma.Web.ChatChannel.ChatChannelState, Pleroma.Web.ShoutChannel.ShoutChannelState,
{Phoenix.PubSub, [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]} {Phoenix.PubSub, [name: Pleroma.PubSub, adapter: Phoenix.PubSub.PG2]}
] ]
end end
defp chat_child(_), do: [] defp shout_child(_), do: []
defp task_children(:test) do defp task_children(:test) do
[ [

View file

@ -34,15 +34,16 @@ defmodule Pleroma.ApplicationRequirements do
defp check_welcome_message_config!(:ok) do defp check_welcome_message_config!(:ok) do
if Pleroma.Config.get([:welcome, :email, :enabled], false) and if Pleroma.Config.get([:welcome, :email, :enabled], false) and
not Pleroma.Emails.Mailer.enabled?() do not Pleroma.Emails.Mailer.enabled?() do
Logger.error(""" Logger.warn("""
To send welcome email do you need to enable mail. To send welcome emails, you need to enable the mailer.
\nconfig :pleroma, Pleroma.Emails.Mailer, enabled: true Welcome emails will NOT be sent with the current config.
""")
{:error, "The mail disabled."} Enable the mailer:
else config :pleroma, Pleroma.Emails.Mailer, enabled: true
:ok """)
end end
:ok
end end
defp check_welcome_message_config!(result), do: result defp check_welcome_message_config!(result), do: result
@ -51,18 +52,21 @@ defmodule Pleroma.ApplicationRequirements do
# #
def check_confirmation_accounts!(:ok) do def check_confirmation_accounts!(:ok) do
if Pleroma.Config.get([:instance, :account_activation_required]) && if Pleroma.Config.get([:instance, :account_activation_required]) &&
not Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do not Pleroma.Emails.Mailer.enabled?() do
Logger.error( Logger.warn("""
"Account activation enabled, but no Mailer settings enabled.\n" <> Account activation is required, but the mailer is disabled.
"Please set config :pleroma, :instance, account_activation_required: false\n" <> Users will NOT be able to confirm their accounts with this config.
"Otherwise setup and enable Mailer." Either disable account activation or enable the mailer.
)
{:error, Disable account activation:
"Account activation enabled, but Mailer is disabled. Cannot send confirmation emails."} config :pleroma, :instance, account_activation_required: false
else
:ok Enable the mailer:
config :pleroma, Pleroma.Emails.Mailer, enabled: true
""")
end end
:ok
end end
def check_confirmation_accounts!(result), do: result def check_confirmation_accounts!(result), do: result
@ -160,9 +164,12 @@ defmodule Pleroma.ApplicationRequirements do
defp check_system_commands!(:ok) do defp check_system_commands!(:ok) do
filter_commands_statuses = [ filter_commands_statuses = [
check_filter(Pleroma.Upload.Filters.Exiftool, "exiftool"), check_filter(Pleroma.Upload.Filter.Exiftool, "exiftool"),
check_filter(Pleroma.Upload.Filters.Mogrify, "mogrify"), check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"),
check_filter(Pleroma.Upload.Filters.Mogrifun, "mogrify") check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"),
check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "mogrify"),
check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "convert"),
check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "ffprobe")
] ]
preview_proxy_commands_status = preview_proxy_commands_status =

View file

@ -99,4 +99,8 @@ defmodule Pleroma.Config do
def oauth_consumer_strategies, do: get([:auth, :oauth_consumer_strategies], []) def oauth_consumer_strategies, do: get([:auth, :oauth_consumer_strategies], [])
def oauth_consumer_enabled?, do: oauth_consumer_strategies() != [] def oauth_consumer_enabled?, do: oauth_consumer_strategies() != []
def feature_enabled?(feature_name) do
get([:features, feature_name]) not in [nil, false, :disabled, :auto]
end
end end

View file

@ -41,7 +41,8 @@ defmodule Pleroma.Config.DeprecationWarnings do
:ok <- check_gun_pool_options(), :ok <- check_gun_pool_options(),
:ok <- check_activity_expiration_config(), :ok <- check_activity_expiration_config(),
:ok <- check_remote_ip_plug_name(), :ok <- check_remote_ip_plug_name(),
:ok <- check_uploders_s3_public_endpoint() do :ok <- check_uploders_s3_public_endpoint(),
:ok <- check_old_chat_shoutbox() do
:ok :ok
else else
_ -> _ ->
@ -215,4 +216,27 @@ defmodule Pleroma.Config.DeprecationWarnings do
:ok :ok
end end
end end
@spec check_old_chat_shoutbox() :: :ok | nil
def check_old_chat_shoutbox do
instance_config = Pleroma.Config.get([:instance])
chat_config = Pleroma.Config.get([:chat]) || []
use_old_config =
Keyword.has_key?(instance_config, :chat_limit) or
Keyword.has_key?(chat_config, :enabled)
if use_old_config do
Logger.error("""
!!!DEPRECATION WARNING!!!
Your config is using the old namespace for the Shoutbox configuration. You need to convert to the new namespace. e.g.,
\n* `config :pleroma, :chat, enabled` and `config :pleroma, :instance, chat_limit` are now equal to:
\n* `config :pleroma, :shout, enabled` and `config :pleroma, :shout, limit`
""")
:error
else
:ok
end
end
end end

View file

@ -3,9 +3,11 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Config.Loader do defmodule Pleroma.Config.Loader do
# These modules are only being used as keys here (for equality check),
# so it's okay to use `Module.concat/1` to have the compiler ignore them.
@reject_keys [ @reject_keys [
Pleroma.Repo, Module.concat(["Pleroma.Repo"]),
Pleroma.Web.Endpoint, Module.concat(["Pleroma.Web.Endpoint"]),
:env, :env,
:configurable_from_database, :configurable_from_database,
:database, :database,

View file

@ -1,6 +1,6 @@
defmodule Pleroma.Config.ReleaseRuntimeProvider do defmodule Pleroma.Config.ReleaseRuntimeProvider do
@moduledoc """ @moduledoc """
Imports `runtime.exs` and `{env}.exported_from_db.secret.exs` for elixir releases. Imports runtime config and `{env}.exported_from_db.secret.exs` for releases.
""" """
@behaviour Config.Provider @behaviour Config.Provider
@ -8,10 +8,11 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do
def init(opts), do: opts def init(opts), do: opts
@impl true @impl true
def load(config, _opts) do def load(config, opts) do
with_defaults = Config.Reader.merge(config, Pleroma.Config.Holder.release_defaults()) with_defaults = Config.Reader.merge(config, Pleroma.Config.Holder.release_defaults())
config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs" config_path =
opts[:config_path] || System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"
with_runtime_config = with_runtime_config =
if File.exists?(config_path) do if File.exists?(config_path) do
@ -24,7 +25,7 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do
warning = [ warning = [
IO.ANSI.red(), IO.ANSI.red(),
IO.ANSI.bright(), IO.ANSI.bright(),
"!!! #{config_path} not found! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file", "!!! Config path is not declared! Please ensure it exists and that PLEROMA_CONFIG_PATH is unset or points to an existing file",
IO.ANSI.reset() IO.ANSI.reset()
] ]
@ -33,13 +34,14 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do
end end
exported_config_path = exported_config_path =
config_path opts[:exported_config_path] ||
|> Path.dirname() config_path
|> Path.join("prod.exported_from_db.secret.exs") |> Path.dirname()
|> Path.join("#{Pleroma.Config.get(:env)}.exported_from_db.secret.exs")
with_exported = with_exported =
if File.exists?(exported_config_path) do if File.exists?(exported_config_path) do
exported_config = Config.Reader.read!(with_runtime_config) exported_config = Config.Reader.read!(exported_config_path)
Config.Reader.merge(with_runtime_config, exported_config) Config.Reader.merge(with_runtime_config, exported_config)
else else
with_runtime_config with_runtime_config

View file

@ -13,23 +13,25 @@ defmodule Pleroma.Config.TransferTask do
@type env() :: :test | :benchmark | :dev | :prod @type env() :: :test | :benchmark | :dev | :prod
@reboot_time_keys [ defp reboot_time_keys,
{:pleroma, :hackney_pools}, do: [
{:pleroma, :chat}, {:pleroma, :hackney_pools},
{:pleroma, Oban}, {:pleroma, :shout},
{:pleroma, :rate_limit}, {:pleroma, Oban},
{:pleroma, :markup}, {:pleroma, :rate_limit},
{:pleroma, :streamer}, {:pleroma, :markup},
{:pleroma, :pools}, {:pleroma, :streamer},
{:pleroma, :connections_pool} {:pleroma, :pools},
] {:pleroma, :connections_pool}
]
@reboot_time_subkeys [ defp reboot_time_subkeys,
{:pleroma, Pleroma.Captcha, [:seconds_valid]}, do: [
{:pleroma, Pleroma.Upload, [:proxy_remote]}, {:pleroma, Pleroma.Captcha, [:seconds_valid]},
{:pleroma, :instance, [:upload_limit]}, {:pleroma, Pleroma.Upload, [:proxy_remote]},
{:pleroma, :gopher, [:enabled]} {:pleroma, :instance, [:upload_limit]},
] {:pleroma, :gopher, [:enabled]}
]
def start_link(restart_pleroma? \\ true) do def start_link(restart_pleroma? \\ true) do
load_and_update_env([], restart_pleroma?) load_and_update_env([], restart_pleroma?)
@ -165,12 +167,12 @@ defmodule Pleroma.Config.TransferTask do
end end
defp group_and_key_need_reboot?(group, key) do defp group_and_key_need_reboot?(group, key) do
Enum.any?(@reboot_time_keys, fn {g, k} -> g == group and k == key end) Enum.any?(reboot_time_keys(), fn {g, k} -> g == group and k == key end)
end end
defp group_and_subkey_need_reboot?(group, key, value) do defp group_and_subkey_need_reboot?(group, key, value) do
Keyword.keyword?(value) and Keyword.keyword?(value) and
Enum.any?(@reboot_time_subkeys, fn {g, k, subkeys} -> Enum.any?(reboot_time_subkeys(), fn {g, k, subkeys} ->
g == group and k == key and g == group and k == key and
Enum.any?(Keyword.keys(value), &(&1 in subkeys)) Enum.any?(Keyword.keys(value), &(&1 in subkeys))
end) end)

View file

@ -387,6 +387,6 @@ defmodule Pleroma.ConfigDB do
@spec module_name?(String.t()) :: boolean() @spec module_name?(String.t()) :: boolean()
def module_name?(string) do def module_name?(string) do
Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or
string in ["Oban", "Ueberauth", "ExSyslogger"] string in ["Oban", "Ueberauth", "ExSyslogger", "ConcurrentLimiter"]
end end
end end

View file

@ -27,6 +27,4 @@ defmodule Pleroma.Constants do
do: do:
~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css) ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css)
) )
def as_local_public, do: Pleroma.Web.base_url() <> "/#Public"
end end

View file

@ -0,0 +1,45 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.DataMigration do
use Ecto.Schema
alias Pleroma.DataMigration
alias Pleroma.DataMigration.State
alias Pleroma.Repo
import Ecto.Changeset
import Ecto.Query
schema "data_migrations" do
field(:name, :string)
field(:state, State, default: :pending)
field(:feature_lock, :boolean, default: false)
field(:params, :map, default: %{})
field(:data, :map, default: %{})
timestamps()
end
def changeset(data_migration, params \\ %{}) do
data_migration
|> cast(params, [:name, :state, :feature_lock, :params, :data])
|> validate_required([:name])
|> unique_constraint(:name)
end
def update_one_by_id(id, params \\ %{}) do
with {1, _} <-
from(dm in DataMigration, where: dm.id == ^id)
|> Repo.update_all(set: params) do
:ok
end
end
def get_by_name(name) do
Repo.get_by(DataMigration, name: name)
end
def populate_hashtags_table, do: get_by_name("populate_hashtags_table")
end

View file

@ -9,7 +9,6 @@ defmodule Pleroma.Delivery do
alias Pleroma.Object alias Pleroma.Object
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.User alias Pleroma.User
alias Pleroma.User
import Ecto.Changeset import Ecto.Changeset
import Ecto.Query import Ecto.Query

View file

@ -1,256 +0,0 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
#
# This file is derived from Earmark, under the following copyright:
# Copyright © 2014 Dave Thomas, The Pragmatic Programmers
# SPDX-License-Identifier: Apache-2.0
# Upstream: https://github.com/pragdave/earmark/blob/master/lib/earmark/html_renderer.ex
defmodule Pleroma.EarmarkRenderer do
@moduledoc false
alias Earmark.Block
alias Earmark.Context
alias Earmark.HtmlRenderer
alias Earmark.Options
import Earmark.Inline, only: [convert: 3]
import Earmark.Helpers.HtmlHelpers
import Earmark.Message, only: [add_messages_from: 2, get_messages: 1, set_messages: 2]
import Earmark.Context, only: [append: 2, set_value: 2]
import Earmark.Options, only: [get_mapper: 1]
@doc false
def render(blocks, %Context{options: %Options{}} = context) do
messages = get_messages(context)
{contexts, html} =
get_mapper(context.options).(
blocks,
&render_block(&1, put_in(context.options.messages, []))
)
|> Enum.unzip()
all_messages =
contexts
|> Enum.reduce(messages, fn ctx, messages1 -> messages1 ++ get_messages(ctx) end)
{put_in(context.options.messages, all_messages), html |> IO.iodata_to_binary()}
end
#############
# Paragraph #
#############
defp render_block(%Block.Para{lnb: lnb, lines: lines, attrs: attrs}, context) do
lines = convert(lines, lnb, context)
add_attrs(lines, "<p>#{lines.value}</p>", attrs, [], lnb)
end
########
# Html #
########
defp render_block(%Block.Html{html: html}, context) do
{context, html}
end
defp render_block(%Block.HtmlComment{lines: lines}, context) do
{context, lines}
end
defp render_block(%Block.HtmlOneline{html: html}, context) do
{context, html}
end
#########
# Ruler #
#########
defp render_block(%Block.Ruler{lnb: lnb, attrs: attrs}, context) do
add_attrs(context, "<hr />", attrs, [], lnb)
end
###########
# Heading #
###########
defp render_block(
%Block.Heading{lnb: lnb, level: level, content: content, attrs: attrs},
context
) do
converted = convert(content, lnb, context)
html = "<h#{level}>#{converted.value}</h#{level}>"
add_attrs(converted, html, attrs, [], lnb)
end
##############
# Blockquote #
##############
defp render_block(%Block.BlockQuote{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
{context1, body} = render(blocks, context)
html = "<blockquote>#{body}</blockquote>"
add_attrs(context1, html, attrs, [], lnb)
end
#########
# Table #
#########
defp render_block(
%Block.Table{lnb: lnb, header: header, rows: rows, alignments: aligns, attrs: attrs},
context
) do
{context1, html} = add_attrs(context, "<table>", attrs, [], lnb)
context2 = set_value(context1, html)
context3 =
if header do
append(add_trs(append(context2, "<thead>"), [header], "th", aligns, lnb), "</thead>")
else
# Maybe an error, needed append(context, html)
context2
end
context4 = append(add_trs(append(context3, "<tbody>"), rows, "td", aligns, lnb), "</tbody>")
{context4, [context4.value, "</table>"]}
end
########
# Code #
########
defp render_block(
%Block.Code{lnb: lnb, language: language, attrs: attrs} = block,
%Context{options: options} = context
) do
class =
if language, do: ~s{ class="#{code_classes(language, options.code_class_prefix)}"}, else: ""
tag = ~s[<pre><code#{class}>]
lines = options.render_code.(block)
html = ~s[#{tag}#{lines}</code></pre>]
add_attrs(context, html, attrs, [], lnb)
end
#########
# Lists #
#########
defp render_block(
%Block.List{lnb: lnb, type: type, blocks: items, attrs: attrs, start: start},
context
) do
{context1, content} = render(items, context)
html = "<#{type}#{start}>#{content}</#{type}>"
add_attrs(context1, html, attrs, [], lnb)
end
# format a single paragraph list item, and remove the para tags
defp render_block(
%Block.ListItem{lnb: lnb, blocks: blocks, spaced: false, attrs: attrs},
context
)
when length(blocks) == 1 do
{context1, content} = render(blocks, context)
content = Regex.replace(~r{</?p>}, content, "")
html = "<li>#{content}</li>"
add_attrs(context1, html, attrs, [], lnb)
end
# format a spaced list item
defp render_block(%Block.ListItem{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
{context1, content} = render(blocks, context)
html = "<li>#{content}</li>"
add_attrs(context1, html, attrs, [], lnb)
end
##################
# Footnote Block #
##################
defp render_block(%Block.FnList{blocks: footnotes}, context) do
items =
Enum.map(footnotes, fn note ->
blocks = append_footnote_link(note)
%Block.ListItem{attrs: "#fn:#{note.number}", type: :ol, blocks: blocks}
end)
{context1, html} = render_block(%Block.List{type: :ol, blocks: items}, context)
{context1, Enum.join([~s[<div class="footnotes">], "<hr />", html, "</div>"])}
end
#######################################
# Isolated IALs are rendered as paras #
#######################################
defp render_block(%Block.Ial{verbatim: verbatim}, context) do
{context, "<p>{:#{verbatim}}</p>"}
end
####################
# IDDef is ignored #
####################
defp render_block(%Block.IdDef{}, context), do: {context, ""}
#####################################
# And here are the inline renderers #
#####################################
defdelegate br, to: HtmlRenderer
defdelegate codespan(text), to: HtmlRenderer
defdelegate em(text), to: HtmlRenderer
defdelegate strong(text), to: HtmlRenderer
defdelegate strikethrough(text), to: HtmlRenderer
defdelegate link(url, text), to: HtmlRenderer
defdelegate link(url, text, title), to: HtmlRenderer
defdelegate image(path, alt, title), to: HtmlRenderer
defdelegate footnote_link(ref, backref, number), to: HtmlRenderer
# Table rows
defp add_trs(context, rows, tag, aligns, lnb) do
numbered_rows =
rows
|> Enum.zip(Stream.iterate(lnb, &(&1 + 1)))
numbered_rows
|> Enum.reduce(context, fn {row, lnb}, ctx ->
append(add_tds(append(ctx, "<tr>"), row, tag, aligns, lnb), "</tr>")
end)
end
defp add_tds(context, row, tag, aligns, lnb) do
Enum.reduce(1..length(row), context, add_td_fn(row, tag, aligns, lnb))
end
defp add_td_fn(row, tag, aligns, lnb) do
fn n, ctx ->
style =
case Enum.at(aligns, n - 1, :default) do
:default -> ""
align -> " style=\"text-align: #{align}\""
end
col = Enum.at(row, n - 1)
converted = convert(col, lnb, set_messages(ctx, []))
append(add_messages_from(ctx, converted), "<#{tag}#{style}>#{converted.value}</#{tag}>")
end
end
###############################
# Append Footnote Return Link #
###############################
defdelegate append_footnote_link(note), to: HtmlRenderer
defdelegate append_footnote_link(note, fnlink), to: HtmlRenderer
defdelegate render_code(lines), to: HtmlRenderer
defp code_classes(language, prefix) do
["" | String.split(prefix || "")]
|> Enum.map(fn pfx -> "#{pfx}#{language}" end)
|> Enum.join(" ")
end
end

View file

@ -17,3 +17,11 @@ defenum(Pleroma.FollowingRelationship.State,
follow_accept: 2, follow_accept: 2,
follow_reject: 3 follow_reject: 3
) )
defenum(Pleroma.DataMigration.State,
pending: 1,
running: 2,
complete: 3,
failed: 4,
manual: 5
)

View file

@ -13,21 +13,33 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients do
cast([object]) cast([object])
end end
def cast(data) when is_list(data) do def cast(object) when is_map(object) do
data case ObjectID.cast(object) do
|> Enum.reduce_while({:ok, []}, fn element, {:ok, list} -> {:ok, data} -> {:ok, [data]}
case ObjectID.cast(element) do _ -> :error
{:ok, id} -> end
{:cont, {:ok, [id | list]}}
_ ->
{:halt, :error}
end
end)
end end
def cast(_) do def cast(data) when is_list(data) do
:error data =
data
|> Enum.reduce_while([], fn element, list ->
case ObjectID.cast(element) do
{:ok, id} ->
{:cont, [id | list]}
_ ->
{:cont, list}
end
end)
|> Enum.sort()
|> Enum.uniq()
{:ok, data}
end
def cast(data) do
{:error, data}
end end
def dump(data) do def dump(data) do

View file

@ -73,7 +73,7 @@ defmodule Pleroma.Emails.AdminEmail do
#{comment_html} #{comment_html}
#{statuses_html} #{statuses_html}
<p> <p>
<a href="#{Pleroma.Web.base_url()}/pleroma/admin/#/reports/index">View Reports in AdminFE</a> <a href="#{Pleroma.Web.Endpoint.url()}/pleroma/admin/#/reports/index">View Reports in AdminFE</a>
""" """
new() new()
@ -87,7 +87,7 @@ defmodule Pleroma.Emails.AdminEmail do
html_body = """ html_body = """
<p>New account for review: <a href="#{account.ap_id}">@#{account.nickname}</a></p> <p>New account for review: <a href="#{account.ap_id}">@#{account.nickname}</a></p>
<blockquote>#{HTML.strip_tags(account.registration_reason)}</blockquote> <blockquote>#{HTML.strip_tags(account.registration_reason)}</blockquote>
<a href="#{Pleroma.Web.base_url()}/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a> <a href="#{Pleroma.Web.Endpoint.url()}/pleroma/admin/#/users/#{account.id}/">Visit AdminFE</a>
""" """
new() new()

View file

@ -5,15 +5,22 @@
defmodule Pleroma.Emails.UserEmail do defmodule Pleroma.Emails.UserEmail do
@moduledoc "User emails" @moduledoc "User emails"
use Phoenix.Swoosh, view: Pleroma.Web.EmailView, layout: {Pleroma.Web.LayoutView, :email}
alias Pleroma.Config alias Pleroma.Config
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.Endpoint alias Pleroma.Web.Endpoint
alias Pleroma.Web.Router alias Pleroma.Web.Router
import Swoosh.Email
import Phoenix.Swoosh, except: [render_body: 3]
import Pleroma.Config.Helpers, only: [instance_name: 0, sender: 0] import Pleroma.Config.Helpers, only: [instance_name: 0, sender: 0]
def render_body(email, template, assigns \\ %{}) do
email
|> put_new_layout({Pleroma.Web.LayoutView, :email})
|> put_new_view(Pleroma.Web.EmailView)
|> Phoenix.Swoosh.render_body(template, assigns)
end
defp recipient(email, nil), do: email defp recipient(email, nil), do: email
defp recipient(email, name), do: {name, email} defp recipient(email, name), do: {name, email}
defp recipient(%User{} = user), do: recipient(user.email, user.name) defp recipient(%User{} = user), do: recipient(user.email, user.name)

View file

@ -5,7 +5,7 @@
defmodule Pleroma.Emoji.Formatter do defmodule Pleroma.Emoji.Formatter do
alias Pleroma.Emoji alias Pleroma.Emoji
alias Pleroma.HTML alias Pleroma.HTML
alias Pleroma.Web alias Pleroma.Web.Endpoint
alias Pleroma.Web.MediaProxy alias Pleroma.Web.MediaProxy
def emojify(text) do def emojify(text) do
@ -44,7 +44,7 @@ defmodule Pleroma.Emoji.Formatter do
Emoji.get_all() Emoji.get_all()
|> Enum.filter(fn {emoji, %Emoji{}} -> String.contains?(text, ":#{emoji}:") end) |> Enum.filter(fn {emoji, %Emoji{}} -> String.contains?(text, ":#{emoji}:") end)
|> Enum.reduce(%{}, fn {name, %Emoji{file: file}}, acc -> |> Enum.reduce(%{}, fn {name, %Emoji{file: file}}, acc ->
Map.put(acc, name, to_string(URI.merge(Web.base_url(), file))) Map.put(acc, name, to_string(URI.merge(Endpoint.url(), file)))
end) end)
end end

View file

@ -62,7 +62,7 @@ defmodule Pleroma.Formatter do
def hashtag_handler("#" <> tag = tag_text, _buffer, _opts, acc) do def hashtag_handler("#" <> tag = tag_text, _buffer, _opts, acc) do
tag = String.downcase(tag) tag = String.downcase(tag)
url = "#{Pleroma.Web.base_url()}/tag/#{tag}" url = "#{Pleroma.Web.Endpoint.url()}/tag/#{tag}"
link = link =
Phoenix.HTML.Tag.content_tag(:a, tag_text, Phoenix.HTML.Tag.content_tag(:a, tag_text,
@ -121,6 +121,10 @@ defmodule Pleroma.Formatter do
end end
end end
def markdown_to_html(text) do
Earmark.as_html!(text, %Earmark.Options{compact_output: true})
end
def html_escape({text, mentions, hashtags}, type) do def html_escape({text, mentions, hashtags}, type) do
{html_escape(text, type), mentions, hashtags} {html_escape(text, type), mentions, hashtags}
end end

View file

@ -11,9 +11,7 @@ defmodule Pleroma.Gun do
@callback await(pid(), reference()) :: {:response, :fin, 200, []} @callback await(pid(), reference()) :: {:response, :fin, 200, []}
@callback set_owner(pid(), pid()) :: :ok @callback set_owner(pid(), pid()) :: :ok
@api Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API) defp api, do: Pleroma.Config.get([Pleroma.Gun], Pleroma.Gun.API)
defp api, do: @api
def open(host, port, opts), do: api().open(host, port, opts) def open(host, port, opts), do: api().open(host, port, opts)

View file

@ -5,11 +5,11 @@
defmodule Pleroma.Gun.ConnectionPool.Reclaimer do defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
use GenServer, restart: :temporary use GenServer, restart: :temporary
@registry Pleroma.Gun.ConnectionPool defp registry, do: Pleroma.Gun.ConnectionPool
def start_monitor do def start_monitor do
pid = pid =
case :gen_server.start(__MODULE__, [], name: {:via, Registry, {@registry, "reclaimer"}}) do case :gen_server.start(__MODULE__, [], name: {:via, Registry, {registry(), "reclaimer"}}) do
{:ok, pid} -> {:ok, pid} ->
pid pid
@ -46,7 +46,7 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
# {worker_pid, crf, last_reference} end) # {worker_pid, crf, last_reference} end)
unused_conns = unused_conns =
Registry.select( Registry.select(
@registry, registry(),
[ [
{{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], [{{:"$1", :"$3", :"$4"}}]} {{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], [{{:"$1", :"$3", :"$4"}}]}
] ]

View file

@ -6,10 +6,10 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
alias Pleroma.Gun alias Pleroma.Gun
use GenServer, restart: :temporary use GenServer, restart: :temporary
@registry Pleroma.Gun.ConnectionPool defp registry, do: Pleroma.Gun.ConnectionPool
def start_link([key | _] = opts) do def start_link([key | _] = opts) do
GenServer.start_link(__MODULE__, opts, name: {:via, Registry, {@registry, key}}) GenServer.start_link(__MODULE__, opts, name: {:via, Registry, {registry(), key}})
end end
@impl true @impl true
@ -24,7 +24,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
time = :erlang.monotonic_time(:millisecond) time = :erlang.monotonic_time(:millisecond)
{_, _} = {_, _} =
Registry.update_value(@registry, key, fn _ -> Registry.update_value(registry(), key, fn _ ->
{conn_pid, [client_pid], 1, time} {conn_pid, [client_pid], 1, time}
end) end)
@ -65,7 +65,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
time = :erlang.monotonic_time(:millisecond) time = :erlang.monotonic_time(:millisecond)
{{conn_pid, used_by, _, _}, _} = {{conn_pid, used_by, _, _}, _} =
Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} -> Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} ->
{conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time} {conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time}
end) end)
@ -92,7 +92,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
@impl true @impl true
def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do def handle_call(:remove_client, {client_pid, _}, %{key: key} = state) do
{{_conn_pid, used_by, _crf, _last_reference}, _} = {{_conn_pid, used_by, _crf, _last_reference}, _} =
Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} -> Registry.update_value(registry(), key, fn {conn_pid, used_by, crf, last_reference} ->
{conn_pid, List.delete(used_by, client_pid), crf, last_reference} {conn_pid, List.delete(used_by, client_pid), crf, last_reference}
end) end)

106
lib/pleroma/hashtag.ex Normal file
View file

@ -0,0 +1,106 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Hashtag do
use Ecto.Schema
import Ecto.Changeset
import Ecto.Query
alias Ecto.Multi
alias Pleroma.Hashtag
alias Pleroma.Object
alias Pleroma.Repo
schema "hashtags" do
field(:name, :string)
many_to_many(:objects, Object, join_through: "hashtags_objects", on_replace: :delete)
timestamps()
end
def normalize_name(name) do
name
|> String.downcase()
|> String.trim()
end
def get_or_create_by_name(name) do
changeset = changeset(%Hashtag{}, %{name: name})
Repo.insert(
changeset,
on_conflict: [set: [name: get_field(changeset, :name)]],
conflict_target: :name,
returning: true
)
end
def get_or_create_by_names(names) when is_list(names) do
names = Enum.map(names, &normalize_name/1)
timestamp = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)
structs =
Enum.map(names, fn name ->
%Hashtag{}
|> changeset(%{name: name})
|> Map.get(:changes)
|> Map.merge(%{inserted_at: timestamp, updated_at: timestamp})
end)
try do
with {:ok, %{query_op: hashtags}} <-
Multi.new()
|> Multi.insert_all(:insert_all_op, Hashtag, structs,
on_conflict: :nothing,
conflict_target: :name
)
|> Multi.run(:query_op, fn _repo, _changes ->
{:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))}
end)
|> Repo.transaction() do
{:ok, hashtags}
else
{:error, _name, value, _changes_so_far} -> {:error, value}
end
rescue
e -> {:error, e}
end
end
def changeset(%Hashtag{} = struct, params) do
struct
|> cast(params, [:name])
|> update_change(:name, &normalize_name/1)
|> validate_required([:name])
|> unique_constraint(:name)
end
def unlink(%Object{id: object_id}) do
with {_, hashtag_ids} <-
from(hto in "hashtags_objects",
where: hto.object_id == ^object_id,
select: hto.hashtag_id
)
|> Repo.delete_all(),
{:ok, unreferenced_count} <- delete_unreferenced(hashtag_ids) do
{:ok, length(hashtag_ids), unreferenced_count}
end
end
@delete_unreferenced_query """
DELETE FROM hashtags WHERE id IN
(SELECT hashtags.id FROM hashtags
LEFT OUTER JOIN hashtags_objects
ON hashtags_objects.hashtag_id = hashtags.id
WHERE hashtags_objects.hashtag_id IS NULL AND hashtags.id = ANY($1));
"""
def delete_unreferenced(ids) do
with {:ok, %{num_rows: deleted_count}} <- Repo.query(@delete_unreferenced_query, [ids]) do
{:ok, deleted_count}
end
end
end

View file

@ -49,31 +49,6 @@ defmodule Pleroma.HTML do
def filter_tags(html), do: filter_tags(html, nil) def filter_tags(html), do: filter_tags(html, nil)
def strip_tags(html), do: filter_tags(html, FastSanitize.Sanitizer.StripTags) def strip_tags(html), do: filter_tags(html, FastSanitize.Sanitizer.StripTags)
def get_cached_scrubbed_html_for_activity(
content,
scrubbers,
activity,
key \\ "",
callback \\ fn x -> x end
) do
key = "#{key}#{generate_scrubber_signature(scrubbers)}|#{activity.id}"
@cachex.fetch!(:scrubber_cache, key, fn _key ->
object = Pleroma.Object.normalize(activity, fetch: false)
ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback)
end)
end
def get_cached_stripped_html_for_activity(content, activity, key) do
get_cached_scrubbed_html_for_activity(
content,
FastSanitize.Sanitizer.StripTags,
activity,
key,
&HtmlEntities.decode/1
)
end
def ensure_scrubbed_html( def ensure_scrubbed_html(
content, content,
scrubbers, scrubbers,
@ -92,16 +67,6 @@ defmodule Pleroma.HTML do
end end
end end
defp generate_scrubber_signature(scrubber) when is_atom(scrubber) do
generate_scrubber_signature([scrubber])
end
defp generate_scrubber_signature(scrubbers) do
Enum.reduce(scrubbers, "", fn scrubber, signature ->
"#{signature}#{to_string(scrubber)}"
end)
end
def extract_first_external_url_from_object(%{data: %{"content" => content}} = object) def extract_first_external_url_from_object(%{data: %{"content" => content}} = object)
when is_binary(content) do when is_binary(content) do
unless object.data["fake"] do unless object.data["fake"] do

View file

@ -54,8 +54,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
Config.get([:pools, pool, :recv_timeout], default) Config.get([:pools, pool, :recv_timeout], default)
end end
@prefix Pleroma.Gun.ConnectionPool
def limiter_setup do def limiter_setup do
prefix = Pleroma.Gun.ConnectionPool
wait = Config.get([:connections_pool, :connection_acquisition_wait]) wait = Config.get([:connections_pool, :connection_acquisition_wait])
retries = Config.get([:connections_pool, :connection_acquisition_retries]) retries = Config.get([:connections_pool, :connection_acquisition_retries])
@ -66,7 +66,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
max_waiting = Keyword.get(opts, :max_waiting, 10) max_waiting = Keyword.get(opts, :max_waiting, 10)
result = result =
ConcurrentLimiter.new(:"#{@prefix}.#{name}", max_running, max_waiting, ConcurrentLimiter.new(:"#{prefix}.#{name}", max_running, max_waiting,
wait: wait, wait: wait,
max_retries: retries max_retries: retries
) )

View file

@ -5,8 +5,8 @@
defmodule Pleroma.HTTP.WebPush do defmodule Pleroma.HTTP.WebPush do
@moduledoc false @moduledoc false
def post(url, payload, headers) do def post(url, payload, headers, options \\ []) do
list_headers = Map.to_list(headers) list_headers = Map.to_list(headers)
Pleroma.HTTP.post(url, payload, list_headers) Pleroma.HTTP.post(url, payload, list_headers, options)
end end
end end

View file

@ -5,13 +5,18 @@
defmodule Pleroma.Instances do defmodule Pleroma.Instances do
@moduledoc "Instances context." @moduledoc "Instances context."
@adapter Pleroma.Instances.Instance alias Pleroma.Instances.Instance
defdelegate filter_reachable(urls_or_hosts), to: @adapter def filter_reachable(urls_or_hosts), do: Instance.filter_reachable(urls_or_hosts)
defdelegate reachable?(url_or_host), to: @adapter
defdelegate set_reachable(url_or_host), to: @adapter def reachable?(url_or_host), do: Instance.reachable?(url_or_host)
defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: @adapter
defdelegate get_consistently_unreachable(), to: @adapter def set_reachable(url_or_host), do: Instance.set_reachable(url_or_host)
def set_unreachable(url_or_host, unreachable_since \\ nil),
do: Instance.set_unreachable(url_or_host, unreachable_since)
def get_consistently_unreachable, do: Instance.get_consistently_unreachable()
def set_consistently_unreachable(url_or_host), def set_consistently_unreachable(url_or_host),
do: set_unreachable(url_or_host, reachability_datetime_threshold()) do: set_unreachable(url_or_host, reachability_datetime_threshold())

View file

@ -12,4 +12,10 @@ defmodule Pleroma.Maps do
_ -> map _ -> map
end end
end end
def safe_put_in(data, keys, value) when is_map(data) and is_list(keys) do
Kernel.put_in(data, keys, value)
rescue
_ -> data
end
end end

View file

@ -0,0 +1,208 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Migrators.HashtagsTableMigrator do
defmodule State do
use Pleroma.Migrators.Support.BaseMigratorState
@impl Pleroma.Migrators.Support.BaseMigratorState
defdelegate data_migration(), to: Pleroma.DataMigration, as: :populate_hashtags_table
end
use Pleroma.Migrators.Support.BaseMigrator
alias Pleroma.Hashtag
alias Pleroma.Migrators.Support.BaseMigrator
alias Pleroma.Object
@impl BaseMigrator
def feature_config_path, do: [:features, :improved_hashtag_timeline]
@impl BaseMigrator
def fault_rate_allowance, do: Config.get([:populate_hashtags_table, :fault_rate_allowance], 0)
@impl BaseMigrator
def perform do
data_migration_id = data_migration_id()
max_processed_id = get_stat(:max_processed_id, 0)
Logger.info("Transferring embedded hashtags to `hashtags` (from oid: #{max_processed_id})...")
query()
|> where([object], object.id > ^max_processed_id)
|> Repo.chunk_stream(100, :batches, timeout: :infinity)
|> Stream.each(fn objects ->
object_ids = Enum.map(objects, & &1.id)
results = Enum.map(objects, &transfer_object_hashtags(&1))
failed_ids =
results
|> Enum.filter(&(elem(&1, 0) == :error))
|> Enum.map(&elem(&1, 1))
# Count of objects with hashtags: `{:noop, id}` is returned for objects having other AS2 tags
chunk_affected_count =
results
|> Enum.filter(&(elem(&1, 0) == :ok))
|> length()
for failed_id <- failed_ids do
_ =
Repo.query(
"INSERT INTO data_migration_failed_ids(data_migration_id, record_id) " <>
"VALUES ($1, $2) ON CONFLICT DO NOTHING;",
[data_migration_id, failed_id]
)
end
_ =
Repo.query(
"DELETE FROM data_migration_failed_ids " <>
"WHERE data_migration_id = $1 AND record_id = ANY($2)",
[data_migration_id, object_ids -- failed_ids]
)
max_object_id = Enum.at(object_ids, -1)
put_stat(:max_processed_id, max_object_id)
increment_stat(:iteration_processed_count, length(object_ids))
increment_stat(:processed_count, length(object_ids))
increment_stat(:failed_count, length(failed_ids))
increment_stat(:affected_count, chunk_affected_count)
put_stat(:records_per_second, records_per_second())
persist_state()
# A quick and dirty approach to controlling the load this background migration imposes
sleep_interval = Config.get([:populate_hashtags_table, :sleep_interval_ms], 0)
Process.sleep(sleep_interval)
end)
|> Stream.run()
end
@impl BaseMigrator
def query do
# Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out)
# Note: not checking activity type, expecting remove_non_create_objects_hashtags/_ to clean up
from(
object in Object,
where:
fragment("(?)->'tag' IS NOT NULL AND (?)->'tag' != '[]'::jsonb", object.data, object.data),
select: %{
id: object.id,
tag: fragment("(?)->'tag'", object.data)
}
)
|> join(:left, [o], hashtags_objects in fragment("SELECT object_id FROM hashtags_objects"),
on: hashtags_objects.object_id == o.id
)
|> where([_o, hashtags_objects], is_nil(hashtags_objects.object_id))
end
@spec transfer_object_hashtags(Map.t()) :: {:noop | :ok | :error, integer()}
defp transfer_object_hashtags(object) do
embedded_tags = if Map.has_key?(object, :tag), do: object.tag, else: object.data["tag"]
hashtags = Object.object_data_hashtags(%{"tag" => embedded_tags})
if Enum.any?(hashtags) do
transfer_object_hashtags(object, hashtags)
else
{:noop, object.id}
end
end
defp transfer_object_hashtags(object, hashtags) do
Repo.transaction(fn ->
with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do
maps = Enum.map(hashtag_records, &%{hashtag_id: &1.id, object_id: object.id})
base_error = "ERROR when inserting hashtags_objects for object with id #{object.id}"
try do
with {rows_count, _} when is_integer(rows_count) <-
Repo.insert_all("hashtags_objects", maps, on_conflict: :nothing) do
object.id
else
e ->
Logger.error("#{base_error}: #{inspect(e)}")
Repo.rollback(object.id)
end
rescue
e ->
Logger.error("#{base_error}: #{inspect(e)}")
Repo.rollback(object.id)
end
else
e ->
error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}"
Logger.error(error)
Repo.rollback(object.id)
end
end)
end
@impl BaseMigrator
def retry_failed do
data_migration_id = data_migration_id()
failed_objects_query()
|> Repo.chunk_stream(100, :one)
|> Stream.each(fn object ->
with {res, _} when res != :error <- transfer_object_hashtags(object) do
_ =
Repo.query(
"DELETE FROM data_migration_failed_ids " <>
"WHERE data_migration_id = $1 AND record_id = $2",
[data_migration_id, object.id]
)
end
end)
|> Stream.run()
put_stat(:failed_count, failures_count())
persist_state()
force_continue()
end
defp failed_objects_query do
from(o in Object)
|> join(:inner, [o], dmf in fragment("SELECT * FROM data_migration_failed_ids"),
on: dmf.record_id == o.id
)
|> where([_o, dmf], dmf.data_migration_id == ^data_migration_id())
|> order_by([o], asc: o.id)
end
@doc """
Service func to delete `hashtags_objects` for legacy objects not associated with Create activity.
Also deletes unreferenced `hashtags` records (might occur after deletion of `hashtags_objects`).
"""
def delete_non_create_activities_hashtags do
hashtags_objects_cleanup_query = """
DELETE FROM hashtags_objects WHERE object_id IN
(SELECT DISTINCT objects.id FROM objects
JOIN hashtags_objects ON hashtags_objects.object_id = objects.id LEFT JOIN activities
ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') =
(objects.data->>'id')
AND activities.data->>'type' = 'Create'
WHERE activities.id IS NULL);
"""
hashtags_cleanup_query = """
DELETE FROM hashtags WHERE id IN
(SELECT hashtags.id FROM hashtags
LEFT OUTER JOIN hashtags_objects
ON hashtags_objects.hashtag_id = hashtags.id
WHERE hashtags_objects.hashtag_id IS NULL);
"""
{:ok, %{num_rows: hashtags_objects_count}} =
Repo.query(hashtags_objects_cleanup_query, [], timeout: :infinity)
{:ok, %{num_rows: hashtags_count}} =
Repo.query(hashtags_cleanup_query, [], timeout: :infinity)
{:ok, hashtags_objects_count, hashtags_count}
end
end

View file

@ -0,0 +1,210 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Migrators.Support.BaseMigrator do
@moduledoc """
Base background migrator functionality.
"""
@callback perform() :: any()
@callback retry_failed() :: any()
@callback feature_config_path() :: list(atom())
@callback query() :: Ecto.Query.t()
@callback fault_rate_allowance() :: integer() | float()
defmacro __using__(_opts) do
quote do
use GenServer
require Logger
import Ecto.Query
alias __MODULE__.State
alias Pleroma.Config
alias Pleroma.Repo
@behaviour Pleroma.Migrators.Support.BaseMigrator
defdelegate data_migration(), to: State
defdelegate data_migration_id(), to: State
defdelegate state(), to: State
defdelegate persist_state(), to: State, as: :persist_to_db
defdelegate get_stat(key, value \\ nil), to: State, as: :get_data_key
defdelegate put_stat(key, value), to: State, as: :put_data_key
defdelegate increment_stat(key, increment), to: State, as: :increment_data_key
@reg_name {:global, __MODULE__}
def whereis, do: GenServer.whereis(@reg_name)
def start_link(_) do
case whereis() do
nil ->
GenServer.start_link(__MODULE__, nil, name: @reg_name)
pid ->
{:ok, pid}
end
end
@impl true
def init(_) do
{:ok, nil, {:continue, :init_state}}
end
@impl true
def handle_continue(:init_state, _state) do
{:ok, _} = State.start_link(nil)
data_migration = data_migration()
manual_migrations = Config.get([:instance, :manual_data_migrations], [])
cond do
Config.get(:env) == :test ->
update_status(:noop)
is_nil(data_migration) ->
message = "Data migration does not exist."
update_status(:failed, message)
Logger.error("#{__MODULE__}: #{message}")
data_migration.state == :manual or data_migration.name in manual_migrations ->
message = "Data migration is in manual execution or manual fix mode."
update_status(:manual, message)
Logger.warn("#{__MODULE__}: #{message}")
data_migration.state == :complete ->
on_complete(data_migration)
true ->
send(self(), :perform)
end
{:noreply, nil}
end
@impl true
def handle_info(:perform, state) do
State.reinit()
update_status(:running)
put_stat(:iteration_processed_count, 0)
put_stat(:started_at, NaiveDateTime.utc_now())
perform()
fault_rate = fault_rate()
put_stat(:fault_rate, fault_rate)
fault_rate_allowance = fault_rate_allowance()
cond do
fault_rate == 0 ->
set_complete()
is_float(fault_rate) and fault_rate <= fault_rate_allowance ->
message = """
Done with fault rate of #{fault_rate} which doesn't exceed #{fault_rate_allowance}.
Putting data migration to manual fix mode. Try running `#{__MODULE__}.retry_failed/0`.
"""
Logger.warn("#{__MODULE__}: #{message}")
update_status(:manual, message)
on_complete(data_migration())
true ->
message = "Too many failures. Try running `#{__MODULE__}.retry_failed/0`."
Logger.error("#{__MODULE__}: #{message}")
update_status(:failed, message)
end
persist_state()
{:noreply, state}
end
defp on_complete(data_migration) do
if data_migration.feature_lock || feature_state() == :disabled do
Logger.warn(
"#{__MODULE__}: migration complete but feature is locked; consider enabling."
)
:noop
else
Config.put(feature_config_path(), :enabled)
:ok
end
end
@doc "Approximate count for current iteration (including processed records count)"
def count(force \\ false, timeout \\ :infinity) do
stored_count = get_stat(:count)
if stored_count && !force do
stored_count
else
processed_count = get_stat(:processed_count, 0)
max_processed_id = get_stat(:max_processed_id, 0)
query = where(query(), [entity], entity.id > ^max_processed_id)
count = Repo.aggregate(query, :count, :id, timeout: timeout) + processed_count
put_stat(:count, count)
persist_state()
count
end
end
def failures_count do
with {:ok, %{rows: [[count]]}} <-
Repo.query(
"SELECT COUNT(record_id) FROM data_migration_failed_ids WHERE data_migration_id = $1;",
[data_migration_id()]
) do
count
end
end
def feature_state, do: Config.get(feature_config_path())
def force_continue do
send(whereis(), :perform)
end
def force_restart do
:ok = State.reset()
force_continue()
end
def set_complete do
update_status(:complete)
persist_state()
on_complete(data_migration())
end
defp update_status(status, message \\ nil) do
put_stat(:state, status)
put_stat(:message, message)
end
defp fault_rate do
with failures_count when is_integer(failures_count) <- failures_count() do
failures_count / Enum.max([get_stat(:affected_count, 0), 1])
else
_ -> :error
end
end
defp records_per_second do
get_stat(:iteration_processed_count, 0) / Enum.max([running_time(), 1])
end
defp running_time do
NaiveDateTime.diff(
NaiveDateTime.utc_now(),
get_stat(:started_at, NaiveDateTime.utc_now())
)
end
end
end
end

View file

@ -0,0 +1,117 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Migrators.Support.BaseMigratorState do
@moduledoc """
Base background migrator state functionality.
"""
@callback data_migration() :: Pleroma.DataMigration.t()
defmacro __using__(_opts) do
quote do
use Agent
alias Pleroma.DataMigration
@behaviour Pleroma.Migrators.Support.BaseMigratorState
@reg_name {:global, __MODULE__}
def start_link(_) do
Agent.start_link(fn -> load_state_from_db() end, name: @reg_name)
end
def data_migration, do: raise("data_migration/0 is not implemented")
defoverridable data_migration: 0
defp load_state_from_db do
data_migration = data_migration()
data =
if data_migration do
Map.new(data_migration.data, fn {k, v} -> {String.to_atom(k), v} end)
else
%{}
end
%{
data_migration_id: data_migration && data_migration.id,
data: data
}
end
def persist_to_db do
%{data_migration_id: data_migration_id, data: data} = state()
if data_migration_id do
DataMigration.update_one_by_id(data_migration_id, data: data)
else
{:error, :nil_data_migration_id}
end
end
def reset do
%{data_migration_id: data_migration_id} = state()
with false <- is_nil(data_migration_id),
:ok <-
DataMigration.update_one_by_id(data_migration_id,
state: :pending,
data: %{}
) do
reinit()
else
true -> {:error, :nil_data_migration_id}
e -> e
end
end
def reinit do
Agent.update(@reg_name, fn _state -> load_state_from_db() end)
end
def state do
Agent.get(@reg_name, & &1)
end
def get_data_key(key, default \\ nil) do
get_in(state(), [:data, key]) || default
end
def put_data_key(key, value) do
_ = persist_non_data_change(key, value)
Agent.update(@reg_name, fn state ->
put_in(state, [:data, key], value)
end)
end
def increment_data_key(key, increment \\ 1) do
Agent.update(@reg_name, fn state ->
initial_value = get_in(state, [:data, key]) || 0
updated_value = initial_value + increment
put_in(state, [:data, key], updated_value)
end)
end
defp persist_non_data_change(:state, value) do
with true <- get_data_key(:state) != value,
true <- value in Pleroma.DataMigration.State.__valid_values__(),
%{data_migration_id: data_migration_id} when not is_nil(data_migration_id) <-
state() do
DataMigration.update_one_by_id(data_migration_id, state: value)
else
false -> :ok
_ -> {:error, :nil_data_migration_id}
end
end
defp persist_non_data_change(_, _) do
nil
end
def data_migration_id, do: Map.get(state(), :data_migration_id)
end
end
end

View file

@ -10,6 +10,7 @@ defmodule Pleroma.Object do
alias Pleroma.Activity alias Pleroma.Activity
alias Pleroma.Config alias Pleroma.Config
alias Pleroma.Hashtag
alias Pleroma.Object alias Pleroma.Object
alias Pleroma.Object.Fetcher alias Pleroma.Object.Fetcher
alias Pleroma.ObjectTombstone alias Pleroma.ObjectTombstone
@ -28,6 +29,8 @@ defmodule Pleroma.Object do
schema "objects" do schema "objects" do
field(:data, :map) field(:data, :map)
many_to_many(:hashtags, Hashtag, join_through: "hashtags_objects", on_replace: :delete)
timestamps() timestamps()
end end
@ -49,7 +52,8 @@ defmodule Pleroma.Object do
end end
def create(data) do def create(data) do
Object.change(%Object{}, %{data: data}) %Object{}
|> Object.change(%{data: data})
|> Repo.insert() |> Repo.insert()
end end
@ -58,8 +62,41 @@ defmodule Pleroma.Object do
|> cast(params, [:data]) |> cast(params, [:data])
|> validate_required([:data]) |> validate_required([:data])
|> unique_constraint(:ap_id, name: :objects_unique_apid_index) |> unique_constraint(:ap_id, name: :objects_unique_apid_index)
# Expecting `maybe_handle_hashtags_change/1` to run last:
|> maybe_handle_hashtags_change(struct)
end end
# Note: not checking activity type (assuming non-legacy objects are associated with Create act.)
defp maybe_handle_hashtags_change(changeset, struct) do
with %Ecto.Changeset{valid?: true} <- changeset,
data_hashtags_change = get_change(changeset, :data),
{_, true} <- {:changed, hashtags_changed?(struct, data_hashtags_change)},
{:ok, hashtag_records} <-
data_hashtags_change
|> object_data_hashtags()
|> Hashtag.get_or_create_by_names() do
put_assoc(changeset, :hashtags, hashtag_records)
else
%{valid?: false} ->
changeset
{:changed, false} ->
changeset
{:error, _} ->
validate_change(changeset, :data, fn _, _ ->
[data: "error referencing hashtags"]
end)
end
end
defp hashtags_changed?(%Object{} = struct, %{"tag" => _} = data) do
Enum.sort(embedded_hashtags(struct)) !=
Enum.sort(object_data_hashtags(data))
end
defp hashtags_changed?(_, _), do: false
def get_by_id(nil), do: nil def get_by_id(nil), do: nil
def get_by_id(id), do: Repo.get(Object, id) def get_by_id(id), do: Repo.get(Object, id)
@ -187,9 +224,13 @@ defmodule Pleroma.Object do
def swap_object_with_tombstone(object) do def swap_object_with_tombstone(object) do
tombstone = make_tombstone(object) tombstone = make_tombstone(object)
object with {:ok, object} <-
|> Object.change(%{data: tombstone}) object
|> Repo.update() |> Object.change(%{data: tombstone})
|> Repo.update() do
Hashtag.unlink(object)
{:ok, object}
end
end end
def delete(%Object{data: %{"id" => id}} = object) do def delete(%Object{data: %{"id" => id}} = object) do
@ -325,7 +366,7 @@ defmodule Pleroma.Object do
end end
def local?(%Object{data: %{"id" => id}}) do def local?(%Object{data: %{"id" => id}}) do
String.starts_with?(id, Pleroma.Web.base_url() <> "/") String.starts_with?(id, Pleroma.Web.Endpoint.url() <> "/")
end end
def replies(object, opts \\ []) do def replies(object, opts \\ []) do
@ -349,4 +390,39 @@ defmodule Pleroma.Object do
def self_replies(object, opts \\ []), def self_replies(object, opts \\ []),
do: replies(object, Keyword.put(opts, :self_only, true)) do: replies(object, Keyword.put(opts, :self_only, true))
def tags(%Object{data: %{"tag" => tags}}) when is_list(tags), do: tags
def tags(_), do: []
def hashtags(%Object{} = object) do
# Note: always using embedded hashtags regardless whether they are migrated to hashtags table
# (embedded hashtags stay in sync anyways, and we avoid extra joins and preload hassle)
embedded_hashtags(object)
end
def embedded_hashtags(%Object{data: data}) do
object_data_hashtags(data)
end
def embedded_hashtags(_), do: []
def object_data_hashtags(%{"tag" => tags}) when is_list(tags) do
tags
|> Enum.filter(fn
%{"type" => "Hashtag"} = data -> Map.has_key?(data, "name")
plain_text when is_bitstring(plain_text) -> true
_ -> false
end)
|> Enum.map(fn
%{"name" => "#" <> hashtag} -> String.downcase(hashtag)
%{"name" => hashtag} -> String.downcase(hashtag)
hashtag when is_bitstring(hashtag) -> String.downcase(hashtag)
end)
|> Enum.uniq()
# Note: "" elements (plain text) might occur in `data.tag` for incoming objects
|> Enum.filter(&(&1 not in [nil, ""]))
end
def object_data_hashtags(_), do: []
end end

View file

@ -71,6 +71,14 @@ defmodule Pleroma.Object.Containment do
compare_uris(id_uri, other_uri) compare_uris(id_uri, other_uri)
end end
# Mastodon pin activities don't have an id, so we check the object field, which will be pinned.
def contain_origin_from_id(id, %{"object" => object}) when is_binary(object) do
id_uri = URI.parse(id)
object_uri = URI.parse(object)
compare_uris(id_uri, object_uri)
end
def contain_origin_from_id(_id, _data), do: :error def contain_origin_from_id(_id, _data), do: :error
def contain_child(%{"object" => %{"id" => id, "attributedTo" => _} = object}), def contain_child(%{"object" => %{"id" => id, "attributedTo" => _} = object}),

View file

@ -4,6 +4,7 @@
defmodule Pleroma.Object.Fetcher do defmodule Pleroma.Object.Fetcher do
alias Pleroma.HTTP alias Pleroma.HTTP
alias Pleroma.Maps
alias Pleroma.Object alias Pleroma.Object
alias Pleroma.Object.Containment alias Pleroma.Object.Containment
alias Pleroma.Repo alias Pleroma.Repo
@ -101,6 +102,9 @@ defmodule Pleroma.Object.Fetcher do
{:transmogrifier, {:error, {:reject, e}}} -> {:transmogrifier, {:error, {:reject, e}}} ->
{:reject, e} {:reject, e}
{:transmogrifier, {:reject, e}} ->
{:reject, e}
{:transmogrifier, _} = e -> {:transmogrifier, _} = e ->
{:error, e} {:error, e}
@ -124,12 +128,14 @@ defmodule Pleroma.Object.Fetcher do
defp prepare_activity_params(data) do defp prepare_activity_params(data) do
%{ %{
"type" => "Create", "type" => "Create",
"to" => data["to"] || [],
"cc" => data["cc"] || [],
# Should we seriously keep this attributedTo thing? # Should we seriously keep this attributedTo thing?
"actor" => data["actor"] || data["attributedTo"], "actor" => data["actor"] || data["attributedTo"],
"object" => data "object" => data
} }
|> Maps.put_if_present("to", data["to"])
|> Maps.put_if_present("cc", data["cc"])
|> Maps.put_if_present("bto", data["bto"])
|> Maps.put_if_present("bcc", data["bcc"])
end end
def fetch_object_from_id!(id, options \\ []) do def fetch_object_from_id!(id, options \\ []) do

View file

@ -93,6 +93,7 @@ defmodule Pleroma.Pagination do
max_id: :string, max_id: :string,
offset: :integer, offset: :integer,
limit: :integer, limit: :integer,
skip_extra_order: :boolean,
skip_order: :boolean skip_order: :boolean
} }
@ -114,6 +115,8 @@ defmodule Pleroma.Pagination do
defp restrict(query, :order, %{skip_order: true}, _), do: query defp restrict(query, :order, %{skip_order: true}, _), do: query
defp restrict(%{order_bys: [_ | _]} = query, :order, %{skip_extra_order: true}, _), do: query
defp restrict(query, :order, %{min_id: _}, table_binding) do defp restrict(query, :order, %{min_id: _}, table_binding) do
order_by( order_by(
query, query,

View file

@ -63,8 +63,8 @@ defmodule Pleroma.Repo do
iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500, :batches) iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500, :batches)
""" """
@spec chunk_stream(Ecto.Query.t(), integer(), atom()) :: Enumerable.t() @spec chunk_stream(Ecto.Query.t(), integer(), atom()) :: Enumerable.t()
def chunk_stream(query, chunk_size, returns_as \\ :one) do def chunk_stream(query, chunk_size, returns_as \\ :one, query_options \\ []) do
# We don't actually need start and end funcitons of resource streaming, # We don't actually need start and end functions of resource streaming,
# but it seems to be the only way to not fetch records one-by-one and # but it seems to be the only way to not fetch records one-by-one and
# have individual records be the elements of the stream, instead of # have individual records be the elements of the stream, instead of
# lists of records # lists of records
@ -76,7 +76,7 @@ defmodule Pleroma.Repo do
|> order_by(asc: :id) |> order_by(asc: :id)
|> where([r], r.id > ^last_id) |> where([r], r.id > ^last_id)
|> limit(^chunk_size) |> limit(^chunk_size)
|> all() |> all(query_options)
|> case do |> case do
[] -> [] ->
{:halt, last_id} {:halt, last_id}

View file

@ -411,7 +411,7 @@ defmodule Pleroma.ReverseProxy do
{:ok, :no_duration_limit, :no_duration_limit} {:ok, :no_duration_limit, :no_duration_limit}
end end
defp client, do: Pleroma.ReverseProxy.Client defp client, do: Pleroma.ReverseProxy.Client.Wrapper
defp track_failed_url(url, error, opts) do defp track_failed_url(url, error, opts) do
ttl = ttl =

View file

@ -17,22 +17,4 @@ defmodule Pleroma.ReverseProxy.Client do
@callback stream_body(map()) :: {:ok, binary(), map()} | :done | {:error, atom() | String.t()} @callback stream_body(map()) :: {:ok, binary(), map()} | :done | {:error, atom() | String.t()}
@callback close(reference() | pid() | map()) :: :ok @callback close(reference() | pid() | map()) :: :ok
def request(method, url, headers, body \\ "", opts \\ []) do
client().request(method, url, headers, body, opts)
end
def stream_body(ref), do: client().stream_body(ref)
def close(ref), do: client().close(ref)
defp client do
:tesla
|> Application.get_env(:adapter)
|> client()
end
defp client(Tesla.Adapter.Hackney), do: Pleroma.ReverseProxy.Client.Hackney
defp client(Tesla.Adapter.Gun), do: Pleroma.ReverseProxy.Client.Tesla
defp client(_), do: Pleroma.Config.get!(Pleroma.ReverseProxy.Client)
end end

View file

@ -0,0 +1,29 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.ReverseProxy.Client.Wrapper do
@moduledoc "Meta-client that calls the appropriate client from the config."
@behaviour Pleroma.ReverseProxy.Client
@impl true
def request(method, url, headers, body \\ "", opts \\ []) do
client().request(method, url, headers, body, opts)
end
@impl true
def stream_body(ref), do: client().stream_body(ref)
@impl true
def close(ref), do: client().close(ref)
defp client do
:tesla
|> Application.get_env(:adapter)
|> client()
end
defp client(Tesla.Adapter.Hackney), do: Pleroma.ReverseProxy.Client.Hackney
defp client(Tesla.Adapter.Gun), do: Pleroma.ReverseProxy.Client.Tesla
defp client(_), do: Pleroma.Config.get!(Pleroma.ReverseProxy.Client)
end

View file

@ -9,7 +9,6 @@ defmodule Pleroma.Tests.AuthTestController do
use Pleroma.Web, :controller use Pleroma.Web, :controller
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.Plugs.EnsurePublicOrAuthenticatedPlug
alias Pleroma.Web.Plugs.OAuthScopesPlug alias Pleroma.Web.Plugs.OAuthScopesPlug
# Serves only with proper OAuth token (:api and :authenticated_api) # Serves only with proper OAuth token (:api and :authenticated_api)
@ -47,10 +46,7 @@ defmodule Pleroma.Tests.AuthTestController do
# Via :authenticated_api, serves if token is present and has requested scopes # Via :authenticated_api, serves if token is present and has requested scopes
# #
# Suggested use: as :fallback_oauth_check but open with nil :user for :api on private instances # Suggested use: as :fallback_oauth_check but open with nil :user for :api on private instances
plug( plug(:skip_public_check when action == :fallback_oauth_skip_publicity_check)
:skip_plug,
EnsurePublicOrAuthenticatedPlug when action == :fallback_oauth_skip_publicity_check
)
plug( plug(
OAuthScopesPlug, OAuthScopesPlug,
@ -62,11 +58,7 @@ defmodule Pleroma.Tests.AuthTestController do
# Via :authenticated_api, serves if :user is set (regardless of token presence and its scopes) # Via :authenticated_api, serves if :user is set (regardless of token presence and its scopes)
# #
# Suggested use: making an :api endpoint always accessible (e.g. email confirmation endpoint) # Suggested use: making an :api endpoint always accessible (e.g. email confirmation endpoint)
plug( plug(:skip_auth when action == :skip_oauth_skip_publicity_check)
:skip_plug,
[OAuthScopesPlug, EnsurePublicOrAuthenticatedPlug]
when action == :skip_oauth_skip_publicity_check
)
# Via :authenticated_api, always fails with 403 (endpoint is insecure) # Via :authenticated_api, always fails with 403 (endpoint is insecure)
# Via :api, drops :user if present and serves if public (private instance rejects on no user) # Via :api, drops :user if present and serves if public (private instance rejects on no user)

View file

@ -23,6 +23,9 @@ defmodule Pleroma.Upload do
is once created permanent and changing it (especially in uploaders) is probably a bad idea! is once created permanent and changing it (especially in uploaders) is probably a bad idea!
* `:tempfile` - path to the temporary file. Prefer in-place changes on the file rather than changing the * `:tempfile` - path to the temporary file. Prefer in-place changes on the file rather than changing the
path as the temporary file is also tracked by `Plug.Upload{}` and automatically deleted once the request is over. path as the temporary file is also tracked by `Plug.Upload{}` and automatically deleted once the request is over.
* `:width` - width of the media in pixels
* `:height` - height of the media in pixels
* `:blurhash` - string hash of the image encoded with the blurhash algorithm (https://blurha.sh/)
Related behaviors: Related behaviors:
@ -32,6 +35,7 @@ defmodule Pleroma.Upload do
""" """
alias Ecto.UUID alias Ecto.UUID
alias Pleroma.Config alias Pleroma.Config
alias Pleroma.Maps
require Logger require Logger
@type source :: @type source ::
@ -53,9 +57,12 @@ defmodule Pleroma.Upload do
name: String.t(), name: String.t(),
tempfile: String.t(), tempfile: String.t(),
content_type: String.t(), content_type: String.t(),
width: integer(),
height: integer(),
blurhash: String.t(),
path: String.t() path: String.t()
} }
defstruct [:id, :name, :tempfile, :content_type, :path] defstruct [:id, :name, :tempfile, :content_type, :width, :height, :blurhash, :path]
defp get_description(opts, upload) do defp get_description(opts, upload) do
case {opts[:description], Pleroma.Config.get([Pleroma.Upload, :default_description])} do case {opts[:description], Pleroma.Config.get([Pleroma.Upload, :default_description])} do
@ -89,9 +96,12 @@ defmodule Pleroma.Upload do
"mediaType" => upload.content_type, "mediaType" => upload.content_type,
"href" => url_from_spec(upload, opts.base_url, url_spec) "href" => url_from_spec(upload, opts.base_url, url_spec)
} }
|> Maps.put_if_present("width", upload.width)
|> Maps.put_if_present("height", upload.height)
], ],
"name" => description "name" => description
}} }
|> Maps.put_if_present("blurhash", upload.blurhash)}
else else
{:description_limit, _} -> {:description_limit, _} ->
{:error, :description_too_long} {:error, :description_too_long}
@ -225,7 +235,7 @@ defmodule Pleroma.Upload do
case uploader do case uploader do
Pleroma.Uploaders.Local -> Pleroma.Uploaders.Local ->
upload_base_url || Pleroma.Web.base_url() <> "/media/" upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
Pleroma.Uploaders.S3 -> Pleroma.Uploaders.S3 ->
bucket = Config.get([Pleroma.Uploaders.S3, :bucket]) bucket = Config.get([Pleroma.Uploaders.S3, :bucket])
@ -251,7 +261,7 @@ defmodule Pleroma.Upload do
end end
_ -> _ ->
public_endpoint || upload_base_url || Pleroma.Web.base_url() <> "/media/" public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/"
end end
end end
end end

View file

@ -15,13 +15,13 @@ defmodule Pleroma.Upload.Filter do
require Logger require Logger
@callback filter(Pleroma.Upload.t()) :: @callback filter(upload :: struct()) ::
{:ok, :filtered} {:ok, :filtered}
| {:ok, :noop} | {:ok, :noop}
| {:ok, :filtered, Pleroma.Upload.t()} | {:ok, :filtered, upload :: struct()}
| {:error, any()} | {:error, any()}
@spec filter([module()], Pleroma.Upload.t()) :: {:ok, Pleroma.Upload.t()} | {:error, any()} @spec filter([module()], upload :: struct()) :: {:ok, upload :: struct()} | {:error, any()}
def filter([], upload) do def filter([], upload) do
{:ok, upload} {:ok, upload}

View file

@ -0,0 +1,83 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Upload.Filter.AnalyzeMetadata do
@moduledoc """
Extracts metadata about the upload, such as width/height
"""
require Logger
@behaviour Pleroma.Upload.Filter
@spec filter(Pleroma.Upload.t()) ::
{:ok, :filtered, Pleroma.Upload.t()} | {:ok, :noop} | {:error, String.t()}
def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do
try do
image =
file
|> Mogrify.open()
|> Mogrify.verbose()
upload =
upload
|> Map.put(:width, image.width)
|> Map.put(:height, image.height)
|> Map.put(:blurhash, get_blurhash(file))
{:ok, :filtered, upload}
rescue
e in ErlangError ->
Logger.warn("#{__MODULE__}: #{inspect(e)}")
{:ok, :noop}
end
end
def filter(%Pleroma.Upload{tempfile: file, content_type: "video" <> _} = upload) do
try do
result = media_dimensions(file)
upload =
upload
|> Map.put(:width, result.width)
|> Map.put(:height, result.height)
{:ok, :filtered, upload}
rescue
e in ErlangError ->
Logger.warn("#{__MODULE__}: #{inspect(e)}")
{:ok, :noop}
end
end
def filter(_), do: {:ok, :noop}
defp get_blurhash(file) do
with {:ok, blurhash} <- :eblurhash.magick(file) do
blurhash
else
_ -> nil
end
end
defp media_dimensions(file) do
with executable when is_binary(executable) <- System.find_executable("ffprobe"),
args = [
"-v",
"error",
"-show_entries",
"stream=width,height",
"-of",
"csv=p=0:s=x",
file
],
{result, 0} <- System.cmd(executable, args),
[width, height] <-
String.split(String.trim(result), "x") |> Enum.map(&String.to_integer(&1)) do
%{width: width, height: height}
else
nil -> {:error, {:ffprobe, :command_not_found}}
{:error, _} = error -> error
end
end
end

View file

@ -35,7 +35,7 @@ defmodule Pleroma.Uploaders.Uploader do
""" """
@type file_spec :: {:file | :url, String.t()} @type file_spec :: {:file | :url, String.t()}
@callback put_file(Pleroma.Upload.t()) :: @callback put_file(upload :: struct()) ::
:ok | {:ok, file_spec()} | {:error, String.t()} | :wait_callback :ok | {:ok, file_spec()} | {:error, String.t()} | :wait_callback
@callback delete_file(file :: String.t()) :: :ok | {:error, String.t()} @callback delete_file(file :: String.t()) :: :ok | {:error, String.t()}
@ -46,7 +46,7 @@ defmodule Pleroma.Uploaders.Uploader do
| {:error, Plug.Conn.t(), String.t()} | {:error, Plug.Conn.t(), String.t()}
@optional_callbacks http_callback: 2 @optional_callbacks http_callback: 2
@spec put_file(module(), Pleroma.Upload.t()) :: {:ok, file_spec()} | {:error, String.t()} @spec put_file(module(), upload :: struct()) :: {:ok, file_spec()} | {:error, String.t()}
def put_file(uploader, upload) do def put_file(uploader, upload) do
case uploader.put_file(upload) do case uploader.put_file(upload) do
:ok -> {:ok, {:file, upload.path}} :ok -> {:ok, {:file, upload.path}}

View file

@ -27,13 +27,13 @@ defmodule Pleroma.User do
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.User alias Pleroma.User
alias Pleroma.UserRelationship alias Pleroma.UserRelationship
alias Pleroma.Web
alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.Builder
alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Pipeline
alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Utils
alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI
alias Pleroma.Web.CommonAPI.Utils, as: CommonUtils alias Pleroma.Web.CommonAPI.Utils, as: CommonUtils
alias Pleroma.Web.Endpoint
alias Pleroma.Web.OAuth alias Pleroma.Web.OAuth
alias Pleroma.Web.RelMe alias Pleroma.Web.RelMe
alias Pleroma.Workers.BackgroundWorker alias Pleroma.Workers.BackgroundWorker
@ -99,6 +99,7 @@ defmodule Pleroma.User do
field(:local, :boolean, default: true) field(:local, :boolean, default: true)
field(:follower_address, :string) field(:follower_address, :string)
field(:following_address, :string) field(:following_address, :string)
field(:featured_address, :string)
field(:search_rank, :float, virtual: true) field(:search_rank, :float, virtual: true)
field(:search_type, :integer, virtual: true) field(:search_type, :integer, virtual: true)
field(:tags, {:array, :string}, default: []) field(:tags, {:array, :string}, default: [])
@ -130,7 +131,6 @@ defmodule Pleroma.User do
field(:hide_followers, :boolean, default: false) field(:hide_followers, :boolean, default: false)
field(:hide_follows, :boolean, default: false) field(:hide_follows, :boolean, default: false)
field(:hide_favorites, :boolean, default: true) field(:hide_favorites, :boolean, default: true)
field(:pinned_activities, {:array, :string}, default: [])
field(:email_notifications, :map, default: %{"digest" => false}) field(:email_notifications, :map, default: %{"digest" => false})
field(:mascot, :map, default: nil) field(:mascot, :map, default: nil)
field(:emoji, :map, default: %{}) field(:emoji, :map, default: %{})
@ -148,6 +148,7 @@ defmodule Pleroma.User do
field(:accepts_chat_messages, :boolean, default: nil) field(:accepts_chat_messages, :boolean, default: nil)
field(:last_active_at, :naive_datetime) field(:last_active_at, :naive_datetime)
field(:disclose_client, :boolean, default: true) field(:disclose_client, :boolean, default: true)
field(:pinned_objects, :map, default: %{})
embeds_one( embeds_one(
:notification_settings, :notification_settings,
@ -359,7 +360,7 @@ defmodule Pleroma.User do
_ -> _ ->
unless options[:no_default] do unless options[:no_default] do
Config.get([:assets, :default_user_avatar], "#{Web.base_url()}/images/avi.png") Config.get([:assets, :default_user_avatar], "#{Endpoint.url()}/images/avi.png")
end end
end end
end end
@ -367,13 +368,15 @@ defmodule Pleroma.User do
def banner_url(user, options \\ []) do def banner_url(user, options \\ []) do
case user.banner do case user.banner do
%{"url" => [%{"href" => href} | _]} -> href %{"url" => [%{"href" => href} | _]} -> href
_ -> !options[:no_default] && "#{Web.base_url()}/images/banner.png" _ -> !options[:no_default] && "#{Endpoint.url()}/images/banner.png"
end end
end end
# Should probably be renamed or removed # Should probably be renamed or removed
def ap_id(%User{nickname: nickname}), do: "#{Web.base_url()}/users/#{nickname}" @spec ap_id(User.t()) :: String.t()
def ap_id(%User{nickname: nickname}), do: "#{Endpoint.url()}/users/#{nickname}"
@spec ap_followers(User.t()) :: String.t()
def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa
def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers" def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers"
@ -381,6 +384,11 @@ defmodule Pleroma.User do
def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa
def ap_following(%User{} = user), do: "#{ap_id(user)}/following" def ap_following(%User{} = user), do: "#{ap_id(user)}/following"
@spec ap_featured_collection(User.t()) :: String.t()
def ap_featured_collection(%User{featured_address: fa}) when is_binary(fa), do: fa
def ap_featured_collection(%User{} = user), do: "#{ap_id(user)}/collections/featured"
defp truncate_fields_param(params) do defp truncate_fields_param(params) do
if Map.has_key?(params, :fields) do if Map.has_key?(params, :fields) do
Map.put(params, :fields, Enum.map(params[:fields], &truncate_field/1)) Map.put(params, :fields, Enum.map(params[:fields], &truncate_field/1))
@ -443,6 +451,7 @@ defmodule Pleroma.User do
:uri, :uri,
:follower_address, :follower_address,
:following_address, :following_address,
:featured_address,
:hide_followers, :hide_followers,
:hide_follows, :hide_follows,
:hide_followers_count, :hide_followers_count,
@ -454,7 +463,8 @@ defmodule Pleroma.User do
:invisible, :invisible,
:actor_type, :actor_type,
:also_known_as, :also_known_as,
:accepts_chat_messages :accepts_chat_messages,
:pinned_objects
] ]
) )
|> cast(params, [:name], empty_values: []) |> cast(params, [:name], empty_values: [])
@ -686,7 +696,7 @@ defmodule Pleroma.User do
|> validate_format(:nickname, local_nickname_regex()) |> validate_format(:nickname, local_nickname_regex())
|> put_ap_id() |> put_ap_id()
|> unique_constraint(:ap_id) |> unique_constraint(:ap_id)
|> put_following_and_follower_address() |> put_following_and_follower_and_featured_address()
end end
def register_changeset(struct, params \\ %{}, opts \\ []) do def register_changeset(struct, params \\ %{}, opts \\ []) do
@ -747,7 +757,7 @@ defmodule Pleroma.User do
|> put_password_hash |> put_password_hash
|> put_ap_id() |> put_ap_id()
|> unique_constraint(:ap_id) |> unique_constraint(:ap_id)
|> put_following_and_follower_address() |> put_following_and_follower_and_featured_address()
end end
def maybe_validate_required_email(changeset, true), do: changeset def maybe_validate_required_email(changeset, true), do: changeset
@ -765,11 +775,16 @@ defmodule Pleroma.User do
put_change(changeset, :ap_id, ap_id) put_change(changeset, :ap_id, ap_id)
end end
defp put_following_and_follower_address(changeset) do defp put_following_and_follower_and_featured_address(changeset) do
followers = ap_followers(%User{nickname: get_field(changeset, :nickname)}) user = %User{nickname: get_field(changeset, :nickname)}
followers = ap_followers(user)
following = ap_following(user)
featured = ap_featured_collection(user)
changeset changeset
|> put_change(:follower_address, followers) |> put_change(:follower_address, followers)
|> put_change(:following_address, following)
|> put_change(:featured_address, featured)
end end
defp autofollow_users(user) do defp autofollow_users(user) do
@ -1680,8 +1695,6 @@ defmodule Pleroma.User do
email: nil, email: nil,
name: nil, name: nil,
password_hash: nil, password_hash: nil,
keys: nil,
public_key: nil,
avatar: %{}, avatar: %{},
tags: [], tags: [],
last_refreshed_at: nil, last_refreshed_at: nil,
@ -1692,9 +1705,7 @@ defmodule Pleroma.User do
follower_count: 0, follower_count: 0,
following_count: 0, following_count: 0,
is_locked: false, is_locked: false,
is_confirmed: true,
password_reset_pending: false, password_reset_pending: false,
is_approved: true,
registration_reason: nil, registration_reason: nil,
confirmation_token: nil, confirmation_token: nil,
domain_blocks: [], domain_blocks: [],
@ -1710,45 +1721,53 @@ defmodule Pleroma.User do
raw_fields: [], raw_fields: [],
is_discoverable: false, is_discoverable: false,
also_known_as: [] also_known_as: []
# id: preserved
# ap_id: preserved
# nickname: preserved
}) })
end end
# Purge doesn't delete the user from the database.
# It just nulls all its fields and deactivates it.
# See `User.purge_user_changeset/1` above.
defp purge(%User{} = user) do
user
|> purge_user_changeset()
|> update_and_set_cache()
end
def delete(users) when is_list(users) do def delete(users) when is_list(users) do
for user <- users, do: delete(user) for user <- users, do: delete(user)
end end
def delete(%User{} = user) do def delete(%User{} = user) do
# Purge the user immediately
purge(user)
BackgroundWorker.enqueue("delete_user", %{"user_id" => user.id}) BackgroundWorker.enqueue("delete_user", %{"user_id" => user.id})
end end
defp delete_and_invalidate_cache(%User{} = user) do # *Actually* delete the user from the DB
defp delete_from_db(%User{} = user) do
invalidate_cache(user) invalidate_cache(user)
Repo.delete(user) Repo.delete(user)
end end
defp delete_or_deactivate(%User{local: false} = user), do: delete_and_invalidate_cache(user) # If the user never finalized their account, it's safe to delete them.
defp maybe_delete_from_db(%User{local: true, is_confirmed: false} = user),
do: delete_from_db(user)
defp delete_or_deactivate(%User{local: true} = user) do defp maybe_delete_from_db(%User{local: true, is_approved: false} = user),
status = account_status(user) do: delete_from_db(user)
case status do defp maybe_delete_from_db(user), do: {:ok, user}
:confirmation_pending ->
delete_and_invalidate_cache(user)
:approval_pending ->
delete_and_invalidate_cache(user)
_ ->
user
|> purge_user_changeset()
|> update_and_set_cache()
end
end
def perform(:force_password_reset, user), do: force_password_reset(user) def perform(:force_password_reset, user), do: force_password_reset(user)
@spec perform(atom(), User.t()) :: {:ok, User.t()} @spec perform(atom(), User.t()) :: {:ok, User.t()}
def perform(:delete, %User{} = user) do def perform(:delete, %User{} = user) do
# Purge the user again, in case perform/2 is called directly
purge(user)
# Remove all relationships # Remove all relationships
user user
|> get_followers() |> get_followers()
@ -1766,10 +1785,9 @@ defmodule Pleroma.User do
delete_user_activities(user) delete_user_activities(user)
delete_notifications_from_user_activities(user) delete_notifications_from_user_activities(user)
delete_outgoing_pending_follow_requests(user) delete_outgoing_pending_follow_requests(user)
delete_or_deactivate(user) maybe_delete_from_db(user)
end end
def perform(:set_activation_async, user, status), do: set_activation(user, status) def perform(:set_activation_async, user, status), do: set_activation(user, status)
@ -2255,13 +2273,6 @@ defmodule Pleroma.User do
|> update_and_set_cache() |> update_and_set_cache()
end end
def roles(%{is_moderator: is_moderator, is_admin: is_admin}) do
%{
admin: is_admin,
moderator: is_moderator
}
end
def validate_fields(changeset, remote? \\ false) do def validate_fields(changeset, remote? \\ false) do
limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields
limit = Config.get([:instance, limit_name], 0) limit = Config.get([:instance, limit_name], 0)
@ -2350,45 +2361,35 @@ defmodule Pleroma.User do
cast(user, %{is_approved: approved?}, [:is_approved]) cast(user, %{is_approved: approved?}, [:is_approved])
end end
def add_pinnned_activity(user, %Pleroma.Activity{id: id}) do @spec add_pinned_object_id(User.t(), String.t()) :: {:ok, User.t()} | {:error, term()}
if id not in user.pinned_activities do def add_pinned_object_id(%User{} = user, object_id) do
max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0) if !user.pinned_objects[object_id] do
params = %{pinned_activities: user.pinned_activities ++ [id]} params = %{pinned_objects: Map.put(user.pinned_objects, object_id, NaiveDateTime.utc_now())}
# if pinned activity was scheduled for deletion, we remove job
if expiration = Pleroma.Workers.PurgeExpiredActivity.get_expiration(id) do
Oban.cancel_job(expiration.id)
end
user user
|> cast(params, [:pinned_activities]) |> cast(params, [:pinned_objects])
|> validate_length(:pinned_activities, |> validate_change(:pinned_objects, fn :pinned_objects, pinned_objects ->
max: max_pinned_statuses, max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0)
message: "You have already pinned the maximum number of statuses"
) if Enum.count(pinned_objects) <= max_pinned_statuses do
[]
else
[pinned_objects: "You have already pinned the maximum number of statuses"]
end
end)
else else
change(user) change(user)
end end
|> update_and_set_cache() |> update_and_set_cache()
end end
def remove_pinnned_activity(user, %Pleroma.Activity{id: id, data: data}) do @spec remove_pinned_object_id(User.t(), String.t()) :: {:ok, t()} | {:error, term()}
params = %{pinned_activities: List.delete(user.pinned_activities, id)} def remove_pinned_object_id(%User{} = user, object_id) do
# if pinned activity was scheduled for deletion, we reschedule it for deletion
if data["expires_at"] do
# MRF.ActivityExpirationPolicy used UTC timestamps for expires_at in original implementation
{:ok, expires_at} =
data["expires_at"] |> Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime.cast()
Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
activity_id: id,
expires_at: expires_at
})
end
user user
|> cast(params, [:pinned_activities]) |> cast(
%{pinned_objects: Map.delete(user.pinned_objects, object_id)},
[:pinned_objects]
)
|> update_and_set_cache() |> update_and_set_cache()
end end

View file

@ -27,7 +27,7 @@ defmodule Pleroma.User.Query do
- e.g. Pleroma.User.Query.build(%{ap_id: ["http://ap_id1", "http://ap_id2"]}) - e.g. Pleroma.User.Query.build(%{ap_id: ["http://ap_id1", "http://ap_id2"]})
""" """
import Ecto.Query import Ecto.Query
import Pleroma.Web.AdminAPI.Search, only: [not_empty_string: 1] import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1]
alias Pleroma.FollowingRelationship alias Pleroma.FollowingRelationship
alias Pleroma.User alias Pleroma.User

View file

@ -11,6 +11,8 @@ defmodule Pleroma.Utils do
eperm epipe erange erofs espipe esrch estale etxtbsy exdev eperm epipe erange erofs espipe esrch estale etxtbsy exdev
)a )a
@repo_timeout Pleroma.Config.get([Pleroma.Repo, :timeout], 15_000)
def compile_dir(dir) when is_binary(dir) do def compile_dir(dir) when is_binary(dir) do
dir dir
|> File.ls!() |> File.ls!()
@ -63,4 +65,21 @@ defmodule Pleroma.Utils do
end end
def posix_error_message(_), do: "" def posix_error_message(_), do: ""
@doc """
Returns [timeout: integer] suitable for passing as an option to Repo functions.
This function detects if the execution was triggered from IEx shell, Mix task, or
./bin/pleroma_ctl and sets the timeout to :infinity, else returns the default timeout value.
"""
@spec query_timeout() :: [timeout: integer]
def query_timeout do
{parent, _, _, _} = Process.info(self(), :current_stacktrace) |> elem(1) |> Enum.fetch!(2)
cond do
parent |> to_string |> String.starts_with?("Elixir.Mix.Task") -> [timeout: :infinity]
parent == :erl_eval -> [timeout: :infinity]
true -> [timeout: @repo_timeout]
end
end
end end

View file

@ -35,9 +35,10 @@ defmodule Pleroma.Web do
import Plug.Conn import Plug.Conn
import Pleroma.Web.Gettext import Pleroma.Web.Gettext
import Pleroma.Web.Router.Helpers
import Pleroma.Web.TranslationHelpers import Pleroma.Web.TranslationHelpers
alias Pleroma.Web.Router.Helpers, as: Routes
plug(:set_put_layout) plug(:set_put_layout)
defp set_put_layout(conn, _) do defp set_put_layout(conn, _) do
@ -61,6 +62,14 @@ defmodule Pleroma.Web do
) )
end end
defp skip_auth(conn, _) do
skip_plug(conn, [OAuthScopesPlug, EnsurePublicOrAuthenticatedPlug])
end
defp skip_public_check(conn, _) do
skip_plug(conn, EnsurePublicOrAuthenticatedPlug)
end
# Executed just before actual controller action, invokes before-action hooks (callbacks) # Executed just before actual controller action, invokes before-action hooks (callbacks)
defp action(conn, params) do defp action(conn, params) do
with %{halted: false} = conn <- with %{halted: false} = conn <-
@ -131,7 +140,8 @@ defmodule Pleroma.Web do
import Pleroma.Web.ErrorHelpers import Pleroma.Web.ErrorHelpers
import Pleroma.Web.Gettext import Pleroma.Web.Gettext
import Pleroma.Web.Router.Helpers
alias Pleroma.Web.Router.Helpers, as: Routes
require Logger require Logger
@ -229,20 +239,4 @@ defmodule Pleroma.Web do
defmacro __using__(which) when is_atom(which) do defmacro __using__(which) when is_atom(which) do
apply(__MODULE__, which, []) apply(__MODULE__, which, [])
end end
def base_url do
Pleroma.Web.Endpoint.url()
end
# TODO: Change to Phoenix.Router.routes/1 for Phoenix 1.6.0+
def get_api_routes do
Pleroma.Web.Router.__routes__()
|> Enum.reject(fn r -> r.plug == Pleroma.Web.Fallback.RedirectController end)
|> Enum.map(fn r ->
r.path
|> String.split("/", trim: true)
|> List.first()
end)
|> Enum.uniq()
end
end end

View file

@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
alias Pleroma.Conversation alias Pleroma.Conversation
alias Pleroma.Conversation.Participation alias Pleroma.Conversation.Participation
alias Pleroma.Filter alias Pleroma.Filter
alias Pleroma.Hashtag
alias Pleroma.Maps alias Pleroma.Maps
alias Pleroma.Notification alias Pleroma.Notification
alias Pleroma.Object alias Pleroma.Object
@ -53,15 +54,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
{recipients, to, cc} {recipients, to, cc}
end end
defp check_actor_is_active(nil), do: true defp check_actor_can_insert(%{"type" => "Delete"}), do: true
defp check_actor_can_insert(%{"type" => "Undo"}), do: true
defp check_actor_is_active(actor) when is_binary(actor) do defp check_actor_can_insert(%{"actor" => actor}) when is_binary(actor) do
case User.get_cached_by_ap_id(actor) do case User.get_cached_by_ap_id(actor) do
%User{is_active: true} -> true %User{is_active: true} -> true
_ -> false _ -> false
end end
end end
defp check_actor_can_insert(_), do: true
defp check_remote_limit(%{"object" => %{"content" => content}}) when not is_nil(content) do defp check_remote_limit(%{"object" => %{"content" => content}}) when not is_nil(content) do
limit = Config.get([:instance, :remote_limit]) limit = Config.get([:instance, :remote_limit])
String.length(content) <= limit String.length(content) <= limit
@ -88,7 +92,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp increase_replies_count_if_reply(_create_data), do: :noop defp increase_replies_count_if_reply(_create_data), do: :noop
@object_types ~w[ChatMessage Question Answer Audio Video Event Article] @object_types ~w[ChatMessage Question Answer Audio Video Event Article Note Page]
@impl true @impl true
def persist(%{"type" => type} = object, meta) when type in @object_types do def persist(%{"type" => type} = object, meta) when type in @object_types do
with {:ok, object} <- Object.create(object) do with {:ok, object} <- Object.create(object) do
@ -117,7 +121,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when is_map(map) do def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when is_map(map) do
with nil <- Activity.normalize(map), with nil <- Activity.normalize(map),
map <- lazy_put_activity_defaults(map, fake), map <- lazy_put_activity_defaults(map, fake),
{_, true} <- {:actor_check, bypass_actor_check || check_actor_is_active(map["actor"])}, {_, true} <- {:actor_check, bypass_actor_check || check_actor_can_insert(map)},
{_, true} <- {:remote_limit_pass, check_remote_limit(map)}, {_, true} <- {:remote_limit_pass, check_remote_limit(map)},
{:ok, map} <- MRF.filter(map), {:ok, map} <- MRF.filter(map),
{recipients, _, _} = get_recipients(map), {recipients, _, _} = get_recipients(map),
@ -472,6 +476,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|> Repo.one() |> Repo.one()
end end
defp fetch_paginated_optimized(query, opts, pagination) do
# Note: tag-filtering funcs may apply "ORDER BY objects.id DESC",
# and extra sorting on "activities.id DESC NULLS LAST" would worse the query plan
opts = Map.put(opts, :skip_extra_order, true)
Pagination.fetch_paginated(query, opts, pagination)
end
def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do
list_memberships = Pleroma.List.memberships(opts[:user])
fetch_activities_query(recipients ++ list_memberships, opts)
|> fetch_paginated_optimized(opts, pagination)
|> Enum.reverse()
|> maybe_update_cc(list_memberships, opts[:user])
end
@spec fetch_public_or_unlisted_activities(map(), Pagination.type()) :: [Activity.t()] @spec fetch_public_or_unlisted_activities(map(), Pagination.type()) :: [Activity.t()]
def fetch_public_or_unlisted_activities(opts \\ %{}, pagination \\ :keyset) do def fetch_public_or_unlisted_activities(opts \\ %{}, pagination \\ :keyset) do
opts = Map.delete(opts, :user) opts = Map.delete(opts, :user)
@ -479,7 +500,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
[Constants.as_public()] [Constants.as_public()]
|> fetch_activities_query(opts) |> fetch_activities_query(opts)
|> restrict_unlisted(opts) |> restrict_unlisted(opts)
|> Pagination.fetch_paginated(opts, pagination) |> fetch_paginated_optimized(opts, pagination)
end end
@spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()] @spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()]
@ -619,7 +640,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
|> Map.put(:type, ["Create", "Announce"]) |> Map.put(:type, ["Create", "Announce"])
|> Map.put(:user, reading_user) |> Map.put(:user, reading_user)
|> Map.put(:actor_id, user.ap_id) |> Map.put(:actor_id, user.ap_id)
|> Map.put(:pinned_activity_ids, user.pinned_activities) |> Map.put(:pinned_object_ids, Map.keys(user.pinned_objects))
params = params =
if User.blocks?(reading_user, user) do if User.blocks?(reading_user, user) do
@ -700,52 +721,144 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp restrict_since(query, _), do: query defp restrict_since(query, _), do: query
defp restrict_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true}) do defp restrict_embedded_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do
raise "Can't use the child object without preloading!" raise_on_missing_preload()
end end
defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do defp restrict_embedded_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
from(
[_activity, object] in query,
where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
)
end
defp restrict_tag_reject(query, _), do: query
defp restrict_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do
raise "Can't use the child object without preloading!"
end
defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
from( from(
[_activity, object] in query, [_activity, object] in query,
where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all) where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
) )
end end
defp restrict_tag_all(query, _), do: query defp restrict_embedded_tag_all(query, %{tag_all: tag}) when is_binary(tag) do
restrict_embedded_tag_any(query, %{tag: tag})
end
defp restrict_tag(_query, %{tag: _tag, skip_preload: true}) do defp restrict_embedded_tag_all(query, _), do: query
defp restrict_embedded_tag_any(_query, %{tag: _tag, skip_preload: true}) do
raise_on_missing_preload()
end
defp restrict_embedded_tag_any(query, %{tag: [_ | _] = tag_any}) do
from(
[_activity, object] in query,
where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag_any)
)
end
defp restrict_embedded_tag_any(query, %{tag: tag}) when is_binary(tag) do
restrict_embedded_tag_any(query, %{tag: [tag]})
end
defp restrict_embedded_tag_any(query, _), do: query
defp restrict_embedded_tag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
raise_on_missing_preload()
end
defp restrict_embedded_tag_reject_any(query, %{tag_reject: [_ | _] = tag_reject}) do
from(
[_activity, object] in query,
where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
)
end
defp restrict_embedded_tag_reject_any(query, %{tag_reject: tag_reject})
when is_binary(tag_reject) do
restrict_embedded_tag_reject_any(query, %{tag_reject: [tag_reject]})
end
defp restrict_embedded_tag_reject_any(query, _), do: query
defp object_ids_query_for_tags(tags) do
from(hto in "hashtags_objects")
|> join(:inner, [hto], ht in Pleroma.Hashtag, on: hto.hashtag_id == ht.id)
|> where([hto, ht], ht.name in ^tags)
|> select([hto], hto.object_id)
|> distinct([hto], true)
end
defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do
raise_on_missing_preload()
end
defp restrict_hashtag_all(query, %{tag_all: [single_tag]}) do
restrict_hashtag_any(query, %{tag: single_tag})
end
defp restrict_hashtag_all(query, %{tag_all: [_ | _] = tags}) do
from(
[_activity, object] in query,
where:
fragment(
"""
(SELECT array_agg(hashtags.name) FROM hashtags JOIN hashtags_objects
ON hashtags_objects.hashtag_id = hashtags.id WHERE hashtags.name = ANY(?)
AND hashtags_objects.object_id = ?) @> ?
""",
^tags,
object.id,
^tags
)
)
end
defp restrict_hashtag_all(query, %{tag_all: tag}) when is_binary(tag) do
restrict_hashtag_all(query, %{tag_all: [tag]})
end
defp restrict_hashtag_all(query, _), do: query
defp restrict_hashtag_any(_query, %{tag: _tag, skip_preload: true}) do
raise_on_missing_preload()
end
defp restrict_hashtag_any(query, %{tag: [_ | _] = tags}) do
hashtag_ids =
from(ht in Hashtag, where: ht.name in ^tags, select: ht.id)
|> Repo.all()
# Note: NO extra ordering should be done on "activities.id desc nulls last" for optimal plan
from(
[_activity, object] in query,
join: hto in "hashtags_objects",
on: hto.object_id == object.id,
where: hto.hashtag_id in ^hashtag_ids,
distinct: [desc: object.id],
order_by: [desc: object.id]
)
end
defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do
restrict_hashtag_any(query, %{tag: [tag]})
end
defp restrict_hashtag_any(query, _), do: query
defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
raise_on_missing_preload()
end
defp restrict_hashtag_reject_any(query, %{tag_reject: [_ | _] = tags_reject}) do
from(
[_activity, object] in query,
where: object.id not in subquery(object_ids_query_for_tags(tags_reject))
)
end
defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do
restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]})
end
defp restrict_hashtag_reject_any(query, _), do: query
defp raise_on_missing_preload do
raise "Can't use the child object without preloading!" raise "Can't use the child object without preloading!"
end end
defp restrict_tag(query, %{tag: tag}) when is_list(tag) do
from(
[_activity, object] in query,
where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag)
)
end
defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do
from(
[_activity, object] in query,
where: fragment("(?)->'tag' \\? (?)", object.data, ^tag)
)
end
defp restrict_tag(query, _), do: query
defp restrict_recipients(query, [], _user), do: query defp restrict_recipients(query, [], _user), do: query
defp restrict_recipients(query, recipients, nil) do defp restrict_recipients(query, recipients, nil) do
@ -972,8 +1085,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp restrict_unlisted(query, _), do: query defp restrict_unlisted(query, _), do: query
defp restrict_pinned(query, %{pinned: true, pinned_activity_ids: ids}) do defp restrict_pinned(query, %{pinned: true, pinned_object_ids: ids}) do
from(activity in query, where: activity.id in ^ids) from(
[activity, object: o] in query,
where:
fragment(
"(?)->>'type' = 'Create' and coalesce((?)->'object'->>'id', (?)->>'object') = any (?)",
activity.data,
activity.data,
activity.data,
^ids
)
)
end end
defp restrict_pinned(query, _), do: query defp restrict_pinned(query, _), do: query
@ -1105,6 +1228,26 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp maybe_order(query, _), do: query defp maybe_order(query, _), do: query
defp normalize_fetch_activities_query_opts(opts) do
Enum.reduce([:tag, :tag_all, :tag_reject], opts, fn key, opts ->
case opts[key] do
value when is_bitstring(value) ->
Map.put(opts, key, Hashtag.normalize_name(value))
value when is_list(value) ->
normalized_value =
value
|> Enum.map(&Hashtag.normalize_name/1)
|> Enum.uniq()
Map.put(opts, key, normalized_value)
_ ->
opts
end
end)
end
defp fetch_activities_query_ap_ids_ops(opts) do defp fetch_activities_query_ap_ids_ops(opts) do
source_user = opts[:muting_user] source_user = opts[:muting_user]
ap_id_relationships = if source_user, do: [:mute, :reblog_mute], else: [] ap_id_relationships = if source_user, do: [:mute, :reblog_mute], else: []
@ -1128,6 +1271,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end end
def fetch_activities_query(recipients, opts \\ %{}) do def fetch_activities_query(recipients, opts \\ %{}) do
opts = normalize_fetch_activities_query_opts(opts)
{restrict_blocked_opts, restrict_muted_opts, restrict_muted_reblogs_opts} = {restrict_blocked_opts, restrict_muted_opts, restrict_muted_reblogs_opts} =
fetch_activities_query_ap_ids_ops(opts) fetch_activities_query_ap_ids_ops(opts)
@ -1135,50 +1280,51 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
skip_thread_containment: Config.get([:instance, :skip_thread_containment]) skip_thread_containment: Config.get([:instance, :skip_thread_containment])
} }
Activity query =
|> maybe_preload_objects(opts) Activity
|> maybe_preload_bookmarks(opts) |> maybe_preload_objects(opts)
|> maybe_preload_report_notes(opts) |> maybe_preload_bookmarks(opts)
|> maybe_set_thread_muted_field(opts) |> maybe_preload_report_notes(opts)
|> maybe_order(opts) |> maybe_set_thread_muted_field(opts)
|> restrict_recipients(recipients, opts[:user]) |> maybe_order(opts)
|> restrict_replies(opts) |> restrict_recipients(recipients, opts[:user])
|> restrict_tag(opts) |> restrict_replies(opts)
|> restrict_tag_reject(opts) |> restrict_since(opts)
|> restrict_tag_all(opts) |> restrict_local(opts)
|> restrict_since(opts) |> restrict_remote(opts)
|> restrict_local(opts) |> restrict_actor(opts)
|> restrict_remote(opts) |> restrict_type(opts)
|> restrict_actor(opts) |> restrict_state(opts)
|> restrict_type(opts) |> restrict_favorited_by(opts)
|> restrict_state(opts) |> restrict_blocked(restrict_blocked_opts)
|> restrict_favorited_by(opts) |> restrict_muted(restrict_muted_opts)
|> restrict_blocked(restrict_blocked_opts) |> restrict_filtered(opts)
|> restrict_muted(restrict_muted_opts) |> restrict_media(opts)
|> restrict_filtered(opts) |> restrict_visibility(opts)
|> restrict_media(opts) |> restrict_thread_visibility(opts, config)
|> restrict_visibility(opts) |> restrict_reblogs(opts)
|> restrict_thread_visibility(opts, config) |> restrict_pinned(opts)
|> restrict_reblogs(opts) |> restrict_muted_reblogs(restrict_muted_reblogs_opts)
|> restrict_pinned(opts) |> restrict_instance(opts)
|> restrict_muted_reblogs(restrict_muted_reblogs_opts) |> restrict_announce_object_actor(opts)
|> restrict_instance(opts) |> restrict_filtered(opts)
|> restrict_announce_object_actor(opts) |> Activity.restrict_deactivated_users()
|> restrict_filtered(opts) |> exclude_poll_votes(opts)
|> Activity.restrict_deactivated_users() |> exclude_chat_messages(opts)
|> exclude_poll_votes(opts) |> exclude_invisible_actors(opts)
|> exclude_chat_messages(opts) |> exclude_visibility(opts)
|> exclude_invisible_actors(opts)
|> exclude_visibility(opts)
end
def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do if Config.feature_enabled?(:improved_hashtag_timeline) do
list_memberships = Pleroma.List.memberships(opts[:user]) query
|> restrict_hashtag_any(opts)
fetch_activities_query(recipients ++ list_memberships, opts) |> restrict_hashtag_all(opts)
|> Pagination.fetch_paginated(opts, pagination) |> restrict_hashtag_reject_any(opts)
|> Enum.reverse() else
|> maybe_update_cc(list_memberships, opts[:user]) query
|> restrict_embedded_tag_any(opts)
|> restrict_embedded_tag_all(opts)
|> restrict_embedded_tag_reject_any(opts)
end
end end
@doc """ @doc """
@ -1257,21 +1403,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp get_actor_url(_url), do: nil defp get_actor_url(_url), do: nil
defp normalize_image(%{"url" => url}) do
%{
"type" => "Image",
"url" => [%{"href" => url}]
}
end
defp normalize_image(urls) when is_list(urls), do: urls |> List.first() |> normalize_image()
defp normalize_image(_), do: nil
defp object_to_user_data(data) do defp object_to_user_data(data) do
avatar =
data["icon"]["url"] &&
%{
"type" => "Image",
"url" => [%{"href" => data["icon"]["url"]}]
}
banner =
data["image"]["url"] &&
%{
"type" => "Image",
"url" => [%{"href" => data["image"]["url"]}]
}
fields = fields =
data data
|> Map.get("attachment", []) |> Map.get("attachment", [])
@ -1297,6 +1439,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
invisible = data["invisible"] || false invisible = data["invisible"] || false
actor_type = data["type"] || "Person" actor_type = data["type"] || "Person"
featured_address = data["featured"]
{:ok, pinned_objects} = fetch_and_prepare_featured_from_ap_id(featured_address)
public_key = public_key =
if is_map(data["publicKey"]) && is_binary(data["publicKey"]["publicKeyPem"]) do if is_map(data["publicKey"]) && is_binary(data["publicKey"]["publicKeyPem"]) do
data["publicKey"]["publicKeyPem"] data["publicKey"]["publicKeyPem"]
@ -1315,23 +1460,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
ap_id: data["id"], ap_id: data["id"],
uri: get_actor_url(data["url"]), uri: get_actor_url(data["url"]),
ap_enabled: true, ap_enabled: true,
banner: banner, banner: normalize_image(data["image"]),
fields: fields, fields: fields,
emoji: emojis, emoji: emojis,
is_locked: is_locked, is_locked: is_locked,
is_discoverable: is_discoverable, is_discoverable: is_discoverable,
invisible: invisible, invisible: invisible,
avatar: avatar, avatar: normalize_image(data["icon"]),
name: data["name"], name: data["name"],
follower_address: data["followers"], follower_address: data["followers"],
following_address: data["following"], following_address: data["following"],
featured_address: featured_address,
bio: data["summary"] || "", bio: data["summary"] || "",
actor_type: actor_type, actor_type: actor_type,
also_known_as: Map.get(data, "alsoKnownAs", []), also_known_as: Map.get(data, "alsoKnownAs", []),
public_key: public_key, public_key: public_key,
inbox: data["inbox"], inbox: data["inbox"],
shared_inbox: shared_inbox, shared_inbox: shared_inbox,
accepts_chat_messages: accepts_chat_messages accepts_chat_messages: accepts_chat_messages,
pinned_objects: pinned_objects
} }
# nickname can be nil because of virtual actors # nickname can be nil because of virtual actors
@ -1469,6 +1616,41 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end end
end end
def pin_data_from_featured_collection(%{
"type" => type,
"orderedItems" => objects
})
when type in ["OrderedCollection", "Collection"] do
Map.new(objects, fn %{"id" => object_ap_id} -> {object_ap_id, NaiveDateTime.utc_now()} end)
end
def fetch_and_prepare_featured_from_ap_id(nil) do
{:ok, %{}}
end
def fetch_and_prepare_featured_from_ap_id(ap_id) do
with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do
{:ok, pin_data_from_featured_collection(data)}
else
e ->
Logger.error("Could not decode featured collection at fetch #{ap_id}, #{inspect(e)}")
{:ok, %{}}
end
end
def pinned_fetch_task(nil), do: nil
def pinned_fetch_task(%{pinned_objects: pins}) do
if Enum.all?(pins, fn {ap_id, _} ->
Object.get_cached_by_ap_id(ap_id) ||
match?({:ok, _object}, Fetcher.fetch_object_from_id(ap_id))
end) do
:ok
else
:error
end
end
def make_user_from_ap_id(ap_id) do def make_user_from_ap_id(ap_id) do
user = User.get_cached_by_ap_id(ap_id) user = User.get_cached_by_ap_id(ap_id)
@ -1476,6 +1658,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
Transmogrifier.upgrade_user_from_ap_id(ap_id) Transmogrifier.upgrade_user_from_ap_id(ap_id)
else else
with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id) do with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id) do
{:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end)
if user do if user do
user user
|> User.remote_user_changeset(data) |> User.remote_user_changeset(data)

View file

@ -3,5 +3,5 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.ActivityPub.Persisting do defmodule Pleroma.Web.ActivityPub.ActivityPub.Persisting do
@callback persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()} @callback persist(map(), keyword()) :: {:ok, struct()}
end end

View file

@ -3,10 +3,6 @@
# SPDX-License-Identifier: AGPL-3.0-only # SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.ActivityPub.Streaming do defmodule Pleroma.Web.ActivityPub.ActivityPub.Streaming do
alias Pleroma.Activity @callback stream_out(struct()) :: any()
alias Pleroma.Object @callback stream_out_participations(struct(), struct()) :: any()
alias Pleroma.User
@callback stream_out(Activity.t()) :: any()
@callback stream_out_participations(Object.t(), User.t()) :: any()
end end

View file

@ -11,7 +11,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
alias Pleroma.Object.Fetcher alias Pleroma.Object.Fetcher
alias Pleroma.User alias Pleroma.User
alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Builder
alias Pleroma.Web.ActivityPub.InternalFetchActor alias Pleroma.Web.ActivityPub.InternalFetchActor
alias Pleroma.Web.ActivityPub.ObjectView alias Pleroma.Web.ActivityPub.ObjectView
alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Pipeline
@ -403,83 +402,90 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
|> json(err) |> json(err)
end end
defp handle_user_activity( defp fix_user_message(%User{ap_id: actor}, %{"type" => "Create", "object" => object} = activity)
%User{} = user, when is_map(object) do
%{"type" => "Create", "object" => %{"type" => "Note"} = object} = params length =
) do [object["content"], object["summary"], object["name"]]
content = if is_binary(object["content"]), do: object["content"], else: "" |> Enum.filter(&is_binary(&1))
name = if is_binary(object["name"]), do: object["name"], else: "" |> Enum.join("")
summary = if is_binary(object["summary"]), do: object["summary"], else: "" |> String.length()
length = String.length(content <> name <> summary)
if length > Pleroma.Config.get([:instance, :limit]) do limit = Pleroma.Config.get([:instance, :limit])
{:error, dgettext("errors", "Note is over the character limit")}
else if length < limit do
object = object =
object object
|> Map.merge(Map.take(params, ["to", "cc"])) |> Transmogrifier.strip_internal_fields()
|> Map.put("attributedTo", user.ap_id) |> Map.put("attributedTo", actor)
|> Transmogrifier.fix_object() |> Map.put("actor", actor)
|> Map.put("id", Utils.generate_object_id())
ActivityPub.create(%{ {:ok, Map.put(activity, "object", object)}
to: params["to"],
actor: user,
context: object["context"],
object: object,
additional: Map.take(params, ["cc"])
})
end
end
defp handle_user_activity(%User{} = user, %{"type" => "Delete"} = params) do
with %Object{} = object <- Object.normalize(params["object"], fetch: false),
true <- user.is_moderator || user.ap_id == object.data["actor"],
{:ok, delete_data, _} <- Builder.delete(user, object.data["id"]),
{:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do
{:ok, delete}
else else
_ -> {:error, dgettext("errors", "Can't delete object")} {:error,
dgettext(
"errors",
"Character limit (%{limit} characters) exceeded, contains %{length} characters",
limit: limit,
length: length
)}
end end
end end
defp handle_user_activity(%User{} = user, %{"type" => "Like"} = params) do defp fix_user_message(
with %Object{} = object <- Object.normalize(params["object"], fetch: false), %User{ap_id: actor} = user,
{_, {:ok, like_object, meta}} <- {:build_object, Builder.like(user, object)}, %{"type" => "Delete", "object" => object} = activity
{_, {:ok, %Activity{} = activity, _meta}} <- ) do
{:common_pipeline, with {_, %Object{data: object_data}} <- {:normalize, Object.normalize(object, fetch: false)},
Pipeline.common_pipeline(like_object, Keyword.put(meta, :local, true))} do {_, true} <- {:permission, user.is_moderator || actor == object_data["actor"]} do
{:ok, activity} {:ok, activity}
else else
_ -> {:error, dgettext("errors", "Can't like object")} {:normalize, _} ->
{:error, "No such object found"}
{:permission, _} ->
{:forbidden, "You can't delete this object"}
end end
end end
defp handle_user_activity(_, _) do defp fix_user_message(%User{}, activity) do
{:error, dgettext("errors", "Unhandled activity type")} {:ok, activity}
end end
def update_outbox( def update_outbox(
%{assigns: %{user: %User{nickname: nickname} = user}} = conn, %{assigns: %{user: %User{nickname: nickname, ap_id: actor} = user}} = conn,
%{"nickname" => nickname} = params %{"nickname" => nickname} = params
) do ) do
actor = user.ap_id
params = params =
params params
|> Map.drop(["id"]) |> Map.drop(["nickname"])
|> Map.put("id", Utils.generate_activity_id())
|> Map.put("actor", actor) |> Map.put("actor", actor)
|> Transmogrifier.fix_addressing()
with {:ok, %Activity{} = activity} <- handle_user_activity(user, params) do with {:ok, params} <- fix_user_message(user, params),
{:ok, activity, _} <- Pipeline.common_pipeline(params, local: true),
%Activity{data: activity_data} <- Activity.normalize(activity) do
conn conn
|> put_status(:created) |> put_status(:created)
|> put_resp_header("location", activity.data["id"]) |> put_resp_header("location", activity_data["id"])
|> json(activity.data) |> json(activity_data)
else else
{:forbidden, message} ->
conn
|> put_status(:forbidden)
|> json(message)
{:error, message} -> {:error, message} ->
conn conn
|> put_status(:bad_request) |> put_status(:bad_request)
|> json(message) |> json(message)
e ->
Logger.warn(fn -> "AP C2S: #{inspect(e)}" end)
conn
|> put_status(:bad_request)
|> json("Bad Request")
end end
end end
@ -543,4 +549,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
|> json(object.data) |> json(object.data)
end end
end end
def pinned(conn, %{"nickname" => nickname}) do
with %User{} = user <- User.get_cached_by_nickname(nickname) do
conn
|> put_resp_header("content-type", "application/activity+json")
|> json(UserView.render("featured.json", %{user: user}))
end
end
end end

View file

@ -223,7 +223,7 @@ defmodule Pleroma.Web.ActivityPub.Builder do
[actor.follower_address] [actor.follower_address]
public? and Visibility.is_local_public?(object) -> public? and Visibility.is_local_public?(object) ->
[actor.follower_address, object.data["actor"], Pleroma.Constants.as_local_public()] [actor.follower_address, object.data["actor"], Utils.as_local_public()]
public? -> public? ->
[actor.follower_address, object.data["actor"], Pleroma.Constants.as_public()] [actor.follower_address, object.data["actor"], Pleroma.Constants.as_public()]
@ -273,4 +273,36 @@ defmodule Pleroma.Web.ActivityPub.Builder do
"context" => object.data["context"] "context" => object.data["context"]
}, []} }, []}
end end
@spec pin(User.t(), Object.t()) :: {:ok, map(), keyword()}
def pin(%User{} = user, object) do
{:ok,
%{
"id" => Utils.generate_activity_id(),
"target" => pinned_url(user.nickname),
"object" => object.data["id"],
"actor" => user.ap_id,
"type" => "Add",
"to" => [Pleroma.Constants.as_public()],
"cc" => [user.follower_address]
}, []}
end
@spec unpin(User.t(), Object.t()) :: {:ok, map, keyword()}
def unpin(%User{} = user, object) do
{:ok,
%{
"id" => Utils.generate_activity_id(),
"target" => pinned_url(user.nickname),
"object" => object.data["id"],
"actor" => user.ap_id,
"type" => "Remove",
"to" => [Pleroma.Constants.as_public()],
"cc" => [user.follower_address]
}, []}
end
defp pinned_url(nickname) when is_binary(nickname) do
Pleroma.Web.Router.Helpers.activity_pub_url(Pleroma.Web.Endpoint, :pinned, nickname)
end
end end

View file

@ -51,17 +51,6 @@ defmodule Pleroma.Web.ActivityPub.MRF do
@required_description_keys [:key, :related_policy] @required_description_keys [:key, :related_policy]
@callback filter(Map.t()) :: {:ok | :reject, Map.t()}
@callback describe() :: {:ok | :error, Map.t()}
@callback config_description() :: %{
optional(:children) => [map()],
key: atom(),
related_policy: String.t(),
label: String.t(),
description: String.t()
}
@optional_callbacks config_description: 0
def filter(policies, %{} = message) do def filter(policies, %{} = message) do
policies policies
|> Enum.reduce({:ok, message}, fn |> Enum.reduce({:ok, message}, fn
@ -92,7 +81,9 @@ defmodule Pleroma.Web.ActivityPub.MRF do
end end
def get_policies do def get_policies do
Pleroma.Config.get([:mrf, :policies], []) |> get_policies() Pleroma.Config.get([:mrf, :policies], [])
|> get_policies()
|> Enum.concat([Pleroma.Web.ActivityPub.MRF.HashtagPolicy])
end end
defp get_policies(policy) when is_atom(policy), do: [policy] defp get_policies(policy) when is_atom(policy), do: [policy]
@ -140,7 +131,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do
def describe, do: get_policies() |> describe() def describe, do: get_policies() |> describe()
def config_descriptions do def config_descriptions do
Pleroma.Web.ActivityPub.MRF Pleroma.Web.ActivityPub.MRF.Policy
|> Pleroma.Docs.Generator.list_behaviour_implementations() |> Pleroma.Docs.Generator.list_behaviour_implementations()
|> config_descriptions() |> config_descriptions()
end end

View file

@ -4,7 +4,7 @@
defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do
@moduledoc "Adds expiration to all local Create activities" @moduledoc "Adds expiration to all local Create activities"
@behaviour Pleroma.Web.ActivityPub.MRF @behaviour Pleroma.Web.ActivityPub.MRF.Policy
@impl true @impl true
def filter(activity) do def filter(activity) do

View file

@ -7,7 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
@moduledoc "Prevent followbots from following with a bit of heuristic" @moduledoc "Prevent followbots from following with a bit of heuristic"
@behaviour Pleroma.Web.ActivityPub.MRF @behaviour Pleroma.Web.ActivityPub.MRF.Policy
# XXX: this should become User.normalize_by_ap_id() or similar, really. # XXX: this should become User.normalize_by_ap_id() or similar, really.
defp normalize_by_ap_id(%{"id" => id}), do: User.get_cached_by_ap_id(id) defp normalize_by_ap_id(%{"id" => id}), do: User.get_cached_by_ap_id(id)

View file

@ -5,7 +5,7 @@
defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do
alias Pleroma.User alias Pleroma.User
@behaviour Pleroma.Web.ActivityPub.MRF @behaviour Pleroma.Web.ActivityPub.MRF.Policy
require Logger require Logger

View file

@ -5,7 +5,7 @@
defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do
require Logger require Logger
@moduledoc "Drop and log everything received" @moduledoc "Drop and log everything received"
@behaviour Pleroma.Web.ActivityPub.MRF @behaviour Pleroma.Web.ActivityPub.MRF.Policy
@impl true @impl true
def filter(object) do def filter(object) do

View file

@ -6,7 +6,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do
alias Pleroma.Object alias Pleroma.Object
@moduledoc "Ensure a re: is prepended on replies to a post with a Subject" @moduledoc "Ensure a re: is prepended on replies to a post with a Subject"
@behaviour Pleroma.Web.ActivityPub.MRF @behaviour Pleroma.Web.ActivityPub.MRF.Policy
@reply_prefix Regex.compile!("^re:[[:space:]]*", [:caseless]) @reply_prefix Regex.compile!("^re:[[:space:]]*", [:caseless])

View file

@ -0,0 +1,59 @@
defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
alias Pleroma.Config
alias Pleroma.User
alias Pleroma.Web.CommonAPI
require Logger
@impl true
def filter(message) do
with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]),
%User{actor_type: "Service"} = follower <-
User.get_cached_by_nickname(follower_nickname),
%{"type" => "Create", "object" => %{"type" => "Note"}} <- message do
try_follow(follower, message)
else
nil ->
Logger.warn(
"#{__MODULE__} skipped because of missing `:mrf_follow_bot, :follower_nickname` configuration, the :follower_nickname
account does not exist, or the account is not correctly configured as a bot."
)
{:ok, message}
_ ->
{:ok, message}
end
end
defp try_follow(follower, message) do
to = Map.get(message, "to", [])
cc = Map.get(message, "cc", [])
actor = [message["actor"]]
Enum.concat([to, cc, actor])
|> List.flatten()
|> Enum.uniq()
|> User.get_all_by_ap_id()
|> Enum.each(fn user ->
with false <- user.local,
false <- User.following?(follower, user),
false <- User.locked?(user),
false <- (user.bio || "") |> String.downcase() |> String.contains?("nobot") do
Logger.debug(
"#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}"
)
CommonAPI.follow(follower, user)
end
end)
{:ok, message}
end
@impl true
def describe do
{:ok, %{}}
end
end

View file

@ -4,7 +4,7 @@
defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do
alias Pleroma.User alias Pleroma.User
@behaviour Pleroma.Web.ActivityPub.MRF @behaviour Pleroma.Web.ActivityPub.MRF.Policy
@moduledoc "Remove bot posts from federated timeline" @moduledoc "Remove bot posts from federated timeline"
require Pleroma.Constants require Pleroma.Constants

View file

@ -0,0 +1,116 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do
require Pleroma.Constants
alias Pleroma.Config
alias Pleroma.Object
@moduledoc """
Reject, TWKN-remove or Set-Sensitive messsages with specific hashtags (without the leading #)
Note: This MRF Policy is always enabled, if you want to disable it you have to set empty lists.
"""
@behaviour Pleroma.Web.ActivityPub.MRF.Policy
defp check_reject(message, hashtags) do
if Enum.any?(Config.get([:mrf_hashtag, :reject]), fn match -> match in hashtags end) do
{:reject, "[HashtagPolicy] Matches with rejected keyword"}
else
{:ok, message}
end
end
defp check_ftl_removal(%{"to" => to} = message, hashtags) do
if Pleroma.Constants.as_public() in to and
Enum.any?(Config.get([:mrf_hashtag, :federated_timeline_removal]), fn match ->
match in hashtags
end) do
to = List.delete(to, Pleroma.Constants.as_public())
cc = [Pleroma.Constants.as_public() | message["cc"] || []]
message =
message
|> Map.put("to", to)
|> Map.put("cc", cc)
|> Kernel.put_in(["object", "to"], to)
|> Kernel.put_in(["object", "cc"], cc)
{:ok, message}
else
{:ok, message}
end
end
defp check_ftl_removal(message, _hashtags), do: {:ok, message}
defp check_sensitive(message, hashtags) do
if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do
{:ok, Kernel.put_in(message, ["object", "sensitive"], true)}
else
{:ok, message}
end
end
@impl true
def filter(%{"type" => "Create", "object" => object} = message) do
hashtags = Object.hashtags(%Object{data: object})
if hashtags != [] do
with {:ok, message} <- check_reject(message, hashtags),
{:ok, message} <- check_ftl_removal(message, hashtags),
{:ok, message} <- check_sensitive(message, hashtags) do
{:ok, message}
end
else
{:ok, message}
end
end
@impl true
def filter(message), do: {:ok, message}
@impl true
def describe do
mrf_hashtag =
Config.get(:mrf_hashtag)
|> Enum.into(%{})
{:ok, %{mrf_hashtag: mrf_hashtag}}
end
@impl true
def config_description do
%{
key: :mrf_hashtag,
related_policy: "Pleroma.Web.ActivityPub.MRF.HashtagPolicy",
label: "MRF Hashtag",
description: @moduledoc,
children: [
%{
key: :reject,
type: {:list, :string},
description: "A list of hashtags which result in message being rejected.",
suggestions: ["foo"]
},
%{
key: :federated_timeline_removal,
type: {:list, :string},
description:
"A list of hashtags which result in message being removed from federated timelines (a.k.a unlisted).",
suggestions: ["foo"]
},
%{
key: :sensitive,
type: {:list, :string},
description:
"A list of hashtags which result in message being set as sensitive (a.k.a NSFW/R-18)",
suggestions: ["nsfw", "r18"]
}
]
}
end
end

View file

@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
@moduledoc "Block messages with too much mentions (configurable)" @moduledoc "Block messages with too much mentions (configurable)"
@behaviour Pleroma.Web.ActivityPub.MRF @behaviour Pleroma.Web.ActivityPub.MRF.Policy
defp delist_message(message, threshold) when threshold > 0 do defp delist_message(message, threshold) when threshold > 0 do
follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address

View file

@ -7,7 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
@moduledoc "Reject or Word-Replace messages with a keyword or regex" @moduledoc "Reject or Word-Replace messages with a keyword or regex"
@behaviour Pleroma.Web.ActivityPub.MRF @behaviour Pleroma.Web.ActivityPub.MRF.Policy
defp string_matches?(string, _) when not is_binary(string) do defp string_matches?(string, _) when not is_binary(string) do
false false
end end

View file

@ -4,7 +4,7 @@
defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
@moduledoc "Preloads any attachments in the MediaProxy cache by prefetching them" @moduledoc "Preloads any attachments in the MediaProxy cache by prefetching them"
@behaviour Pleroma.Web.ActivityPub.MRF @behaviour Pleroma.Web.ActivityPub.MRF.Policy
alias Pleroma.HTTP alias Pleroma.HTTP
alias Pleroma.Web.MediaProxy alias Pleroma.Web.MediaProxy

Some files were not shown because too many files have changed in this diff Show more