Merge branch 'develop' of https://git.pleroma.social/pleroma/pleroma into develop

.gitlab-ci.yml

@@ -16,6 +16,7 @@ stages:
   - build
   - test
   - deploy
+  - release
 
 before_script:
   - mix local.hex --force
@@ -42,6 +43,7 @@ docs-build:
     paths:
       - priv/static/doc
 
+
 unit-testing:
   stage: test
   services:
@@ -140,3 +142,103 @@ stop_review_app:
     - ssh-keyscan -H "pleroma.online" >> ~/.ssh/known_hosts
     - ssh -t dokku@pleroma.online -- --force apps:destroy "$CI_ENVIRONMENT_SLUG"
     - ssh -t dokku@pleroma.online -- --force postgres:destroy $(echo $CI_ENVIRONMENT_SLUG | sed -e 's/-/_/g')_db
+
+amd64:
+  stage: release 
+  # TODO: Replace with upstream image when 1.9.0 comes out
+  image: rinpatch/elixir:1.9.0-rc.0
+  only: &release-only
+  - master@pleroma/pleroma
+  - develop@pleroma/pleroma
+  artifacts: &release-artifacts
+    name: "pleroma-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA-$CI_JOB_NAME"
+    paths:
+      - release/*
+    # Ideally it would be never for master branch and with the next commit for develop,
+    # but Gitlab does not support neither `only` for artifacts
+    # nor setting it to never from .gitlab-ci.yml
+    # nor expiring with the next commit
+    expire_in: 42 yrs
+
+  cache: &release-cache
+    key: $CI_COMMIT_REF_NAME-$CI_JOB_NAME
+    paths:
+          - deps
+  variables: &release-variables
+    MIX_ENV: prod
+  before_script: &before-release
+  - echo "import Mix.Config" > config/prod.secret.exs
+  - mix local.hex --force
+  - mix local.rebar --force
+  script: &release
+    - mix deps.get --only prod
+    - mkdir release
+    - mix release --path release
+
+
+amd64-musl:
+  stage: release
+  artifacts: *release-artifacts
+  only: *release-only
+  # TODO: Replace with upstream image when 1.9.0 comes out
+  image: rinpatch/elixir:1.9.0-rc.0-alpine
+  cache: *release-cache
+  variables: *release-variables
+  before_script: &before-release-musl
+  - apk add git gcc g++ musl-dev make
+  - echo "import Mix.Config" > config/prod.secret.exs
+  - mix local.hex --force
+  - mix local.rebar --force
+  script: *release
+
+arm:
+  stage: release
+  artifacts: *release-artifacts
+  only: *release-only
+  tags:
+    - arm32
+  # TODO: Replace with upstream image when 1.9.0 comes out
+  image: rinpatch/elixir:1.9.0-rc.0-arm
+  cache: *release-cache
+  variables: *release-variables
+  before_script: *before-release
+  script: *release
+
+arm-musl:
+  stage: release
+  artifacts: *release-artifacts
+  only: *release-only
+  tags:
+    - arm32
+  # TODO: Replace with upstream image when 1.9.0 comes out
+  image: rinpatch/elixir:1.9.0-rc.0-arm-alpine
+  cache: *release-cache
+  variables: *release-variables
+  before_script: *before-release-musl
+  script: *release
+
+arm64:
+  stage: release
+  artifacts: *release-artifacts
+  only: *release-only
+  tags:
+    - arm
+  # TODO: Replace with upstream image when 1.9.0 comes out
+  image: rinpatch/elixir:1.9.0-rc.0-arm64
+  cache: *release-cache
+  variables: *release-variables
+  before_script: *before-release
+  script: *release
+
+arm64-musl:
+  stage: release
+  artifacts: *release-artifacts
+  only: *release-only
+  tags:
+    - arm
+  # TODO: Replace with upstream image when 1.9.0 comes out
+  image: rinpatch/elixir:1.9.0-rc.0-arm64-alpine
+  cache: *release-cache
+  variables: *release-variables
+  before_script: *before-release-musl
+  script: *release

CHANGELOG.md

@@ -19,6 +19,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mix Tasks: `mix pleroma.database remove_embedded_objects`
 - Mix Tasks: `mix pleroma.database update_users_following_followers_counts`
 - Mix Tasks: `mix pleroma.user toggle_confirmed`
+- Mix Tasks: `mix pleroma.config migrate_to_db`
+- Mix Tasks: `mix pleroma.config migrate_from_db`
 - Federation: Support for `Question` and `Answer` objects
 - Federation: Support for reports
 - Configuration: `poll_limits` option
@@ -37,7 +39,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Admin API: added filters (role, tags, email, name) for users endpoint
 - Admin API: Endpoints for managing reports
 - Admin API: Endpoints for deleting and changing the scope of individual reported statuses
+- Admin API: Endpoints to view and change config settings.
 - AdminFE: initial release with basic user management accessible at /pleroma/admin/
+- Mastodon API: Add chat token to `verify_credentials` response
+- Mastodon API: Add background image setting to `update_credentials`
 - Mastodon API: [Scheduled statuses](https://docs.joinmastodon.org/api/rest/scheduled-statuses/)
 - Mastodon API: `/api/v1/notifications/destroy_multiple` (glitch-soc extension)
 - Mastodon API: `/api/v1/pleroma/accounts/:id/favourites` (API extension)

config/config.exs

@@ -245,7 +245,8 @@
   healthcheck: false,
   remote_post_retention_days: 90,
   skip_thread_containment: true,
-  limit_to_local_content: :unauthenticated
+  limit_to_local_content: :unauthenticated,
+  dynamic_configuration: false
 
 config :pleroma, :markup,
   # XXX - unfortunately, inline images must be enabled by default right now, because

config/dev.exs

@@ -59,3 +59,6 @@
     "!!! RUNNING IN LOCALHOST DEV MODE! !!!\nFEDERATION WON'T WORK UNTIL YOU CONFIGURE A dev.secret.exs"
   )
 end
+
+if File.exists?("./config/dev.migrated.secret.exs"),
+  do: import_config("./config/dev.migrated.secret.exs")

config/prod.exs

@@ -63,3 +63,6 @@
 # Finally import the config/prod.secret.exs
 # which should be versioned separately.
 import_config "prod.secret.exs"
+
+if File.exists?("./config/prod.migrated.secret.exs"),
+  do: import_config("./config/prod.migrated.secret.exs")

config/releases.exs

@@ -1,5 +1,8 @@
 import Config
 
+config :pleroma, :instance, static_dir: "/var/lib/pleroma/static"
+config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads"
+
 config_path = System.get_env("PLEROMA_CONFIG_PATH") || "/etc/pleroma/config.exs"
 
 if File.exists?(config_path) do

config/test.exs

@@ -60,7 +60,7 @@
   total_user_limit: 3,
   enabled: false
 
-config :pleroma, :rate_limit, app_account_creation: {1000, 5}
+config :pleroma, :rate_limit, app_account_creation: {10_000, 5}
 
 config :pleroma, :http_security, report_uri: "https://endpoint.com"
 

docs/api/admin_api.md

@@ -289,7 +289,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - `limit`: optional, the number of records to retrieve
   - `since_id`: optional, returns results that are more recent than the specified id
   - `max_id`: optional, returns results that are older than the specified id
-- Response: 
+- Response:
   - On failure: 403 Forbidden error `{"error": "error_msg"}` when requested by anonymous or non-admin
   - On success: JSON, returns a list of reports, where:
     - `account`: the user who has been reported
@@ -443,7 +443,7 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 - Params:
   - `id`
 - Response:
-  - On failure: 
+  - On failure:
     - 403 Forbidden `{"error": "error_msg"}`
     - 404 Not Found `"Not found"`
   - On success: JSON, Report object (see above)
@@ -454,8 +454,8 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 - Params:
   - `id`
   - `state`: required, the new state. Valid values are `open`, `closed` and `resolved`
-- Response: 
+- Response:
-  - On failure: 
+  - On failure:
     - 400 Bad Request `"Unsupported state"`
     - 403 Forbidden `{"error": "error_msg"}`
     - 404 Not Found `"Not found"`
@@ -467,10 +467,10 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 - Params:
   - `id`
   - `status`: required, the message
-- Response: 
+- Response:
-  - On failure: 
+  - On failure:
-    - 400 Bad Request `"Invalid parameters"` when `status` is missing 
+    - 400 Bad Request `"Invalid parameters"` when `status` is missing
-    - 403 Forbidden `{"error": "error_msg"}` 
+    - 403 Forbidden `{"error": "error_msg"}`
     - 404 Not Found `"Not found"`
   - On success: JSON, created Mastodon Status entity
 
@@ -540,10 +540,10 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
   - `id`
   - `sensitive`: optional, valid values are `true` or `false`
   - `visibility`: optional, valid values are `public`, `private` and `unlisted`
-- Response: 
+- Response:
-  - On failure: 
+  - On failure:
     - 400 Bad Request `"Unsupported visibility"`
-    - 403 Forbidden `{"error": "error_msg"}` 
+    - 403 Forbidden `{"error": "error_msg"}`
     - 404 Not Found `"Not found"`
   - On success: JSON, Mastodon Status entity
 
@@ -552,8 +552,88 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret
 - Method `DELETE`
 - Params:
   - `id`
-- Response: 
+- Response:
-  - On failure: 
+  - On failure:
-    - 403 Forbidden `{"error": "error_msg"}` 
+    - 403 Forbidden `{"error": "error_msg"}`
     - 404 Not Found `"Not found"`
   - On success: 200 OK `{}`
+
+## `/api/pleroma/admin/config`
+### List config settings
+- Method `GET`
+- Params: none
+- Response:
+
+```json
+{
+  configs: [
+    {
+      "key": string,
+      "value": string or {} or []
+     }
+  ]
+}
+```
+
+## `/api/pleroma/admin/config`
+### Update config settings
+Module name can be passed as string, which starts with `Pleroma`, e.g. `"Pleroma.Upload"`.
+Atom or boolean value can be passed with `:` in the beginning, e.g. `":true"`, `":upload"`.
+Integer with `i:`, e.g. `"i:150"`.
+
+Compile time settings (need instance reboot):
+- all settings by this keys:
+  - `:hackney_pools`
+  - `:chat`
+  - `Pleroma.Web.Endpoint`
+  - `Pleroma.Repo`
+- part settings:
+  - `Pleroma.Captcha` -> `:seconds_valid`
+  - `Pleroma.Upload` -> `:proxy_remote`
+  - `:instance` -> `:upload_limit`
+
+- Method `POST`
+- Params:
+  - `configs` => [
+    - `key` (string)
+    - `value` (string, [], {})
+    - `delete` = true (optional, if parameter must be deleted)
+  ]
+
+- Request (example):
+
+```json
+{
+  configs: [
+    {
+      "key": "Pleroma.Upload",
+      "value": {
+        "uploader": "Pleroma.Uploaders.Local",
+        "filters": ["Pleroma.Upload.Filter.Dedupe"],
+        "link_name": ":true",
+        "proxy_remote": ":false",
+        "proxy_opts": {
+          "redirect_on_failure": ":false",
+          "max_body_length": "i:1048576",
+          "http": {
+            "follow_redirect": ":true",
+            "pool": ":upload"
+          }
+        }
+      }
+     }
+  ]
+}
+
+- Response:
+
+```json
+{
+  configs: [
+    {
+      "key": string,
+      "value": string or {} or []
+     }
+  ]
+}
+```

docs/api/differences_in_mastoapi_responses.md

@@ -44,6 +44,7 @@ Has these additional fields under the `pleroma` object:
 - `hide_followers`: boolean, true when the user has follower hiding enabled
 - `hide_follows`: boolean, true when the user has follow hiding enabled
 - `settings_store`: A generic map of settings for frontends. Opaque to the backend. Only returned in `verify_credentials` and `update_credentials`
+- `chat_token`: The token needed for Pleroma chat. Only returned in `verify_credentials`
 
 ### Source
 
@@ -84,6 +85,7 @@ Additional parameters can be added to the JSON body/Form data:
 - `default_scope` - the scope returned under `privacy` key in Source subentity
 - `pleroma_settings_store` - Opaque user settings to be saved on the backend.
 - `skip_thread_containment` - if true, skip filtering out broken threads
+- `pleroma_background_image` - sets the background image of the user.
 
 ### Pleroma Settings Store
 Pleroma has mechanism that allows frontends to save blobs of json for each user on the backend. This can be used to save frontend-specific settings for a user that the backend does not need to know about.

docs/config.md

@@ -114,6 +114,7 @@ config :pleroma, Pleroma.Emails.Mailer,
 * `remote_post_retention_days`: The default amount of days to retain remote posts when pruning the database.
 * `skip_thread_containment`: Skip filter out broken threads. The default is `false`.
 * `limit_to_local_content`: Limit unauthenticated users to search for local statutes and users only. Possible values: `:unauthenticated`, `:all` and `false`. The default is `:unauthenticated`.
+* `dynamic_configuration`: Allow transferring configuration to DB with the subsequent customization from Admin api.
 
 
 ## :logger

lib/mix/tasks/pleroma/config.ex

@@ -0,0 +1,68 @@
+defmodule Mix.Tasks.Pleroma.Config do
+  use Mix.Task
+  alias Mix.Tasks.Pleroma.Common
+  alias Pleroma.Repo
+  alias Pleroma.Web.AdminAPI.Config
+  @shortdoc "Manages the location of the config"
+  @moduledoc """
+  Manages the location of the config.
+
+  ## Transfers config from file to DB.
+
+      mix pleroma.config migrate_to_db
+
+  ## Transfers config from DB to file.
+
+      mix pleroma.config migrate_from_db ENV
+  """
+
+  def run(["migrate_to_db"]) do
+    Common.start_pleroma()
+
+    if Pleroma.Config.get([:instance, :dynamic_configuration]) do
+      Application.get_all_env(:pleroma)
+      |> Enum.reject(fn {k, _v} -> k in [Pleroma.Repo, :env] end)
+      |> Enum.each(fn {k, v} ->
+        key = to_string(k) |> String.replace("Elixir.", "")
+        {:ok, _} = Config.update_or_create(%{key: key, value: v})
+        Mix.shell().info("#{key} is migrated.")
+      end)
+
+      Mix.shell().info("Settings migrated.")
+    else
+      Mix.shell().info(
+        "Migration is not allowed by config. You can change this behavior in instance settings."
+      )
+    end
+  end
+
+  def run(["migrate_from_db", env]) do
+    Common.start_pleroma()
+
+    if Pleroma.Config.get([:instance, :dynamic_configuration]) do
+      config_path = "config/#{env}.migrated.secret.exs"
+
+      {:ok, file} = File.open(config_path, [:write])
+
+      Repo.all(Config)
+      |> Enum.each(fn config ->
+        mark = if String.starts_with?(config.key, "Pleroma."), do: ",", else: ":"
+
+        IO.write(
+          file,
+          "config :pleroma, #{config.key}#{mark} #{inspect(Config.from_binary(config.value))}\r\n"
+        )
+
+        {:ok, _} = Repo.delete(config)
+        Mix.shell().info("#{config.key} deleted from DB.")
+      end)
+
+      File.close(file)
+      System.cmd("mix", ["format", config_path])
+    else
+      Mix.shell().info(
+        "Migration is not allowed by config. You can change this behavior in instance settings."
+      )
+    end
+  end
+end

lib/mix/tasks/pleroma/emoji.ex

@@ -55,15 +55,13 @@ defmodule Mix.Tasks.Pleroma.Emoji do
   are extracted).
   """
 
-  @default_manifest Pleroma.Config.get!([:emoji, :default_manifest])
-
   def run(["ls-packs" | args]) do
     Application.ensure_all_started(:hackney)
 
     {options, [], []} = parse_global_opts(args)
 
     manifest =
-      fetch_manifest(if options[:manifest], do: options[:manifest], else: @default_manifest)
+      fetch_manifest(if options[:manifest], do: options[:manifest], else: default_manifest())
 
     Enum.each(manifest, fn {name, info} ->
       to_print = [
@@ -88,7 +86,7 @@ def run(["get-packs" | args]) do
 
     {options, pack_names, []} = parse_global_opts(args)
 
-    manifest_url = if options[:manifest], do: options[:manifest], else: @default_manifest
+    manifest_url = if options[:manifest], do: options[:manifest], else: default_manifest()
 
     manifest = fetch_manifest(manifest_url)
 
@@ -298,4 +296,6 @@ defp client do
 
     Tesla.client(middleware)
   end
+
+  defp default_manifest, do: Pleroma.Config.get!([:emoji, :default_manifest])
 end

lib/mix/tasks/pleroma/instance.ex

@@ -30,6 +30,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
   - `--dbuser DBUSER` - the user (aka role) to use for the database connection
   - `--dbpass DBPASS` - the password to use for the database connection
   - `--indexable Y/N` - Allow/disallow indexing site by search engines
+  - `--db-configurable Y/N` - Allow/disallow configuring instance from admin part
   """
 
   def run(["gen" | rest]) do
@@ -48,7 +49,8 @@ def run(["gen" | rest]) do
           dbname: :string,
           dbuser: :string,
           dbpass: :string,
-          indexable: :string
+          indexable: :string,
+          db_configurable: :string
         ],
         aliases: [
           o: :output,
@@ -101,6 +103,14 @@ def run(["gen" | rest]) do
           "y"
         ) === "y"
 
+      db_configurable? =
+        Common.get_option(
+          options,
+          :db_configurable,
+          "Do you want to be able to configure instance from admin part? (y/n)",
+          "y"
+        ) === "y"
+
       dbhost =
         Common.get_option(options, :dbhost, "What is the hostname of your database?", "localhost")
 
@@ -144,7 +154,8 @@ def run(["gen" | rest]) do
           secret: secret,
           signing_salt: signing_salt,
           web_push_public_key: Base.url_encode64(web_push_public_key, padding: false),
-          web_push_private_key: Base.url_encode64(web_push_private_key, padding: false)
+          web_push_private_key: Base.url_encode64(web_push_private_key, padding: false),
+          db_configurable?: db_configurable?
         )
 
       result_psql =

lib/mix/tasks/pleroma/sample_config.eex

@@ -16,7 +16,8 @@ config :pleroma, :instance,
   notify_email: "<%= notify_email %>",
   limit: 5000,
   registrations_open: true,
-  dedupe_media: false
+  dedupe_media: false,
+  dynamic_configuration: <%= db_configurable? %>
 
 config :pleroma, :media_proxy,
   enabled: false,

lib/pleroma/application.ex

@@ -31,6 +31,7 @@ def start(_type, _args) do
       [
         # Start the Ecto repository
         %{id: Pleroma.Repo, start: {Pleroma.Repo, :start_link, []}, type: :supervisor},
+        %{id: Pleroma.Config.TransferTask, start: {Pleroma.Config.TransferTask, :start_link, []}},
         %{id: Pleroma.Emoji, start: {Pleroma.Emoji, :start_link, []}},
         %{id: Pleroma.Captcha, start: {Pleroma.Captcha, :start_link, []}},
         %{
@@ -174,7 +175,6 @@ defp setup_instrumenters do
       Pleroma.Repo.Instrumenter.setup()
     end
 
-    Prometheus.Registry.register_collector(:prometheus_process_collector)
     Pleroma.Web.Endpoint.MetricsExporter.setup()
     Pleroma.Web.Endpoint.PipelineInstrumenter.setup()
     Pleroma.Web.Endpoint.Instrumenter.setup()
@@ -187,7 +187,7 @@ def enabled_hackney_pools do
       else
         []
       end ++
-      if Pleroma.Config.get([Pleroma.Uploader, :proxy_remote]) do
+      if Pleroma.Config.get([Pleroma.Upload, :proxy_remote]) do
         [:upload]
       else
         []

lib/pleroma/config/transfer_task.ex

@@ -0,0 +1,42 @@
+defmodule Pleroma.Config.TransferTask do
+  use Task
+  alias Pleroma.Web.AdminAPI.Config
+
+  def start_link do
+    load_and_update_env()
+    if Pleroma.Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Pleroma.Repo)
+    :ignore
+  end
+
+  def load_and_update_env do
+    if Pleroma.Config.get([:instance, :dynamic_configuration]) and
+         Ecto.Adapters.SQL.table_exists?(Pleroma.Repo, "config") do
+      Pleroma.Repo.all(Config)
+      |> Enum.each(&update_env(&1))
+    end
+  end
+
+  defp update_env(setting) do
+    try do
+      key =
+        if String.starts_with?(setting.key, "Pleroma.") do
+          "Elixir." <> setting.key
+        else
+          setting.key
+        end
+
+      Application.put_env(
+        :pleroma,
+        String.to_existing_atom(key),
+        Config.from_binary(setting.value)
+      )
+    rescue
+      e ->
+        require Logger
+
+        Logger.warn(
+          "updating env causes error, key: #{inspect(setting.key)}, error: #{inspect(e)}"
+        )
+    end
+  end
+end

lib/pleroma/emoji.ex

@@ -22,7 +22,6 @@ defmodule Pleroma.Emoji do
 
   @ets __MODULE__.Ets
   @ets_options [:ordered_set, :protected, :named_table, {:read_concurrency, true}]
-  @groups Pleroma.Config.get([:emoji, :groups])
 
   @doc false
   def start_link do
@@ -87,6 +86,8 @@ defp load do
         "emoji"
       )
 
+    emoji_groups = Pleroma.Config.get([:emoji, :groups])
+
     case File.ls(emoji_dir_path) do
       {:error, :enoent} ->
         # The custom emoji directory doesn't exist,
@@ -118,7 +119,7 @@ defp load do
         emojis =
           Enum.flat_map(
             packs,
-            fn pack -> load_pack(Path.join(emoji_dir_path, pack)) end
+            fn pack -> load_pack(Path.join(emoji_dir_path, pack), emoji_groups) end
           )
 
         true = :ets.insert(@ets, emojis)
@@ -129,9 +130,9 @@ defp load do
     shortcode_globs = Pleroma.Config.get([:emoji, :shortcode_globs], [])
 
     emojis =
-      (load_from_file("config/emoji.txt") ++
+      (load_from_file("config/emoji.txt", emoji_groups) ++
-         load_from_file("config/custom_emoji.txt") ++
+         load_from_file("config/custom_emoji.txt", emoji_groups) ++
-         load_from_globs(shortcode_globs))
+         load_from_globs(shortcode_globs, emoji_groups))
       |> Enum.reject(fn value -> value == nil end)
 
     true = :ets.insert(@ets, emojis)
@@ -139,13 +140,13 @@ defp load do
     :ok
   end
 
-  defp load_pack(pack_dir) do
+  defp load_pack(pack_dir, emoji_groups) do
     pack_name = Path.basename(pack_dir)
 
     emoji_txt = Path.join(pack_dir, "emoji.txt")
 
     if File.exists?(emoji_txt) do
-      load_from_file(emoji_txt)
+      load_from_file(emoji_txt, emoji_groups)
     else
       Logger.info(
         "No emoji.txt found for pack \"#{pack_name}\", assuming all .png files are emoji"
@@ -155,7 +156,7 @@ defp load_pack(pack_dir) do
       |> Enum.map(fn {shortcode, rel_file} ->
         filename = Path.join("/emoji/#{pack_name}", rel_file)
 
-        {shortcode, filename, [to_string(match_extra(@groups, filename))]}
+        {shortcode, filename, [to_string(match_extra(emoji_groups, filename))]}
       end)
     end
   end
@@ -184,21 +185,21 @@ def find_all_emoji(dir, exts) do
     |> Enum.filter(fn f -> Path.extname(f) in exts end)
   end
 
-  defp load_from_file(file) do
+  defp load_from_file(file, emoji_groups) do
     if File.exists?(file) do
-      load_from_file_stream(File.stream!(file))
+      load_from_file_stream(File.stream!(file), emoji_groups)
     else
       []
     end
   end
 
-  defp load_from_file_stream(stream) do
+  defp load_from_file_stream(stream, emoji_groups) do
     stream
     |> Stream.map(&String.trim/1)
     |> Stream.map(fn line ->
       case String.split(line, ~r/,\s*/) do
         [name, file] ->
-          {name, file, [to_string(match_extra(@groups, file))]}
+          {name, file, [to_string(match_extra(emoji_groups, file))]}
 
         [name, file | tags] ->
           {name, file, tags}
@@ -210,7 +211,7 @@ defp load_from_file_stream(stream) do
     |> Enum.to_list()
   end
 
-  defp load_from_globs(globs) do
+  defp load_from_globs(globs, emoji_groups) do
     static_path = Path.join(:code.priv_dir(:pleroma), "static")
 
     paths =
@@ -221,7 +222,7 @@ defp load_from_globs(globs) do
       |> Enum.concat()
 
     Enum.map(paths, fn path ->
-      tag = match_extra(@groups, Path.join("/", Path.relative_to(path, static_path)))
+      tag = match_extra(emoji_groups, Path.join("/", Path.relative_to(path, static_path)))
       shortcode = Path.basename(path, Path.extname(path))
       external_path = Path.join("/", Path.relative_to(path, static_path))
       {shortcode, external_path, [to_string(tag)]}

lib/pleroma/helpers/uri_helper.ex

@@ -0,0 +1,27 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Helpers.UriHelper do
+  def append_uri_params(uri, appended_params) do
+    uri = URI.parse(uri)
+    appended_params = for {k, v} <- appended_params, into: %{}, do: {to_string(k), v}
+    existing_params = URI.query_decoder(uri.query || "") |> Enum.into(%{})
+    updated_params_keys = Enum.uniq(Map.keys(existing_params) ++ Map.keys(appended_params))
+
+    updated_params =
+      for k <- updated_params_keys, do: {k, appended_params[k] || existing_params[k]}
+
+    uri
+    |> Map.put(:query, URI.encode_query(updated_params))
+    |> URI.to_string()
+  end
+
+  def append_param_if_present(%{} = params, param_name, param_value) do
+    if param_value do
+      Map.put(params, param_name, param_value)
+    else
+      params
+    end
+  end
+end

lib/pleroma/html.ex

@@ -89,7 +89,7 @@ def extract_first_external_url(object, content) do
     Cachex.fetch!(:scrubber_cache, key, fn _key ->
       result =
         content
-        |> Floki.filter_out("a.mention")
+        |> Floki.filter_out("a.mention,a.hashtag")
         |> Floki.attribute("a", "href")
         |> Enum.at(0)
 

lib/pleroma/instances.ex

@@ -13,7 +13,7 @@ def set_consistently_unreachable(url_or_host),
 
   def reachability_datetime_threshold do
     federation_reachability_timeout_days =
-      Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 0
+      Pleroma.Config.get([:instance, :federation_reachability_timeout_days], 0)
 
     if federation_reachability_timeout_days > 0 do
       NaiveDateTime.add(

lib/pleroma/notification.ex

@@ -13,6 +13,8 @@ defmodule Pleroma.Notification do
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
+  alias Pleroma.Web.Push
+  alias Pleroma.Web.Streamer
 
   import Ecto.Query
   import Ecto.Changeset
@@ -145,8 +147,9 @@ def create_notification(%Activity{} = activity, %User{} = user) do
     unless skip?(activity, user) do
       notification = %Notification{user_id: user.id, activity: activity}
       {:ok, notification} = Repo.insert(notification)
-      Pleroma.Web.Streamer.stream("user", notification)
+      Streamer.stream("user", notification)
-      Pleroma.Web.Push.send(notification)
+      Streamer.stream("user:notification", notification)
+      Push.send(notification)
       notification
     end
   end

lib/pleroma/object/containment.ex

@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Object.Containment do
   @moduledoc """
   This module contains some useful functions for containing objects to specific

lib/pleroma/object/fetcher.ex

@@ -85,6 +85,9 @@ def fetch_and_contain_remote_object_from_id(id) do
          :ok <- Containment.contain_origin_from_id(id, data) do
       {:ok, data}
     else
+      {:ok, %{status: code}} when code in [404, 410] ->
+        {:error, "Object has been deleted"}
+
       e ->
         {:error, e}
     end

lib/pleroma/plugs/rate_limiter.ex

@@ -14,13 +14,20 @@ defmodule Pleroma.Plugs.RateLimiter do
 
   It is also possible to have different limits for unauthenticated and authenticated users: the keyword value must be a list of two tuples where the first one is a config for unauthenticated users and the second one is for authenticated.
 
+  To disable a limiter set its value to `nil`.
+
   ### Example
 
       config :pleroma, :rate_limit,
         one: {1000, 10},
-        two: [{10_000, 10}, {10_000, 50}]
+        two: [{10_000, 10}, {10_000, 50}],
+        foobar: nil
 
-  Here we have two limiters: `one` which is not over 10req/1s and `two` which has two limits 10req/10s for unauthenticated users and 50req/10s for authenticated users.
+  Here we have three limiters:
+
+  * `one` which is not over 10req/1s
+  * `two` which has two limits: 10req/10s for unauthenticated users and 50req/10s for authenticated users
+  * `foobar` which is disabled
 
   ## Usage
 

lib/pleroma/plugs/uploaded_media.ex

@@ -36,7 +36,7 @@ def call(%{request_path: <<"/", @path, "/", file::binary>>} = conn, opts) do
           conn
       end
 
-    config = Pleroma.Config.get([Pleroma.Upload])
+    config = Pleroma.Config.get(Pleroma.Upload)
 
     with uploader <- Keyword.fetch!(config, :uploader),
          proxy_remote = Keyword.get(config, :proxy_remote, false),

lib/pleroma/reverse_proxy.ex

@@ -146,7 +146,7 @@ defp request(method, url, headers, hackney_opts) do
     Logger.debug("#{__MODULE__} #{method} #{url} #{inspect(headers)}")
     method = method |> String.downcase() |> String.to_existing_atom()
 
-    case :hackney.request(method, url, headers, "", hackney_opts) do
+    case hackney().request(method, url, headers, "", hackney_opts) do
       {:ok, code, headers, client} when code in @valid_resp_codes ->
         {:ok, code, downcase_headers(headers), client}
 
@@ -196,7 +196,7 @@ defp chunk_reply(conn, client, opts, sent_so_far, duration) do
              duration,
              Keyword.get(opts, :max_read_duration, @max_read_duration)
            ),
-         {:ok, data} <- :hackney.stream_body(client),
+         {:ok, data} <- hackney().stream_body(client),
          {:ok, duration} <- increase_read_duration(duration),
          sent_so_far = sent_so_far + byte_size(data),
          :ok <- body_size_constraint(sent_so_far, Keyword.get(opts, :max_body_size)),
@@ -377,4 +377,6 @@ defp increase_read_duration({previous_duration, started})
   defp increase_read_duration(_) do
     {:ok, :no_duration_limit, :no_duration_limit}
   end
+
+  defp hackney, do: Pleroma.Config.get(:hackney, :hackney)
 end

lib/pleroma/user.ex

@@ -1036,9 +1036,7 @@ def html_filter_policy(%User{info: %{no_rich_text: true}}) do
     Pleroma.HTML.Scrubber.TwitterText
   end
 
-  @default_scrubbers Pleroma.Config.get([:markup, :scrub_policy])
+  def html_filter_policy(_), do: Pleroma.Config.get([:markup, :scrub_policy])
-
-  def html_filter_policy(_), do: @default_scrubbers
 
   def fetch_by_ap_id(ap_id) do
     ap_try = ActivityPub.make_user_from_ap_id(ap_id)

lib/pleroma/user/search.ex

@@ -7,45 +7,69 @@ defmodule Pleroma.User.Search do
   alias Pleroma.User
   import Ecto.Query
 
-  def search(query, opts \\ []) do
+  @similarity_threshold 0.25
+  @limit 20
+
+  def search(query_string, opts \\ []) do
     resolve = Keyword.get(opts, :resolve, false)
+    following = Keyword.get(opts, :following, false)
+    result_limit = Keyword.get(opts, :limit, @limit)
+    offset = Keyword.get(opts, :offset, 0)
+
     for_user = Keyword.get(opts, :for_user)
 
     # Strip the beginning @ off if there is a query
-    query = String.trim_leading(query, "@")
+    query_string = String.trim_leading(query_string, "@")
 
-    maybe_resolve(resolve, for_user, query)
+    maybe_resolve(resolve, for_user, query_string)
 
     {:ok, results} =
       Repo.transaction(fn ->
-        Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", [])
+        Ecto.Adapters.SQL.query(
+          Repo,
+          "select set_limit(#{@similarity_threshold})",
+          []
+        )
 
-        query
+        query_string
-        |> search_query(for_user)
+        |> search_query(for_user, following)
+        |> paginate(result_limit, offset)
         |> Repo.all()
       end)
 
     results
   end
 
-  defp search_query(query, for_user) do
+  defp search_query(query_string, for_user, following) do
-    query
+    for_user
-    |> union_query()
+    |> base_query(following)
+    |> search_subqueries(query_string)
+    |> union_subqueries
     |> distinct_query()
     |> boost_search_rank_query(for_user)
     |> subquery()
     |> order_by(desc: :search_rank)
-    |> limit(20)
     |> maybe_restrict_local(for_user)
   end
 
-  defp union_query(query) do
+  defp base_query(_user, false), do: User
-    fts_subquery = fts_search_subquery(query)
+  defp base_query(user, true), do: User.get_followers_query(user)
-    trigram_subquery = trigram_search_subquery(query)
 
+  defp paginate(query, limit, offset) do
+    from(q in query, limit: ^limit, offset: ^offset)
+  end
+
+  defp union_subqueries({fts_subquery, trigram_subquery}) do
     from(s in trigram_subquery, union_all: ^fts_subquery)
   end
 
+  defp search_subqueries(base_query, query_string) do
+    {
+      fts_search_subquery(base_query, query_string),
+      trigram_search_subquery(base_query, query_string)
+    }
+  end
+
   defp distinct_query(q) do
     from(s in subquery(q), order_by: s.search_type, distinct: s.id)
   end
@@ -102,7 +126,8 @@ defp boost_search_rank_query(query, for_user) do
     )
   end
 
-  defp fts_search_subquery(term, query \\ User) do
+  @spec fts_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t()
+  defp fts_search_subquery(query, term) do
     processed_query =
       term
       |> String.replace(~r/\W+/, " ")
@@ -144,9 +169,10 @@ defp fts_search_subquery(term, query \\ User) do
     |> User.restrict_deactivated()
   end
 
-  defp trigram_search_subquery(term) do
+  @spec trigram_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t()
+  defp trigram_search_subquery(query, term) do
     from(
-      u in User,
+      u in query,
       select_merge: %{
         # ^1 gives 'Postgrex expected a binary, got 1' for some weird reason
         search_type: fragment("?", 1),

lib/pleroma/web/activity_pub/publisher.ex

@@ -88,7 +88,7 @@ defp should_federate?(inbox, public) do
       true
     else
       inbox_info = URI.parse(inbox)
-      !Enum.member?(Pleroma.Config.get([:instance, :quarantined_instances], []), inbox_info.host)
+      !Enum.member?(Config.get([:instance, :quarantined_instances], []), inbox_info.host)
     end
   end
 

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -10,6 +10,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.AdminAPI.AccountView
+  alias Pleroma.Web.AdminAPI.Config
+  alias Pleroma.Web.AdminAPI.ConfigView
   alias Pleroma.Web.AdminAPI.ReportView
   alias Pleroma.Web.AdminAPI.Search
   alias Pleroma.Web.CommonAPI
@@ -362,6 +364,41 @@ def status_delete(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     end
   end
 
+  def config_show(conn, _params) do
+    configs = Pleroma.Repo.all(Config)
+
+    conn
+    |> put_view(ConfigView)
+    |> render("index.json", %{configs: configs})
+  end
+
+  def config_update(conn, %{"configs" => configs}) do
+    updated =
+      if Pleroma.Config.get([:instance, :dynamic_configuration]) do
+        updated =
+          Enum.map(configs, fn
+            %{"key" => key, "value" => value} ->
+              {:ok, config} = Config.update_or_create(%{key: key, value: value})
+              config
+
+            %{"key" => key, "delete" => "true"} ->
+              {:ok, _} = Config.delete(key)
+              nil
+          end)
+          |> Enum.reject(&is_nil(&1))
+
+        Pleroma.Config.TransferTask.load_and_update_env()
+        Mix.Tasks.Pleroma.Config.run(["migrate_from_db", Pleroma.Config.get(:env)])
+        updated
+      else
+        []
+      end
+
+    conn
+    |> put_view(ConfigView)
+    |> render("index.json", %{configs: updated})
+  end
+
   def errors(conn, {:error, :not_found}) do
     conn
     |> put_status(404)

lib/pleroma/web/admin_api/config.ex

@@ -0,0 +1,144 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.Config do
+  use Ecto.Schema
+  import Ecto.Changeset
+  alias __MODULE__
+  alias Pleroma.Repo
+
+  @type t :: %__MODULE__{}
+
+  schema "config" do
+    field(:key, :string)
+    field(:value, :binary)
+
+    timestamps()
+  end
+
+  @spec get_by_key(String.t()) :: Config.t() | nil
+  def get_by_key(key), do: Repo.get_by(Config, key: key)
+
+  @spec changeset(Config.t(), map()) :: Changeset.t()
+  def changeset(config, params \\ %{}) do
+    config
+    |> cast(params, [:key, :value])
+    |> validate_required([:key, :value])
+    |> unique_constraint(:key)
+  end
+
+  @spec create(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
+  def create(%{key: key, value: value}) do
+    %Config{}
+    |> changeset(%{key: key, value: transform(value)})
+    |> Repo.insert()
+  end
+
+  @spec update(Config.t(), map()) :: {:ok, Config} | {:error, Changeset.t()}
+  def update(%Config{} = config, %{value: value}) do
+    config
+    |> change(value: transform(value))
+    |> Repo.update()
+  end
+
+  @spec update_or_create(map()) :: {:ok, Config.t()} | {:error, Changeset.t()}
+  def update_or_create(%{key: key} = params) do
+    with %Config{} = config <- Config.get_by_key(key) do
+      Config.update(config, params)
+    else
+      nil -> Config.create(params)
+    end
+  end
+
+  @spec delete(String.t()) :: {:ok, Config.t()} | {:error, Changeset.t()}
+  def delete(key) do
+    with %Config{} = config <- Config.get_by_key(key) do
+      Repo.delete(config)
+    else
+      nil -> {:error, "Config with key #{key} not found"}
+    end
+  end
+
+  @spec from_binary(binary()) :: term()
+  def from_binary(value), do: :erlang.binary_to_term(value)
+
+  @spec from_binary_to_map(binary()) :: any()
+  def from_binary_to_map(binary) do
+    from_binary(binary)
+    |> do_convert()
+  end
+
+  defp do_convert([{k, v}] = value) when is_list(value) and length(value) == 1,
+    do: %{k => do_convert(v)}
+
+  defp do_convert(values) when is_list(values), do: for(val <- values, do: do_convert(val))
+
+  defp do_convert({k, v} = value) when is_tuple(value),
+    do: %{k => do_convert(v)}
+
+  defp do_convert(value) when is_binary(value) or is_atom(value) or is_map(value),
+    do: value
+
+  @spec transform(any()) :: binary()
+  def transform(entity) when is_map(entity) do
+    tuples =
+      for {k, v} <- entity,
+          into: [],
+          do: {if(is_atom(k), do: k, else: String.to_atom(k)), do_transform(v)}
+
+    Enum.reject(tuples, fn {_k, v} -> is_nil(v) end)
+    |> Enum.sort()
+    |> :erlang.term_to_binary()
+  end
+
+  def transform(entity) when is_list(entity) do
+    list = Enum.map(entity, &do_transform(&1))
+    :erlang.term_to_binary(list)
+  end
+
+  def transform(entity), do: :erlang.term_to_binary(entity)
+
+  defp do_transform(%Regex{} = value) when is_map(value), do: value
+
+  defp do_transform(value) when is_map(value) do
+    values =
+      for {key, val} <- value,
+          into: [],
+          do: {String.to_atom(key), do_transform(val)}
+
+    Enum.sort(values)
+  end
+
+  defp do_transform(value) when is_list(value) do
+    Enum.map(value, &do_transform(&1))
+  end
+
+  defp do_transform(entity) when is_list(entity) and length(entity) == 1, do: hd(entity)
+
+  defp do_transform(value) when is_binary(value) do
+    value = String.trim(value)
+
+    case String.length(value) do
+      0 ->
+        nil
+
+      _ ->
+        cond do
+          String.starts_with?(value, "Pleroma") ->
+            String.to_existing_atom("Elixir." <> value)
+
+          String.starts_with?(value, ":") ->
+            String.replace(value, ":", "") |> String.to_existing_atom()
+
+          String.starts_with?(value, "i:") ->
+            String.replace(value, "i:", "") |> String.to_integer()
+
+          true ->
+            value
+        end
+    end
+  end
+
+  defp do_transform(value), do: value
+end

lib/pleroma/web/admin_api/views/config_view.ex

@@ -0,0 +1,16 @@
+defmodule Pleroma.Web.AdminAPI.ConfigView do
+  use Pleroma.Web, :view
+
+  def render("index.json", %{configs: configs}) do
+    %{
+      configs: render_many(configs, __MODULE__, "show.json", as: :config)
+    }
+  end
+
+  def render("show.json", %{config: config}) do
+    %{
+      key: config.key,
+      value: Pleroma.Web.AdminAPI.Config.from_binary_to_map(config.value)
+    }
+  end
+end

lib/pleroma/web/controller_helper.ex

@@ -15,4 +15,22 @@ def json_response(conn, status, json) do
     |> put_status(status)
     |> json(json)
   end
+
+  @spec fetch_integer_param(map(), String.t(), integer() | nil) :: integer() | nil
+  def fetch_integer_param(params, name, default \\ nil) do
+    params
+    |> Map.get(name, default)
+    |> param_to_integer(default)
+  end
+
+  defp param_to_integer(val, _) when is_integer(val), do: val
+
+  defp param_to_integer(val, default) when is_binary(val) do
+    case Integer.parse(val) do
+      {res, _} -> res
+      _ -> default
+    end
+  end
+
+  defp param_to_integer(_, default), do: default
 end

lib/pleroma/web/endpoint.ex

@@ -91,7 +91,7 @@ defmodule Pleroma.Web.Endpoint do
     Plug.Session,
     store: :cookie,
     key: cookie_name,
-    signing_salt: {Pleroma.Config, :get, [[__MODULE__, :signing_salt], "CqaoopA2"]},
+    signing_salt: Pleroma.Config.get([__MODULE__, :signing_salt], "CqaoopA2"),
     http_only: true,
     secure: secure_cookies,
     extra: extra

lib/pleroma/web/mastodon_api/mastodon_api_controller.ex

@@ -136,6 +136,14 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do
           _ -> :error
         end
       end)
+      |> add_if_present(params, "pleroma_background_image", :background, fn value ->
+        with %Plug.Upload{} <- value,
+             {:ok, object} <- ActivityPub.upload(value, type: :background) do
+          {:ok, object.data}
+        else
+          _ -> :error
+        end
+      end)
       |> Map.put(:emoji, user_info_emojis)
 
     info_cng = User.Info.profile_update(user.info, info_params)
@@ -160,8 +168,15 @@ def update_credentials(%{assigns: %{user: user}} = conn, params) do
   end
 
   def verify_credentials(%{assigns: %{user: user}} = conn, _) do
+    chat_token = Phoenix.Token.sign(conn, "user socket", user.id)
+
     account =
-      AccountView.render("account.json", %{user: user, for: user, with_pleroma_settings: true})
+      AccountView.render("account.json", %{
+        user: user,
+        for: user,
+        with_pleroma_settings: true,
+        with_chat_token: chat_token
+      })
 
     json(conn, account)
   end
@@ -439,12 +454,26 @@ def get_poll(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     end
   end
 
+  defp get_cached_vote_or_vote(user, object, choices) do
+    idempotency_key = "polls:#{user.id}:#{object.data["id"]}"
+
+    {_, res} =
+      Cachex.fetch(:idempotency_cache, idempotency_key, fn _ ->
+        case CommonAPI.vote(user, object, choices) do
+          {:error, _message} = res -> {:ignore, res}
+          res -> {:commit, res}
+        end
+      end)
+
+    res
+  end
+
   def poll_vote(%{assigns: %{user: user}} = conn, %{"id" => id, "choices" => choices}) do
     with %Object{} = object <- Object.get_by_id(id),
          true <- object.data["type"] == "Question",
          %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]),
          true <- Visibility.visible_for_user?(activity, user),
-         {:ok, _activities, object} <- CommonAPI.vote(user, object, choices) do
+         {:ok, _activities, object} <- get_cached_vote_or_vote(user, object, choices) do
       conn
       |> put_view(StatusView)
       |> try_render("poll.json", %{object: object, for: user})
@@ -1118,58 +1147,6 @@ def unsubscribe(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     end
   end
 
-  def search2(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
-    accounts = User.search(query, resolve: params["resolve"] == "true", for_user: user)
-    statuses = Activity.search(user, query)
-    tags_path = Web.base_url() <> "/tag/"
-
-    tags =
-      query
-      |> String.split()
-      |> Enum.uniq()
-      |> Enum.filter(fn tag -> String.starts_with?(tag, "#") end)
-      |> Enum.map(fn tag -> String.slice(tag, 1..-1) end)
-      |> Enum.map(fn tag -> %{name: tag, url: tags_path <> tag} end)
-
-    res = %{
-      "accounts" => AccountView.render("accounts.json", users: accounts, for: user, as: :user),
-      "statuses" =>
-        StatusView.render("index.json", activities: statuses, for: user, as: :activity),
-      "hashtags" => tags
-    }
-
-    json(conn, res)
-  end
-
-  def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
-    accounts = User.search(query, resolve: params["resolve"] == "true", for_user: user)
-    statuses = Activity.search(user, query)
-
-    tags =
-      query
-      |> String.split()
-      |> Enum.uniq()
-      |> Enum.filter(fn tag -> String.starts_with?(tag, "#") end)
-      |> Enum.map(fn tag -> String.slice(tag, 1..-1) end)
-
-    res = %{
-      "accounts" => AccountView.render("accounts.json", users: accounts, for: user, as: :user),
-      "statuses" =>
-        StatusView.render("index.json", activities: statuses, for: user, as: :activity),
-      "hashtags" => tags
-    }
-
-    json(conn, res)
-  end
-
-  def account_search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
-    accounts = User.search(query, resolve: params["resolve"] == "true", for_user: user)
-
-    res = AccountView.render("accounts.json", users: accounts, for: user, as: :user)
-
-    json(conn, res)
-  end
-
   def favourites(%{assigns: %{user: user}} = conn, params) do
     params =
       params

lib/pleroma/web/mastodon_api/search_controller.ex

@@ -0,0 +1,79 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MastodonAPI.SearchController do
+  use Pleroma.Web, :controller
+  alias Pleroma.Activity
+  alias Pleroma.User
+  alias Pleroma.Web
+  alias Pleroma.Web.MastodonAPI.AccountView
+  alias Pleroma.Web.MastodonAPI.StatusView
+
+  alias Pleroma.Web.ControllerHelper
+
+  require Logger
+
+  plug(Pleroma.Plugs.RateLimiter, :search when action in [:search, :search2, :account_search])
+
+  def search2(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
+    accounts = User.search(query, search_options(params, user))
+    statuses = Activity.search(user, query)
+    tags_path = Web.base_url() <> "/tag/"
+
+    tags =
+      query
+      |> String.split()
+      |> Enum.uniq()
+      |> Enum.filter(fn tag -> String.starts_with?(tag, "#") end)
+      |> Enum.map(fn tag -> String.slice(tag, 1..-1) end)
+      |> Enum.map(fn tag -> %{name: tag, url: tags_path <> tag} end)
+
+    res = %{
+      "accounts" => AccountView.render("accounts.json", users: accounts, for: user, as: :user),
+      "statuses" =>
+        StatusView.render("index.json", activities: statuses, for: user, as: :activity),
+      "hashtags" => tags
+    }
+
+    json(conn, res)
+  end
+
+  def search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
+    accounts = User.search(query, search_options(params, user))
+    statuses = Activity.search(user, query)
+
+    tags =
+      query
+      |> String.split()
+      |> Enum.uniq()
+      |> Enum.filter(fn tag -> String.starts_with?(tag, "#") end)
+      |> Enum.map(fn tag -> String.slice(tag, 1..-1) end)
+
+    res = %{
+      "accounts" => AccountView.render("accounts.json", users: accounts, for: user, as: :user),
+      "statuses" =>
+        StatusView.render("index.json", activities: statuses, for: user, as: :activity),
+      "hashtags" => tags
+    }
+
+    json(conn, res)
+  end
+
+  def account_search(%{assigns: %{user: user}} = conn, %{"q" => query} = params) do
+    accounts = User.search(query, search_options(params, user))
+    res = AccountView.render("accounts.json", users: accounts, for: user, as: :user)
+
+    json(conn, res)
+  end
+
+  defp search_options(params, user) do
+    [
+      resolve: params["resolve"] == "true",
+      following: params["following"] == "true",
+      limit: ControllerHelper.fetch_integer_param(params, "limit"),
+      offset: ControllerHelper.fetch_integer_param(params, "offset"),
+      for_user: user
+    ]
+  end
+end

lib/pleroma/web/mastodon_api/views/account_view.ex

@@ -125,13 +125,15 @@ defp do_render("account.json", %{user: user} = opts) do
         hide_follows: user.info.hide_follows,
         hide_favorites: user.info.hide_favorites,
         relationship: relationship,
-        skip_thread_containment: user.info.skip_thread_containment
+        skip_thread_containment: user.info.skip_thread_containment,
+        background_image: image_url(user.info.background) |> MediaProxy.url()
       }
     }
     |> maybe_put_role(user, opts[:for])
     |> maybe_put_settings(user, opts[:for], user_info)
     |> maybe_put_notification_settings(user, opts[:for])
     |> maybe_put_settings_store(user, opts[:for], opts)
+    |> maybe_put_chat_token(user, opts[:for], opts)
   end
 
   defp username_from_nickname(string) when is_binary(string) do
@@ -163,6 +165,15 @@ defp maybe_put_settings_store(data, %User{info: info, id: id}, %User{id: id}, %{
 
   defp maybe_put_settings_store(data, _, _, _), do: data
 
+  defp maybe_put_chat_token(data, %User{id: id}, %User{id: id}, %{
+         with_chat_token: token
+       }) do
+    data
+    |> Kernel.put_in([:pleroma, :chat_token], token)
+  end
+
+  defp maybe_put_chat_token(data, _, _, _), do: data
+
   defp maybe_put_role(data, %User{info: %{show_role: true}} = user, _) do
     data
     |> Kernel.put_in([:pleroma, :is_admin], user.info.is_admin)
@@ -182,4 +193,7 @@ defp maybe_put_notification_settings(data, %User{id: user_id} = user, %User{id:
   end
 
   defp maybe_put_notification_settings(data, _, _), do: data
+
+  defp image_url(%{"url" => [%{"href" => href} | _]}), do: href
+  defp image_url(_), do: nil
 end

lib/pleroma/web/mastodon_api/websocket_handler.ex

@@ -17,6 +17,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
     "public:media",
     "public:local:media",
     "user",
+    "user:notification",
     "direct",
     "list",
     "hashtag"

lib/pleroma/web/oauth/oauth_controller.ex

@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.OAuth.OAuthController do
   use Pleroma.Web, :controller
 
+  alias Pleroma.Helpers.UriHelper
   alias Pleroma.Registration
   alias Pleroma.Repo
   alias Pleroma.User
@@ -26,34 +27,25 @@ defmodule Pleroma.Web.OAuth.OAuthController do
 
   action_fallback(Pleroma.Web.OAuth.FallbackController)
 
+  @oob_token_redirect_uri "urn:ietf:wg:oauth:2.0:oob"
+
   # Note: this definition is only called from error-handling methods with `conn.params` as 2nd arg
-  def authorize(conn, %{"authorization" => _} = params) do
+  def authorize(%Plug.Conn{} = conn, %{"authorization" => _} = params) do
     {auth_attrs, params} = Map.pop(params, "authorization")
     authorize(conn, Map.merge(params, auth_attrs))
   end
 
-  def authorize(%{assigns: %{token: %Token{} = token}} = conn, params) do
+  def authorize(%Plug.Conn{assigns: %{token: %Token{}}} = conn, params) do
     if ControllerHelper.truthy_param?(params["force_login"]) do
       do_authorize(conn, params)
     else
-      redirect_uri =
+      handle_existing_authorization(conn, params)
-        if is_binary(params["redirect_uri"]) do
-          params["redirect_uri"]
-        else
-          app = Repo.preload(token, :app).app
-
-          app.redirect_uris
-          |> String.split()
-          |> Enum.at(0)
-        end
-
-      redirect(conn, external: redirect_uri(conn, redirect_uri))
     end
   end
 
-  def authorize(conn, params), do: do_authorize(conn, params)
+  def authorize(%Plug.Conn{} = conn, params), do: do_authorize(conn, params)
 
-  defp do_authorize(conn, params) do
+  defp do_authorize(%Plug.Conn{} = conn, params) do
     app = Repo.get_by(App, client_id: params["client_id"])
     available_scopes = (app && app.scopes) || []
     scopes = Scopes.fetch_scopes(params, available_scopes)
@@ -70,8 +62,33 @@ defp do_authorize(conn, params) do
     })
   end
 
+  defp handle_existing_authorization(
+         %Plug.Conn{assigns: %{token: %Token{} = token}} = conn,
+         params
+       ) do
+    token = Repo.preload(token, :app)
+
+    redirect_uri =
+      if is_binary(params["redirect_uri"]) do
+        params["redirect_uri"]
+      else
+        default_redirect_uri(token.app)
+      end
+
+    redirect_uri = redirect_uri(conn, redirect_uri)
+
+    if redirect_uri == @oob_token_redirect_uri do
+      render(conn, "oob_token_exists.html", %{token: token})
+    else
+      url_params = %{access_token: token.token}
+      url_params = UriHelper.append_param_if_present(url_params, :state, params["state"])
+      url = UriHelper.append_uri_params(redirect_uri, url_params)
+      redirect(conn, external: url)
+    end
+  end
+
   def create_authorization(
-        conn,
+        %Plug.Conn{} = conn,
         %{"authorization" => _} = params,
         opts \\ []
       ) do
@@ -83,35 +100,23 @@ def create_authorization(
     end
   end
 
-  def after_create_authorization(conn, auth, %{
+  def after_create_authorization(%Plug.Conn{} = conn, %Authorization{} = auth, %{
         "authorization" => %{"redirect_uri" => redirect_uri} = auth_attrs
       }) do
     redirect_uri = redirect_uri(conn, redirect_uri)
 
-    if redirect_uri == "urn:ietf:wg:oauth:2.0:oob" do
+    if redirect_uri == @oob_token_redirect_uri do
-      render(conn, "results.html", %{
+      render(conn, "oob_authorization_created.html", %{auth: auth})
-        auth: auth
-      })
     else
-      connector = if String.contains?(redirect_uri, "?"), do: "&", else: "?"
+      url_params = %{code: auth.token}
-      url = "#{redirect_uri}#{connector}"
+      url_params = UriHelper.append_param_if_present(url_params, :state, auth_attrs["state"])
-      url_params = %{:code => auth.token}
+      url = UriHelper.append_uri_params(redirect_uri, url_params)
-
-      url_params =
-        if auth_attrs["state"] do
-          Map.put(url_params, :state, auth_attrs["state"])
-        else
-          url_params
-        end
-
-      url = "#{url}#{Plug.Conn.Query.encode(url_params)}"
-
       redirect(conn, external: url)
     end
   end
 
   defp handle_create_authorization_error(
-         conn,
+         %Plug.Conn{} = conn,
          {:error, scopes_issue},
          %{"authorization" => _} = params
        )
@@ -125,7 +130,7 @@ defp handle_create_authorization_error(
   end
 
   defp handle_create_authorization_error(
-         conn,
+         %Plug.Conn{} = conn,
          {:auth_active, false},
          %{"authorization" => _} = params
        ) do
@@ -137,13 +142,13 @@ defp handle_create_authorization_error(
     |> authorize(params)
   end
 
-  defp handle_create_authorization_error(conn, error, %{"authorization" => _}) do
+  defp handle_create_authorization_error(%Plug.Conn{} = conn, error, %{"authorization" => _}) do
     Authenticator.handle_error(conn, error)
   end
 
   @doc "Renew access_token with refresh_token"
   def token_exchange(
-        conn,
+        %Plug.Conn{} = conn,
         %{"grant_type" => "refresh_token", "refresh_token" => token} = _params
       ) do
     with {:ok, app} <- Token.Utils.fetch_app(conn),
@@ -159,7 +164,7 @@ def token_exchange(
     end
   end
 
-  def token_exchange(conn, %{"grant_type" => "authorization_code"} = params) do
+  def token_exchange(%Plug.Conn{} = conn, %{"grant_type" => "authorization_code"} = params) do
     with {:ok, app} <- Token.Utils.fetch_app(conn),
          fixed_token = Token.Utils.fix_padding(params["code"]),
          {:ok, auth} <- Authorization.get_by_token(app, fixed_token),
@@ -176,7 +181,7 @@ def token_exchange(conn, %{"grant_type" => "authorization_code"} = params) do
   end
 
   def token_exchange(
-        conn,
+        %Plug.Conn{} = conn,
         %{"grant_type" => "password"} = params
       ) do
     with {:ok, %User{} = user} <- Authenticator.get_user(conn),
@@ -207,7 +212,7 @@ def token_exchange(
   end
 
   def token_exchange(
-        conn,
+        %Plug.Conn{} = conn,
         %{"grant_type" => "password", "name" => name, "password" => _password} = params
       ) do
     params =
@@ -218,7 +223,7 @@ def token_exchange(
     token_exchange(conn, params)
   end
 
-  def token_exchange(conn, %{"grant_type" => "client_credentials"} = _params) do
+  def token_exchange(%Plug.Conn{} = conn, %{"grant_type" => "client_credentials"} = _params) do
     with {:ok, app} <- Token.Utils.fetch_app(conn),
          {:ok, auth} <- Authorization.create_authorization(app, %User{}),
          {:ok, token} <- Token.exchange_token(app, auth) do
@@ -231,9 +236,9 @@ def token_exchange(conn, %{"grant_type" => "client_credentials"} = _params) do
   end
 
   # Bad request
-  def token_exchange(conn, params), do: bad_request(conn, params)
+  def token_exchange(%Plug.Conn{} = conn, params), do: bad_request(conn, params)
 
-  def token_revoke(conn, %{"token" => _token} = params) do
+  def token_revoke(%Plug.Conn{} = conn, %{"token" => _token} = params) do
     with {:ok, app} <- Token.Utils.fetch_app(conn),
          {:ok, _token} <- RevokeToken.revoke(app, params) do
       json(conn, %{})
@@ -244,17 +249,20 @@ def token_revoke(conn, %{"token" => _token} = params) do
     end
   end
 
-  def token_revoke(conn, params), do: bad_request(conn, params)
+  def token_revoke(%Plug.Conn{} = conn, params), do: bad_request(conn, params)
 
   # Response for bad request
-  defp bad_request(conn, _) do
+  defp bad_request(%Plug.Conn{} = conn, _) do
     conn
     |> put_status(500)
     |> json(%{error: "Bad request"})
   end
 
   @doc "Prepares OAuth request to provider for Ueberauth"
-  def prepare_request(conn, %{"provider" => provider, "authorization" => auth_attrs}) do
+  def prepare_request(%Plug.Conn{} = conn, %{
+        "provider" => provider,
+        "authorization" => auth_attrs
+      }) do
     scope =
       auth_attrs
       |> Scopes.fetch_scopes([])
@@ -275,7 +283,7 @@ def prepare_request(conn, %{"provider" => provider, "authorization" => auth_attr
     redirect(conn, to: o_auth_path(conn, :request, provider, params))
   end
 
-  def request(conn, params) do
+  def request(%Plug.Conn{} = conn, params) do
     message =
       if params["provider"] do
         "Unsupported OAuth provider: #{params["provider"]}."
@@ -288,7 +296,7 @@ def request(conn, params) do
     |> redirect(to: "/")
   end
 
-  def callback(%{assigns: %{ueberauth_failure: failure}} = conn, params) do
+  def callback(%Plug.Conn{assigns: %{ueberauth_failure: failure}} = conn, params) do
     params = callback_params(params)
     messages = for e <- Map.get(failure, :errors, []), do: e.message
     message = Enum.join(messages, "; ")
@@ -298,7 +306,7 @@ def callback(%{assigns: %{ueberauth_failure: failure}} = conn, params) do
     |> redirect(external: redirect_uri(conn, params["redirect_uri"]))
   end
 
-  def callback(conn, params) do
+  def callback(%Plug.Conn{} = conn, params) do
     params = callback_params(params)
 
     with {:ok, registration} <- Authenticator.get_registration(conn) do
@@ -333,7 +341,7 @@ defp callback_params(%{"state" => state} = params) do
     Map.merge(params, Jason.decode!(state))
   end
 
-  def registration_details(conn, %{"authorization" => auth_attrs}) do
+  def registration_details(%Plug.Conn{} = conn, %{"authorization" => auth_attrs}) do
     render(conn, "register.html", %{
       client_id: auth_attrs["client_id"],
       redirect_uri: auth_attrs["redirect_uri"],
@@ -344,7 +352,7 @@ def registration_details(conn, %{"authorization" => auth_attrs}) do
     })
   end
 
-  def register(conn, %{"authorization" => _, "op" => "connect"} = params) do
+  def register(%Plug.Conn{} = conn, %{"authorization" => _, "op" => "connect"} = params) do
     with registration_id when not is_nil(registration_id) <- get_session_registration_id(conn),
          %Registration{} = registration <- Repo.get(Registration, registration_id),
          {_, {:ok, auth}} <-
@@ -363,7 +371,7 @@ def register(conn, %{"authorization" => _, "op" => "connect"} = params) do
     end
   end
 
-  def register(conn, %{"authorization" => _, "op" => "register"} = params) do
+  def register(%Plug.Conn{} = conn, %{"authorization" => _, "op" => "register"} = params) do
     with registration_id when not is_nil(registration_id) <- get_session_registration_id(conn),
          %Registration{} = registration <- Repo.get(Registration, registration_id),
          {:ok, user} <- Authenticator.create_from_registration(conn, registration) do
@@ -399,7 +407,7 @@ def register(conn, %{"authorization" => _, "op" => "register"} = params) do
   end
 
   defp do_create_authorization(
-         conn,
+         %Plug.Conn{} = conn,
          %{
            "authorization" =>
              %{
@@ -420,13 +428,13 @@ defp do_create_authorization(
   end
 
   # Special case: Local MastodonFE
-  defp redirect_uri(conn, "."), do: mastodon_api_url(conn, :login)
+  defp redirect_uri(%Plug.Conn{} = conn, "."), do: mastodon_api_url(conn, :login)
 
-  defp redirect_uri(_conn, redirect_uri), do: redirect_uri
+  defp redirect_uri(%Plug.Conn{}, redirect_uri), do: redirect_uri
 
-  defp get_session_registration_id(conn), do: get_session(conn, :registration_id)
+  defp get_session_registration_id(%Plug.Conn{} = conn), do: get_session(conn, :registration_id)
 
-  defp put_session_registration_id(conn, registration_id),
+  defp put_session_registration_id(%Plug.Conn{} = conn, registration_id),
     do: put_session(conn, :registration_id, registration_id)
 
   @spec validate_scopes(App.t(), map()) ::
@@ -436,4 +444,10 @@ defp validate_scopes(app, params) do
     |> Scopes.fetch_scopes(app.scopes)
     |> Scopes.validates(app.scopes)
   end
+
+  defp default_redirect_uri(%App{} = app) do
+    app.redirect_uris
+    |> String.split()
+    |> Enum.at(0)
+  end
 end

lib/pleroma/web/oauth/token.ex

@@ -14,7 +14,6 @@ defmodule Pleroma.Web.OAuth.Token do
   alias Pleroma.Web.OAuth.Token
   alias Pleroma.Web.OAuth.Token.Query
 
-  @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600)
   @type t :: %__MODULE__{}
 
   schema "oauth_tokens" do
@@ -78,7 +77,7 @@ defp put_refresh_token(changeset, attrs) do
 
   defp put_valid_until(changeset, attrs) do
     expires_in =
-      Map.get(attrs, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), @expires_in))
+      Map.get(attrs, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), expires_in()))
 
     changeset
     |> change(%{valid_until: expires_in})
@@ -123,4 +122,6 @@ def is_expired?(%__MODULE__{valid_until: valid_until}) do
   end
 
   def is_expired?(_), do: false
+
+  defp expires_in, do: Pleroma.Config.get([:oauth2, :token_expires_in], 600)
 end

lib/pleroma/web/oauth/token/response.ex

@@ -4,15 +4,13 @@ defmodule Pleroma.Web.OAuth.Token.Response do
   alias Pleroma.User
   alias Pleroma.Web.OAuth.Token.Utils
 
-  @expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600)
-
   @doc false
   def build(%User{} = user, token, opts \\ %{}) do
     %{
       token_type: "Bearer",
       access_token: token.token,
       refresh_token: token.refresh_token,
-      expires_in: @expires_in,
+      expires_in: expires_in(),
       scope: Enum.join(token.scopes, " "),
       me: user.ap_id
     }
@@ -25,8 +23,10 @@ def build_for_client_credentials(token) do
       access_token: token.token,
       refresh_token: token.refresh_token,
       created_at: Utils.format_created_at(token),
-      expires_in: @expires_in,
+      expires_in: expires_in(),
       scope: Enum.join(token.scopes, " ")
     }
   end
+
+  defp expires_in, do: Pleroma.Config.get([:oauth2, :token_expires_in], 600)
 end

lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex

@@ -1,15 +1,19 @@
 defmodule Pleroma.Web.RichMedia.Parsers.MetaTagsParser do
   def parse(html, data, prefix, error_message, key_name, value_name \\ "content") do
-    with elements = [_ | _] <- get_elements(html, key_name, prefix),
+    meta_data =
-         meta_data =
+      html
-           Enum.reduce(elements, data, fn el, acc ->
+      |> get_elements(key_name, prefix)
-             attributes = normalize_attributes(el, prefix, key_name, value_name)
+      |> Enum.reduce(data, fn el, acc ->
+        attributes = normalize_attributes(el, prefix, key_name, value_name)
 
-             Map.merge(acc, attributes)
+        Map.merge(acc, attributes)
-           end) do
+      end)
-      {:ok, meta_data}
+      |> maybe_put_title(html)
+
+    if Enum.empty?(meta_data) do
+      {:error, error_message}
     else
-      _e -> {:error, error_message}
+      {:ok, meta_data}
     end
   end
 
@@ -27,4 +31,17 @@ defp normalize_attributes(html_node, prefix, key_name, value_name) do
 
     %{String.to_atom(data[key_name]) => data[value_name]}
   end
+
+  defp maybe_put_title(%{title: _} = meta, _), do: meta
+
+  defp maybe_put_title(meta, html) do
+    case get_page_title(html) do
+      "" -> meta
+      title -> Map.put_new(meta, :title, title)
+    end
+  end
+
+  defp get_page_title(html) do
+    Floki.find(html, "title") |> Floki.text()
+  end
 end

lib/pleroma/web/router.ex

@@ -202,6 +202,9 @@ defmodule Pleroma.Web.Router do
 
     put("/statuses/:id", AdminAPIController, :status_update)
     delete("/statuses/:id", AdminAPIController, :status_delete)
+
+    get("/config", AdminAPIController, :config_show)
+    post("/config", AdminAPIController, :config_update)
   end
 
   scope "/", Pleroma.Web.TwitterAPI do
@@ -412,7 +415,7 @@ defmodule Pleroma.Web.Router do
 
     get("/trends", MastodonAPIController, :empty_array)
 
-    get("/accounts/search", MastodonAPIController, :account_search)
+    get("/accounts/search", SearchController, :account_search)
 
     scope [] do
       pipe_through(:oauth_read_or_public)
@@ -431,7 +434,7 @@ defmodule Pleroma.Web.Router do
       get("/accounts/:id/following", MastodonAPIController, :following)
       get("/accounts/:id", MastodonAPIController, :user)
 
-      get("/search", MastodonAPIController, :search)
+      get("/search", SearchController, :search)
 
       get("/pleroma/accounts/:id/favourites", MastodonAPIController, :user_favourites)
     end
@@ -439,7 +442,7 @@ defmodule Pleroma.Web.Router do
 
   scope "/api/v2", Pleroma.Web.MastodonAPI do
     pipe_through([:api, :oauth_read_or_public])
-    get("/search", MastodonAPIController, :search2)
+    get("/search", SearchController, :search2)
   end
 
   scope "/api", Pleroma.Web do
@@ -604,12 +607,6 @@ defmodule Pleroma.Web.Router do
     post("/push/subscriptions/:id", Websub.WebsubController, :websub_incoming)
   end
 
-  scope "/", Pleroma.Web do
-    pipe_through(:oembed)
-
-    get("/oembed", OEmbed.OEmbedController, :url)
-  end
-
   pipeline :activitypub do
     plug(:accepts, ["activity+json", "json"])
     plug(Pleroma.Web.Plugs.HTTPSignaturePlug)

lib/pleroma/web/streamer.ex

@@ -110,23 +110,18 @@ def handle_cast(%{action: :stream, topic: "list", item: item}, topics) do
     {:noreply, topics}
   end
 
-  def handle_cast(%{action: :stream, topic: "user", item: %Notification{} = item}, topics) do
+  def handle_cast(
-    topic = "user:#{item.user_id}"
+        %{action: :stream, topic: topic, item: %Notification{} = item},
-
+        topics
-    Enum.each(topics[topic] || [], fn socket ->
+      )
-      json =
+      when topic in ["user", "user:notification"] do
-        %{
+    topics
-          event: "notification",
+    |> Map.get("#{topic}:#{item.user_id}", [])
-          payload:
+    |> Enum.each(fn socket ->
-            NotificationView.render("show.json", %{
+      send(
-              notification: item,
+        socket.transport_pid,
-              for: socket.assigns["user"]
+        {:text, represent_notification(socket.assigns[:user], item)}
-            })
+      )
-            |> Jason.encode!()
-        }
-        |> Jason.encode!()
-
-      send(socket.transport_pid, {:text, json})
     end)
 
     {:noreply, topics}
@@ -216,6 +211,20 @@ def represent_conversation(%Participation{} = participation) do
     |> Jason.encode!()
   end
 
+  @spec represent_notification(User.t(), Notification.t()) :: binary()
+  defp represent_notification(%User{} = user, %Notification{} = notify) do
+    %{
+      event: "notification",
+      payload:
+        NotificationView.render(
+          "show.json",
+          %{notification: notify, for: user}
+        )
+        |> Jason.encode!()
+    }
+    |> Jason.encode!()
+  end
+
   def push_to_socket(topics, topic, %Activity{data: %{"type" => "Announce"}} = item) do
     Enum.each(topics[topic] || [], fn socket ->
       # Get the current user so we have up-to-date blocks etc.
@@ -274,7 +283,7 @@ def push_to_socket(topics, topic, item) do
     end)
   end
 
-  defp internal_topic(topic, socket) when topic in ~w[user direct] do
+  defp internal_topic(topic, socket) when topic in ~w[user user:notification direct] do
     "#{topic}:#{socket.assigns[:user].id}"
   end
 

lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex

@@ -0,0 +1,2 @@
+<h1>Authorization exists</h1>
+<h2>Access token is <%= @token.token %></h2>

mix.exs

@@ -85,7 +85,7 @@ defp oauth_deps do
   # Type `mix help deps` for examples and options.
   defp deps do
     [
-      {:phoenix, "~> 1.4.1"},
+      {:phoenix, "~> 1.4.8"},
       {:plug_cowboy, "~> 2.0"},
       {:phoenix_pubsub, "~> 1.1"},
       {:phoenix_ecto, "~> 4.0"},
@@ -136,7 +136,6 @@ defp deps do
       {:prometheus_plugs, "~> 1.1"},
       {:prometheus_phoenix, "~> 1.2"},
       {:prometheus_ecto, "~> 1.4"},
-      {:prometheus_process_collector, "~> 1.4"},
       {:recon, github: "ferd/recon", tag: "2.4.0"},
       {:quack, "~> 0.1.1"},
       {:benchee, "~> 1.0"},
@@ -177,7 +176,9 @@ defp version(version) do
            ahead <- String.replace(describe, tag, "") do
         {String.replace_prefix(tag, "v", ""), if(ahead != "", do: String.trim(ahead))}
       else
-        _ -> {nil, nil}
+        _ ->
+          {commit_hash, 0} = System.cmd("git", ["rev-parse", "--short", "HEAD"])
+          {nil, "-0-g" <> String.trim(commit_hash)}
       end
 
     if git_tag && version != git_tag do
@@ -204,7 +205,17 @@ defp version(version) do
             string -> "+" <> string
           end).()
 
-    [version, git_pre_release, build]
+    branch_name =
+      with {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]),
+           true <- branch_name != "master" do
+        branch_name =
+          String.trim(branch_name)
+          |> String.replace(~r/\W+/, "-")
+
+        "-" <> branch_name
+      end
+
+    [version, git_pre_release, branch_name, build]
     |> Enum.filter(fn string -> string && string != "" end)
     |> Enum.join()
   end

mix.lock

@@ -12,8 +12,8 @@
   "comeonin": {:hex, :comeonin, "4.1.1", "c7304fc29b45b897b34142a91122bc72757bc0c295e9e824999d5179ffc08416", [:mix], [{:argon2_elixir, "~> 1.2", [hex: :argon2_elixir, repo: "hexpm", optional: true]}, {:bcrypt_elixir, "~> 0.12.1 or ~> 1.0", [hex: :bcrypt_elixir, repo: "hexpm", optional: true]}, {:pbkdf2_elixir, "~> 0.12", [hex: :pbkdf2_elixir, repo: "hexpm", optional: true]}], "hexpm"},
   "connection": {:hex, :connection, "1.0.4", "a1cae72211f0eef17705aaededacac3eb30e6625b04a6117c1b2db6ace7d5976", [:mix], [], "hexpm"},
   "cors_plug": {:hex, :cors_plug, "1.5.2", "72df63c87e4f94112f458ce9d25800900cc88608c1078f0e4faddf20933eda6e", [:mix], [{:plug, "~> 1.3 or ~> 1.4 or ~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
-  "cowboy": {:hex, :cowboy, "2.6.1", "f2e06f757c337b3b311f9437e6e072b678fcd71545a7b2865bdaa154d078593f", [:rebar3], [{:cowlib, "~> 2.7.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"},
+  "cowboy": {:hex, :cowboy, "2.6.3", "99aa50e94e685557cad82e704457336a453d4abcb77839ad22dbe71f311fcc06", [:rebar3], [{:cowlib, "~> 2.7.3", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "~> 1.7.1", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm"},
-  "cowlib": {:hex, :cowlib, "2.7.0", "3ef16e77562f9855a2605900cedb15c1462d76fb1be6a32fc3ae91973ee543d2", [:rebar3], [], "hexpm"},
+  "cowlib": {:hex, :cowlib, "2.7.3", "a7ffcd0917e6d50b4d5fb28e9e2085a0ceb3c97dea310505f7460ff5ed764ce9", [:rebar3], [], "hexpm"},
   "credo": {:hex, :credo, "0.9.3", "76fa3e9e497ab282e0cf64b98a624aa11da702854c52c82db1bf24e54ab7c97a", [:mix], [{:bunt, "~> 0.2.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:poison, ">= 0.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"},
   "crypt": {:git, "https://github.com/msantos/crypt", "1f2b58927ab57e72910191a7ebaeff984382a1d3", [ref: "1f2b58927ab57e72910191a7ebaeff984382a1d3"]},
   "db_connection": {:hex, :db_connection, "2.0.6", "bde2f85d047969c5b5800cb8f4b3ed6316c8cb11487afedac4aa5f93fd39abfa", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, repo: "hexpm", optional: false]}], "hexpm"},
@@ -55,13 +55,13 @@
   "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm"},
   "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"},
   "pbkdf2_elixir": {:hex, :pbkdf2_elixir, "0.12.3", "6706a148809a29c306062862c803406e88f048277f6e85b68faf73291e820b84", [:mix], [], "hexpm"},
-  "phoenix": {:hex, :phoenix, "1.4.1", "801f9d632808657f1f7c657c8bbe624caaf2ba91429123ebe3801598aea4c3d9", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"},
+  "phoenix": {:hex, :phoenix, "1.4.8", "c72dc3adeb49c70eb963a0ea24f7a064ec1588e651e84e1b7ad5ed8253c0b4a2", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.8.1 or ~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
   "phoenix_ecto": {:hex, :phoenix_ecto, "4.0.0", "c43117a136e7399ea04ecaac73f8f23ee0ffe3e07acfcb8062fe5f4c9f0f6531", [:mix], [{:ecto, "~> 3.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.9", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
   "phoenix_html": {:hex, :phoenix_html, "2.13.1", "fa8f034b5328e2dfa0e4131b5569379003f34bc1fafdaa84985b0b9d2f12e68b", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
-  "phoenix_pubsub": {:hex, :phoenix_pubsub, "1.1.1", "6668d787e602981f24f17a5fbb69cc98f8ab085114ebfac6cc36e10a90c8e93c", [:mix], [], "hexpm"},
+  "phoenix_pubsub": {:hex, :phoenix_pubsub, "1.1.2", "496c303bdf1b2e98a9d26e89af5bba3ab487ba3a3735f74bf1f4064d2a845a3e", [:mix], [], "hexpm"},
   "pleroma_job_queue": {:hex, :pleroma_job_queue, "0.2.0", "879e660aa1cebe8dc6f0aaaa6aa48b4875e89cd961d4a585fd128e0773b31a18", [:mix], [], "hexpm"},
   "plug": {:hex, :plug, "1.8.2", "0bcce1daa420f189a6491f3940cc77ea7fb1919761175c9c3b59800d897440fc", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm"},
-  "plug_cowboy": {:hex, :plug_cowboy, "2.0.1", "d798f8ee5acc86b7d42dbe4450b8b0dadf665ce588236eb0a751a132417a980e", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
+  "plug_cowboy": {:hex, :plug_cowboy, "2.0.2", "6055f16868cc4882b24b6e1d63d2bada94fb4978413377a3b32ac16c18dffba2", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
   "plug_crypto": {:hex, :plug_crypto, "1.0.0", "18e49317d3fa343f24620ed22795ec29d4a5e602d52d1513ccea0b07d8ea7d4d", [:mix], [], "hexpm"},
   "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm"},
   "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm"},

priv/repo/migrations/20190518032627_create_config.exs

@@ -0,0 +1,13 @@
+defmodule Pleroma.Repo.Migrations.CreateConfig do
+  use Ecto.Migration
+
+  def change do
+    create table(:config) do
+      add(:key, :string)
+      add(:value, :binary)
+      timestamps()
+    end
+
+    create(unique_index(:config, :key))
+  end
+end

rel/pleroma_ctl

@@ -1,6 +1,6 @@
 #!/bin/sh
 # XXX: This should be removed when elixir's releases get custom command support
-if [ -z "$1" ] || [ "$1" == "help" ]; then
+if [ -z "$1" ] || [ "$1" = "help" ]; then
   echo "Usage: $(basename "$0") COMMAND [ARGS]
 
     The known commands are:
@@ -15,5 +15,5 @@ if [ -z "$1" ] || [ "$1" == "help" ]; then
 else
   SCRIPT=$(readlink -f "$0")
   SCRIPTPATH=$(dirname "$SCRIPT")
-  $SCRIPTPATH/pleroma eval 'Pleroma.ReleaseTasks.run("'"$*"'")'
+  "$SCRIPTPATH"/pleroma eval 'Pleroma.ReleaseTasks.run("'"$*"'")'
 fi

test/config/transfer_task_test.exs

@@ -0,0 +1,35 @@
+defmodule Pleroma.Config.TransferTaskTest do
+  use Pleroma.DataCase
+
+  setup do
+    dynamic = Pleroma.Config.get([:instance, :dynamic_configuration])
+
+    Pleroma.Config.put([:instance, :dynamic_configuration], true)
+
+    on_exit(fn ->
+      Pleroma.Config.put([:instance, :dynamic_configuration], dynamic)
+    end)
+  end
+
+  test "transfer config values from db to env" do
+    refute Application.get_env(:pleroma, :test_key)
+    Pleroma.Web.AdminAPI.Config.create(%{key: "test_key", value: [live: 2, com: 3]})
+
+    Pleroma.Config.TransferTask.start_link()
+
+    assert Application.get_env(:pleroma, :test_key) == [live: 2, com: 3]
+
+    on_exit(fn ->
+      Application.delete_env(:pleroma, :test_key)
+    end)
+  end
+
+  test "non existing atom" do
+    Pleroma.Web.AdminAPI.Config.create(%{key: "undefined_atom_key", value: [live: 2, com: 3]})
+
+    assert ExUnit.CaptureLog.capture_log(fn ->
+             Pleroma.Config.TransferTask.start_link()
+           end) =~
+             "updating env causes error, key: \"undefined_atom_key\", error: %ArgumentError{message: \"argument error\"}"
+  end
+end

test/fixtures/rich_media/ogp-missing-title.html

@@ -0,0 +1,12 @@
+<html prefix="og: http://ogp.me/ns#">
+
+<head>
+  <title>The Rock (1996)</title>
+  <meta property="og:type" content="video.movie" />
+  <meta property="og:url" content="http://www.imdb.com/title/tt0117500/" />
+  <meta property="og:image" content="http://ia.media-imdb.com/images/rock.jpg" />
+  <meta property="og:description"
+    content="Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.">
+</head>
+
+</html>

test/html_test.exs

@@ -4,8 +4,12 @@
 
 defmodule Pleroma.HTMLTest do
   alias Pleroma.HTML
+  alias Pleroma.Object
+  alias Pleroma.Web.CommonAPI
   use Pleroma.DataCase
 
+  import Pleroma.Factory
+
   @html_sample """
     <b>this is in bold</b>
     <p>this is a paragraph</p>
@@ -160,4 +164,53 @@ test "filters invalid microformats markup" do
                )
     end
   end
+
+  describe "extract_first_external_url" do
+    test "extracts the url" do
+      user = insert(:user)
+
+      {:ok, activity} =
+        CommonAPI.post(user, %{
+          "status" =>
+            "I think I just found the best github repo https://github.com/komeiji-satori/Dress"
+        })
+
+      object = Object.normalize(activity)
+      {:ok, url} = HTML.extract_first_external_url(object, object.data["content"])
+      assert url == "https://github.com/komeiji-satori/Dress"
+    end
+
+    test "skips mentions" do
+      user = insert(:user)
+      other_user = insert(:user)
+
+      {:ok, activity} =
+        CommonAPI.post(user, %{
+          "status" =>
+            "@#{other_user.nickname} install misskey! https://github.com/syuilo/misskey/blob/develop/docs/setup.en.md"
+        })
+
+      object = Object.normalize(activity)
+      {:ok, url} = HTML.extract_first_external_url(object, object.data["content"])
+
+      assert url == "https://github.com/syuilo/misskey/blob/develop/docs/setup.en.md"
+
+      refute url == other_user.ap_id
+    end
+
+    test "skips hashtags" do
+      user = insert(:user)
+
+      {:ok, activity} =
+        CommonAPI.post(user, %{
+          "status" =>
+            "#cofe https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72255140"
+        })
+
+      object = Object.normalize(activity)
+      {:ok, url} = HTML.extract_first_external_url(object, object.data["content"])
+
+      assert url == "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72255140"
+    end
+  end
 end

test/integration/mastodon_websocket_test.exs

@@ -97,5 +97,15 @@ test "receives well formatted events" do
     test "accepts valid tokens", state do
       assert {:ok, _} = start_socket("?stream=user&access_token=#{state.token.token}")
     end
+
+    test "accepts the 'user' stream", %{token: token} = _state do
+      assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}")
+      assert {:error, {403, "Forbidden"}} = start_socket("?stream=user")
+    end
+
+    test "accepts the 'user:notification' stream", %{token: token} = _state do
+      assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}")
+      assert {:error, {403, "Forbidden"}} = start_socket("?stream=user:notification")
+    end
   end
 end

test/notification_test.exs

@@ -8,6 +8,7 @@ defmodule Pleroma.NotificationTest do
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.Streamer
   alias Pleroma.Web.TwitterAPI.TwitterAPI
   import Pleroma.Factory
 
@@ -44,13 +45,42 @@ test "it creates a notification for subscribed users" do
   end
 
   describe "create_notification" do
+    setup do
+      GenServer.start(Streamer, %{}, name: Streamer)
+
+      on_exit(fn ->
+        if pid = Process.whereis(Streamer) do
+          Process.exit(pid, :kill)
+        end
+      end)
+    end
+
+    test "it creates a notification for user and send to the 'user' and the 'user:notification' stream" do
+      user = insert(:user)
+      task = Task.async(fn -> assert_receive {:text, _}, 4_000 end)
+      task_user_notification = Task.async(fn -> assert_receive {:text, _}, 4_000 end)
+      Streamer.add_socket("user", %{transport_pid: task.pid, assigns: %{user: user}})
+
+      Streamer.add_socket(
+        "user:notification",
+        %{transport_pid: task_user_notification.pid, assigns: %{user: user}}
+      )
+
+      activity = insert(:note_activity)
+
+      notify = Notification.create_notification(activity, user)
+      assert notify.user_id == user.id
+      Task.await(task)
+      Task.await(task_user_notification)
+    end
+
     test "it doesn't create a notification for user if the user blocks the activity author" do
       activity = insert(:note_activity)
       author = User.get_cached_by_ap_id(activity.data["actor"])
       user = insert(:user)
       {:ok, user} = User.block(user, author)
 
-      assert nil == Notification.create_notification(activity, user)
+      refute Notification.create_notification(activity, user)
     end
 
     test "it doesn't create a notificatin for the user if the user mutes the activity author" do
@@ -60,7 +90,7 @@ test "it doesn't create a notificatin for the user if the user mutes the activit
       muter = Repo.get(User, muter.id)
       {:ok, activity} = CommonAPI.post(muted, %{"status" => "Hi @#{muter.nickname}"})
 
-      assert nil == Notification.create_notification(activity, muter)
+      refute Notification.create_notification(activity, muter)
     end
 
     test "it doesn't create a notification for an activity from a muted thread" do
@@ -75,7 +105,7 @@ test "it doesn't create a notification for an activity from a muted thread" do
           "in_reply_to_status_id" => activity.id
         })
 
-      assert nil == Notification.create_notification(activity, muter)
+      refute Notification.create_notification(activity, muter)
     end
 
     test "it disables notifications from followers" do
@@ -83,14 +113,14 @@ test "it disables notifications from followers" do
       followed = insert(:user, info: %{notification_settings: %{"followers" => false}})
       User.follow(follower, followed)
       {:ok, activity} = CommonAPI.post(follower, %{"status" => "hey @#{followed.nickname}"})
-      assert nil == Notification.create_notification(activity, followed)
+      refute Notification.create_notification(activity, followed)
     end
 
     test "it disables notifications from non-followers" do
       follower = insert(:user)
       followed = insert(:user, info: %{notification_settings: %{"non_followers" => false}})
       {:ok, activity} = CommonAPI.post(follower, %{"status" => "hey @#{followed.nickname}"})
-      assert nil == Notification.create_notification(activity, followed)
+      refute Notification.create_notification(activity, followed)
     end
 
     test "it disables notifications from people the user follows" do
@@ -99,21 +129,21 @@ test "it disables notifications from people the user follows" do
       User.follow(follower, followed)
       follower = Repo.get(User, follower.id)
       {:ok, activity} = CommonAPI.post(followed, %{"status" => "hey @#{follower.nickname}"})
-      assert nil == Notification.create_notification(activity, follower)
+      refute Notification.create_notification(activity, follower)
     end
 
     test "it disables notifications from people the user does not follow" do
       follower = insert(:user, info: %{notification_settings: %{"non_follows" => false}})
       followed = insert(:user)
       {:ok, activity} = CommonAPI.post(followed, %{"status" => "hey @#{follower.nickname}"})
-      assert nil == Notification.create_notification(activity, follower)
+      refute Notification.create_notification(activity, follower)
     end
 
     test "it doesn't create a notification for user if he is the activity author" do
       activity = insert(:note_activity)
       author = User.get_cached_by_ap_id(activity.data["actor"])
 
-      assert nil == Notification.create_notification(activity, author)
+      refute Notification.create_notification(activity, author)
     end
 
     test "it doesn't create a notification for follow-unfollow-follow chains" do
@@ -123,7 +153,7 @@ test "it doesn't create a notification for follow-unfollow-follow chains" do
       Notification.create_notification(activity, followed_user)
       TwitterAPI.unfollow(user, %{"user_id" => followed_user.id})
       {:ok, _, _, activity_dupe} = TwitterAPI.follow(user, %{"user_id" => followed_user.id})
-      assert nil == Notification.create_notification(activity_dupe, followed_user)
+      refute Notification.create_notification(activity_dupe, followed_user)
     end
 
     test "it doesn't create a notification for like-unlike-like chains" do
@@ -134,7 +164,7 @@ test "it doesn't create a notification for like-unlike-like chains" do
       Notification.create_notification(fav_status, liked_user)
       TwitterAPI.unfav(user, status.id)
       {:ok, dupe} = TwitterAPI.fav(user, status.id)
-      assert nil == Notification.create_notification(dupe, liked_user)
+      refute Notification.create_notification(dupe, liked_user)
     end
 
     test "it doesn't create a notification for repeat-unrepeat-repeat chains" do
@@ -150,7 +180,7 @@ test "it doesn't create a notification for repeat-unrepeat-repeat chains" do
       Notification.create_notification(retweeted_activity, retweeted_user)
       TwitterAPI.unrepeat(user, status.id)
       {:ok, dupe} = TwitterAPI.repeat(user, status.id)
-      assert nil == Notification.create_notification(dupe, retweeted_user)
+      refute Notification.create_notification(dupe, retweeted_user)
     end
 
     test "it doesn't create duplicate notifications for follow+subscribed users" do

test/object/fetcher_test.exs

@@ -7,7 +7,17 @@ defmodule Pleroma.Object.FetcherTest do
   import Tesla.Mock
 
   setup do
-    mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    mock(fn
+      %{method: :get, url: "https://mastodon.example.org/users/userisgone"} ->
+        %Tesla.Env{status: 410}
+
+      %{method: :get, url: "https://mastodon.example.org/users/userisgone404"} ->
+        %Tesla.Env{status: 404}
+
+      env ->
+        apply(HttpRequestMock, :request, [env])
+    end)
+
     :ok
   end
 
@@ -81,10 +91,24 @@ test "it can fetch peertube videos" do
     end
 
     test "all objects with fake directions are rejected by the object fetcher" do
-      {:error, _} =
+      assert {:error, _} =
-        Fetcher.fetch_and_contain_remote_object_from_id(
+               Fetcher.fetch_and_contain_remote_object_from_id(
-          "https://info.pleroma.site/activity4.json"
+                 "https://info.pleroma.site/activity4.json"
-        )
+               )
+    end
+
+    test "handle HTTP 410 Gone response" do
+      assert {:error, "Object has been deleted"} ==
+               Fetcher.fetch_and_contain_remote_object_from_id(
+                 "https://mastodon.example.org/users/userisgone"
+               )
+    end
+
+    test "handle HTTP 404 response" do
+      assert {:error, "Object has been deleted"} ==
+               Fetcher.fetch_and_contain_remote_object_from_id(
+                 "https://mastodon.example.org/users/userisgone404"
+               )
     end
   end
 

test/support/factory.ex

@@ -310,4 +310,17 @@ def registration_factory do
       }
     }
   end
+
+  def config_factory do
+    %Pleroma.Web.AdminAPI.Config{
+      key: sequence(:key, &"some_key_#{&1}"),
+      value:
+        sequence(
+          :value,
+          fn key ->
+            :erlang.term_to_binary(%{another_key: "#{key}somevalue", another: "#{key}somevalue"})
+          end
+        )
+    }
+  end
 end

test/tasks/config_test.exs

@@ -0,0 +1,54 @@
+defmodule Mix.Tasks.Pleroma.ConfigTest do
+  use Pleroma.DataCase
+  alias Pleroma.Repo
+  alias Pleroma.Web.AdminAPI.Config
+
+  setup_all do
+    Mix.shell(Mix.Shell.Process)
+    temp_file = "config/temp.migrated.secret.exs"
+
+    dynamic = Pleroma.Config.get([:instance, :dynamic_configuration])
+
+    Pleroma.Config.put([:instance, :dynamic_configuration], true)
+
+    on_exit(fn ->
+      Mix.shell(Mix.Shell.IO)
+      Application.delete_env(:pleroma, :first_setting)
+      Application.delete_env(:pleroma, :second_setting)
+      Pleroma.Config.put([:instance, :dynamic_configuration], dynamic)
+      :ok = File.rm(temp_file)
+    end)
+
+    {:ok, temp_file: temp_file}
+  end
+
+  test "settings are migrated to db" do
+    assert Repo.all(Config) == []
+
+    Application.put_env(:pleroma, :first_setting, key: "value", key2: [Pleroma.Repo])
+    Application.put_env(:pleroma, :second_setting, key: "value2", key2: [Pleroma.Activity])
+
+    Mix.Tasks.Pleroma.Config.run(["migrate_to_db"])
+
+    first_db = Config.get_by_key("first_setting")
+    second_db = Config.get_by_key("second_setting")
+    refute Config.get_by_key("Pleroma.Repo")
+
+    assert Config.from_binary(first_db.value) == [key: "value", key2: [Pleroma.Repo]]
+    assert Config.from_binary(second_db.value) == [key: "value2", key2: [Pleroma.Activity]]
+  end
+
+  test "settings are migrated to file and deleted from db", %{temp_file: temp_file} do
+    Config.create(%{key: "setting_first", value: [key: "value", key2: [Pleroma.Activity]]})
+    Config.create(%{key: "setting_second", value: [key: "valu2", key2: [Pleroma.Repo]]})
+
+    Mix.Tasks.Pleroma.Config.run(["migrate_from_db", "temp"])
+
+    assert Repo.all(Config) == []
+    assert File.exists?(temp_file)
+    {:ok, file} = File.read(temp_file)
+
+    assert file =~ "config :pleroma, setting_first:"
+    assert file =~ "config :pleroma, setting_second:"
+  end
+end

test/tasks/instance.exs

@@ -36,6 +36,8 @@ test "running gen" do
         "--dbpass",
         "dbpass",
         "--indexable",
+        "y",
+        "--db-configurable",
         "y"
       ])
     end
@@ -53,6 +55,7 @@ test "running gen" do
     assert generated_config =~ "database: \"dbname\""
     assert generated_config =~ "username: \"dbuser\""
     assert generated_config =~ "password: \"dbpass\""
+    assert generated_config =~ "dynamic_configuration: true"
     assert File.read!(tmp_path() <> "setup.psql") == generated_setup_psql()
   end
 

test/user_test.exs

@@ -1011,6 +1011,18 @@ test "User.delete() plugs any possible zombie objects" do
   end
 
   describe "User.search" do
+    test "accepts limit parameter" do
+      Enum.each(0..4, &insert(:user, %{nickname: "john#{&1}"}))
+      assert length(User.search("john", limit: 3)) == 3
+      assert length(User.search("john")) == 5
+    end
+
+    test "accepts offset parameter" do
+      Enum.each(0..4, &insert(:user, %{nickname: "john#{&1}"}))
+      assert length(User.search("john", limit: 3)) == 3
+      assert length(User.search("john", limit: 3, offset: 3)) == 2
+    end
+
     test "finds a user by full or partial nickname" do
       user = insert(:user, %{nickname: "john"})
 
@@ -1077,6 +1089,24 @@ test "finds users, boosting ranks of friends and followers" do
                Enum.map(User.search("doe", resolve: false, for_user: u1), & &1.id) == []
     end
 
+    test "finds followers of user by partial name" do
+      u1 = insert(:user)
+      u2 = insert(:user, %{name: "Jimi"})
+      follower_jimi = insert(:user, %{name: "Jimi Hendrix"})
+      follower_lizz = insert(:user, %{name: "Lizz Wright"})
+      friend = insert(:user, %{name: "Jimi"})
+
+      {:ok, follower_jimi} = User.follow(follower_jimi, u1)
+      {:ok, _follower_lizz} = User.follow(follower_lizz, u2)
+      {:ok, u1} = User.follow(u1, friend)
+
+      assert Enum.map(User.search("jimi", following: true, for_user: u1), & &1.id) == [
+               follower_jimi.id
+             ]
+
+      assert User.search("lizz", following: true, for_user: u1) == []
+    end
+
     test "find local and remote users for authenticated users" do
       u1 = insert(:user, %{name: "lain"})
       u2 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social", local: false})

test/web/admin_api/admin_api_controller_test.exs

@@ -1292,4 +1292,176 @@ test "returns error when status is not exist", %{conn: conn} do
       assert json_response(conn, :bad_request) == "Could not delete"
     end
   end
+
+  describe "GET /api/pleroma/admin/config" do
+    setup %{conn: conn} do
+      admin = insert(:user, info: %{is_admin: true})
+
+      %{conn: assign(conn, :user, admin)}
+    end
+
+    test "without any settings in db", %{conn: conn} do
+      conn = get(conn, "/api/pleroma/admin/config")
+
+      assert json_response(conn, 200) == %{"configs" => []}
+    end
+
+    test "with settings in db", %{conn: conn} do
+      config1 = insert(:config)
+      config2 = insert(:config)
+
+      conn = get(conn, "/api/pleroma/admin/config")
+
+      %{
+        "configs" => [
+          %{
+            "key" => key1,
+            "value" => _
+          },
+          %{
+            "key" => key2,
+            "value" => _
+          }
+        ]
+      } = json_response(conn, 200)
+
+      assert key1 == config1.key
+      assert key2 == config2.key
+    end
+  end
+
+  describe "POST /api/pleroma/admin/config" do
+    setup %{conn: conn} do
+      admin = insert(:user, info: %{is_admin: true})
+
+      temp_file = "config/test.migrated.secret.exs"
+
+      on_exit(fn ->
+        Application.delete_env(:pleroma, :key1)
+        Application.delete_env(:pleroma, :key2)
+        Application.delete_env(:pleroma, :key3)
+        Application.delete_env(:pleroma, :key4)
+        Application.delete_env(:pleroma, :keyaa1)
+        Application.delete_env(:pleroma, :keyaa2)
+        :ok = File.rm(temp_file)
+      end)
+
+      dynamic = Pleroma.Config.get([:instance, :dynamic_configuration])
+
+      Pleroma.Config.put([:instance, :dynamic_configuration], true)
+
+      on_exit(fn ->
+        Pleroma.Config.put([:instance, :dynamic_configuration], dynamic)
+      end)
+
+      %{conn: assign(conn, :user, admin)}
+    end
+
+    test "create new config setting in db", %{conn: conn} do
+      conn =
+        post(conn, "/api/pleroma/admin/config", %{
+          configs: [
+            %{key: "key1", value: "value1"},
+            %{
+              key: "key2",
+              value: %{
+                "nested_1" => "nested_value1",
+                "nested_2" => [
+                  %{"nested_22" => "nested_value222"},
+                  %{"nested_33" => %{"nested_44" => "nested_444"}}
+                ]
+              }
+            },
+            %{
+              key: "key3",
+              value: [
+                %{"nested_3" => ":nested_3", "nested_33" => "nested_33"},
+                %{"nested_4" => ":true"}
+              ]
+            },
+            %{
+              key: "key4",
+              value: %{"nested_5" => ":upload", "endpoint" => "https://example.com"}
+            }
+          ]
+        })
+
+      assert json_response(conn, 200) == %{
+               "configs" => [
+                 %{
+                   "key" => "key1",
+                   "value" => "value1"
+                 },
+                 %{
+                   "key" => "key2",
+                   "value" => [
+                     %{"nested_1" => "nested_value1"},
+                     %{
+                       "nested_2" => [
+                         %{"nested_22" => "nested_value222"},
+                         %{"nested_33" => %{"nested_44" => "nested_444"}}
+                       ]
+                     }
+                   ]
+                 },
+                 %{
+                   "key" => "key3",
+                   "value" => [
+                     [%{"nested_3" => "nested_3"}, %{"nested_33" => "nested_33"}],
+                     %{"nested_4" => true}
+                   ]
+                 },
+                 %{
+                   "key" => "key4",
+                   "value" => [%{"endpoint" => "https://example.com"}, %{"nested_5" => "upload"}]
+                 }
+               ]
+             }
+
+      assert Application.get_env(:pleroma, :key1) == "value1"
+
+      assert Application.get_env(:pleroma, :key2) == [
+               nested_1: "nested_value1",
+               nested_2: [
+                 [nested_22: "nested_value222"],
+                 [nested_33: [nested_44: "nested_444"]]
+               ]
+             ]
+
+      assert Application.get_env(:pleroma, :key3) == [
+               [nested_3: :nested_3, nested_33: "nested_33"],
+               [nested_4: true]
+             ]
+
+      assert Application.get_env(:pleroma, :key4) == [
+               endpoint: "https://example.com",
+               nested_5: :upload
+             ]
+    end
+
+    test "update config setting & delete", %{conn: conn} do
+      config1 = insert(:config, key: "keyaa1")
+      config2 = insert(:config, key: "keyaa2")
+
+      conn =
+        post(conn, "/api/pleroma/admin/config", %{
+          configs: [
+            %{key: config1.key, value: "another_value"},
+            %{key: config2.key, delete: "true"}
+          ]
+        })
+
+      assert json_response(conn, 200) == %{
+               "configs" => [
+                 %{
+                   "key" => config1.key,
+                   "value" => "another_value"
+                 }
+               ]
+             }
+
+      assert Application.get_env(:pleroma, :keyaa1) == "another_value"
+      refute Application.get_env(:pleroma, :keyaa2)
+    end
+  end
 end

test/web/admin_api/config_test.exs

@@ -0,0 +1,183 @@
+defmodule Pleroma.Web.AdminAPI.ConfigTest do
+  use Pleroma.DataCase, async: true
+  import Pleroma.Factory
+  alias Pleroma.Web.AdminAPI.Config
+
+  test "get_by_key/1" do
+    config = insert(:config)
+    insert(:config)
+
+    assert config == Config.get_by_key(config.key)
+  end
+
+  test "create/1" do
+    {:ok, config} = Config.create(%{key: "some_key", value: "some_value"})
+    assert config == Config.get_by_key("some_key")
+  end
+
+  test "update/1" do
+    config = insert(:config)
+    {:ok, updated} = Config.update(config, %{value: "some_value"})
+    loaded = Config.get_by_key(config.key)
+    assert loaded == updated
+  end
+
+  test "update_or_create/1" do
+    config = insert(:config)
+    key2 = "another_key"
+
+    params = [
+      %{key: key2, value: "another_value"},
+      %{key: config.key, value: "new_value"}
+    ]
+
+    assert Repo.all(Config) |> length() == 1
+
+    Enum.each(params, &Config.update_or_create(&1))
+
+    assert Repo.all(Config) |> length() == 2
+
+    config1 = Config.get_by_key(config.key)
+    config2 = Config.get_by_key(key2)
+
+    assert config1.value == Config.transform("new_value")
+    assert config2.value == Config.transform("another_value")
+  end
+
+  test "delete/1" do
+    config = insert(:config)
+    {:ok, _} = Config.delete(config.key)
+    refute Config.get_by_key(config.key)
+  end
+
+  describe "transform/1" do
+    test "string" do
+      binary = Config.transform("value as string")
+      assert binary == :erlang.term_to_binary("value as string")
+      assert Config.from_binary(binary) == "value as string"
+    end
+
+    test "list of modules" do
+      binary = Config.transform(["Pleroma.Repo", "Pleroma.Activity"])
+      assert binary == :erlang.term_to_binary([Pleroma.Repo, Pleroma.Activity])
+      assert Config.from_binary(binary) == [Pleroma.Repo, Pleroma.Activity]
+    end
+
+    test "list of strings" do
+      binary = Config.transform(["string1", "string2"])
+      assert binary == :erlang.term_to_binary(["string1", "string2"])
+      assert Config.from_binary(binary) == ["string1", "string2"]
+    end
+
+    test "map" do
+      binary =
+        Config.transform(%{
+          "types" => "Pleroma.PostgresTypes",
+          "telemetry_event" => ["Pleroma.Repo.Instrumenter"],
+          "migration_lock" => ""
+        })
+
+      assert binary ==
+               :erlang.term_to_binary(
+                 telemetry_event: [Pleroma.Repo.Instrumenter],
+                 types: Pleroma.PostgresTypes
+               )
+
+      assert Config.from_binary(binary) == [
+               telemetry_event: [Pleroma.Repo.Instrumenter],
+               types: Pleroma.PostgresTypes
+             ]
+    end
+
+    test "complex map with nested integers, lists and atoms" do
+      binary =
+        Config.transform(%{
+          "uploader" => "Pleroma.Uploaders.Local",
+          "filters" => ["Pleroma.Upload.Filter.Dedupe"],
+          "link_name" => ":true",
+          "proxy_remote" => ":false",
+          "proxy_opts" => %{
+            "redirect_on_failure" => ":false",
+            "max_body_length" => "i:1048576",
+            "http" => %{
+              "follow_redirect" => ":true",
+              "pool" => ":upload"
+            }
+          }
+        })
+
+      assert binary ==
+               :erlang.term_to_binary(
+                 filters: [Pleroma.Upload.Filter.Dedupe],
+                 link_name: true,
+                 proxy_opts: [
+                   http: [
+                     follow_redirect: true,
+                     pool: :upload
+                   ],
+                   max_body_length: 1_048_576,
+                   redirect_on_failure: false
+                 ],
+                 proxy_remote: false,
+                 uploader: Pleroma.Uploaders.Local
+               )
+
+      assert Config.from_binary(binary) ==
+               [
+                 filters: [Pleroma.Upload.Filter.Dedupe],
+                 link_name: true,
+                 proxy_opts: [
+                   http: [
+                     follow_redirect: true,
+                     pool: :upload
+                   ],
+                   max_body_length: 1_048_576,
+                   redirect_on_failure: false
+                 ],
+                 proxy_remote: false,
+                 uploader: Pleroma.Uploaders.Local
+               ]
+    end
+
+    test "keyword" do
+      binary =
+        Config.transform(%{
+          "level" => ":warn",
+          "meta" => [":all"],
+          "webhook_url" => "https://hooks.slack.com/services/YOUR-KEY-HERE"
+        })
+
+      assert binary ==
+               :erlang.term_to_binary(
+                 level: :warn,
+                 meta: [:all],
+                 webhook_url: "https://hooks.slack.com/services/YOUR-KEY-HERE"
+               )
+
+      assert Config.from_binary(binary) == [
+               level: :warn,
+               meta: [:all],
+               webhook_url: "https://hooks.slack.com/services/YOUR-KEY-HERE"
+             ]
+    end
+
+    test "complex map with sigil" do
+      binary =
+        Config.transform(%{
+          federated_timeline_removal: [],
+          reject: [~r/comp[lL][aA][iI][nN]er/],
+          replace: []
+        })
+
+      assert binary ==
+               :erlang.term_to_binary(
+                 federated_timeline_removal: [],
+                 reject: [~r/comp[lL][aA][iI][nN]er/],
+                 replace: []
+               )
+
+      assert Config.from_binary(binary) ==
+               [federated_timeline_removal: [], reject: [~r/comp[lL][aA][iI][nN]er/], replace: []]
+    end
+  end
+end

test/web/mastodon_api/account_view_test.exs

@@ -19,9 +19,18 @@ test "Represent a user account" do
       ]
     }
 
+    background_image = %{
+      "url" => [%{"href" => "https://example.com/images/asuka_hospital.png"}]
+    }
+
     user =
       insert(:user, %{
-        info: %{note_count: 5, follower_count: 3, source_data: source_data},
+        info: %{
+          note_count: 5,
+          follower_count: 3,
+          source_data: source_data,
+          background: background_image
+        },
         nickname: "shp@shitposter.club",
         name: ":karjalanpiirakka: shp",
         bio: "<script src=\"invalid-html\"></script><span>valid html</span>",
@@ -60,6 +69,7 @@ test "Represent a user account" do
         pleroma: %{}
       },
       pleroma: %{
+        background_image: "https://example.com/images/asuka_hospital.png",
         confirmation_pending: false,
         tags: [],
         is_admin: false,
@@ -126,6 +136,7 @@ test "Represent a Service(bot) account" do
         pleroma: %{}
       },
       pleroma: %{
+        background_image: nil,
         confirmation_pending: false,
         tags: [],
         is_admin: false,
@@ -216,6 +227,7 @@ test "represent an embedded relationship" do
         pleroma: %{}
       },
       pleroma: %{
+        background_image: nil,
         confirmation_pending: false,
         tags: [],
         is_admin: false,

test/web/mastodon_api/mastodon_api_controller/update_credentials_test.exs

@@ -0,0 +1,304 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do
+  alias Pleroma.Repo
+  alias Pleroma.User
+
+  use Pleroma.Web.ConnCase
+
+  import Pleroma.Factory
+
+  describe "updating credentials" do
+    test "sets user settings in a generic way", %{conn: conn} do
+      user = insert(:user)
+
+      res_conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{
+          "pleroma_settings_store" => %{
+            pleroma_fe: %{
+              theme: "bla"
+            }
+          }
+        })
+
+      assert user = json_response(res_conn, 200)
+      assert user["pleroma"]["settings_store"] == %{"pleroma_fe" => %{"theme" => "bla"}}
+
+      user = Repo.get(User, user["id"])
+
+      res_conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{
+          "pleroma_settings_store" => %{
+            masto_fe: %{
+              theme: "bla"
+            }
+          }
+        })
+
+      assert user = json_response(res_conn, 200)
+
+      assert user["pleroma"]["settings_store"] ==
+               %{
+                 "pleroma_fe" => %{"theme" => "bla"},
+                 "masto_fe" => %{"theme" => "bla"}
+               }
+
+      user = Repo.get(User, user["id"])
+
+      res_conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{
+          "pleroma_settings_store" => %{
+            masto_fe: %{
+              theme: "blub"
+            }
+          }
+        })
+
+      assert user = json_response(res_conn, 200)
+
+      assert user["pleroma"]["settings_store"] ==
+               %{
+                 "pleroma_fe" => %{"theme" => "bla"},
+                 "masto_fe" => %{"theme" => "blub"}
+               }
+    end
+
+    test "updates the user's bio", %{conn: conn} do
+      user = insert(:user)
+      user2 = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{
+          "note" => "I drink #cofe with @#{user2.nickname}"
+        })
+
+      assert user = json_response(conn, 200)
+
+      assert user["note"] ==
+               ~s(I drink <a class="hashtag" data-tag="cofe" href="http://localhost:4001/tag/cofe" rel="tag">#cofe</a> with <span class="h-card"><a data-user=") <>
+                 user2.id <>
+                 ~s(" class="u-url mention" href=") <>
+                 user2.ap_id <> ~s(">@<span>) <> user2.nickname <> ~s(</span></a></span>)
+    end
+
+    test "updates the user's locking status", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{locked: "true"})
+
+      assert user = json_response(conn, 200)
+      assert user["locked"] == true
+    end
+
+    test "updates the user's default scope", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{default_scope: "cofe"})
+
+      assert user = json_response(conn, 200)
+      assert user["source"]["privacy"] == "cofe"
+    end
+
+    test "updates the user's hide_followers status", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{hide_followers: "true"})
+
+      assert user = json_response(conn, 200)
+      assert user["pleroma"]["hide_followers"] == true
+    end
+
+    test "updates the user's skip_thread_containment option", %{conn: conn} do
+      user = insert(:user)
+
+      response =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{skip_thread_containment: "true"})
+        |> json_response(200)
+
+      assert response["pleroma"]["skip_thread_containment"] == true
+      assert refresh_record(user).info.skip_thread_containment
+    end
+
+    test "updates the user's hide_follows status", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{hide_follows: "true"})
+
+      assert user = json_response(conn, 200)
+      assert user["pleroma"]["hide_follows"] == true
+    end
+
+    test "updates the user's hide_favorites status", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{hide_favorites: "true"})
+
+      assert user = json_response(conn, 200)
+      assert user["pleroma"]["hide_favorites"] == true
+    end
+
+    test "updates the user's show_role status", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{show_role: "false"})
+
+      assert user = json_response(conn, 200)
+      assert user["source"]["pleroma"]["show_role"] == false
+    end
+
+    test "updates the user's no_rich_text status", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{no_rich_text: "true"})
+
+      assert user = json_response(conn, 200)
+      assert user["source"]["pleroma"]["no_rich_text"] == true
+    end
+
+    test "updates the user's name", %{conn: conn} do
+      user = insert(:user)
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{"display_name" => "markorepairs"})
+
+      assert user = json_response(conn, 200)
+      assert user["display_name"] == "markorepairs"
+    end
+
+    test "updates the user's avatar", %{conn: conn} do
+      user = insert(:user)
+
+      new_avatar = %Plug.Upload{
+        content_type: "image/jpg",
+        path: Path.absname("test/fixtures/image.jpg"),
+        filename: "an_image.jpg"
+      }
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{"avatar" => new_avatar})
+
+      assert user_response = json_response(conn, 200)
+      assert user_response["avatar"] != User.avatar_url(user)
+    end
+
+    test "updates the user's banner", %{conn: conn} do
+      user = insert(:user)
+
+      new_header = %Plug.Upload{
+        content_type: "image/jpg",
+        path: Path.absname("test/fixtures/image.jpg"),
+        filename: "an_image.jpg"
+      }
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{"header" => new_header})
+
+      assert user_response = json_response(conn, 200)
+      assert user_response["header"] != User.banner_url(user)
+    end
+
+    test "updates the user's background", %{conn: conn} do
+      user = insert(:user)
+
+      new_header = %Plug.Upload{
+        content_type: "image/jpg",
+        path: Path.absname("test/fixtures/image.jpg"),
+        filename: "an_image.jpg"
+      }
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{
+          "pleroma_background_image" => new_header
+        })
+
+      assert user_response = json_response(conn, 200)
+      assert user_response["pleroma"]["background_image"]
+    end
+
+    test "requires 'write' permission", %{conn: conn} do
+      token1 = insert(:oauth_token, scopes: ["read"])
+      token2 = insert(:oauth_token, scopes: ["write", "follow"])
+
+      for token <- [token1, token2] do
+        conn =
+          conn
+          |> put_req_header("authorization", "Bearer #{token.token}")
+          |> patch("/api/v1/accounts/update_credentials", %{})
+
+        if token == token1 do
+          assert %{"error" => "Insufficient permissions: write."} == json_response(conn, 403)
+        else
+          assert json_response(conn, 200)
+        end
+      end
+    end
+
+    test "updates profile emojos", %{conn: conn} do
+      user = insert(:user)
+
+      note = "*sips :blank:*"
+      name = "I am :firefox:"
+
+      conn =
+        conn
+        |> assign(:user, user)
+        |> patch("/api/v1/accounts/update_credentials", %{
+          "note" => note,
+          "display_name" => name
+        })
+
+      assert json_response(conn, 200)
+
+      conn =
+        conn
+        |> get("/api/v1/accounts/#{user.id}")
+
+      assert user = json_response(conn, 200)
+
+      assert user["note"] == note
+      assert user["display_name"] == name
+      assert [%{"shortcode" => "blank"}, %{"shortcode" => "firefox"}] = user["emojis"]
+    end
+  end
+end

test/web/mastodon_api/mastodon_api_controller_test.exs

@@ -542,7 +542,10 @@ test "verify_credentials", %{conn: conn} do
       |> assign(:user, user)
       |> get("/api/v1/accounts/verify_credentials")
 
-    assert %{"id" => id, "source" => %{"privacy" => "public"}} = json_response(conn, 200)
+    response = json_response(conn, 200)
+
+    assert %{"id" => id, "source" => %{"privacy" => "public"}} = response
+    assert response["pleroma"]["chat_token"]
     assert id == to_string(user.id)
   end
 
@@ -2134,116 +2137,6 @@ test "unimplemented follow_requests, blocks, domain blocks" do
     end)
   end
 
-  test "account search", %{conn: conn} do
-    user = insert(:user)
-    user_two = insert(:user, %{nickname: "shp@shitposter.club"})
-    user_three = insert(:user, %{nickname: "shp@heldscal.la", name: "I love 2hu"})
-
-    results =
-      conn
-      |> assign(:user, user)
-      |> get("/api/v1/accounts/search", %{"q" => "shp"})
-      |> json_response(200)
-
-    result_ids = for result <- results, do: result["acct"]
-
-    assert user_two.nickname in result_ids
-    assert user_three.nickname in result_ids
-
-    results =
-      conn
-      |> assign(:user, user)
-      |> get("/api/v1/accounts/search", %{"q" => "2hu"})
-      |> json_response(200)
-
-    result_ids = for result <- results, do: result["acct"]
-
-    assert user_three.nickname in result_ids
-  end
-
-  test "search", %{conn: conn} do
-    user = insert(:user)
-    user_two = insert(:user, %{nickname: "shp@shitposter.club"})
-    user_three = insert(:user, %{nickname: "shp@heldscal.la", name: "I love 2hu"})
-
-    {:ok, activity} = CommonAPI.post(user, %{"status" => "This is about 2hu"})
-
-    {:ok, _activity} =
-      CommonAPI.post(user, %{
-        "status" => "This is about 2hu, but private",
-        "visibility" => "private"
-      })
-
-    {:ok, _} = CommonAPI.post(user_two, %{"status" => "This isn't"})
-
-    conn =
-      conn
-      |> get("/api/v1/search", %{"q" => "2hu"})
-
-    assert results = json_response(conn, 200)
-
-    [account | _] = results["accounts"]
-    assert account["id"] == to_string(user_three.id)
-
-    assert results["hashtags"] == []
-
-    [status] = results["statuses"]
-    assert status["id"] == to_string(activity.id)
-  end
-
-  test "search fetches remote statuses", %{conn: conn} do
-    capture_log(fn ->
-      conn =
-        conn
-        |> get("/api/v1/search", %{"q" => "https://shitposter.club/notice/2827873"})
-
-      assert results = json_response(conn, 200)
-
-      [status] = results["statuses"]
-      assert status["uri"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment"
-    end)
-  end
-
-  test "search doesn't show statuses that it shouldn't", %{conn: conn} do
-    {:ok, activity} =
-      CommonAPI.post(insert(:user), %{
-        "status" => "This is about 2hu, but private",
-        "visibility" => "private"
-      })
-
-    capture_log(fn ->
-      conn =
-        conn
-        |> get("/api/v1/search", %{"q" => Object.normalize(activity).data["id"]})
-
-      assert results = json_response(conn, 200)
-
-      [] = results["statuses"]
-    end)
-  end
-
-  test "search fetches remote accounts", %{conn: conn} do
-    user = insert(:user)
-
-    conn =
-      conn
-      |> assign(:user, user)
-      |> get("/api/v1/search", %{"q" => "shp@social.heldscal.la", "resolve" => "true"})
-
-    assert results = json_response(conn, 200)
-    [account] = results["accounts"]
-    assert account["acct"] == "shp@social.heldscal.la"
-  end
-
-  test "search doesn't fetch remote accounts if resolve is false", %{conn: conn} do
-    conn =
-      conn
-      |> get("/api/v1/search", %{"q" => "shp@social.heldscal.la", "resolve" => "false"})
-
-    assert results = json_response(conn, 200)
-    assert [] == results["accounts"]
-  end
-
   test "returns the favorites of a user", %{conn: conn} do
     user = insert(:user)
     other_user = insert(:user)
@@ -2484,278 +2377,6 @@ test "hides favorites for new users by default", %{conn: conn, current_user: cur
     end
   end
 
-  describe "updating credentials" do
-    test "sets user settings in a generic way", %{conn: conn} do
-      user = insert(:user)
-
-      res_conn =
-        conn
-        |> assign(:user, user)
-        |> patch("/api/v1/accounts/update_credentials", %{
-          "pleroma_settings_store" => %{
-            pleroma_fe: %{
-              theme: "bla"
-            }
-          }
-        })
-
-      assert user = json_response(res_conn, 200)
-      assert user["pleroma"]["settings_store"] == %{"pleroma_fe" => %{"theme" => "bla"}}
-
-      user = Repo.get(User, user["id"])
-
-      res_conn =
-        conn
-        |> assign(:user, user)
-        |> patch("/api/v1/accounts/update_credentials", %{
-          "pleroma_settings_store" => %{
-            masto_fe: %{
-              theme: "bla"
-            }
-          }
-        })
-
-      assert user = json_response(res_conn, 200)
-
-      assert user["pleroma"]["settings_store"] ==
-               %{
-                 "pleroma_fe" => %{"theme" => "bla"},
-                 "masto_fe" => %{"theme" => "bla"}
-               }
-
-      user = Repo.get(User, user["id"])
-
-      res_conn =
-        conn
-        |> assign(:user, user)
-        |> patch("/api/v1/accounts/update_credentials", %{
-          "pleroma_settings_store" => %{
-            masto_fe: %{
-              theme: "blub"
-            }
-          }
-        })
-
-      assert user = json_response(res_conn, 200)
-
-      assert user["pleroma"]["settings_store"] ==
-               %{
-                 "pleroma_fe" => %{"theme" => "bla"},
-                 "masto_fe" => %{"theme" => "blub"}
-               }
-    end
-
-    test "updates the user's bio", %{conn: conn} do
-      user = insert(:user)
-      user2 = insert(:user)
-
-      conn =
-        conn
-        |> assign(:user, user)
-        |> patch("/api/v1/accounts/update_credentials", %{
-          "note" => "I drink #cofe with @#{user2.nickname}"
-        })
-
-      assert user = json_response(conn, 200)
-
-      assert user["note"] ==
-               ~s(I drink <a class="hashtag" data-tag="cofe" href="http://localhost:4001/tag/cofe" rel="tag">#cofe</a> with <span class="h-card"><a data-user=") <>
-                 user2.id <>
-                 ~s(" class="u-url mention" href=") <>
-                 user2.ap_id <> ~s(">@<span>) <> user2.nickname <> ~s(</span></a></span>)
-    end
-
-    test "updates the user's locking status", %{conn: conn} do
-      user = insert(:user)
-
-      conn =
-        conn
-        |> assign(:user, user)
-        |> patch("/api/v1/accounts/update_credentials", %{locked: "true"})
-
-      assert user = json_response(conn, 200)
-      assert user["locked"] == true
-    end
-
-    test "updates the user's default scope", %{conn: conn} do
-      user = insert(:user)
-
-      conn =
-        conn
-        |> assign(:user, user)
-        |> patch("/api/v1/accounts/update_credentials", %{default_scope: "cofe"})
-
-      assert user = json_response(conn, 200)
-      assert user["source"]["privacy"] == "cofe"
-    end
-
-    test "updates the user's hide_followers status", %{conn: conn} do
-      user = insert(:user)
-
-      conn =
-        conn
-        |> assign(:user, user)
-        |> patch("/api/v1/accounts/update_credentials", %{hide_followers: "true"})
-
-      assert user = json_response(conn, 200)
-      assert user["pleroma"]["hide_followers"] == true
-    end
-
-    test "updates the user's skip_thread_containment option", %{conn: conn} do
-      user = insert(:user)
-
-      response =
-        conn
-        |> assign(:user, user)
-        |> patch("/api/v1/accounts/update_credentials", %{skip_thread_containment: "true"})
-        |> json_response(200)
-
-      assert response["pleroma"]["skip_thread_containment"] == true
-      assert refresh_record(user).info.skip_thread_containment
-    end
-
-    test "updates the user's hide_follows status", %{conn: conn} do
-      user = insert(:user)
-
-      conn =
-        conn
-        |> assign(:user, user)
-        |> patch("/api/v1/accounts/update_credentials", %{hide_follows: "true"})
-
-      assert user = json_response(conn, 200)
-      assert user["pleroma"]["hide_follows"] == true
-    end
-
-    test "updates the user's hide_favorites status", %{conn: conn} do
-      user = insert(:user)
-
-      conn =
-        conn
-        |> assign(:user, user)
-        |> patch("/api/v1/accounts/update_credentials", %{hide_favorites: "true"})
-
-      assert user = json_response(conn, 200)
-      assert user["pleroma"]["hide_favorites"] == true
-    end
-
-    test "updates the user's show_role status", %{conn: conn} do
-      user = insert(:user)
-
-      conn =
-        conn
-        |> assign(:user, user)
-        |> patch("/api/v1/accounts/update_credentials", %{show_role: "false"})
-
-      assert user = json_response(conn, 200)
-      assert user["source"]["pleroma"]["show_role"] == false
-    end
-
-    test "updates the user's no_rich_text status", %{conn: conn} do
-      user = insert(:user)
-
-      conn =
-        conn
-        |> assign(:user, user)
-        |> patch("/api/v1/accounts/update_credentials", %{no_rich_text: "true"})
-
-      assert user = json_response(conn, 200)
-      assert user["source"]["pleroma"]["no_rich_text"] == true
-    end
-
-    test "updates the user's name", %{conn: conn} do
-      user = insert(:user)
-
-      conn =
-        conn
-        |> assign(:user, user)
-        |> patch("/api/v1/accounts/update_credentials", %{"display_name" => "markorepairs"})
-
-      assert user = json_response(conn, 200)
-      assert user["display_name"] == "markorepairs"
-    end
-
-    test "updates the user's avatar", %{conn: conn} do
-      user = insert(:user)
-
-      new_avatar = %Plug.Upload{
-        content_type: "image/jpg",
-        path: Path.absname("test/fixtures/image.jpg"),
-        filename: "an_image.jpg"
-      }
-
-      conn =
-        conn
-        |> assign(:user, user)
-        |> patch("/api/v1/accounts/update_credentials", %{"avatar" => new_avatar})
-
-      assert user_response = json_response(conn, 200)
-      assert user_response["avatar"] != User.avatar_url(user)
-    end
-
-    test "updates the user's banner", %{conn: conn} do
-      user = insert(:user)
-
-      new_header = %Plug.Upload{
-        content_type: "image/jpg",
-        path: Path.absname("test/fixtures/image.jpg"),
-        filename: "an_image.jpg"
-      }
-
-      conn =
-        conn
-        |> assign(:user, user)
-        |> patch("/api/v1/accounts/update_credentials", %{"header" => new_header})
-
-      assert user_response = json_response(conn, 200)
-      assert user_response["header"] != User.banner_url(user)
-    end
-
-    test "requires 'write' permission", %{conn: conn} do
-      token1 = insert(:oauth_token, scopes: ["read"])
-      token2 = insert(:oauth_token, scopes: ["write", "follow"])
-
-      for token <- [token1, token2] do
-        conn =
-          conn
-          |> put_req_header("authorization", "Bearer #{token.token}")
-          |> patch("/api/v1/accounts/update_credentials", %{})
-
-        if token == token1 do
-          assert %{"error" => "Insufficient permissions: write."} == json_response(conn, 403)
-        else
-          assert json_response(conn, 200)
-        end
-      end
-    end
-
-    test "updates profile emojos", %{conn: conn} do
-      user = insert(:user)
-
-      note = "*sips :blank:*"
-      name = "I am :firefox:"
-
-      conn =
-        conn
-        |> assign(:user, user)
-        |> patch("/api/v1/accounts/update_credentials", %{
-          "note" => note,
-          "display_name" => name
-        })
-
-      assert json_response(conn, 200)
-
-      conn =
-        conn
-        |> get("/api/v1/accounts/#{user.id}")
-
-      assert user = json_response(conn, 200)
-
-      assert user["note"] == note
-      assert user["display_name"] == name
-      assert [%{"shortcode" => "blank"}, %{"shortcode" => "firefox"}] = user["emojis"]
-    end
-  end
-
   test "get instance information", %{conn: conn} do
     conn = get(conn, "/api/v1/instance")
     assert result = json_response(conn, 200)

test/web/mastodon_api/search_controller_test.exs

@@ -0,0 +1,128 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.MastodonAPI.SearchControllerTest do
+  use Pleroma.Web.ConnCase
+
+  alias Pleroma.Object
+  alias Pleroma.Web.CommonAPI
+  import Pleroma.Factory
+  import ExUnit.CaptureLog
+  import Tesla.Mock
+
+  setup do
+    mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
+    :ok
+  end
+
+  test "account search", %{conn: conn} do
+    user = insert(:user)
+    user_two = insert(:user, %{nickname: "shp@shitposter.club"})
+    user_three = insert(:user, %{nickname: "shp@heldscal.la", name: "I love 2hu"})
+
+    results =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/accounts/search", %{"q" => "shp"})
+      |> json_response(200)
+
+    result_ids = for result <- results, do: result["acct"]
+
+    assert user_two.nickname in result_ids
+    assert user_three.nickname in result_ids
+
+    results =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/accounts/search", %{"q" => "2hu"})
+      |> json_response(200)
+
+    result_ids = for result <- results, do: result["acct"]
+
+    assert user_three.nickname in result_ids
+  end
+
+  test "search", %{conn: conn} do
+    user = insert(:user)
+    user_two = insert(:user, %{nickname: "shp@shitposter.club"})
+    user_three = insert(:user, %{nickname: "shp@heldscal.la", name: "I love 2hu"})
+
+    {:ok, activity} = CommonAPI.post(user, %{"status" => "This is about 2hu"})
+
+    {:ok, _activity} =
+      CommonAPI.post(user, %{
+        "status" => "This is about 2hu, but private",
+        "visibility" => "private"
+      })
+
+    {:ok, _} = CommonAPI.post(user_two, %{"status" => "This isn't"})
+
+    conn =
+      conn
+      |> get("/api/v1/search", %{"q" => "2hu"})
+
+    assert results = json_response(conn, 200)
+
+    [account | _] = results["accounts"]
+    assert account["id"] == to_string(user_three.id)
+
+    assert results["hashtags"] == []
+
+    [status] = results["statuses"]
+    assert status["id"] == to_string(activity.id)
+  end
+
+  test "search fetches remote statuses", %{conn: conn} do
+    capture_log(fn ->
+      conn =
+        conn
+        |> get("/api/v1/search", %{"q" => "https://shitposter.club/notice/2827873"})
+
+      assert results = json_response(conn, 200)
+
+      [status] = results["statuses"]
+      assert status["uri"] == "tag:shitposter.club,2017-05-05:noticeId=2827873:objectType=comment"
+    end)
+  end
+
+  test "search doesn't show statuses that it shouldn't", %{conn: conn} do
+    {:ok, activity} =
+      CommonAPI.post(insert(:user), %{
+        "status" => "This is about 2hu, but private",
+        "visibility" => "private"
+      })
+
+    capture_log(fn ->
+      conn =
+        conn
+        |> get("/api/v1/search", %{"q" => Object.normalize(activity).data["id"]})
+
+      assert results = json_response(conn, 200)
+
+      [] = results["statuses"]
+    end)
+  end
+
+  test "search fetches remote accounts", %{conn: conn} do
+    user = insert(:user)
+
+    conn =
+      conn
+      |> assign(:user, user)
+      |> get("/api/v1/search", %{"q" => "shp@social.heldscal.la", "resolve" => "true"})
+
+    assert results = json_response(conn, 200)
+    [account] = results["accounts"]
+    assert account["acct"] == "shp@social.heldscal.la"
+  end
+
+  test "search doesn't fetch remote accounts if resolve is false", %{conn: conn} do
+    conn =
+      conn
+      |> get("/api/v1/search", %{"q" => "shp@social.heldscal.la", "resolve" => "false"})
+
+    assert results = json_response(conn, 200)
+    assert [] == results["accounts"]
+  end
+end

test/web/oauth/oauth_controller_test.exs

@@ -408,7 +408,11 @@ test "renders authentication page if user is already authenticated but `force_lo
       assert html_response(conn, 200) =~ ~s(type="submit")
     end
 
-    test "redirects to app if user is already authenticated", %{app: app, conn: conn} do
+    test "with existing authentication and non-OOB `redirect_uri`, redirects to app with `token` and `state` params",
+         %{
+           app: app,
+           conn: conn
+         } do
       token = insert(:oauth_token, app_id: app.id)
 
       conn =
@@ -420,11 +424,36 @@ test "redirects to app if user is already authenticated", %{app: app, conn: conn
             "response_type" => "code",
             "client_id" => app.client_id,
             "redirect_uri" => app.redirect_uris,
+            "state" => "specific_client_state",
             "scope" => "read"
           }
         )
 
-      assert redirected_to(conn) == "https://redirect.url"
+      assert URI.decode(redirected_to(conn)) ==
+               "https://redirect.url?access_token=#{token.token}&state=specific_client_state"
+    end
+
+    test "with existing authentication and OOB `redirect_uri`, redirects to app with `token` and `state` params",
+         %{
+           app: app,
+           conn: conn
+         } do
+      token = insert(:oauth_token, app_id: app.id)
+
+      conn =
+        conn
+        |> put_session(:oauth_token, token.token)
+        |> get(
+          "/oauth/authorize",
+          %{
+            "response_type" => "code",
+            "client_id" => app.client_id,
+            "redirect_uri" => "urn:ietf:wg:oauth:2.0:oob",
+            "scope" => "read"
+          }
+        )
+
+      assert html_response(conn, 200) =~ "Authorization exists"
     end
   end
 

test/web/rich_media/parser_test.exs

@@ -9,6 +9,15 @@ defmodule Pleroma.Web.RichMedia.ParserTest do
       } ->
         %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}
 
+      %{
+        method: :get,
+        url: "http://example.com/ogp-missing-title"
+      } ->
+        %Tesla.Env{
+          status: 200,
+          body: File.read!("test/fixtures/rich_media/ogp-missing-title.html")
+        }
+
       %{
         method: :get,
         url: "http://example.com/twitter-card"
@@ -51,6 +60,19 @@ test "parses ogp" do
               }}
   end
 
+  test "falls back to <title> when ogp:title is missing" do
+    assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/ogp-missing-title") ==
+             {:ok,
+              %{
+                image: "http://ia.media-imdb.com/images/rock.jpg",
+                title: "The Rock (1996)",
+                description:
+                  "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.",
+                type: "video.movie",
+                url: "http://www.imdb.com/title/tt0117500/"
+              }}
+  end
+
   test "parses twitter card" do
     assert Pleroma.Web.RichMedia.Parser.parse("http://example.com/twitter-card") ==
              {:ok,

test/web/streamer_test.exs

@@ -21,6 +21,52 @@ defmodule Pleroma.Web.StreamerTest do
     :ok
   end
 
+  describe "user streams" do
+    setup do
+      GenServer.start(Streamer, %{}, name: Streamer)
+
+      on_exit(fn ->
+        if pid = Process.whereis(Streamer) do
+          Process.exit(pid, :kill)
+        end
+      end)
+
+      user = insert(:user)
+      notify = insert(:notification, user: user, activity: build(:note_activity))
+      {:ok, %{user: user, notify: notify}}
+    end
+
+    test "it sends notify to in the 'user' stream", %{user: user, notify: notify} do
+      task =
+        Task.async(fn ->
+          assert_receive {:text, _}, 4_000
+        end)
+
+      Streamer.add_socket(
+        "user",
+        %{transport_pid: task.pid, assigns: %{user: user}}
+      )
+
+      Streamer.stream("user", notify)
+      Task.await(task)
+    end
+
+    test "it sends notify to in the 'user:notification' stream", %{user: user, notify: notify} do
+      task =
+        Task.async(fn ->
+          assert_receive {:text, _}, 4_000
+        end)
+
+      Streamer.add_socket(
+        "user:notification",
+        %{transport_pid: task.pid, assigns: %{user: user}}
+      )
+
+      Streamer.stream("user:notification", notify)
+      Task.await(task)
+    end
+  end
+
   test "it sends to public" do
     user = insert(:user)
     other_user = insert(:user)