From 37a1001b97be5b103dab9dc0f62d73487e8d5450 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Sun, 14 Aug 2022 23:13:49 +0000 Subject: [PATCH 01/44] add finch outbound proxy support (#158) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/158 --- CHANGELOG.md | 5 ++ config/description.exs | 7 +- docs/docs/configuration/cheatsheet.md | 2 +- lib/pleroma/application.ex | 4 + lib/pleroma/http/adapter_helper.ex | 76 ++++++++++++----- lib/pleroma/http/adapter_helper/default.ex | 2 +- lib/pleroma/http/adapter_helper/gun.ex | 82 ------------------- lib/pleroma/http/adapter_helper/hackney.ex | 40 --------- test/pleroma/http/adapter_helper/gun_test.exs | 77 ----------------- .../http/adapter_helper/hackney_test.exs | 35 -------- test/pleroma/http/adapter_helper_test.exs | 30 ++++++- 11 files changed, 98 insertions(+), 262 deletions(-) delete mode 100644 lib/pleroma/http/adapter_helper/gun.ex delete mode 100644 lib/pleroma/http/adapter_helper/hackney.ex delete mode 100644 test/pleroma/http/adapter_helper/gun_test.exs delete mode 100644 test/pleroma/http/adapter_helper/hackney_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index c658af460..7c7cd8601 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Removed +- Non-finch HTTP adapters. `:tesla, :adapter` is now highly recommended to be set to the default. + +## 2022.08 + ### Added - extended runtime module support, see config cheatsheet - quote posting; quotes are limited to public posts diff --git a/config/description.exs b/config/description.exs index a44ab2432..9f93265d1 100644 --- a/config/description.exs +++ b/config/description.exs @@ -2598,9 +2598,10 @@ %{ key: :proxy_url, label: "Proxy URL", - type: [:string, :tuple], - description: "Proxy URL", - suggestions: ["localhost:9020", {:socks5, :localhost, 3090}] + type: :string, + description: + "Proxy URL - of the format http://host:port. Advise setting in .exs instead of admin-fe due to this being set at boot-time.", + suggestions: ["http://localhost:3128"] }, %{ key: :user_agent, diff --git a/docs/docs/configuration/cheatsheet.md b/docs/docs/configuration/cheatsheet.md index 71ebf28dc..8fa188de1 100644 --- a/docs/docs/configuration/cheatsheet.md +++ b/docs/docs/configuration/cheatsheet.md @@ -521,7 +521,7 @@ Available caches: ### :http -* `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`) +* `proxy_url`: an upstream proxy to fetch posts and/or media with, (default: `nil`); for example `http://127.0.0.1:3192`. Does not support SOCKS5 proxy, only http(s). * `send_user_agent`: should we include a user agent with HTTP requests? (default: `true`) * `user_agent`: what user agent should we use? (default: `:default`), must be string or `:default` * `adapter`: array of adapter options diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index e29bf3ca3..cb619232f 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -248,9 +248,13 @@ def limiters_setup do end defp http_children do + proxy_url = Config.get([:http, :proxy_url]) + proxy = Pleroma.HTTP.AdapterHelper.format_proxy(proxy_url) + config = [:http, :adapter] |> Config.get([]) + |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy_pool(proxy) |> Keyword.put(:name, MyFinch) [{Finch, config}] diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index f9b489616..4949dd727 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -6,7 +6,7 @@ defmodule Pleroma.HTTP.AdapterHelper do @moduledoc """ Configure Tesla.Client with default and customized adapter options. """ - @defaults [name: MyFinch, connect_timeout: 5_000, recv_timeout: 5_000] + @defaults [name: MyFinch, pool_timeout: 5_000, receive_timeout: 5_000] @type proxy_type() :: :socks4 | :socks5 @type host() :: charlist() | :inet.ip_address() @@ -25,15 +25,58 @@ def format_proxy(nil), do: nil def format_proxy(proxy_url) do case parse_proxy(proxy_url) do - {:ok, host, port} -> {host, port} - {:ok, type, host, port} -> {type, host, port} + {:ok, host, port} -> {:http, host, port, []} + {:ok, type, host, port} -> {type, host, port, []} _ -> nil end end @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() def maybe_add_proxy(opts, nil), do: opts - def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) + + def maybe_add_proxy(opts, proxy) do + Keyword.put(opts, :proxy, proxy) + end + + def maybe_add_proxy_pool(opts, nil), do: opts + + def maybe_add_proxy_pool(opts, proxy) do + Logger.info("Using HTTP Proxy: #{inspect(proxy)}") + + opts + |> maybe_add_pools() + |> maybe_add_default_pool() + |> maybe_add_conn_opts() + |> put_in([:pools, :default, :conn_opts, :proxy], proxy) + end + + defp maybe_add_pools(opts) do + if Keyword.has_key?(opts, :pools) do + opts + else + Keyword.put(opts, :pools, %{}) + end + end + + defp maybe_add_default_pool(opts) do + pools = Keyword.get(opts, :pools) + + if Map.has_key?(pools, :default) do + opts + else + put_in(opts, [:pools, :default], []) + end + end + + defp maybe_add_conn_opts(opts) do + conn_opts = get_in(opts, [:pools, :default, :conn_opts]) + + unless is_nil(conn_opts) do + opts + else + put_in(opts, [:pools, :default, :conn_opts], []) + end + end @doc """ Merge default connection & adapter options with received ones. @@ -46,36 +89,31 @@ def options(%URI{} = uri, opts \\ []) do |> AdapterHelper.Default.options(uri) end + defp proxy_type("http"), do: {:ok, :http} + defp proxy_type("https"), do: {:ok, :https} + defp proxy_type(_), do: {:error, :unknown} + @spec parse_proxy(String.t() | tuple() | nil) :: {:ok, host(), pos_integer()} | {:ok, proxy_type(), host(), pos_integer()} | {:error, atom()} | nil - def parse_proxy(nil), do: nil def parse_proxy(proxy) when is_binary(proxy) do - with [host, port] <- String.split(proxy, ":"), - {port, ""} <- Integer.parse(port) do - {:ok, parse_host(host), port} + with %URI{} = uri <- URI.parse(proxy), + {:ok, type} <- proxy_type(uri.scheme) do + {:ok, type, uri.host, uri.port} else - {_, _} -> - Logger.warn("Parsing port failed #{inspect(proxy)}") - {:error, :invalid_proxy_port} - - :error -> - Logger.warn("Parsing port failed #{inspect(proxy)}") - {:error, :invalid_proxy_port} - - _ -> - Logger.warn("Parsing proxy failed #{inspect(proxy)}") + e -> + Logger.warn("Parsing proxy failed #{inspect(proxy)}, #{inspect(e)}") {:error, :invalid_proxy} end end def parse_proxy(proxy) when is_tuple(proxy) do with {type, host, port} <- proxy do - {:ok, type, parse_host(host), port} + {:ok, type, host, port} else _ -> Logger.warn("Parsing proxy failed #{inspect(proxy)}") diff --git a/lib/pleroma/http/adapter_helper/default.ex b/lib/pleroma/http/adapter_helper/default.ex index a1614b9c5..630536871 100644 --- a/lib/pleroma/http/adapter_helper/default.ex +++ b/lib/pleroma/http/adapter_helper/default.ex @@ -9,7 +9,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Default do @spec options(keyword(), URI.t()) :: keyword() def options(opts, _uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) + proxy = Pleroma.Config.get([:http, :proxy_url]) AdapterHelper.maybe_add_proxy(opts, AdapterHelper.format_proxy(proxy)) end diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex deleted file mode 100644 index 251539f34..000000000 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ /dev/null @@ -1,82 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.AdapterHelper.Gun do - @behaviour Pleroma.HTTP.AdapterHelper - - alias Pleroma.Config - alias Pleroma.HTTP.AdapterHelper - - require Logger - - @defaults [ - retry: 1, - retry_timeout: 1_000 - ] - - @type pool() :: :federation | :upload | :media | :default - - @spec options(keyword(), URI.t()) :: keyword() - def options(incoming_opts \\ [], %URI{} = uri) do - proxy = - [:http, :proxy_url] - |> Config.get() - |> AdapterHelper.format_proxy() - - config_opts = Config.get([:http, :adapter], []) - - @defaults - |> Keyword.merge(config_opts) - |> add_scheme_opts(uri) - |> AdapterHelper.maybe_add_proxy(proxy) - |> Keyword.merge(incoming_opts) - |> put_timeout() - end - - defp add_scheme_opts(opts, %{scheme: "http"}), do: opts - - defp add_scheme_opts(opts, %{scheme: "https"}) do - Keyword.put(opts, :certificates_verification, true) - end - - defp put_timeout(opts) do - {recv_timeout, opts} = Keyword.pop(opts, :recv_timeout, pool_timeout(opts[:pool])) - # this is the timeout to receive a message from Gun - # `:timeout` key is used in Tesla - Keyword.put(opts, :timeout, recv_timeout) - end - - @spec pool_timeout(pool()) :: non_neg_integer() - def pool_timeout(pool) do - default = Config.get([:pools, :default, :recv_timeout], 5_000) - - Config.get([:pools, pool, :recv_timeout], default) - end - - def limiter_setup do - prefix = Pleroma.Gun.ConnectionPool - wait = Config.get([:connections_pool, :connection_acquisition_wait]) - retries = Config.get([:connections_pool, :connection_acquisition_retries]) - - :pools - |> Config.get([]) - |> Enum.each(fn {name, opts} -> - max_running = Keyword.get(opts, :size, 50) - max_waiting = Keyword.get(opts, :max_waiting, 10) - - result = - ConcurrentLimiter.new(:"#{prefix}.#{name}", max_running, max_waiting, - wait: wait, - max_retries: retries - ) - - case result do - :ok -> :ok - {:error, :existing} -> :ok - end - end) - - :ok - end -end diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex deleted file mode 100644 index af0ada1e7..000000000 --- a/lib/pleroma/http/adapter_helper/hackney.ex +++ /dev/null @@ -1,40 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.AdapterHelper.Hackney do - @behaviour Pleroma.HTTP.AdapterHelper - - @defaults [ - follow_redirect: true, - force_redirect: true - ] - - @spec options(keyword(), URI.t()) :: keyword() - def options(connection_opts \\ [], %URI{} = uri) do - proxy = Pleroma.Config.get([:http, :proxy_url]) - - config_opts = Pleroma.Config.get([:http, :adapter], []) - - @defaults - |> Keyword.merge(config_opts) - |> Keyword.merge(connection_opts) - |> add_scheme_opts(uri) - |> maybe_add_with_body() - |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy) - end - - defp add_scheme_opts(opts, %URI{scheme: "https"}) do - Keyword.put(opts, :ssl_options, versions: [:"tlsv1.3", :"tlsv1.2", :"tlsv1.1", :tlsv1]) - end - - defp add_scheme_opts(opts, _), do: opts - - defp maybe_add_with_body(opts) do - if opts[:max_body] do - Keyword.put(opts, :with_body, true) - else - opts - end - end -end diff --git a/test/pleroma/http/adapter_helper/gun_test.exs b/test/pleroma/http/adapter_helper/gun_test.exs deleted file mode 100644 index cfb68557d..000000000 --- a/test/pleroma/http/adapter_helper/gun_test.exs +++ /dev/null @@ -1,77 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.AdapterHelper.GunTest do - use ExUnit.Case - use Pleroma.Tests.Helpers - - import Mox - - alias Pleroma.HTTP.AdapterHelper.Gun - - setup :verify_on_exit! - - describe "options/1" do - setup do: clear_config([:http, :adapter], a: 1, b: 2) - - test "https url with default port" do - uri = URI.parse("https://example.com") - - opts = Gun.options([receive_conn: false], uri) - assert opts[:certificates_verification] - end - - test "https ipv4 with default port" do - uri = URI.parse("https://127.0.0.1") - - opts = Gun.options([receive_conn: false], uri) - assert opts[:certificates_verification] - end - - test "https ipv6 with default port" do - uri = URI.parse("https://[2a03:2880:f10c:83:face:b00c:0:25de]") - - opts = Gun.options([receive_conn: false], uri) - assert opts[:certificates_verification] - end - - test "https url with non standart port" do - uri = URI.parse("https://example.com:115") - - opts = Gun.options([receive_conn: false], uri) - - assert opts[:certificates_verification] - end - - test "merges with defaul http adapter config" do - defaults = Gun.options([receive_conn: false], URI.parse("https://example.com")) - assert Keyword.has_key?(defaults, :a) - assert Keyword.has_key?(defaults, :b) - end - - test "parses string proxy host & port" do - clear_config([:http, :proxy_url], "localhost:8123") - - uri = URI.parse("https://some-domain.com") - opts = Gun.options([receive_conn: false], uri) - assert opts[:proxy] == {'localhost', 8123} - end - - test "parses tuple proxy scheme host and port" do - clear_config([:http, :proxy_url], {:socks, 'localhost', 1234}) - - uri = URI.parse("https://some-domain.com") - opts = Gun.options([receive_conn: false], uri) - assert opts[:proxy] == {:socks, 'localhost', 1234} - end - - test "passed opts have more weight than defaults" do - clear_config([:http, :proxy_url], {:socks5, 'localhost', 1234}) - uri = URI.parse("https://some-domain.com") - opts = Gun.options([receive_conn: false, proxy: {'example.com', 4321}], uri) - - assert opts[:proxy] == {'example.com', 4321} - end - end -end diff --git a/test/pleroma/http/adapter_helper/hackney_test.exs b/test/pleroma/http/adapter_helper/hackney_test.exs deleted file mode 100644 index 85150a65c..000000000 --- a/test/pleroma/http/adapter_helper/hackney_test.exs +++ /dev/null @@ -1,35 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.AdapterHelper.HackneyTest do - use ExUnit.Case, async: true - use Pleroma.Tests.Helpers - - alias Pleroma.HTTP.AdapterHelper.Hackney - - setup_all do - uri = URI.parse("http://domain.com") - {:ok, uri: uri} - end - - describe "options/2" do - setup do: clear_config([:http, :adapter], a: 1, b: 2) - - test "add proxy and opts from config", %{uri: uri} do - opts = Hackney.options([proxy: "localhost:8123"], uri) - - assert opts[:a] == 1 - assert opts[:b] == 2 - assert opts[:proxy] == "localhost:8123" - end - - test "respect connection opts and no proxy", %{uri: uri} do - opts = Hackney.options([a: 2, b: 1], uri) - - assert opts[:a] == 2 - assert opts[:b] == 1 - refute Keyword.has_key?(opts, :proxy) - end - end -end diff --git a/test/pleroma/http/adapter_helper_test.exs b/test/pleroma/http/adapter_helper_test.exs index 3c8c89164..55ffe4921 100644 --- a/test/pleroma/http/adapter_helper_test.exs +++ b/test/pleroma/http/adapter_helper_test.exs @@ -13,16 +13,38 @@ test "with nil" do end test "with string" do - assert AdapterHelper.format_proxy("127.0.0.1:8123") == {{127, 0, 0, 1}, 8123} + assert AdapterHelper.format_proxy("http://127.0.0.1:8123") == {:http, "127.0.0.1", 8123, []} end test "localhost with port" do - assert AdapterHelper.format_proxy("localhost:8123") == {'localhost', 8123} + assert AdapterHelper.format_proxy("https://localhost:8123") == + {:https, "localhost", 8123, []} end test "tuple" do - assert AdapterHelper.format_proxy({:socks4, :localhost, 9050}) == - {:socks4, 'localhost', 9050} + assert AdapterHelper.format_proxy({:http, "localhost", 9050}) == + {:http, "localhost", 9050, []} + end + end + + describe "maybe_add_proxy_pool/1" do + test "should do nothing with nil" do + assert AdapterHelper.maybe_add_proxy_pool([], nil) == [] + end + + test "should create pools" do + assert AdapterHelper.maybe_add_proxy_pool([], "proxy") == [ + pools: %{default: [conn_opts: [proxy: "proxy"]]} + ] + end + + test "should not override conn_opts if set" do + assert AdapterHelper.maybe_add_proxy_pool( + [pools: %{default: [conn_opts: [already: "set"]]}], + "proxy" + ) == [ + pools: %{default: [conn_opts: [proxy: "proxy", already: "set"]]} + ] end end end From 61641957cb4cc42f52b4efe41d1a251fe235d42f Mon Sep 17 00:00:00 2001 From: floatingghost Date: Tue, 16 Aug 2022 22:56:49 +0000 Subject: [PATCH 02/44] fix compatibility with meilisearch (#164) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/164 --- CHANGELOG.md | 4 ++++ lib/mix/tasks/pleroma/search/meilisearch.ex | 8 ++++---- lib/pleroma/release_tasks.ex | 9 ++++++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c7cd8601..3618211e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Fixed +- Compatibility with latest meilisearch +- Resolution of nested mix tasks (i.e search.meilisearch) in OTP releases + ### Removed - Non-finch HTTP adapters. `:tesla, :adapter` is now highly recommended to be set to the default. diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index d4a83c3cd..27a31afcf 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -9,7 +9,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do import Ecto.Query import Pleroma.Search.Meilisearch, - only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete!: 1] + only: [meili_put: 2, meili_get: 1, meili_delete!: 1] def run(["index"]) do start_pleroma() @@ -27,7 +27,7 @@ def run(["index"]) do end {:ok, _} = - meili_post( + meili_put( "/indexes/objects/settings/ranking-rules", [ "published:desc", @@ -41,7 +41,7 @@ def run(["index"]) do ) {:ok, _} = - meili_post( + meili_put( "/indexes/objects/settings/searchable-attributes", [ "content" @@ -91,7 +91,7 @@ def run(["index"]) do ) with {:ok, res} <- result do - if not Map.has_key?(res, "uid") do + if not Map.has_key?(res, "indexUid") do IO.puts("\nFailed to index: #{inspect(result)}") end else diff --git a/lib/pleroma/release_tasks.ex b/lib/pleroma/release_tasks.ex index 1e06aafe4..e43eef070 100644 --- a/lib/pleroma/release_tasks.ex +++ b/lib/pleroma/release_tasks.ex @@ -25,7 +25,7 @@ defp mix_task(task, args) do module = Module.split(module) match?(["Mix", "Tasks", "Pleroma" | _], module) and - String.downcase(List.last(module)) == task + task_match?(module, task) end) if module do @@ -35,6 +35,13 @@ defp mix_task(task, args) do end end + defp task_match?(["Mix", "Tasks", "Pleroma" | module_path], task) do + module_path + |> Enum.join(".") + |> String.downcase() + |> String.equivalent?(String.downcase(task)) + end + def migrate(args) do Mix.Tasks.Pleroma.Ecto.Migrate.run(args) end From 89ffc01c23cf89dc88245eb95f035e0404a5fcbb Mon Sep 17 00:00:00 2001 From: floatingghost Date: Tue, 16 Aug 2022 23:24:19 +0000 Subject: [PATCH 03/44] only return create objects for ES search (#165) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/165 --- CHANGELOG.md | 1 + lib/pleroma/search/elasticsearch.ex | 10 +++++++--- lib/pleroma/search/elasticsearch/store.ex | 1 - 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3618211e4..c1e1d01c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Fixed - Compatibility with latest meilisearch - Resolution of nested mix tasks (i.e search.meilisearch) in OTP releases +- Elasticsearch returning likes and repeats, displaying as posts ### Removed - Non-finch HTTP adapters. `:tesla, :adapter` is now highly recommended to be set to the default. diff --git a/lib/pleroma/search/elasticsearch.ex b/lib/pleroma/search/elasticsearch.ex index 7c7ca82c8..16b01101a 100644 --- a/lib/pleroma/search/elasticsearch.ex +++ b/lib/pleroma/search/elasticsearch.ex @@ -23,7 +23,7 @@ def es_query(:activity, query, offset, limit) do timeout: "5s", sort: [ "_score", - %{_timestamp: %{order: "desc", format: "basic_date_time"}} + %{"_timestamp" => %{order: "desc", format: "basic_date_time"}} ], query: %{ bool: %{ @@ -62,8 +62,12 @@ def search(user, query, options) do Task.async(fn -> q = es_query(:activity, parsed_query, offset, limit) - Pleroma.Search.Elasticsearch.Store.search(:activities, q) - |> Enum.filter(fn x -> Visibility.visible_for_user?(x, user) end) + :activities + |> Pleroma.Search.Elasticsearch.Store.search(q) + |> Enum.filter(fn x -> + x.data["type"] == "Create" && x.object.data["type"] == "Note" && + Visibility.visible_for_user?(x, user) + end) end) activity_results = Task.await(activity_task) diff --git a/lib/pleroma/search/elasticsearch/store.ex b/lib/pleroma/search/elasticsearch/store.ex index 895b76d7f..3b7bbb838 100644 --- a/lib/pleroma/search/elasticsearch/store.ex +++ b/lib/pleroma/search/elasticsearch/store.ex @@ -42,7 +42,6 @@ def search(:activities, q) do results |> Enum.map(fn result -> result["_id"] end) |> Pleroma.Activity.all_by_ids_with_object() - |> Enum.sort(&(&1.inserted_at >= &2.inserted_at)) else e -> Logger.error(e) From 11ec9daa5b742f8a1b408497321392e144f45019 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Wed, 17 Aug 2022 00:22:59 +0000 Subject: [PATCH 04/44] API compatibility with fedibird, frontend config (#163) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/163 --- CHANGELOG.md | 3 + config/config.exs | 8 +++ .../docs/configuration/frontend_management.md | 14 ++++- lib/pleroma/web/masto_fe_controller.ex | 14 ++++- .../web/mastodon_api/views/status_view.ex | 4 +- .../web/mastodon_api/websocket_handler.ex | 46 ++++++++++++-- .../controllers/emoji_reaction_controller.ex | 2 + lib/pleroma/web/router.ex | 5 ++ lib/pleroma/web/streamer.ex | 15 +++-- .../masto_fe/fedibird.index.html.eex | 34 +++++++++++ ...ndex.html.eex => glitchsoc.index.html.eex} | 0 lib/pleroma/web/views/masto_fe_view.ex | 8 ++- lib/pleroma/web/views/streamer_view.ex | 15 +++-- .../integration/mastodon_websocket_test.exs | 4 +- test/pleroma/notification_test.exs | 4 +- test/pleroma/web/masto_fe_controller_test.exs | 38 ++++++++++++ .../mastodon_api/views/status_view_test.exs | 31 ++++++---- .../emoji_reaction_controller_test.exs | 11 +++- test/pleroma/web/streamer_test.exs | 60 ++++++++++++------- 19 files changed, 256 insertions(+), 60 deletions(-) create mode 100644 lib/pleroma/web/templates/masto_fe/fedibird.index.html.eex rename lib/pleroma/web/templates/masto_fe/{index.html.eex => glitchsoc.index.html.eex} (100%) create mode 100644 test/pleroma/web/masto_fe_controller_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index c1e1d01c4..b32ae2e16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +### Added +- support for fedibird-fe, and non-breaking API parity for it to function + ### Fixed - Compatibility with latest meilisearch - Resolution of nested mix tasks (i.e search.meilisearch) in OTP releases diff --git a/config/config.exs b/config/config.exs index f49ec861c..83977da19 100644 --- a/config/config.exs +++ b/config/config.exs @@ -734,6 +734,14 @@ "build_dir" => "distribution", "ref" => "akkoma" }, + "fedibird-fe" => %{ + "name" => "fedibird-fe", + "git" => "https://akkoma.dev/AkkomaGang/fedibird-fe", + "build_url" => + "https://akkoma-updates.s3-website.fr-par.scw.cloud/frontend/${ref}/fedibird-fe.zip", + "build_dir" => "distribution", + "ref" => "akkoma" + }, "admin-fe" => %{ "name" => "admin-fe", "git" => "https://akkoma.dev/AkkomaGang/admin-fe", diff --git a/docs/docs/configuration/frontend_management.md b/docs/docs/configuration/frontend_management.md index a25120589..5e4b9b051 100644 --- a/docs/docs/configuration/frontend_management.md +++ b/docs/docs/configuration/frontend_management.md @@ -19,6 +19,10 @@ config :pleroma, :frontends, admin: %{ "name" => "admin-fe", "ref" => "stable" + }, + mastodon: %{ + "name" => "mastodon-fe", + "ref" => "akkoma" } ``` @@ -26,12 +30,18 @@ This would serve the frontend from the the folder at `$instance_static/frontends Refer to [the frontend CLI task](../../administration/CLI_tasks/frontend) for how to install the frontend's files -If you wish masto-fe to also be enabled, you will also need to run the install task for `mastodon-fe`. Not doing this will lead to the frontend not working. - If you choose not to install a frontend for whatever reason, it is recommended that you enable [`:static_fe`](#static_fe) to allow remote users to click "view remote source". Don't bother with this if you've got no unauthenticated access though. You can also replace the default "no frontend" page by placing an `index.html` file under your `instance/static/` directory. +## Mastodon-FE + +Akkoma supports both [glitchsoc](https://github.com/glitch-soc/mastodon)'s more "vanilla" mastodon frontend, +as well as [fedibird](https://github.com/fedibird/mastodon)'s extended frontend which has near-feature-parity with akkoma (with quoting and reactions). + +To enable either one, you must run the `frontend.install` task for either `mastodon-fe` or `fedibird-fe` (both `--ref akkoma`), then make sure +`:pleroma, :frontends, :mastodon` references the one you want. + ## Swagger (openAPI) documentation viewer If you're a developer and you'd like a human-readable rendering of the diff --git a/lib/pleroma/web/masto_fe_controller.ex b/lib/pleroma/web/masto_fe_controller.ex index d2460f51d..7b6e01aad 100644 --- a/lib/pleroma/web/masto_fe_controller.ex +++ b/lib/pleroma/web/masto_fe_controller.ex @@ -27,9 +27,21 @@ defmodule Pleroma.Web.MastoFEController do def index(conn, _params) do with %{assigns: %{user: %User{} = user, token: %Token{app_id: token_app_id} = token}} <- conn, {:ok, %{id: ^token_app_id}} <- AuthController.local_mastofe_app() do + flavour = + [:frontends, :mastodon] + |> Pleroma.Config.get() + |> Map.get("name", "mastodon-fe") + + index = + if flavour == "fedibird-fe" do + "fedibird.index.html" + else + "glitchsoc.index.html" + end + conn |> put_layout(false) - |> render("index.html", + |> render(index, token: token.token, user: user, custom_emojis: Pleroma.Emoji.get_all() diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index f0fe9a4ba..d099c4901 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -375,6 +375,7 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} emojis: build_emojis(object.data["emoji"]), quote_id: if(quote, do: quote.id, else: nil), quote: maybe_render_quote(quote, opts), + emoji_reactions: emoji_reactions, pleroma: %{ local: activity.local, conversation_id: get_context_id(activity), @@ -589,7 +590,8 @@ defp build_emoji_map(emoji, users, url, current_user) do name: emoji, count: length(users), url: MediaProxy.url(url), - me: !!(current_user && current_user.ap_id in users) + me: !!(current_user && current_user.ap_id in users), + account_ids: Enum.map(users, fn user -> User.get_cached_by_ap_id(user).id end) } end diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 861a7ce3e..582e65d70 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -32,8 +32,15 @@ def init(%{qs: qs} = req, state) do req end - {:cowboy_websocket, req, %{user: user, topic: topic, count: 0, timer: nil}, - %{idle_timeout: @timeout}} + {:cowboy_websocket, req, + %{ + user: user, + topic: topic, + count: 0, + timer: nil, + subscriptions: [], + oauth_token: oauth_token + }, %{idle_timeout: @timeout}} else {:error, :bad_topic} -> Logger.debug("#{__MODULE__} bad topic #{inspect(req)}") @@ -65,21 +72,50 @@ def websocket_handle(:pong, state) do # We only receive pings for now def websocket_handle(:ping, state), do: {:ok, state} - def websocket_handle({:text, "ping"}, state) do + def websocket_handle({:text, ping}, state) when ping in ~w[ping PING] do if state.timer, do: Process.cancel_timer(state.timer) {:reply, {:text, "pong"}, %{state | timer: timer()}} end + def websocket_handle({:text, text}, state) do + with {:ok, json} <- Jason.decode(text) do + websocket_handle({:json, json}, state) + else + _ -> + Logger.error("#{__MODULE__} received text frame: #{text}") + {:ok, state} + end + end + + def websocket_handle( + {:json, %{"type" => "subscribe", "stream" => stream_name}}, + %{user: user, oauth_token: token} = state + ) do + with {:ok, topic} <- Streamer.get_topic(stream_name, user, token, %{}) do + new_subscriptions = + [topic | Map.get(state, :subscriptions, [])] + |> Enum.uniq() + + {:ok, _topic} = Streamer.add_socket(topic, user) + + {:ok, Map.put(state, :subscriptions, new_subscriptions)} + else + _ -> + Logger.error("#{__MODULE__} received invalid topic: #{stream_name}") + {:ok, state} + end + end + def websocket_handle(frame, state) do Logger.error("#{__MODULE__} received frame: #{inspect(frame)}") {:ok, state} end - def websocket_info({:render_with_user, view, template, item}, state) do + def websocket_info({:render_with_user, view, template, item, topic}, state) do user = %User{} = User.get_cached_by_ap_id(state.user.ap_id) unless Streamer.filtered_by_user?(user, item) do - websocket_info({:text, view.render(template, item, user)}, %{state | user: user}) + websocket_info({:text, view.render(template, item, user, topic)}, %{state | user: user}) else {:ok, state} end diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex index 1de02faf8..91658587a 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex @@ -74,6 +74,8 @@ defp filter(reactions, %{emoji: emoji}) when is_binary(emoji) do defp filter(reactions, _), do: reactions def create(%{assigns: %{user: user}} = conn, %{id: activity_id, emoji: emoji}) do + emoji = Pleroma.Emoji.maybe_quote(emoji) + with {:ok, _activity} <- CommonAPI.react_with_emoji(activity_id, user, emoji) do activity = Activity.get_by_id(activity_id) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a0310bbb5..647d99278 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -457,6 +457,11 @@ defmodule Pleroma.Web.Router do get("/federation_status", InstancesController, :show) end + scope "/api/v1", Pleroma.Web.PleromaAPI do + pipe_through(:authenticated_api) + put("/statuses/:id/emoji_reactions/:emoji", EmojiReactionController, :create) + end + scope "/api/v1", Pleroma.Web.MastodonAPI do pipe_through(:authenticated_api) diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 9a4ac1317..d5b1d0678 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -114,6 +114,11 @@ def get_topic("list", _user, _oauth_token, _params) do {:error, :unauthorized} end + # mastodon multi-topic WS + def get_topic(nil, _user, _oauth_token, _params) do + {:ok, :multi} + end + def get_topic(_stream, _user, _oauth_token, _params) do {:error, :bad_topic} end @@ -186,8 +191,8 @@ defp do_stream("direct", item) do end defp do_stream("follow_relationship", item) do - text = StreamerView.render("follow_relationships_update.json", item) user_topic = "user:#{item.follower.id}" + text = StreamerView.render("follow_relationships_update.json", item, user_topic) Logger.debug("Trying to push follow relationship update to #{user_topic}\n\n") @@ -235,7 +240,7 @@ defp do_stream(topic, %Notification{} = item) when topic in ["user", "user:notification"] do Registry.dispatch(@registry, "#{topic}:#{item.user_id}", fn list -> Enum.each(list, fn {pid, _auth} -> - send(pid, {:render_with_user, StreamerView, "notification.json", item}) + send(pid, {:render_with_user, StreamerView, "notification.json", item, topic}) end) end) end @@ -259,7 +264,7 @@ defp do_stream(topic, item) do end defp push_to_socket(topic, %Participation{} = participation) do - rendered = StreamerView.render("conversation.json", participation) + rendered = StreamerView.render("conversation.json", participation, topic) Registry.dispatch(@registry, topic, fn list -> Enum.each(list, fn {pid, _} -> @@ -283,12 +288,12 @@ defp push_to_socket(topic, %Activity{ defp push_to_socket(_topic, %Activity{data: %{"type" => "Delete"}}), do: :noop defp push_to_socket(topic, item) do - anon_render = StreamerView.render("update.json", item) + anon_render = StreamerView.render("update.json", item, topic) Registry.dispatch(@registry, topic, fn list -> Enum.each(list, fn {pid, auth?} -> if auth? do - send(pid, {:render_with_user, StreamerView, "update.json", item}) + send(pid, {:render_with_user, StreamerView, "update.json", item, topic}) else send(pid, {:text, anon_render}) end diff --git a/lib/pleroma/web/templates/masto_fe/fedibird.index.html.eex b/lib/pleroma/web/templates/masto_fe/fedibird.index.html.eex new file mode 100644 index 000000000..02c421831 --- /dev/null +++ b/lib/pleroma/web/templates/masto_fe/fedibird.index.html.eex @@ -0,0 +1,34 @@ + + + + + + +<%= Config.get([:instance, :name]) %> + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + diff --git a/lib/pleroma/web/templates/masto_fe/index.html.eex b/lib/pleroma/web/templates/masto_fe/glitchsoc.index.html.eex similarity index 100% rename from lib/pleroma/web/templates/masto_fe/index.html.eex rename to lib/pleroma/web/templates/masto_fe/glitchsoc.index.html.eex diff --git a/lib/pleroma/web/views/masto_fe_view.ex b/lib/pleroma/web/views/masto_fe_view.ex index 63a9c8179..305368c9d 100644 --- a/lib/pleroma/web/views/masto_fe_view.ex +++ b/lib/pleroma/web/views/masto_fe_view.ex @@ -14,6 +14,7 @@ def initial_state(token, user, custom_emojis) do %{ meta: %{ + title: Config.get([:instance, :name]), streaming_api_base_url: Pleroma.Web.Endpoint.websocket_url(), access_token: token, locale: "en", @@ -27,7 +28,11 @@ def initial_state(token, user, custom_emojis) do display_sensitive_media: false, reduce_motion: false, max_toot_chars: limit, - mascot: User.get_mascot(user)["url"] + mascot: User.get_mascot(user)["url"], + show_quote_button: true, + enable_reaction: true, + compact_reaction: false, + advanced_layout: true }, poll_limits: Config.get([:instance, :poll_limits]), rights: %{ @@ -56,6 +61,7 @@ def initial_state(token, user, custom_emojis) do "video\/mp4" ] }, + lists: [], settings: user.mastofe_settings || %{}, push_subscription: nil, accounts: %{user.id => render(AccountView, "show.json", user: user, for: user)}, diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index de2e4d1e9..f455f941e 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -11,8 +11,9 @@ defmodule Pleroma.Web.StreamerView do alias Pleroma.User alias Pleroma.Web.MastodonAPI.NotificationView - def render("update.json", %Activity{} = activity, %User{} = user) do + def render("update.json", %Activity{} = activity, %User{} = user, topic) do %{ + stream: [topic], event: "update", payload: Pleroma.Web.MastodonAPI.StatusView.render( @@ -25,8 +26,9 @@ def render("update.json", %Activity{} = activity, %User{} = user) do |> Jason.encode!() end - def render("notification.json", %Notification{} = notify, %User{} = user) do + def render("notification.json", %Notification{} = notify, %User{} = user, topic) do %{ + stream: [topic], event: "notification", payload: NotificationView.render( @@ -38,8 +40,9 @@ def render("notification.json", %Notification{} = notify, %User{} = user) do |> Jason.encode!() end - def render("update.json", %Activity{} = activity) do + def render("update.json", %Activity{} = activity, topic) do %{ + stream: [topic], event: "update", payload: Pleroma.Web.MastodonAPI.StatusView.render( @@ -51,8 +54,9 @@ def render("update.json", %Activity{} = activity) do |> Jason.encode!() end - def render("follow_relationships_update.json", item) do + def render("follow_relationships_update.json", item, topic) do %{ + stream: [topic], event: "pleroma:follow_relationships_update", payload: %{ @@ -73,8 +77,9 @@ def render("follow_relationships_update.json", item) do |> Jason.encode!() end - def render("conversation.json", %Participation{} = participation) do + def render("conversation.json", %Participation{} = participation, topic) do %{ + stream: [topic], event: "conversation", payload: Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs index 43ec57893..356bfa48d 100644 --- a/test/pleroma/integration/mastodon_websocket_test.exs +++ b/test/pleroma/integration/mastodon_websocket_test.exs @@ -31,9 +31,9 @@ def start_socket(qs \\ nil, headers \\ []) do WebsocketClient.start_link(self(), path, headers) end - test "refuses invalid requests" do + test "allows multi-streams" do capture_log(fn -> - assert {:error, {404, _}} = start_socket() + assert {:ok, _} = start_socket() assert {:error, {404, _}} = start_socket("?stream=ncjdk") Process.sleep(30) end) diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs index b47edd0a3..4354dd2b6 100644 --- a/test/pleroma/notification_test.exs +++ b/test/pleroma/notification_test.exs @@ -224,7 +224,7 @@ test "it creates a notification for user and send to the 'user' and the 'user:no task = Task.async(fn -> {:ok, _topic} = Streamer.get_topic_and_add_socket("user", user, oauth_token) - assert_receive {:render_with_user, _, _, _}, 4_000 + assert_receive {:render_with_user, _, _, _, "user"}, 4_000 end) task_user_notification = @@ -232,7 +232,7 @@ test "it creates a notification for user and send to the 'user' and the 'user:no {:ok, _topic} = Streamer.get_topic_and_add_socket("user:notification", user, oauth_token) - assert_receive {:render_with_user, _, _, _}, 4_000 + assert_receive {:render_with_user, _, _, _, "user:notification"}, 4_000 end) activity = insert(:note_activity) diff --git a/test/pleroma/web/masto_fe_controller_test.exs b/test/pleroma/web/masto_fe_controller_test.exs new file mode 100644 index 000000000..924b45352 --- /dev/null +++ b/test/pleroma/web/masto_fe_controller_test.exs @@ -0,0 +1,38 @@ +defmodule Pleroma.Web.MastoFEControllerTest do + use Pleroma.Web.ConnCase, async: true + alias Pleroma.Web.MastodonAPI.AuthController + + describe "index/2 (main page)" do + test "GET /web/ (glitch-soc)" do + clear_config([:frontends, :mastodon], %{"name" => "mastodon-fe"}) + + {:ok, masto_app} = AuthController.local_mastofe_app() + user = Pleroma.Factory.insert(:user) + token = Pleroma.Factory.insert(:oauth_token, app: masto_app, user: user) + %{conn: conn} = oauth_access(["read", "write"], oauth_token: token, user: user) + + resp = + conn + |> get("/web/getting-started") + |> html_response(200) + + assert resp =~ "glitch" + end + + test "GET /web/ (fedibird)" do + clear_config([:frontends, :mastodon], %{"name" => "fedibird-fe"}) + + {:ok, masto_app} = AuthController.local_mastofe_app() + user = Pleroma.Factory.insert(:user) + token = Pleroma.Factory.insert(:oauth_token, app: masto_app, user: user) + %{conn: conn} = oauth_access(["read", "write"], oauth_token: token, user: user) + + resp = + conn + |> get("/web/getting-started") + |> html_response(200) + + refute resp =~ "glitch" + end + end +end diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index fb3255927..a6f8f3fc8 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -44,14 +44,15 @@ test "has an emoji reaction list" do assert_schema(status, "Status", Pleroma.Web.ApiSpec.spec()) assert status[:pleroma][:emoji_reactions] == [ - %{name: "☕", count: 2, me: false, url: nil}, + %{name: "☕", count: 2, me: false, url: nil, account_ids: [other_user.id, user.id]}, %{ count: 2, me: false, name: "dinosaur", - url: "http://localhost:4001/emoji/dino walking.gif" + url: "http://localhost:4001/emoji/dino walking.gif", + account_ids: [other_user.id, user.id] }, - %{name: "🍵", count: 1, me: false, url: nil} + %{name: "🍵", count: 1, me: false, url: nil, account_ids: [third_user.id]} ] status = StatusView.render("show.json", activity: activity, for: user) @@ -59,14 +60,15 @@ test "has an emoji reaction list" do assert_schema(status, "Status", Pleroma.Web.ApiSpec.spec()) assert status[:pleroma][:emoji_reactions] == [ - %{name: "☕", count: 2, me: true, url: nil}, + %{name: "☕", count: 2, me: true, url: nil, account_ids: [other_user.id, user.id]}, %{ count: 2, me: true, name: "dinosaur", - url: "http://localhost:4001/emoji/dino walking.gif" + url: "http://localhost:4001/emoji/dino walking.gif", + account_ids: [other_user.id, user.id] }, - %{name: "🍵", count: 1, me: false, url: nil} + %{name: "🍵", count: 1, me: false, url: nil, account_ids: [third_user.id]} ] end @@ -82,7 +84,7 @@ test "works correctly with badly formatted emojis" do status = StatusView.render("show.json", activity: activity, for: user) assert status[:pleroma][:emoji_reactions] == [ - %{name: "☕", count: 1, me: true, url: nil} + %{name: "☕", count: 1, me: true, url: nil, account_ids: [user.id]} ] end @@ -102,7 +104,7 @@ test "doesn't show reactions from muted and blocked users" do status = StatusView.render("show.json", activity: activity) assert status[:pleroma][:emoji_reactions] == [ - %{name: "☕", count: 1, me: false, url: nil} + %{name: "☕", count: 1, me: false, url: nil, account_ids: [other_user.id]} ] status = StatusView.render("show.json", activity: activity, for: user) @@ -114,19 +116,25 @@ test "doesn't show reactions from muted and blocked users" do status = StatusView.render("show.json", activity: activity) assert status[:pleroma][:emoji_reactions] == [ - %{name: "☕", count: 2, me: false, url: nil} + %{ + name: "☕", + count: 2, + me: false, + url: nil, + account_ids: [third_user.id, other_user.id] + } ] status = StatusView.render("show.json", activity: activity, for: user) assert status[:pleroma][:emoji_reactions] == [ - %{name: "☕", count: 1, me: false, url: nil} + %{name: "☕", count: 1, me: false, url: nil, account_ids: [third_user.id]} ] status = StatusView.render("show.json", activity: activity, for: other_user) assert status[:pleroma][:emoji_reactions] == [ - %{name: "☕", count: 1, me: true, url: nil} + %{name: "☕", count: 1, me: true, url: nil, account_ids: [other_user.id]} ] end @@ -272,6 +280,7 @@ test "a note activity" do spoiler_text: HTML.filter_tags(object_data["summary"]), visibility: "public", media_attachments: [], + emoji_reactions: [], mentions: [], tags: [ %{ diff --git a/test/pleroma/web/pleroma_api/controllers/emoji_reaction_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/emoji_reaction_controller_test.exs index 65bb22e27..4898179e6 100644 --- a/test/pleroma/web/pleroma_api/controllers/emoji_reaction_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/emoji_reaction_controller_test.exs @@ -31,7 +31,13 @@ test "PUT /api/v1/pleroma/statuses/:id/reactions/:emoji", %{conn: conn} do assert to_string(activity.id) == id assert result["pleroma"]["emoji_reactions"] == [ - %{"name" => "☕", "count" => 1, "me" => true, "url" => nil} + %{ + "name" => "☕", + "count" => 1, + "me" => true, + "url" => nil, + "account_ids" => [other_user.id] + } ] {:ok, activity} = CommonAPI.post(user, %{status: "#cofe"}) @@ -54,7 +60,8 @@ test "PUT /api/v1/pleroma/statuses/:id/reactions/:emoji", %{conn: conn} do "name" => "dinosaur", "count" => 1, "me" => true, - "url" => "http://localhost:4001/emoji/dino walking.gif" + "url" => "http://localhost:4001/emoji/dino walking.gif", + "account_ids" => [other_user.id] } ] diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs index 841db0e91..07129ff11 100644 --- a/test/pleroma/web/streamer_test.exs +++ b/test/pleroma/web/streamer_test.exs @@ -157,7 +157,8 @@ test "it streams the user's post in the 'user' stream", %{user: user, token: oau Streamer.get_topic_and_add_socket("user", user, oauth_token) {:ok, activity} = CommonAPI.post(user, %{status: "hey"}) - assert_receive {:render_with_user, _, _, ^activity} + stream_name = "user:#{user.id}" + assert_receive {:render_with_user, _, _, ^activity, ^stream_name} refute Streamer.filtered_by_user?(user, activity) end @@ -168,7 +169,11 @@ test "it streams boosts of the user in the 'user' stream", %{user: user, token: {:ok, activity} = CommonAPI.post(other_user, %{status: "hey"}) {:ok, announce} = CommonAPI.repeat(activity.id, user) - assert_receive {:render_with_user, Pleroma.Web.StreamerView, "update.json", ^announce} + stream_name = "user:#{user.id}" + + assert_receive {:render_with_user, Pleroma.Web.StreamerView, "update.json", ^announce, + ^stream_name} + refute Streamer.filtered_by_user?(user, announce) end @@ -221,7 +226,11 @@ test "it streams boosts of mastodon user in the 'user' stream", %{ {:ok, %Pleroma.Activity{data: _data, local: false} = announce} = Pleroma.Web.ActivityPub.Transmogrifier.handle_incoming(data) - assert_receive {:render_with_user, Pleroma.Web.StreamerView, "update.json", ^announce} + stream_name = "user:#{user.id}" + + assert_receive {:render_with_user, Pleroma.Web.StreamerView, "update.json", ^announce, + ^stream_name} + refute Streamer.filtered_by_user?(user, announce) end @@ -233,7 +242,7 @@ test "it sends notify to in the 'user' stream", %{ Streamer.get_topic_and_add_socket("user", user, oauth_token) Streamer.stream("user", notify) - assert_receive {:render_with_user, _, _, ^notify} + assert_receive {:render_with_user, _, _, ^notify, "user"} refute Streamer.filtered_by_user?(user, notify) end @@ -245,7 +254,7 @@ test "it sends notify to in the 'user:notification' stream", %{ Streamer.get_topic_and_add_socket("user:notification", user, oauth_token) Streamer.stream("user:notification", notify) - assert_receive {:render_with_user, _, _, ^notify} + assert_receive {:render_with_user, _, _, ^notify, "user:notification"} refute Streamer.filtered_by_user?(user, notify) end @@ -291,7 +300,7 @@ test "it sends favorite to 'user:notification' stream'", %{ Streamer.get_topic_and_add_socket("user:notification", user, oauth_token) {:ok, favorite_activity} = CommonAPI.favorite(user2, activity.id) - assert_receive {:render_with_user, _, "notification.json", notif} + assert_receive {:render_with_user, _, "notification.json", notif, "user:notification"} assert notif.activity.id == favorite_activity.id refute Streamer.filtered_by_user?(user, notif) end @@ -320,7 +329,7 @@ test "it sends follow activities to the 'user:notification' stream", %{ Streamer.get_topic_and_add_socket("user:notification", user, oauth_token) {:ok, _follower, _followed, follow_activity} = CommonAPI.follow(user2, user) - assert_receive {:render_with_user, _, "notification.json", notif} + assert_receive {:render_with_user, _, "notification.json", notif, "user:notification"} assert notif.activity.id == follow_activity.id refute Streamer.filtered_by_user?(user, notif) end @@ -384,7 +393,7 @@ test "it sends to public (authenticated)" do Streamer.get_topic_and_add_socket("public", user, oauth_token) {:ok, activity} = CommonAPI.post(other_user, %{status: "Test"}) - assert_receive {:render_with_user, _, _, ^activity} + assert_receive {:render_with_user, _, _, ^activity, "public"} refute Streamer.filtered_by_user?(other_user, activity) end @@ -436,7 +445,7 @@ test "it filters to user if recipients invalid and thread containment is enabled Streamer.get_topic_and_add_socket("public", user, oauth_token) Streamer.stream("public", activity) - assert_receive {:render_with_user, _, _, ^activity} + assert_receive {:render_with_user, _, _, ^activity, "public"} assert Streamer.filtered_by_user?(user, activity) end @@ -458,7 +467,7 @@ test "it sends message if recipients invalid and thread containment is disabled" Streamer.get_topic_and_add_socket("public", user, oauth_token) Streamer.stream("public", activity) - assert_receive {:render_with_user, _, _, ^activity} + assert_receive {:render_with_user, _, _, ^activity, "public"} refute Streamer.filtered_by_user?(user, activity) end @@ -481,7 +490,7 @@ test "it sends message if recipients invalid and thread containment is enabled b Streamer.get_topic_and_add_socket("public", user, oauth_token) Streamer.stream("public", activity) - assert_receive {:render_with_user, _, _, ^activity} + assert_receive {:render_with_user, _, _, ^activity, "public"} refute Streamer.filtered_by_user?(user, activity) end end @@ -495,7 +504,7 @@ test "it filters messages involving blocked users", %{user: user, token: oauth_t Streamer.get_topic_and_add_socket("public", user, oauth_token) {:ok, activity} = CommonAPI.post(blocked_user, %{status: "Test"}) - assert_receive {:render_with_user, _, _, ^activity} + assert_receive {:render_with_user, _, _, ^activity, "public"} assert Streamer.filtered_by_user?(user, activity) end @@ -512,17 +521,17 @@ test "it filters messages transitively involving blocked users", %{ {:ok, activity_one} = CommonAPI.post(friend, %{status: "hey! @#{blockee.nickname}"}) - assert_receive {:render_with_user, _, _, ^activity_one} + assert_receive {:render_with_user, _, _, ^activity_one, "public"} assert Streamer.filtered_by_user?(blocker, activity_one) {:ok, activity_two} = CommonAPI.post(blockee, %{status: "hey! @#{friend.nickname}"}) - assert_receive {:render_with_user, _, _, ^activity_two} + assert_receive {:render_with_user, _, _, ^activity_two, "public"} assert Streamer.filtered_by_user?(blocker, activity_two) {:ok, activity_three} = CommonAPI.post(blockee, %{status: "hey! @#{blocker.nickname}"}) - assert_receive {:render_with_user, _, _, ^activity_three} + assert_receive {:render_with_user, _, _, ^activity_three, "public"} assert Streamer.filtered_by_user?(blocker, activity_three) end end @@ -583,7 +592,8 @@ test "it sends wanted private posts to list", %{user: user_a, token: user_a_toke visibility: "private" }) - assert_receive {:render_with_user, _, _, ^activity} + stream_name = "list:#{list.id}" + assert_receive {:render_with_user, _, _, ^activity, ^stream_name} refute Streamer.filtered_by_user?(user_a, activity) end end @@ -601,7 +611,8 @@ test "it filters muted reblogs", %{user: user1, token: user1_token} do Streamer.get_topic_and_add_socket("user", user1, user1_token) {:ok, announce_activity} = CommonAPI.repeat(create_activity.id, user2) - assert_receive {:render_with_user, _, _, ^announce_activity} + stream_name = "user:#{user1.id}" + assert_receive {:render_with_user, _, _, ^announce_activity, ^stream_name} assert Streamer.filtered_by_user?(user1, announce_activity) end @@ -617,7 +628,7 @@ test "it filters reblog notification for reblog-muted actors", %{ Streamer.get_topic_and_add_socket("user", user1, user1_token) {:ok, _announce_activity} = CommonAPI.repeat(create_activity.id, user2) - assert_receive {:render_with_user, _, "notification.json", notif} + assert_receive {:render_with_user, _, "notification.json", notif, "user"} assert Streamer.filtered_by_user?(user1, notif) end @@ -633,7 +644,7 @@ test "it send non-reblog notification for reblog-muted actors", %{ Streamer.get_topic_and_add_socket("user", user1, user1_token) {:ok, _favorite_activity} = CommonAPI.favorite(user2, create_activity.id) - assert_receive {:render_with_user, _, "notification.json", notif} + assert_receive {:render_with_user, _, "notification.json", notif, "user"} refute Streamer.filtered_by_user?(user1, notif) end end @@ -648,7 +659,8 @@ test "it filters posts from muted threads" do {:ok, activity} = CommonAPI.post(user, %{status: "super hot take"}) {:ok, _} = CommonAPI.add_mute(user2, activity) - assert_receive {:render_with_user, _, _, ^activity} + stream_name = "user:#{user2.id}" + assert_receive {:render_with_user, _, _, ^activity, ^stream_name} assert Streamer.filtered_by_user?(user2, activity) end end @@ -690,7 +702,8 @@ test "it doesn't send conversation update to the 'direct' stream when the last m }) create_activity_id = create_activity.id - assert_receive {:render_with_user, _, _, ^create_activity} + stream_name = "direct:#{user.id}" + assert_receive {:render_with_user, _, _, ^create_activity, ^stream_name} assert_receive {:text, received_conversation1} assert %{"event" => "conversation", "payload" => _} = Jason.decode!(received_conversation1) @@ -725,8 +738,9 @@ test "it sends conversation update to the 'direct' stream when a message is dele visibility: "direct" }) - assert_receive {:render_with_user, _, _, ^create_activity} - assert_receive {:render_with_user, _, _, ^create_activity2} + stream_name = "direct:#{user.id}" + assert_receive {:render_with_user, _, _, ^create_activity, ^stream_name} + assert_receive {:render_with_user, _, _, ^create_activity2, ^stream_name} assert_receive {:text, received_conversation1} assert %{"event" => "conversation", "payload" => _} = Jason.decode!(received_conversation1) assert_receive {:text, received_conversation1} From aaf78e2b52c3aa1e75206e7dbe41b45874978e0a Mon Sep 17 00:00:00 2001 From: floatingghost Date: Wed, 17 Aug 2022 09:35:11 +0000 Subject: [PATCH 05/44] only put linked mfm in source (#171) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/171 --- config/description.exs | 43 ++++++++++++++++--- .../article_note_page_validator.ex | 5 +-- test/fixtures/misskey/mfm_x_format.json | 2 +- .../article_note_page_validator_test.exs | 13 +++--- 4 files changed, 44 insertions(+), 19 deletions(-) diff --git a/config/description.exs b/config/description.exs index 9f93265d1..b70982cd2 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1169,7 +1169,6 @@ hideFilteredStatuses: false, hideMutedPosts: false, hidePostStats: false, - hideSitename: false, hideUserStats: false, loginMethod: "password", logo: "/static/logo.svg", @@ -1235,12 +1234,6 @@ type: :boolean, description: "Hide notices statistics (repeats, favorites, ...)" }, - %{ - key: :hideSitename, - label: "Hide Sitename", - type: :boolean, - description: "Hides instance name from PleromaFE banner" - }, %{ key: :hideUserStats, label: "Hide user stats", @@ -1350,6 +1343,42 @@ type: :string, description: "Which theme to use. Available themes are defined in styles.json", suggestions: ["pleroma-dark"] + }, + %{ + key: :showPanelNavShortcuts, + label: "Show timeline panel nav shortcuts", + type: :boolean, + description: "Whether to put timeline nav tabs on the top of the panel" + }, + %{ + key: :showNavShortcuts, + label: "Show navbar shortcuts", + type: :boolean, + description: "Whether to put extra navigation options on the navbar" + }, + %{ + key: :showWiderShortcuts, + label: "Increase navbar shortcut spacing", + type: :boolean, + description: "Whether to add extra space between navbar icons" + }, + %{ + key: :hideSiteFavicon, + label: "Hide site favicon", + type: :boolean, + description: "Whether to hide the instance favicon from the navbar" + }, + %{ + key: :hideSiteName, + label: "Hide site name", + type: :boolean, + description: "Whether to hide the site name from the navbar" + }, + %{ + key: :renderMisskeyMarkdown, + label: "Render misskey markdown", + type: :boolean, + description: "Whether to render Misskey-flavoured markdown" } ] }, diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index eee7629ad..f2779432e 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -119,7 +119,7 @@ defp fix_misskey_content( {linked, _, _} = Utils.format_input(content, "text/x.misskeymarkdown", mention_handler: mention_handler) - Map.put(object, "content", linked) + put_in(object, ["source", "content"], linked) end defp fix_misskey_content(%{"_misskey_content" => content} = object) when is_binary(content) do @@ -132,10 +132,9 @@ defp fix_misskey_content(%{"_misskey_content" => content} = object) when is_bina object |> Map.put("source", %{ - "content" => content, + "content" => linked, "mediaType" => "text/x.misskeymarkdown" }) - |> Map.put("content", linked) |> Map.delete("_misskey_content") end diff --git a/test/fixtures/misskey/mfm_x_format.json b/test/fixtures/misskey/mfm_x_format.json index 21aae9204..590e399fe 100644 --- a/test/fixtures/misskey/mfm_x_format.json +++ b/test/fixtures/misskey/mfm_x_format.json @@ -3,7 +3,7 @@ "type": "Note", "attributedTo": "https://misskey.local.live/users/92hzkskwgy", "summary": null, - "content": "this gets replaced", + "content": "this does not get replaced", "source": { "content": "@akkoma_user @remote_user @full_tag_remote_user@misskey.local.live @oops_not_a_mention linkifylink #dancedance $[jelly mfm goes here] \n\n## aaa", "mediaType": "text/x.misskeymarkdown" diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index 09cd1a964..f419770f2 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -96,10 +96,9 @@ test "a misskey MFM status with a content field should work and be linked", _ do %{ valid?: true, changes: %{ - content: content, + content: "this does not get replaced", source: %{ - "content" => - "@akkoma_user @remote_user @full_tag_remote_user@misskey.local.live @oops_not_a_mention linkifylink #dancedance $[jelly mfm goes here] \n\n## aaa", + "content" => content, "mediaType" => "text/x.misskeymarkdown" } } @@ -129,22 +128,20 @@ test "a misskey MFM status with a _misskey_content field should work and be link |> File.read!() |> Jason.decode!() - expected_content = - "@akkoma_user linkifylink #dancedance $[jelly mfm goes here]

## aaa" - changes = ArticleNotePageValidator.cast_and_validate(note) %{ valid?: true, changes: %{ source: %{ - "content" => "@akkoma_user linkifylink #dancedance $[jelly mfm goes here] \n\n## aaa", + "content" => content, "mediaType" => "text/x.misskeymarkdown" } } } = changes - assert changes.changes[:content] == expected_content + assert content =~ + "@akkoma_user" end end end From e9f1897cfdb32c890e9eaf2e894128be5c7e1123 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Thu, 18 Aug 2022 03:14:48 +0000 Subject: [PATCH 06/44] parser MFM server-side (#172) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/172 --- .../article_note_page_validator.ex | 7 ++-- lib/pleroma/web/common_api/utils.ex | 8 ++--- mix.exs | 5 ++- mix.lock | 1 + priv/scrubbers/default.ex | 34 +++++++++++++++++-- .../article_note_page_validator_test.exs | 12 ++++--- 6 files changed, 53 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index f2779432e..28053ea3a 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -108,6 +108,8 @@ defp remote_mention_resolver( end # https://github.com/misskey-dev/misskey/pull/8787 + # Misskey has an awful tendency to drop all custom formatting when it sends remotely + # So this basically reprocesses their MFM source defp fix_misskey_content( %{"source" => %{"mediaType" => "text/x.misskeymarkdown", "content" => content}} = object ) @@ -119,7 +121,7 @@ defp fix_misskey_content( {linked, _, _} = Utils.format_input(content, "text/x.misskeymarkdown", mention_handler: mention_handler) - put_in(object, ["source", "content"], linked) + Map.put(object, "content", linked) end defp fix_misskey_content(%{"_misskey_content" => content} = object) when is_binary(content) do @@ -132,9 +134,10 @@ defp fix_misskey_content(%{"_misskey_content" => content} = object) when is_bina object |> Map.put("source", %{ - "content" => linked, + "content" => content, "mediaType" => "text/x.misskeymarkdown" }) + |> Map.put("content", linked) |> Map.delete("_misskey_content") end diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 61af71acd..15016eb47 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -285,11 +285,11 @@ def format_input(text, "text/html", options) do def format_input(text, "text/x.misskeymarkdown", options) do text + |> Formatter.markdown_to_html() + |> MfmParser.Parser.parse() + |> MfmParser.Encoder.to_html() |> Formatter.linkify(options) - |> Formatter.html_escape("text/x.misskeymarkdown") - |> (fn {text, mentions, tags} -> - {String.replace(text, ~r/\r?\n/, "
"), mentions, tags} - end).() + |> Formatter.html_escape("text/html") end def format_input(text, "text/markdown", options) do diff --git a/mix.exs b/mix.exs index c6bd0e28f..e7f491997 100644 --- a/mix.exs +++ b/mix.exs @@ -129,7 +129,7 @@ defp deps do override: true}, {:bcrypt_elixir, "~> 2.2"}, {:trailing_format_plug, "~> 0.0.7"}, - {:fast_sanitize, "~> 0.2.0"}, + {:fast_sanitize, "~> 0.2.3"}, {:html_entities, "~> 0.5", override: true}, {:phoenix_html, "~> 3.1", override: true}, {:calendar, "~> 1.0"}, @@ -191,6 +191,9 @@ defp deps do {:ecto_psql_extras, "~> 0.6"}, {:elasticsearch, git: "https://akkoma.dev/AkkomaGang/elasticsearch-elixir.git", ref: "main"}, + {:mfm_parser, + git: "https://akkoma.dev/AkkomaGang/mfm-parser.git", + ref: "5054e0ba1ebcbd9a7916aec219528e3e58057241"}, # indirect dependency version override {:plug, "~> 1.10.4", override: true}, diff --git a/mix.lock b/mix.lock index 2d3c9f33e..1c3b550e7 100644 --- a/mix.lock +++ b/mix.lock @@ -67,6 +67,7 @@ "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "meck": {:hex, :meck, "0.9.2", "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", [:rebar3], [], "hexpm", "81344f561357dc40a8344afa53767c32669153355b626ea9fcbc8da6b3045826"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, + "mfm_parser": {:git, "https://akkoma.dev/AkkomaGang/mfm-parser.git", "5054e0ba1ebcbd9a7916aec219528e3e58057241", [ref: "5054e0ba1ebcbd9a7916aec219528e3e58057241"]}, "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "mint": {:hex, :mint, "1.4.2", "50330223429a6e1260b2ca5415f69b0ab086141bc76dc2fbf34d7c389a6675b2", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "ce75a5bbcc59b4d7d8d70f8b2fc284b1751ffb35c7b6a6302b5192f8ab4ddd80"}, diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index 153b0be45..68ac06e32 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -56,8 +56,36 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes(:u, []) Meta.allow_tag_with_these_attributes(:ul, []) - Meta.allow_tag_with_this_attribute_values(:span, "class", ["h-card", "quote-inline"]) - Meta.allow_tag_with_these_attributes(:span, []) + Meta.allow_tags_with_style_attributes([:span]) + + Meta.allow_tag_with_this_attribute_values(:span, "class", [ + "h-card", + "quote-inline", + "mfm", + "_mfm_tada_", + "_mfm_jelly_", + "_mfm_twitch_", + "_mfm_shake_", + "_mfm_spin_", + "_mfm_jump_", + "_mfm_bounce_", + "_mfm_flip_", + "_mfm_x2_", + "_mfm_x3_", + "_mfm_x4_", + "_mfm_blur_", + "_mfm_rainbow_", + "_mfm_rotate_" + ]) + + Meta.allow_tag_with_these_attributes(:span, [ + "data-x", + "data-y", + "data-h", + "data-v", + "data-left", + "data-right" + ]) Meta.allow_tag_with_this_attribute_values(:code, "class", ["inline"]) @@ -101,4 +129,6 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_these_attributes(:small, []) Meta.strip_everything_not_covered() + + defp scrub_css(value), do: value end diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index f419770f2..c766414a6 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -96,9 +96,8 @@ test "a misskey MFM status with a content field should work and be linked", _ do %{ valid?: true, changes: %{ - content: "this does not get replaced", + content: content, source: %{ - "content" => content, "mediaType" => "text/x.misskeymarkdown" } } @@ -114,7 +113,9 @@ test "a misskey MFM status with a content field should work and be linked", _ do "@full_tag_remote_user" assert content =~ "@oops_not_a_mention" - assert content =~ "$[jelly mfm goes here]

## aaa" + + assert content =~ + "mfm goes here

aaa" end test "a misskey MFM status with a _misskey_content field should work and be linked", _ do @@ -133,9 +134,10 @@ test "a misskey MFM status with a _misskey_content field should work and be link %{ valid?: true, changes: %{ + content: content, source: %{ - "content" => content, - "mediaType" => "text/x.misskeymarkdown" + "mediaType" => "text/x.misskeymarkdown", + "content" => "@akkoma_user linkifylink #dancedance $[jelly mfm goes here] \n\n## aaa" } } } = changes From a8f8ecce31238309e677802ad57d6521d58b814d Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Thu, 18 Aug 2022 04:23:07 +0100 Subject: [PATCH 07/44] add changelog entry --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b32ae2e16..41b352f21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - support for fedibird-fe, and non-breaking API parity for it to function +### Changed +- MFM parsing is now done on the backend by a modified version of ilja's parser -> https://akkoma.dev/AkkomaGang/mfm-parser + ### Fixed - Compatibility with latest meilisearch - Resolution of nested mix tasks (i.e search.meilisearch) in OTP releases From 22333f13e834f163f32574e89aed4d3bed5f0684 Mon Sep 17 00:00:00 2001 From: Karen Konou Date: Thu, 18 Aug 2022 12:43:20 +0200 Subject: [PATCH 08/44] Update OTP systemd service --- .../{pleroma.service => akkoma.service} | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) rename rel/files/installation/{pleroma.service => akkoma.service} (63%) diff --git a/rel/files/installation/pleroma.service b/rel/files/installation/akkoma.service similarity index 63% rename from rel/files/installation/pleroma.service rename to rel/files/installation/akkoma.service index e47cf58dc..0719dbf73 100644 --- a/rel/files/installation/pleroma.service +++ b/rel/files/installation/akkoma.service @@ -1,27 +1,27 @@ [Unit] -Description=Pleroma social network +Description=Akkoma social network After=network.target postgresql.service nginx.service [Service] KillMode=process Restart=on-failure -; Name of the user that runs the Pleroma service. -User=pleroma +; Name of the user that runs the Akkoma service. +User=akkoma ; Make sure that all paths fit your installation. -; Path to the home directory of the user running the Pleroma service. -Environment="HOME=/opt/pleroma" -; Path to the folder containing the Pleroma installation. -WorkingDirectory=/opt/pleroma -; Path to the Pleroma binary. -ExecStart=/opt/pleroma/bin/pleroma start -ExecStop=/opt/pleroma/bin/pleroma stop +; Path to the home directory of the user running the Akkoma service. +Environment="HOME=/opt/akkoma" +; Path to the folder containing the Akkoma installation. +WorkingDirectory=/opt/akkoma +; Path to the Mix binary. +ExecStart=/opt/akkoma/bin/pleroma start +ExecStop=/opt/akkoma/bin/pleroma stop ; Some security directives. ; Use private /tmp and /var/tmp folders inside a new file system namespace, which are discarded after the process stops. PrivateTmp=true -; The /home, /root, and /run/user folders can not be accessed by this service anymore. If your Pleroma user has its home folder in one of the restricted places, or use one of these folders as its working directory, you have to set this to false. +; The /home, /root, and /run/user folders can not be accessed by this service anymore. If your Akkoma user has its home folder in one of the restricted places, or use one of these folders as its working directory, you have to set this to false. ProtectHome=true ; Mount /usr, /boot, and /etc as read-only for processes invoked by this service. ProtectSystem=full From ffbf8304e057631f4d8bdf56cb87f8bacafb751e Mon Sep 17 00:00:00 2001 From: Norm Date: Thu, 18 Aug 2022 23:13:09 +0000 Subject: [PATCH 09/44] Update OTP OpenRC service This makes the paths match that of the OTP install guide on OpenRC distros. --- rel/files/installation/init.d/{pleroma => akkoma} | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename rel/files/installation/init.d/{pleroma => akkoma} (70%) diff --git a/rel/files/installation/init.d/pleroma b/rel/files/installation/init.d/akkoma similarity index 70% rename from rel/files/installation/init.d/pleroma rename to rel/files/installation/init.d/akkoma index dea1db26c..ea6ea3580 100755 --- a/rel/files/installation/init.d/pleroma +++ b/rel/files/installation/init.d/akkoma @@ -3,17 +3,17 @@ supervisor=supervise-daemon # Requires OpenRC >= 0.35 -directory=/opt/pleroma +directory=/opt/akkoma -command=/opt/pleroma/bin/pleroma +command=/opt/akkoma/bin/pleroma command_args="start" -command_user=pleroma +command_user=akkoma command_background=1 # Ask process to terminate within 30 seconds, otherwise kill it retry="SIGTERM/30/SIGKILL/5" -pidfile="/var/run/pleroma.pid" +pidfile="/var/run/akkoma.pid" depend() { want nginx From 429e2ac832a874ae8ba8a9c116da61a6273c8a87 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Sun, 21 Aug 2022 14:46:52 +0000 Subject: [PATCH 10/44] oauth2 fixes (#177) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/177 --- lib/pleroma/helpers/auth_helper.ex | 18 ----- .../controllers/auth_controller.ex | 11 +-- lib/pleroma/web/o_auth/o_auth_controller.ex | 77 ++++++++++--------- lib/pleroma/web/o_auth/token.ex | 6 ++ lib/pleroma/web/o_auth/token/query.ex | 8 +- lib/pleroma/web/plugs/o_auth_plug.ex | 13 +--- .../web/plugs/set_user_session_id_plug.ex | 18 ----- lib/pleroma/web/router.ex | 5 +- mix.exs | 4 +- .../web/o_auth/o_auth_controller_test.exs | 53 ++----------- test/pleroma/web/plugs/o_auth_plug_test.exs | 56 -------------- .../plugs/set_user_session_id_plug_test.exs | 43 ----------- 12 files changed, 69 insertions(+), 243 deletions(-) delete mode 100644 lib/pleroma/web/plugs/set_user_session_id_plug.ex delete mode 100644 test/pleroma/web/plugs/set_user_session_id_plug_test.exs diff --git a/lib/pleroma/helpers/auth_helper.ex b/lib/pleroma/helpers/auth_helper.ex index 13e4c8158..d56f6f461 100644 --- a/lib/pleroma/helpers/auth_helper.ex +++ b/lib/pleroma/helpers/auth_helper.ex @@ -4,12 +4,9 @@ defmodule Pleroma.Helpers.AuthHelper do alias Pleroma.Web.Plugs.OAuthScopesPlug - alias Plug.Conn import Plug.Conn - @oauth_token_session_key :oauth_token - @doc """ Skips OAuth permissions (scopes) checks, assigns nil `:token`. Intended to be used with explicit authentication and only when OAuth token cannot be determined. @@ -28,19 +25,4 @@ def drop_auth_info(conn) do |> assign(:token, nil) |> put_private(:authentication_ignored, true) end - - @doc "Gets OAuth token string from session" - def get_session_token(%Conn{} = conn) do - get_session(conn, @oauth_token_session_key) - end - - @doc "Updates OAuth token string in session" - def put_session_token(%Conn{} = conn, token) when is_binary(token) do - put_session(conn, @oauth_token_session_key, token) - end - - @doc "Deletes OAuth token string from session" - def delete_session_token(%Conn{} = conn) do - delete_session(conn, @oauth_token_session_key) - end end diff --git a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex index 4920d65da..f415e5931 100644 --- a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex @@ -7,7 +7,6 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do import Pleroma.Web.ControllerHelper, only: [json_response: 3] - alias Pleroma.Helpers.AuthHelper alias Pleroma.Helpers.UriHelper alias Pleroma.User alias Pleroma.Web.OAuth.App @@ -34,7 +33,6 @@ def login(conn, %{"code" => auth_token} = params) do |> UriHelper.modify_uri_params(%{"access_token" => oauth_token.token}) conn - |> AuthHelper.put_session_token(oauth_token.token) |> redirect(to: redirect_to) else _ -> redirect_to_oauth_form(conn, params) @@ -42,9 +40,9 @@ def login(conn, %{"code" => auth_token} = params) do end def login(conn, params) do - with %{assigns: %{user: %User{}, token: %Token{app_id: app_id}}} <- conn, + with %{assigns: %{user: %User{}, token: %Token{app_id: app_id, token: token}}} <- conn, {:ok, %{id: ^app_id}} <- local_mastofe_app() do - redirect(conn, to: local_mastodon_post_login_path(conn)) + redirect(conn, to: local_mastodon_post_login_path(conn) <> "?access_token=#{token}") else _ -> redirect_to_oauth_form(conn, params) end @@ -68,9 +66,8 @@ defp redirect_to_oauth_form(conn, _params) do def logout(conn, _) do conn = with %{assigns: %{token: %Token{} = oauth_token}} <- conn, - session_token = AuthHelper.get_session_token(conn), - {:ok, %Token{token: ^session_token}} <- RevokeToken.revoke(oauth_token) do - AuthHelper.delete_session_token(conn) + {:ok, %Token{token: _session_token}} <- RevokeToken.revoke(oauth_token) do + conn else _ -> conn end diff --git a/lib/pleroma/web/o_auth/o_auth_controller.ex b/lib/pleroma/web/o_auth/o_auth_controller.ex index 358120fe6..43536f95d 100644 --- a/lib/pleroma/web/o_auth/o_auth_controller.ex +++ b/lib/pleroma/web/o_auth/o_auth_controller.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.OAuth.OAuthController do use Pleroma.Web, :controller - alias Pleroma.Helpers.AuthHelper alias Pleroma.Helpers.UriHelper alias Pleroma.Maps alias Pleroma.MFA @@ -77,33 +76,46 @@ defp do_authorize(%Plug.Conn{} = conn, params) do available_scopes = (app && app.scopes) || [] scopes = Scopes.fetch_scopes(params, available_scopes) - user = - with %{assigns: %{user: %User{} = user}} <- conn do - user - else - _ -> nil - end + # if we already have a token for this specific setup, we can use that + with false <- Params.truthy_param?(params["force_login"]), + %App{} <- app, + {:ok, _} <- Scopes.validate(scopes, app.scopes), + {:ok, %Token{} = token} <- Token.get_by_app(app) do + token = Repo.preload(token, :app) - scopes = - if scopes == [] do - available_scopes - else - scopes - end + conn + |> assign(:token, token) + |> handle_existing_authorization(params) + else + _ -> + user = + with %{assigns: %{user: %User{} = user}} <- conn do + user + else + _ -> nil + end - # Note: `params` might differ from `conn.params`; use `@params` not `@conn.params` in template - render(conn, Authenticator.auth_template(), %{ - user: user, - app: app && Map.delete(app, :client_secret), - response_type: params["response_type"], - client_id: params["client_id"], - available_scopes: available_scopes, - scopes: scopes, - redirect_uri: params["redirect_uri"], - state: params["state"], - params: params, - view_module: OAuthView - }) + scopes = + if scopes == [] do + available_scopes + else + scopes + end + + # Note: `params` might differ from `conn.params`; use `@params` not `@conn.params` in template + render(conn, Authenticator.auth_template(), %{ + user: user, + app: app && Map.delete(app, :client_secret), + response_type: params["response_type"], + client_id: params["client_id"], + available_scopes: available_scopes, + scopes: scopes, + redirect_uri: params["redirect_uri"], + state: params["state"], + params: params, + view_module: OAuthView + }) + end end defp handle_existing_authorization( @@ -318,9 +330,8 @@ def token_exchange(%Plug.Conn{} = conn, %{"grant_type" => "client_credentials"} # Bad request def token_exchange(%Plug.Conn{} = conn, params), do: bad_request(conn, params) - def after_token_exchange(%Plug.Conn{} = conn, %{token: token} = view_params) do + def after_token_exchange(%Plug.Conn{} = conn, %{token: _token} = view_params) do conn - |> AuthHelper.put_session_token(token.token) |> json(OAuthView.render("token.json", view_params)) end @@ -379,15 +390,7 @@ defp handle_token_exchange_error(%Plug.Conn{} = conn, _error) do def token_revoke(%Plug.Conn{} = conn, %{"token" => token}) do with {:ok, %Token{} = oauth_token} <- Token.get_by_token(token), - {:ok, oauth_token} <- RevokeToken.revoke(oauth_token) do - conn = - with session_token = AuthHelper.get_session_token(conn), - %Token{token: ^session_token} <- oauth_token do - AuthHelper.delete_session_token(conn) - else - _ -> conn - end - + {:ok, _oauth_token} <- RevokeToken.revoke(oauth_token) do json(conn, %{}) else _error -> diff --git a/lib/pleroma/web/o_auth/token.ex b/lib/pleroma/web/o_auth/token.ex index 9d69e9db4..08c8cd298 100644 --- a/lib/pleroma/web/o_auth/token.ex +++ b/lib/pleroma/web/o_auth/token.ex @@ -39,6 +39,12 @@ def get_by_token(token) do |> Repo.find_resource() end + def get_by_app(%App{} = app) do + app.id + |> Query.get_unexpired_by_app() + |> Repo.find_resource() + end + @doc "Gets token for app by access token" @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found} def get_by_token(%App{id: app_id} = _app, token) do diff --git a/lib/pleroma/web/o_auth/token/query.ex b/lib/pleroma/web/o_auth/token/query.ex index d16a759d8..8edfcf5d7 100644 --- a/lib/pleroma/web/o_auth/token/query.ex +++ b/lib/pleroma/web/o_auth/token/query.ex @@ -23,9 +23,15 @@ def get_by_token(query \\ Token, token) do from(q in query, where: q.token == ^token) end + @spec get_unexpired_by_app(query, String.t()) :: query + def get_unexpired_by_app(query \\ Token, app_id) do + time = NaiveDateTime.utc_now() + from(q in query, where: q.app_id == ^app_id and q.valid_until > ^time, limit: 1) + end + @spec get_by_app(query, String.t()) :: query def get_by_app(query \\ Token, app_id) do - from(q in query, where: q.app_id == ^app_id) + from(q in query, where: q.app_id == ^app_id, limit: 1) end @spec get_by_id(query, String.t()) :: query diff --git a/lib/pleroma/web/plugs/o_auth_plug.ex b/lib/pleroma/web/plugs/o_auth_plug.ex index 5e06ac3f6..29b3316b3 100644 --- a/lib/pleroma/web/plugs/o_auth_plug.ex +++ b/lib/pleroma/web/plugs/o_auth_plug.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.Plugs.OAuthPlug do import Plug.Conn import Ecto.Query - alias Pleroma.Helpers.AuthHelper alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.OAuth.App @@ -18,8 +17,6 @@ defmodule Pleroma.Web.Plugs.OAuthPlug do def init(options), do: options - def call(%{assigns: %{user: %User{}}} = conn, _), do: conn - def call(conn, _) do with {:ok, token_str} <- fetch_token_str(conn) do with {:ok, user, user_token} <- fetch_user_and_token(token_str), @@ -82,7 +79,7 @@ defp fetch_token_str(%Plug.Conn{} = conn) do with {:ok, token} <- fetch_token_str(headers) do {:ok, token} else - _ -> fetch_token_from_session(conn) + _ -> :no_token_found end end @@ -96,12 +93,4 @@ defp fetch_token_str([token | tail]) do end defp fetch_token_str([]), do: :no_token_found - - @spec fetch_token_from_session(Plug.Conn.t()) :: :no_token_found | {:ok, String.t()} - defp fetch_token_from_session(conn) do - case AuthHelper.get_session_token(conn) do - nil -> :no_token_found - token -> {:ok, token} - end - end end diff --git a/lib/pleroma/web/plugs/set_user_session_id_plug.ex b/lib/pleroma/web/plugs/set_user_session_id_plug.ex deleted file mode 100644 index a1cfa0915..000000000 --- a/lib/pleroma/web/plugs/set_user_session_id_plug.ex +++ /dev/null @@ -1,18 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Plugs.SetUserSessionIdPlug do - alias Pleroma.Helpers.AuthHelper - alias Pleroma.Web.OAuth.Token - - def init(opts) do - opts - end - - def call(%{assigns: %{token: %Token{} = oauth_token}} = conn, _) do - AuthHelper.put_session_token(conn, oauth_token.token) - end - - def call(conn, _), do: conn -end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 647d99278..f2d6b0aff 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -57,7 +57,6 @@ defmodule Pleroma.Web.Router do pipeline :after_auth do plug(Pleroma.Web.Plugs.UserEnabledPlug) - plug(Pleroma.Web.Plugs.SetUserSessionIdPlug) plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug) plug(Pleroma.Web.Plugs.UserTrackingPlug) end @@ -793,10 +792,8 @@ defmodule Pleroma.Web.Router do get("/web/login", MastodonAPI.AuthController, :login) delete("/auth/sign_out", MastodonAPI.AuthController, :logout) - - post("/auth/password", MastodonAPI.AuthController, :password_reset) - get("/web/*path", MastoFEController, :index) + post("/auth/password", MastodonAPI.AuthController, :password_reset) get("/embed/:id", EmbedController, :show) end diff --git a/mix.exs b/mix.exs index e7f491997..170276b0a 100644 --- a/mix.exs +++ b/mix.exs @@ -131,7 +131,7 @@ defp deps do {:trailing_format_plug, "~> 0.0.7"}, {:fast_sanitize, "~> 0.2.3"}, {:html_entities, "~> 0.5", override: true}, - {:phoenix_html, "~> 3.1", override: true}, + {:phoenix_html, "~> 3.0", override: true}, {:calendar, "~> 1.0"}, {:cachex, "~> 3.4"}, {:poison, "~> 3.0", override: true}, @@ -152,7 +152,7 @@ defp deps do ref: "f75cd55325e33cbea198fb41fe41871392f8fb76"}, {:cors_plug, "~> 2.0"}, {:web_push_encryption, "~> 0.3.1"}, - {:swoosh, "~> 1.0"}, + {:swoosh, "~> 1.3"}, {:phoenix_swoosh, "~> 0.3"}, {:gen_smtp, "~> 0.13"}, {:ex_syslogger, "~> 1.4"}, diff --git a/test/pleroma/web/o_auth/o_auth_controller_test.exs b/test/pleroma/web/o_auth/o_auth_controller_test.exs index 0fdd5b8e9..4e197a485 100644 --- a/test/pleroma/web/o_auth/o_auth_controller_test.exs +++ b/test/pleroma/web/o_auth/o_auth_controller_test.exs @@ -7,7 +7,6 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do import Pleroma.Factory - alias Pleroma.Helpers.AuthHelper alias Pleroma.MFA alias Pleroma.MFA.TOTP alias Pleroma.Repo @@ -456,7 +455,7 @@ test "renders authentication page if user is already authenticated but `force_lo conn = conn - |> AuthHelper.put_session_token(token.token) + |> put_req_header("authorization", "Bearer #{token.token}") |> get( "/oauth/authorize", %{ @@ -480,7 +479,7 @@ test "renders authentication page if user is already authenticated but user requ conn = conn - |> AuthHelper.put_session_token(token.token) + |> put_req_header("authorization", "Bearer #{token.token}") |> get( "/oauth/authorize", %{ @@ -503,7 +502,7 @@ test "with existing authentication and non-OOB `redirect_uri`, redirects to app conn = conn - |> AuthHelper.put_session_token(token.token) + |> put_req_header("authorization", "Bearer #{token.token}") |> get( "/oauth/authorize", %{ @@ -529,7 +528,7 @@ test "with existing authentication and unlisted non-OOB `redirect_uri`, redirect conn = conn - |> AuthHelper.put_session_token(token.token) + |> put_req_header("authorization", "Bearer #{token.token}") |> get( "/oauth/authorize", %{ @@ -553,7 +552,7 @@ test "with existing authentication and OOB `redirect_uri`, redirects to app with conn = conn - |> AuthHelper.put_session_token(token.token) + |> put_req_header("authorization", "Bearer #{token.token}") |> get( "/oauth/authorize", %{ @@ -611,41 +610,6 @@ test "redirects with oauth authorization, " <> end end - test "authorize from cookie" do - user = insert(:user) - app = insert(:oauth_app) - oauth_token = insert(:oauth_token, user: user, app: app) - redirect_uri = OAuthController.default_redirect_uri(app) - - conn = - build_conn() - |> Plug.Session.call(Plug.Session.init(@session_opts)) - |> fetch_session() - |> AuthHelper.put_session_token(oauth_token.token) - |> post( - "/oauth/authorize", - %{ - "authorization" => %{ - "name" => user.nickname, - "client_id" => app.client_id, - "redirect_uri" => redirect_uri, - "scope" => app.scopes, - "state" => "statepassed" - } - } - ) - - target = redirected_to(conn) - assert target =~ redirect_uri - - query = URI.parse(target).query |> URI.query_decoder() |> Map.new() - - assert %{"state" => "statepassed", "code" => code} = query - auth = Repo.get_by(Authorization, token: code) - assert auth - assert auth.scopes == app.scopes - end - test "redirect to on two-factor auth page" do otp_secret = TOTP.generate_secret() @@ -1218,6 +1182,7 @@ test "issues a new token if token expired" do response = build_conn() + |> put_req_header("authorization", "Bearer #{access_token.token}") |> post("/oauth/token", %{ "grant_type" => "refresh_token", "refresh_token" => access_token.refresh_token, @@ -1267,12 +1232,11 @@ test "when authenticated with request token, revokes it and clears it from sessi build_conn() |> Plug.Session.call(Plug.Session.init(@session_opts)) |> fetch_session() - |> AuthHelper.put_session_token(oauth_token.token) + |> put_req_header("authorization", "Bearer #{oauth_token.token}") |> post("/oauth/revoke", %{"token" => oauth_token.token}) assert json_response(conn, 200) - refute AuthHelper.get_session_token(conn) assert Token.get_by_token(oauth_token.token) == {:error, :not_found} end @@ -1286,12 +1250,11 @@ test "if request is authenticated with a different token, " <> build_conn() |> Plug.Session.call(Plug.Session.init(@session_opts)) |> fetch_session() - |> AuthHelper.put_session_token(oauth_token.token) + |> put_req_header("authorization", "Bearer #{oauth_token.token}") |> post("/oauth/revoke", %{"token" => other_app_oauth_token.token}) assert json_response(conn, 200) - assert AuthHelper.get_session_token(conn) == oauth_token.token assert Token.get_by_token(other_app_oauth_token.token) == {:error, :not_found} end diff --git a/test/pleroma/web/plugs/o_auth_plug_test.exs b/test/pleroma/web/plugs/o_auth_plug_test.exs index 9e4be5559..caabfc1cb 100644 --- a/test/pleroma/web/plugs/o_auth_plug_test.exs +++ b/test/pleroma/web/plugs/o_auth_plug_test.exs @@ -5,11 +5,8 @@ defmodule Pleroma.Web.Plugs.OAuthPlugTest do use Pleroma.Web.ConnCase, async: true - alias Pleroma.Helpers.AuthHelper alias Pleroma.Web.OAuth.Token - alias Pleroma.Web.OAuth.Token.Strategy.Revoke alias Pleroma.Web.Plugs.OAuthPlug - alias Plug.Session import Pleroma.Factory @@ -72,57 +69,4 @@ test "with invalid token, it does not assign the user", %{conn: conn} do refute conn.assigns[:user] end - - describe "with :oauth_token in session, " do - setup %{token: oauth_token, conn: conn} do - session_opts = [ - store: :cookie, - key: "_test", - signing_salt: "cooldude" - ] - - conn = - conn - |> Session.call(Session.init(session_opts)) - |> fetch_session() - |> AuthHelper.put_session_token(oauth_token.token) - - %{conn: conn} - end - - test "if session-stored token matches a valid OAuth token, assigns :user and :token", %{ - conn: conn, - user: user, - token: oauth_token - } do - conn = OAuthPlug.call(conn, %{}) - - assert conn.assigns.user && conn.assigns.user.id == user.id - assert conn.assigns.token && conn.assigns.token.id == oauth_token.id - end - - test "if session-stored token matches an expired OAuth token, does nothing", %{ - conn: conn, - token: oauth_token - } do - expired_valid_until = NaiveDateTime.add(NaiveDateTime.utc_now(), -3600 * 24, :second) - - oauth_token - |> Ecto.Changeset.change(valid_until: expired_valid_until) - |> Pleroma.Repo.update() - - ret_conn = OAuthPlug.call(conn, %{}) - assert ret_conn == conn - end - - test "if session-stored token matches a revoked OAuth token, does nothing", %{ - conn: conn, - token: oauth_token - } do - Revoke.revoke(oauth_token) - - ret_conn = OAuthPlug.call(conn, %{}) - assert ret_conn == conn - end - end end diff --git a/test/pleroma/web/plugs/set_user_session_id_plug_test.exs b/test/pleroma/web/plugs/set_user_session_id_plug_test.exs deleted file mode 100644 index 9814c80d8..000000000 --- a/test/pleroma/web/plugs/set_user_session_id_plug_test.exs +++ /dev/null @@ -1,43 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Plugs.SetUserSessionIdPlugTest do - use Pleroma.Web.ConnCase, async: true - - alias Pleroma.Helpers.AuthHelper - alias Pleroma.Web.Plugs.SetUserSessionIdPlug - - setup %{conn: conn} do - session_opts = [ - store: :cookie, - key: "_test", - signing_salt: "cooldude" - ] - - conn = - conn - |> Plug.Session.call(Plug.Session.init(session_opts)) - |> fetch_session() - - %{conn: conn} - end - - test "doesn't do anything if the user isn't set", %{conn: conn} do - ret_conn = SetUserSessionIdPlug.call(conn, %{}) - - assert ret_conn == conn - end - - test "sets session token basing on :token assign", %{conn: conn} do - %{user: user, token: oauth_token} = oauth_access(["read"]) - - ret_conn = - conn - |> assign(:user, user) - |> assign(:token, oauth_token) - |> SetUserSessionIdPlug.call(%{}) - - assert AuthHelper.get_session_token(ret_conn) == oauth_token.token - end -end From d72f9e39d9f76ee8bbd26c068b2870ea945705b7 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Sun, 21 Aug 2022 15:17:01 +0000 Subject: [PATCH 11/44] add visibility check on quote (#178) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/178 --- .../web/mastodon_api/views/status_view.ex | 10 +++++-- .../mastodon_api/views/status_view_test.exs | 28 +++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index d099c4901..d838c4673 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -623,15 +623,19 @@ defp build_image_url(_, _), do: nil defp maybe_render_quote(nil, _), do: nil defp maybe_render_quote(quote, opts) do - if opts[:do_not_recurse] || !visible_for_user?(quote, opts[:for]) do - nil - else + with %User{} = quoted_user <- User.get_cached_by_ap_id(quote.actor), + false <- Map.get(opts, :do_not_recurse, false), + true <- visible_for_user?(quote, opts[:for]), + false <- User.blocks?(opts[:for], quoted_user), + false <- User.mutes?(opts[:for], quoted_user) do opts = opts |> Map.put(:activity, quote) |> Map.put(:do_not_recurse, true) render("show.json", opts) + else + _ -> nil end end end diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index a6f8f3fc8..f46dded7c 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -428,6 +428,34 @@ test "a quote that we can't resolve" do assert is_nil(status.quote) end + test "a quote from a user we block" do + user = insert(:user) + other_user = insert(:user) + blocked_user = insert(:user) + + {:ok, _relationship} = User.block(user, blocked_user) + + {:ok, activity} = CommonAPI.post(blocked_user, %{status: ":< i am ANGERY"}) + {:ok, quote_activity} = CommonAPI.post(other_user, %{status: "hehe", quote_id: activity.id}) + + status = StatusView.render("show.json", %{activity: quote_activity, for: user}) + assert is_nil(status.quote) + end + + test "a quote from a user we mute" do + user = insert(:user) + other_user = insert(:user) + blocked_user = insert(:user) + + {:ok, _relationship} = User.mute(user, blocked_user) + + {:ok, activity} = CommonAPI.post(blocked_user, %{status: ":< i am ANGERY"}) + {:ok, quote_activity} = CommonAPI.post(other_user, %{status: "hehe", quote_id: activity.id}) + + status = StatusView.render("show.json", %{activity: quote_activity, for: user}) + assert is_nil(status.quote) + end + test "contains mentions" do user = insert(:user) mentioned = insert(:user) From b0130bfa7b420550aa7acba6a88c71aa22c51246 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sun, 21 Aug 2022 16:22:15 +0100 Subject: [PATCH 12/44] Revert "oauth2 fixes (#177)" This reverts commit 429e2ac832a874ae8ba8a9c116da61a6273c8a87. --- lib/pleroma/helpers/auth_helper.ex | 18 +++++ .../controllers/auth_controller.ex | 11 ++- lib/pleroma/web/o_auth/o_auth_controller.ex | 77 +++++++++---------- lib/pleroma/web/o_auth/token.ex | 6 -- lib/pleroma/web/o_auth/token/query.ex | 8 +- lib/pleroma/web/plugs/o_auth_plug.ex | 13 +++- .../web/plugs/set_user_session_id_plug.ex | 18 +++++ lib/pleroma/web/router.ex | 5 +- mix.exs | 4 +- .../web/o_auth/o_auth_controller_test.exs | 53 +++++++++++-- test/pleroma/web/plugs/o_auth_plug_test.exs | 56 ++++++++++++++ .../plugs/set_user_session_id_plug_test.exs | 43 +++++++++++ 12 files changed, 243 insertions(+), 69 deletions(-) create mode 100644 lib/pleroma/web/plugs/set_user_session_id_plug.ex create mode 100644 test/pleroma/web/plugs/set_user_session_id_plug_test.exs diff --git a/lib/pleroma/helpers/auth_helper.ex b/lib/pleroma/helpers/auth_helper.ex index d56f6f461..13e4c8158 100644 --- a/lib/pleroma/helpers/auth_helper.ex +++ b/lib/pleroma/helpers/auth_helper.ex @@ -4,9 +4,12 @@ defmodule Pleroma.Helpers.AuthHelper do alias Pleroma.Web.Plugs.OAuthScopesPlug + alias Plug.Conn import Plug.Conn + @oauth_token_session_key :oauth_token + @doc """ Skips OAuth permissions (scopes) checks, assigns nil `:token`. Intended to be used with explicit authentication and only when OAuth token cannot be determined. @@ -25,4 +28,19 @@ def drop_auth_info(conn) do |> assign(:token, nil) |> put_private(:authentication_ignored, true) end + + @doc "Gets OAuth token string from session" + def get_session_token(%Conn{} = conn) do + get_session(conn, @oauth_token_session_key) + end + + @doc "Updates OAuth token string in session" + def put_session_token(%Conn{} = conn, token) when is_binary(token) do + put_session(conn, @oauth_token_session_key, token) + end + + @doc "Deletes OAuth token string from session" + def delete_session_token(%Conn{} = conn) do + delete_session(conn, @oauth_token_session_key) + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex index f415e5931..4920d65da 100644 --- a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do import Pleroma.Web.ControllerHelper, only: [json_response: 3] + alias Pleroma.Helpers.AuthHelper alias Pleroma.Helpers.UriHelper alias Pleroma.User alias Pleroma.Web.OAuth.App @@ -33,6 +34,7 @@ def login(conn, %{"code" => auth_token} = params) do |> UriHelper.modify_uri_params(%{"access_token" => oauth_token.token}) conn + |> AuthHelper.put_session_token(oauth_token.token) |> redirect(to: redirect_to) else _ -> redirect_to_oauth_form(conn, params) @@ -40,9 +42,9 @@ def login(conn, %{"code" => auth_token} = params) do end def login(conn, params) do - with %{assigns: %{user: %User{}, token: %Token{app_id: app_id, token: token}}} <- conn, + with %{assigns: %{user: %User{}, token: %Token{app_id: app_id}}} <- conn, {:ok, %{id: ^app_id}} <- local_mastofe_app() do - redirect(conn, to: local_mastodon_post_login_path(conn) <> "?access_token=#{token}") + redirect(conn, to: local_mastodon_post_login_path(conn)) else _ -> redirect_to_oauth_form(conn, params) end @@ -66,8 +68,9 @@ defp redirect_to_oauth_form(conn, _params) do def logout(conn, _) do conn = with %{assigns: %{token: %Token{} = oauth_token}} <- conn, - {:ok, %Token{token: _session_token}} <- RevokeToken.revoke(oauth_token) do - conn + session_token = AuthHelper.get_session_token(conn), + {:ok, %Token{token: ^session_token}} <- RevokeToken.revoke(oauth_token) do + AuthHelper.delete_session_token(conn) else _ -> conn end diff --git a/lib/pleroma/web/o_auth/o_auth_controller.ex b/lib/pleroma/web/o_auth/o_auth_controller.ex index 43536f95d..358120fe6 100644 --- a/lib/pleroma/web/o_auth/o_auth_controller.ex +++ b/lib/pleroma/web/o_auth/o_auth_controller.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do use Pleroma.Web, :controller + alias Pleroma.Helpers.AuthHelper alias Pleroma.Helpers.UriHelper alias Pleroma.Maps alias Pleroma.MFA @@ -76,46 +77,33 @@ defp do_authorize(%Plug.Conn{} = conn, params) do available_scopes = (app && app.scopes) || [] scopes = Scopes.fetch_scopes(params, available_scopes) - # if we already have a token for this specific setup, we can use that - with false <- Params.truthy_param?(params["force_login"]), - %App{} <- app, - {:ok, _} <- Scopes.validate(scopes, app.scopes), - {:ok, %Token{} = token} <- Token.get_by_app(app) do - token = Repo.preload(token, :app) + user = + with %{assigns: %{user: %User{} = user}} <- conn do + user + else + _ -> nil + end - conn - |> assign(:token, token) - |> handle_existing_authorization(params) - else - _ -> - user = - with %{assigns: %{user: %User{} = user}} <- conn do - user - else - _ -> nil - end + scopes = + if scopes == [] do + available_scopes + else + scopes + end - scopes = - if scopes == [] do - available_scopes - else - scopes - end - - # Note: `params` might differ from `conn.params`; use `@params` not `@conn.params` in template - render(conn, Authenticator.auth_template(), %{ - user: user, - app: app && Map.delete(app, :client_secret), - response_type: params["response_type"], - client_id: params["client_id"], - available_scopes: available_scopes, - scopes: scopes, - redirect_uri: params["redirect_uri"], - state: params["state"], - params: params, - view_module: OAuthView - }) - end + # Note: `params` might differ from `conn.params`; use `@params` not `@conn.params` in template + render(conn, Authenticator.auth_template(), %{ + user: user, + app: app && Map.delete(app, :client_secret), + response_type: params["response_type"], + client_id: params["client_id"], + available_scopes: available_scopes, + scopes: scopes, + redirect_uri: params["redirect_uri"], + state: params["state"], + params: params, + view_module: OAuthView + }) end defp handle_existing_authorization( @@ -330,8 +318,9 @@ def token_exchange(%Plug.Conn{} = conn, %{"grant_type" => "client_credentials"} # Bad request def token_exchange(%Plug.Conn{} = conn, params), do: bad_request(conn, params) - def after_token_exchange(%Plug.Conn{} = conn, %{token: _token} = view_params) do + def after_token_exchange(%Plug.Conn{} = conn, %{token: token} = view_params) do conn + |> AuthHelper.put_session_token(token.token) |> json(OAuthView.render("token.json", view_params)) end @@ -390,7 +379,15 @@ defp handle_token_exchange_error(%Plug.Conn{} = conn, _error) do def token_revoke(%Plug.Conn{} = conn, %{"token" => token}) do with {:ok, %Token{} = oauth_token} <- Token.get_by_token(token), - {:ok, _oauth_token} <- RevokeToken.revoke(oauth_token) do + {:ok, oauth_token} <- RevokeToken.revoke(oauth_token) do + conn = + with session_token = AuthHelper.get_session_token(conn), + %Token{token: ^session_token} <- oauth_token do + AuthHelper.delete_session_token(conn) + else + _ -> conn + end + json(conn, %{}) else _error -> diff --git a/lib/pleroma/web/o_auth/token.ex b/lib/pleroma/web/o_auth/token.ex index 08c8cd298..9d69e9db4 100644 --- a/lib/pleroma/web/o_auth/token.ex +++ b/lib/pleroma/web/o_auth/token.ex @@ -39,12 +39,6 @@ def get_by_token(token) do |> Repo.find_resource() end - def get_by_app(%App{} = app) do - app.id - |> Query.get_unexpired_by_app() - |> Repo.find_resource() - end - @doc "Gets token for app by access token" @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found} def get_by_token(%App{id: app_id} = _app, token) do diff --git a/lib/pleroma/web/o_auth/token/query.ex b/lib/pleroma/web/o_auth/token/query.ex index 8edfcf5d7..d16a759d8 100644 --- a/lib/pleroma/web/o_auth/token/query.ex +++ b/lib/pleroma/web/o_auth/token/query.ex @@ -23,15 +23,9 @@ def get_by_token(query \\ Token, token) do from(q in query, where: q.token == ^token) end - @spec get_unexpired_by_app(query, String.t()) :: query - def get_unexpired_by_app(query \\ Token, app_id) do - time = NaiveDateTime.utc_now() - from(q in query, where: q.app_id == ^app_id and q.valid_until > ^time, limit: 1) - end - @spec get_by_app(query, String.t()) :: query def get_by_app(query \\ Token, app_id) do - from(q in query, where: q.app_id == ^app_id, limit: 1) + from(q in query, where: q.app_id == ^app_id) end @spec get_by_id(query, String.t()) :: query diff --git a/lib/pleroma/web/plugs/o_auth_plug.ex b/lib/pleroma/web/plugs/o_auth_plug.ex index 29b3316b3..5e06ac3f6 100644 --- a/lib/pleroma/web/plugs/o_auth_plug.ex +++ b/lib/pleroma/web/plugs/o_auth_plug.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.Plugs.OAuthPlug do import Plug.Conn import Ecto.Query + alias Pleroma.Helpers.AuthHelper alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.OAuth.App @@ -17,6 +18,8 @@ defmodule Pleroma.Web.Plugs.OAuthPlug do def init(options), do: options + def call(%{assigns: %{user: %User{}}} = conn, _), do: conn + def call(conn, _) do with {:ok, token_str} <- fetch_token_str(conn) do with {:ok, user, user_token} <- fetch_user_and_token(token_str), @@ -79,7 +82,7 @@ defp fetch_token_str(%Plug.Conn{} = conn) do with {:ok, token} <- fetch_token_str(headers) do {:ok, token} else - _ -> :no_token_found + _ -> fetch_token_from_session(conn) end end @@ -93,4 +96,12 @@ defp fetch_token_str([token | tail]) do end defp fetch_token_str([]), do: :no_token_found + + @spec fetch_token_from_session(Plug.Conn.t()) :: :no_token_found | {:ok, String.t()} + defp fetch_token_from_session(conn) do + case AuthHelper.get_session_token(conn) do + nil -> :no_token_found + token -> {:ok, token} + end + end end diff --git a/lib/pleroma/web/plugs/set_user_session_id_plug.ex b/lib/pleroma/web/plugs/set_user_session_id_plug.ex new file mode 100644 index 000000000..a1cfa0915 --- /dev/null +++ b/lib/pleroma/web/plugs/set_user_session_id_plug.ex @@ -0,0 +1,18 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.SetUserSessionIdPlug do + alias Pleroma.Helpers.AuthHelper + alias Pleroma.Web.OAuth.Token + + def init(opts) do + opts + end + + def call(%{assigns: %{token: %Token{} = oauth_token}} = conn, _) do + AuthHelper.put_session_token(conn, oauth_token.token) + end + + def call(conn, _), do: conn +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index f2d6b0aff..647d99278 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -57,6 +57,7 @@ defmodule Pleroma.Web.Router do pipeline :after_auth do plug(Pleroma.Web.Plugs.UserEnabledPlug) + plug(Pleroma.Web.Plugs.SetUserSessionIdPlug) plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug) plug(Pleroma.Web.Plugs.UserTrackingPlug) end @@ -792,9 +793,11 @@ defmodule Pleroma.Web.Router do get("/web/login", MastodonAPI.AuthController, :login) delete("/auth/sign_out", MastodonAPI.AuthController, :logout) - get("/web/*path", MastoFEController, :index) + post("/auth/password", MastodonAPI.AuthController, :password_reset) + get("/web/*path", MastoFEController, :index) + get("/embed/:id", EmbedController, :show) end diff --git a/mix.exs b/mix.exs index 170276b0a..e7f491997 100644 --- a/mix.exs +++ b/mix.exs @@ -131,7 +131,7 @@ defp deps do {:trailing_format_plug, "~> 0.0.7"}, {:fast_sanitize, "~> 0.2.3"}, {:html_entities, "~> 0.5", override: true}, - {:phoenix_html, "~> 3.0", override: true}, + {:phoenix_html, "~> 3.1", override: true}, {:calendar, "~> 1.0"}, {:cachex, "~> 3.4"}, {:poison, "~> 3.0", override: true}, @@ -152,7 +152,7 @@ defp deps do ref: "f75cd55325e33cbea198fb41fe41871392f8fb76"}, {:cors_plug, "~> 2.0"}, {:web_push_encryption, "~> 0.3.1"}, - {:swoosh, "~> 1.3"}, + {:swoosh, "~> 1.0"}, {:phoenix_swoosh, "~> 0.3"}, {:gen_smtp, "~> 0.13"}, {:ex_syslogger, "~> 1.4"}, diff --git a/test/pleroma/web/o_auth/o_auth_controller_test.exs b/test/pleroma/web/o_auth/o_auth_controller_test.exs index 4e197a485..0fdd5b8e9 100644 --- a/test/pleroma/web/o_auth/o_auth_controller_test.exs +++ b/test/pleroma/web/o_auth/o_auth_controller_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do import Pleroma.Factory + alias Pleroma.Helpers.AuthHelper alias Pleroma.MFA alias Pleroma.MFA.TOTP alias Pleroma.Repo @@ -455,7 +456,7 @@ test "renders authentication page if user is already authenticated but `force_lo conn = conn - |> put_req_header("authorization", "Bearer #{token.token}") + |> AuthHelper.put_session_token(token.token) |> get( "/oauth/authorize", %{ @@ -479,7 +480,7 @@ test "renders authentication page if user is already authenticated but user requ conn = conn - |> put_req_header("authorization", "Bearer #{token.token}") + |> AuthHelper.put_session_token(token.token) |> get( "/oauth/authorize", %{ @@ -502,7 +503,7 @@ test "with existing authentication and non-OOB `redirect_uri`, redirects to app conn = conn - |> put_req_header("authorization", "Bearer #{token.token}") + |> AuthHelper.put_session_token(token.token) |> get( "/oauth/authorize", %{ @@ -528,7 +529,7 @@ test "with existing authentication and unlisted non-OOB `redirect_uri`, redirect conn = conn - |> put_req_header("authorization", "Bearer #{token.token}") + |> AuthHelper.put_session_token(token.token) |> get( "/oauth/authorize", %{ @@ -552,7 +553,7 @@ test "with existing authentication and OOB `redirect_uri`, redirects to app with conn = conn - |> put_req_header("authorization", "Bearer #{token.token}") + |> AuthHelper.put_session_token(token.token) |> get( "/oauth/authorize", %{ @@ -610,6 +611,41 @@ test "redirects with oauth authorization, " <> end end + test "authorize from cookie" do + user = insert(:user) + app = insert(:oauth_app) + oauth_token = insert(:oauth_token, user: user, app: app) + redirect_uri = OAuthController.default_redirect_uri(app) + + conn = + build_conn() + |> Plug.Session.call(Plug.Session.init(@session_opts)) + |> fetch_session() + |> AuthHelper.put_session_token(oauth_token.token) + |> post( + "/oauth/authorize", + %{ + "authorization" => %{ + "name" => user.nickname, + "client_id" => app.client_id, + "redirect_uri" => redirect_uri, + "scope" => app.scopes, + "state" => "statepassed" + } + } + ) + + target = redirected_to(conn) + assert target =~ redirect_uri + + query = URI.parse(target).query |> URI.query_decoder() |> Map.new() + + assert %{"state" => "statepassed", "code" => code} = query + auth = Repo.get_by(Authorization, token: code) + assert auth + assert auth.scopes == app.scopes + end + test "redirect to on two-factor auth page" do otp_secret = TOTP.generate_secret() @@ -1182,7 +1218,6 @@ test "issues a new token if token expired" do response = build_conn() - |> put_req_header("authorization", "Bearer #{access_token.token}") |> post("/oauth/token", %{ "grant_type" => "refresh_token", "refresh_token" => access_token.refresh_token, @@ -1232,11 +1267,12 @@ test "when authenticated with request token, revokes it and clears it from sessi build_conn() |> Plug.Session.call(Plug.Session.init(@session_opts)) |> fetch_session() - |> put_req_header("authorization", "Bearer #{oauth_token.token}") + |> AuthHelper.put_session_token(oauth_token.token) |> post("/oauth/revoke", %{"token" => oauth_token.token}) assert json_response(conn, 200) + refute AuthHelper.get_session_token(conn) assert Token.get_by_token(oauth_token.token) == {:error, :not_found} end @@ -1250,11 +1286,12 @@ test "if request is authenticated with a different token, " <> build_conn() |> Plug.Session.call(Plug.Session.init(@session_opts)) |> fetch_session() - |> put_req_header("authorization", "Bearer #{oauth_token.token}") + |> AuthHelper.put_session_token(oauth_token.token) |> post("/oauth/revoke", %{"token" => other_app_oauth_token.token}) assert json_response(conn, 200) + assert AuthHelper.get_session_token(conn) == oauth_token.token assert Token.get_by_token(other_app_oauth_token.token) == {:error, :not_found} end diff --git a/test/pleroma/web/plugs/o_auth_plug_test.exs b/test/pleroma/web/plugs/o_auth_plug_test.exs index caabfc1cb..9e4be5559 100644 --- a/test/pleroma/web/plugs/o_auth_plug_test.exs +++ b/test/pleroma/web/plugs/o_auth_plug_test.exs @@ -5,8 +5,11 @@ defmodule Pleroma.Web.Plugs.OAuthPlugTest do use Pleroma.Web.ConnCase, async: true + alias Pleroma.Helpers.AuthHelper alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.OAuth.Token.Strategy.Revoke alias Pleroma.Web.Plugs.OAuthPlug + alias Plug.Session import Pleroma.Factory @@ -69,4 +72,57 @@ test "with invalid token, it does not assign the user", %{conn: conn} do refute conn.assigns[:user] end + + describe "with :oauth_token in session, " do + setup %{token: oauth_token, conn: conn} do + session_opts = [ + store: :cookie, + key: "_test", + signing_salt: "cooldude" + ] + + conn = + conn + |> Session.call(Session.init(session_opts)) + |> fetch_session() + |> AuthHelper.put_session_token(oauth_token.token) + + %{conn: conn} + end + + test "if session-stored token matches a valid OAuth token, assigns :user and :token", %{ + conn: conn, + user: user, + token: oauth_token + } do + conn = OAuthPlug.call(conn, %{}) + + assert conn.assigns.user && conn.assigns.user.id == user.id + assert conn.assigns.token && conn.assigns.token.id == oauth_token.id + end + + test "if session-stored token matches an expired OAuth token, does nothing", %{ + conn: conn, + token: oauth_token + } do + expired_valid_until = NaiveDateTime.add(NaiveDateTime.utc_now(), -3600 * 24, :second) + + oauth_token + |> Ecto.Changeset.change(valid_until: expired_valid_until) + |> Pleroma.Repo.update() + + ret_conn = OAuthPlug.call(conn, %{}) + assert ret_conn == conn + end + + test "if session-stored token matches a revoked OAuth token, does nothing", %{ + conn: conn, + token: oauth_token + } do + Revoke.revoke(oauth_token) + + ret_conn = OAuthPlug.call(conn, %{}) + assert ret_conn == conn + end + end end diff --git a/test/pleroma/web/plugs/set_user_session_id_plug_test.exs b/test/pleroma/web/plugs/set_user_session_id_plug_test.exs new file mode 100644 index 000000000..9814c80d8 --- /dev/null +++ b/test/pleroma/web/plugs/set_user_session_id_plug_test.exs @@ -0,0 +1,43 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.SetUserSessionIdPlugTest do + use Pleroma.Web.ConnCase, async: true + + alias Pleroma.Helpers.AuthHelper + alias Pleroma.Web.Plugs.SetUserSessionIdPlug + + setup %{conn: conn} do + session_opts = [ + store: :cookie, + key: "_test", + signing_salt: "cooldude" + ] + + conn = + conn + |> Plug.Session.call(Plug.Session.init(session_opts)) + |> fetch_session() + + %{conn: conn} + end + + test "doesn't do anything if the user isn't set", %{conn: conn} do + ret_conn = SetUserSessionIdPlug.call(conn, %{}) + + assert ret_conn == conn + end + + test "sets session token basing on :token assign", %{conn: conn} do + %{user: user, token: oauth_token} = oauth_access(["read"]) + + ret_conn = + conn + |> assign(:user, user) + |> assign(:token, oauth_token) + |> SetUserSessionIdPlug.call(%{}) + + assert AuthHelper.get_session_token(ret_conn) == oauth_token.token + end +end From aa681d7e15f6170e7e92d86146d5ba96be6433bc Mon Sep 17 00:00:00 2001 From: floatingghost Date: Sun, 21 Aug 2022 16:24:37 +0000 Subject: [PATCH 13/44] Fix oauth2 (for real) (#179) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/179 --- lib/pleroma/helpers/auth_helper.ex | 18 -- .../controllers/auth_controller.ex | 11 +- lib/pleroma/web/o_auth/o_auth_controller.ex | 88 +++++---- lib/pleroma/web/o_auth/token.ex | 6 + lib/pleroma/web/o_auth/token/query.ex | 12 +- lib/pleroma/web/plugs/o_auth_plug.ex | 13 +- .../web/plugs/set_user_session_id_plug.ex | 18 -- lib/pleroma/web/router.ex | 5 +- mix.exs | 4 +- .../web/o_auth/o_auth_controller_test.exs | 167 ++++++++++++------ test/pleroma/web/plugs/o_auth_plug_test.exs | 56 ------ .../plugs/set_user_session_id_plug_test.exs | 43 ----- 12 files changed, 193 insertions(+), 248 deletions(-) delete mode 100644 lib/pleroma/web/plugs/set_user_session_id_plug.ex delete mode 100644 test/pleroma/web/plugs/set_user_session_id_plug_test.exs diff --git a/lib/pleroma/helpers/auth_helper.ex b/lib/pleroma/helpers/auth_helper.ex index 13e4c8158..d56f6f461 100644 --- a/lib/pleroma/helpers/auth_helper.ex +++ b/lib/pleroma/helpers/auth_helper.ex @@ -4,12 +4,9 @@ defmodule Pleroma.Helpers.AuthHelper do alias Pleroma.Web.Plugs.OAuthScopesPlug - alias Plug.Conn import Plug.Conn - @oauth_token_session_key :oauth_token - @doc """ Skips OAuth permissions (scopes) checks, assigns nil `:token`. Intended to be used with explicit authentication and only when OAuth token cannot be determined. @@ -28,19 +25,4 @@ def drop_auth_info(conn) do |> assign(:token, nil) |> put_private(:authentication_ignored, true) end - - @doc "Gets OAuth token string from session" - def get_session_token(%Conn{} = conn) do - get_session(conn, @oauth_token_session_key) - end - - @doc "Updates OAuth token string in session" - def put_session_token(%Conn{} = conn, token) when is_binary(token) do - put_session(conn, @oauth_token_session_key, token) - end - - @doc "Deletes OAuth token string from session" - def delete_session_token(%Conn{} = conn) do - delete_session(conn, @oauth_token_session_key) - end end diff --git a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex index 4920d65da..f415e5931 100644 --- a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex @@ -7,7 +7,6 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do import Pleroma.Web.ControllerHelper, only: [json_response: 3] - alias Pleroma.Helpers.AuthHelper alias Pleroma.Helpers.UriHelper alias Pleroma.User alias Pleroma.Web.OAuth.App @@ -34,7 +33,6 @@ def login(conn, %{"code" => auth_token} = params) do |> UriHelper.modify_uri_params(%{"access_token" => oauth_token.token}) conn - |> AuthHelper.put_session_token(oauth_token.token) |> redirect(to: redirect_to) else _ -> redirect_to_oauth_form(conn, params) @@ -42,9 +40,9 @@ def login(conn, %{"code" => auth_token} = params) do end def login(conn, params) do - with %{assigns: %{user: %User{}, token: %Token{app_id: app_id}}} <- conn, + with %{assigns: %{user: %User{}, token: %Token{app_id: app_id, token: token}}} <- conn, {:ok, %{id: ^app_id}} <- local_mastofe_app() do - redirect(conn, to: local_mastodon_post_login_path(conn)) + redirect(conn, to: local_mastodon_post_login_path(conn) <> "?access_token=#{token}") else _ -> redirect_to_oauth_form(conn, params) end @@ -68,9 +66,8 @@ defp redirect_to_oauth_form(conn, _params) do def logout(conn, _) do conn = with %{assigns: %{token: %Token{} = oauth_token}} <- conn, - session_token = AuthHelper.get_session_token(conn), - {:ok, %Token{token: ^session_token}} <- RevokeToken.revoke(oauth_token) do - AuthHelper.delete_session_token(conn) + {:ok, %Token{token: _session_token}} <- RevokeToken.revoke(oauth_token) do + conn else _ -> conn end diff --git a/lib/pleroma/web/o_auth/o_auth_controller.ex b/lib/pleroma/web/o_auth/o_auth_controller.ex index 358120fe6..6a8006d31 100644 --- a/lib/pleroma/web/o_auth/o_auth_controller.ex +++ b/lib/pleroma/web/o_auth/o_auth_controller.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.OAuth.OAuthController do use Pleroma.Web, :controller - alias Pleroma.Helpers.AuthHelper alias Pleroma.Helpers.UriHelper alias Pleroma.Maps alias Pleroma.MFA @@ -72,38 +71,62 @@ def authorize( def authorize(%Plug.Conn{} = conn, params), do: do_authorize(conn, params) + defp maybe_remove_token(%Plug.Conn{assigns: %{token: %{app: id}}} = conn, %App{id: id}) do + conn + end + + defp maybe_remove_token(conn, _app) do + conn + |> assign(:token, nil) + end + defp do_authorize(%Plug.Conn{} = conn, params) do app = Repo.get_by(App, client_id: params["client_id"]) + conn = maybe_remove_token(conn, app) available_scopes = (app && app.scopes) || [] scopes = Scopes.fetch_scopes(params, available_scopes) - user = - with %{assigns: %{user: %User{} = user}} <- conn do - user - else - _ -> nil - end + # if we already have a token for this specific setup, we can use that + with false <- Params.truthy_param?(params["force_login"]), + %App{} <- app, + %{assigns: %{user: %Pleroma.User{} = user}} <- conn, + {:ok, %Token{} = token} <- Token.get_preexisting_by_app_and_user(app, user), + true <- scopes == token.scopes do + token = Repo.preload(token, :app) - scopes = - if scopes == [] do - available_scopes - else - scopes - end + conn + |> assign(:token, token) + |> handle_existing_authorization(params) + else + _ -> + user = + with %{assigns: %{user: %User{} = user}} <- conn do + user + else + _ -> nil + end - # Note: `params` might differ from `conn.params`; use `@params` not `@conn.params` in template - render(conn, Authenticator.auth_template(), %{ - user: user, - app: app && Map.delete(app, :client_secret), - response_type: params["response_type"], - client_id: params["client_id"], - available_scopes: available_scopes, - scopes: scopes, - redirect_uri: params["redirect_uri"], - state: params["state"], - params: params, - view_module: OAuthView - }) + scopes = + if scopes == [] do + available_scopes + else + scopes + end + + # Note: `params` might differ from `conn.params`; use `@params` not `@conn.params` in template + render(conn, Authenticator.auth_template(), %{ + user: user, + app: app && Map.delete(app, :client_secret), + response_type: params["response_type"], + client_id: params["client_id"], + available_scopes: available_scopes, + scopes: scopes, + redirect_uri: params["redirect_uri"], + state: params["state"], + params: params, + view_module: OAuthView + }) + end end defp handle_existing_authorization( @@ -318,9 +341,8 @@ def token_exchange(%Plug.Conn{} = conn, %{"grant_type" => "client_credentials"} # Bad request def token_exchange(%Plug.Conn{} = conn, params), do: bad_request(conn, params) - def after_token_exchange(%Plug.Conn{} = conn, %{token: token} = view_params) do + def after_token_exchange(%Plug.Conn{} = conn, %{token: _token} = view_params) do conn - |> AuthHelper.put_session_token(token.token) |> json(OAuthView.render("token.json", view_params)) end @@ -379,15 +401,7 @@ defp handle_token_exchange_error(%Plug.Conn{} = conn, _error) do def token_revoke(%Plug.Conn{} = conn, %{"token" => token}) do with {:ok, %Token{} = oauth_token} <- Token.get_by_token(token), - {:ok, oauth_token} <- RevokeToken.revoke(oauth_token) do - conn = - with session_token = AuthHelper.get_session_token(conn), - %Token{token: ^session_token} <- oauth_token do - AuthHelper.delete_session_token(conn) - else - _ -> conn - end - + {:ok, _oauth_token} <- RevokeToken.revoke(oauth_token) do json(conn, %{}) else _error -> diff --git a/lib/pleroma/web/o_auth/token.ex b/lib/pleroma/web/o_auth/token.ex index 9d69e9db4..6e91b6216 100644 --- a/lib/pleroma/web/o_auth/token.ex +++ b/lib/pleroma/web/o_auth/token.ex @@ -39,6 +39,12 @@ def get_by_token(token) do |> Repo.find_resource() end + def get_preexisting_by_app_and_user(%App{} = app, %User{} = user) do + app.id + |> Query.get_unexpired_by_app_and_user(user) + |> Repo.find_resource() + end + @doc "Gets token for app by access token" @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found} def get_by_token(%App{id: app_id} = _app, token) do diff --git a/lib/pleroma/web/o_auth/token/query.ex b/lib/pleroma/web/o_auth/token/query.ex index d16a759d8..acddf0533 100644 --- a/lib/pleroma/web/o_auth/token/query.ex +++ b/lib/pleroma/web/o_auth/token/query.ex @@ -23,9 +23,19 @@ def get_by_token(query \\ Token, token) do from(q in query, where: q.token == ^token) end + @spec get_unexpired_by_app_and_user(query, String.t()) :: query + def get_unexpired_by_app_and_user(query \\ Token, app_id, %Pleroma.User{id: user_id}) do + time = NaiveDateTime.utc_now() + + from(q in query, + where: q.app_id == ^app_id and q.valid_until > ^time and q.user_id == ^user_id, + limit: 1 + ) + end + @spec get_by_app(query, String.t()) :: query def get_by_app(query \\ Token, app_id) do - from(q in query, where: q.app_id == ^app_id) + from(q in query, where: q.app_id == ^app_id, limit: 1) end @spec get_by_id(query, String.t()) :: query diff --git a/lib/pleroma/web/plugs/o_auth_plug.ex b/lib/pleroma/web/plugs/o_auth_plug.ex index 5e06ac3f6..29b3316b3 100644 --- a/lib/pleroma/web/plugs/o_auth_plug.ex +++ b/lib/pleroma/web/plugs/o_auth_plug.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.Plugs.OAuthPlug do import Plug.Conn import Ecto.Query - alias Pleroma.Helpers.AuthHelper alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.OAuth.App @@ -18,8 +17,6 @@ defmodule Pleroma.Web.Plugs.OAuthPlug do def init(options), do: options - def call(%{assigns: %{user: %User{}}} = conn, _), do: conn - def call(conn, _) do with {:ok, token_str} <- fetch_token_str(conn) do with {:ok, user, user_token} <- fetch_user_and_token(token_str), @@ -82,7 +79,7 @@ defp fetch_token_str(%Plug.Conn{} = conn) do with {:ok, token} <- fetch_token_str(headers) do {:ok, token} else - _ -> fetch_token_from_session(conn) + _ -> :no_token_found end end @@ -96,12 +93,4 @@ defp fetch_token_str([token | tail]) do end defp fetch_token_str([]), do: :no_token_found - - @spec fetch_token_from_session(Plug.Conn.t()) :: :no_token_found | {:ok, String.t()} - defp fetch_token_from_session(conn) do - case AuthHelper.get_session_token(conn) do - nil -> :no_token_found - token -> {:ok, token} - end - end end diff --git a/lib/pleroma/web/plugs/set_user_session_id_plug.ex b/lib/pleroma/web/plugs/set_user_session_id_plug.ex deleted file mode 100644 index a1cfa0915..000000000 --- a/lib/pleroma/web/plugs/set_user_session_id_plug.ex +++ /dev/null @@ -1,18 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Plugs.SetUserSessionIdPlug do - alias Pleroma.Helpers.AuthHelper - alias Pleroma.Web.OAuth.Token - - def init(opts) do - opts - end - - def call(%{assigns: %{token: %Token{} = oauth_token}} = conn, _) do - AuthHelper.put_session_token(conn, oauth_token.token) - end - - def call(conn, _), do: conn -end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 647d99278..f2d6b0aff 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -57,7 +57,6 @@ defmodule Pleroma.Web.Router do pipeline :after_auth do plug(Pleroma.Web.Plugs.UserEnabledPlug) - plug(Pleroma.Web.Plugs.SetUserSessionIdPlug) plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug) plug(Pleroma.Web.Plugs.UserTrackingPlug) end @@ -793,10 +792,8 @@ defmodule Pleroma.Web.Router do get("/web/login", MastodonAPI.AuthController, :login) delete("/auth/sign_out", MastodonAPI.AuthController, :logout) - - post("/auth/password", MastodonAPI.AuthController, :password_reset) - get("/web/*path", MastoFEController, :index) + post("/auth/password", MastodonAPI.AuthController, :password_reset) get("/embed/:id", EmbedController, :show) end diff --git a/mix.exs b/mix.exs index e7f491997..170276b0a 100644 --- a/mix.exs +++ b/mix.exs @@ -131,7 +131,7 @@ defp deps do {:trailing_format_plug, "~> 0.0.7"}, {:fast_sanitize, "~> 0.2.3"}, {:html_entities, "~> 0.5", override: true}, - {:phoenix_html, "~> 3.1", override: true}, + {:phoenix_html, "~> 3.0", override: true}, {:calendar, "~> 1.0"}, {:cachex, "~> 3.4"}, {:poison, "~> 3.0", override: true}, @@ -152,7 +152,7 @@ defp deps do ref: "f75cd55325e33cbea198fb41fe41871392f8fb76"}, {:cors_plug, "~> 2.0"}, {:web_push_encryption, "~> 0.3.1"}, - {:swoosh, "~> 1.0"}, + {:swoosh, "~> 1.3"}, {:phoenix_swoosh, "~> 0.3"}, {:gen_smtp, "~> 0.13"}, {:ex_syslogger, "~> 1.4"}, diff --git a/test/pleroma/web/o_auth/o_auth_controller_test.exs b/test/pleroma/web/o_auth/o_auth_controller_test.exs index 0fdd5b8e9..9f984b26f 100644 --- a/test/pleroma/web/o_auth/o_auth_controller_test.exs +++ b/test/pleroma/web/o_auth/o_auth_controller_test.exs @@ -7,7 +7,6 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do import Pleroma.Factory - alias Pleroma.Helpers.AuthHelper alias Pleroma.MFA alias Pleroma.MFA.TOTP alias Pleroma.Repo @@ -456,7 +455,7 @@ test "renders authentication page if user is already authenticated but `force_lo conn = conn - |> AuthHelper.put_session_token(token.token) + |> put_req_header("authorization", "Bearer #{token.token}") |> get( "/oauth/authorize", %{ @@ -471,26 +470,130 @@ test "renders authentication page if user is already authenticated but `force_lo assert html_response(conn, 200) =~ ~s(type="submit") end - test "renders authentication page if user is already authenticated but user request with another client", + test "reuses authentication if the user is authenticated with another client", %{ - app: app, conn: conn } do - token = insert(:oauth_token, app: app) + user = insert(:user) + + app = insert(:oauth_app, redirect_uris: "https://redirect.url") + other_app = insert(:oauth_app, redirect_uris: "https://redirect.url") + + token = insert(:oauth_token, user: user, app: app) + reusable_token = insert(:oauth_token, app: other_app, user: user) conn = conn - |> AuthHelper.put_session_token(token.token) + |> put_req_header("authorization", "Bearer #{token.token}") |> get( "/oauth/authorize", %{ "response_type" => "code", - "client_id" => "another_client_id", - "redirect_uri" => OAuthController.default_redirect_uri(app), + "client_id" => other_app.client_id, + "redirect_uri" => OAuthController.default_redirect_uri(other_app), "scope" => "read" } ) + assert URI.decode(redirected_to(conn)) == + "https://redirect.url?access_token=#{reusable_token.token}" + end + + test "does not reuse other people's tokens", + %{ + conn: conn + } do + user = insert(:user) + other_user = insert(:user) + + app = insert(:oauth_app, redirect_uris: "https://redirect.url") + other_app = insert(:oauth_app, redirect_uris: "https://redirect.url") + + token = insert(:oauth_token, user: user, app: app) + _not_reusable_token = insert(:oauth_token, app: other_app, user: other_user) + + conn = + conn + |> put_req_header("authorization", "Bearer #{token.token}") + |> get( + "/oauth/authorize", + %{ + "response_type" => "code", + "client_id" => other_app.client_id, + "redirect_uri" => OAuthController.default_redirect_uri(other_app), + "scope" => "read" + } + ) + + assert html_response(conn, 200) =~ ~s(type="submit") + end + + test "does not reuse expired tokens", + %{ + conn: conn + } do + user = insert(:user) + + app = insert(:oauth_app, redirect_uris: "https://redirect.url") + + other_app = insert(:oauth_app, redirect_uris: "https://redirect.url") + + token = insert(:oauth_token, user: user, app: app) + + _not_reusable_token = + insert(:oauth_token, + app: other_app, + user: user, + valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), -60 * 100) + ) + + conn = + conn + |> put_req_header("authorization", "Bearer #{token.token}") + |> get( + "/oauth/authorize", + %{ + "response_type" => "code", + "client_id" => other_app.client_id, + "redirect_uri" => OAuthController.default_redirect_uri(other_app), + "scope" => "read" + } + ) + + assert html_response(conn, 200) =~ ~s(type="submit") + end + + test "does not reuse tokens with the wrong scopes", + %{ + conn: conn + } do + user = insert(:user) + + app = insert(:oauth_app, redirect_uris: "https://redirect.url") + + other_app = insert(:oauth_app, redirect_uris: "https://redirect.url") + + token = insert(:oauth_token, user: user, app: app, scopes: ["read"]) + + _not_reusable_token = + insert(:oauth_token, + app: other_app, + user: user + ) + + conn = + conn + |> put_req_header("authorization", "Bearer #{token.token}") + |> get( + "/oauth/authorize", + %{ + "response_type" => "code", + "client_id" => other_app.client_id, + "redirect_uri" => OAuthController.default_redirect_uri(other_app), + "scope" => "read write" + } + ) + assert html_response(conn, 200) =~ ~s(type="submit") end @@ -503,7 +606,7 @@ test "with existing authentication and non-OOB `redirect_uri`, redirects to app conn = conn - |> AuthHelper.put_session_token(token.token) + |> put_req_header("authorization", "Bearer #{token.token}") |> get( "/oauth/authorize", %{ @@ -529,7 +632,7 @@ test "with existing authentication and unlisted non-OOB `redirect_uri`, redirect conn = conn - |> AuthHelper.put_session_token(token.token) + |> put_req_header("authorization", "Bearer #{token.token}") |> get( "/oauth/authorize", %{ @@ -553,7 +656,7 @@ test "with existing authentication and OOB `redirect_uri`, redirects to app with conn = conn - |> AuthHelper.put_session_token(token.token) + |> put_req_header("authorization", "Bearer #{token.token}") |> get( "/oauth/authorize", %{ @@ -611,41 +714,6 @@ test "redirects with oauth authorization, " <> end end - test "authorize from cookie" do - user = insert(:user) - app = insert(:oauth_app) - oauth_token = insert(:oauth_token, user: user, app: app) - redirect_uri = OAuthController.default_redirect_uri(app) - - conn = - build_conn() - |> Plug.Session.call(Plug.Session.init(@session_opts)) - |> fetch_session() - |> AuthHelper.put_session_token(oauth_token.token) - |> post( - "/oauth/authorize", - %{ - "authorization" => %{ - "name" => user.nickname, - "client_id" => app.client_id, - "redirect_uri" => redirect_uri, - "scope" => app.scopes, - "state" => "statepassed" - } - } - ) - - target = redirected_to(conn) - assert target =~ redirect_uri - - query = URI.parse(target).query |> URI.query_decoder() |> Map.new() - - assert %{"state" => "statepassed", "code" => code} = query - auth = Repo.get_by(Authorization, token: code) - assert auth - assert auth.scopes == app.scopes - end - test "redirect to on two-factor auth page" do otp_secret = TOTP.generate_secret() @@ -1218,6 +1286,7 @@ test "issues a new token if token expired" do response = build_conn() + |> put_req_header("authorization", "Bearer #{access_token.token}") |> post("/oauth/token", %{ "grant_type" => "refresh_token", "refresh_token" => access_token.refresh_token, @@ -1267,12 +1336,11 @@ test "when authenticated with request token, revokes it and clears it from sessi build_conn() |> Plug.Session.call(Plug.Session.init(@session_opts)) |> fetch_session() - |> AuthHelper.put_session_token(oauth_token.token) + |> put_req_header("authorization", "Bearer #{oauth_token.token}") |> post("/oauth/revoke", %{"token" => oauth_token.token}) assert json_response(conn, 200) - refute AuthHelper.get_session_token(conn) assert Token.get_by_token(oauth_token.token) == {:error, :not_found} end @@ -1286,12 +1354,11 @@ test "if request is authenticated with a different token, " <> build_conn() |> Plug.Session.call(Plug.Session.init(@session_opts)) |> fetch_session() - |> AuthHelper.put_session_token(oauth_token.token) + |> put_req_header("authorization", "Bearer #{oauth_token.token}") |> post("/oauth/revoke", %{"token" => other_app_oauth_token.token}) assert json_response(conn, 200) - assert AuthHelper.get_session_token(conn) == oauth_token.token assert Token.get_by_token(other_app_oauth_token.token) == {:error, :not_found} end diff --git a/test/pleroma/web/plugs/o_auth_plug_test.exs b/test/pleroma/web/plugs/o_auth_plug_test.exs index 9e4be5559..caabfc1cb 100644 --- a/test/pleroma/web/plugs/o_auth_plug_test.exs +++ b/test/pleroma/web/plugs/o_auth_plug_test.exs @@ -5,11 +5,8 @@ defmodule Pleroma.Web.Plugs.OAuthPlugTest do use Pleroma.Web.ConnCase, async: true - alias Pleroma.Helpers.AuthHelper alias Pleroma.Web.OAuth.Token - alias Pleroma.Web.OAuth.Token.Strategy.Revoke alias Pleroma.Web.Plugs.OAuthPlug - alias Plug.Session import Pleroma.Factory @@ -72,57 +69,4 @@ test "with invalid token, it does not assign the user", %{conn: conn} do refute conn.assigns[:user] end - - describe "with :oauth_token in session, " do - setup %{token: oauth_token, conn: conn} do - session_opts = [ - store: :cookie, - key: "_test", - signing_salt: "cooldude" - ] - - conn = - conn - |> Session.call(Session.init(session_opts)) - |> fetch_session() - |> AuthHelper.put_session_token(oauth_token.token) - - %{conn: conn} - end - - test "if session-stored token matches a valid OAuth token, assigns :user and :token", %{ - conn: conn, - user: user, - token: oauth_token - } do - conn = OAuthPlug.call(conn, %{}) - - assert conn.assigns.user && conn.assigns.user.id == user.id - assert conn.assigns.token && conn.assigns.token.id == oauth_token.id - end - - test "if session-stored token matches an expired OAuth token, does nothing", %{ - conn: conn, - token: oauth_token - } do - expired_valid_until = NaiveDateTime.add(NaiveDateTime.utc_now(), -3600 * 24, :second) - - oauth_token - |> Ecto.Changeset.change(valid_until: expired_valid_until) - |> Pleroma.Repo.update() - - ret_conn = OAuthPlug.call(conn, %{}) - assert ret_conn == conn - end - - test "if session-stored token matches a revoked OAuth token, does nothing", %{ - conn: conn, - token: oauth_token - } do - Revoke.revoke(oauth_token) - - ret_conn = OAuthPlug.call(conn, %{}) - assert ret_conn == conn - end - end end diff --git a/test/pleroma/web/plugs/set_user_session_id_plug_test.exs b/test/pleroma/web/plugs/set_user_session_id_plug_test.exs deleted file mode 100644 index 9814c80d8..000000000 --- a/test/pleroma/web/plugs/set_user_session_id_plug_test.exs +++ /dev/null @@ -1,43 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2021 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Plugs.SetUserSessionIdPlugTest do - use Pleroma.Web.ConnCase, async: true - - alias Pleroma.Helpers.AuthHelper - alias Pleroma.Web.Plugs.SetUserSessionIdPlug - - setup %{conn: conn} do - session_opts = [ - store: :cookie, - key: "_test", - signing_salt: "cooldude" - ] - - conn = - conn - |> Plug.Session.call(Plug.Session.init(session_opts)) - |> fetch_session() - - %{conn: conn} - end - - test "doesn't do anything if the user isn't set", %{conn: conn} do - ret_conn = SetUserSessionIdPlug.call(conn, %{}) - - assert ret_conn == conn - end - - test "sets session token basing on :token assign", %{conn: conn} do - %{user: user, token: oauth_token} = oauth_access(["read"]) - - ret_conn = - conn - |> assign(:user, user) - |> assign(:token, oauth_token) - |> SetUserSessionIdPlug.call(%{}) - - assert AuthHelper.get_session_token(ret_conn) == oauth_token.token - end -end From 8d7b63a766e8e2e254843a28ff417d3da57a12be Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sun, 21 Aug 2022 17:52:02 +0100 Subject: [PATCH 14/44] Revert "Fix oauth2 (for real) (#179)" This reverts commit aa681d7e15f6170e7e92d86146d5ba96be6433bc. --- lib/pleroma/helpers/auth_helper.ex | 18 ++ .../controllers/auth_controller.ex | 11 +- lib/pleroma/web/o_auth/o_auth_controller.ex | 88 ++++----- lib/pleroma/web/o_auth/token.ex | 6 - lib/pleroma/web/o_auth/token/query.ex | 12 +- lib/pleroma/web/plugs/o_auth_plug.ex | 13 +- .../web/plugs/set_user_session_id_plug.ex | 18 ++ lib/pleroma/web/router.ex | 5 +- mix.exs | 4 +- .../web/o_auth/o_auth_controller_test.exs | 167 ++++++------------ test/pleroma/web/plugs/o_auth_plug_test.exs | 56 ++++++ .../plugs/set_user_session_id_plug_test.exs | 43 +++++ 12 files changed, 248 insertions(+), 193 deletions(-) create mode 100644 lib/pleroma/web/plugs/set_user_session_id_plug.ex create mode 100644 test/pleroma/web/plugs/set_user_session_id_plug_test.exs diff --git a/lib/pleroma/helpers/auth_helper.ex b/lib/pleroma/helpers/auth_helper.ex index d56f6f461..13e4c8158 100644 --- a/lib/pleroma/helpers/auth_helper.ex +++ b/lib/pleroma/helpers/auth_helper.ex @@ -4,9 +4,12 @@ defmodule Pleroma.Helpers.AuthHelper do alias Pleroma.Web.Plugs.OAuthScopesPlug + alias Plug.Conn import Plug.Conn + @oauth_token_session_key :oauth_token + @doc """ Skips OAuth permissions (scopes) checks, assigns nil `:token`. Intended to be used with explicit authentication and only when OAuth token cannot be determined. @@ -25,4 +28,19 @@ def drop_auth_info(conn) do |> assign(:token, nil) |> put_private(:authentication_ignored, true) end + + @doc "Gets OAuth token string from session" + def get_session_token(%Conn{} = conn) do + get_session(conn, @oauth_token_session_key) + end + + @doc "Updates OAuth token string in session" + def put_session_token(%Conn{} = conn, token) when is_binary(token) do + put_session(conn, @oauth_token_session_key, token) + end + + @doc "Deletes OAuth token string from session" + def delete_session_token(%Conn{} = conn) do + delete_session(conn, @oauth_token_session_key) + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex index f415e5931..4920d65da 100644 --- a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do import Pleroma.Web.ControllerHelper, only: [json_response: 3] + alias Pleroma.Helpers.AuthHelper alias Pleroma.Helpers.UriHelper alias Pleroma.User alias Pleroma.Web.OAuth.App @@ -33,6 +34,7 @@ def login(conn, %{"code" => auth_token} = params) do |> UriHelper.modify_uri_params(%{"access_token" => oauth_token.token}) conn + |> AuthHelper.put_session_token(oauth_token.token) |> redirect(to: redirect_to) else _ -> redirect_to_oauth_form(conn, params) @@ -40,9 +42,9 @@ def login(conn, %{"code" => auth_token} = params) do end def login(conn, params) do - with %{assigns: %{user: %User{}, token: %Token{app_id: app_id, token: token}}} <- conn, + with %{assigns: %{user: %User{}, token: %Token{app_id: app_id}}} <- conn, {:ok, %{id: ^app_id}} <- local_mastofe_app() do - redirect(conn, to: local_mastodon_post_login_path(conn) <> "?access_token=#{token}") + redirect(conn, to: local_mastodon_post_login_path(conn)) else _ -> redirect_to_oauth_form(conn, params) end @@ -66,8 +68,9 @@ defp redirect_to_oauth_form(conn, _params) do def logout(conn, _) do conn = with %{assigns: %{token: %Token{} = oauth_token}} <- conn, - {:ok, %Token{token: _session_token}} <- RevokeToken.revoke(oauth_token) do - conn + session_token = AuthHelper.get_session_token(conn), + {:ok, %Token{token: ^session_token}} <- RevokeToken.revoke(oauth_token) do + AuthHelper.delete_session_token(conn) else _ -> conn end diff --git a/lib/pleroma/web/o_auth/o_auth_controller.ex b/lib/pleroma/web/o_auth/o_auth_controller.ex index 6a8006d31..358120fe6 100644 --- a/lib/pleroma/web/o_auth/o_auth_controller.ex +++ b/lib/pleroma/web/o_auth/o_auth_controller.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do use Pleroma.Web, :controller + alias Pleroma.Helpers.AuthHelper alias Pleroma.Helpers.UriHelper alias Pleroma.Maps alias Pleroma.MFA @@ -71,62 +72,38 @@ def authorize( def authorize(%Plug.Conn{} = conn, params), do: do_authorize(conn, params) - defp maybe_remove_token(%Plug.Conn{assigns: %{token: %{app: id}}} = conn, %App{id: id}) do - conn - end - - defp maybe_remove_token(conn, _app) do - conn - |> assign(:token, nil) - end - defp do_authorize(%Plug.Conn{} = conn, params) do app = Repo.get_by(App, client_id: params["client_id"]) - conn = maybe_remove_token(conn, app) available_scopes = (app && app.scopes) || [] scopes = Scopes.fetch_scopes(params, available_scopes) - # if we already have a token for this specific setup, we can use that - with false <- Params.truthy_param?(params["force_login"]), - %App{} <- app, - %{assigns: %{user: %Pleroma.User{} = user}} <- conn, - {:ok, %Token{} = token} <- Token.get_preexisting_by_app_and_user(app, user), - true <- scopes == token.scopes do - token = Repo.preload(token, :app) + user = + with %{assigns: %{user: %User{} = user}} <- conn do + user + else + _ -> nil + end - conn - |> assign(:token, token) - |> handle_existing_authorization(params) - else - _ -> - user = - with %{assigns: %{user: %User{} = user}} <- conn do - user - else - _ -> nil - end + scopes = + if scopes == [] do + available_scopes + else + scopes + end - scopes = - if scopes == [] do - available_scopes - else - scopes - end - - # Note: `params` might differ from `conn.params`; use `@params` not `@conn.params` in template - render(conn, Authenticator.auth_template(), %{ - user: user, - app: app && Map.delete(app, :client_secret), - response_type: params["response_type"], - client_id: params["client_id"], - available_scopes: available_scopes, - scopes: scopes, - redirect_uri: params["redirect_uri"], - state: params["state"], - params: params, - view_module: OAuthView - }) - end + # Note: `params` might differ from `conn.params`; use `@params` not `@conn.params` in template + render(conn, Authenticator.auth_template(), %{ + user: user, + app: app && Map.delete(app, :client_secret), + response_type: params["response_type"], + client_id: params["client_id"], + available_scopes: available_scopes, + scopes: scopes, + redirect_uri: params["redirect_uri"], + state: params["state"], + params: params, + view_module: OAuthView + }) end defp handle_existing_authorization( @@ -341,8 +318,9 @@ def token_exchange(%Plug.Conn{} = conn, %{"grant_type" => "client_credentials"} # Bad request def token_exchange(%Plug.Conn{} = conn, params), do: bad_request(conn, params) - def after_token_exchange(%Plug.Conn{} = conn, %{token: _token} = view_params) do + def after_token_exchange(%Plug.Conn{} = conn, %{token: token} = view_params) do conn + |> AuthHelper.put_session_token(token.token) |> json(OAuthView.render("token.json", view_params)) end @@ -401,7 +379,15 @@ defp handle_token_exchange_error(%Plug.Conn{} = conn, _error) do def token_revoke(%Plug.Conn{} = conn, %{"token" => token}) do with {:ok, %Token{} = oauth_token} <- Token.get_by_token(token), - {:ok, _oauth_token} <- RevokeToken.revoke(oauth_token) do + {:ok, oauth_token} <- RevokeToken.revoke(oauth_token) do + conn = + with session_token = AuthHelper.get_session_token(conn), + %Token{token: ^session_token} <- oauth_token do + AuthHelper.delete_session_token(conn) + else + _ -> conn + end + json(conn, %{}) else _error -> diff --git a/lib/pleroma/web/o_auth/token.ex b/lib/pleroma/web/o_auth/token.ex index 6e91b6216..9d69e9db4 100644 --- a/lib/pleroma/web/o_auth/token.ex +++ b/lib/pleroma/web/o_auth/token.ex @@ -39,12 +39,6 @@ def get_by_token(token) do |> Repo.find_resource() end - def get_preexisting_by_app_and_user(%App{} = app, %User{} = user) do - app.id - |> Query.get_unexpired_by_app_and_user(user) - |> Repo.find_resource() - end - @doc "Gets token for app by access token" @spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found} def get_by_token(%App{id: app_id} = _app, token) do diff --git a/lib/pleroma/web/o_auth/token/query.ex b/lib/pleroma/web/o_auth/token/query.ex index acddf0533..d16a759d8 100644 --- a/lib/pleroma/web/o_auth/token/query.ex +++ b/lib/pleroma/web/o_auth/token/query.ex @@ -23,19 +23,9 @@ def get_by_token(query \\ Token, token) do from(q in query, where: q.token == ^token) end - @spec get_unexpired_by_app_and_user(query, String.t()) :: query - def get_unexpired_by_app_and_user(query \\ Token, app_id, %Pleroma.User{id: user_id}) do - time = NaiveDateTime.utc_now() - - from(q in query, - where: q.app_id == ^app_id and q.valid_until > ^time and q.user_id == ^user_id, - limit: 1 - ) - end - @spec get_by_app(query, String.t()) :: query def get_by_app(query \\ Token, app_id) do - from(q in query, where: q.app_id == ^app_id, limit: 1) + from(q in query, where: q.app_id == ^app_id) end @spec get_by_id(query, String.t()) :: query diff --git a/lib/pleroma/web/plugs/o_auth_plug.ex b/lib/pleroma/web/plugs/o_auth_plug.ex index 29b3316b3..5e06ac3f6 100644 --- a/lib/pleroma/web/plugs/o_auth_plug.ex +++ b/lib/pleroma/web/plugs/o_auth_plug.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.Plugs.OAuthPlug do import Plug.Conn import Ecto.Query + alias Pleroma.Helpers.AuthHelper alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.OAuth.App @@ -17,6 +18,8 @@ defmodule Pleroma.Web.Plugs.OAuthPlug do def init(options), do: options + def call(%{assigns: %{user: %User{}}} = conn, _), do: conn + def call(conn, _) do with {:ok, token_str} <- fetch_token_str(conn) do with {:ok, user, user_token} <- fetch_user_and_token(token_str), @@ -79,7 +82,7 @@ defp fetch_token_str(%Plug.Conn{} = conn) do with {:ok, token} <- fetch_token_str(headers) do {:ok, token} else - _ -> :no_token_found + _ -> fetch_token_from_session(conn) end end @@ -93,4 +96,12 @@ defp fetch_token_str([token | tail]) do end defp fetch_token_str([]), do: :no_token_found + + @spec fetch_token_from_session(Plug.Conn.t()) :: :no_token_found | {:ok, String.t()} + defp fetch_token_from_session(conn) do + case AuthHelper.get_session_token(conn) do + nil -> :no_token_found + token -> {:ok, token} + end + end end diff --git a/lib/pleroma/web/plugs/set_user_session_id_plug.ex b/lib/pleroma/web/plugs/set_user_session_id_plug.ex new file mode 100644 index 000000000..a1cfa0915 --- /dev/null +++ b/lib/pleroma/web/plugs/set_user_session_id_plug.ex @@ -0,0 +1,18 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.SetUserSessionIdPlug do + alias Pleroma.Helpers.AuthHelper + alias Pleroma.Web.OAuth.Token + + def init(opts) do + opts + end + + def call(%{assigns: %{token: %Token{} = oauth_token}} = conn, _) do + AuthHelper.put_session_token(conn, oauth_token.token) + end + + def call(conn, _), do: conn +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index f2d6b0aff..647d99278 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -57,6 +57,7 @@ defmodule Pleroma.Web.Router do pipeline :after_auth do plug(Pleroma.Web.Plugs.UserEnabledPlug) + plug(Pleroma.Web.Plugs.SetUserSessionIdPlug) plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug) plug(Pleroma.Web.Plugs.UserTrackingPlug) end @@ -792,9 +793,11 @@ defmodule Pleroma.Web.Router do get("/web/login", MastodonAPI.AuthController, :login) delete("/auth/sign_out", MastodonAPI.AuthController, :logout) - get("/web/*path", MastoFEController, :index) + post("/auth/password", MastodonAPI.AuthController, :password_reset) + get("/web/*path", MastoFEController, :index) + get("/embed/:id", EmbedController, :show) end diff --git a/mix.exs b/mix.exs index 170276b0a..e7f491997 100644 --- a/mix.exs +++ b/mix.exs @@ -131,7 +131,7 @@ defp deps do {:trailing_format_plug, "~> 0.0.7"}, {:fast_sanitize, "~> 0.2.3"}, {:html_entities, "~> 0.5", override: true}, - {:phoenix_html, "~> 3.0", override: true}, + {:phoenix_html, "~> 3.1", override: true}, {:calendar, "~> 1.0"}, {:cachex, "~> 3.4"}, {:poison, "~> 3.0", override: true}, @@ -152,7 +152,7 @@ defp deps do ref: "f75cd55325e33cbea198fb41fe41871392f8fb76"}, {:cors_plug, "~> 2.0"}, {:web_push_encryption, "~> 0.3.1"}, - {:swoosh, "~> 1.3"}, + {:swoosh, "~> 1.0"}, {:phoenix_swoosh, "~> 0.3"}, {:gen_smtp, "~> 0.13"}, {:ex_syslogger, "~> 1.4"}, diff --git a/test/pleroma/web/o_auth/o_auth_controller_test.exs b/test/pleroma/web/o_auth/o_auth_controller_test.exs index 9f984b26f..0fdd5b8e9 100644 --- a/test/pleroma/web/o_auth/o_auth_controller_test.exs +++ b/test/pleroma/web/o_auth/o_auth_controller_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.OAuth.OAuthControllerTest do import Pleroma.Factory + alias Pleroma.Helpers.AuthHelper alias Pleroma.MFA alias Pleroma.MFA.TOTP alias Pleroma.Repo @@ -455,7 +456,7 @@ test "renders authentication page if user is already authenticated but `force_lo conn = conn - |> put_req_header("authorization", "Bearer #{token.token}") + |> AuthHelper.put_session_token(token.token) |> get( "/oauth/authorize", %{ @@ -470,130 +471,26 @@ test "renders authentication page if user is already authenticated but `force_lo assert html_response(conn, 200) =~ ~s(type="submit") end - test "reuses authentication if the user is authenticated with another client", + test "renders authentication page if user is already authenticated but user request with another client", %{ + app: app, conn: conn } do - user = insert(:user) - - app = insert(:oauth_app, redirect_uris: "https://redirect.url") - other_app = insert(:oauth_app, redirect_uris: "https://redirect.url") - - token = insert(:oauth_token, user: user, app: app) - reusable_token = insert(:oauth_token, app: other_app, user: user) + token = insert(:oauth_token, app: app) conn = conn - |> put_req_header("authorization", "Bearer #{token.token}") + |> AuthHelper.put_session_token(token.token) |> get( "/oauth/authorize", %{ "response_type" => "code", - "client_id" => other_app.client_id, - "redirect_uri" => OAuthController.default_redirect_uri(other_app), + "client_id" => "another_client_id", + "redirect_uri" => OAuthController.default_redirect_uri(app), "scope" => "read" } ) - assert URI.decode(redirected_to(conn)) == - "https://redirect.url?access_token=#{reusable_token.token}" - end - - test "does not reuse other people's tokens", - %{ - conn: conn - } do - user = insert(:user) - other_user = insert(:user) - - app = insert(:oauth_app, redirect_uris: "https://redirect.url") - other_app = insert(:oauth_app, redirect_uris: "https://redirect.url") - - token = insert(:oauth_token, user: user, app: app) - _not_reusable_token = insert(:oauth_token, app: other_app, user: other_user) - - conn = - conn - |> put_req_header("authorization", "Bearer #{token.token}") - |> get( - "/oauth/authorize", - %{ - "response_type" => "code", - "client_id" => other_app.client_id, - "redirect_uri" => OAuthController.default_redirect_uri(other_app), - "scope" => "read" - } - ) - - assert html_response(conn, 200) =~ ~s(type="submit") - end - - test "does not reuse expired tokens", - %{ - conn: conn - } do - user = insert(:user) - - app = insert(:oauth_app, redirect_uris: "https://redirect.url") - - other_app = insert(:oauth_app, redirect_uris: "https://redirect.url") - - token = insert(:oauth_token, user: user, app: app) - - _not_reusable_token = - insert(:oauth_token, - app: other_app, - user: user, - valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), -60 * 100) - ) - - conn = - conn - |> put_req_header("authorization", "Bearer #{token.token}") - |> get( - "/oauth/authorize", - %{ - "response_type" => "code", - "client_id" => other_app.client_id, - "redirect_uri" => OAuthController.default_redirect_uri(other_app), - "scope" => "read" - } - ) - - assert html_response(conn, 200) =~ ~s(type="submit") - end - - test "does not reuse tokens with the wrong scopes", - %{ - conn: conn - } do - user = insert(:user) - - app = insert(:oauth_app, redirect_uris: "https://redirect.url") - - other_app = insert(:oauth_app, redirect_uris: "https://redirect.url") - - token = insert(:oauth_token, user: user, app: app, scopes: ["read"]) - - _not_reusable_token = - insert(:oauth_token, - app: other_app, - user: user - ) - - conn = - conn - |> put_req_header("authorization", "Bearer #{token.token}") - |> get( - "/oauth/authorize", - %{ - "response_type" => "code", - "client_id" => other_app.client_id, - "redirect_uri" => OAuthController.default_redirect_uri(other_app), - "scope" => "read write" - } - ) - assert html_response(conn, 200) =~ ~s(type="submit") end @@ -606,7 +503,7 @@ test "with existing authentication and non-OOB `redirect_uri`, redirects to app conn = conn - |> put_req_header("authorization", "Bearer #{token.token}") + |> AuthHelper.put_session_token(token.token) |> get( "/oauth/authorize", %{ @@ -632,7 +529,7 @@ test "with existing authentication and unlisted non-OOB `redirect_uri`, redirect conn = conn - |> put_req_header("authorization", "Bearer #{token.token}") + |> AuthHelper.put_session_token(token.token) |> get( "/oauth/authorize", %{ @@ -656,7 +553,7 @@ test "with existing authentication and OOB `redirect_uri`, redirects to app with conn = conn - |> put_req_header("authorization", "Bearer #{token.token}") + |> AuthHelper.put_session_token(token.token) |> get( "/oauth/authorize", %{ @@ -714,6 +611,41 @@ test "redirects with oauth authorization, " <> end end + test "authorize from cookie" do + user = insert(:user) + app = insert(:oauth_app) + oauth_token = insert(:oauth_token, user: user, app: app) + redirect_uri = OAuthController.default_redirect_uri(app) + + conn = + build_conn() + |> Plug.Session.call(Plug.Session.init(@session_opts)) + |> fetch_session() + |> AuthHelper.put_session_token(oauth_token.token) + |> post( + "/oauth/authorize", + %{ + "authorization" => %{ + "name" => user.nickname, + "client_id" => app.client_id, + "redirect_uri" => redirect_uri, + "scope" => app.scopes, + "state" => "statepassed" + } + } + ) + + target = redirected_to(conn) + assert target =~ redirect_uri + + query = URI.parse(target).query |> URI.query_decoder() |> Map.new() + + assert %{"state" => "statepassed", "code" => code} = query + auth = Repo.get_by(Authorization, token: code) + assert auth + assert auth.scopes == app.scopes + end + test "redirect to on two-factor auth page" do otp_secret = TOTP.generate_secret() @@ -1286,7 +1218,6 @@ test "issues a new token if token expired" do response = build_conn() - |> put_req_header("authorization", "Bearer #{access_token.token}") |> post("/oauth/token", %{ "grant_type" => "refresh_token", "refresh_token" => access_token.refresh_token, @@ -1336,11 +1267,12 @@ test "when authenticated with request token, revokes it and clears it from sessi build_conn() |> Plug.Session.call(Plug.Session.init(@session_opts)) |> fetch_session() - |> put_req_header("authorization", "Bearer #{oauth_token.token}") + |> AuthHelper.put_session_token(oauth_token.token) |> post("/oauth/revoke", %{"token" => oauth_token.token}) assert json_response(conn, 200) + refute AuthHelper.get_session_token(conn) assert Token.get_by_token(oauth_token.token) == {:error, :not_found} end @@ -1354,11 +1286,12 @@ test "if request is authenticated with a different token, " <> build_conn() |> Plug.Session.call(Plug.Session.init(@session_opts)) |> fetch_session() - |> put_req_header("authorization", "Bearer #{oauth_token.token}") + |> AuthHelper.put_session_token(oauth_token.token) |> post("/oauth/revoke", %{"token" => other_app_oauth_token.token}) assert json_response(conn, 200) + assert AuthHelper.get_session_token(conn) == oauth_token.token assert Token.get_by_token(other_app_oauth_token.token) == {:error, :not_found} end diff --git a/test/pleroma/web/plugs/o_auth_plug_test.exs b/test/pleroma/web/plugs/o_auth_plug_test.exs index caabfc1cb..9e4be5559 100644 --- a/test/pleroma/web/plugs/o_auth_plug_test.exs +++ b/test/pleroma/web/plugs/o_auth_plug_test.exs @@ -5,8 +5,11 @@ defmodule Pleroma.Web.Plugs.OAuthPlugTest do use Pleroma.Web.ConnCase, async: true + alias Pleroma.Helpers.AuthHelper alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.OAuth.Token.Strategy.Revoke alias Pleroma.Web.Plugs.OAuthPlug + alias Plug.Session import Pleroma.Factory @@ -69,4 +72,57 @@ test "with invalid token, it does not assign the user", %{conn: conn} do refute conn.assigns[:user] end + + describe "with :oauth_token in session, " do + setup %{token: oauth_token, conn: conn} do + session_opts = [ + store: :cookie, + key: "_test", + signing_salt: "cooldude" + ] + + conn = + conn + |> Session.call(Session.init(session_opts)) + |> fetch_session() + |> AuthHelper.put_session_token(oauth_token.token) + + %{conn: conn} + end + + test "if session-stored token matches a valid OAuth token, assigns :user and :token", %{ + conn: conn, + user: user, + token: oauth_token + } do + conn = OAuthPlug.call(conn, %{}) + + assert conn.assigns.user && conn.assigns.user.id == user.id + assert conn.assigns.token && conn.assigns.token.id == oauth_token.id + end + + test "if session-stored token matches an expired OAuth token, does nothing", %{ + conn: conn, + token: oauth_token + } do + expired_valid_until = NaiveDateTime.add(NaiveDateTime.utc_now(), -3600 * 24, :second) + + oauth_token + |> Ecto.Changeset.change(valid_until: expired_valid_until) + |> Pleroma.Repo.update() + + ret_conn = OAuthPlug.call(conn, %{}) + assert ret_conn == conn + end + + test "if session-stored token matches a revoked OAuth token, does nothing", %{ + conn: conn, + token: oauth_token + } do + Revoke.revoke(oauth_token) + + ret_conn = OAuthPlug.call(conn, %{}) + assert ret_conn == conn + end + end end diff --git a/test/pleroma/web/plugs/set_user_session_id_plug_test.exs b/test/pleroma/web/plugs/set_user_session_id_plug_test.exs new file mode 100644 index 000000000..9814c80d8 --- /dev/null +++ b/test/pleroma/web/plugs/set_user_session_id_plug_test.exs @@ -0,0 +1,43 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.SetUserSessionIdPlugTest do + use Pleroma.Web.ConnCase, async: true + + alias Pleroma.Helpers.AuthHelper + alias Pleroma.Web.Plugs.SetUserSessionIdPlug + + setup %{conn: conn} do + session_opts = [ + store: :cookie, + key: "_test", + signing_salt: "cooldude" + ] + + conn = + conn + |> Plug.Session.call(Plug.Session.init(session_opts)) + |> fetch_session() + + %{conn: conn} + end + + test "doesn't do anything if the user isn't set", %{conn: conn} do + ret_conn = SetUserSessionIdPlug.call(conn, %{}) + + assert ret_conn == conn + end + + test "sets session token basing on :token assign", %{conn: conn} do + %{user: user, token: oauth_token} = oauth_access(["read"]) + + ret_conn = + conn + |> assign(:user, user) + |> assign(:token, oauth_token) + |> SetUserSessionIdPlug.call(%{}) + + assert AuthHelper.get_session_token(ret_conn) == oauth_token.token + end +end From 152c43ac9e391a7db92013ef4b45a8dbc6db6049 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Tue, 23 Aug 2022 12:09:01 +0100 Subject: [PATCH 15/44] update mfm_parser --- mix.exs | 2 +- mix.lock | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index e7f491997..5312be47b 100644 --- a/mix.exs +++ b/mix.exs @@ -193,7 +193,7 @@ defp deps do git: "https://akkoma.dev/AkkomaGang/elasticsearch-elixir.git", ref: "main"}, {:mfm_parser, git: "https://akkoma.dev/AkkomaGang/mfm-parser.git", - ref: "5054e0ba1ebcbd9a7916aec219528e3e58057241"}, + ref: "48d0da81e060fdfb161696fc4e7e8e19190c8746"}, # indirect dependency version override {:plug, "~> 1.10.4", override: true}, diff --git a/mix.lock b/mix.lock index 1c3b550e7..e488648af 100644 --- a/mix.lock +++ b/mix.lock @@ -67,7 +67,7 @@ "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "meck": {:hex, :meck, "0.9.2", "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", [:rebar3], [], "hexpm", "81344f561357dc40a8344afa53767c32669153355b626ea9fcbc8da6b3045826"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, - "mfm_parser": {:git, "https://akkoma.dev/AkkomaGang/mfm-parser.git", "5054e0ba1ebcbd9a7916aec219528e3e58057241", [ref: "5054e0ba1ebcbd9a7916aec219528e3e58057241"]}, + "mfm_parser": {:git, "https://akkoma.dev/AkkomaGang/mfm-parser.git", "48d0da81e060fdfb161696fc4e7e8e19190c8746", [ref: "48d0da81e060fdfb161696fc4e7e8e19190c8746"]}, "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "mint": {:hex, :mint, "1.4.2", "50330223429a6e1260b2ca5415f69b0ab086141bc76dc2fbf34d7c389a6675b2", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "ce75a5bbcc59b4d7d8d70f8b2fc284b1751ffb35c7b6a6302b5192f8ab4ddd80"}, @@ -110,6 +110,7 @@ "table_rex": {:hex, :table_rex, "3.1.1", "0c67164d1714b5e806d5067c1e96ff098ba7ae79413cc075973e17c38a587caa", [:mix], [], "hexpm", "678a23aba4d670419c23c17790f9dcd635a4a89022040df7d5d772cb21012490"}, "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.1", "315d9163a1d4660aedc3fee73f33f1d355dcc76c5c3ab3d59e76e3edf80eef1f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7be9e0871c41732c233be71e4be11b96e56177bf15dde64a8ac9ce72ac9834c6"}, + "temple": {:git, "git@akkoma.dev:floatingghost/temple.git", "066a699ade472d8fa42a9d730b29a61af9bc8b59", [ref: "066a699ade472d8fa42a9d730b29a61af9bc8b59"]}, "tesla": {:hex, :tesla, "1.4.4", "bb89aa0c9745190930366f6a2ac612cdf2d0e4d7fff449861baa7875afd797b2", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "d5503a49f9dec1b287567ea8712d085947e247cb11b06bc54adb05bfde466457"}, "timex": {:hex, :timex, "3.7.8", "0e6e8bf7c0aba95f1e13204889b2446e7a5297b1c8e408f15ab58b2c8dc85f81", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8f3b8edc5faab5205d69e5255a1d64a83b190bab7f16baa78aefcb897cf81435"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, From 3cf8c1eb31844a2a46ba7f3b950a39068c555ae1 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Tue, 23 Aug 2022 12:13:35 +0100 Subject: [PATCH 16/44] use public temple dep --- mix.exs | 2 +- mix.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mix.exs b/mix.exs index 5312be47b..2c2d63b01 100644 --- a/mix.exs +++ b/mix.exs @@ -193,7 +193,7 @@ defp deps do git: "https://akkoma.dev/AkkomaGang/elasticsearch-elixir.git", ref: "main"}, {:mfm_parser, git: "https://akkoma.dev/AkkomaGang/mfm-parser.git", - ref: "48d0da81e060fdfb161696fc4e7e8e19190c8746"}, + ref: "51282dd6a784f4e75d6987ae3ceb91671e46dcfb"}, # indirect dependency version override {:plug, "~> 1.10.4", override: true}, diff --git a/mix.lock b/mix.lock index e488648af..9f9d5b035 100644 --- a/mix.lock +++ b/mix.lock @@ -67,7 +67,7 @@ "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "meck": {:hex, :meck, "0.9.2", "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", [:rebar3], [], "hexpm", "81344f561357dc40a8344afa53767c32669153355b626ea9fcbc8da6b3045826"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, - "mfm_parser": {:git, "https://akkoma.dev/AkkomaGang/mfm-parser.git", "48d0da81e060fdfb161696fc4e7e8e19190c8746", [ref: "48d0da81e060fdfb161696fc4e7e8e19190c8746"]}, + "mfm_parser": {:git, "https://akkoma.dev/AkkomaGang/mfm-parser.git", "51282dd6a784f4e75d6987ae3ceb91671e46dcfb", [ref: "51282dd6a784f4e75d6987ae3ceb91671e46dcfb"]}, "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "mint": {:hex, :mint, "1.4.2", "50330223429a6e1260b2ca5415f69b0ab086141bc76dc2fbf34d7c389a6675b2", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "ce75a5bbcc59b4d7d8d70f8b2fc284b1751ffb35c7b6a6302b5192f8ab4ddd80"}, @@ -110,7 +110,7 @@ "table_rex": {:hex, :table_rex, "3.1.1", "0c67164d1714b5e806d5067c1e96ff098ba7ae79413cc075973e17c38a587caa", [:mix], [], "hexpm", "678a23aba4d670419c23c17790f9dcd635a4a89022040df7d5d772cb21012490"}, "telemetry": {:hex, :telemetry, "0.4.3", "a06428a514bdbc63293cd9a6263aad00ddeb66f608163bdec7c8995784080818", [:rebar3], [], "hexpm", "eb72b8365ffda5bed68a620d1da88525e326cb82a75ee61354fc24b844768041"}, "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.1", "315d9163a1d4660aedc3fee73f33f1d355dcc76c5c3ab3d59e76e3edf80eef1f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7be9e0871c41732c233be71e4be11b96e56177bf15dde64a8ac9ce72ac9834c6"}, - "temple": {:git, "git@akkoma.dev:floatingghost/temple.git", "066a699ade472d8fa42a9d730b29a61af9bc8b59", [ref: "066a699ade472d8fa42a9d730b29a61af9bc8b59"]}, + "temple": {:git, "https://akkoma.dev/AkkomaGang/temple.git", "066a699ade472d8fa42a9d730b29a61af9bc8b59", [ref: "066a699ade472d8fa42a9d730b29a61af9bc8b59"]}, "tesla": {:hex, :tesla, "1.4.4", "bb89aa0c9745190930366f6a2ac612cdf2d0e4d7fff449861baa7875afd797b2", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: true]}, {:exjsx, ">= 3.0.0", [hex: :exjsx, repo: "hexpm", optional: true]}, {:finch, "~> 0.3", [hex: :finch, repo: "hexpm", optional: true]}, {:fuse, "~> 2.4", [hex: :fuse, repo: "hexpm", optional: true]}, {:gun, "~> 1.3", [hex: :gun, repo: "hexpm", optional: true]}, {:hackney, "~> 1.6", [hex: :hackney, repo: "hexpm", optional: true]}, {:ibrowse, "4.4.0", [hex: :ibrowse, repo: "hexpm", optional: true]}, {:jason, ">= 1.0.0", [hex: :jason, repo: "hexpm", optional: true]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.0", [hex: :mint, repo: "hexpm", optional: true]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "d5503a49f9dec1b287567ea8712d085947e247cb11b06bc54adb05bfde466457"}, "timex": {:hex, :timex, "3.7.8", "0e6e8bf7c0aba95f1e13204889b2446e7a5297b1c8e408f15ab58b2c8dc85f81", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8f3b8edc5faab5205d69e5255a1d64a83b190bab7f16baa78aefcb897cf81435"}, "trailing_format_plug": {:hex, :trailing_format_plug, "0.0.7", "64b877f912cf7273bed03379936df39894149e35137ac9509117e59866e10e45", [:mix], [{:plug, "> 0.12.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "bd4fde4c15f3e993a999e019d64347489b91b7a9096af68b2bdadd192afa693f"}, From 9b6feb6657763113ad5092a58afd40967933509b Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Tue, 23 Aug 2022 16:10:19 +0100 Subject: [PATCH 17/44] add language tags --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5debafaea..c3ead7fc1 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ *a smallish microblogging platform, aka the cooler pleroma* +![English OK](https://img.shields.io/badge/English-OK-blueviolet) ![日本語OK](https://img.shields.io/badge/%E6%97%A5%E6%9C%AC%E8%AA%9E-OK-blueviolet) + ## About This is a fork of Pleroma, which is a microblogging server software that can federate (= exchange messages with) other servers that support ActivityPub. What that means is that you can host a server for yourself or your friends and stay in control of your online identity, but still exchange messages with people on larger servers. Akkoma will federate with all servers that implement ActivityPub, like Friendica, GNU Social, Hubzilla, Mastodon, Misskey, Peertube, and Pixelfed. From fd7f4874baeaa68984cb41995efb8c8dee9f8627 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Wed, 24 Aug 2022 10:06:48 +0100 Subject: [PATCH 18/44] allow new mfm classes --- mix.exs | 2 +- mix.lock | 2 +- priv/scrubbers/default.ex | 28 ++++++++++++++-------------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/mix.exs b/mix.exs index 2c2d63b01..f6cdac595 100644 --- a/mix.exs +++ b/mix.exs @@ -193,7 +193,7 @@ defp deps do git: "https://akkoma.dev/AkkomaGang/elasticsearch-elixir.git", ref: "main"}, {:mfm_parser, git: "https://akkoma.dev/AkkomaGang/mfm-parser.git", - ref: "51282dd6a784f4e75d6987ae3ceb91671e46dcfb"}, + ref: "912fba81152d4d572e457fd5427f9875b2bc3dbe"}, # indirect dependency version override {:plug, "~> 1.10.4", override: true}, diff --git a/mix.lock b/mix.lock index 9f9d5b035..bda77cf5a 100644 --- a/mix.lock +++ b/mix.lock @@ -67,7 +67,7 @@ "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "meck": {:hex, :meck, "0.9.2", "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", [:rebar3], [], "hexpm", "81344f561357dc40a8344afa53767c32669153355b626ea9fcbc8da6b3045826"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, - "mfm_parser": {:git, "https://akkoma.dev/AkkomaGang/mfm-parser.git", "51282dd6a784f4e75d6987ae3ceb91671e46dcfb", [ref: "51282dd6a784f4e75d6987ae3ceb91671e46dcfb"]}, + "mfm_parser": {:git, "https://akkoma.dev/AkkomaGang/mfm-parser.git", "912fba81152d4d572e457fd5427f9875b2bc3dbe", [ref: "912fba81152d4d572e457fd5427f9875b2bc3dbe"]}, "mime": {:hex, :mime, "1.6.0", "dabde576a497cef4bbdd60aceee8160e02a6c89250d6c0b29e56c0dfb00db3d2", [:mix], [], "hexpm", "31a1a8613f8321143dde1dafc36006a17d28d02bdfecb9e95a880fa7aabd19a7"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, "mint": {:hex, :mint, "1.4.2", "50330223429a6e1260b2ca5415f69b0ab086141bc76dc2fbf34d7c389a6675b2", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "ce75a5bbcc59b4d7d8d70f8b2fc284b1751ffb35c7b6a6302b5192f8ab4ddd80"}, diff --git a/priv/scrubbers/default.ex b/priv/scrubbers/default.ex index 68ac06e32..950b6c21e 100644 --- a/priv/scrubbers/default.ex +++ b/priv/scrubbers/default.ex @@ -62,20 +62,20 @@ defmodule Pleroma.HTML.Scrubber.Default do "h-card", "quote-inline", "mfm", - "_mfm_tada_", - "_mfm_jelly_", - "_mfm_twitch_", - "_mfm_shake_", - "_mfm_spin_", - "_mfm_jump_", - "_mfm_bounce_", - "_mfm_flip_", - "_mfm_x2_", - "_mfm_x3_", - "_mfm_x4_", - "_mfm_blur_", - "_mfm_rainbow_", - "_mfm_rotate_" + "mfm _mfm_tada_", + "mfm _mfm_jelly_", + "mfm _mfm_twitch_", + "mfm _mfm_shake_", + "mfm _mfm_spin_", + "mfm _mfm_jump_", + "mfm _mfm_bounce_", + "mfm _mfm_flip_", + "mfm _mfm_x2_", + "mfm _mfm_x3_", + "mfm _mfm_x4_", + "mfm _mfm_blur_", + "mfm _mfm_rainbow_", + "mfm _mfm_rotate_" ]) Meta.allow_tag_with_these_attributes(:span, [ From 92ba2802fb0d71a0bbca676ecc3af40c0a27db53 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Wed, 24 Aug 2022 14:36:33 +0000 Subject: [PATCH 19/44] generate-keys-at-registration-time (#181) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/181 --- lib/mix/pleroma.ex | 11 +++++- lib/mix/tasks/pleroma/user.ex | 39 +++++++++++++++++++ lib/pleroma/user.ex | 7 ++++ test/pleroma/user_test.exs | 3 +- .../article_note_page_validator_test.exs | 2 +- 5 files changed, 59 insertions(+), 3 deletions(-) diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index f4a6bcf63..6431f0a1c 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -23,7 +23,15 @@ def start_pleroma do Pleroma.Config.Oban.warn() Pleroma.Application.limiters_setup() Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) - Finch.start_link(name: MyFinch) + + proxy_url = Pleroma.Config.get([:http, :proxy_url]) + proxy = Pleroma.HTTP.AdapterHelper.format_proxy(proxy_url) + + finch_config = + [:http, :adapter] + |> Pleroma.Config.get([]) + |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy_pool(proxy) + |> Keyword.put(:name, MyFinch) unless System.get_env("DEBUG") do Logger.remove_backend(:console) @@ -45,6 +53,7 @@ def start_pleroma do Pleroma.Emoji, {Pleroma.Config.TransferTask, false}, Pleroma.Web.Endpoint, + {Finch, finch_config}, {Oban, oban_config}, {Majic.Pool, [name: Pleroma.MajicPool, pool_size: Pleroma.Config.get([:majic_pool, :size], 2)]} diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index d2d416655..f420d68bb 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -258,6 +258,25 @@ def run(["untag", nickname | tags]) do end end + def run(["refetch_public_keys"]) do + start_pleroma() + + Pleroma.User.Query.build(%{ + external: true, + is_active: true + }) + |> refetch_public_keys() + end + + def run(["refetch_public_keys" | rest]) do + start_pleroma() + + Pleroma.User.Query.build(%{ + ap_id: rest + }) + |> refetch_public_keys() + end + def run(["invite" | rest]) do {options, [], []} = OptionParser.parse(rest, @@ -519,6 +538,26 @@ def run(["fix_follow_state", local_user, remote_user]) do end end + defp refetch_public_keys(query) do + query + |> Pleroma.Repo.chunk_stream(50, :batches) + |> Stream.each(fn users -> + users + |> Enum.each(fn user -> + IO.puts("Re-Resolving: #{user.ap_id}") + + with {:ok, user} <- Pleroma.User.fetch_by_ap_id(user.ap_id), + changeset <- Pleroma.User.update_changeset(user), + {:ok, _user} <- Pleroma.User.update_and_set_cache(changeset) do + :ok + else + error -> IO.puts("Could not resolve: #{user.ap_id}, #{inspect(error)}") + end + end) + end) + |> Stream.run() + end + defp set_moderator(user, value) do {:ok, user} = user diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 2a1b5af94..4383f8f53 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -681,6 +681,7 @@ def register_changeset_ldap(struct, params = %{password: password}) |> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames])) |> validate_format(:nickname, local_nickname_regex()) |> put_ap_id() + |> put_keys() |> unique_constraint(:ap_id) |> put_following_and_follower_and_featured_address() end @@ -740,6 +741,7 @@ def register_changeset(struct, params \\ %{}, opts \\ []) do |> validate_length(:registration_reason, max: reason_limit) |> maybe_validate_required_email(opts[:external]) |> put_password_hash + |> put_keys() |> put_ap_id() |> unique_constraint(:ap_id) |> put_following_and_follower_and_featured_address() @@ -755,6 +757,11 @@ def maybe_validate_required_email(changeset, _) do end end + def put_keys(changeset) do + {:ok, pem} = Keys.generate_rsa_pem() + put_change(changeset, :keys, pem) + end + def put_ap_id(changeset) do ap_id = ap_id(%User{nickname: get_field(changeset, :nickname)}) put_change(changeset, :ap_id, ap_id) diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 67136e95b..645622e43 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -620,13 +620,14 @@ test "it blocks blacklisted email domains" do assert changeset.valid? end - test "it sets the password_hash and ap_id" do + test "it sets the password_hash, ap_id and PEM key" do changeset = User.register_changeset(%User{}, @full_user_data) assert changeset.valid? assert is_binary(changeset.changes[:password_hash]) assert changeset.changes[:ap_id] == User.ap_id(%User{nickname: @full_user_data.nickname}) + assert is_binary(changeset.changes[:keys]) assert changeset.changes.follower_address == "#{changeset.changes.ap_id}/followers" end diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index c766414a6..1d73d6765 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -115,7 +115,7 @@ test "a misskey MFM status with a content field should work and be linked", _ do assert content =~ "@oops_not_a_mention" assert content =~ - "mfm goes here

aaa" + "mfm goes here

aaa" end test "a misskey MFM status with a _misskey_content field should work and be linked", _ do From 017b50550b88c9320ed111a54fd770d19dc1416d Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Wed, 24 Aug 2022 15:38:02 +0100 Subject: [PATCH 20/44] add changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 41b352f21..3966bb10b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Compatibility with latest meilisearch - Resolution of nested mix tasks (i.e search.meilisearch) in OTP releases - Elasticsearch returning likes and repeats, displaying as posts +- Ensure key generation happens at registration-time to prevent potential race-conditions ### Removed - Non-finch HTTP adapters. `:tesla, :adapter` is now highly recommended to be set to the default. From 618cf7ff7f60f9a81a5d85160416107032ad7b34 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Thu, 25 Aug 2022 14:37:51 +0000 Subject: [PATCH 21/44] reuse valid oauth tokens (#182) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/182 --- lib/pleroma/helpers/auth_helper.ex | 13 ++ .../controllers/auth_controller.ex | 3 +- lib/pleroma/web/o_auth/authorization.ex | 5 + lib/pleroma/web/o_auth/o_auth_controller.ex | 32 ++++- lib/pleroma/web/o_auth/token.ex | 18 +++ lib/pleroma/web/o_auth/token/query.ex | 13 ++ .../web/o_auth/o_auth_controller_test.exs | 123 ++++++++++++++++++ 7 files changed, 202 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/helpers/auth_helper.ex b/lib/pleroma/helpers/auth_helper.ex index 13e4c8158..37765da4d 100644 --- a/lib/pleroma/helpers/auth_helper.ex +++ b/lib/pleroma/helpers/auth_helper.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Helpers.AuthHelper do import Plug.Conn @oauth_token_session_key :oauth_token + @oauth_user_session_key :oauth_user @doc """ Skips OAuth permissions (scopes) checks, assigns nil `:token`. @@ -43,4 +44,16 @@ def put_session_token(%Conn{} = conn, token) when is_binary(token) do def delete_session_token(%Conn{} = conn) do delete_session(conn, @oauth_token_session_key) end + + def put_session_user(%Conn{} = conn, user) do + put_session(conn, @oauth_user_session_key, user) + end + + def delete_session_user(%Conn{} = conn) do + delete_session(conn, @oauth_user_session_key) + end + + def get_session_user(%Conn{} = conn) do + get_session(conn, @oauth_user_session_key) + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex index 4920d65da..a9ccaa982 100644 --- a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex @@ -27,7 +27,8 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do def login(conn, %{"code" => auth_token} = params) do with {:ok, app} <- local_mastofe_app(), {:ok, auth} <- Authorization.get_by_token(app, auth_token), - {:ok, oauth_token} <- Token.exchange_token(app, auth) do + %User{} = user <- User.get_cached_by_id(auth.user_id), + {:ok, oauth_token} <- Token.get_or_exchange_token(auth, app, user) do redirect_to = conn |> local_mastodon_post_login_path() diff --git a/lib/pleroma/web/o_auth/authorization.ex b/lib/pleroma/web/o_auth/authorization.ex index e0ecb0f4f..e56704164 100644 --- a/lib/pleroma/web/o_auth/authorization.ex +++ b/lib/pleroma/web/o_auth/authorization.ex @@ -94,4 +94,9 @@ def get_by_token(%App{id: app_id} = _app, token) do from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token) |> Repo.find_resource() end + + def get_preeexisting_by_app_and_user(%App{id: app_id} = _app, %User{id: user_id} = _user) do + from(t in __MODULE__, where: t.app_id == ^app_id and t.user_id == ^user_id, limit: 1) + |> Repo.find_resource() + end end diff --git a/lib/pleroma/web/o_auth/o_auth_controller.ex b/lib/pleroma/web/o_auth/o_auth_controller.ex index 358120fe6..455af11d7 100644 --- a/lib/pleroma/web/o_auth/o_auth_controller.ex +++ b/lib/pleroma/web/o_auth/o_auth_controller.ex @@ -59,18 +59,39 @@ def authorize(%Plug.Conn{assigns: %{token: %Token{}}} = conn, %{"force_login" => # after user already authorized to MastodonFE. # So we have to check client and token. def authorize( - %Plug.Conn{assigns: %{token: %Token{} = token}} = conn, + %Plug.Conn{assigns: %{token: %Token{} = token, user: %User{} = user}} = conn, %{"client_id" => client_id} = params ) do with %Token{} = t <- Repo.get_by(Token, token: token.token) |> Repo.preload(:app), ^client_id <- t.app.client_id do handle_existing_authorization(conn, params) + else + _ -> + maybe_reuse_token(conn, params, user.id) + end + end + + def authorize(%Plug.Conn{} = conn, params) do + # if we have a user in the session, attempt to authenticate as them + # otherwise show the login form + maybe_reuse_token(conn, params, AuthHelper.get_session_user(conn)) + end + + defp maybe_reuse_token(conn, params, user_id) when is_binary(user_id) do + with %User{} = user <- User.get_cached_by_id(user_id), + %App{} = app <- Repo.get_by(App, client_id: params["client_id"]), + {:ok, %Token{} = token} <- Token.get_preeexisting_by_app_and_user(app, user), + {:ok, %Authorization{} = auth} <- + Authorization.get_preeexisting_by_app_and_user(app, user) do + conn + |> assign(:token, token) + |> after_create_authorization(auth, %{"authorization" => params}) else _ -> do_authorize(conn, params) end end - def authorize(%Plug.Conn{} = conn, params), do: do_authorize(conn, params) + defp maybe_reuse_token(conn, params, _user), do: do_authorize(conn, params) defp do_authorize(%Plug.Conn{} = conn, params) do app = Repo.get_by(App, client_id: params["client_id"]) @@ -148,7 +169,9 @@ def create_authorization(%Plug.Conn{assigns: %{user: %User{} = user}} = conn, pa def create_authorization(%Plug.Conn{} = conn, %{"authorization" => _} = params, opts) do with {:ok, auth, user} <- do_create_authorization(conn, params, opts[:user]), {:mfa_required, _, _, false} <- {:mfa_required, user, auth, MFA.require?(user)} do - after_create_authorization(conn, auth, params) + conn + |> AuthHelper.put_session_user(user.id) + |> after_create_authorization(auth, params) else error -> handle_create_authorization_error(conn, error, params) @@ -269,7 +292,7 @@ def token_exchange(%Plug.Conn{} = conn, %{"grant_type" => "authorization_code"} fixed_token = Token.Utils.fix_padding(params["code"]), {:ok, auth} <- Authorization.get_by_token(app, fixed_token), %User{} = user <- User.get_cached_by_id(auth.user_id), - {:ok, token} <- Token.exchange_token(app, auth) do + {:ok, token} <- Token.get_or_exchange_token(auth, app, user) do after_token_exchange(conn, %{user: user, token: token}) else error -> @@ -321,6 +344,7 @@ def token_exchange(%Plug.Conn{} = conn, params), do: bad_request(conn, params) def after_token_exchange(%Plug.Conn{} = conn, %{token: token} = view_params) do conn |> AuthHelper.put_session_token(token.token) + |> AuthHelper.put_session_user(token.user_id) |> json(OAuthView.render("token.json", view_params)) end diff --git a/lib/pleroma/web/o_auth/token.ex b/lib/pleroma/web/o_auth/token.ex index 9d69e9db4..c9398aeaa 100644 --- a/lib/pleroma/web/o_auth/token.ex +++ b/lib/pleroma/web/o_auth/token.ex @@ -70,6 +70,16 @@ def exchange_token(app, auth) do end end + def get_preeexisting_by_app_and_user(app, user) do + Query.get_by_app(app.id) + |> Query.get_by_user(user.id) + |> Query.get_unexpired() + |> Query.preload([:user]) + |> Query.sort_by_inserted_at() + |> Query.limit(1) + |> Repo.find_resource() + end + defp put_token(changeset) do changeset |> change(%{token: Token.Utils.generate_token()}) @@ -86,6 +96,14 @@ defp put_refresh_token(changeset, attrs) do |> unique_constraint(:refresh_token) end + def get_or_exchange_token(%Authorization{} = auth, %App{} = app, %User{} = user) do + if auth.used do + get_preeexisting_by_app_and_user(app, user) + else + exchange_token(app, auth) + end + end + defp put_valid_until(changeset, attrs) do valid_until = Map.get(attrs, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), lifespan())) diff --git a/lib/pleroma/web/o_auth/token/query.ex b/lib/pleroma/web/o_auth/token/query.ex index d16a759d8..662e7856d 100644 --- a/lib/pleroma/web/o_auth/token/query.ex +++ b/lib/pleroma/web/o_auth/token/query.ex @@ -38,6 +38,19 @@ def get_by_user(query \\ Token, user_id) do from(q in query, where: q.user_id == ^user_id) end + def get_unexpired(query) do + now = NaiveDateTime.utc_now() + from(q in query, where: q.valid_until > ^now) + end + + def limit(query, limit) do + from(q in query, limit: ^limit) + end + + def sort_by_inserted_at(query) do + from(q in query, order_by: [desc: :updated_at]) + end + @spec preload(query, any) :: query def preload(query \\ Token, assoc_preload \\ []) diff --git a/test/pleroma/web/o_auth/o_auth_controller_test.exs b/test/pleroma/web/o_auth/o_auth_controller_test.exs index 0fdd5b8e9..5a1258ec3 100644 --- a/test/pleroma/web/o_auth/o_auth_controller_test.exs +++ b/test/pleroma/web/o_auth/o_auth_controller_test.exs @@ -494,6 +494,129 @@ test "renders authentication page if user is already authenticated but user requ assert html_response(conn, 200) =~ ~s(type="submit") end + test "allows access if the user has a prior authorization but is authenticated with another client", + %{ + app: app, + conn: conn + } do + user = insert(:user) + token = insert(:oauth_token, app: app, user: user) + + other_app = insert(:oauth_app, redirect_uris: "https://other_redirect.url") + authorization = insert(:oauth_authorization, user: user, app: other_app) + _reusable_token = insert(:oauth_token, app: other_app, user: user) + + conn = + conn + |> AuthHelper.put_session_token(token.token) + |> AuthHelper.put_session_user(user.id) + |> get( + "/oauth/authorize", + %{ + "response_type" => "code", + "client_id" => other_app.client_id, + "redirect_uri" => OAuthController.default_redirect_uri(other_app), + "scope" => "read" + } + ) + + assert URI.decode(redirected_to(conn)) == + "https://other_redirect.url?code=#{authorization.token}" + end + + test "renders login page if the user has an authorization but no token", + %{ + app: app, + conn: conn + } do + user = insert(:user) + token = insert(:oauth_token, app: app, user: user) + + other_app = insert(:oauth_app, redirect_uris: "https://other_redirect.url") + _authorization = insert(:oauth_authorization, user: user, app: other_app) + + conn = + conn + |> AuthHelper.put_session_token(token.token) + |> AuthHelper.put_session_user(user.id) + |> get( + "/oauth/authorize", + %{ + "response_type" => "code", + "client_id" => other_app.client_id, + "redirect_uri" => OAuthController.default_redirect_uri(other_app), + "scope" => "read" + } + ) + + assert html_response(conn, 200) =~ ~s(type="submit") + end + + test "does not reuse other people's tokens", + %{ + app: app, + conn: conn + } do + user = insert(:user) + other_user = insert(:user) + token = insert(:oauth_token, app: app, user: user) + + other_app = insert(:oauth_app, redirect_uris: "https://other_redirect.url") + _authorization = insert(:oauth_authorization, user: other_user, app: other_app) + _reusable_token = insert(:oauth_token, app: other_app, user: other_user) + + conn = + conn + |> AuthHelper.put_session_token(token.token) + |> AuthHelper.put_session_user(user.id) + |> get( + "/oauth/authorize", + %{ + "response_type" => "code", + "client_id" => other_app.client_id, + "redirect_uri" => OAuthController.default_redirect_uri(other_app), + "scope" => "read" + } + ) + + assert html_response(conn, 200) =~ ~s(type="submit") + end + + test "does not reuse expired tokens", + %{ + app: app, + conn: conn + } do + user = insert(:user) + token = insert(:oauth_token, app: app, user: user) + + other_app = insert(:oauth_app, redirect_uris: "https://other_redirect.url") + _authorization = insert(:oauth_authorization, user: user, app: other_app) + + _reusable_token = + insert(:oauth_token, + app: other_app, + user: user, + valid_until: NaiveDateTime.add(NaiveDateTime.utc_now(), -100) + ) + + conn = + conn + |> AuthHelper.put_session_token(token.token) + |> AuthHelper.put_session_user(user.id) + |> get( + "/oauth/authorize", + %{ + "response_type" => "code", + "client_id" => other_app.client_id, + "redirect_uri" => OAuthController.default_redirect_uri(other_app), + "scope" => "read" + } + ) + + assert html_response(conn, 200) =~ ~s(type="submit") + end + test "with existing authentication and non-OOB `redirect_uri`, redirects to app with `token` and `state` params", %{ app: app, From e4f2251e0f5744de66b9b3bee2a0086bc7ab2bb1 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Thu, 25 Aug 2022 16:11:21 +0000 Subject: [PATCH 22/44] Add support for setting language in instance metadata (#183) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/183 --- CHANGELOG.md | 2 ++ config/config.exs | 1 + config/description.exs | 10 ++++++++++ lib/pleroma/web/mastodon_api/views/instance_view.ex | 2 +- .../controllers/instance_controller_test.exs | 5 +++-- 5 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3966bb10b..df34d2bb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Added - support for fedibird-fe, and non-breaking API parity for it to function +- support for setting instance languages in metadata +- support for reusing oauth tokens, and not requiring new authorizations ### Changed - MFM parsing is now done on the backend by a modified version of ilja's parser -> https://akkoma.dev/AkkomaGang/mfm-parser diff --git a/config/config.exs b/config/config.exs index 83977da19..ec82e872a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -197,6 +197,7 @@ avatar_upload_limit: 2_000_000, background_upload_limit: 4_000_000, banner_upload_limit: 4_000_000, + languages: ["en"], poll_limits: %{ max_options: 20, max_option_chars: 200, diff --git a/config/description.exs b/config/description.exs index b70982cd2..61ef8f449 100644 --- a/config/description.exs +++ b/config/description.exs @@ -509,6 +509,16 @@ "Pleroma" ] }, + %{ + key: :languages, + type: {:list, :string}, + description: "Languages the instance uses", + suggestions: [ + "en", + "ja", + "fr" + ] + }, %{ key: :email, label: "Admin Email Address", diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 4cfa8d85c..7ae357e23 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -26,7 +26,7 @@ def render("show.json", _) do thumbnail: URI.merge(Pleroma.Web.Endpoint.url(), Keyword.get(instance, :instance_thumbnail)) |> to_string, - languages: ["en"], + languages: Keyword.get(instance, :languages, ["en"]), registrations: Keyword.get(instance, :registrations_open), approval_required: Keyword.get(instance, :account_approval_required), # Extra (not present in Mastodon): diff --git a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs index 90801a90a..bc3d35819 100644 --- a/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/instance_controller_test.exs @@ -10,10 +10,11 @@ defmodule Pleroma.Web.MastodonAPI.InstanceControllerTest do import Pleroma.Factory test "get instance information", %{conn: conn} do + clear_config([:instance, :languages], ["en", "ja"]) conn = get(conn, "/api/v1/instance") assert result = json_response_and_validate_schema(conn, 200) - email = Pleroma.Config.get([:instance, :email]) + thumbnail = Pleroma.Web.Endpoint.url() <> Pleroma.Config.get([:instance, :instance_thumbnail]) background = Pleroma.Web.Endpoint.url() <> Pleroma.Config.get([:instance, :background_image]) @@ -29,7 +30,7 @@ test "get instance information", %{conn: conn} do }, "stats" => _, "thumbnail" => from_config_thumbnail, - "languages" => _, + "languages" => ["en", "ja"], "registrations" => _, "approval_required" => _, "poll_limits" => _, From db7ad08d1e2d4c59b52a41bb0790c073fda1649f Mon Sep 17 00:00:00 2001 From: Norm Date: Thu, 25 Aug 2022 18:30:19 +0000 Subject: [PATCH 23/44] Update min elixir version in mix.exs to 1.12 The install docs already mention 1.12 as the minimum supported version, so this should also be reflected in `mix.exs`. --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index e7f491997..56c854021 100644 --- a/mix.exs +++ b/mix.exs @@ -5,7 +5,7 @@ def project do [ app: :pleroma, version: version("3.1.0"), - elixir: "~> 1.9", + elixir: "~> 1.12", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(), elixirc_options: [warnings_as_errors: warnings_as_errors()], From 85137f591f95169d385d690c48dcbeccb1306058 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sat, 27 Aug 2022 11:57:57 +0100 Subject: [PATCH 24/44] Add ability to obfuscate domains in MRF transparency --- CHANGELOG.md | 1 + config/config.exs | 3 +- docs/docs/configuration/cheatsheet.md | 1 + lib/pleroma/web/activity_pub/mrf.ex | 10 +++++ .../web/activity_pub/mrf/simple_policy.ex | 31 +++++++++++++++- .../activity_pub/mrf/simple_policy_test.exs | 37 +++++++++++++++++++ 6 files changed, 80 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df34d2bb6..183a60d10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - support for fedibird-fe, and non-breaking API parity for it to function - support for setting instance languages in metadata - support for reusing oauth tokens, and not requiring new authorizations +- the ability to obfuscate domains in your MRF descriptions ### Changed - MFM parsing is now done on the backend by a modified version of ilja's parser -> https://akkoma.dev/AkkomaGang/mfm-parser diff --git a/config/config.exs b/config/config.exs index ec82e872a..5ae7a33a2 100644 --- a/config/config.exs +++ b/config/config.exs @@ -794,7 +794,8 @@ config :pleroma, :mrf, policies: [Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy, Pleroma.Web.ActivityPub.MRF.TagPolicy], transparency: true, - transparency_exclusions: [] + transparency_exclusions: [], + transparency_obfuscate_domains: [] config :ex_aws, http_client: Pleroma.HTTP.ExAws diff --git a/docs/docs/configuration/cheatsheet.md b/docs/docs/configuration/cheatsheet.md index 8fa188de1..a29db208c 100644 --- a/docs/docs/configuration/cheatsheet.md +++ b/docs/docs/configuration/cheatsheet.md @@ -120,6 +120,7 @@ To add configuration to your config file, you can copy it from the base config. * `Pleroma.Web.ActivityPub.MRF.KeywordPolicy`: Rejects or removes from the federated timeline or replaces keywords. (See [`:mrf_keyword`](#mrf_keyword)). * `transparency`: Make the content of your Message Rewrite Facility settings public (via nodeinfo). * `transparency_exclusions`: Exclude specific instance names from MRF transparency. The use of the exclusions feature will be disclosed in nodeinfo as a boolean value. +* `transparency_obfuscate_domains`: Show domains with `*` in the middle, to censor them if needed. For example, `ridingho.me` will show as `rid*****.me` ## Federation ### MRF policies diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index bd6f6777f..5606dac83 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -41,6 +41,16 @@ defmodule Pleroma.Web.ActivityPub.MRF do suggestions: [ "exclusion.com" ] + }, + %{ + key: :transparency_obfuscate_domains, + label: "MRF domain obfuscation", + type: {:list, :string}, + description: + "Obfuscate domains in MRF transparency. This is useful if the domain you're blocking contains words you don't want displayed, but still want to disclose the MRF settings.", + suggestions: [ + "badword.com" + ] } ] } diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index c631cc85f..415c5d2dd 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -256,10 +256,35 @@ def filter(object) when is_binary(object) do def filter(object), do: {:ok, object} + defp obfuscate(string) when is_binary(string) do + string + |> to_charlist() + |> Enum.with_index() + |> Enum.map(fn + {?., _index} -> + ?. + + {char, index} -> + if 3 <= index && index < String.length(string) - 3, do: ?*, else: char + end) + |> to_string() + end + + defp maybe_obfuscate(host, obfuscations) do + if MRF.subdomain_match?(obfuscations, host) do + obfuscate(host) + else + host + end + end + @impl true def describe do exclusions = Config.get([:mrf, :transparency_exclusions]) |> MRF.instance_list_from_tuples() + obfuscations = + Config.get([:mrf, :transparency_obfuscate_domains], []) |> MRF.subdomains_regex() + mrf_simple_excluded = Config.get(:mrf_simple) |> Enum.map(fn {rule, instances} -> @@ -269,7 +294,7 @@ def describe do mrf_simple = mrf_simple_excluded |> Enum.map(fn {rule, instances} -> - {rule, Enum.map(instances, fn {host, _} -> host end)} + {rule, Enum.map(instances, fn {host, _} -> maybe_obfuscate(host, obfuscations) end)} end) |> Map.new() @@ -286,7 +311,9 @@ def describe do |> Enum.map(fn {rule, instances} -> instances = instances - |> Enum.map(fn {host, reason} -> {host, %{"reason" => reason}} end) + |> Enum.map(fn {host, reason} -> + {maybe_obfuscate(host, obfuscations), %{"reason" => reason}} + end) |> Map.new() {rule, instances} diff --git a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs index 0a0f51bdb..0569bfed3 100644 --- a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs @@ -216,6 +216,43 @@ test "has a matching host but only as:Public in to" do end end + describe "describe/1" do + test "returns a description of the policy" do + clear_config([:mrf_simple, :reject], [ + {"remote.instance", "did not give my catboy a burg"} + ]) + + assert {:ok, %{mrf_simple: %{reject: ["remote.instance"]}}} = SimplePolicy.describe() + end + + test "excludes domains listed in :transparency_exclusions" do + clear_config([:mrf, :transparency_exclusions], [{"remote.instance", ":("}]) + + clear_config([:mrf_simple, :reject], [ + {"remote.instance", "did not give my catboy a burg"} + ]) + + {:ok, description} = SimplePolicy.describe() + assert %{mrf_simple: %{reject: []}} = description + assert description[:mrf_simple_info][:reject] == nil + end + + test "obfuscates domains listed in :transparency_obfuscate_domains" do + clear_config([:mrf, :transparency_obfuscate_domains], ["remote.instance", "a.b"]) + + clear_config([:mrf_simple, :reject], [ + {"remote.instance", "did not give my catboy a burg"}, + {"a.b", "spam-poked me on facebook in 2006"} + ]) + + assert {:ok, + %{ + mrf_simple: %{reject: ["rem***.*****nce", "a.b"]}, + mrf_simple_info: %{reject: %{"rem***.*****nce" => %{}}} + }} = SimplePolicy.describe() + end + end + defp build_ftl_actor_and_message do actor = insert(:user) From 772c209914d5cbfd4f763edc266d0f1541f656f8 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Sat, 27 Aug 2022 18:05:48 +0000 Subject: [PATCH 25/44] GTS: cherry-picks and collection usage (#186) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3725?commit_id=61254111e59f02118cad15de49d1e0704c07030e what is this, a yoink of a yoink? good times Co-authored-by: Hélène Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/186 --- lib/pleroma/collections/fetcher.ex | 38 +++++++----- .../article_note_page_validator.ex | 19 ++---- .../object_validators/common_fixes.ex | 29 ++++++++- .../create_generic_validator.ex | 3 +- .../web/activity_pub/transmogrifier.ex | 26 +------- lib/pleroma/web/plugs/http_signature_plug.ex | 8 ++- .../collections/collections_fetcher_test.exs | 59 +++++++++++++++++-- .../activity_pub_controller_test.exs | 11 ++++ .../article_note_page_validator_test.exs | 13 ++++ .../transmogrifier/note_handling_test.exs | 1 - .../web/plugs/http_signature_plug_test.exs | 6 +- test/support/http_request_mock.ex | 9 +++ 12 files changed, 155 insertions(+), 67 deletions(-) diff --git a/lib/pleroma/collections/fetcher.ex b/lib/pleroma/collections/fetcher.ex index 0c81f0b56..ab69f4b84 100644 --- a/lib/pleroma/collections/fetcher.ex +++ b/lib/pleroma/collections/fetcher.ex @@ -11,10 +11,7 @@ defmodule Akkoma.Collections.Fetcher do alias Pleroma.Config require Logger - def fetch_collection_by_ap_id(ap_id) when is_binary(ap_id) do - fetch_collection(ap_id) - end - + @spec fetch_collection(String.t() | map()) :: {:ok, [Pleroma.Object.t()]} | {:error, any()} def fetch_collection(ap_id) when is_binary(ap_id) do with {:ok, page} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do {:ok, objects_from_collection(page)} @@ -26,7 +23,7 @@ def fetch_collection(ap_id) when is_binary(ap_id) do end def fetch_collection(%{"type" => type} = page) - when type in ["Collection", "OrderedCollection"] do + when type in ["Collection", "OrderedCollection", "CollectionPage", "OrderedCollectionPage"] do {:ok, objects_from_collection(page)} end @@ -38,12 +35,13 @@ defp items_in_page(%{"type" => type, "items" => items}) when is_list(items) and type in ["Collection", "CollectionPage"], do: items - defp objects_from_collection(%{"type" => "OrderedCollection", "orderedItems" => items}) - when is_list(items), - do: items + defp objects_from_collection(%{"type" => type, "orderedItems" => items} = page) + when is_list(items) and type in ["OrderedCollection", "OrderedCollectionPage"], + do: maybe_next_page(page, items) - defp objects_from_collection(%{"type" => "Collection", "items" => items}) when is_list(items), - do: items + defp objects_from_collection(%{"type" => type, "items" => items} = page) + when is_list(items) and type in ["Collection", "CollectionPage"], + do: maybe_next_page(page, items) defp objects_from_collection(%{"type" => type, "first" => first}) when is_binary(first) and type in ["Collection", "OrderedCollection"] do @@ -55,17 +53,27 @@ defp objects_from_collection(%{"type" => type, "first" => %{"id" => id}}) fetch_page_items(id) end + defp objects_from_collection(_page), do: [] + defp fetch_page_items(id, items \\ []) do if Enum.count(items) >= Config.get([:activitypub, :max_collection_objects]) do items else - {:ok, page} = Fetcher.fetch_and_contain_remote_object_from_id(id) - objects = items_in_page(page) + with {:ok, page} <- Fetcher.fetch_and_contain_remote_object_from_id(id) do + objects = items_in_page(page) - if Enum.count(objects) > 0 do - maybe_next_page(page, items ++ objects) + if Enum.count(objects) > 0 do + maybe_next_page(page, items ++ objects) + else + items + end else - items + {:error, "Object has been deleted"} -> + items + + {:error, error} -> + Logger.error("Could not fetch page #{id} - #{inspect(error)}") + {:error, error} end end end diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index 28053ea3a..55323bc2e 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do use Ecto.Schema alias Pleroma.User alias Pleroma.EctoType.ActivityPub.ObjectValidators - alias Pleroma.Object.Fetcher alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations @@ -58,19 +57,10 @@ defp fix_tag(%{"tag" => tag} = data) when is_list(tag), do: data defp fix_tag(%{"tag" => tag} = data) when is_map(tag), do: Map.put(data, "tag", [tag]) defp fix_tag(data), do: Map.drop(data, ["tag"]) - defp fix_replies(%{"replies" => %{"first" => %{"items" => replies}}} = data) - when is_list(replies), - do: Map.put(data, "replies", replies) - - defp fix_replies(%{"replies" => %{"items" => replies}} = data) when is_list(replies), - do: Map.put(data, "replies", replies) - - defp fix_replies(%{"replies" => replies} = data) when is_bitstring(replies), - do: Map.drop(data, ["replies"]) + defp fix_replies(%{"replies" => replies} = data) when is_list(replies), do: data defp fix_replies(%{"replies" => %{"first" => first}} = data) do - with {:ok, %{"orderedItems" => replies}} <- - Fetcher.fetch_and_contain_remote_object_from_id(first) do + with {:ok, replies} <- Akkoma.Collections.Fetcher.fetch_collection(first) do Map.put(data, "replies", replies) else {:error, _} -> @@ -79,7 +69,10 @@ defp fix_replies(%{"replies" => %{"first" => first}} = data) do end end - defp fix_replies(data), do: data + defp fix_replies(%{"replies" => %{"items" => replies}} = data) when is_list(replies), + do: Map.put(data, "replies", replies) + + defp fix_replies(data), do: Map.delete(data, "replies") defp remote_mention_resolver( %{"id" => ap_id, "tag" => tags}, diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 779c8b622..6fa2bbb99 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -7,8 +7,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do alias Pleroma.Object alias Pleroma.Object.Containment alias Pleroma.User - alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils + require Pleroma.Constants def cast_and_filter_recipients(message, field, follower_collection, field_fallback \\ []) do {:ok, data} = ObjectValidators.Recipients.cast(message[field] || field_fallback) @@ -32,7 +32,7 @@ def fix_object_defaults(data) do |> cast_and_filter_recipients("cc", follower_collection) |> cast_and_filter_recipients("bto", follower_collection) |> cast_and_filter_recipients("bcc", follower_collection) - |> Transmogrifier.fix_implicit_addressing(follower_collection) + |> fix_implicit_addressing(follower_collection) end def fix_activity_addressing(activity) do @@ -43,7 +43,7 @@ def fix_activity_addressing(activity) do |> cast_and_filter_recipients("cc", follower_collection) |> cast_and_filter_recipients("bto", follower_collection) |> cast_and_filter_recipients("bcc", follower_collection) - |> Transmogrifier.fix_implicit_addressing(follower_collection) + |> fix_implicit_addressing(follower_collection) end def fix_actor(data) do @@ -73,4 +73,27 @@ def fix_object_action_recipients(data, %Object{data: %{"actor" => actor}}) do Map.put(data, "to", to) end + + # if as:Public is addressed, then make sure the followers collection is also addressed + # so that the activities will be delivered to local users. + def fix_implicit_addressing(%{"to" => to, "cc" => cc} = object, followers_collection) do + recipients = to ++ cc + + if followers_collection not in recipients do + cond do + Pleroma.Constants.as_public() in cc -> + to = to ++ [followers_collection] + Map.put(object, "to", to) + + Pleroma.Constants.as_public() in to -> + cc = cc ++ [followers_collection] + Map.put(object, "cc", cc) + + true -> + object + end + else + object + end + end end diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index 803b5d5a1..d868c3915 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -13,7 +13,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations - alias Pleroma.Web.ActivityPub.Transmogrifier import Ecto.Changeset @@ -67,7 +66,7 @@ defp fix_addressing(data, object) do |> CommonFixes.cast_and_filter_recipients("cc", follower_collection, object["cc"]) |> CommonFixes.cast_and_filter_recipients("bto", follower_collection, object["bto"]) |> CommonFixes.cast_and_filter_recipients("bcc", follower_collection, object["bcc"]) - |> Transmogrifier.fix_implicit_addressing(follower_collection) + |> CommonFixes.fix_implicit_addressing(follower_collection) end def fix(data, meta) do diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index d2077967c..8ec4b0fec 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -19,6 +19,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes alias Pleroma.Web.Federator alias Pleroma.Workers.TransmogrifierWorker @@ -95,29 +96,6 @@ def fix_explicit_addressing(%{"to" => to, "cc" => cc} = object, follower_collect |> Map.put("cc", final_cc) end - # if as:Public is addressed, then make sure the followers collection is also addressed - # so that the activities will be delivered to local users. - def fix_implicit_addressing(%{"to" => to, "cc" => cc} = object, followers_collection) do - recipients = to ++ cc - - if followers_collection not in recipients do - cond do - Pleroma.Constants.as_public() in cc -> - to = to ++ [followers_collection] - Map.put(object, "to", to) - - Pleroma.Constants.as_public() in to -> - cc = cc ++ [followers_collection] - Map.put(object, "cc", cc) - - true -> - object - end - else - object - end - end - def fix_addressing(object) do {:ok, %User{follower_address: follower_collection}} = object @@ -130,7 +108,7 @@ def fix_addressing(object) do |> fix_addressing_list("bto") |> fix_addressing_list("bcc") |> fix_explicit_addressing(follower_collection) - |> fix_implicit_addressing(follower_collection) + |> CommonFixes.fix_implicit_addressing(follower_collection) end def fix_actor(%{"attributedTo" => actor} = object) do diff --git a/lib/pleroma/web/plugs/http_signature_plug.ex b/lib/pleroma/web/plugs/http_signature_plug.ex index cfee392c8..c906a4eec 100644 --- a/lib/pleroma/web/plugs/http_signature_plug.ex +++ b/lib/pleroma/web/plugs/http_signature_plug.ex @@ -27,11 +27,11 @@ def call(conn, _opts) do end end - def route_aliases(%{path_info: ["objects", id]}) do + def route_aliases(%{path_info: ["objects", id], query_string: query_string}) do ap_id = Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :object, id) with %Activity{} = activity <- Activity.get_by_object_ap_id_with_object(ap_id) do - ["/notice/#{activity.id}"] + ["/notice/#{activity.id}", "/notice/#{activity.id}?#{query_string}"] else _ -> [] end @@ -64,7 +64,9 @@ defp maybe_assign_valid_signature(conn) do if has_signature_header?(conn) do # set (request-target) header to the appropriate value # we also replace the digest header with the one we computed - possible_paths = route_aliases(conn) ++ [conn.request_path] + possible_paths = + route_aliases(conn) ++ [conn.request_path, conn.request_path <> "?#{conn.query_string}"] + assign_valid_signature_on_route_aliases(conn, possible_paths) else Logger.debug("No signature header!") diff --git a/test/pleroma/collections/collections_fetcher_test.exs b/test/pleroma/collections/collections_fetcher_test.exs index b9f84f5c4..7a582a3d7 100644 --- a/test/pleroma/collections/collections_fetcher_test.exs +++ b/test/pleroma/collections/collections_fetcher_test.exs @@ -30,7 +30,7 @@ test "it should extract items from an embedded array in a Collection" do } end) - {:ok, objects} = Fetcher.fetch_collection_by_ap_id(ap_id) + {:ok, objects} = Fetcher.fetch_collection(ap_id) assert [%{"type" => "Create"}, %{"type" => "Like"}] = objects end @@ -53,7 +53,7 @@ test "it should extract items from an embedded array in an OrderedCollection" do } end) - {:ok, objects} = Fetcher.fetch_collection_by_ap_id(ap_id) + {:ok, objects} = Fetcher.fetch_collection(ap_id) assert [%{"type" => "Create"}, %{"type" => "Like"}] = objects end @@ -106,7 +106,7 @@ test "it should extract items from an referenced first page in a Collection" do } end) - {:ok, objects} = Fetcher.fetch_collection_by_ap_id(ap_id) + {:ok, objects} = Fetcher.fetch_collection(ap_id) assert [%{"type" => "Create"}, %{"type" => "Like"}] = objects end @@ -161,7 +161,58 @@ test "it should stop fetching when we hit :max_collection_objects" do } end) - {:ok, objects} = Fetcher.fetch_collection_by_ap_id(ap_id) + {:ok, objects} = Fetcher.fetch_collection(ap_id) + assert [%{"type" => "Create"}] = objects + end + + test "it should stop fetching when we hit a 404" do + clear_config([:activitypub, :max_collection_objects], 1) + + unordered_collection = + "test/fixtures/collections/unordered_page_reference.json" + |> File.read!() + + first_page = + "test/fixtures/collections/unordered_page_first.json" + |> File.read!() + + ap_id = "https://example.com/collection/unordered_page_reference" + first_page_id = "https://example.com/collection/unordered_page_reference?page=1" + second_page_id = "https://example.com/collection/unordered_page_reference?page=2" + + Tesla.Mock.mock(fn + %{ + method: :get, + url: ^ap_id + } -> + %Tesla.Env{ + status: 200, + body: unordered_collection, + headers: [{"content-type", "application/activity+json"}] + } + + %{ + method: :get, + url: ^first_page_id + } -> + %Tesla.Env{ + status: 200, + body: first_page, + headers: [{"content-type", "application/activity+json"}] + } + + %{ + method: :get, + url: ^second_page_id + } -> + %Tesla.Env{ + status: 404, + body: nil, + headers: [{"content-type", "application/activity+json"}] + } + end) + + {:ok, objects} = Fetcher.fetch_collection(ap_id) assert [%{"type" => "Create"}] = objects end end diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index da5c87bd8..e209bb46b 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -782,6 +782,7 @@ test "mastodon pin/unpin", %{conn: conn} do |> String.replace("{{status_id}}", status_id) status_url = "https://example.com/users/lain/statuses/#{status_id}" + replies_url = status_url <> "/replies?only_other_accounts=true&page=true" user = File.read!("test/fixtures/users_mock/user.json") @@ -820,6 +821,16 @@ test "mastodon pin/unpin", %{conn: conn} do |> String.replace("{{nickname}}", "lain"), headers: [{"content-type", "application/activity+json"}] } + + %{ + method: :get, + url: ^replies_url + } -> + %Tesla.Env{ + status: 404, + body: "", + headers: [{"content-type", "application/activity+json"}] + } end) data = %{ diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index 1d73d6765..7c8e5a4e1 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -146,4 +146,17 @@ test "a misskey MFM status with a _misskey_content field should work and be link "@akkoma_user" end end + + test "a Note without replies/first/items validates" do + insert(:user, ap_id: "https://mastodon.social/users/emelie") + + note = + "test/fixtures/tesla_mock/status.emelie.json" + |> File.read!() + |> Jason.decode!() + |> pop_in(["replies", "first", "items"]) + |> elem(1) + + %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note) + end end diff --git a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs index 24df5ea61..002042802 100644 --- a/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier/note_handling_test.exs @@ -380,7 +380,6 @@ test "schedules background fetching of `replies` items if max thread depth limit clear_config([:instance, :federation_incoming_replies_max_depth], 10) {:ok, activity} = Transmogrifier.handle_incoming(data) - object = Object.normalize(activity.data["object"]) assert object.data["replies"] == items diff --git a/test/pleroma/web/plugs/http_signature_plug_test.exs b/test/pleroma/web/plugs/http_signature_plug_test.exs index 02e8b3092..8ce956510 100644 --- a/test/pleroma/web/plugs/http_signature_plug_test.exs +++ b/test/pleroma/web/plugs/http_signature_plug_test.exs @@ -86,10 +86,12 @@ test "halts the connection when `signature` header is not present", %{conn: conn test "aliases redirected /object endpoints", _ do obj = insert(:note) act = insert(:note_activity, note: obj) - params = %{"actor" => "http://mastodon.example.org/users/admin"} + params = %{"actor" => "someparam"} path = URI.parse(obj.data["id"]).path conn = build_conn(:get, path, params) - assert ["/notice/#{act.id}"] == HTTPSignaturePlug.route_aliases(conn) + + assert ["/notice/#{act.id}", "/notice/#{act.id}?actor=someparam"] == + HTTPSignaturePlug.route_aliases(conn) end end end diff --git a/test/support/http_request_mock.ex b/test/support/http_request_mock.ex index 476e0ce04..ab44c489b 100644 --- a/test/support/http_request_mock.ex +++ b/test/support/http_request_mock.ex @@ -407,6 +407,15 @@ def get("http://mastodon.example.org/users/admin", _, _, _) do }} end + def get( + "http://mastodon.example.org/users/admin/statuses/99512778738411822/replies?min_id=99512778738411824&page=true", + _, + _, + _ + ) do + {:ok, %Tesla.Env{status: 404, body: ""}} + end + def get("http://mastodon.example.org/users/relay", _, _, [ {"accept", "application/activity+json"} ]) do From 95e4018c1a17bd96331cdeb19d1c62a599061351 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Fri, 19 Aug 2022 13:19:38 -0400 Subject: [PATCH 26/44] Disconnect streaming sessions when token is revoked Use Websockex to replace websocket_client Test that server will disconnect websocket upon token revocation Lint Execute session disconnect in background Refactor streamer test allow multi-streams rebase websocket change --- lib/pleroma/application.ex | 3 +- .../web/mastodon_api/websocket_handler.ex | 6 +- .../web/o_auth/token/strategy/revoke.ex | 14 ++- lib/pleroma/web/streamer.ex | 24 ++++- mix.exs | 2 +- mix.lock | 2 +- .../integration/mastodon_websocket_test.exs | 37 +++++-- test/pleroma/web/streamer_test.exs | 101 ++++++++++++++++++ test/support/websocket_client.ex | 34 +++--- 9 files changed, 192 insertions(+), 31 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index cb619232f..e11e5495a 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -63,7 +63,8 @@ def start(_type, _args) do Pleroma.Repo, Config.TransferTask, Pleroma.Emoji, - Pleroma.Web.Plugs.RateLimiter.Supervisor + Pleroma.Web.Plugs.RateLimiter.Supervisor, + {Task.Supervisor, name: Pleroma.TaskSupervisor} ] ++ cachex_children() ++ http_children() ++ diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 582e65d70..bd7c56243 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -59,7 +59,7 @@ def websocket_init(state) do "#{__MODULE__} accepted websocket connection for user #{(state.user || %{id: "anonymous"}).id}, topic #{state.topic}" ) - Streamer.add_socket(state.topic, state.user) + Streamer.add_socket(state.topic, state.oauth_token) {:ok, %{state | timer: timer()}} end @@ -139,6 +139,10 @@ def websocket_info(:tick, state) do {:reply, :ping, %{state | timer: nil, count: 0}, :hibernate} end + def websocket_info(:close, state) do + {:stop, state} + end + # State can be `[]` only in case we terminate before switching to websocket, # we already log errors for these cases in `init/1`, so just do nothing here def terminate(_reason, _req, []), do: :ok diff --git a/lib/pleroma/web/o_auth/token/strategy/revoke.ex b/lib/pleroma/web/o_auth/token/strategy/revoke.ex index 8d6572704..de99bc137 100644 --- a/lib/pleroma/web/o_auth/token/strategy/revoke.ex +++ b/lib/pleroma/web/o_auth/token/strategy/revoke.ex @@ -21,6 +21,18 @@ def revoke(%App{} = app, %{"token" => token} = _attrs) do @doc "Revokes access token" @spec revoke(Token.t()) :: {:ok, Token.t()} | {:error, Ecto.Changeset.t()} def revoke(%Token{} = token) do - Repo.delete(token) + with {:ok, token} <- Repo.delete(token) do + Task.Supervisor.start_child( + Pleroma.TaskSupervisor, + Pleroma.Web.Streamer, + :close_streams_by_oauth_token, + [token], + restart: :transient + ) + + {:ok, token} + else + result -> result + end end end diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index d5b1d0678..fba5d1c02 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -36,7 +36,7 @@ def registry, do: @registry {:ok, topic :: String.t()} | {:error, :bad_topic} | {:error, :unauthorized} def get_topic_and_add_socket(stream, user, oauth_token, params \\ %{}) do with {:ok, topic} <- get_topic(stream, user, oauth_token, params) do - add_socket(topic, user) + add_socket(topic, oauth_token) end end @@ -124,10 +124,10 @@ def get_topic(_stream, _user, _oauth_token, _params) do end @doc "Registers the process for streaming. Use `get_topic/3` to get the full authorized topic." - def add_socket(topic, user) do + def add_socket(topic, oauth_token) do if should_env_send?() do - auth? = if user, do: true - Registry.register(@registry, topic, auth?) + oauth_token_id = if oauth_token, do: oauth_token.id, else: false + Registry.register(@registry, topic, oauth_token_id) end {:ok, topic} @@ -311,6 +311,22 @@ defp thread_containment(activity, user) do end end + def close_streams_by_oauth_token(oauth_token) do + if should_env_send?() do + Registry.select( + @registry, + [ + { + {:"$1", :"$2", :"$3"}, + [{:==, :"$3", oauth_token.id}], + [:"$2"] + } + ] + ) + |> Enum.each(fn pid -> send(pid, :close) end) + end + end + # In test environement, only return true if the registry is started. # In benchmark environment, returns false. # In any other environment, always returns true. diff --git a/mix.exs b/mix.exs index 768c8cbd6..ef038ce74 100644 --- a/mix.exs +++ b/mix.exs @@ -206,7 +206,7 @@ defp deps do # temporary downgrade for excoveralls, hackney until hackney max_connections bug will be fixed {:excoveralls, "0.12.3", only: :test}, {:mox, "~> 1.0", only: :test}, - {:websocket_client, git: "https://github.com/jeremyong/websocket_client.git", only: :test} + {:websockex, "~> 0.4.3", only: :test} ] ++ oauth_deps() end diff --git a/mix.lock b/mix.lock index bda77cf5a..7eeb5c138 100644 --- a/mix.lock +++ b/mix.lock @@ -120,5 +120,5 @@ "unsafe": {:hex, :unsafe, "1.0.1", "a27e1874f72ee49312e0a9ec2e0b27924214a05e3ddac90e91727bc76f8613d8", [:mix], [], "hexpm", "6c7729a2d214806450d29766abc2afaa7a2cbecf415be64f36a6691afebb50e5"}, "vex": {:hex, :vex, "0.9.0", "613ea5eb3055662e7178b83e25b2df0975f68c3d8bb67c1645f0573e1a78d606", [:mix], [], "hexpm", "c69fff44d5c8aa3f1faee71bba1dcab05dd36364c5a629df8bb11751240c857f"}, "web_push_encryption": {:hex, :web_push_encryption, "0.3.1", "76d0e7375142dfee67391e7690e89f92578889cbcf2879377900b5620ee4708d", [:mix], [{:httpoison, "~> 1.0", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jose, "~> 1.11.1", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "4f82b2e57622fb9337559058e8797cb0df7e7c9790793bdc4e40bc895f70e2a2"}, - "websocket_client": {:git, "https://github.com/jeremyong/websocket_client.git", "9a6f65d05ebf2725d62fb19262b21f1805a59fbf", []}, + "websockex": {:hex, :websockex, "0.4.3", "92b7905769c79c6480c02daacaca2ddd49de936d912976a4d3c923723b647bf0", [:mix], [], "hexpm", "95f2e7072b85a3a4cc385602d42115b73ce0b74a9121d0d6dbbf557645ac53e4"}, } diff --git a/test/pleroma/integration/mastodon_websocket_test.exs b/test/pleroma/integration/mastodon_websocket_test.exs index 356bfa48d..9e266868d 100644 --- a/test/pleroma/integration/mastodon_websocket_test.exs +++ b/test/pleroma/integration/mastodon_websocket_test.exs @@ -34,15 +34,20 @@ def start_socket(qs \\ nil, headers \\ []) do test "allows multi-streams" do capture_log(fn -> assert {:ok, _} = start_socket() - assert {:error, {404, _}} = start_socket("?stream=ncjdk") + + assert {:error, %WebSockex.RequestError{code: 404, message: "Not Found"}} = + start_socket("?stream=ncjdk") + Process.sleep(30) end) end test "requires authentication and a valid token for protected streams" do capture_log(fn -> - assert {:error, {401, _}} = start_socket("?stream=user&access_token=aaaaaaaaaaaa") - assert {:error, {401, _}} = start_socket("?stream=user") + assert {:error, %WebSockex.RequestError{code: 401}} = + start_socket("?stream=user&access_token=aaaaaaaaaaaa") + + assert {:error, %WebSockex.RequestError{code: 401}} = start_socket("?stream=user") Process.sleep(30) end) end @@ -91,7 +96,7 @@ test "receives well formatted events" do {:ok, token} = OAuth.Token.exchange_token(app, auth) - %{user: user, token: token} + %{app: app, user: user, token: token} end test "accepts valid tokens", state do @@ -102,7 +107,7 @@ test "accepts the 'user' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}") capture_log(fn -> - assert {:error, {401, _}} = start_socket("?stream=user") + assert {:error, %WebSockex.RequestError{code: 401}} = start_socket("?stream=user") Process.sleep(30) end) end @@ -111,7 +116,9 @@ test "accepts the 'user:notification' stream", %{token: token} = _state do assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}") capture_log(fn -> - assert {:error, {401, _}} = start_socket("?stream=user:notification") + assert {:error, %WebSockex.RequestError{code: 401}} = + start_socket("?stream=user:notification") + Process.sleep(30) end) end @@ -120,11 +127,27 @@ test "accepts valid token on Sec-WebSocket-Protocol header", %{token: token} do assert {:ok, _} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", token.token}]) capture_log(fn -> - assert {:error, {401, _}} = + assert {:error, %WebSockex.RequestError{code: 401}} = start_socket("?stream=user", [{"Sec-WebSocket-Protocol", "I am a friend"}]) Process.sleep(30) end) end + + test "disconnect when token is revoked", %{app: app, user: user, token: token} do + assert {:ok, _} = start_socket("?stream=user:notification&access_token=#{token.token}") + assert {:ok, _} = start_socket("?stream=user&access_token=#{token.token}") + + {:ok, auth} = OAuth.Authorization.create_authorization(app, user) + + {:ok, token2} = OAuth.Token.exchange_token(app, auth) + assert {:ok, _} = start_socket("?stream=user&access_token=#{token2.token}") + + OAuth.Token.Strategy.Revoke.revoke(token) + + assert_receive {:close, _} + assert_receive {:close, _} + refute_receive {:close, _} + end end end diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs index 07129ff11..9ae733fc6 100644 --- a/test/pleroma/web/streamer_test.exs +++ b/test/pleroma/web/streamer_test.exs @@ -760,4 +760,105 @@ test "it sends conversation update to the 'direct' stream when a message is dele assert last_status["id"] == to_string(create_activity.id) end end + + describe "stop streaming if token got revoked" do + setup do + child_proc = fn start, finalize -> + fn -> + start.() + + receive do + {StreamerTest, :ready} -> + assert_receive {:render_with_user, _, "update.json", _, _} + + receive do + {StreamerTest, :revoked} -> finalize.() + end + end + end + end + + starter = fn user, token -> + fn -> Streamer.get_topic_and_add_socket("user", user, token) end + end + + hit = fn -> assert_receive :close end + miss = fn -> refute_receive :close end + + send_all = fn tasks, thing -> Enum.each(tasks, &send(&1.pid, thing)) end + + %{ + child_proc: child_proc, + starter: starter, + hit: hit, + miss: miss, + send_all: send_all + } + end + + test "do not revoke other tokens", %{ + child_proc: child_proc, + starter: starter, + hit: hit, + miss: miss, + send_all: send_all + } do + %{user: user, token: token} = oauth_access(["read"]) + %{token: token2} = oauth_access(["read"], user: user) + %{user: user2, token: user2_token} = oauth_access(["read"]) + + post_user = insert(:user) + CommonAPI.follow(user, post_user) + CommonAPI.follow(user2, post_user) + + tasks = [ + Task.async(child_proc.(starter.(user, token), hit)), + Task.async(child_proc.(starter.(user, token2), miss)), + Task.async(child_proc.(starter.(user2, user2_token), miss)) + ] + + {:ok, _} = + CommonAPI.post(post_user, %{ + status: "hi" + }) + + send_all.(tasks, {StreamerTest, :ready}) + + Pleroma.Web.OAuth.Token.Strategy.Revoke.revoke(token) + + send_all.(tasks, {StreamerTest, :revoked}) + + Enum.each(tasks, &Task.await/1) + end + + test "revoke all streams for this token", %{ + child_proc: child_proc, + starter: starter, + hit: hit, + send_all: send_all + } do + %{user: user, token: token} = oauth_access(["read"]) + + post_user = insert(:user) + CommonAPI.follow(user, post_user) + + tasks = [ + Task.async(child_proc.(starter.(user, token), hit)), + Task.async(child_proc.(starter.(user, token), hit)) + ] + + {:ok, _} = + CommonAPI.post(post_user, %{ + status: "hi" + }) + + send_all.(tasks, {StreamerTest, :ready}) + + Pleroma.Web.OAuth.Token.Strategy.Revoke.revoke(token) + + send_all.(tasks, {StreamerTest, :revoked}) + + Enum.each(tasks, &Task.await/1) + end + end end diff --git a/test/support/websocket_client.ex b/test/support/websocket_client.ex index 34b955474..70d331999 100644 --- a/test/support/websocket_client.ex +++ b/test/support/websocket_client.ex @@ -5,18 +5,17 @@ defmodule Pleroma.Integration.WebsocketClient do # https://github.com/phoenixframework/phoenix/blob/master/test/support/websocket_client.exs + use WebSockex + @doc """ Starts the WebSocket server for given ws URL. Received Socket.Message's are forwarded to the sender pid """ def start_link(sender, url, headers \\ []) do - :crypto.start() - :ssl.start() - - :websocket_client.start_link( - String.to_charlist(url), + WebSockex.start_link( + url, __MODULE__, - [sender], + %{sender: sender}, extra_headers: headers ) end @@ -36,27 +35,32 @@ def send_text(server_pid, msg) do end @doc false - def init([sender], _conn_state) do - {:ok, %{sender: sender}} - end - - @doc false - def websocket_handle(frame, _conn_state, state) do + @impl true + def handle_frame(frame, state) do send(state.sender, frame) {:ok, state} end + @impl true + def handle_disconnect(conn_status, state) do + send(state.sender, {:close, conn_status}) + {:ok, state} + end + @doc false - def websocket_info({:text, msg}, _conn_state, state) do + @impl true + def handle_info({:text, msg}, state) do {:reply, {:text, msg}, state} end - def websocket_info(:close, _conn_state, _state) do + @impl true + def handle_info(:close, _state) do {:close, <<>>, "done"} end @doc false - def websocket_terminate(_reason, _conn_state, _state) do + @impl true + def terminate(_reason, _state) do :ok end end From 722e56b3086318d01210884b8aa0ae7945833bc6 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sat, 27 Aug 2022 19:12:15 +0100 Subject: [PATCH 27/44] add changelog entry --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 183a60d10..1a71255ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Resolution of nested mix tasks (i.e search.meilisearch) in OTP releases - Elasticsearch returning likes and repeats, displaying as posts - Ensure key generation happens at registration-time to prevent potential race-conditions +- Ensured websockets get closed on logout +- Allowed GoToSocial-style `?query_string` signatures ### Removed - Non-finch HTTP adapters. `:tesla, :adapter` is now highly recommended to be set to the default. From df39cab9c13e470e2b2d62550079ae5df0c18fec Mon Sep 17 00:00:00 2001 From: floatingghost Date: Mon, 29 Aug 2022 19:42:22 +0000 Subject: [PATCH 28/44] Automatic status translation (#187) Fixes #115 Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/187 --- CHANGELOG.md | 1 + config/config.exs | 13 +++ config/description.exs | 73 ++++++++++++++- docs/docs/configuration/cheatsheet.md | 25 +++++ lib/pleroma/akkoma/translators/deepl.ex | 58 ++++++++++++ .../akkoma/translators/libre_translate.ex | 51 +++++++++++ lib/pleroma/akkoma/translators/translator.ex | 3 + lib/pleroma/application.ex | 3 +- .../api_spec/operations/status_operation.ex | 36 ++++++++ .../controllers/status_controller.ex | 45 ++++++++- .../web/mastodon_api/views/instance_view.ex | 3 + lib/pleroma/web/router.ex | 1 + test/pleroma/translators/deepl_test.exs | 75 +++++++++++++++ .../translators/libre_translate_test.exs | 91 +++++++++++++++++++ .../controllers/status_controller_test.exs | 72 +++++++++++++++ 15 files changed, 543 insertions(+), 7 deletions(-) create mode 100644 lib/pleroma/akkoma/translators/deepl.ex create mode 100644 lib/pleroma/akkoma/translators/libre_translate.ex create mode 100644 lib/pleroma/akkoma/translators/translator.ex create mode 100644 test/pleroma/translators/deepl_test.exs create mode 100644 test/pleroma/translators/libre_translate_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a71255ff..05cb69c40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - support for setting instance languages in metadata - support for reusing oauth tokens, and not requiring new authorizations - the ability to obfuscate domains in your MRF descriptions +- automatic translation of statuses via DeepL or LibreTranslate ### Changed - MFM parsing is now done on the backend by a modified version of ilja's parser -> https://akkoma.dev/AkkomaGang/mfm-parser diff --git a/config/config.exs b/config/config.exs index 5ae7a33a2..330e572fe 100644 --- a/config/config.exs +++ b/config/config.exs @@ -843,6 +843,19 @@ } } +config :pleroma, :translator, + enabled: false, + module: Akkoma.Translators.DeepL + +config :pleroma, :deepl, + # either :free or :pro + tier: :free, + api_key: "" + +config :pleroma, :libre_translate, + url: "http://127.0.0.1:5000", + api_key: nil + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/description.exs b/config/description.exs index 61ef8f449..a17897b98 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3226,13 +3226,14 @@ group: :pleroma, key: Pleroma.Search, type: :group, + label: "Search", description: "General search settings.", children: [ %{ key: :module, - type: :keyword, + type: :module, description: "Selected search module.", - suggestion: [Pleroma.Search.DatabaseSearch, Pleroma.Search.Meilisearch] + suggestions: {:list_behaviour_implementations, Pleroma.Search.SearchBackend} } ] }, @@ -3257,7 +3258,7 @@ }, %{ key: :initial_indexing_chunk_size, - type: :int, + type: :integer, description: "Amount of posts in a batch when running the initial indexing operation. Should probably not be more than 100000" <> " since there's a limit on maximum insert size", @@ -3268,6 +3269,7 @@ %{ group: :pleroma, key: Pleroma.Search.Elasticsearch.Cluster, + label: "Elasticsearch", type: :group, description: "Elasticsearch settings.", children: [ @@ -3334,13 +3336,13 @@ }, %{ key: :bulk_page_size, - type: :int, + type: :integer, description: "Size for bulk put requests, mostly used on building the index", suggestion: [5000] }, %{ key: :bulk_wait_interval, - type: :int, + type: :integer, description: "Time to wait between bulk put requests (in ms)", suggestion: [15_000] } @@ -3349,5 +3351,66 @@ ] } ] + }, + %{ + group: :pleroma, + key: :translator, + type: :group, + description: "Translation Settings", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "Is translation enabled?", + suggestion: [true, false] + }, + %{ + key: :module, + type: :module, + description: "Translation module.", + suggestions: {:list_behaviour_implementations, Pleroma.Akkoma.Translator} + } + ] + }, + %{ + group: :pleroma, + key: :deepl, + label: "DeepL", + type: :group, + description: "DeepL Settings.", + children: [ + %{ + key: :tier, + type: {:dropdown, :atom}, + description: "API Tier", + suggestions: [:free, :pro] + }, + %{ + key: :api_key, + type: :string, + description: "API key for DeepL", + suggestions: [nil] + } + ] + }, + %{ + group: :pleroma, + key: :libre_translate, + type: :group, + description: "LibreTranslate Settings.", + children: [ + %{ + key: :url, + type: :string, + description: "URL for libretranslate", + suggestion: [nil] + }, + %{ + key: :api_key, + type: :string, + description: "API key for libretranslate", + suggestion: [nil] + } + ] } ] diff --git a/docs/docs/configuration/cheatsheet.md b/docs/docs/configuration/cheatsheet.md index a29db208c..90041d3d6 100644 --- a/docs/docs/configuration/cheatsheet.md +++ b/docs/docs/configuration/cheatsheet.md @@ -1159,3 +1159,28 @@ Each job has these settings: * `:max_running` - max concurrently runnings jobs * `:max_waiting` - max waiting jobs + +### Translation Settings + +Settings to automatically translate statuses for end users. Currently supported +translation services are DeepL and LibreTranslate. + +Translations are available at `/api/v1/statuses/:id/translations/:language`, where +`language` is the target language code (e.g `en`) + +### `:translator` + +- `:enabled` - enables translation +- `:module` - Sets module to be used + - Either `Pleroma.Akkoma.Translators.DeepL` or `Pleroma.Akkoma.Translators.LibreTranslate` + +### `:deepl` + +- `:api_key` - API key for DeepL +- `:tier` - API tier + - either `:free` or `:pro` + +### `:libre_translate` + +- `:url` - URL of LibreTranslate instance +- `:api_key` - API key for LibreTranslate \ No newline at end of file diff --git a/lib/pleroma/akkoma/translators/deepl.ex b/lib/pleroma/akkoma/translators/deepl.ex new file mode 100644 index 000000000..0a4a7fe10 --- /dev/null +++ b/lib/pleroma/akkoma/translators/deepl.ex @@ -0,0 +1,58 @@ +defmodule Pleroma.Akkoma.Translators.DeepL do + @behaviour Pleroma.Akkoma.Translator + + alias Pleroma.HTTP + alias Pleroma.Config + require Logger + + defp base_url(:free) do + "https://api-free.deepl.com/v2/" + end + + defp base_url(:pro) do + "https://api.deepl.com/v2/" + end + + defp api_key do + Config.get([:deepl, :api_key]) + end + + defp tier do + Config.get([:deepl, :tier]) + end + + @impl Pleroma.Akkoma.Translator + def translate(string, to_language) do + with {:ok, %{status: 200} = response} <- do_request(api_key(), tier(), string, to_language), + {:ok, body} <- Jason.decode(response.body) do + %{"translations" => [%{"text" => translated, "detected_source_language" => detected}]} = + body + + {:ok, detected, translated} + else + {:ok, %{status: status} = response} -> + Logger.warning("DeepL: Request rejected: #{inspect(response)}") + {:error, "DeepL request failed (code #{status})"} + + {:error, reason} -> + {:error, reason} + end + end + + defp do_request(api_key, tier, string, to_language) do + HTTP.post( + base_url(tier) <> "translate", + URI.encode_query( + %{ + text: string, + target_lang: to_language + }, + :rfc3986 + ), + [ + {"authorization", "DeepL-Auth-Key #{api_key}"}, + {"content-type", "application/x-www-form-urlencoded"} + ] + ) + end +end diff --git a/lib/pleroma/akkoma/translators/libre_translate.ex b/lib/pleroma/akkoma/translators/libre_translate.ex new file mode 100644 index 000000000..615d04192 --- /dev/null +++ b/lib/pleroma/akkoma/translators/libre_translate.ex @@ -0,0 +1,51 @@ +defmodule Pleroma.Akkoma.Translators.LibreTranslate do + @behaviour Pleroma.Akkoma.Translator + + alias Pleroma.Config + alias Pleroma.HTTP + require Logger + + defp api_key do + Config.get([:libre_translate, :api_key]) + end + + defp url do + Config.get([:libre_translate, :url]) + end + + @impl Pleroma.Akkoma.Translator + def translate(string, to_language) do + with {:ok, %{status: 200} = response} <- do_request(string, to_language), + {:ok, body} <- Jason.decode(response.body) do + %{"translatedText" => translated, "detectedLanguage" => %{"language" => detected}} = body + + {:ok, detected, translated} + else + {:ok, %{status: status} = response} -> + Logger.warning("libre_translate: request failed, #{inspect(response)}") + {:error, "libre_translate: request failed (code #{status})"} + + {:error, reason} -> + {:error, reason} + end + end + + defp do_request(string, to_language) do + url = URI.parse(url()) + url = %{url | path: "/translate"} + + HTTP.post( + to_string(url), + Jason.encode!(%{ + q: string, + source: "auto", + target: to_language, + format: "html", + api_key: api_key() + }), + [ + {"content-type", "application/json"} + ] + ) + end +end diff --git a/lib/pleroma/akkoma/translators/translator.ex b/lib/pleroma/akkoma/translators/translator.ex new file mode 100644 index 000000000..0276ed6c2 --- /dev/null +++ b/lib/pleroma/akkoma/translators/translator.ex @@ -0,0 +1,3 @@ +defmodule Pleroma.Akkoma.Translator do + @callback translate(String.t(), String.t()) :: {:ok, String.t(), String.t()} | {:error, any()} +end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index e11e5495a..b809f7733 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -154,7 +154,8 @@ defp cachex_children do build_cachex("web_resp", limit: 2500), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("failed_proxy_url", limit: 2500), - build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) + build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000), + build_cachex("translations", default_ttl: :timer.hours(24 * 30), limit: 2500) ] end diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index a5da8b58e..04a7bf5db 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -406,6 +406,22 @@ def bookmarks_operation do } end + def translate_operation do + %Operation{ + tags: ["Retrieve status translation"], + summary: "Translate status", + description: "View the translation of a given status", + operationId: "StatusController.translation", + security: [%{"oAuth" => ["read:statuses"]}], + parameters: [id_param(), language_param()], + responses: %{ + 200 => Operation.response("Translation", "application/json", translation()), + 400 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } + end + def array_of_statuses do %Schema{type: :array, items: Status, example: [Status.schema().example]} end @@ -552,6 +568,10 @@ def id_param do ) end + defp language_param do + Operation.parameter(:language, :path, :string, "ISO 639 language code", example: "en") + end + defp status_response do Operation.response("Status", "application/json", Status) end @@ -573,4 +593,20 @@ defp context do } } end + + defp translation do + %Schema{ + title: "StatusTranslation", + description: "The translation of a status.", + type: :object, + required: [:detected_language, :text], + properties: %{ + detected_language: %Schema{ + type: :string, + description: "The detected language of the text" + }, + text: %Schema{type: :string, description: "The translated text"} + } + } + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 9ab30742b..d9b93ca5e 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do alias Pleroma.Bookmark alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.Config alias Pleroma.ScheduledActivity alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -30,6 +31,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do plug(:skip_public_check when action in [:index, :show]) @unauthenticated_access %{fallback: :proceed_unauthenticated, scopes: []} + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) plug( OAuthScopesPlug, @@ -37,7 +39,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do when action in [ :index, :show, - :context + :context, + :translate ] ) @@ -418,6 +421,46 @@ def bookmarks(%{assigns: %{user: user}} = conn, params) do ) end + @doc "GET /api/v1/statuses/:id/translations/:language" + def translate(%{assigns: %{user: user}} = conn, %{id: id, language: language}) do + with {:enabled, true} <- {:enabled, Config.get([:translator, :enabled])}, + %Activity{} = activity <- Activity.get_by_id_with_object(id), + {:visible, true} <- {:visible, Visibility.visible_for_user?(activity, user)}, + translation_module <- Config.get([:translator, :module]), + {:ok, detected, translation} <- + fetch_or_translate( + activity.id, + activity.object.data["content"], + language, + translation_module + ) do + json(conn, %{detected_language: detected, text: translation}) + else + {:enabled, false} -> + conn + |> put_status(:bad_request) + |> json(%{"error" => "Translation is not enabled"}) + + {:visible, false} -> + {:error, :not_found} + + e -> + e + end + end + + defp fetch_or_translate(status_id, text, language, translation_module) do + @cachex.fetch!(:user_cache, "translations:#{status_id}:#{language}", fn _ -> + value = translation_module.translate(text, language) + + with {:ok, _, _} <- value do + value + else + _ -> {:ignore, value} + end + end) + end + defp put_application(params, %{assigns: %{token: %Token{user: %User{} = user} = token}} = _conn) do if user.disclose_client do %{client_name: client_name, website: website} = Repo.preload(token, :app).app diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 7ae357e23..436519439 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -81,6 +81,9 @@ def features do if Config.get([:instance, :profile_directory]) do "profile_directory" end, + if Config.get([:translator, :enabled], false) do + "akkoma:machine_translation" + end, "custom_emoji_reactions" ] |> Enum.filter(& &1) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 647d99278..aff7b67db 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -553,6 +553,7 @@ defmodule Pleroma.Web.Router do post("/statuses/:id/unbookmark", StatusController, :unbookmark) post("/statuses/:id/mute", StatusController, :mute_conversation) post("/statuses/:id/unmute", StatusController, :unmute_conversation) + get("/statuses/:id/translations/:language", StatusController, :translate) post("/push/subscription", SubscriptionController, :create) get("/push/subscription", SubscriptionController, :show) diff --git a/test/pleroma/translators/deepl_test.exs b/test/pleroma/translators/deepl_test.exs new file mode 100644 index 000000000..286d21d3e --- /dev/null +++ b/test/pleroma/translators/deepl_test.exs @@ -0,0 +1,75 @@ +defmodule Pleroma.Akkoma.Translators.DeepLTest do + use Pleroma.DataCase, async: true + + alias Pleroma.Akkoma.Translators.DeepL + + describe "translating with deepl" do + setup do + clear_config([:deepl, :api_key], "deepl_api_key") + end + + test "should work with the free tier" do + clear_config([:deepl, :tier], :free) + + Tesla.Mock.mock(fn + %{method: :post, url: "https://api-free.deepl.com/v2/translate"} = env -> + auth_header = Enum.find(env.headers, fn {k, _v} -> k == "authorization" end) + assert {"authorization", "DeepL-Auth-Key deepl_api_key"} = auth_header + + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + translations: [ + %{ + "text" => "I will crush you", + "detected_source_language" => "ja" + } + ] + }) + } + end) + + assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", "en") + end + + test "should work with the pro tier" do + clear_config([:deepl, :tier], :pro) + + Tesla.Mock.mock(fn + %{method: :post, url: "https://api.deepl.com/v2/translate"} = env -> + auth_header = Enum.find(env.headers, fn {k, _v} -> k == "authorization" end) + assert {"authorization", "DeepL-Auth-Key deepl_api_key"} = auth_header + + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + translations: [ + %{ + "text" => "I will crush you", + "detected_source_language" => "ja" + } + ] + }) + } + end) + + assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", "en") + end + + test "should gracefully fail if the API errors" do + clear_config([:deepl, :tier], :free) + + Tesla.Mock.mock(fn + %{method: :post, url: "https://api-free.deepl.com/v2/translate"} -> + %Tesla.Env{ + status: 403, + body: "" + } + end) + + assert {:error, "DeepL request failed (code 403)"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", "en") + end + end +end diff --git a/test/pleroma/translators/libre_translate_test.exs b/test/pleroma/translators/libre_translate_test.exs new file mode 100644 index 000000000..9ed2c5323 --- /dev/null +++ b/test/pleroma/translators/libre_translate_test.exs @@ -0,0 +1,91 @@ +defmodule Pleroma.Akkoma.Translators.LibreTranslateTest do + use Pleroma.DataCase, async: true + + alias Pleroma.Akkoma.Translators.LibreTranslate + + describe "translating with libre translate" do + setup do + clear_config([:libre_translate, :url], "http://libre.translate/translate") + end + + test "should work without an API key" do + Tesla.Mock.mock(fn + %{method: :post, url: "http://libre.translate/translate"} = env -> + assert {:ok, %{"api_key" => nil}} = Jason.decode(env.body) + + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + detectedLanguage: %{ + confidence: 83, + language: "ja" + }, + translatedText: "I will crush you" + }) + } + end) + + assert {:ok, "ja", "I will crush you"} = LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "en") + end + + test "should work with an API key" do + clear_config([:libre_translate, :api_key], "libre_translate_api_key") + + Tesla.Mock.mock(fn + %{method: :post, url: "http://libre.translate/translate"} = env -> + assert {:ok, %{"api_key" => "libre_translate_api_key"}} = Jason.decode(env.body) + + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + detectedLanguage: %{ + confidence: 83, + language: "ja" + }, + translatedText: "I will crush you" + }) + } + end) + + assert {:ok, "ja", "I will crush you"} = LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "en") + end + + test "should gracefully handle API key errors" do + clear_config([:libre_translate, :api_key], "") + + Tesla.Mock.mock(fn + %{method: :post, url: "http://libre.translate/translate"} -> + %Tesla.Env{ + status: 403, + body: + Jason.encode!(%{ + error: "Please contact the server operator to obtain an API key" + }) + } + end) + + assert {:error, "libre_translate: request failed (code 403)"} = + LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "en") + end + + test "should gracefully handle an unsupported language" do + clear_config([:libre_translate, :api_key], "") + + Tesla.Mock.mock(fn + %{method: :post, url: "http://libre.translate/translate"} -> + %Tesla.Env{ + status: 400, + body: + Jason.encode!(%{ + error: "zoop is not supported" + }) + } + end) + + assert {:error, "libre_translate: request failed (code 400)"} = + LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "zoop") + end + end +end diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index ea168f6c5..e38f5fe58 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2071,4 +2071,76 @@ test "posting a quote of a status that doesn't exist", %{conn: conn} do |> json_response_and_validate_schema(422) end end + + describe "translating statuses" do + setup do + clear_config([:translator, :enabled], true) + clear_config([:translator, :module], Pleroma.Akkoma.Translators.DeepL) + clear_config([:deepl, :api_key], "deepl_api_key") + oauth_access(["read:statuses"]) + end + + test "should return text and detected language", %{conn: conn} do + clear_config([:deepl, :tier], :free) + + Tesla.Mock.mock_global(fn + %{method: :post, url: "https://api-free.deepl.com/v2/translate"} -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + translations: [ + %{ + "text" => "Tell me, for whom do you fight?", + "detected_source_language" => "ja" + } + ] + }) + } + end) + + user = insert(:user) + {:ok, to_translate} = CommonAPI.post(user, %{status: "何のために闘う?"}) + + conn = + conn + |> put_req_header("content-type", "application/json") + |> get("/api/v1/statuses/#{to_translate.id}/translations/en") + + response = json_response_and_validate_schema(conn, 200) + + assert response["text"] == "Tell me, for whom do you fight?" + assert response["detected_language"] == "ja" + end + + test "should not allow translating of statuses you cannot see", %{conn: conn} do + clear_config([:deepl, :tier], :free) + + Tesla.Mock.mock_global(fn + %{method: :post, url: "https://api-free.deepl.com/v2/translate"} -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + translations: [ + %{ + "text" => "Tell me, for whom do you fight?", + "detected_source_language" => "ja" + } + ] + }) + } + end) + + user = insert(:user) + {:ok, to_translate} = CommonAPI.post(user, %{status: "何のために闘う?", visibility: "private"}) + + conn = + conn + |> put_req_header("content-type", "application/json") + |> get("/api/v1/statuses/#{to_translate.id}/translations/en") + + json_response_and_validate_schema(conn, 404) + end + end end From 7759187de90b8894b8efa2ba111fc73ae48ed324 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 29 Aug 2022 22:20:47 +0100 Subject: [PATCH 29/44] ensure default value is sane --- config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 330e572fe..bf7e7db44 100644 --- a/config/config.exs +++ b/config/config.exs @@ -845,7 +845,7 @@ config :pleroma, :translator, enabled: false, - module: Akkoma.Translators.DeepL + module: Pleroma.Akkoma.Translators.DeepL config :pleroma, :deepl, # either :free or :pro From 9cb41b6d7bb6788c0416952e64a582bc06f6d2dd Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Tue, 30 Aug 2022 10:39:36 +0100 Subject: [PATCH 30/44] add extra instructions to placeholder page --- priv/static/index.html | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/priv/static/index.html b/priv/static/index.html index 4a304f576..e60d31966 100644 --- a/priv/static/index.html +++ b/priv/static/index.html @@ -6,7 +6,18 @@

Welcome to Akkoma!

If you're seeing this page, your server works!

-

In order to get a frontend to show here, you'll need to set up :pleroma, :frontends, primary and install your frontend of choice

- Documentation +

In order to get a frontend to show here, you'll need to set up :pleroma, :frontends, primary and install your frontend of choice, in most cases this will just be:

+ 
+        
+        # OTP
+        ./bin/pleroma_ctl frontend install pleroma-fe --ref stable
+        # Source
+        mix pleroma.frontend install pleroma-fe --ref stable
+
+        ## you can do the same thing for admin-fe if you so wish
+        
+
+

Installation Command Documentation

+

Config Documentation

From 25111bb407f9e94939160258f8a8ad59ed04f93f Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Tue, 30 Aug 2022 10:56:33 +0100 Subject: [PATCH 31/44] include frontend installation document on all install guides --- docs/docs/configuration/cheatsheet.md | 2 +- docs/docs/installation/alpine_linux_en.md | 2 ++ docs/docs/installation/arch_linux_en.md | 2 ++ docs/docs/installation/debian_based_en.md | 2 ++ docs/docs/installation/fedora_based_en.md | 2 ++ docs/docs/installation/freebsd_en.md | 3 ++ docs/docs/installation/frontends.include | 25 ++++++++++++++++ docs/docs/installation/gentoo_en.md | 2 ++ docs/docs/installation/migrating_to_akkoma.md | 29 ++++++++++--------- docs/docs/installation/netbsd_en.md | 2 ++ docs/docs/installation/openbsd_en.md | 2 ++ docs/docs/installation/otp_en.md | 2 ++ docs/docs/installation/otp_redhat_en.md | 1 + 13 files changed, 61 insertions(+), 15 deletions(-) create mode 100644 docs/docs/installation/frontends.include diff --git a/docs/docs/configuration/cheatsheet.md b/docs/docs/configuration/cheatsheet.md index 90041d3d6..52062eaa0 100644 --- a/docs/docs/configuration/cheatsheet.md +++ b/docs/docs/configuration/cheatsheet.md @@ -1183,4 +1183,4 @@ Translations are available at `/api/v1/statuses/:id/translations/:language`, whe ### `:libre_translate` - `:url` - URL of LibreTranslate instance -- `:api_key` - API key for LibreTranslate \ No newline at end of file +- `:api_key` - API key for LibreTranslate diff --git a/docs/docs/installation/alpine_linux_en.md b/docs/docs/installation/alpine_linux_en.md index f98998fb8..aae8f9626 100644 --- a/docs/docs/installation/alpine_linux_en.md +++ b/docs/docs/installation/alpine_linux_en.md @@ -221,6 +221,8 @@ If your instance is up and running, you can create your first user with administ doas -u akkoma env MIX_ENV=prod mix pleroma.user new --admin ``` +{! installation/frontends.include !} + #### Further reading {! installation/further_reading.include !} diff --git a/docs/docs/installation/arch_linux_en.md b/docs/docs/installation/arch_linux_en.md index f7a7d6239..639c9c798 100644 --- a/docs/docs/installation/arch_linux_en.md +++ b/docs/docs/installation/arch_linux_en.md @@ -212,6 +212,8 @@ If your instance is up and running, you can create your first user with administ sudo -Hu akkoma MIX_ENV=prod mix pleroma.user new --admin ``` +{! installation/frontends.include !} + #### Further reading {! installation/further_reading.include !} diff --git a/docs/docs/installation/debian_based_en.md b/docs/docs/installation/debian_based_en.md index 40503db0c..139c789bc 100644 --- a/docs/docs/installation/debian_based_en.md +++ b/docs/docs/installation/debian_based_en.md @@ -175,6 +175,8 @@ If your instance is up and running, you can create your first user with administ sudo -Hu akkoma MIX_ENV=prod mix pleroma.user new --admin ``` +{! installation/frontends.include !} + #### Further reading {! installation/further_reading.include !} diff --git a/docs/docs/installation/fedora_based_en.md b/docs/docs/installation/fedora_based_en.md index 30d68d97f..d8c7b3e74 100644 --- a/docs/docs/installation/fedora_based_en.md +++ b/docs/docs/installation/fedora_based_en.md @@ -199,6 +199,8 @@ If your instance is up and running, you can create your first user with administ sudo -Hu akkoma MIX_ENV=prod mix pleroma.user new --admin ``` +{! installation/frontends.include !} + #### Further reading {! installation/further_reading.include !} diff --git a/docs/docs/installation/freebsd_en.md b/docs/docs/installation/freebsd_en.md index be735a998..53c029d27 100644 --- a/docs/docs/installation/freebsd_en.md +++ b/docs/docs/installation/freebsd_en.md @@ -206,6 +206,9 @@ If your instance is up and running, you can create your first user with administ ```shell sudo -Hu akkoma MIX_ENV=prod mix pleroma.user new --admin ``` + +{! installation/frontends.include !} + ## Conclusion Restart nginx with `# service nginx restart` and you should be up and running. diff --git a/docs/docs/installation/frontends.include b/docs/docs/installation/frontends.include new file mode 100644 index 000000000..585be71ae --- /dev/null +++ b/docs/docs/installation/frontends.include @@ -0,0 +1,25 @@ +#### Installing Frontends + +Once your backend server is functional, you'll also want to +probably install frontends. + +These are no longer bundled with the distribution and need an extra +command to install. + +For most installations, the following will suffice: + +=== "OTP" + ```sh + ./bin/pleroma_ctl frontend install pleroma-fe --ref stable + # and also, if desired + ./bin/pleroma_ctl frontend install admin-fe --ref stable + ``` + +=== "From Source" + ```sh + mix pleroma.frontend install pleroma-fe --ref stable + mix pleroma.frontend install admin-fe --ref stable + ``` + +For more customised installations, refer to [Frontend Management](../../configuration/frontend_management) + diff --git a/docs/docs/installation/gentoo_en.md b/docs/docs/installation/gentoo_en.md index 4649b63bf..9450c9b38 100644 --- a/docs/docs/installation/gentoo_en.md +++ b/docs/docs/installation/gentoo_en.md @@ -293,6 +293,8 @@ akkoma$ MIX_ENV=prod mix pleroma.user new --admin If you opted to allow sudo for the `akkoma` user but would like to remove the ability for greater security, now might be a good time to edit `/etc/sudoers` and/or change the groups the `akkoma` user belongs to. Be sure to restart the akkoma service afterwards to ensure it picks up on the changes. +{! installation/frontends.include !} + #### Further reading {! installation/further_reading.include !} diff --git a/docs/docs/installation/migrating_to_akkoma.md b/docs/docs/installation/migrating_to_akkoma.md index 74b87e318..d8ea0ea25 100644 --- a/docs/docs/installation/migrating_to_akkoma.md +++ b/docs/docs/installation/migrating_to_akkoma.md @@ -1,7 +1,5 @@ # Migrating to Akkoma -**Akkoma does not currently have a stable release, until 3.0, all builds should be considered "develop"** - ## Why should you migrate? aside from actually responsive maintainer(s)? let's lookie here, we've got: @@ -11,6 +9,8 @@ aside from actually responsive maintainer(s)? let's lookie here, we've got: - elasticsearch support (because pleroma search is GARBAGE) - latest develop pleroma-fe additions - local-only posting +- automatic post translation +- the mastodon frontend back in all its glory - probably more, this is like 3.5 years of IHBA additions finally compiled ## Actually migrating @@ -43,14 +43,14 @@ This will just be setting the update URL - find your flavour from the [mapping o ```bash export FLAVOUR=[the flavour you found above] -./bin/pleroma_ctl update --zip-url https://akkoma-updates.s3-website.fr-par.scw.cloud/develop/akkoma-$FLAVOUR.zip +./bin/pleroma_ctl update --zip-url https://akkoma-updates.s3-website.fr-par.scw.cloud/stable/akkoma-$FLAVOUR.zip ./bin/pleroma_ctl migrate ``` Then restart. When updating in the future, you canjust use ```bash -./bin/pleroma_ctl update --branch develop +./bin/pleroma_ctl update --branch stable ``` ## Frontend changes @@ -62,17 +62,18 @@ your upgrade path here depends on your setup You'll need to run a couple of commands, -```bash -# From source -mix pleroma.frontend install pleroma-fe -# you'll probably want this too -mix pleroma.frontend install admin-fe +=== "OTP" + ```sh + ./bin/pleroma_ctl frontend install pleroma-fe --ref stable + # and also, if desired + ./bin/pleroma_ctl frontend install admin-fe --ref stable + ``` -# OTP -./bin/pleroma_ctl frontend install pleroma-fe -# you'll probably want this too -./bin/pleroma_ctl frontend install admin-fe -``` +=== "From Source" + ```sh + mix pleroma.frontend install pleroma-fe --ref stable + mix pleroma.frontend install admin-fe --ref stable + ``` ### I've run the mix task to install a frontend diff --git a/docs/docs/installation/netbsd_en.md b/docs/docs/installation/netbsd_en.md index c00a32e34..f13a3ee89 100644 --- a/docs/docs/installation/netbsd_en.md +++ b/docs/docs/installation/netbsd_en.md @@ -202,6 +202,8 @@ incorrect timestamps. You should have ntpd running. * +{! installation/frontends.include !} + #### Further reading {! installation/further_reading.include !} diff --git a/docs/docs/installation/openbsd_en.md b/docs/docs/installation/openbsd_en.md index c7e8cf0c0..581942f99 100644 --- a/docs/docs/installation/openbsd_en.md +++ b/docs/docs/installation/openbsd_en.md @@ -250,6 +250,8 @@ If your instance is up and running, you can create your first user with administ LC_ALL=en_US.UTF-8 MIX_ENV=prod mix pleroma.user new --admin ``` +{! installation/frontends.include !} + #### Further reading {! installation/further_reading.include !} diff --git a/docs/docs/installation/otp_en.md b/docs/docs/installation/otp_en.md index 022716fec..329afe967 100644 --- a/docs/docs/installation/otp_en.md +++ b/docs/docs/installation/otp_en.md @@ -306,6 +306,8 @@ su akkoma -s $SHELL -lc "./bin/pleroma_ctl user new joeuser joeuser@sld.tld --ad ``` This will create an account withe the username of 'joeuser' with the email address of joeuser@sld.tld, and set that user's account as an admin. This will result in a link that you can paste into the browser, which logs you in and enables you to set the password. +{! installation/frontends.include !} + ## Further reading {! installation/further_reading.include !} diff --git a/docs/docs/installation/otp_redhat_en.md b/docs/docs/installation/otp_redhat_en.md index 2e6b58c9e..ec6c30bcf 100644 --- a/docs/docs/installation/otp_redhat_en.md +++ b/docs/docs/installation/otp_redhat_en.md @@ -279,6 +279,7 @@ After that, run the `pleroma_ctl migrate` command as usual to perform database m As it currently stands, your OTP build will only be compatible for the specific RedHat distribution you've built it on. Fedora builds only work on Fedora, Centos builds only on Centos, RedHat builds only on RedHat. Secondly, for Fedora, they will also be bound to the specific Fedora release. This is because different releases of Fedora may have significant changes made in some of the required packages and libraries. +{! installation/frontends.include !} {! installation/further_reading.include !} From c3fde9577d5822243a7567a55cefa2fd2f289559 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Tue, 30 Aug 2022 14:58:54 +0000 Subject: [PATCH 32/44] Allow listing languages, setting source language (#192) Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/192 --- lib/pleroma/akkoma/translators/deepl.ex | 43 +++++++++++-- .../akkoma/translators/libre_translate.ex | 40 ++++++++++-- lib/pleroma/akkoma/translators/translator.ex | 4 +- .../controllers/translation_controller.ex | 43 +++++++++++++ .../api_spec/operations/status_operation.ex | 6 +- .../operations/translate_operation.ex | 41 ++++++++++++ .../controllers/status_controller.ex | 23 ++++--- lib/pleroma/web/router.ex | 5 ++ test/pleroma/translators/deepl_test.exs | 63 ++++++++++++++++++- .../translators/libre_translate_test.exs | 55 ++++++++++++++-- 10 files changed, 294 insertions(+), 29 deletions(-) create mode 100644 lib/pleroma/web/akkoma_api/controllers/translation_controller.ex create mode 100644 lib/pleroma/web/api_spec/operations/translate_operation.ex diff --git a/lib/pleroma/akkoma/translators/deepl.ex b/lib/pleroma/akkoma/translators/deepl.ex index 0a4a7fe10..f93fb7e59 100644 --- a/lib/pleroma/akkoma/translators/deepl.ex +++ b/lib/pleroma/akkoma/translators/deepl.ex @@ -22,8 +22,27 @@ defp tier do end @impl Pleroma.Akkoma.Translator - def translate(string, to_language) do - with {:ok, %{status: 200} = response} <- do_request(api_key(), tier(), string, to_language), + def languages do + with {:ok, %{status: 200} = response} <- do_languages(), + {:ok, body} <- Jason.decode(response.body) do + resp = + Enum.map(body, fn %{"language" => code, "name" => name} -> %{code: code, name: name} end) + + {:ok, resp} + else + {:ok, %{status: status} = response} -> + Logger.warning("DeepL: Request rejected: #{inspect(response)}") + {:error, "DeepL request failed (code #{status})"} + + {:error, reason} -> + {:error, reason} + end + end + + @impl Pleroma.Akkoma.Translator + def translate(string, from_language, to_language) do + with {:ok, %{status: 200} = response} <- + do_request(api_key(), tier(), string, from_language, to_language), {:ok, body} <- Jason.decode(response.body) do %{"translations" => [%{"text" => translated, "detected_source_language" => detected}]} = body @@ -39,14 +58,16 @@ def translate(string, to_language) do end end - defp do_request(api_key, tier, string, to_language) do + defp do_request(api_key, tier, string, from_language, to_language) do HTTP.post( base_url(tier) <> "translate", URI.encode_query( %{ text: string, - target_lang: to_language - }, + target_lang: to_language, + tag_handling: "html" + } + |> maybe_add_source(from_language), :rfc3986 ), [ @@ -55,4 +76,16 @@ defp do_request(api_key, tier, string, to_language) do ] ) end + + defp maybe_add_source(opts, nil), do: opts + defp maybe_add_source(opts, lang), do: Map.put(opts, :source_lang, lang) + + defp do_languages() do + HTTP.get( + base_url(tier()) <> "languages?type=target", + [ + {"authorization", "DeepL-Auth-Key #{api_key()}"} + ] + ) + end end diff --git a/lib/pleroma/akkoma/translators/libre_translate.ex b/lib/pleroma/akkoma/translators/libre_translate.ex index 615d04192..319907c2f 100644 --- a/lib/pleroma/akkoma/translators/libre_translate.ex +++ b/lib/pleroma/akkoma/translators/libre_translate.ex @@ -14,10 +14,33 @@ defp url do end @impl Pleroma.Akkoma.Translator - def translate(string, to_language) do - with {:ok, %{status: 200} = response} <- do_request(string, to_language), + def languages do + with {:ok, %{status: 200} = response} <- do_languages(), {:ok, body} <- Jason.decode(response.body) do - %{"translatedText" => translated, "detectedLanguage" => %{"language" => detected}} = body + resp = Enum.map(body, fn %{"code" => code, "name" => name} -> %{code: code, name: name} end) + {:ok, resp} + else + {:ok, %{status: status} = response} -> + Logger.warning("LibreTranslate: Request rejected: #{inspect(response)}") + {:error, "LibreTranslate request failed (code #{status})"} + + {:error, reason} -> + {:error, reason} + end + end + + @impl Pleroma.Akkoma.Translator + def translate(string, from_language, to_language) do + with {:ok, %{status: 200} = response} <- do_request(string, from_language, to_language), + {:ok, body} <- Jason.decode(response.body) do + %{"translatedText" => translated} = body + + detected = + if Map.has_key?(body, "detectedLanguage") do + get_in(body, ["detectedLanguage", "language"]) + else + from_language + end {:ok, detected, translated} else @@ -30,7 +53,7 @@ def translate(string, to_language) do end end - defp do_request(string, to_language) do + defp do_request(string, from_language, to_language) do url = URI.parse(url()) url = %{url | path: "/translate"} @@ -38,7 +61,7 @@ defp do_request(string, to_language) do to_string(url), Jason.encode!(%{ q: string, - source: "auto", + source: if(is_nil(from_language), do: "auto", else: from_language), target: to_language, format: "html", api_key: api_key() @@ -48,4 +71,11 @@ defp do_request(string, to_language) do ] ) end + + defp do_languages() do + url = URI.parse(url()) + url = %{url | path: "/languages"} + + HTTP.get(to_string(url)) + end end diff --git a/lib/pleroma/akkoma/translators/translator.ex b/lib/pleroma/akkoma/translators/translator.ex index 0276ed6c2..aa49b0655 100644 --- a/lib/pleroma/akkoma/translators/translator.ex +++ b/lib/pleroma/akkoma/translators/translator.ex @@ -1,3 +1,5 @@ defmodule Pleroma.Akkoma.Translator do - @callback translate(String.t(), String.t()) :: {:ok, String.t(), String.t()} | {:error, any()} + @callback translate(String.t(), String.t() | nil, String.t()) :: + {:ok, String.t(), String.t()} | {:error, any()} + @callback languages() :: {:ok, [%{name: String.t(), code: String.t()}]} | {:error, any()} end diff --git a/lib/pleroma/web/akkoma_api/controllers/translation_controller.ex b/lib/pleroma/web/akkoma_api/controllers/translation_controller.ex new file mode 100644 index 000000000..49ef89a50 --- /dev/null +++ b/lib/pleroma/web/akkoma_api/controllers/translation_controller.ex @@ -0,0 +1,43 @@ +defmodule Pleroma.Web.AkkomaAPI.TranslationController do + use Pleroma.Web, :controller + + alias Pleroma.Web.Plugs.OAuthScopesPlug + + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + + @unauthenticated_access %{fallback: :proceed_unauthenticated, scopes: []} + plug( + OAuthScopesPlug, + %{@unauthenticated_access | scopes: ["read:statuses"]} + when action in [ + :languages + ] + ) + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.TranslationOperation + + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + + @doc "GET /api/v1/akkoma/translation/languages" + def languages(conn, _params) do + with {:ok, languages} <- get_languages() do + conn + |> json(languages) + else + e -> IO.inspect(e) + end + end + + defp get_languages do + module = Pleroma.Config.get([:translator, :module]) + + @cachex.fetch!(:translations_cache, "languages:#{module}}", fn _ -> + with {:ok, languages} <- module.languages() do + {:ok, languages} + else + {:error, err} -> {:ignore, {:error, err}} + end + end) + end +end diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 04a7bf5db..5332c9dca 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -413,7 +413,7 @@ def translate_operation do description: "View the translation of a given status", operationId: "StatusController.translation", security: [%{"oAuth" => ["read:statuses"]}], - parameters: [id_param(), language_param()], + parameters: [id_param(), language_param(), source_language_param()], responses: %{ 200 => Operation.response("Translation", "application/json", translation()), 400 => Operation.response("Error", "application/json", ApiError), @@ -572,6 +572,10 @@ defp language_param do Operation.parameter(:language, :path, :string, "ISO 639 language code", example: "en") end + defp source_language_param do + Operation.parameter(:from, :query, :string, "ISO 639 language code", example: "en") + end + defp status_response do Operation.response("Status", "application/json", Status) end diff --git a/lib/pleroma/web/api_spec/operations/translate_operation.ex b/lib/pleroma/web/api_spec/operations/translate_operation.ex new file mode 100644 index 000000000..aa3b69a18 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/translate_operation.ex @@ -0,0 +1,41 @@ +defmodule Pleroma.Web.ApiSpec.TranslationOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + + @spec open_api_operation(atom) :: Operation.t() + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + @spec languages_operation() :: Operation.t() + def languages_operation() do + %Operation{ + tags: ["Retrieve status translation"], + summary: "Translate status", + description: "View the translation of a given status", + operationId: "AkkomaAPI.TranslationController.languages", + security: [%{"oAuth" => ["read:statuses"]}], + responses: %{ + 200 => Operation.response("Translation", "application/json", languages_schema()) + } + } + end + + defp languages_schema do + %Schema{ + type: "array", + items: %Schema{ + type: "object", + properties: %{ + code: %Schema{ + type: "string" + }, + name: %Schema{ + type: "string" + } + } + } + } + end +end diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index d9b93ca5e..41fbd7acf 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -422,7 +422,7 @@ def bookmarks(%{assigns: %{user: user}} = conn, params) do end @doc "GET /api/v1/statuses/:id/translations/:language" - def translate(%{assigns: %{user: user}} = conn, %{id: id, language: language}) do + def translate(%{assigns: %{user: user}} = conn, %{id: id, language: language} = params) do with {:enabled, true} <- {:enabled, Config.get([:translator, :enabled])}, %Activity{} = activity <- Activity.get_by_id_with_object(id), {:visible, true} <- {:visible, Visibility.visible_for_user?(activity, user)}, @@ -431,6 +431,7 @@ def translate(%{assigns: %{user: user}} = conn, %{id: id, language: language}) d fetch_or_translate( activity.id, activity.object.data["content"], + Map.get(params, :from, nil), language, translation_module ) do @@ -449,16 +450,20 @@ def translate(%{assigns: %{user: user}} = conn, %{id: id, language: language}) d end end - defp fetch_or_translate(status_id, text, language, translation_module) do - @cachex.fetch!(:user_cache, "translations:#{status_id}:#{language}", fn _ -> - value = translation_module.translate(text, language) + defp fetch_or_translate(status_id, text, source_language, target_language, translation_module) do + @cachex.fetch!( + :translations_cache, + "translations:#{status_id}:#{source_language}:#{target_language}", + fn _ -> + value = translation_module.translate(text, source_language, target_language) - with {:ok, _, _} <- value do - value - else - _ -> {:ignore, value} + with {:ok, _, _} <- value do + value + else + _ -> {:ignore, value} + end end - end) + ) end defp put_application(params, %{assigns: %{token: %Token{user: %User{} = user} = token}} = _conn) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index aff7b67db..175b1c4c0 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -462,6 +462,11 @@ defmodule Pleroma.Web.Router do put("/statuses/:id/emoji_reactions/:emoji", EmojiReactionController, :create) end + scope "/api/v1/akkoma", Pleroma.Web.AkkomaAPI do + pipe_through(:authenticated_api) + get("/translation/languages", TranslationController, :languages) + end + scope "/api/v1", Pleroma.Web.MastodonAPI do pipe_through(:authenticated_api) diff --git a/test/pleroma/translators/deepl_test.exs b/test/pleroma/translators/deepl_test.exs index 286d21d3e..58f23fe26 100644 --- a/test/pleroma/translators/deepl_test.exs +++ b/test/pleroma/translators/deepl_test.exs @@ -8,6 +8,36 @@ defmodule Pleroma.Akkoma.Translators.DeepLTest do clear_config([:deepl, :api_key], "deepl_api_key") end + test "should list supported languages" do + clear_config([:deepl, :tier], :free) + + Tesla.Mock.mock(fn + %{method: :get, url: "https://api-free.deepl.com/v2/languages?type=target"} = env -> + auth_header = Enum.find(env.headers, fn {k, _v} -> k == "authorization" end) + assert {"authorization", "DeepL-Auth-Key deepl_api_key"} = auth_header + + %Tesla.Env{ + status: 200, + body: + Jason.encode!([ + %{ + "language" => "BG", + "name" => "Bulgarian", + "supports_formality" => false + }, + %{ + "language" => "CS", + "name" => "Czech", + "supports_formality" => false + } + ]) + } + end) + + assert {:ok, [%{code: "BG", name: "Bulgarian"}, %{code: "CS", name: "Czech"}]} = + DeepL.languages() + end + test "should work with the free tier" do clear_config([:deepl, :tier], :free) @@ -30,7 +60,7 @@ test "should work with the free tier" do } end) - assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", "en") + assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", nil, "en") end test "should work with the pro tier" do @@ -55,7 +85,33 @@ test "should work with the pro tier" do } end) - assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", "en") + assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", nil, "en") + end + + test "should assign source language if set" do + clear_config([:deepl, :tier], :pro) + + Tesla.Mock.mock(fn + %{method: :post, url: "https://api.deepl.com/v2/translate"} = env -> + auth_header = Enum.find(env.headers, fn {k, _v} -> k == "authorization" end) + assert {"authorization", "DeepL-Auth-Key deepl_api_key"} = auth_header + assert String.contains?(env.body, "source_lang=ja") + + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + translations: [ + %{ + "text" => "I will crush you", + "detected_source_language" => "ja" + } + ] + }) + } + end) + + assert {:ok, "ja", "I will crush you"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", "ja", "en") end test "should gracefully fail if the API errors" do @@ -69,7 +125,8 @@ test "should gracefully fail if the API errors" do } end) - assert {:error, "DeepL request failed (code 403)"} = DeepL.translate("ギュギュ握りつぶしちゃうぞ", "en") + assert {:error, "DeepL request failed (code 403)"} = + DeepL.translate("ギュギュ握りつぶしちゃうぞ", nil, "en") end end end diff --git a/test/pleroma/translators/libre_translate_test.exs b/test/pleroma/translators/libre_translate_test.exs index 9ed2c5323..d28d9278a 100644 --- a/test/pleroma/translators/libre_translate_test.exs +++ b/test/pleroma/translators/libre_translate_test.exs @@ -8,10 +8,35 @@ defmodule Pleroma.Akkoma.Translators.LibreTranslateTest do clear_config([:libre_translate, :url], "http://libre.translate/translate") end + test "should list supported languages" do + clear_config([:deepl, :tier], :free) + + Tesla.Mock.mock(fn + %{method: :get, url: "http://libre.translate/languages"} = _ -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!([ + %{ + "code" => "en", + "name" => "English" + }, + %{ + "code" => "ar", + "name" => "Arabic" + } + ]) + } + end) + + assert {:ok, [%{code: "en", name: "English"}, %{code: "ar", name: "Arabic"}]} = + LibreTranslate.languages() + end + test "should work without an API key" do Tesla.Mock.mock(fn %{method: :post, url: "http://libre.translate/translate"} = env -> - assert {:ok, %{"api_key" => nil}} = Jason.decode(env.body) + assert {:ok, %{"api_key" => nil, "source" => "auto"}} = Jason.decode(env.body) %Tesla.Env{ status: 200, @@ -26,7 +51,8 @@ test "should work without an API key" do } end) - assert {:ok, "ja", "I will crush you"} = LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "en") + assert {:ok, "ja", "I will crush you"} = + LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", nil, "en") end test "should work with an API key" do @@ -49,7 +75,8 @@ test "should work with an API key" do } end) - assert {:ok, "ja", "I will crush you"} = LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "en") + assert {:ok, "ja", "I will crush you"} = + LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", nil, "en") end test "should gracefully handle API key errors" do @@ -67,7 +94,25 @@ test "should gracefully handle API key errors" do end) assert {:error, "libre_translate: request failed (code 403)"} = - LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "en") + LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", nil, "en") + end + + test "should set a source language if requested" do + Tesla.Mock.mock(fn + %{method: :post, url: "http://libre.translate/translate"} = env -> + assert {:ok, %{"api_key" => nil, "source" => "ja"}} = Jason.decode(env.body) + + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + translatedText: "I will crush you" + }) + } + end) + + assert {:ok, "ja", "I will crush you"} = + LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "ja", "en") end test "should gracefully handle an unsupported language" do @@ -85,7 +130,7 @@ test "should gracefully handle an unsupported language" do end) assert {:error, "libre_translate: request failed (code 400)"} = - LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", "zoop") + LibreTranslate.translate("ギュギュ握りつぶしちゃうぞ", nil, "zoop") end end end From decbca0c91d78c5e63c62bf9501deff304178b44 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Tue, 30 Aug 2022 16:59:33 +0000 Subject: [PATCH 33/44] add seperate source and dest entries in language listing (#193) Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/193 --- lib/pleroma/akkoma/translators/deepl.ex | 23 +++++++++---- .../akkoma/translators/libre_translate.ex | 3 +- lib/pleroma/akkoma/translators/translator.ex | 5 ++- .../controllers/translation_controller.ex | 8 ++--- .../operations/translate_operation.ex | 22 +++++++++--- test/pleroma/translators/deepl_test.exs | 16 ++++++++- .../translators/libre_translate_test.exs | 3 +- .../controllers/status_controller_test.exs | 34 +++++++++++++++++++ 8 files changed, 94 insertions(+), 20 deletions(-) diff --git a/lib/pleroma/akkoma/translators/deepl.ex b/lib/pleroma/akkoma/translators/deepl.ex index f93fb7e59..da6b8a582 100644 --- a/lib/pleroma/akkoma/translators/deepl.ex +++ b/lib/pleroma/akkoma/translators/deepl.ex @@ -23,12 +23,21 @@ defp tier do @impl Pleroma.Akkoma.Translator def languages do - with {:ok, %{status: 200} = response} <- do_languages(), - {:ok, body} <- Jason.decode(response.body) do - resp = - Enum.map(body, fn %{"language" => code, "name" => name} -> %{code: code, name: name} end) + with {:ok, %{status: 200} = source_response} <- do_languages("source"), + {:ok, %{status: 200} = dest_response} <- do_languages("target"), + {:ok, source_body} <- Jason.decode(source_response.body), + {:ok, dest_body} <- Jason.decode(dest_response.body) do + source_resp = + Enum.map(source_body, fn %{"language" => code, "name" => name} -> + %{code: code, name: name} + end) - {:ok, resp} + dest_resp = + Enum.map(dest_body, fn %{"language" => code, "name" => name} -> + %{code: code, name: name} + end) + + {:ok, source_resp, dest_resp} else {:ok, %{status: status} = response} -> Logger.warning("DeepL: Request rejected: #{inspect(response)}") @@ -80,9 +89,9 @@ defp do_request(api_key, tier, string, from_language, to_language) do defp maybe_add_source(opts, nil), do: opts defp maybe_add_source(opts, lang), do: Map.put(opts, :source_lang, lang) - defp do_languages() do + defp do_languages(type) do HTTP.get( - base_url(tier()) <> "languages?type=target", + base_url(tier()) <> "languages?type=#{type}", [ {"authorization", "DeepL-Auth-Key #{api_key()}"} ] diff --git a/lib/pleroma/akkoma/translators/libre_translate.ex b/lib/pleroma/akkoma/translators/libre_translate.ex index 319907c2f..3a8d9d827 100644 --- a/lib/pleroma/akkoma/translators/libre_translate.ex +++ b/lib/pleroma/akkoma/translators/libre_translate.ex @@ -18,7 +18,8 @@ def languages do with {:ok, %{status: 200} = response} <- do_languages(), {:ok, body} <- Jason.decode(response.body) do resp = Enum.map(body, fn %{"code" => code, "name" => name} -> %{code: code, name: name} end) - {:ok, resp} + # No separate source/dest + {:ok, resp, resp} else {:ok, %{status: status} = response} -> Logger.warning("LibreTranslate: Request rejected: #{inspect(response)}") diff --git a/lib/pleroma/akkoma/translators/translator.ex b/lib/pleroma/akkoma/translators/translator.ex index aa49b0655..93fbeb3b9 100644 --- a/lib/pleroma/akkoma/translators/translator.ex +++ b/lib/pleroma/akkoma/translators/translator.ex @@ -1,5 +1,8 @@ defmodule Pleroma.Akkoma.Translator do @callback translate(String.t(), String.t() | nil, String.t()) :: {:ok, String.t(), String.t()} | {:error, any()} - @callback languages() :: {:ok, [%{name: String.t(), code: String.t()}]} | {:error, any()} + @callback languages() :: + {:ok, [%{name: String.t(), code: String.t()}], + [%{name: String.t(), code: String.t()}]} + | {:error, any()} end diff --git a/lib/pleroma/web/akkoma_api/controllers/translation_controller.ex b/lib/pleroma/web/akkoma_api/controllers/translation_controller.ex index 49ef89a50..9983a7e39 100644 --- a/lib/pleroma/web/akkoma_api/controllers/translation_controller.ex +++ b/lib/pleroma/web/akkoma_api/controllers/translation_controller.ex @@ -21,9 +21,9 @@ defmodule Pleroma.Web.AkkomaAPI.TranslationController do @doc "GET /api/v1/akkoma/translation/languages" def languages(conn, _params) do - with {:ok, languages} <- get_languages() do + with {:ok, source_languages, dest_languages} <- get_languages() do conn - |> json(languages) + |> json(%{source: source_languages, target: dest_languages}) else e -> IO.inspect(e) end @@ -33,8 +33,8 @@ defp get_languages do module = Pleroma.Config.get([:translator, :module]) @cachex.fetch!(:translations_cache, "languages:#{module}}", fn _ -> - with {:ok, languages} <- module.languages() do - {:ok, languages} + with {:ok, source_languages, dest_languages} <- module.languages() do + {:ok, source_languages, dest_languages} else {:error, err} -> {:ignore, {:error, err}} end diff --git a/lib/pleroma/web/api_spec/operations/translate_operation.ex b/lib/pleroma/web/api_spec/operations/translate_operation.ex index aa3b69a18..bf0280319 100644 --- a/lib/pleroma/web/api_spec/operations/translate_operation.ex +++ b/lib/pleroma/web/api_spec/operations/translate_operation.ex @@ -17,22 +17,34 @@ def languages_operation() do operationId: "AkkomaAPI.TranslationController.languages", security: [%{"oAuth" => ["read:statuses"]}], responses: %{ - 200 => Operation.response("Translation", "application/json", languages_schema()) + 200 => + Operation.response("Translation", "application/json", source_dest_languages_schema()) + } + } + end + + defp source_dest_languages_schema do + %Schema{ + type: :object, + required: [:source, :target], + properties: %{ + source: languages_schema(), + target: languages_schema() } } end defp languages_schema do %Schema{ - type: "array", + type: :array, items: %Schema{ - type: "object", + type: :object, properties: %{ code: %Schema{ - type: "string" + type: :string }, name: %Schema{ - type: "string" + type: :string } } } diff --git a/test/pleroma/translators/deepl_test.exs b/test/pleroma/translators/deepl_test.exs index 58f23fe26..d85bef982 100644 --- a/test/pleroma/translators/deepl_test.exs +++ b/test/pleroma/translators/deepl_test.exs @@ -32,9 +32,23 @@ test "should list supported languages" do } ]) } + + %{method: :get, url: "https://api-free.deepl.com/v2/languages?type=source"} -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!([ + %{ + "language" => "JA", + "name" => "Japanese", + "supports_formality" => false + } + ]) + } end) - assert {:ok, [%{code: "BG", name: "Bulgarian"}, %{code: "CS", name: "Czech"}]} = + assert {:ok, [%{code: "JA", name: "Japanese"}], + [%{code: "BG", name: "Bulgarian"}, %{code: "CS", name: "Czech"}]} = DeepL.languages() end diff --git a/test/pleroma/translators/libre_translate_test.exs b/test/pleroma/translators/libre_translate_test.exs index d28d9278a..3c81c3d76 100644 --- a/test/pleroma/translators/libre_translate_test.exs +++ b/test/pleroma/translators/libre_translate_test.exs @@ -29,7 +29,8 @@ test "should list supported languages" do } end) - assert {:ok, [%{code: "en", name: "English"}, %{code: "ar", name: "Arabic"}]} = + assert {:ok, [%{code: "en", name: "English"}, %{code: "ar", name: "Arabic"}], + [%{code: "en", name: "English"}, %{code: "ar", name: "Arabic"}]} = LibreTranslate.languages() end diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index e38f5fe58..f76ab3d0d 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2080,6 +2080,40 @@ test "posting a quote of a status that doesn't exist", %{conn: conn} do oauth_access(["read:statuses"]) end + test "listing languages", %{conn: conn} do + Tesla.Mock.mock_global(fn + %{method: :get, url: "https://api-free.deepl.com/v2/languages?type=source"} -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!([ + %{language: "en", name: "English"} + ]) + } + + %{method: :get, url: "https://api-free.deepl.com/v2/languages?type=target"} -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!([ + %{language: "ja", name: "Japanese"} + ]) + } + end) + + conn = + conn + |> put_req_header("content-type", "application/json") + |> get("/api/v1/akkoma/translation/languages") + + response = json_response_and_validate_schema(conn, 200) + + assert %{ + "source" => [%{"code" => "en", "name" => "English"}], + "target" => [%{"code" => "ja", "name" => "Japanese"}] + } = response + end + test "should return text and detected language", %{conn: conn} do clear_config([:deepl, :tier], :free) From 8e4de118c1f108437c43c2bdbca67d1084d69ba5 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Wed, 31 Aug 2022 18:00:36 +0000 Subject: [PATCH 34/44] Don't persist local undone follow (#194) same deal but backwards this time Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/194 --- lib/pleroma/web/activity_pub/activity_pub.ex | 8 +++--- lib/pleroma/web/activity_pub/utils.ex | 12 --------- ...1170605_remove_local_cancelled_follows.exs | 22 ++++++++++++++++ test/mix/tasks/pleroma/relay_test.exs | 6 ++--- test/pleroma/notification_test.exs | 5 ++-- .../web/activity_pub/activity_pub_test.exs | 26 +++++++++++++++++-- test/pleroma/web/activity_pub/utils_test.exs | 23 ---------------- test/pleroma/web/common_api_test.exs | 16 +++++------- 8 files changed, 61 insertions(+), 57 deletions(-) create mode 100644 priv/repo/migrations/20220831170605_remove_local_cancelled_follows.exs diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 03e72be58..20acdf86e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -331,9 +331,9 @@ defp do_unfollow(follower, followed, activity_id, local) defp do_unfollow(follower, followed, activity_id, local) when local == true do with %Activity{} = follow_activity <- fetch_latest_follow(follower, followed), - {:ok, follow_activity} <- update_follow_state(follow_activity, "cancelled"), unfollow_data <- make_unfollow_data(follower, followed, follow_activity, activity_id), {:ok, activity} <- insert(unfollow_data, local), + {:ok, _activity} <- Repo.delete(follow_activity), _ <- notify_and_stream(activity), :ok <- maybe_federate(activity) do {:ok, activity} @@ -349,7 +349,7 @@ defp do_unfollow(follower, followed, activity_id, false) do with %Activity{} = follow_activity <- fetch_latest_follow(follower, followed), {:ok, _activity} <- Repo.delete(follow_activity), unfollow_data <- make_unfollow_data(follower, followed, follow_activity, activity_id), - unfollow_activity <- remote_unfollow_data(unfollow_data), + unfollow_activity <- make_unfollow_activity(unfollow_data, false), _ <- notify_and_stream(unfollow_activity) do {:ok, unfollow_activity} else @@ -358,12 +358,12 @@ defp do_unfollow(follower, followed, activity_id, false) do end end - defp remote_unfollow_data(data) do + defp make_unfollow_activity(data, local) do {recipients, _, _} = get_recipients(data) %Activity{ data: data, - local: false, + local: local, actor: data["actor"], recipients: recipients } diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 5e5df4888..b920e8c1d 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -472,18 +472,6 @@ def update_follow_state_for_all( {:ok, activity} end - def update_follow_state( - %Activity{} = activity, - state - ) do - new_data = Map.put(activity.data, "state", state) - changeset = Changeset.change(activity, data: new_data) - - with {:ok, activity} <- Repo.update(changeset) do - {:ok, activity} - end - end - @doc """ Makes a follow activity data for the given follower and followed """ diff --git a/priv/repo/migrations/20220831170605_remove_local_cancelled_follows.exs b/priv/repo/migrations/20220831170605_remove_local_cancelled_follows.exs new file mode 100644 index 000000000..16597f848 --- /dev/null +++ b/priv/repo/migrations/20220831170605_remove_local_cancelled_follows.exs @@ -0,0 +1,22 @@ +defmodule Pleroma.Repo.Migrations.RemoveLocalCancelledFollows do + use Ecto.Migration + + def up do + statement = """ + DELETE FROM + activities + WHERE + (data->>'type') = 'Follow' + AND + (data->>'state') = 'cancelled' + AND + local = true; + """ + + execute(statement) + end + + def down do + :ok + end +end diff --git a/test/mix/tasks/pleroma/relay_test.exs b/test/mix/tasks/pleroma/relay_test.exs index db75b3630..d45690418 100644 --- a/test/mix/tasks/pleroma/relay_test.exs +++ b/test/mix/tasks/pleroma/relay_test.exs @@ -65,7 +65,7 @@ test "relay is unfollowed" do Mix.Tasks.Pleroma.Relay.run(["unfollow", target_instance]) cancelled_activity = Activity.get_by_ap_id(follow_activity.data["id"]) - assert cancelled_activity.data["state"] == "cancelled" + assert is_nil(cancelled_activity) [undo_activity] = ActivityPub.fetch_activities([], %{ @@ -78,7 +78,6 @@ test "relay is unfollowed" do assert undo_activity.data["type"] == "Undo" assert undo_activity.data["actor"] == local_user.ap_id - assert undo_activity.data["object"]["id"] == cancelled_activity.data["id"] refute "#{target_instance}/followers" in User.following(local_user) end @@ -142,7 +141,7 @@ test "force unfollow when relay is dead" do Mix.Tasks.Pleroma.Relay.run(["unfollow", target_instance, "--force"]) cancelled_activity = Activity.get_by_ap_id(follow_activity.data["id"]) - assert cancelled_activity.data["state"] == "cancelled" + assert is_nil(cancelled_activity) [undo_activity] = ActivityPub.fetch_activities( @@ -152,7 +151,6 @@ test "force unfollow when relay is dead" do assert undo_activity.data["type"] == "Undo" assert undo_activity.data["actor"] == local_user.ap_id - assert undo_activity.data["object"]["id"] == cancelled_activity.data["id"] refute "#{target_instance}/followers" in User.following(local_user) end end diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs index 4354dd2b6..8db208878 100644 --- a/test/pleroma/notification_test.exs +++ b/test/pleroma/notification_test.exs @@ -427,13 +427,12 @@ test "it doesn't create a notification for follow-unfollow-follow chains" do {:ok, _, _, _activity} = CommonAPI.follow(user, followed_user) assert FollowingRelationship.following?(user, followed_user) - assert [notification] = Notification.for_user(followed_user) + assert [_notification] = Notification.for_user(followed_user) CommonAPI.unfollow(user, followed_user) {:ok, _, _, _activity_dupe} = CommonAPI.follow(user, followed_user) - notification_id = notification.id - assert [%{id: ^notification_id}] = Notification.for_user(followed_user) + assert Enum.count(Notification.for_user(followed_user)) == 1 end test "dismisses the notification on follow request rejection" do diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index 50eff9431..ec562ac7b 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -1373,6 +1373,25 @@ test "creates an undo activity for a pending follow request" do assert embedded_object["id"] == follow_activity.data["id"] end + test "it removes the follow activity if it was local" do + follower = insert(:user, local: true) + followed = insert(:user) + + {:ok, _, _, follow_activity} = CommonAPI.follow(follower, followed) + {:ok, activity} = ActivityPub.unfollow(follower, followed, nil, true) + + assert activity.data["type"] == "Undo" + assert activity.data["actor"] == follower.ap_id + + follow_activity = Activity.get_by_id(follow_activity.id) + assert is_nil(follow_activity) + assert is_nil(Utils.fetch_latest_follow(follower, followed)) + + # We need to keep our own undo + undo_activity = Activity.get_by_ap_id(activity.data["id"]) + refute is_nil(undo_activity) + end + test "it removes the follow activity if it was remote" do follower = insert(:user, local: false) followed = insert(:user) @@ -1383,9 +1402,12 @@ test "it removes the follow activity if it was remote" do assert activity.data["type"] == "Undo" assert activity.data["actor"] == follower.ap_id - activity = Activity.get_by_id(follow_activity.id) - assert is_nil(activity) + follow_activity = Activity.get_by_id(follow_activity.id) + assert is_nil(follow_activity) assert is_nil(Utils.fetch_latest_follow(follower, followed)) + + undo_activity = Activity.get_by_ap_id(activity.data["id"]) + assert is_nil(undo_activity) end end diff --git a/test/pleroma/web/activity_pub/utils_test.exs b/test/pleroma/web/activity_pub/utils_test.exs index 0d88303e3..e45af3aec 100644 --- a/test/pleroma/web/activity_pub/utils_test.exs +++ b/test/pleroma/web/activity_pub/utils_test.exs @@ -229,29 +229,6 @@ test "also updates the state of accepted follows" do end end - describe "update_follow_state/2" do - test "updates the state of the given follow activity" do - user = insert(:user, is_locked: true) - follower = insert(:user) - - {:ok, _, _, follow_activity} = CommonAPI.follow(follower, user) - {:ok, _, _, follow_activity_two} = CommonAPI.follow(follower, user) - - data = - follow_activity_two.data - |> Map.put("state", "accept") - - cng = Ecto.Changeset.change(follow_activity_two, data: data) - - {:ok, follow_activity_two} = Repo.update(cng) - - {:ok, follow_activity_two} = Utils.update_follow_state(follow_activity_two, "reject") - - assert refresh_record(follow_activity).data["state"] == "pending" - assert refresh_record(follow_activity_two).data["state"] == "reject" - end - end - describe "update_element_in_object/3" do test "updates likes" do user = insert(:user) diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index fa751bf60..840d74d2f 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1058,24 +1058,23 @@ test "also unsubscribes a user" do refute User.subscribed_to?(follower, followed) end - test "cancels a pending follow for a local user" do + test "removes a pending follow for a local user" do follower = insert(:user) followed = insert(:user, is_locked: true) - assert {:ok, follower, followed, %{id: activity_id, data: %{"state" => "pending"}}} = + assert {:ok, follower, followed, %{id: _activity_id, data: %{"state" => "pending"}}} = CommonAPI.follow(follower, followed) assert User.get_follow_state(follower, followed) == :follow_pending assert {:ok, follower} = CommonAPI.unfollow(follower, followed) assert User.get_follow_state(follower, followed) == nil - assert %{id: ^activity_id, data: %{"state" => "cancelled"}} = - Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(follower, followed) + assert is_nil(Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(follower, followed)) assert %{ data: %{ "type" => "Undo", - "object" => %{"type" => "Follow", "state" => "cancelled"} + "object" => %{"type" => "Follow"} } } = Pleroma.Web.ActivityPub.Utils.fetch_latest_undo(follower) end @@ -1084,20 +1083,19 @@ test "cancels a pending follow for a remote user" do follower = insert(:user) followed = insert(:user, is_locked: true, local: false, ap_enabled: true) - assert {:ok, follower, followed, %{id: activity_id, data: %{"state" => "pending"}}} = + assert {:ok, follower, followed, %{id: _activity_id, data: %{"state" => "pending"}}} = CommonAPI.follow(follower, followed) assert User.get_follow_state(follower, followed) == :follow_pending assert {:ok, follower} = CommonAPI.unfollow(follower, followed) assert User.get_follow_state(follower, followed) == nil - assert %{id: ^activity_id, data: %{"state" => "cancelled"}} = - Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(follower, followed) + assert is_nil(Pleroma.Web.ActivityPub.Utils.fetch_latest_follow(follower, followed)) assert %{ data: %{ "type" => "Undo", - "object" => %{"type" => "Follow", "state" => "cancelled"} + "object" => %{"type" => "Follow"} } } = Pleroma.Web.ActivityPub.Utils.fetch_latest_undo(follower) end From 7a90d71e8d2bc0a5af5522be2881b36277e1eebe Mon Sep 17 00:00:00 2001 From: floatingghost Date: Fri, 2 Sep 2022 22:05:39 +0000 Subject: [PATCH 35/44] ensure .exs config is used before default (#197) Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/197 --- lib/pleroma/config/transfer_task.ex | 8 +++- test/pleroma/config/transfer_task_test.exs | 44 +++++++++++++++++++++- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 6a3184e6c..81dc847cf 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -38,7 +38,6 @@ def start_link(restart_pleroma? \\ true) do def load_and_update_env(deleted_settings \\ [], restart_pleroma? \\ true) do with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do # We need to restart applications for loaded settings take effect - {logger, other} = (Repo.all(ConfigDB) ++ deleted_settings) |> Enum.map(&merge_with_default/1) @@ -85,7 +84,12 @@ defp maybe_set_pleroma_last(apps) do end defp merge_with_default(%{group: group, key: key, value: value} = setting) do - default = Config.Holder.default_config(group, key) + default = + if group == :pleroma do + Config.get([key], Config.Holder.default_config(group, key)) + else + Config.Holder.default_config(group, key) + end merged = cond do diff --git a/test/pleroma/config/transfer_task_test.exs b/test/pleroma/config/transfer_task_test.exs index c56f20ec5..30cb92fa7 100644 --- a/test/pleroma/config/transfer_task_test.exs +++ b/test/pleroma/config/transfer_task_test.exs @@ -10,13 +10,16 @@ defmodule Pleroma.Config.TransferTaskTest do alias Pleroma.Config.TransferTask - setup do: clear_config(:configurable_from_database, true) + setup do + clear_config(:configurable_from_database, true) + end test "transfer config values from db to env" do refute Application.get_env(:pleroma, :test_key) refute Application.get_env(:idna, :test_key) refute Application.get_env(:quack, :test_key) refute Application.get_env(:postgrex, :test_key) + initial = Application.get_env(:logger, :level) insert(:config, key: :test_key, value: [live: 2, com: 3]) @@ -24,7 +27,7 @@ test "transfer config values from db to env" do insert(:config, group: :quack, key: :test_key, value: [:test_value1, :test_value2]) insert(:config, group: :postgrex, key: :test_key, value: :value) insert(:config, group: :logger, key: :level, value: :debug) - + insert(:config, group: :pleroma, key: :instance, value: [static_dir: "static_dir_from_db"]) TransferTask.start_link([]) assert Application.get_env(:pleroma, :test_key) == [live: 2, com: 3] @@ -32,6 +35,7 @@ test "transfer config values from db to env" do assert Application.get_env(:quack, :test_key) == [:test_value1, :test_value2] assert Application.get_env(:logger, :level) == :debug assert Application.get_env(:postgrex, :test_key) == :value + assert Application.get_env(:pleroma, :instance)[:static_dir] == "static_dir_from_db" on_exit(fn -> Application.delete_env(:pleroma, :test_key) @@ -39,6 +43,42 @@ test "transfer config values from db to env" do Application.delete_env(:quack, :test_key) Application.delete_env(:postgrex, :test_key) Application.put_env(:logger, :level, initial) + System.delete_env("RELEASE_NAME") + end) + end + + test "transfer task falls back to env before default" do + instance = Application.get_env(:pleroma, :instance) + + insert(:config, key: :instance, value: [name: "wow"]) + clear_config([:instance, :static_dir], "static_dir_from_env") + TransferTask.start_link([]) + + assert Application.get_env(:pleroma, :instance)[:name] == "wow" + assert Application.get_env(:pleroma, :instance)[:static_dir] == "static_dir_from_env" + + on_exit(fn -> + Application.put_env(:pleroma, :instance, instance) + end) + end + + test "transfer task falls back to release defaults if no other values found" do + instance = Application.get_env(:pleroma, :instance) + + System.put_env("RELEASE_NAME", "akkoma") + Pleroma.Config.Holder.save_default() + insert(:config, key: :instance, value: [name: "wow"]) + Application.delete_env(:pleroma, :instance) + + TransferTask.start_link([]) + + assert Application.get_env(:pleroma, :instance)[:name] == "wow" + assert Application.get_env(:pleroma, :instance)[:static_dir] == "/var/lib/akkoma/static" + + on_exit(fn -> + System.delete_env("RELEASE_NAME") + Pleroma.Config.Holder.save_default() + Application.put_env(:pleroma, :instance, instance) end) end From 1b826eea543d5210d9004c8b418d41285238f5b4 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Sun, 4 Sep 2022 23:31:41 +0000 Subject: [PATCH 36/44] Allow reacting with remote emoji when they exist on the post (#200) Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/200 --- lib/pleroma/web/activity_pub/builder.ex | 97 +++++++++++++----- .../emoji_react_validator.ex | 2 +- lib/pleroma/web/activity_pub/utils.ex | 32 ++++-- lib/pleroma/web/common_api.ex | 3 +- .../web/mastodon_api/views/status_view.ex | 2 +- .../controllers/emoji_reaction_controller.ex | 10 +- .../pleroma_api/views/emoji_reaction_view.ex | 14 ++- .../emoji_reaction_controller_test.exs | 98 +++++++++++++++++-- 8 files changed, 211 insertions(+), 47 deletions(-) diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index 97ceaf08e..71ccbdef5 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -55,37 +55,84 @@ def follow(follower, followed) do {:ok, data, []} end + defp unicode_emoji_react(_object, data, emoji) do + data + |> Map.put("content", emoji) + |> Map.put("type", "EmojiReact") + end + + defp add_emoji_content(data, emoji, url) do + data + |> Map.put("content", Emoji.maybe_quote(emoji)) + |> Map.put("type", "EmojiReact") + |> Map.put("tag", [ + %{} + |> Map.put("id", url) + |> Map.put("type", "Emoji") + |> Map.put("name", Emoji.maybe_quote(emoji)) + |> Map.put( + "icon", + %{} + |> Map.put("type", "Image") + |> Map.put("url", url) + ) + ]) + end + + defp remote_custom_emoji_react( + %{data: %{"reactions" => existing_reactions}} = object, + data, + emoji + ) do + [emoji_code, instance] = String.split(Emoji.stripped_name(emoji), "@") + + matching_reaction = + Enum.find( + existing_reactions, + fn [name, _, url] -> + url = URI.parse(url) + url.host == instance && name == emoji_code + end + ) + + if matching_reaction do + [name, _, url] = matching_reaction + add_emoji_content(data, name, url) + else + {:error, "Could not react"} + end + end + + defp remote_custom_emoji_react(_object, data, emoji) do + {:error, "Could not react"} + end + + defp local_custom_emoji_react(data, emoji) do + with %{} = emojo <- Emoji.get(emoji) do + path = emojo |> Map.get(:file) + url = "#{Endpoint.url()}#{path}" + add_emoji_content(data, emojo.code, url) + else + _ -> {:error, "Emoji does not exist"} + end + end + + defp custom_emoji_react(object, data, emoji) do + if String.contains?(emoji, "@") do + remote_custom_emoji_react(object, data, emoji) + else + local_custom_emoji_react(data, emoji) + end + end + @spec emoji_react(User.t(), Object.t(), String.t()) :: {:ok, map(), keyword()} def emoji_react(actor, object, emoji) do with {:ok, data, meta} <- object_action(actor, object) do data = if Emoji.is_unicode_emoji?(emoji) do - data - |> Map.put("content", emoji) - |> Map.put("type", "EmojiReact") + unicode_emoji_react(object, data, emoji) else - with %{} = emojo <- Emoji.get(emoji) do - path = emojo |> Map.get(:file) - url = "#{Endpoint.url()}#{path}" - - data - |> Map.put("content", emoji) - |> Map.put("type", "EmojiReact") - |> Map.put("tag", [ - %{} - |> Map.put("id", url) - |> Map.put("type", "Emoji") - |> Map.put("name", emojo.code) - |> Map.put( - "icon", - %{} - |> Map.put("type", "Image") - |> Map.put("url", url) - ) - ]) - else - _ -> {:error, "Emoji does not exist"} - end + custom_emoji_react(object, data, emoji) end {:ok, data, meta} diff --git a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex index 306a57a93..6109a0355 100644 --- a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations @primary_key false - @emoji_regex ~r/:[A-Za-z0-9_-]+:/ + @emoji_regex ~r/:[A-Za-z0-9_-]+(@.+)?:/ embedded_schema do quote do diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index b920e8c1d..008aec475 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -329,7 +329,7 @@ def add_emoji_reaction_to_object( object ) do reactions = get_cached_emoji_reactions(object) - emoji = stripped_emoji_name(emoji) + emoji = Pleroma.Emoji.stripped_name(emoji) url = emoji_url(emoji, activity) new_reactions = @@ -356,12 +356,6 @@ def add_emoji_reaction_to_object( update_element_in_object("reaction", new_reactions, object, count) end - defp stripped_emoji_name(name) do - name - |> String.replace_leading(":", "") - |> String.replace_trailing(":", "") - end - defp emoji_url( name, %Activity{ @@ -384,7 +378,7 @@ def remove_emoji_reaction_from_object( %Activity{data: %{"content" => emoji, "actor" => actor}} = activity, object ) do - emoji = stripped_emoji_name(emoji) + emoji = Pleroma.Emoji.stripped_name(emoji) reactions = get_cached_emoji_reactions(object) url = emoji_url(emoji, activity) @@ -513,19 +507,37 @@ def fetch_latest_undo(%User{ap_id: ap_id}) do def get_latest_reaction(internal_activity_id, %{ap_id: ap_id}, emoji) do %{data: %{"object" => object_ap_id}} = Activity.get_by_id(internal_activity_id) - emoji = Pleroma.Emoji.maybe_quote(emoji) "EmojiReact" |> Activity.Queries.by_type() |> where(actor: ^ap_id) - |> where([activity], fragment("?->>'content' = ?", activity.data, ^emoji)) + |> custom_emoji_discriminator(emoji) |> Activity.Queries.by_object_id(object_ap_id) |> order_by([activity], fragment("? desc nulls last", activity.id)) |> limit(1) |> Repo.one() end + defp custom_emoji_discriminator(query, emoji) do + if String.contains?(emoji, "@") do + stripped = Pleroma.Emoji.stripped_name(emoji) + [name, domain] = String.split(stripped, "@") + domain_pattern = "%" <> domain <> "%" + emoji_pattern = Pleroma.Emoji.maybe_quote(name) + + query + |> where([activity], fragment("?->>'content' = ? + AND EXISTS ( + SELECT FROM jsonb_array_elements(?->'tag') elem + WHERE elem->>'id' ILIKE ? + )", activity.data, ^emoji_pattern, activity.data, ^domain_pattern)) + else + query + |> where([activity], fragment("?->>'content' = ?", activity.data, ^emoji)) + end + end + #### Announce-related helpers @doc """ diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index bc5e26cf7..23d353dc2 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -209,7 +209,8 @@ def react_with_emoji(id, user, emoji) do {:ok, activity, _} <- Pipeline.common_pipeline(emoji_react, local: true) do {:ok, activity} else - _ -> {:error, dgettext("errors", "Could not add reaction emoji")} + _ -> + {:error, dgettext("errors", "Could not add reaction emoji")} end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index d838c4673..0d2571ab8 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -587,7 +587,7 @@ defp pin_data(%Object{data: %{"id" => object_id}}, %User{pinned_objects: pinned_ defp build_emoji_map(emoji, users, url, current_user) do %{ - name: emoji, + name: Pleroma.Web.PleromaAPI.EmojiReactionView.emoji_name(emoji, url), count: length(users), url: MediaProxy.url(url), me: !!(current_user && current_user.ap_id in users), diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex index 91658587a..0933363a6 100644 --- a/lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/emoji_reaction_controller.ex @@ -74,7 +74,10 @@ defp filter(reactions, %{emoji: emoji}) when is_binary(emoji) do defp filter(reactions, _), do: reactions def create(%{assigns: %{user: user}} = conn, %{id: activity_id, emoji: emoji}) do - emoji = Pleroma.Emoji.maybe_quote(emoji) + emoji = + emoji + |> Pleroma.Emoji.fully_qualify_emoji() + |> Pleroma.Emoji.maybe_quote() with {:ok, _activity} <- CommonAPI.react_with_emoji(activity_id, user, emoji) do activity = Activity.get_by_id(activity_id) @@ -86,6 +89,11 @@ def create(%{assigns: %{user: user}} = conn, %{id: activity_id, emoji: emoji}) d end def delete(%{assigns: %{user: user}} = conn, %{id: activity_id, emoji: emoji}) do + emoji = + emoji + |> Pleroma.Emoji.fully_qualify_emoji() + |> Pleroma.Emoji.maybe_quote() + with {:ok, _activity} <- CommonAPI.unreact_with_emoji(activity_id, user, emoji) do activity = Activity.get_by_id(activity_id) diff --git a/lib/pleroma/web/pleroma_api/views/emoji_reaction_view.ex b/lib/pleroma/web/pleroma_api/views/emoji_reaction_view.ex index 9993480db..4335228b6 100644 --- a/lib/pleroma/web/pleroma_api/views/emoji_reaction_view.ex +++ b/lib/pleroma/web/pleroma_api/views/emoji_reaction_view.ex @@ -8,6 +8,18 @@ defmodule Pleroma.Web.PleromaAPI.EmojiReactionView do alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.MediaProxy + def emoji_name(emoji, nil), do: emoji + + def emoji_name(emoji, url) do + url = URI.parse(url) + + if url.host == Pleroma.Web.Endpoint.host() do + emoji + else + "#{emoji}@#{url.host}" + end + end + def render("index.json", %{emoji_reactions: emoji_reactions} = opts) do render_many(emoji_reactions, __MODULE__, "show.json", opts) end @@ -16,7 +28,7 @@ def render("show.json", %{emoji_reaction: {emoji, user_ap_ids, url}, user: user} users = fetch_users(user_ap_ids) %{ - name: emoji, + name: emoji_name(emoji, url), count: length(users), accounts: render(AccountView, "index.json", users: users, for: user), url: MediaProxy.url(url), diff --git a/test/pleroma/web/pleroma_api/controllers/emoji_reaction_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/emoji_reaction_controller_test.exs index 4898179e6..6864b37cb 100644 --- a/test/pleroma/web/pleroma_api/controllers/emoji_reaction_controller_test.exs +++ b/test/pleroma/web/pleroma_api/controllers/emoji_reaction_controller_test.exs @@ -17,22 +17,29 @@ test "PUT /api/v1/pleroma/statuses/:id/reactions/:emoji", %{conn: conn} do user = insert(:user) other_user = insert(:user) - {:ok, activity} = CommonAPI.post(user, %{status: "#cofe"}) + note = insert(:note, user: user, data: %{"reactions" => [["👍", [other_user.ap_id], nil]]}) + activity = insert(:note_activity, note: note, user: user) result = conn |> assign(:user, other_user) |> assign(:token, insert(:oauth_token, user: other_user, scopes: ["write:statuses"])) - |> put("/api/v1/pleroma/statuses/#{activity.id}/reactions/☕") + |> put("/api/v1/pleroma/statuses/#{activity.id}/reactions/\u26A0") |> json_response_and_validate_schema(200) - # We return the status, but this our implementation detail. assert %{"id" => id} = result assert to_string(activity.id) == id assert result["pleroma"]["emoji_reactions"] == [ %{ - "name" => "☕", + "name" => "👍", + "count" => 1, + "me" => true, + "url" => nil, + "account_ids" => [other_user.id] + }, + %{ + "name" => "\u26A0\uFE0F", "count" => 1, "me" => true, "url" => nil, @@ -43,6 +50,7 @@ test "PUT /api/v1/pleroma/statuses/:id/reactions/:emoji", %{conn: conn} do {:ok, activity} = CommonAPI.post(user, %{status: "#cofe"}) ObanHelpers.perform_all() + # Reacting with a custom emoji result = conn @@ -51,7 +59,6 @@ test "PUT /api/v1/pleroma/statuses/:id/reactions/:emoji", %{conn: conn} do |> put("/api/v1/pleroma/statuses/#{activity.id}/reactions/:dinosaur:") |> json_response_and_validate_schema(200) - # We return the status, but this our implementation detail. assert %{"id" => id} = result assert to_string(activity.id) == id @@ -65,6 +72,46 @@ test "PUT /api/v1/pleroma/statuses/:id/reactions/:emoji", %{conn: conn} do } ] + # Reacting with a remote emoji + note = + insert(:note, + user: user, + data: %{"reactions" => [["wow", [other_user.ap_id], "https://remote/emoji/wow"]]} + ) + + activity = insert(:note_activity, note: note, user: user) + + result = + conn + |> assign(:user, user) + |> assign(:token, insert(:oauth_token, user: user, scopes: ["write:statuses"])) + |> put("/api/v1/pleroma/statuses/#{activity.id}/reactions/:wow@remote:") + |> json_response(200) + + assert result["pleroma"]["emoji_reactions"] == [ + %{ + "name" => "wow@remote", + "count" => 2, + "me" => true, + "url" => "https://remote/emoji/wow", + "account_ids" => [user.id, other_user.id] + } + ] + + # Reacting with a remote custom emoji that hasn't been reacted with yet + note = + insert(:note, + user: user + ) + + activity = insert(:note_activity, note: note, user: user) + + assert conn + |> assign(:user, user) + |> assign(:token, insert(:oauth_token, user: user, scopes: ["write:statuses"])) + |> put("/api/v1/pleroma/statuses/#{activity.id}/reactions/:wow@remote:") + |> json_response(400) + # Reacting with a non-emoji assert conn |> assign(:user, other_user) @@ -77,10 +124,22 @@ test "DELETE /api/v1/pleroma/statuses/:id/reactions/:emoji", %{conn: conn} do user = insert(:user) other_user = insert(:user) - {:ok, activity} = CommonAPI.post(user, %{status: "#cofe"}) + note = + insert(:note, + user: user, + data: %{"reactions" => [["wow", [user.ap_id], "https://remote/emoji/wow"]]} + ) + + activity = insert(:note_activity, note: note, user: user) + + ObanHelpers.perform_all() + {:ok, _reaction_activity} = CommonAPI.react_with_emoji(activity.id, other_user, "☕") {:ok, _reaction_activity} = CommonAPI.react_with_emoji(activity.id, other_user, ":dinosaur:") + {:ok, _reaction_activity} = + CommonAPI.react_with_emoji(activity.id, other_user, ":wow@remote:") + ObanHelpers.perform_all() result = @@ -107,7 +166,32 @@ test "DELETE /api/v1/pleroma/statuses/:id/reactions/:emoji", %{conn: conn} do object = Object.get_by_ap_id(activity.data["object"]) - assert object.data["reaction_count"] == 0 + assert object.data["reaction_count"] == 2 + + # Remove custom remote emoji + result = + conn + |> assign(:user, other_user) + |> assign(:token, insert(:oauth_token, user: other_user, scopes: ["write:statuses"])) + |> delete("/api/v1/pleroma/statuses/#{activity.id}/reactions/:wow@remote:") + |> json_response(200) + + assert result["pleroma"]["emoji_reactions"] == [ + %{ + "name" => "wow@remote", + "count" => 1, + "me" => false, + "url" => "https://remote/emoji/wow", + "account_ids" => [user.id] + } + ] + + # Remove custom remote emoji that hasn't been reacted with yet + assert conn + |> assign(:user, other_user) + |> assign(:token, insert(:oauth_token, user: other_user, scopes: ["write:statuses"])) + |> delete("/api/v1/pleroma/statuses/#{activity.id}/reactions/:zoop@remote:") + |> json_response(400) end test "GET /api/v1/pleroma/statuses/:id/reactions", %{conn: conn} do From 1c7d7845c3a01b203a2c6e17805ee91732044e4d Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 5 Sep 2022 00:39:32 +0100 Subject: [PATCH 37/44] fix compilation warnings --- lib/pleroma/web/activity_pub/builder.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index 71ccbdef5..ba756ed64 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -80,7 +80,7 @@ defp add_emoji_content(data, emoji, url) do end defp remote_custom_emoji_react( - %{data: %{"reactions" => existing_reactions}} = object, + %{data: %{"reactions" => existing_reactions}}, data, emoji ) do @@ -103,7 +103,7 @@ defp remote_custom_emoji_react( end end - defp remote_custom_emoji_react(_object, data, emoji) do + defp remote_custom_emoji_react(_object, _data, _emoji) do {:error, "Could not react"} end From f6304cfd7836e646df013e6230304b5915b92899 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 5 Sep 2022 01:24:40 +0100 Subject: [PATCH 38/44] add extra tests for builder --- lib/pleroma/emoji.ex | 5 ++ .../pleroma/web/activity_pub/builder_test.exs | 53 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 24eafda41..dbe9abe8d 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -188,6 +188,11 @@ def emoji_url(%{"type" => "EmojiReact", "content" => emoji, "tag" => tags}) do def emoji_url(_), do: nil + def emoji_name_with_instance(name, url) do + url = url |> URI.parse() |> Map.get(:host) + "#{name}@#{url}" + end + emoji_qualification_map = emojis |> Enum.filter(&String.contains?(&1, "\uFE0F")) diff --git a/test/pleroma/web/activity_pub/builder_test.exs b/test/pleroma/web/activity_pub/builder_test.exs index 640caa2b6..98d20f022 100644 --- a/test/pleroma/web/activity_pub/builder_test.exs +++ b/test/pleroma/web/activity_pub/builder_test.exs @@ -48,4 +48,57 @@ test "returns note data" do assert {:ok, ^expected, []} = Builder.note(draft) end end + + describe "emoji_react/1" do + test "unicode emoji" do + user = insert(:user) + note = insert(:note) + + assert {:ok, %{"content" => "👍", "type" => "EmojiReact"}, []} = + Builder.emoji_react(user, note, "👍") + end + + test "custom emoji" do + user = insert(:user) + note = insert(:note) + + assert {:ok, + %{ + "content" => ":dinosaur:", + "type" => "EmojiReact", + "tag" => [ + %{ + "name" => ":dinosaur:", + "id" => "http://localhost:4001/emoji/dino walking.gif", + "icon" => %{ + "type" => "Image", + "url" => "http://localhost:4001/emoji/dino walking.gif" + } + } + ] + }, []} = Builder.emoji_react(user, note, ":dinosaur:") + end + + test "remote custom emoji" do + user = insert(:user) + other_user = insert(:user, local: false) + note = insert(:note, data: %{"reactions" => [["wow", [other_user.ap_id], "https://remote/emoji/wow"]]}) + + assert {:ok, + %{ + "content" => ":wow:", + "type" => "EmojiReact", + "tag" => [ + %{ + "name" => ":wow:", + "id" => "https://remote/emoji/wow", + "icon" => %{ + "type" => "Image", + "url" => "https://remote/emoji/wow" + } + } + ] + }, []} = Builder.emoji_react(user, note, ":wow@remote:") + end + end end From 6c80977b064dc1aea1fe50a8d636cdf40875a757 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 5 Sep 2022 17:22:33 +0100 Subject: [PATCH 39/44] turn inlineQuotePolicy on by default --- lib/pleroma/web/activity_pub/mrf.ex | 6 +++- .../pleroma/web/activity_pub/builder_test.exs | 34 +++++++++++-------- test/pleroma/web/activity_pub/mrf_test.exs | 4 +-- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 5606dac83..20bce0d5f 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -95,7 +95,11 @@ def pipeline_filter(%{} = message, meta) do def get_policies do Pleroma.Config.get([:mrf, :policies], []) |> get_policies() - |> Enum.concat([Pleroma.Web.ActivityPub.MRF.HashtagPolicy]) + |> Enum.concat([ + Pleroma.Web.ActivityPub.MRF.HashtagPolicy, + Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy + ]) + |> Enum.uniq() end defp get_policies(policy) when is_atom(policy), do: [policy] diff --git a/test/pleroma/web/activity_pub/builder_test.exs b/test/pleroma/web/activity_pub/builder_test.exs index 98d20f022..9269733b7 100644 --- a/test/pleroma/web/activity_pub/builder_test.exs +++ b/test/pleroma/web/activity_pub/builder_test.exs @@ -82,23 +82,27 @@ test "custom emoji" do test "remote custom emoji" do user = insert(:user) other_user = insert(:user, local: false) - note = insert(:note, data: %{"reactions" => [["wow", [other_user.ap_id], "https://remote/emoji/wow"]]}) + + note = + insert(:note, + data: %{"reactions" => [["wow", [other_user.ap_id], "https://remote/emoji/wow"]]} + ) assert {:ok, - %{ - "content" => ":wow:", - "type" => "EmojiReact", - "tag" => [ - %{ - "name" => ":wow:", - "id" => "https://remote/emoji/wow", - "icon" => %{ - "type" => "Image", - "url" => "https://remote/emoji/wow" - } - } - ] - }, []} = Builder.emoji_react(user, note, ":wow@remote:") + %{ + "content" => ":wow:", + "type" => "EmojiReact", + "tag" => [ + %{ + "name" => ":wow:", + "id" => "https://remote/emoji/wow", + "icon" => %{ + "type" => "Image", + "url" => "https://remote/emoji/wow" + } + } + ] + }, []} = Builder.emoji_react(user, note, ":wow@remote:") end end end diff --git a/test/pleroma/web/activity_pub/mrf_test.exs b/test/pleroma/web/activity_pub/mrf_test.exs index 6ab27bc86..ed3233758 100644 --- a/test/pleroma/web/activity_pub/mrf_test.exs +++ b/test/pleroma/web/activity_pub/mrf_test.exs @@ -77,7 +77,7 @@ test "it works as expected with noop policy" do clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.NoOpPolicy]) expected = %{ - mrf_policies: ["NoOpPolicy", "HashtagPolicy"], + mrf_policies: ["NoOpPolicy", "HashtagPolicy", "InlineQuotePolicy"], mrf_hashtag: %{ federated_timeline_removal: [], reject: [], @@ -93,7 +93,7 @@ test "it works as expected with mock policy" do clear_config([:mrf, :policies], [MRFModuleMock]) expected = %{ - mrf_policies: ["MRFModuleMock", "HashtagPolicy"], + mrf_policies: ["MRFModuleMock", "HashtagPolicy", "InlineQuotePolicy"], mrf_module_mock: "some config data", mrf_hashtag: %{ federated_timeline_removal: [], From 2641dcdd15791cdee908fd167309a646c1742e31 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Tue, 6 Sep 2022 19:24:02 +0000 Subject: [PATCH 40/44] Post editing (#202) Rebased from #103 Co-authored-by: Tusooa Zhu Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/202 --- CHANGELOG.md | 1 + .../API/differences_in_mastoapi_responses.md | 4 + lib/pleroma/activity/html.ex | 36 +++ lib/pleroma/application.ex | 1 + lib/pleroma/constants.ex | 36 +++ lib/pleroma/notification.ex | 31 +- lib/pleroma/object/fetcher.ex | 34 ++ lib/pleroma/object/updater.ex | 240 ++++++++++++++ lib/pleroma/upload.ex | 2 + lib/pleroma/web/activity_pub/activity_pub.ex | 13 +- lib/pleroma/web/activity_pub/builder.ex | 13 +- lib/pleroma/web/activity_pub/mrf.ex | 45 ++- .../activity_pub/mrf/anti_link_spam_policy.ex | 3 + .../activity_pub/mrf/ensure_re_prepended.ex | 6 +- .../web/activity_pub/mrf/hashtag_policy.ex | 47 ++- .../web/activity_pub/mrf/keyword_policy.ex | 55 +++- .../mrf/media_proxy_warming_policy.ex | 9 +- .../web/activity_pub/mrf/no_empty_policy.ex | 29 +- .../mrf/no_placeholder_text_policy.ex | 7 +- .../web/activity_pub/mrf/normalize_markup.ex | 6 +- lib/pleroma/web/activity_pub/mrf/policy.ex | 3 +- .../web/activity_pub/object_validator.ex | 111 ++++++- .../article_note_page_validator.ex | 5 +- .../object_validators/attachment_validator.ex | 3 +- .../object_validators/common_fields.ex | 1 + .../object_validators/update_validator.ex | 4 +- lib/pleroma/web/activity_pub/side_effects.ex | 97 +++++- .../web/activity_pub/transmogrifier.ex | 33 ++ .../api_spec/operations/status_operation.ex | 192 +++++++++++ lib/pleroma/web/api_spec/schemas/status.ex | 6 + lib/pleroma/web/common_api.ex | 35 +++ lib/pleroma/web/common_api/activity_draft.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 10 +- .../controllers/notification_controller.ex | 1 + .../controllers/status_controller.ex | 60 +++- .../web/mastodon_api/views/instance_view.ex | 1 + .../mastodon_api/views/notification_view.ex | 15 +- .../web/mastodon_api/views/status_view.ex | 154 ++++++++- lib/pleroma/web/router.ex | 3 + lib/pleroma/web/streamer.ex | 21 ++ lib/pleroma/web/views/streamer_view.ex | 33 ++ ...85734_add_update_to_notifications_enum.exs | 51 +++ priv/static/schemas/litepub-0.1.jsonld | 4 +- test/pleroma/notification_test.exs | 46 +++ test/pleroma/object/fetcher_test.exs | 267 ++++++++++++++++ test/pleroma/object/updater_test.exs | 76 +++++ test/pleroma/upload_test.exs | 30 +- .../mrf/anti_link_spam_policy_test.exs | 33 +- .../mrf/ensure_re_prepended_test.exs | 51 ++- .../activity_pub/mrf/hashtag_policy_test.exs | 70 +++++ .../activity_pub/mrf/keyword_policy_test.exs | 131 ++++++++ .../mrf/media_proxy_warming_policy_test.exs | 44 +++ .../activity_pub/mrf/no_empty_policy_test.exs | 23 ++ .../mrf/no_placeholder_text_policy_test.exs | 41 +++ .../mrf/normalize_markup_test.exs | 59 +++- .../article_note_page_validator_test.exs | 49 +++ .../update_handling_test.exs | 126 +++++++- .../web/activity_pub/side_effects_test.exs | 297 +++++++++++++++++- .../web/activity_pub/transmogrifier_test.exs | 61 ++++ test/pleroma/web/common_api_test.exs | 124 ++++++++ .../controllers/status_controller_test.exs | 174 ++++++++++ .../views/notification_view_test.exs | 26 ++ .../mastodon_api/views/status_view_test.exs | 52 +++ test/pleroma/web/metadata/utils_test.exs | 16 +- test/pleroma/web/streamer_test.exs | 75 +++++ test/support/factory.ex | 12 + 66 files changed, 3216 insertions(+), 130 deletions(-) create mode 100644 lib/pleroma/object/updater.ex create mode 100644 priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs create mode 100644 test/pleroma/object/updater_test.exs diff --git a/CHANGELOG.md b/CHANGELOG.md index 05cb69c40..e63cc1f6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - support for reusing oauth tokens, and not requiring new authorizations - the ability to obfuscate domains in your MRF descriptions - automatic translation of statuses via DeepL or LibreTranslate +- ability to edit posts ### Changed - MFM parsing is now done on the backend by a modified version of ilja's parser -> https://akkoma.dev/AkkomaGang/mfm-parser diff --git a/docs/docs/development/API/differences_in_mastoapi_responses.md b/docs/docs/development/API/differences_in_mastoapi_responses.md index 4465784bf..752be1762 100644 --- a/docs/docs/development/API/differences_in_mastoapi_responses.md +++ b/docs/docs/development/API/differences_in_mastoapi_responses.md @@ -40,6 +40,10 @@ Has these additional fields under the `pleroma` object: - `parent_visible`: If the parent of this post is visible to the user or not. - `pinned_at`: a datetime (iso8601) when status was pinned, `null` otherwise. +The `GET /api/v1/statuses/:id/source` endpoint additionally has the following attributes: + +- `content_type`: The content type of the status source. + ## Scheduled statuses Has these additional fields in `params`: diff --git a/lib/pleroma/activity/html.ex b/lib/pleroma/activity/html.ex index 0bf393836..30409d93d 100644 --- a/lib/pleroma/activity/html.ex +++ b/lib/pleroma/activity/html.ex @@ -8,6 +8,40 @@ defmodule Pleroma.Activity.HTML do @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + # We store a list of cache keys related to an activity in a + # separate cache, scrubber_management_cache. It has the same + # size as scrubber_cache (see application.ex). Every time we add + # a cache to scrubber_cache, we update scrubber_management_cache. + # + # The most recent write of a certain key in the management cache + # is the same as the most recent write of any record related to that + # key in the main cache. + # Assuming LRW ( https://hexdocs.pm/cachex/Cachex.Policy.LRW.html ), + # this means when the management cache is evicted by cachex, all + # related records in the main cache will also have been evicted. + + defp get_cache_keys_for(activity_id) do + with {:ok, list} when is_list(list) <- @cachex.get(:scrubber_management_cache, activity_id) do + list + else + _ -> [] + end + end + + defp add_cache_key_for(activity_id, additional_key) do + current = get_cache_keys_for(activity_id) + + unless additional_key in current do + @cachex.put(:scrubber_management_cache, activity_id, [additional_key | current]) + end + end + + def invalidate_cache_for(activity_id) do + keys = get_cache_keys_for(activity_id) + Enum.map(keys, &@cachex.del(:scrubber_cache, &1)) + @cachex.del(:scrubber_management_cache, activity_id) + end + def get_cached_scrubbed_html_for_activity( content, scrubbers, @@ -19,6 +53,8 @@ def get_cached_scrubbed_html_for_activity( @cachex.fetch!(:scrubber_cache, key, fn _key -> object = Object.normalize(activity, fetch: false) + + add_cache_key_for(activity.id, key) HTML.ensure_scrubbed_html(content, scrubbers, object.data["fake"] || false, callback) end) end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index b809f7733..adccd7c5d 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -150,6 +150,7 @@ defp cachex_children do build_cachex("object", default_ttl: 25_000, ttl_interval: 1000, limit: 2500), build_cachex("rich_media", default_ttl: :timer.minutes(120), limit: 5000), build_cachex("scrubber", limit: 2500), + build_cachex("scrubber_management", limit: 2500), build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500), build_cachex("web_resp", limit: 2500), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index bf92f65cb..7343ef8c3 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -27,4 +27,40 @@ defmodule Pleroma.Constants do do: ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc embed.js embed.css) ) + + const(status_updatable_fields, + do: [ + "source", + "tag", + "updated", + "emoji", + "content", + "summary", + "sensitive", + "attachment", + "generator" + ] + ) + + const(updatable_object_types, + do: [ + "Note", + "Question", + "Audio", + "Video", + "Event", + "Article", + "Page" + ] + ) + + const(actor_types, + do: [ + "Application", + "Group", + "Organization", + "Person", + "Service" + ] + ) end diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index d8878338e..593448713 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -384,7 +384,7 @@ def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = act end def create_notifications(%Activity{data: %{"type" => type}} = activity, options) - when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag"] do + when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag", "Update"] do do_create_notifications(activity, options) end @@ -438,6 +438,9 @@ defp type_from_activity(%{data: %{"type" => type}} = activity) do activity |> type_from_activity_object() + "Update" -> + "update" + t -> raise "No notification type for activity type #{t}" end @@ -503,7 +506,16 @@ def create_poll_notifications(%Activity{} = activity) do def get_notified_from_activity(activity, local_only \\ true) def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only) - when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact", "Flag"] do + when type in [ + "Create", + "Like", + "Announce", + "Follow", + "Move", + "EmojiReact", + "Flag", + "Update" + ] do potential_receiver_ap_ids = get_potential_receiver_ap_ids(activity) potential_receivers = @@ -543,6 +555,21 @@ def get_potential_receiver_ap_ids(%{data: %{"type" => "Flag", "actor" => actor}} (User.all_superusers() |> Enum.map(fn user -> user.ap_id end)) -- [actor] end + # Update activity: notify all who repeated this + def get_potential_receiver_ap_ids(%{data: %{"type" => "Update", "actor" => actor}} = activity) do + with %Object{data: %{"id" => object_id}} <- Object.normalize(activity, fetch: false) do + repeaters = + Activity.Queries.by_type("Announce") + |> Activity.Queries.by_object_id(object_id) + |> Activity.with_joined_user_actor() + |> where([a, u], u.local) + |> select([a, u], u.ap_id) + |> Repo.all() + + repeaters -- [actor] + end + end + def get_potential_receiver_ap_ids(activity) do [] |> Utils.maybe_notify_to_recipients(activity) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 4ca67f0fd..8ec28345f 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -26,8 +26,42 @@ defp touch_changeset(changeset) do end defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do + has_history? = fn + %{"formerRepresentations" => %{"orderedItems" => list}} when is_list(list) -> true + _ -> false + end + internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields()) + remote_history_exists? = has_history?.(new_data) + + # If the remote history exists, we treat that as the only source of truth. + new_data = + if has_history?.(old_data) and not remote_history_exists? do + Map.put(new_data, "formerRepresentations", old_data["formerRepresentations"]) + else + new_data + end + + # If the remote does not have history information, we need to manage it ourselves + new_data = + if not remote_history_exists? do + changed? = + Pleroma.Constants.status_updatable_fields() + |> Enum.any?(fn field -> Map.get(old_data, field) != Map.get(new_data, field) end) + + %{updated_object: updated_object} = + new_data + |> Object.Updater.maybe_update_history(old_data, + updated: changed?, + use_history_in_new_object?: false + ) + + updated_object + else + new_data + end + Map.merge(new_data, internal_fields) end diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex new file mode 100644 index 000000000..ab38d3ed2 --- /dev/null +++ b/lib/pleroma/object/updater.ex @@ -0,0 +1,240 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Object.Updater do + require Pleroma.Constants + + def update_content_fields(orig_object_data, updated_object) do + Pleroma.Constants.status_updatable_fields() + |> Enum.reduce( + %{data: orig_object_data, updated: false}, + fn field, %{data: data, updated: updated} -> + updated = + updated or + (field != "updated" and + Map.get(updated_object, field) != Map.get(orig_object_data, field)) + + data = + if Map.has_key?(updated_object, field) do + Map.put(data, field, updated_object[field]) + else + Map.drop(data, [field]) + end + + %{data: data, updated: updated} + end + ) + end + + def maybe_history(object) do + with history <- Map.get(object, "formerRepresentations"), + true <- is_map(history), + "OrderedCollection" <- Map.get(history, "type"), + true <- is_list(Map.get(history, "orderedItems")), + true <- is_integer(Map.get(history, "totalItems")) do + history + else + _ -> nil + end + end + + def history_for(object) do + with history when not is_nil(history) <- maybe_history(object) do + history + else + _ -> history_skeleton() + end + end + + defp history_skeleton do + %{ + "type" => "OrderedCollection", + "totalItems" => 0, + "orderedItems" => [] + } + end + + def maybe_update_history( + updated_object, + orig_object_data, + opts + ) do + updated = opts[:updated] + use_history_in_new_object? = opts[:use_history_in_new_object?] + + if not updated do + %{updated_object: updated_object, used_history_in_new_object?: false} + else + # Put edit history + # Note that we may have got the edit history by first fetching the object + {new_history, used_history_in_new_object?} = + with true <- use_history_in_new_object?, + updated_history when not is_nil(updated_history) <- maybe_history(opts[:new_data]) do + {updated_history, true} + else + _ -> + history = history_for(orig_object_data) + + latest_history_item = + orig_object_data + |> Map.drop(["id", "formerRepresentations"]) + + updated_history = + history + |> Map.put("orderedItems", [latest_history_item | history["orderedItems"]]) + |> Map.put("totalItems", history["totalItems"] + 1) + + {updated_history, false} + end + + updated_object = + updated_object + |> Map.put("formerRepresentations", new_history) + + %{updated_object: updated_object, used_history_in_new_object?: used_history_in_new_object?} + end + end + + defp maybe_update_poll(to_be_updated, updated_object) do + choice_key = fn data -> + if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf" + end + + with true <- to_be_updated["type"] == "Question", + key <- choice_key.(updated_object), + true <- key == choice_key.(to_be_updated), + orig_choices <- to_be_updated[key] |> Enum.map(&Map.drop(&1, ["replies"])), + new_choices <- updated_object[key] |> Enum.map(&Map.drop(&1, ["replies"])), + true <- orig_choices == new_choices do + # Choices are the same, but counts are different + to_be_updated + |> Map.put(key, updated_object[key]) + else + # Choices (or vote type) have changed, do not allow this + _ -> to_be_updated + end + end + + # This calculates the data to be sent as the object of an Update. + # new_data's formerRepresentations is not considered. + # formerRepresentations is added to the returned data. + def make_update_object_data(original_data, new_data, date) do + %{data: updated_data, updated: updated} = + original_data + |> update_content_fields(new_data) + + if not updated do + updated_data + else + %{updated_object: updated_data} = + updated_data + |> maybe_update_history(original_data, updated: updated, use_history_in_new_object?: false) + + updated_data + |> Map.put("updated", date) + end + end + + # This calculates the data of the new Object from an Update. + # new_data's formerRepresentations is considered. + def make_new_object_data_from_update_object(original_data, new_data) do + update_is_reasonable = + with {_, updated} when not is_nil(updated) <- {:cur_updated, new_data["updated"]}, + {_, {:ok, updated_time, _}} <- {:cur_updated, DateTime.from_iso8601(updated)}, + {_, last_updated} when not is_nil(last_updated) <- + {:last_updated, original_data["updated"] || original_data["published"]}, + {_, {:ok, last_updated_time, _}} <- + {:last_updated, DateTime.from_iso8601(last_updated)}, + :gt <- DateTime.compare(updated_time, last_updated_time) do + :update_everything + else + # only allow poll updates + {:cur_updated, _} -> :no_content_update + :eq -> :no_content_update + # allow all updates + {:last_updated, _} -> :update_everything + # allow no updates + _ -> false + end + + %{ + updated_object: updated_data, + used_history_in_new_object?: used_history_in_new_object?, + updated: updated + } = + if update_is_reasonable == :update_everything do + %{data: updated_data, updated: updated} = + original_data + |> update_content_fields(new_data) + + updated_data + |> maybe_update_history(original_data, + updated: updated, + use_history_in_new_object?: true, + new_data: new_data + ) + |> Map.put(:updated, updated) + else + %{ + updated_object: original_data, + used_history_in_new_object?: false, + updated: false + } + end + + updated_data = + if update_is_reasonable != false do + updated_data + |> maybe_update_poll(new_data) + else + updated_data + end + + %{ + updated_data: updated_data, + updated: updated, + used_history_in_new_object?: used_history_in_new_object? + } + end + + def for_each_history_item(%{"orderedItems" => items} = history, _object, fun) do + new_items = + Enum.map(items, fun) + |> Enum.reduce_while( + {:ok, []}, + fn + {:ok, item}, {:ok, acc} -> {:cont, {:ok, acc ++ [item]}} + e, _acc -> {:halt, e} + end + ) + + case new_items do + {:ok, items} -> {:ok, Map.put(history, "orderedItems", items)} + e -> e + end + end + + def for_each_history_item(history, _, _) do + {:ok, history} + end + + def do_with_history(object, fun) do + with history <- object["formerRepresentations"], + object <- Map.drop(object, ["formerRepresentations"]), + {_, {:ok, object}} <- {:main_body, fun.(object)}, + {_, {:ok, history}} <- {:history_items, for_each_history_item(history, object, fun)} do + object = + if history do + Map.put(object, "formerRepresentations", history) + else + object + end + + {:ok, object} + else + {:main_body, e} -> e + {:history_items, e} -> e + end + end +end diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 17822dc5e..9bf8e03df 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -36,6 +36,7 @@ defmodule Pleroma.Upload do alias Ecto.UUID alias Pleroma.Config alias Pleroma.Maps + alias Pleroma.Web.ActivityPub.Utils require Logger @type source :: @@ -88,6 +89,7 @@ def store(upload, opts \\ []) do {:ok, url_spec} <- Pleroma.Uploaders.Uploader.put_file(opts.uploader, upload) do {:ok, %{ + "id" => Utils.generate_object_id(), "type" => opts.activity_type, "mediaType" => upload.content_type, "url" => [ diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 20acdf86e..dcdc7085f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -194,7 +194,16 @@ defp insert_activity_with_expiration(data, local, recipients) do def notify_and_stream(activity) do Notification.create_notifications(activity) - conversation = create_or_bump_conversation(activity, activity.actor) + original_activity = + case activity do + %{data: %{"type" => "Update"}, object: %{data: %{"id" => id}}} -> + Activity.get_create_by_object_ap_id_with_object(id) + + _ -> + activity + end + + conversation = create_or_bump_conversation(original_activity, original_activity.actor) participations = get_participations(conversation) stream_out(activity) stream_out_participations(participations) @@ -260,7 +269,7 @@ def stream_out_participations(_, _), do: :noop @impl true def stream_out(%Activity{data: %{"type" => data_type}} = activity) - when data_type in ["Create", "Announce", "Delete"] do + when data_type in ["Create", "Announce", "Delete", "Update"] do activity |> Topics.get_activity_topics() |> Streamer.stream(activity) diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index ba756ed64..6d39ad3a8 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -278,10 +278,16 @@ def like(actor, object) do end end - # Retricted to user updates for now, always public @spec update(User.t(), Object.t()) :: {:ok, map(), keyword()} def update(actor, object) do - to = [Pleroma.Constants.as_public(), actor.follower_address] + {to, cc} = + if object["type"] in Pleroma.Constants.actor_types() do + # User updates, always public + {[Pleroma.Constants.as_public(), actor.follower_address], []} + else + # Status updates, follow the recipients in the object + {object["to"] || [], object["cc"] || []} + end {:ok, %{ @@ -289,7 +295,8 @@ def update(actor, object) do "type" => "Update", "actor" => actor.ap_id, "object" => object, - "to" => to + "to" => to, + "cc" => cc }, []} end diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 20bce0d5f..4df226e80 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -63,10 +63,53 @@ defmodule Pleroma.Web.ActivityPub.MRF do @required_description_keys [:key, :related_policy] + def filter_one(policy, message) do + should_plug_history? = + if function_exported?(policy, :history_awareness, 0) do + policy.history_awareness() + else + :manual + end + |> Kernel.==(:auto) + + if not should_plug_history? do + policy.filter(message) + else + main_result = policy.filter(message) + + with {_, {:ok, main_message}} <- {:main, main_result}, + {_, + %{ + "formerRepresentations" => %{ + "orderedItems" => [_ | _] + } + }} = {_, object} <- {:object, message["object"]}, + {_, {:ok, new_history}} <- + {:history, + Pleroma.Object.Updater.for_each_history_item( + object["formerRepresentations"], + object, + fn item -> + with {:ok, filtered} <- policy.filter(Map.put(message, "object", item)) do + {:ok, filtered["object"]} + else + e -> e + end + end + )} do + {:ok, put_in(main_message, ["object", "formerRepresentations"], new_history)} + else + {:main, _} -> main_result + {:object, _} -> main_result + {:history, e} -> e + end + end + end + def filter(policies, %{} = message) do policies |> Enum.reduce({:ok, message}, fn - policy, {:ok, message} -> policy.filter(message) + policy, {:ok, message} -> filter_one(policy, message) _, error -> error end) end diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index cdf17fd28..ba7c8400b 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -9,6 +9,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do require Logger + @impl true + def history_awareness, do: :auto + # has the user successfully posted before? defp old_user?(%User{} = u) do u.note_count > 0 || u.follower_count > 0 diff --git a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex index fad8d873b..c438b8f70 100644 --- a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex +++ b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex @@ -10,6 +10,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do @reply_prefix Regex.compile!("^re:[[:space:]]*", [:caseless]) + def history_awareness, do: :auto + def filter_by_summary( %{data: %{"summary" => parent_summary}} = _in_reply_to, %{"summary" => child_summary} = child @@ -27,8 +29,8 @@ def filter_by_summary( def filter_by_summary(_in_reply_to, child), do: child - def filter(%{"type" => "Create", "object" => child_object} = object) - when is_map(child_object) do + def filter(%{"type" => type, "object" => child_object} = object) + when type in ["Create", "Update"] and is_map(child_object) do child = child_object["inReplyTo"] |> Object.normalize(fetch: false) diff --git a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex index b7db4fa3d..b5ad8b5b4 100644 --- a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex @@ -16,6 +16,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do @behaviour Pleroma.Web.ActivityPub.MRF.Policy + @impl true + def history_awareness, do: :manual + defp check_reject(message, hashtags) do if Enum.any?(Config.get([:mrf_hashtag, :reject]), fn match -> match in hashtags end) do {:reject, "[HashtagPolicy] Matches with rejected keyword"} @@ -47,22 +50,46 @@ defp check_ftl_removal(%{"to" => to} = message, hashtags) do defp check_ftl_removal(message, _hashtags), do: {:ok, message} - defp check_sensitive(message, hashtags) do - if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do - {:ok, Kernel.put_in(message, ["object", "sensitive"], true)} - else - {:ok, message} - end + defp check_sensitive(message) do + {:ok, new_object} = + Object.Updater.do_with_history(message["object"], fn object -> + hashtags = Object.hashtags(%Object{data: object}) + + if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do + {:ok, Map.put(object, "sensitive", true)} + else + {:ok, object} + end + end) + + {:ok, Map.put(message, "object", new_object)} end @impl true - def filter(%{"type" => "Create", "object" => object} = message) do - hashtags = Object.hashtags(%Object{data: object}) + def filter(%{"type" => type, "object" => object} = message) when type in ["Create", "Update"] do + history_items = + with %{"formerRepresentations" => %{"orderedItems" => items}} <- object do + items + else + _ -> [] + end + + historical_hashtags = + Enum.reduce(history_items, [], fn item, acc -> + acc ++ Object.hashtags(%Object{data: item}) + end) + + hashtags = Object.hashtags(%Object{data: object}) ++ historical_hashtags if hashtags != [] do with {:ok, message} <- check_reject(message, hashtags), - {:ok, message} <- check_ftl_removal(message, hashtags), - {:ok, message} <- check_sensitive(message, hashtags) do + {:ok, message} <- + (if "type" == "Create" do + check_ftl_removal(message, hashtags) + else + {:ok, message} + end), + {:ok, message} <- check_sensitive(message) do {:ok, message} end else diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index 1383fa757..7c921fc76 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -27,24 +27,46 @@ defp object_payload(%{} = object) do end defp check_reject(%{"object" => %{} = object} = message) do - payload = object_payload(object) + with {:ok, _new_object} <- + Pleroma.Object.Updater.do_with_history(object, fn object -> + payload = object_payload(object) - if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern -> - string_matches?(payload, pattern) - end) do - {:reject, "[KeywordPolicy] Matches with rejected keyword"} - else + if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern -> + string_matches?(payload, pattern) + end) do + {:reject, "[KeywordPolicy] Matches with rejected keyword"} + else + {:ok, message} + end + end) do {:ok, message} + else + e -> e end end - defp check_ftl_removal(%{"to" => to, "object" => %{} = object} = message) do - payload = object_payload(object) + defp check_ftl_removal(%{"type" => "Create", "to" => to, "object" => %{} = object} = message) do + check_keyword = fn object -> + payload = object_payload(object) - if Pleroma.Constants.as_public() in to and - Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern -> + if Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern -> string_matches?(payload, pattern) end) do + {:should_delist, nil} + else + {:ok, %{}} + end + end + + should_delist? = fn object -> + with {:ok, _} <- Pleroma.Object.Updater.do_with_history(object, check_keyword) do + false + else + _ -> true + end + end + + if Pleroma.Constants.as_public() in to and should_delist?.(object) do to = List.delete(to, Pleroma.Constants.as_public()) cc = [Pleroma.Constants.as_public() | message["cc"] || []] @@ -59,8 +81,12 @@ defp check_ftl_removal(%{"to" => to, "object" => %{} = object} = message) do end end + defp check_ftl_removal(message) do + {:ok, message} + end + defp check_replace(%{"object" => %{} = object} = message) do - object = + replace_kw = fn object -> ["content", "name", "summary"] |> Enum.filter(fn field -> Map.has_key?(object, field) && object[field] end) |> Enum.reduce(object, fn field, object -> @@ -73,6 +99,10 @@ defp check_replace(%{"object" => %{} = object} = message) do Map.put(object, field, data) end) + |> (fn object -> {:ok, object} end).() + end + + {:ok, object} = Pleroma.Object.Updater.do_with_history(object, replace_kw) message = Map.put(message, "object", object) @@ -80,7 +110,8 @@ defp check_replace(%{"object" => %{} = object} = message) do end @impl true - def filter(%{"type" => "Create", "object" => %{"content" => _content}} = message) do + def filter(%{"type" => type, "object" => %{"content" => _content}} = message) + when type in ["Create", "Update"] do with {:ok, message} <- check_reject(message), {:ok, message} <- check_ftl_removal(message), {:ok, message} <- check_replace(message) do diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index f60a76adf..72455afd0 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -15,6 +15,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do recv_timeout: 10_000 ] + @impl true + def history_awareness, do: :auto + defp prefetch(url) do # Fetching only proxiable resources if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do @@ -53,10 +56,8 @@ defp preload(%{"object" => %{"attachment" => attachments}} = _message) do end @impl true - def filter( - %{"type" => "Create", "object" => %{"attachment" => attachments} = _object} = message - ) - when is_list(attachments) and length(attachments) > 0 do + def filter(%{"type" => type, "object" => %{"attachment" => attachments} = _object} = message) + when type in ["Create", "Update"] and is_list(attachments) and length(attachments) > 0 do preload(message) {:ok, message} diff --git a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex index b2939a4d6..19637a38d 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do @impl true def filter(%{"actor" => actor} = object) do with true <- is_local?(actor), + true <- is_eligible_type?(object), true <- is_note?(object), false <- has_attachment?(object), true <- only_mentions?(object) do @@ -32,7 +33,6 @@ defp is_local?(actor) do end defp has_attachment?(%{ - "type" => "Create", "object" => %{"type" => "Note", "attachment" => attachments} }) when length(attachments) > 0, @@ -40,23 +40,13 @@ defp has_attachment?(%{ defp has_attachment?(_), do: false - defp only_mentions?(%{"type" => "Create", "object" => %{"type" => "Note", "source" => source}}) - when is_binary(source) do - non_mentions = - source |> String.split() |> Enum.filter(&(not String.starts_with?(&1, "@"))) |> length + defp only_mentions?(%{"object" => %{"type" => "Note", "source" => source}}) do + source = + case source do + %{"content" => text} -> text + _ -> source + end - if non_mentions > 0 do - false - else - true - end - end - - defp only_mentions?(%{ - "type" => "Create", - "object" => %{"type" => "Note", "source" => %{"content" => source}} - }) - when is_binary(source) do non_mentions = source |> String.split() |> Enum.filter(&(not String.starts_with?(&1, "@"))) |> length @@ -69,9 +59,12 @@ defp only_mentions?(%{ defp only_mentions?(_), do: false - defp is_note?(%{"type" => "Create", "object" => %{"type" => "Note"}}), do: true + defp is_note?(%{"object" => %{"type" => "Note"}}), do: true defp is_note?(_), do: false + defp is_eligible_type?(%{"type" => type}) when type in ["Create", "Update"], do: true + defp is_eligible_type?(_), do: false + @impl true def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index 90272766c..f25bb4efd 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -6,14 +6,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do @moduledoc "Ensure no content placeholder is present (such as the dot from mastodon)" @behaviour Pleroma.Web.ActivityPub.MRF.Policy + @impl true + def history_awareness, do: :auto + @impl true def filter( %{ - "type" => "Create", + "type" => type, "object" => %{"content" => content, "attachment" => _} = _child_object } = object ) - when content in [".", "

.

"] do + when type in ["Create", "Update"] and content in [".", "

.

"] do {:ok, put_in(object, ["object", "content"], "")} end diff --git a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex index 0d7146738..151c6ed20 100644 --- a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex +++ b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex @@ -9,7 +9,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do @behaviour Pleroma.Web.ActivityPub.MRF.Policy @impl true - def filter(%{"type" => "Create", "object" => child_object} = object) do + def history_awareness, do: :auto + + @impl true + def filter(%{"type" => type, "object" => child_object} = object) + when type in ["Create", "Update"] do scrub_policy = Pleroma.Config.get([:mrf_normalize_markup, :scrub_policy]) content = diff --git a/lib/pleroma/web/activity_pub/mrf/policy.ex b/lib/pleroma/web/activity_pub/mrf/policy.ex index a4a960c01..75209b2db 100644 --- a/lib/pleroma/web/activity_pub/mrf/policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/policy.ex @@ -12,5 +12,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.Policy do label: String.t(), description: String.t() } - @optional_callbacks config_description: 0 + @callback history_awareness() :: :auto | :manual + @optional_callbacks config_description: 0, history_awareness: 0 end diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 283cd884c..cb0cc9ed7 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -86,8 +86,8 @@ def validate( meta ) when objtype in ~w[Question Answer Audio Video Event Article Note Page] do - with {:ok, object_data} <- cast_and_apply(object), - meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), + with {:ok, object_data} <- cast_and_apply_and_stringify_with_history(object), + meta = Keyword.put(meta, :object_data, object_data), {:ok, create_activity} <- create_activity |> CreateGenericValidator.cast_and_validate(meta) @@ -111,19 +111,53 @@ def validate(%{"type" => type} = object, meta) end with {:ok, object} <- - object - |> validator.cast_and_validate() - |> Ecto.Changeset.apply_action(:insert) do - object = stringify_keys(object) + do_separate_with_history(object, fn object -> + with {:ok, object} <- + object + |> validator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) - # Insert copy of hashtags as strings for the non-hashtag table indexing - tag = (object["tag"] || []) ++ Object.hashtags(%Object{data: object}) - object = Map.put(object, "tag", tag) + # Insert copy of hashtags as strings for the non-hashtag table indexing + tag = (object["tag"] || []) ++ Object.hashtags(%Object{data: object}) + object = Map.put(object, "tag", tag) + {:ok, object} + end + end) do {:ok, object, meta} end end + def validate( + %{"type" => "Update", "object" => %{"type" => objtype} = object} = update_activity, + meta + ) + when objtype in ~w[Question Answer Audio Video Event Article Note Page] do + with {_, false} <- {:local, Access.get(meta, :local, false)}, + {_, {:ok, object_data, _}} <- {:object_validation, validate(object, meta)}, + meta = Keyword.put(meta, :object_data, object_data), + {:ok, update_activity} <- + update_activity + |> UpdateValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + update_activity = stringify_keys(update_activity) + {:ok, update_activity, meta} + else + {:local, _} -> + with {:ok, object} <- + update_activity + |> UpdateValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end + + {:object_validation, e} -> + e + end + end + def validate(%{"type" => type} = object, meta) when type in ~w[Accept Reject Follow Update Like EmojiReact Announce Answer] do @@ -160,6 +194,15 @@ def validate(%{"type" => type} = object, meta) when type in ~w(Add Remove) do def validate(o, m), do: {:error, {:validator_not_set, {o, m}}} + def cast_and_apply_and_stringify_with_history(object) do + do_separate_with_history(object, fn object -> + with {:ok, object_data} <- cast_and_apply(object), + object_data <- object_data |> stringify_keys() do + {:ok, object_data} + end + end) + end + def cast_and_apply(%{"type" => "Question"} = object) do QuestionValidator.cast_and_apply(object) end @@ -214,4 +257,54 @@ def fetch_actor_and_object(object) do Object.normalize(object["object"], fetch: true) :ok end + + defp for_each_history_item( + %{"type" => "OrderedCollection", "orderedItems" => items} = history, + object, + fun + ) do + processed_items = + Enum.map(items, fn item -> + with item <- Map.put(item, "id", object["id"]), + {:ok, item} <- fun.(item) do + item + else + _ -> nil + end + end) + + if Enum.all?(processed_items, &(not is_nil(&1))) do + {:ok, Map.put(history, "orderedItems", processed_items)} + else + {:error, :invalid_history} + end + end + + defp for_each_history_item(nil, _object, _fun) do + {:ok, nil} + end + + defp for_each_history_item(_, _object, _fun) do + {:error, :invalid_history} + end + + # fun is (object -> {:ok, validated_object_with_string_keys}) + defp do_separate_with_history(object, fun) do + with history <- object["formerRepresentations"], + object <- Map.drop(object, ["formerRepresentations"]), + {_, {:ok, object}} <- {:main_body, fun.(object)}, + {_, {:ok, history}} <- {:history_items, for_each_history_item(history, object, fun)} do + object = + if history do + Map.put(object, "formerRepresentations", history) + else + object + end + + {:ok, object} + else + {:main_body, e} -> e + {:history_items, e} -> e + end + end end diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index 55323bc2e..0d45421e2 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -53,7 +53,10 @@ defp fix_url(%{"url" => url} = data) when is_bitstring(url), do: data defp fix_url(%{"url" => url} = data) when is_map(url), do: Map.put(data, "url", url["href"]) defp fix_url(data), do: data - defp fix_tag(%{"tag" => tag} = data) when is_list(tag), do: data + defp fix_tag(%{"tag" => tag} = data) when is_list(tag) do + Map.put(data, "tag", Enum.filter(tag, &is_map/1)) + end + defp fix_tag(%{"tag" => tag} = data) when is_map(tag), do: Map.put(data, "tag", [tag]) defp fix_tag(data), do: Map.drop(data, ["tag"]) diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index ffdb16976..dba18a3d0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do @primary_key false embedded_schema do + field(:id, :string) field(:type, :string) field(:mediaType, :string, default: "application/octet-stream") field(:name, :string) @@ -43,7 +44,7 @@ def changeset(struct, data) do |> fix_url() struct - |> cast(data, [:type, :mediaType, :name, :blurhash]) + |> cast(data, [:id, :type, :mediaType, :name, :blurhash]) |> cast_embed(:url, with: &url_changeset/2, required: true) |> validate_inclusion(:type, ~w[Link Document Audio Image Video]) |> validate_required([:type, :mediaType]) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex index 49aba68af..db28c38ef 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex @@ -33,6 +33,7 @@ defmacro object_fields do field(:content, :string) field(:published, ObjectValidators.DateTime) + field(:updated, ObjectValidators.DateTime) field(:emoji, ObjectValidators.Emoji, default: %{}) embeds_many(:attachment, AttachmentValidator) end diff --git a/lib/pleroma/web/activity_pub/object_validators/update_validator.ex b/lib/pleroma/web/activity_pub/object_validators/update_validator.ex index a1fae47f5..2f0839c5b 100644 --- a/lib/pleroma/web/activity_pub/object_validators/update_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/update_validator.ex @@ -51,7 +51,9 @@ def validate_updating_rights(cng) do with actor = get_field(cng, :actor), object = get_field(cng, :object), {:ok, object_id} <- ObjectValidators.ObjectID.cast(object), - true <- actor == object_id do + actor_uri <- URI.parse(actor), + object_uri <- URI.parse(object_id), + true <- actor_uri.host == object_uri.host do cng else _e -> diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 439268470..43b1b089b 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -23,6 +23,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do alias Pleroma.Web.Streamer alias Pleroma.Workers.PollWorker + require Pleroma.Constants require Logger @logger Pleroma.Config.get([:side_effects, :logger], Logger) @@ -150,23 +151,26 @@ def handle( # Tasks this handles: # - Update the user + # - Update a non-user object (Note, Question, etc.) # # For a local user, we also get a changeset with the full information, so we # can update non-federating, non-activitypub settings as well. @impl true def handle(%{data: %{"type" => "Update", "object" => updated_object}} = object, meta) do - if changeset = Keyword.get(meta, :user_update_changeset) do - changeset - |> User.update_and_set_cache() + updated_object_id = updated_object["id"] + + with {_, true} <- {:has_id, is_binary(updated_object_id)}, + %{"type" => type} <- updated_object, + {_, is_user} <- {:is_user, type in Pleroma.Constants.actor_types()} do + if is_user do + handle_update_user(object, meta) + else + handle_update_object(object, meta) + end else - {:ok, new_user_data} = ActivityPub.user_data_from_user_object(updated_object) - - User.get_by_ap_id(updated_object["id"]) - |> User.remote_user_changeset(new_user_data) - |> User.update_and_set_cache() + _ -> + {:ok, object, meta} end - - {:ok, object, meta} end # Tasks this handles: @@ -395,6 +399,79 @@ def handle(object, meta) do {:ok, object, meta} end + defp handle_update_user( + %{data: %{"type" => "Update", "object" => updated_object}} = object, + meta + ) do + if changeset = Keyword.get(meta, :user_update_changeset) do + changeset + |> User.update_and_set_cache() + else + {:ok, new_user_data} = ActivityPub.user_data_from_user_object(updated_object) + + User.get_by_ap_id(updated_object["id"]) + |> User.remote_user_changeset(new_user_data) + |> User.update_and_set_cache() + end + + {:ok, object, meta} + end + + defp handle_update_object( + %{data: %{"type" => "Update", "object" => updated_object}} = object, + meta + ) do + orig_object_ap_id = updated_object["id"] + orig_object = Object.get_by_ap_id(orig_object_ap_id) + orig_object_data = orig_object.data + + updated_object = + if meta[:local] do + # If this is a local Update, we don't process it by transmogrifier, + # so we use the embedded object as-is. + updated_object + else + meta[:object_data] + end + + if orig_object_data["type"] in Pleroma.Constants.updatable_object_types() do + %{ + updated_data: updated_object_data, + updated: updated, + used_history_in_new_object?: used_history_in_new_object? + } = Object.Updater.make_new_object_data_from_update_object(orig_object_data, updated_object) + + changeset = + orig_object + |> Repo.preload(:hashtags) + |> Object.change(%{data: updated_object_data}) + + with {:ok, new_object} <- Repo.update(changeset), + {:ok, _} <- Object.invalid_object_cache(new_object), + {:ok, _} <- Object.set_cache(new_object), + # The metadata/utils.ex uses the object id for the cache. + {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do + if used_history_in_new_object? do + with create_activity when not is_nil(create_activity) <- + Pleroma.Activity.get_create_by_object_ap_id(orig_object_ap_id), + {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(create_activity.id) do + nil + else + _ -> nil + end + end + + if updated do + object + |> Activity.normalize() + |> ActivityPub.notify_and_stream() + end + end + end + + {:ok, object, meta} + end + def handle_object_creation(%{"type" => "Question"} = object, activity, meta) do with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do PollWorker.schedule_poll_end(activity) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 8ec4b0fec..b9d853610 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -699,6 +699,24 @@ def prepare_object(object) do |> strip_internal_fields |> strip_internal_tags |> set_type + |> maybe_process_history + end + + defp maybe_process_history(%{"formerRepresentations" => %{"orderedItems" => history}} = object) do + processed_history = + Enum.map( + history, + fn + item when is_map(item) -> prepare_object(item) + item -> item + end + ) + + put_in(object, ["formerRepresentations", "orderedItems"], processed_history) + end + + defp maybe_process_history(object) do + object end # @doc @@ -723,6 +741,21 @@ def prepare_outgoing(%{"type" => activity_type, "object" => object_id} = data) {:ok, data} end + def prepare_outgoing(%{"type" => "Update", "object" => %{"type" => objtype} = object} = data) + when objtype in Pleroma.Constants.updatable_object_types() do + object = + object + |> prepare_object + + data = + data + |> Map.put("object", object) + |> Map.merge(Utils.make_json_ld_header()) + |> Map.delete("bcc") + + {:ok, data} + end + def prepare_outgoing(%{"type" => "Announce", "actor" => ap_id, "object" => object_id} = data) do object = object_id diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 5332c9dca..65877cc64 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -6,9 +6,13 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do alias OpenApiSpex.Operation alias OpenApiSpex.Schema alias Pleroma.Web.ApiSpec.AccountOperation + alias Pleroma.Web.ApiSpec.Schemas.Account alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.Attachment alias Pleroma.Web.ApiSpec.Schemas.BooleanLike + alias Pleroma.Web.ApiSpec.Schemas.Emoji alias Pleroma.Web.ApiSpec.Schemas.FlakeID + alias Pleroma.Web.ApiSpec.Schemas.Poll alias Pleroma.Web.ApiSpec.Schemas.ScheduledStatus alias Pleroma.Web.ApiSpec.Schemas.Status alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope @@ -422,6 +426,59 @@ def translate_operation do } end + def show_history_operation do + %Operation{ + tags: ["Retrieve status history"], + summary: "Status history", + description: "View history of a status", + operationId: "StatusController.show_history", + security: [%{"oAuth" => ["read:statuses"]}], + parameters: [ + id_param() + ], + responses: %{ + 200 => status_history_response(), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } + end + + def show_source_operation do + %Operation{ + tags: ["Retrieve status source"], + summary: "Status source", + description: "View source of a status", + operationId: "StatusController.show_source", + security: [%{"oAuth" => ["read:statuses"]}], + parameters: [ + id_param() + ], + responses: %{ + 200 => status_source_response(), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } + end + + def update_operation do + %Operation{ + tags: ["Update status"], + summary: "Update status", + description: "Change the content of a status", + operationId: "StatusController.update", + security: [%{"oAuth" => ["write:statuses"]}], + parameters: [ + id_param() + ], + requestBody: request_body("Parameters", update_request(), required: true), + responses: %{ + 200 => status_response(), + 403 => Operation.response("Forbidden", "application/json", ApiError), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } + end + def array_of_statuses do %Schema{type: :array, items: Status, example: [Status.schema().example]} end @@ -530,6 +587,60 @@ defp create_request do } end + defp update_request do + %Schema{ + title: "StatusUpdateRequest", + type: :object, + properties: %{ + status: %Schema{ + type: :string, + nullable: true, + description: + "Text content of the status. If `media_ids` is provided, this becomes optional. Attaching a `poll` is optional while `status` is provided." + }, + media_ids: %Schema{ + nullable: true, + type: :array, + items: %Schema{type: :string}, + description: "Array of Attachment ids to be attached as media." + }, + poll: poll_params(), + sensitive: %Schema{ + allOf: [BooleanLike], + nullable: true, + description: "Mark status and attached media as sensitive?" + }, + spoiler_text: %Schema{ + type: :string, + nullable: true, + description: + "Text to be shown as a warning or subject before the actual content. Statuses are generally collapsed behind this field." + }, + content_type: %Schema{ + type: :string, + nullable: true, + description: + "The MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint." + }, + to: %Schema{ + type: :array, + nullable: true, + items: %Schema{type: :string}, + description: + "A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for for post visibility are not affected by this and will still apply" + } + }, + example: %{ + "status" => "What time is it?", + "sensitive" => "false", + "poll" => %{ + "options" => ["Cofe", "Adventure"], + "expires_in" => 420 + } + } + } + end + def poll_params do %Schema{ nullable: true, @@ -580,6 +691,87 @@ defp status_response do Operation.response("Status", "application/json", Status) end + defp status_history_response do + Operation.response( + "Status History", + "application/json", + %Schema{ + title: "Status history", + description: "Response schema for history of a status", + type: :array, + items: %Schema{ + type: :object, + properties: %{ + account: %Schema{ + allOf: [Account], + description: "The account that authored this status" + }, + content: %Schema{ + type: :string, + format: :html, + description: "HTML-encoded status content" + }, + sensitive: %Schema{ + type: :boolean, + description: "Is this status marked as sensitive content?" + }, + spoiler_text: %Schema{ + type: :string, + description: + "Subject or summary line, below which status content is collapsed until expanded" + }, + created_at: %Schema{ + type: :string, + format: "date-time", + description: "The date when this status was created" + }, + media_attachments: %Schema{ + type: :array, + items: Attachment, + description: "Media that is attached to this status" + }, + emojis: %Schema{ + type: :array, + items: Emoji, + description: "Custom emoji to be used when rendering status content" + }, + poll: %Schema{ + allOf: [Poll], + nullable: true, + description: "The poll attached to the status" + } + } + } + } + ) + end + + defp status_source_response do + Operation.response( + "Status Source", + "application/json", + %Schema{ + type: :object, + properties: %{ + id: FlakeID, + text: %Schema{ + type: :string, + description: "Raw source of status content" + }, + spoiler_text: %Schema{ + type: :string, + description: + "Subject or summary line, below which status content is collapsed until expanded" + }, + content_type: %Schema{ + type: :string, + description: "The content type of the source" + } + } + } + ) + end + defp context do %Schema{ title: "StatusContext", diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index c5d9119ef..a6df9be94 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -73,6 +73,12 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do format: "date-time", description: "The date when this status was created" }, + edited_at: %Schema{ + type: :string, + format: "date-time", + nullable: true, + description: "The date when this status was last edited" + }, emojis: %Schema{ type: :array, items: Emoji, diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 23d353dc2..f1f51acf5 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -347,6 +347,41 @@ def post(user, %{status: _} = data) do end end + def update(user, orig_activity, changes) do + with orig_object <- Object.normalize(orig_activity), + {:ok, new_object} <- make_update_data(user, orig_object, changes), + {:ok, update_data, _} <- Builder.update(user, new_object), + {:ok, update, _} <- Pipeline.common_pipeline(update_data, local: true) do + {:ok, update} + else + _ -> {:error, nil} + end + end + + defp make_update_data(user, orig_object, changes) do + kept_params = %{ + visibility: Visibility.get_visibility(orig_object), + in_reply_to_id: + with replied_id when is_binary(replied_id) <- orig_object.data["inReplyTo"], + %Activity{id: activity_id} <- Activity.get_create_by_object_ap_id(replied_id) do + activity_id + else + _ -> nil + end + } + + params = Map.merge(changes, kept_params) + + with {:ok, draft} <- ActivityDraft.create(user, params) do + change = + Object.Updater.make_update_object_data(orig_object.data, draft.object, Utils.make_date()) + + {:ok, change} + else + _ -> {:error, nil} + end + end + @spec pin(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, term()} def pin(id, %User{} = user) do with %Activity{} = activity <- create_activity_by_id(id), diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 767b2bf0f..b3a49de44 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -221,7 +221,7 @@ defp object(draft) do |> Map.put("emoji", emoji) |> Map.put("source", %{ "content" => draft.status, - "mediaType" => draft.params[:content_type] + "mediaType" => Utils.get_content_type(draft.params[:content_type]) }) |> Map.put("generator", draft.params[:generator]) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 15016eb47..bf03b0a82 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -37,7 +37,7 @@ def attachments_from_ids_no_descs([]), do: [] def attachments_from_ids_no_descs(ids) do Enum.map(ids, fn media_id -> - case Repo.get(Object, media_id) do + case get_attachment(media_id) do %Object{data: data} -> data _ -> nil end @@ -51,13 +51,17 @@ def attachments_from_ids_descs(ids, descs_str) do {_, descs} = Jason.decode(descs_str) Enum.map(ids, fn media_id -> - with %Object{data: data} <- Repo.get(Object, media_id) do + with %Object{data: data} <- get_attachment(media_id) do Map.put(data, "name", descs[media_id]) end end) |> Enum.reject(&is_nil/1) end + defp get_attachment(media_id) do + Repo.get(Object, media_id) + end + @spec get_to_and_cc(ActivityDraft.t()) :: {list(String.t()), list(String.t())} def get_to_and_cc(%{in_reply_to_conversation: %Participation{} = participation}) do @@ -219,7 +223,7 @@ def make_content_html(%ActivityDraft{} = draft) do |> maybe_add_attachments(draft.attachments, attachment_links) end - defp get_content_type(content_type) do + def get_content_type(content_type) do if Enum.member?(Config.get([:instance, :allowed_post_formats]), content_type) do content_type else diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index ae4432e85..8e6cf2a6a 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -51,6 +51,7 @@ def index(conn, %{account_id: account_id} = params) do move pleroma:emoji_reaction poll + update } def index(%{assigns: %{user: user}} = conn, params) do params = diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 41fbd7acf..31f3b3a8d 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -40,7 +40,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do :index, :show, :context, - :translate + :translate, + :show_history, + :show_source ] ) @@ -51,7 +53,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do :create, :delete, :reblog, - :unreblog + :unreblog, + :update ] ) @@ -193,6 +196,59 @@ def create(%{assigns: %{user: _user}, body_params: %{media_ids: _} = params} = c create(%Plug.Conn{conn | body_params: params}, %{}) end + @doc "GET /api/v1/statuses/:id/history" + def show_history(%{assigns: assigns} = conn, %{id: id} = params) do + with user = assigns[:user], + %Activity{} = activity <- Activity.get_by_id_with_object(id), + true <- Visibility.visible_for_user?(activity, user) do + try_render(conn, "history.json", + activity: activity, + for: user, + with_direct_conversation_id: true, + with_muted: Map.get(params, :with_muted, false) + ) + else + _ -> {:error, :not_found} + end + end + + @doc "GET /api/v1/statuses/:id/source" + def show_source(%{assigns: assigns} = conn, %{id: id} = _params) do + with user = assigns[:user], + %Activity{} = activity <- Activity.get_by_id_with_object(id), + true <- Visibility.visible_for_user?(activity, user) do + try_render(conn, "source.json", + activity: activity, + for: user + ) + else + _ -> {:error, :not_found} + end + end + + @doc "PUT /api/v1/statuses/:id" + def update(%{assigns: %{user: user}, body_params: body_params} = conn, %{id: id} = params) do + with {_, %Activity{}} = {_, activity} <- {:activity, Activity.get_by_id_with_object(id)}, + {_, true} <- {:visible, Visibility.visible_for_user?(activity, user)}, + {_, true} <- {:is_create, activity.data["type"] == "Create"}, + actor <- Activity.user_actor(activity), + {_, true} <- {:own_status, actor.id == user.id}, + changes <- body_params |> put_application(conn), + {_, {:ok, _update_activity}} <- {:pipeline, CommonAPI.update(user, activity, changes)}, + {_, %Activity{}} = {_, activity} <- {:refetched, Activity.get_by_id_with_object(id)} do + try_render(conn, "show.json", + activity: activity, + for: user, + with_direct_conversation_id: true, + with_muted: Map.get(params, :with_muted, false) + ) + else + {:own_status, _} -> {:error, :forbidden} + {:pipeline, _} -> {:error, :internal_server_error} + _ -> {:error, :not_found} + end + end + @doc "GET /api/v1/statuses/:id" def show(%{assigns: %{user: user}} = conn, %{id: id} = params) do with %Activity{} = activity <- Activity.get_by_id_with_object(id), diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 436519439..6612a7ec1 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -65,6 +65,7 @@ def features do "shareable_emoji_packs", "multifetch", "pleroma:api/v1/notifications:include_types_filter", + "editing", if Config.get([:media_proxy, :enabled]) do "media_proxy" end, diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 83914a275..463d31d1a 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -17,7 +17,11 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do alias Pleroma.Web.MastodonAPI.NotificationView alias Pleroma.Web.MastodonAPI.StatusView - @parent_types ~w{Like Announce EmojiReact} + defp object_id_for(%{data: %{"object" => %{"id" => id}}}) when is_binary(id), do: id + + defp object_id_for(%{data: %{"object" => id}}) when is_binary(id), do: id + + @parent_types ~w{Like Announce EmojiReact Update} def render("index.json", %{notifications: notifications, for: reading_user} = opts) do activities = Enum.map(notifications, & &1.activity) @@ -28,7 +32,7 @@ def render("index.json", %{notifications: notifications, for: reading_user} = op %{data: %{"type" => type}} -> type in @parent_types end) - |> Enum.map(& &1.data["object"]) + |> Enum.map(&object_id_for/1) |> Activity.create_by_object_ap_id() |> Activity.with_preloaded_object(:left) |> Pleroma.Repo.all() @@ -76,9 +80,9 @@ def render( parent_activity_fn = fn -> if opts[:parent_activities] do - Activity.Queries.find_by_object_ap_id(opts[:parent_activities], activity.data["object"]) + Activity.Queries.find_by_object_ap_id(opts[:parent_activities], object_id_for(activity)) else - Activity.get_create_by_object_ap_id(activity.data["object"]) + Activity.get_create_by_object_ap_id(object_id_for(activity)) end end @@ -107,6 +111,9 @@ def render( "reblog" -> put_status(response, parent_activity_fn.(), reading_user, status_render_opts) + "update" -> + put_status(response, parent_activity_fn.(), reading_user, status_render_opts) + "move" -> put_target(response, activity, reading_user, %{}) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 0d2571ab8..b3a35526e 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -265,10 +265,30 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} created_at = Utils.to_masto_date(object.data["published"]) + edited_at = + with %{"updated" => updated} <- object.data, + date <- Utils.to_masto_date(updated), + true <- date != "" do + date + else + _ -> + nil + end + reply_to = get_reply_to(activity, opts) reply_to_user = reply_to && CommonAPI.get_user(reply_to.data["actor"]) + history_len = + 1 + + (Object.Updater.history_for(object.data) + |> Map.get("orderedItems") + |> length()) + + # See render("history.json", ...) for more details + # Here the implicit index of the current content is 0 + chrono_order = history_len - 1 + content = object |> render_content() @@ -278,14 +298,14 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} |> Activity.HTML.get_cached_scrubbed_html_for_activity( User.html_filter_policy(opts[:for]), activity, - "mastoapi:content" + "mastoapi:content:#{chrono_order}" ) content_plaintext = content |> Activity.HTML.get_cached_stripped_html_for_activity( activity, - "mastoapi:content" + "mastoapi:content:#{chrono_order}" ) summary = object.data["summary"] || "" @@ -353,8 +373,9 @@ def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} reblog: nil, card: card, content: content_html, - text: opts[:with_source] && object.data["source"], + text: opts[:with_source] && get_source_text(object.data["source"]), created_at: created_at, + edited_at: edited_at, reblogs_count: announcement_count, replies_count: object.data["repliesCount"] || 0, favourites_count: like_count, @@ -400,6 +421,100 @@ def render("show.json", _) do nil end + def render("history.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do + object = Object.normalize(activity, fetch: false) + + hashtags = Object.hashtags(object) + + user = CommonAPI.get_user(activity.data["actor"]) + + past_history = + Object.Updater.history_for(object.data) + |> Map.get("orderedItems") + |> Enum.map(&Map.put(&1, "id", object.data["id"])) + |> Enum.map(&%Object{data: &1, id: object.id}) + + history = + [object | past_history] + # Mastodon expects the original to be at the first + |> Enum.reverse() + |> Enum.with_index() + |> Enum.map(fn {object, chrono_order} -> + %{ + # The history is prepended every time there is a new edit. + # In chrono_order, the oldest item is always at 0, and so on. + # The chrono_order is an invariant kept between edits. + chrono_order: chrono_order, + object: object + } + end) + + individual_opts = + opts + |> Map.put(:as, :item) + |> Map.put(:user, user) + |> Map.put(:hashtags, hashtags) + + render_many(history, StatusView, "history_item.json", individual_opts) + end + + def render( + "history_item.json", + %{ + activity: activity, + user: user, + item: %{object: object, chrono_order: chrono_order}, + hashtags: hashtags + } = opts + ) do + sensitive = object.data["sensitive"] || Enum.member?(hashtags, "nsfw") + + attachment_data = object.data["attachment"] || [] + attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment) + + created_at = Utils.to_masto_date(object.data["updated"] || object.data["published"]) + + content = + object + |> render_content() + + content_html = + content + |> Activity.HTML.get_cached_scrubbed_html_for_activity( + User.html_filter_policy(opts[:for]), + activity, + "mastoapi:content:#{chrono_order}" + ) + + summary = object.data["summary"] || "" + + %{ + account: + AccountView.render("show.json", %{ + user: user, + for: opts[:for] + }), + content: content_html, + sensitive: sensitive, + spoiler_text: summary, + created_at: created_at, + media_attachments: attachments, + emojis: build_emojis(object.data["emoji"]), + poll: render(PollView, "show.json", object: object, for: opts[:for]) + } + end + + def render("source.json", %{activity: %{data: %{"object" => _object}} = activity} = _opts) do + object = Object.normalize(activity, fetch: false) + + %{ + id: activity.id, + text: get_source_text(Map.get(object.data, "source", "")), + spoiler_text: Map.get(object.data, "summary", ""), + content_type: get_source_content_type(object.data["source"]) + } + end + def render("card.json", %{rich_media: rich_media, page_url: page_url}) do page_url_data = URI.parse(page_url) @@ -452,10 +567,19 @@ def render("attachment.json", %{attachment: attachment}) do true -> "unknown" end - <> = :crypto.hash(:md5, href) + attachment_id = + with {_, ap_id} when is_binary(ap_id) <- {:ap_id, attachment["id"]}, + {_, %Object{data: _object_data, id: object_id}} <- + {:object, Object.get_by_ap_id(ap_id)} do + to_string(object_id) + else + _ -> + <> = :crypto.hash(:md5, href) + to_string(attachment["id"] || hash_id) + end %{ - id: to_string(attachment["id"] || hash_id), + id: attachment_id, url: href, remote_url: href, preview_url: href_preview, @@ -638,4 +762,24 @@ defp maybe_render_quote(quote, opts) do _ -> nil end end + + defp get_source_text(%{"content" => content} = _source) do + content + end + + defp get_source_text(source) when is_binary(source) do + source + end + + defp get_source_text(_) do + "" + end + + defp get_source_content_type(%{"mediaType" => type} = _source) do + type + end + + defp get_source_content_type(_source) do + Utils.get_content_type(nil) + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 175b1c4c0..cc63b2b04 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -547,6 +547,7 @@ defmodule Pleroma.Web.Router do get("/bookmarks", StatusController, :bookmarks) post("/statuses", StatusController, :create) + put("/statuses/:id", StatusController, :update) delete("/statuses/:id", StatusController, :delete) post("/statuses/:id/reblog", StatusController, :reblog) post("/statuses/:id/unreblog", StatusController, :unreblog) @@ -612,6 +613,8 @@ defmodule Pleroma.Web.Router do get("/statuses/:id/context", StatusController, :context) get("/statuses/:id/favourited_by", StatusController, :favourited_by) get("/statuses/:id/reblogged_by", StatusController, :reblogged_by) + get("/statuses/:id/history", StatusController, :show_history) + get("/statuses/:id/source", StatusController, :show_source) get("/custom_emojis", CustomEmojiController, :index) diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index fba5d1c02..c03e7fc30 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -287,6 +287,27 @@ defp push_to_socket(topic, %Activity{ defp push_to_socket(_topic, %Activity{data: %{"type" => "Delete"}}), do: :noop + defp push_to_socket(topic, %Activity{data: %{"type" => "Update"}} = item) do + create_activity = + Pleroma.Activity.get_create_by_object_ap_id(item.object.data["id"]) + |> Map.put(:object, item.object) + + anon_render = StreamerView.render("status_update.json", create_activity, topic) + + Registry.dispatch(@registry, topic, fn list -> + Enum.each(list, fn {pid, auth?} -> + if auth? do + send( + pid, + {:render_with_user, StreamerView, "status_update.json", create_activity, topic} + ) + else + send(pid, {:text, anon_render}) + end + end) + end) + end + defp push_to_socket(topic, item) do anon_render = StreamerView.render("update.json", item, topic) diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index f455f941e..eba3d96ec 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -26,6 +26,23 @@ def render("update.json", %Activity{} = activity, %User{} = user, topic) do |> Jason.encode!() end + def render("status_update.json", %Activity{} = activity, %User{} = user, topic) do + activity = Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"]) + + %{ + stream: [topic], + event: "status.update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "show.json", + activity: activity, + for: user + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + def render("notification.json", %Notification{} = notify, %User{} = user, topic) do %{ stream: [topic], @@ -54,6 +71,22 @@ def render("update.json", %Activity{} = activity, topic) do |> Jason.encode!() end + def render("status_update.json", %Activity{} = activity, topic) do + activity = Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"]) + + %{ + stream: [topic], + event: "status.update", + payload: + Pleroma.Web.MastodonAPI.StatusView.render( + "show.json", + activity: activity + ) + |> Jason.encode!() + } + |> Jason.encode!() + end + def render("follow_relationships_update.json", item, topic) do %{ stream: [topic], diff --git a/priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs b/priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs new file mode 100644 index 000000000..0656c885f --- /dev/null +++ b/priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs @@ -0,0 +1,51 @@ +defmodule Pleroma.Repo.Migrations.AddUpdateToNotificationsEnum do + use Ecto.Migration + + @disable_ddl_transaction true + + def up do + """ + alter type notification_type add value 'update' + """ + |> execute() + end + + # 20210717000000_add_poll_to_notifications_enum.exs + def down do + alter table(:notifications) do + modify(:type, :string) + end + + """ + delete from notifications where type = 'update' + """ + |> execute() + + """ + drop type if exists notification_type + """ + |> execute() + + """ + create type notification_type as enum ( + 'follow', + 'follow_request', + 'mention', + 'move', + 'pleroma:emoji_reaction', + 'pleroma:chat_mention', + 'reblog', + 'favourite', + 'pleroma:report', + 'poll' + ) + """ + |> execute() + + """ + alter table notifications + alter column type type notification_type using (type::notification_type) + """ + |> execute() + end +end diff --git a/priv/static/schemas/litepub-0.1.jsonld b/priv/static/schemas/litepub-0.1.jsonld index d2b62ba77..f582ed42c 100644 --- a/priv/static/schemas/litepub-0.1.jsonld +++ b/priv/static/schemas/litepub-0.1.jsonld @@ -39,7 +39,9 @@ "alsoKnownAs": { "@id": "as:alsoKnownAs", "@type": "@id" - } + }, + "vcard": "http://www.w3.org/2006/vcard/ns#", + "formerRepresentations": "litepub:formerRepresentations" } ] } diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs index 8db208878..68330465b 100644 --- a/test/pleroma/notification_test.exs +++ b/test/pleroma/notification_test.exs @@ -127,6 +127,28 @@ test "does not create a notification for subscribed users if status is a reply" subscriber_notifications = Notification.for_user(subscriber) assert Enum.empty?(subscriber_notifications) end + + test "it sends edited notifications to those who repeated a status" do + user = insert(:user) + repeated_user = insert(:user) + other_user = insert(:user) + + {:ok, activity_one} = + CommonAPI.post(user, %{ + status: "hey @#{other_user.nickname}!" + }) + + {:ok, _activity_two} = CommonAPI.repeat(activity_one.id, repeated_user) + + {:ok, _edit_activity} = + CommonAPI.update(user, activity_one, %{ + status: "hey @#{other_user.nickname}! mew mew" + }) + + assert [%{type: "reblog"}] = Notification.for_user(user) + assert [%{type: "update"}] = Notification.for_user(repeated_user) + assert [%{type: "mention"}] = Notification.for_user(other_user) + end end test "create_poll_notifications/1" do @@ -838,6 +860,30 @@ test "it returns following domain-blocking recipient in enabled recipients list" assert [other_user] == enabled_receivers assert [] == disabled_receivers end + + test "it sends edited notifications to those who repeated a status" do + user = insert(:user) + repeated_user = insert(:user) + other_user = insert(:user) + + {:ok, activity_one} = + CommonAPI.post(user, %{ + status: "hey @#{other_user.nickname}!" + }) + + {:ok, _activity_two} = CommonAPI.repeat(activity_one.id, repeated_user) + + {:ok, edit_activity} = + CommonAPI.update(user, activity_one, %{ + status: "hey @#{other_user.nickname}! mew mew" + }) + + {enabled_receivers, _disabled_receivers} = + Notification.get_notified_from_activity(edit_activity) + + assert repeated_user in enabled_receivers + assert other_user not in enabled_receivers + end end describe "notification lifecycle" do diff --git a/test/pleroma/object/fetcher_test.exs b/test/pleroma/object/fetcher_test.exs index bd0a6e497..c321032ad 100644 --- a/test/pleroma/object/fetcher_test.exs +++ b/test/pleroma/object/fetcher_test.exs @@ -269,4 +269,271 @@ test "it can refetch pruned objects" do refute called(Pleroma.Signature.sign(:_, :_)) end end + + describe "refetching" do + setup do + object1 = %{ + "id" => "https://mastodon.social/1", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "type" => "Note", + "content" => "test 1", + "bcc" => [], + "bto" => [], + "cc" => [], + "to" => [], + "summary" => "" + } + + object2 = %{ + "id" => "https://mastodon.social/2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "type" => "Note", + "content" => "test 2", + "bcc" => [], + "bto" => [], + "cc" => [], + "to" => [], + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "type" => "Note", + "content" => "orig 2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "bcc" => [], + "bto" => [], + "cc" => [], + "to" => [], + "summary" => "" + } + ], + "totalItems" => 1 + } + } + + mock(fn + %{ + method: :get, + url: "https://mastodon.social/1" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: Jason.encode!(object1) + } + + %{ + method: :get, + url: "https://mastodon.social/2" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: Jason.encode!(object2) + } + + %{ + method: :get, + url: "https://mastodon.social/users/emelie/collections/featured" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: + Jason.encode!(%{ + "id" => "https://mastodon.social/users/emelie/collections/featured", + "type" => "OrderedCollection", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "orderedItems" => [], + "totalItems" => 0 + }) + } + + env -> + apply(HttpRequestMock, :request, [env]) + end) + + %{object1: object1, object2: object2} + end + + test "it keeps formerRepresentations if remote does not have this attr", %{object1: object1} do + full_object1 = + object1 + |> Map.merge(%{ + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "type" => "Note", + "content" => "orig 2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "bcc" => [], + "bto" => [], + "cc" => [], + "to" => [], + "summary" => "" + } + ], + "totalItems" => 1 + } + }) + + {:ok, o} = Object.create(full_object1) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{"formerRepresentations" => %{"orderedItems" => [%{"content" => "orig 2"}]}} = + refetched.data + end + + test "it uses formerRepresentations from remote if possible", %{object2: object2} do + {:ok, o} = Object.create(object2) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{"formerRepresentations" => %{"orderedItems" => [%{"content" => "orig 2"}]}} = + refetched.data + end + + test "it replaces formerRepresentations with the one from remote", %{object2: object2} do + full_object2 = + object2 + |> Map.merge(%{ + "content" => "mew mew #def", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{"type" => "Note", "content" => "mew mew 2"} + ], + "totalItems" => 1 + } + }) + + {:ok, o} = Object.create(full_object2) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{ + "content" => "test 2", + "formerRepresentations" => %{"orderedItems" => [%{"content" => "orig 2"}]} + } = refetched.data + end + + test "it adds to formerRepresentations if the remote does not have one and the object has changed", + %{object1: object1} do + full_object1 = + object1 + |> Map.merge(%{ + "content" => "mew mew #def", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{"type" => "Note", "content" => "mew mew 1"} + ], + "totalItems" => 1 + } + }) + + {:ok, o} = Object.create(full_object1) + + assert {:ok, refetched} = Fetcher.refetch_object(o) + + assert %{ + "content" => "test 1", + "formerRepresentations" => %{ + "orderedItems" => [ + %{"content" => "mew mew #def"}, + %{"content" => "mew mew 1"} + ], + "totalItems" => 2 + } + } = refetched.data + end + end + + describe "fetch with history" do + setup do + object2 = %{ + "id" => "https://mastodon.social/2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "type" => "Note", + "content" => "test 2", + "bcc" => [], + "bto" => [], + "cc" => ["https://mastodon.social/users/emelie/followers"], + "to" => [], + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "type" => "Note", + "content" => "orig 2", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "bcc" => [], + "bto" => [], + "cc" => ["https://mastodon.social/users/emelie/followers"], + "to" => [], + "summary" => "" + } + ], + "totalItems" => 1 + } + } + + mock(fn + %{ + method: :get, + url: "https://mastodon.social/2" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: Jason.encode!(object2) + } + + %{ + method: :get, + url: "https://mastodon.social/users/emelie/collections/featured" + } -> + %Tesla.Env{ + status: 200, + headers: [{"content-type", "application/activity+json"}], + body: + Jason.encode!(%{ + "id" => "https://mastodon.social/users/emelie/collections/featured", + "type" => "OrderedCollection", + "actor" => "https://mastodon.social/users/emelie", + "attributedTo" => "https://mastodon.social/users/emelie", + "orderedItems" => [], + "totalItems" => 0 + }) + } + + env -> + apply(HttpRequestMock, :request, [env]) + end) + + %{object2: object2} + end + + test "it gets history", %{object2: object2} do + {:ok, object} = Fetcher.fetch_object_from_id(object2["id"]) + + assert %{ + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [%{}] + } + } = object.data + end + end end diff --git a/test/pleroma/object/updater_test.exs b/test/pleroma/object/updater_test.exs new file mode 100644 index 000000000..7e9b44823 --- /dev/null +++ b/test/pleroma/object/updater_test.exs @@ -0,0 +1,76 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Object.UpdaterTest do + use Pleroma.DataCase + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + + alias Pleroma.Object.Updater + + describe "make_update_object_data/3" do + setup do + note = insert(:note) + %{original_data: note.data} + end + + test "it makes an updated field", %{original_data: original_data} do + new_data = Map.put(original_data, "content", "new content") + + date = Pleroma.Web.ActivityPub.Utils.make_date() + update_object_data = Updater.make_update_object_data(original_data, new_data, date) + assert %{"updated" => ^date} = update_object_data + end + + test "it creates formerRepresentations", %{original_data: original_data} do + new_data = Map.put(original_data, "content", "new content") + + date = Pleroma.Web.ActivityPub.Utils.make_date() + update_object_data = Updater.make_update_object_data(original_data, new_data, date) + + history_item = original_data |> Map.drop(["id", "formerRepresentations"]) + + assert %{ + "formerRepresentations" => %{ + "totalItems" => 1, + "orderedItems" => [^history_item] + } + } = update_object_data + end + end + + describe "make_new_object_data_from_update_object/2" do + test "it reuses formerRepresentations if it exists" do + %{data: original_data} = insert(:note) + + new_data = + original_data + |> Map.put("content", "edited") + + date = Pleroma.Web.ActivityPub.Utils.make_date() + update_object_data = Updater.make_update_object_data(original_data, new_data, date) + + history = update_object_data["formerRepresentations"]["orderedItems"] + + update_object_data = + update_object_data + |> put_in( + ["formerRepresentations", "orderedItems"], + history ++ [Map.put(original_data, "summary", "additional summary")] + ) + |> put_in(["formerRepresentations", "totalItems"], length(history) + 1) + + %{ + updated_data: updated_data, + updated: updated, + used_history_in_new_object?: used_history_in_new_object? + } = Updater.make_new_object_data_from_update_object(original_data, update_object_data) + + assert updated + assert used_history_in_new_object? + assert updated_data["formerRepresentations"] == update_object_data["formerRepresentations"] + end + end +end diff --git a/test/pleroma/upload_test.exs b/test/pleroma/upload_test.exs index f1ab82a57..8f242630f 100644 --- a/test/pleroma/upload_test.exs +++ b/test/pleroma/upload_test.exs @@ -49,20 +49,22 @@ def put_file(upload), do: TestUploaderBase.put_file(upload, __MODULE__) test "it returns file" do File.cp!("test/fixtures/image.jpg", "test/fixtures/image_tmp.jpg") - assert Upload.store(@upload_file) == - {:ok, - %{ - "name" => "image.jpg", - "type" => "Document", - "mediaType" => "image/jpeg", - "url" => [ - %{ - "href" => "http://localhost:4001/media/post-process-file.jpg", - "mediaType" => "image/jpeg", - "type" => "Link" - } - ] - }} + assert {:ok, result} = Upload.store(@upload_file) + + assert result == + %{ + "id" => result["id"], + "name" => "image.jpg", + "type" => "Document", + "mediaType" => "image/jpeg", + "url" => [ + %{ + "href" => "http://localhost:4001/media/post-process-file.jpg", + "mediaType" => "image/jpeg", + "type" => "Link" + } + ] + } Task.await(Agent.get(TestUploaderSuccess, fn task_pid -> task_pid end)) end diff --git a/test/pleroma/web/activity_pub/mrf/anti_link_spam_policy_test.exs b/test/pleroma/web/activity_pub/mrf/anti_link_spam_policy_test.exs index 5b990451c..c3ee03a05 100644 --- a/test/pleroma/web/activity_pub/mrf/anti_link_spam_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/anti_link_spam_policy_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicyTest do import Pleroma.Factory import ExUnit.CaptureLog + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy @linkless_message %{ @@ -49,11 +50,23 @@ test "it disallows posts with links" do assert user.note_count == 0 - message = - @linkful_message - |> Map.put("actor", user.ap_id) + message = %{ + "type" => "Create", + "actor" => user.ap_id, + "object" => %{ + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "content" => "hi world!" + } + ] + }, + "content" => "mew" + } + } - {:reject, _} = AntiLinkSpamPolicy.filter(message) + {:reject, _} = MRF.filter_one(AntiLinkSpamPolicy, message) end test "it allows posts with links for local users" do @@ -67,6 +80,18 @@ test "it allows posts with links for local users" do {:ok, _message} = AntiLinkSpamPolicy.filter(message) end + + test "it disallows posts with links in history" do + user = insert(:user, local: false) + + assert user.note_count == 0 + + message = + @linkful_message + |> Map.put("actor", user.ap_id) + + {:reject, _} = AntiLinkSpamPolicy.filter(message) + end end describe "with old user" do diff --git a/test/pleroma/web/activity_pub/mrf/ensure_re_prepended_test.exs b/test/pleroma/web/activity_pub/mrf/ensure_re_prepended_test.exs index 89439b65f..e174a83f7 100644 --- a/test/pleroma/web/activity_pub/mrf/ensure_re_prepended_test.exs +++ b/test/pleroma/web/activity_pub/mrf/ensure_re_prepended_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrependedTest do alias Pleroma.Activity alias Pleroma.Object + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.EnsureRePrepended describe "rewrites summary" do @@ -35,10 +36,58 @@ test "it adds `re:` to summary object when child summary containts re-subject of assert {:ok, res} = EnsureRePrepended.filter(message) assert res["object"]["summary"] == "re: object-summary" end + + test "it adds `re:` to history" do + message = %{ + "type" => "Create", + "object" => %{ + "summary" => "object-summary", + "inReplyTo" => %Activity{object: %Object{data: %{"summary" => "object-summary"}}}, + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "summary" => "object-summary", + "inReplyTo" => %Activity{object: %Object{data: %{"summary" => "object-summary"}}} + } + ] + } + } + } + + assert {:ok, res} = MRF.filter_one(EnsureRePrepended, message) + assert res["object"]["summary"] == "re: object-summary" + + assert Enum.at(res["object"]["formerRepresentations"]["orderedItems"], 0)["summary"] == + "re: object-summary" + end + + test "it accepts Updates" do + message = %{ + "type" => "Update", + "object" => %{ + "summary" => "object-summary", + "inReplyTo" => %Activity{object: %Object{data: %{"summary" => "object-summary"}}}, + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "summary" => "object-summary", + "inReplyTo" => %Activity{object: %Object{data: %{"summary" => "object-summary"}}} + } + ] + } + } + } + + assert {:ok, res} = MRF.filter_one(EnsureRePrepended, message) + assert res["object"]["summary"] == "re: object-summary" + + assert Enum.at(res["object"]["formerRepresentations"]["orderedItems"], 0)["summary"] == + "re: object-summary" + end end describe "skip filter" do - test "it skip if type isn't 'Create'" do + test "it skip if type isn't 'Create' or 'Update'" do message = %{ "type" => "Annotation", "object" => %{"summary" => "object-summary"} diff --git a/test/pleroma/web/activity_pub/mrf/hashtag_policy_test.exs b/test/pleroma/web/activity_pub/mrf/hashtag_policy_test.exs index 13415bb79..b88090869 100644 --- a/test/pleroma/web/activity_pub/mrf/hashtag_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/hashtag_policy_test.exs @@ -20,6 +20,76 @@ test "it sets the sensitive property with relevant hashtags" do assert modified["object"]["sensitive"] end + test "it is history-aware" do + activity = %{ + "type" => "Create", + "object" => %{ + "content" => "hey", + "tag" => [] + } + } + + activity_data = + activity + |> put_in( + ["object", "formerRepresentations"], + %{ + "type" => "OrderedCollection", + "orderedItems" => [ + Map.put( + activity["object"], + "tag", + [%{"type" => "Hashtag", "name" => "#nsfw"}] + ) + ] + } + ) + + {:ok, modified} = + Pleroma.Web.ActivityPub.MRF.filter_one( + Pleroma.Web.ActivityPub.MRF.HashtagPolicy, + activity_data + ) + + refute modified["object"]["sensitive"] + assert Enum.at(modified["object"]["formerRepresentations"]["orderedItems"], 0)["sensitive"] + end + + test "it works with Update" do + activity = %{ + "type" => "Update", + "object" => %{ + "content" => "hey", + "tag" => [] + } + } + + activity_data = + activity + |> put_in( + ["object", "formerRepresentations"], + %{ + "type" => "OrderedCollection", + "orderedItems" => [ + Map.put( + activity["object"], + "tag", + [%{"type" => "Hashtag", "name" => "#nsfw"}] + ) + ] + } + ) + + {:ok, modified} = + Pleroma.Web.ActivityPub.MRF.filter_one( + Pleroma.Web.ActivityPub.MRF.HashtagPolicy, + activity_data + ) + + refute modified["object"]["sensitive"] + assert Enum.at(modified["object"]["formerRepresentations"]["orderedItems"], 0)["sensitive"] + end + test "it doesn't sets the sensitive property with irrelevant hashtags" do user = insert(:user) diff --git a/test/pleroma/web/activity_pub/mrf/keyword_policy_test.exs b/test/pleroma/web/activity_pub/mrf/keyword_policy_test.exs index 8af4c5efa..9bc8c8355 100644 --- a/test/pleroma/web/activity_pub/mrf/keyword_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/keyword_policy_test.exs @@ -79,6 +79,54 @@ test "rejects if regex matches in summary" do KeywordPolicy.filter(message) end) end + + test "rejects if string matches in history" do + clear_config([:mrf_keyword, :reject], ["pun"]) + + message = %{ + "type" => "Create", + "object" => %{ + "content" => "just a daily reminder that compLAINer is a good", + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "content" => "just a daily reminder that compLAINer is a good pun", + "summary" => "" + } + ] + } + } + } + + assert {:reject, "[KeywordPolicy] Matches with rejected keyword"} = + KeywordPolicy.filter(message) + end + + test "rejects Updates" do + clear_config([:mrf_keyword, :reject], ["pun"]) + + message = %{ + "type" => "Update", + "object" => %{ + "content" => "just a daily reminder that compLAINer is a good", + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "content" => "just a daily reminder that compLAINer is a good pun", + "summary" => "" + } + ] + } + } + } + + assert {:reject, "[KeywordPolicy] Matches with rejected keyword"} = + KeywordPolicy.filter(message) + end end describe "delisting from ftl based on keywords" do @@ -157,6 +205,31 @@ test "delists if regex matches in summary" do not (["https://www.w3.org/ns/activitystreams#Public"] == result["to"]) end) end + + test "delists if string matches in history" do + clear_config([:mrf_keyword, :federated_timeline_removal], ["pun"]) + + message = %{ + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "type" => "Create", + "object" => %{ + "content" => "just a daily reminder that compLAINer is a good", + "summary" => "", + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "content" => "just a daily reminder that compLAINer is a good pun", + "summary" => "" + } + ] + } + } + } + + {:ok, result} = KeywordPolicy.filter(message) + assert ["https://www.w3.org/ns/activitystreams#Public"] == result["cc"] + refute ["https://www.w3.org/ns/activitystreams#Public"] == result["to"] + end end describe "replacing keywords" do @@ -221,5 +294,63 @@ test "replaces keyword if regex matches in summary" do result == "ZFS is free software" end) end + + test "replaces keyword if string matches in history" do + clear_config([:mrf_keyword, :replace], [{"opensource", "free software"}]) + + message = %{ + "type" => "Create", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "object" => %{ + "content" => "ZFS is opensource", + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{"content" => "ZFS is opensource mew mew", "summary" => ""} + ] + } + } + } + + {:ok, + %{ + "object" => %{ + "content" => "ZFS is free software", + "formerRepresentations" => %{ + "orderedItems" => [%{"content" => "ZFS is free software mew mew"}] + } + } + }} = KeywordPolicy.filter(message) + end + + test "replaces keyword in Updates" do + clear_config([:mrf_keyword, :replace], [{"opensource", "free software"}]) + + message = %{ + "type" => "Update", + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "object" => %{ + "content" => "ZFS is opensource", + "summary" => "", + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{"content" => "ZFS is opensource mew mew", "summary" => ""} + ] + } + } + } + + {:ok, + %{ + "object" => %{ + "content" => "ZFS is free software", + "formerRepresentations" => %{ + "orderedItems" => [%{"content" => "ZFS is free software mew mew"}] + } + } + }} = KeywordPolicy.filter(message) + end end end diff --git a/test/pleroma/web/activity_pub/mrf/media_proxy_warming_policy_test.exs b/test/pleroma/web/activity_pub/mrf/media_proxy_warming_policy_test.exs index 96e715d0d..3268e2321 100644 --- a/test/pleroma/web/activity_pub/mrf/media_proxy_warming_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/media_proxy_warming_policy_test.exs @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do use Pleroma.Tests.Helpers alias Pleroma.HTTP + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy import Mock @@ -22,6 +23,25 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicyTest do } } + @message_with_history %{ + "type" => "Create", + "object" => %{ + "type" => "Note", + "content" => "content", + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "type" => "Note", + "content" => "content", + "attachment" => [ + %{"url" => [%{"href" => "http://example.com/image.jpg"}]} + ] + } + ] + } + } + } + setup do: clear_config([:media_proxy, :enabled], true) test "it prefetches media proxy URIs" do @@ -50,4 +70,28 @@ test "it does nothing when no attachments are present" do refute called(HTTP.get(:_, :_, :_)) end end + + test "history-aware" do + Tesla.Mock.mock(fn %{method: :get, url: "http://example.com/image.jpg"} -> + {:ok, %Tesla.Env{status: 200, body: ""}} + end) + + with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do + MRF.filter_one(MediaProxyWarmingPolicy, @message_with_history) + + assert called(HTTP.get(:_, :_, :_)) + end + end + + test "works with Updates" do + Tesla.Mock.mock(fn %{method: :get, url: "http://example.com/image.jpg"} -> + {:ok, %Tesla.Env{status: 200, body: ""}} + end) + + with_mock HTTP, get: fn _, _, _ -> {:ok, []} end do + MRF.filter_one(MediaProxyWarmingPolicy, @message_with_history |> Map.put("type", "Update")) + + assert called(HTTP.get(:_, :_, :_)) + end + end end diff --git a/test/pleroma/web/activity_pub/mrf/no_empty_policy_test.exs b/test/pleroma/web/activity_pub/mrf/no_empty_policy_test.exs index 2c6fcbc41..d9e05d313 100644 --- a/test/pleroma/web/activity_pub/mrf/no_empty_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/no_empty_policy_test.exs @@ -153,4 +153,27 @@ test "Notes with no content are denied" do assert NoEmptyPolicy.filter(message) == {:reject, "[NoEmptyPolicy]"} end + + test "works with Update" do + message = %{ + "actor" => "http://localhost:4001/users/testuser", + "cc" => ["http://localhost:4001/users/testuser/followers"], + "object" => %{ + "actor" => "http://localhost:4001/users/testuser", + "attachment" => [], + "cc" => ["http://localhost:4001/users/testuser/followers"], + "source" => "", + "to" => [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "type" => "Note" + }, + "to" => [ + "https://www.w3.org/ns/activitystreams#Public" + ], + "type" => "Update" + } + + assert NoEmptyPolicy.filter(message) == {:reject, "[NoEmptyPolicy]"} + end end diff --git a/test/pleroma/web/activity_pub/mrf/no_placeholder_text_policy_test.exs b/test/pleroma/web/activity_pub/mrf/no_placeholder_text_policy_test.exs index 81a6e0f50..59456d790 100644 --- a/test/pleroma/web/activity_pub/mrf/no_placeholder_text_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/no_placeholder_text_policy_test.exs @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicyTest do use Pleroma.DataCase, async: true + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy test "it clears content object" do @@ -20,6 +21,46 @@ test "it clears content object" do assert res["object"]["content"] == "" end + test "history-aware" do + message = %{ + "type" => "Create", + "object" => %{ + "content" => ".", + "attachment" => "image", + "formerRepresentations" => %{ + "orderedItems" => [%{"content" => ".", "attachment" => "image"}] + } + } + } + + assert {:ok, res} = MRF.filter_one(NoPlaceholderTextPolicy, message) + + assert %{ + "content" => "", + "formerRepresentations" => %{"orderedItems" => [%{"content" => ""}]} + } = res["object"] + end + + test "works with Updates" do + message = %{ + "type" => "Update", + "object" => %{ + "content" => ".", + "attachment" => "image", + "formerRepresentations" => %{ + "orderedItems" => [%{"content" => ".", "attachment" => "image"}] + } + } + } + + assert {:ok, res} = MRF.filter_one(NoPlaceholderTextPolicy, message) + + assert %{ + "content" => "", + "formerRepresentations" => %{"orderedItems" => [%{"content" => ""}]} + } = res["object"] + end + @messages [ %{ "type" => "Create", diff --git a/test/pleroma/web/activity_pub/mrf/normalize_markup_test.exs b/test/pleroma/web/activity_pub/mrf/normalize_markup_test.exs index edc330b6c..52a23fdca 100644 --- a/test/pleroma/web/activity_pub/mrf/normalize_markup_test.exs +++ b/test/pleroma/web/activity_pub/mrf/normalize_markup_test.exs @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkupTest do use Pleroma.DataCase, async: true + alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.ActivityPub.MRF.NormalizeMarkup @html_sample """ @@ -16,24 +17,58 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkupTest do """ - test "it filter html tags" do - expected = """ - this is in bold -

this is a paragraph

- this is a linebreak
- this is a link with allowed "rel" attribute: - this is a link with not allowed "rel" attribute: example.com - this is an image:
- alert('hacked') - """ + @expected """ + this is in bold +

this is a paragraph

+ this is a linebreak
+ this is a link with allowed "rel" attribute: + this is a link with not allowed "rel" attribute: example.com + this is an image:
+ alert('hacked') + """ + test "it filter html tags" do message = %{"type" => "Create", "object" => %{"content" => @html_sample}} assert {:ok, res} = NormalizeMarkup.filter(message) - assert res["object"]["content"] == expected + assert res["object"]["content"] == @expected end - test "it skips filter if type isn't `Create`" do + test "history-aware" do + message = %{ + "type" => "Create", + "object" => %{ + "content" => @html_sample, + "formerRepresentations" => %{"orderedItems" => [%{"content" => @html_sample}]} + } + } + + assert {:ok, res} = MRF.filter_one(NormalizeMarkup, message) + + assert %{ + "content" => @expected, + "formerRepresentations" => %{"orderedItems" => [%{"content" => @expected}]} + } = res["object"] + end + + test "works with Updates" do + message = %{ + "type" => "Update", + "object" => %{ + "content" => @html_sample, + "formerRepresentations" => %{"orderedItems" => [%{"content" => @html_sample}]} + } + } + + assert {:ok, res} = MRF.filter_one(NormalizeMarkup, message) + + assert %{ + "content" => @expected, + "formerRepresentations" => %{"orderedItems" => [%{"content" => @expected}]} + } = res["object"] + end + + test "it skips filter if type isn't `Create` or `Update`" do message = %{"type" => "Note", "object" => %{}} assert {:ok, res} = NormalizeMarkup.filter(message) diff --git a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs index 7c8e5a4e1..5b95ebc51 100644 --- a/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/article_note_page_validator_test.exs @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidatorTest do use Pleroma.DataCase, async: true + alias Pleroma.Web.ActivityPub.ObjectValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator alias Pleroma.Web.ActivityPub.Utils @@ -38,6 +39,11 @@ test "a basic note validates", %{note: note} do %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note) end + test "a note from factory validates" do + note = insert(:note) + %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note.data) + end + test "a note with a remote replies collection should validate", _ do insert(:user, %{ap_id: "https://bookwyrm.com/user/TestUser"}) collection = File.read!("test/fixtures/bookwyrm-replies-collection.json") @@ -159,4 +165,47 @@ test "a Note without replies/first/items validates" do %{valid?: true} = ArticleNotePageValidator.cast_and_validate(note) end + + describe "Note with history" do + setup do + user = insert(:user) + {:ok, activity} = Pleroma.Web.CommonAPI.post(user, %{status: "mew mew :dinosaur:"}) + {:ok, edit} = Pleroma.Web.CommonAPI.update(user, activity, %{status: "edited :blank:"}) + + {:ok, %{"object" => external_rep}} = + Pleroma.Web.ActivityPub.Transmogrifier.prepare_outgoing(edit.data) + + %{external_rep: external_rep} + end + + test "edited note", %{external_rep: external_rep} do + assert %{"formerRepresentations" => %{"orderedItems" => [%{"tag" => [_]}]}} = external_rep + + {:ok, validate_res, []} = ObjectValidator.validate(external_rep, []) + + assert %{"formerRepresentations" => %{"orderedItems" => [%{"emoji" => %{"dinosaur" => _}}]}} = + validate_res + end + + test "edited note, badly-formed formerRepresentations", %{external_rep: external_rep} do + external_rep = Map.put(external_rep, "formerRepresentations", %{}) + + assert {:error, _} = ObjectValidator.validate(external_rep, []) + end + + test "edited note, badly-formed history item", %{external_rep: external_rep} do + history_item = + Enum.at(external_rep["formerRepresentations"]["orderedItems"], 0) + |> Map.put("type", "Foo") + + external_rep = + put_in( + external_rep, + ["formerRepresentations", "orderedItems"], + [history_item] + ) + + assert {:error, _} = ObjectValidator.validate(external_rep, []) + end + end end diff --git a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs index 15e4a82cd..a74ee2416 100644 --- a/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs +++ b/test/pleroma/web/activity_pub/object_validators/update_handling_test.exs @@ -32,7 +32,7 @@ test "validates a basic object", %{valid_update: valid_update} do test "returns an error if the object can't be updated by the actor", %{ valid_update: valid_update } do - other_user = insert(:user) + other_user = insert(:user, local: false) update = valid_update @@ -40,5 +40,129 @@ test "returns an error if the object can't be updated by the actor", %{ assert {:error, _cng} = ObjectValidator.validate(update, []) end + + test "validates as long as the object is same-origin with the actor", %{ + valid_update: valid_update + } do + other_user = insert(:user) + + update = + valid_update + |> Map.put("actor", other_user.ap_id) + + assert {:ok, _update, []} = ObjectValidator.validate(update, []) + end + + test "validates if the object is not of an Actor type" do + note = insert(:note) + updated_note = note.data |> Map.put("content", "edited content") + other_user = insert(:user) + + {:ok, update, _} = Builder.update(other_user, updated_note) + + assert {:ok, _update, _} = ObjectValidator.validate(update, []) + end + end + + describe "update note" do + test "converts object into Pleroma's format" do + mastodon_tags = [ + %{ + "icon" => %{ + "mediaType" => "image/png", + "type" => "Image", + "url" => "https://somewhere.org/emoji/url/1.png" + }, + "id" => "https://somewhere.org/emoji/1", + "name" => ":some_emoji:", + "type" => "Emoji", + "updated" => "2021-04-07T11:00:00Z" + } + ] + + user = insert(:user) + note = insert(:note, user: user) + + updated_note = + note.data + |> Map.put("content", "edited content") + |> Map.put("tag", mastodon_tags) + + {:ok, update, _} = Builder.update(user, updated_note) + + assert {:ok, _update, meta} = ObjectValidator.validate(update, []) + + assert %{"emoji" => %{"some_emoji" => "https://somewhere.org/emoji/url/1.png"}} = + meta[:object_data] + end + + test "returns no object_data in meta for a local Update" do + user = insert(:user) + note = insert(:note, user: user) + + updated_note = + note.data + |> Map.put("content", "edited content") + + {:ok, update, _} = Builder.update(user, updated_note) + + assert {:ok, _update, meta} = ObjectValidator.validate(update, local: true) + assert is_nil(meta[:object_data]) + end + + test "returns object_data in meta for a remote Update" do + user = insert(:user) + note = insert(:note, user: user) + + updated_note = + note.data + |> Map.put("content", "edited content") + + {:ok, update, _} = Builder.update(user, updated_note) + + assert {:ok, _update, meta} = ObjectValidator.validate(update, local: false) + assert meta[:object_data] + + assert {:ok, _update, meta} = ObjectValidator.validate(update, []) + assert meta[:object_data] + end + end + + describe "update with history" do + setup do + user = insert(:user) + {:ok, activity} = Pleroma.Web.CommonAPI.post(user, %{status: "mew mew :dinosaur:"}) + {:ok, edit} = Pleroma.Web.CommonAPI.update(user, activity, %{status: "edited :blank:"}) + {:ok, external_rep} = Pleroma.Web.ActivityPub.Transmogrifier.prepare_outgoing(edit.data) + %{external_rep: external_rep} + end + + test "edited note", %{external_rep: external_rep} do + {:ok, _validate_res, meta} = ObjectValidator.validate(external_rep, []) + + assert %{"formerRepresentations" => %{"orderedItems" => [%{"emoji" => %{"dinosaur" => _}}]}} = + meta[:object_data] + end + + test "edited note, badly-formed formerRepresentations", %{external_rep: external_rep} do + external_rep = put_in(external_rep, ["object", "formerRepresentations"], %{}) + + assert {:error, _} = ObjectValidator.validate(external_rep, []) + end + + test "edited note, badly-formed history item", %{external_rep: external_rep} do + history_item = + Enum.at(external_rep["object"]["formerRepresentations"]["orderedItems"], 0) + |> Map.put("type", "Foo") + + external_rep = + put_in( + external_rep, + ["object", "formerRepresentations", "orderedItems"], + [history_item] + ) + + assert {:error, _} = ObjectValidator.validate(external_rep, []) + end end end diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index e542c06f5..fa8171eab 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -123,7 +123,10 @@ test "it blocks but does not unfollow if the relevant setting is set", %{ describe "update users" do setup do user = insert(:user, local: false) - {:ok, update_data, []} = Builder.update(user, %{"id" => user.ap_id, "name" => "new name!"}) + + {:ok, update_data, []} = + Builder.update(user, %{"id" => user.ap_id, "type" => "Person", "name" => "new name!"}) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) %{user: user, update_data: update_data, update: update} @@ -145,6 +148,298 @@ test "it uses a given changeset to update", %{user: user, update: update} do end end + describe "update notes" do + setup do + make_time = fn -> + Pleroma.Web.ActivityPub.Utils.make_date() + end + + user = insert(:user) + note = insert(:note, user: user, data: %{"published" => make_time.()}) + _note_activity = insert(:note_activity, note: note) + + updated_note = + note.data + |> Map.put("summary", "edited summary") + |> Map.put("content", "edited content") + |> Map.put("updated", make_time.()) + + {:ok, update_data, []} = Builder.update(user, updated_note) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) + + %{ + user: user, + note: note, + object_id: note.id, + update_data: update_data, + update: update, + updated_note: updated_note + } + end + + test "it updates the note", %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + updated_time = updated_note["updated"] + + new_note = Pleroma.Object.get_by_id(object_id) + + assert %{ + "summary" => "edited summary", + "content" => "edited content", + "updated" => ^updated_time + } = new_note.data + end + + test "it rejects updates with no updated attribute in object", %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + old_note = Pleroma.Object.get_by_id(object_id) + updated_note = Map.drop(updated_note, ["updated"]) + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + new_note = Pleroma.Object.get_by_id(object_id) + assert old_note.data == new_note.data + end + + test "it rejects updates with updated attribute older than what we have in the original object", + %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + old_note = Pleroma.Object.get_by_id(object_id) + {:ok, creation_time, _} = DateTime.from_iso8601(old_note.data["published"]) + + updated_note = + Map.put(updated_note, "updated", DateTime.to_iso8601(DateTime.add(creation_time, -10))) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + new_note = Pleroma.Object.get_by_id(object_id) + assert old_note.data == new_note.data + end + + test "it rejects updates with updated attribute older than the last Update", %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + old_note = Pleroma.Object.get_by_id(object_id) + {:ok, creation_time, _} = DateTime.from_iso8601(old_note.data["published"]) + + updated_note = + Map.put(updated_note, "updated", DateTime.to_iso8601(DateTime.add(creation_time, +10))) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + + old_note = Pleroma.Object.get_by_id(object_id) + {:ok, update_time, _} = DateTime.from_iso8601(old_note.data["updated"]) + + updated_note = + Map.put(updated_note, "updated", DateTime.to_iso8601(DateTime.add(update_time, -5))) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + + new_note = Pleroma.Object.get_by_id(object_id) + assert old_note.data == new_note.data + end + + test "it updates using object_data", %{ + object_id: object_id, + update: update, + updated_note: updated_note + } do + updated_note = Map.put(updated_note, "summary", "mew mew") + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + new_note = Pleroma.Object.get_by_id(object_id) + assert %{"summary" => "mew mew", "content" => "edited content"} = new_note.data + end + + test "it records the original note in formerRepresentations", %{ + note: note, + object_id: object_id, + update: update, + updated_note: updated_note + } do + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + %{data: new_note} = Pleroma.Object.get_by_id(object_id) + assert %{"summary" => "edited summary", "content" => "edited content"} = new_note + + assert [Map.drop(note.data, ["id", "formerRepresentations"])] == + new_note["formerRepresentations"]["orderedItems"] + + assert new_note["formerRepresentations"]["totalItems"] == 1 + end + + test "it puts the original note at the front of formerRepresentations", %{ + user: user, + note: note, + object_id: object_id, + update: update, + updated_note: updated_note + } do + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + %{data: first_edit} = Pleroma.Object.get_by_id(object_id) + + second_updated_note = + note.data + |> Map.put("summary", "edited summary 2") + |> Map.put("content", "edited content 2") + |> Map.put( + "updated", + first_edit["updated"] + |> DateTime.from_iso8601() + |> elem(1) + |> DateTime.add(10) + |> DateTime.to_iso8601() + ) + + {:ok, second_update_data, []} = Builder.update(user, second_updated_note) + {:ok, update, _meta} = ActivityPub.persist(second_update_data, local: true) + {:ok, _, _} = SideEffects.handle(update, object_data: second_updated_note) + %{data: new_note} = Pleroma.Object.get_by_id(object_id) + assert %{"summary" => "edited summary 2", "content" => "edited content 2"} = new_note + + original_version = Map.drop(note.data, ["id", "formerRepresentations"]) + first_edit = Map.drop(first_edit, ["id", "formerRepresentations"]) + + assert [first_edit, original_version] == + new_note["formerRepresentations"]["orderedItems"] + + assert new_note["formerRepresentations"]["totalItems"] == 2 + end + + test "it does not prepend to formerRepresentations if no actual changes are made", %{ + note: note, + object_id: object_id, + update: update, + updated_note: updated_note + } do + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + %{data: first_edit} = Pleroma.Object.get_by_id(object_id) + + updated_note = + updated_note + |> Map.put( + "updated", + first_edit["updated"] + |> DateTime.from_iso8601() + |> elem(1) + |> DateTime.add(10) + |> DateTime.to_iso8601() + ) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_note) + %{data: new_note} = Pleroma.Object.get_by_id(object_id) + assert %{"summary" => "edited summary", "content" => "edited content"} = new_note + + original_version = Map.drop(note.data, ["id", "formerRepresentations"]) + + assert [original_version] == + new_note["formerRepresentations"]["orderedItems"] + + assert new_note["formerRepresentations"]["totalItems"] == 1 + end + end + + describe "update questions" do + setup do + user = insert(:user) + + question = + insert(:question, + user: user, + data: %{"published" => Pleroma.Web.ActivityPub.Utils.make_date()} + ) + + %{user: user, data: question.data, id: question.id} + end + + test "allows updating choice count without generating edit history", %{ + user: user, + data: data, + id: id + } do + new_choices = + data["oneOf"] + |> Enum.map(fn choice -> put_in(choice, ["replies", "totalItems"], 5) end) + + updated_question = + data + |> Map.put("oneOf", new_choices) + |> Map.put("updated", Pleroma.Web.ActivityPub.Utils.make_date()) + + {:ok, update_data, []} = Builder.update(user, updated_question) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_question) + + %{data: new_question} = Pleroma.Object.get_by_id(id) + + assert [%{"replies" => %{"totalItems" => 5}}, %{"replies" => %{"totalItems" => 5}}] = + new_question["oneOf"] + + refute Map.has_key?(new_question, "formerRepresentations") + end + + test "allows updating choice count without updated field", %{ + user: user, + data: data, + id: id + } do + new_choices = + data["oneOf"] + |> Enum.map(fn choice -> put_in(choice, ["replies", "totalItems"], 5) end) + + updated_question = + data + |> Map.put("oneOf", new_choices) + + {:ok, update_data, []} = Builder.update(user, updated_question) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_question) + + %{data: new_question} = Pleroma.Object.get_by_id(id) + + assert [%{"replies" => %{"totalItems" => 5}}, %{"replies" => %{"totalItems" => 5}}] = + new_question["oneOf"] + + refute Map.has_key?(new_question, "formerRepresentations") + end + + test "allows updating choice count with updated field same as the creation date", %{ + user: user, + data: data, + id: id + } do + new_choices = + data["oneOf"] + |> Enum.map(fn choice -> put_in(choice, ["replies", "totalItems"], 5) end) + + updated_question = + data + |> Map.put("oneOf", new_choices) + |> Map.put("updated", data["published"]) + + {:ok, update_data, []} = Builder.update(user, updated_question) + {:ok, update, _meta} = ActivityPub.persist(update_data, local: true) + + {:ok, _, _} = SideEffects.handle(update, object_data: updated_question) + + %{data: new_question} = Pleroma.Object.get_by_id(id) + + assert [%{"replies" => %{"totalItems" => 5}}, %{"replies" => %{"totalItems" => 5}}] = + new_question["oneOf"] + + refute Map.has_key?(new_question, "formerRepresentations") + end + end + describe "EmojiReact objects" do setup do poster = insert(:user) diff --git a/test/pleroma/web/activity_pub/transmogrifier_test.exs b/test/pleroma/web/activity_pub/transmogrifier_test.exs index ae2fc067a..a10708481 100644 --- a/test/pleroma/web/activity_pub/transmogrifier_test.exs +++ b/test/pleroma/web/activity_pub/transmogrifier_test.exs @@ -301,6 +301,28 @@ test "custom emoji urls are URI encoded" do assert url == "http://localhost:4001/emoji/dino%20walking.gif" end + + test "Updates of Notes are handled" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "everybody do the dinosaur :dinosaur:"}) + {:ok, update} = CommonAPI.update(user, activity, %{status: "mew mew :blank:"}) + + {:ok, prepared} = Transmogrifier.prepare_outgoing(update.data) + + assert %{ + "content" => "mew mew :blank:", + "tag" => [%{"name" => ":blank:", "type" => "Emoji"}], + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "content" => "everybody do the dinosaur :dinosaur:", + "tag" => [%{"name" => ":dinosaur:", "type" => "Emoji"}] + } + ] + } + } = prepared["object"] + end end describe "user upgrade" do @@ -564,4 +586,43 @@ test "puts dimensions into attachment url field" do assert Transmogrifier.fix_attachments(object) == expected end end + + describe "prepare_object/1" do + test "it processes history" do + original = %{ + "formerRepresentations" => %{ + "orderedItems" => [ + %{ + "generator" => %{}, + "emoji" => %{"blobcat" => "http://localhost:4001/emoji/blobcat.png"} + } + ] + } + } + + processed = Transmogrifier.prepare_object(original) + + history_item = Enum.at(processed["formerRepresentations"]["orderedItems"], 0) + + refute Map.has_key?(history_item, "generator") + + assert [%{"name" => ":blobcat:"}] = history_item["tag"] + end + + test "it works when there is no or bad history" do + original = %{ + "formerRepresentations" => %{ + "items" => [ + %{ + "generator" => %{}, + "emoji" => %{"blobcat" => "http://localhost:4001/emoji/blobcat.png"} + } + ] + } + } + + processed = Transmogrifier.prepare_object(original) + assert processed["formerRepresentations"] == original["formerRepresentations"] + end + end end diff --git a/test/pleroma/web/common_api_test.exs b/test/pleroma/web/common_api_test.exs index 840d74d2f..2b7a34be2 100644 --- a/test/pleroma/web/common_api_test.exs +++ b/test/pleroma/web/common_api_test.exs @@ -1313,4 +1313,128 @@ test "unreact_with_emoji" do end end end + + describe "update/3" do + test "updates a post" do + user = insert(:user) + {:ok, activity} = CommonAPI.post(user, %{status: "foo1", spoiler_text: "title 1"}) + + {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2"}) + + updated_object = Object.normalize(updated) + assert updated_object.data["content"] == "updated 2" + assert Map.get(updated_object.data, "summary", "") == "" + assert Map.has_key?(updated_object.data, "updated") + end + + test "does not change visibility" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{status: "foo1", spoiler_text: "title 1", visibility: "private"}) + + {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2"}) + + updated_object = Object.normalize(updated) + assert updated_object.data["content"] == "updated 2" + assert Map.get(updated_object.data, "summary", "") == "" + assert Visibility.get_visibility(updated_object) == "private" + assert Visibility.get_visibility(updated) == "private" + end + + test "updates a post with emoji" do + [{emoji1, _}, {emoji2, _} | _] = Pleroma.Emoji.get_all() + + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{status: "foo1", spoiler_text: "title 1 :#{emoji1}:"}) + + {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2 :#{emoji2}:"}) + + updated_object = Object.normalize(updated) + assert updated_object.data["content"] == "updated 2 :#{emoji2}:" + assert %{^emoji2 => _} = updated_object.data["emoji"] + end + + test "updates a post with emoji and federate properly" do + [{emoji1, _}, {emoji2, _} | _] = Pleroma.Emoji.get_all() + + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{status: "foo1", spoiler_text: "title 1 :#{emoji1}:"}) + + clear_config([:instance, :federating], true) + + with_mock Pleroma.Web.Federator, + publish: fn _p -> nil end do + {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2 :#{emoji2}:"}) + + assert updated.data["object"]["content"] == "updated 2 :#{emoji2}:" + assert %{^emoji2 => _} = updated.data["object"]["emoji"] + + assert called(Pleroma.Web.Federator.publish(updated)) + end + end + + test "editing a post that copied a remote title with remote emoji should keep that emoji" do + remote_emoji_uri = "https://remote.org/emoji.png" + + note = + insert( + :note, + data: %{ + "summary" => ":remoteemoji:", + "emoji" => %{ + "remoteemoji" => remote_emoji_uri + }, + "tag" => [ + %{ + "type" => "Emoji", + "name" => "remoteemoji", + "icon" => %{"url" => remote_emoji_uri} + } + ] + } + ) + + note_activity = insert(:note_activity, note: note) + + user = insert(:user) + + {:ok, reply} = + CommonAPI.post(user, %{ + status: "reply", + spoiler_text: ":remoteemoji:", + in_reply_to_id: note_activity.id + }) + + assert reply.object.data["emoji"]["remoteemoji"] == remote_emoji_uri + + {:ok, edit} = + CommonAPI.update(user, reply, %{status: "reply mew mew", spoiler_text: ":remoteemoji:"}) + + edited_note = Pleroma.Object.normalize(edit) + + assert edited_note.data["emoji"]["remoteemoji"] == remote_emoji_uri + end + + test "respects MRF" do + user = insert(:user) + + clear_config([:mrf, :policies], [Pleroma.Web.ActivityPub.MRF.KeywordPolicy]) + clear_config([:mrf_keyword, :replace], [{"updated", "mewmew"}]) + + {:ok, activity} = CommonAPI.post(user, %{status: "foo1", spoiler_text: "updated 1"}) + assert Object.normalize(activity).data["summary"] == "mewmew 1" + + {:ok, updated} = CommonAPI.update(user, activity, %{status: "updated 2"}) + + updated_object = Object.normalize(updated) + assert updated_object.data["content"] == "mewmew 2" + assert Map.get(updated_object.data, "summary", "") == "" + assert Map.has_key?(updated_object.data, "updated") + end + end end diff --git a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs index f76ab3d0d..ea6ace69f 100644 --- a/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/status_controller_test.exs @@ -2072,6 +2072,52 @@ test "posting a quote of a status that doesn't exist", %{conn: conn} do end end + describe "get status history" do + setup do + %{conn: build_conn()} + end + + test "unedited post", %{conn: conn} do + activity = insert(:note_activity) + + conn = get(conn, "/api/v1/statuses/#{activity.id}/history") + + assert [_] = json_response_and_validate_schema(conn, 200) + end + + test "edited post", %{conn: conn} do + note = + insert( + :note, + data: %{ + "formerRepresentations" => %{ + "type" => "OrderedCollection", + "orderedItems" => [ + %{ + "type" => "Note", + "content" => "mew mew 2", + "summary" => "title 2" + }, + %{ + "type" => "Note", + "content" => "mew mew 1", + "summary" => "title 1" + } + ], + "totalItems" => 2 + } + } + ) + + activity = insert(:note_activity, note: note) + + conn = get(conn, "/api/v1/statuses/#{activity.id}/history") + + assert [%{"spoiler_text" => "title 1"}, %{"spoiler_text" => "title 2"}, _] = + json_response_and_validate_schema(conn, 200) + end + end + describe "translating statuses" do setup do clear_config([:translator, :enabled], true) @@ -2177,4 +2223,132 @@ test "should not allow translating of statuses you cannot see", %{conn: conn} do json_response_and_validate_schema(conn, 404) end end + + describe "get status source" do + setup do + %{conn: build_conn()} + end + + test "it returns the source", %{conn: conn} do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "mew mew #abc", spoiler_text: "#def"}) + + conn = get(conn, "/api/v1/statuses/#{activity.id}/source") + + id = activity.id + + assert %{"id" => ^id, "text" => "mew mew #abc", "spoiler_text" => "#def"} = + json_response_and_validate_schema(conn, 200) + end + end + + describe "update status" do + setup do + oauth_access(["write:statuses"]) + end + + test "it updates the status" do + %{conn: conn, user: user} = oauth_access(["write:statuses", "read:statuses"]) + + {:ok, activity} = CommonAPI.post(user, %{status: "mew mew #abc", spoiler_text: "#def"}) + + conn + |> get("/api/v1/statuses/#{activity.id}") + |> json_response_and_validate_schema(200) + + response = + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/statuses/#{activity.id}", %{ + "status" => "edited", + "spoiler_text" => "lol" + }) + |> json_response_and_validate_schema(200) + + assert response["content"] == "edited" + assert response["spoiler_text"] == "lol" + + response = + conn + |> get("/api/v1/statuses/#{activity.id}") + |> json_response_and_validate_schema(200) + + assert response["content"] == "edited" + assert response["spoiler_text"] == "lol" + end + + test "it updates the attachments", %{conn: conn, user: user} do + attachment = insert(:attachment, user: user) + attachment_id = to_string(attachment.id) + + {:ok, activity} = CommonAPI.post(user, %{status: "mew mew #abc", spoiler_text: "#def"}) + + response = + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/statuses/#{activity.id}", %{ + "status" => "mew mew #abc", + "spoiler_text" => "#def", + "media_ids" => [attachment_id] + }) + |> json_response_and_validate_schema(200) + + assert [%{"id" => ^attachment_id}] = response["media_attachments"] + end + + test "it does not update visibility", %{conn: conn, user: user} do + {:ok, activity} = + CommonAPI.post(user, %{ + status: "mew mew #abc", + spoiler_text: "#def", + visibility: "private" + }) + + response = + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/statuses/#{activity.id}", %{ + "status" => "edited", + "spoiler_text" => "lol" + }) + |> json_response_and_validate_schema(200) + + assert response["visibility"] == "private" + end + + test "it refuses to update when original post is not by the user", %{conn: conn} do + another_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(another_user, %{status: "mew mew #abc", spoiler_text: "#def"}) + + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/statuses/#{activity.id}", %{ + "status" => "edited", + "spoiler_text" => "lol" + }) + |> json_response_and_validate_schema(:forbidden) + end + + test "it returns 404 if the user cannot see the post", %{conn: conn} do + another_user = insert(:user) + + {:ok, activity} = + CommonAPI.post(another_user, %{ + status: "mew mew #abc", + spoiler_text: "#def", + visibility: "private" + }) + + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/statuses/#{activity.id}", %{ + "status" => "edited", + "spoiler_text" => "lol" + }) + |> json_response_and_validate_schema(:not_found) + end + end end diff --git a/test/pleroma/web/mastodon_api/views/notification_view_test.exs b/test/pleroma/web/mastodon_api/views/notification_view_test.exs index 803b1f438..64d2c8a2e 100644 --- a/test/pleroma/web/mastodon_api/views/notification_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/notification_view_test.exs @@ -285,6 +285,32 @@ test "Report notification" do test_notifications_rendering([notification], moderator_user, [expected]) end + test "Edit notification" do + user = insert(:user) + repeat_user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "mew"}) + {:ok, _} = CommonAPI.repeat(activity.id, repeat_user) + {:ok, update} = CommonAPI.update(user, activity, %{status: "mew mew"}) + + user = Pleroma.User.get_by_ap_id(user.ap_id) + activity = Pleroma.Activity.normalize(activity) + update = Pleroma.Activity.normalize(update) + + {:ok, [notification]} = Notification.create_notifications(update) + + expected = %{ + id: to_string(notification.id), + pleroma: %{is_seen: false, is_muted: false}, + type: "update", + account: AccountView.render("show.json", %{user: user, for: repeat_user}), + created_at: Utils.to_masto_date(notification.inserted_at), + status: StatusView.render("show.json", %{activity: activity, for: repeat_user}) + } + + test_notifications_rendering([notification], repeat_user, [expected]) + end + test "muted notification" do user = insert(:user) another_user = insert(:user) diff --git a/test/pleroma/web/mastodon_api/views/status_view_test.exs b/test/pleroma/web/mastodon_api/views/status_view_test.exs index f46dded7c..b3f0a1781 100644 --- a/test/pleroma/web/mastodon_api/views/status_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/status_view_test.exs @@ -267,6 +267,7 @@ test "a note activity" do content: HTML.filter_tags(object_data["content"]), text: nil, created_at: created_at, + edited_at: nil, reblogs_count: 0, replies_count: 0, favourites_count: 0, @@ -788,4 +789,55 @@ test "has a field for parent visibility" do status = StatusView.render("show.json", activity: visible, for: poster) assert status.pleroma.parent_visible end + + test "it shows edited_at" do + poster = insert(:user) + + {:ok, post} = CommonAPI.post(poster, %{status: "hey"}) + + status = StatusView.render("show.json", activity: post) + refute status.edited_at + + {:ok, _} = CommonAPI.update(poster, post, %{status: "mew mew"}) + edited = Pleroma.Activity.normalize(post) + + status = StatusView.render("show.json", activity: edited) + assert status.edited_at + end + + test "with a source object" do + note = + insert(:note, + data: %{"source" => %{"content" => "object source", "mediaType" => "text/markdown"}} + ) + + activity = insert(:note_activity, note: note) + + status = StatusView.render("show.json", activity: activity, with_source: true) + assert status.text == "object source" + end + + describe "source.json" do + test "with a source object, renders both source and content type" do + note = + insert(:note, + data: %{"source" => %{"content" => "object source", "mediaType" => "text/markdown"}} + ) + + activity = insert(:note_activity, note: note) + + status = StatusView.render("source.json", activity: activity) + assert status.text == "object source" + assert status.content_type == "text/markdown" + end + + test "with a source string, renders source and put text/plain as the content type" do + note = insert(:note, data: %{"source" => "string source"}) + activity = insert(:note_activity, note: note) + + status = StatusView.render("source.json", activity: activity) + assert status.text == "string source" + assert status.content_type == "text/plain" + end + end end diff --git a/test/pleroma/web/metadata/utils_test.exs b/test/pleroma/web/metadata/utils_test.exs index 074bd2e2f..c99d11596 100644 --- a/test/pleroma/web/metadata/utils_test.exs +++ b/test/pleroma/web/metadata/utils_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Metadata.UtilsTest do - use Pleroma.DataCase, async: true + use Pleroma.DataCase, async: false import Pleroma.Factory alias Pleroma.Web.Metadata.Utils @@ -22,6 +22,20 @@ test "it returns text without encode HTML" do assert Utils.scrub_html_and_truncate(note) == "Pleroma's really cool!" end + + test "it does not return old content after editing" do + user = insert(:user) + + {:ok, activity} = Pleroma.Web.CommonAPI.post(user, %{status: "mew mew #def"}) + + object = Pleroma.Object.normalize(activity) + assert Utils.scrub_html_and_truncate(object) == "mew mew #def" + + {:ok, update} = Pleroma.Web.CommonAPI.update(user, activity, %{status: "mew mew #abc"}) + update = Pleroma.Activity.normalize(update) + object = Pleroma.Object.normalize(update) + assert Utils.scrub_html_and_truncate(object) == "mew mew #abc" + end end describe "scrub_html_and_truncate/2" do diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs index 9ae733fc6..8e2ab5016 100644 --- a/test/pleroma/web/streamer_test.exs +++ b/test/pleroma/web/streamer_test.exs @@ -383,6 +383,33 @@ test "it sends follow relationships updates to the 'user' stream", %{ "state" => "follow_accept" } = Jason.decode!(payload) end + + test "it streams edits in the 'user' stream", %{user: user, token: oauth_token} do + sender = insert(:user) + {:ok, _, _, _} = CommonAPI.follow(user, sender) + + {:ok, activity} = CommonAPI.post(sender, %{status: "hey"}) + + Streamer.get_topic_and_add_socket("user", user, oauth_token) + {:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew"}) + create = Pleroma.Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"]) + + stream = "user:#{user.id}" + assert_receive {:render_with_user, _, "status_update.json", ^create, ^stream} + refute Streamer.filtered_by_user?(user, edited) + end + + test "it streams own edits in the 'user' stream", %{user: user, token: oauth_token} do + {:ok, activity} = CommonAPI.post(user, %{status: "hey"}) + + Streamer.get_topic_and_add_socket("user", user, oauth_token) + {:ok, edited} = CommonAPI.update(user, activity, %{status: "mew mew"}) + create = Pleroma.Activity.get_create_by_object_ap_id_with_object(activity.object.data["id"]) + + stream = "user:#{user.id}" + assert_receive {:render_with_user, _, "status_update.json", ^create, ^stream} + refute Streamer.filtered_by_user?(user, edited) + end end describe "public streams" do @@ -425,6 +452,54 @@ test "handles deletions" do assert_receive {:text, event} assert %{"event" => "delete", "payload" => ^activity_id} = Jason.decode!(event) end + + test "it streams edits in the 'public' stream" do + sender = insert(:user) + + Streamer.get_topic_and_add_socket("public", nil, nil) + {:ok, activity} = CommonAPI.post(sender, %{status: "hey"}) + assert_receive {:text, _} + + {:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew"}) + + edited = Pleroma.Activity.normalize(edited) + + %{id: activity_id} = Pleroma.Activity.get_create_by_object_ap_id(edited.object.data["id"]) + + assert_receive {:text, event} + assert %{"event" => "status.update", "payload" => payload} = Jason.decode!(event) + assert %{"id" => ^activity_id} = Jason.decode!(payload) + refute Streamer.filtered_by_user?(sender, edited) + end + + test "it streams multiple edits in the 'public' stream correctly" do + sender = insert(:user) + + Streamer.get_topic_and_add_socket("public", nil, nil) + {:ok, activity} = CommonAPI.post(sender, %{status: "hey"}) + assert_receive {:text, _} + + {:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew"}) + + edited = Pleroma.Activity.normalize(edited) + + %{id: activity_id} = Pleroma.Activity.get_create_by_object_ap_id(edited.object.data["id"]) + + assert_receive {:text, event} + assert %{"event" => "status.update", "payload" => payload} = Jason.decode!(event) + assert %{"id" => ^activity_id} = Jason.decode!(payload) + refute Streamer.filtered_by_user?(sender, edited) + + {:ok, edited} = CommonAPI.update(sender, activity, %{status: "mew mew 2"}) + + edited = Pleroma.Activity.normalize(edited) + + %{id: activity_id} = Pleroma.Activity.get_create_by_object_ap_id(edited.object.data["id"]) + assert_receive {:text, event} + assert %{"event" => "status.update", "payload" => payload} = Jason.decode!(event) + assert %{"id" => ^activity_id, "content" => "mew mew 2"} = Jason.decode!(payload) + refute Streamer.filtered_by_user?(sender, edited) + end end describe "thread_containment/2" do diff --git a/test/support/factory.ex b/test/support/factory.ex index 64d983663..6695886dc 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -111,6 +111,18 @@ def note_factory(attrs \\ %{}) do } end + def attachment_factory(attrs \\ %{}) do + user = attrs[:user] || insert(:user) + + data = + attachment_data(user.ap_id, nil) + |> Map.put("id", Pleroma.Web.ActivityPub.Utils.generate_object_id()) + + %Pleroma.Object{ + data: merge_attributes(data, Map.get(attrs, :data, %{})) + } + end + def attachment_note_factory(attrs \\ %{}) do user = attrs[:user] || insert(:user) {length, attrs} = Map.pop(attrs, :length, 1) From 7af32634be4ea4fde7ff495a6ebe293c9233fe7b Mon Sep 17 00:00:00 2001 From: Norm Date: Thu, 8 Sep 2022 09:54:02 +0000 Subject: [PATCH 41/44] Remove gitlab files (#203) These don't really serve a purpose now and aren't even recognized by Gitea. Co-authored-by: Francis Dinh Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/203 Co-authored-by: Norm Co-committed-by: Norm --- .gitlab/issue_templates/Bug.md | 18 ------------------ .gitlab/merge_request_templates/Release.md | 6 ------ 2 files changed, 24 deletions(-) delete mode 100644 .gitlab/issue_templates/Bug.md delete mode 100644 .gitlab/merge_request_templates/Release.md diff --git a/.gitlab/issue_templates/Bug.md b/.gitlab/issue_templates/Bug.md deleted file mode 100644 index dd0d6eb24..000000000 --- a/.gitlab/issue_templates/Bug.md +++ /dev/null @@ -1,18 +0,0 @@ - - -### Environment - -* Installation type (OTP or From Source): -* Pleroma version (could be found in the "Version" tab of settings in Pleroma-FE): -* Elixir version (`elixir -v` for from source installations, N/A for OTP): -* Operating system: -* PostgreSQL version (`psql -V`): - - -### Bug description diff --git a/.gitlab/merge_request_templates/Release.md b/.gitlab/merge_request_templates/Release.md deleted file mode 100644 index b2c772696..000000000 --- a/.gitlab/merge_request_templates/Release.md +++ /dev/null @@ -1,6 +0,0 @@ -### Release checklist -* [ ] Bump version in `mix.exs` -* [ ] Compile a changelog -* [ ] Create an MR with an announcement to pleroma.social -* [ ] Tag the release -* [ ] Merge `stable` into `develop` (in case the fixes are already in develop, use `git merge -s ours --no-commit` and manually merge the changelogs) From a6d85003fe0cedab924a14f1065c181a2d468a1b Mon Sep 17 00:00:00 2001 From: Norm Date: Thu, 8 Sep 2022 10:19:22 +0000 Subject: [PATCH 42/44] Remote interaction with posts (#198) Grabbed from https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3587 Co-authored-by: Tusooa Zhu Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/198 Co-authored-by: Norm Co-committed-by: Norm --- CHANGELOG.md | 1 + .../operations/twitter_util_operation.ex | 10 ++ .../web/mastodon_api/views/instance_view.ex | 3 +- lib/pleroma/web/router.ex | 1 + .../twitter_api/util/status_interact.html.eex | 10 ++ .../controllers/util_controller.ex | 95 +++++++++++++++++- .../web/twitter_api/views/util_view.ex | 2 + .../web/twitter_api/util_controller_test.exs | 96 +++++++++++++++++++ 8 files changed, 212 insertions(+), 6 deletions(-) create mode 100644 lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex diff --git a/CHANGELOG.md b/CHANGELOG.md index e63cc1f6e..f63fa540c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - amd64 is built for debian stable. Compatible with ubuntu 20. - ubuntu-jammy is built for... well, ubuntu 22 (LTS) - amd64-musl is built for alpine 3.16 +- Enable remote users to interact with posts ### Fixed - Updated mastoFE path, for the newer version diff --git a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex index 4a2a246f5..c025867a2 100644 --- a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex +++ b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex @@ -405,6 +405,16 @@ defp remote_interaction_request do } end + def show_subscribe_form_operation do + %Operation{ + tags: ["Accounts"], + summary: "Show remote subscribe form", + operationId: "UtilController.show_subscribe_form", + parameters: [], + responses: %{200 => Operation.response("Web Page", "test/html", %Schema{type: :string})} + } + end + defp delete_account_request do %Schema{ title: "AccountDeleteRequest", diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 6612a7ec1..4fed1af74 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -85,7 +85,8 @@ def features do if Config.get([:translator, :enabled], false) do "akkoma:machine_translation" end, - "custom_emoji_reactions" + "custom_emoji_reactions", + "pleroma:get:main/ostatus" ] |> Enum.filter(& &1) end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index cc63b2b04..f722d94f7 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -337,6 +337,7 @@ defmodule Pleroma.Web.Router do pipe_through(:pleroma_html) post("/main/ostatus", UtilController, :remote_subscribe) + get("/main/ostatus", UtilController, :show_subscribe_form) get("/ostatus_subscribe", RemoteFollowController, :follow) post("/ostatus_subscribe", RemoteFollowController, :do_follow) end diff --git a/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex b/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex new file mode 100644 index 000000000..d77174967 --- /dev/null +++ b/lib/pleroma/web/templates/twitter_api/util/status_interact.html.eex @@ -0,0 +1,10 @@ +<%= if @error do %> +

<%= Gettext.dpgettext("static_pages", "status interact error", "Error: %{error}", error: @error) %>

+<% else %> +

<%= raw Gettext.dpgettext("static_pages", "status interact header", "Interacting with %{nickname}'s %{status_link}", nickname: safe_to_string(html_escape(@nickname)), status_link: safe_to_string(link(Gettext.dpgettext("static_pages", "status interact header - status link text", "status"), to: @status_link))) %>

+ <%= form_for @conn, Routes.util_path(@conn, :remote_subscribe), [as: "status"], fn f -> %> + <%= hidden_input f, :status_id, value: @status_id %> + <%= text_input f, :profile, placeholder: Gettext.dpgettext("static_pages", "placeholder text for account id", "Your account ID, e.g. lain@quitter.se") %> + <%= submit Gettext.dpgettext("static_pages", "status interact authorization button", "Interact") %> + <% end %> +<% end %> diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index b8abc666e..a0c3e5c52 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do require Logger + alias Pleroma.Activity alias Pleroma.Config alias Pleroma.Emoji alias Pleroma.Healthcheck @@ -16,8 +17,16 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Pleroma.Web.Plugs.OAuthScopesPlug alias Pleroma.Web.WebFinger - plug(Pleroma.Web.ApiSpec.CastAndValidate when action != :remote_subscribe) - plug(Pleroma.Web.Plugs.FederatingPlug when action == :remote_subscribe) + plug( + Pleroma.Web.ApiSpec.CastAndValidate + when action != :remote_subscribe and action != :show_subscribe_form + ) + + plug( + Pleroma.Web.Plugs.FederatingPlug + when action == :remote_subscribe + when action == :show_subscribe_form + ) plug( OAuthScopesPlug, @@ -44,7 +53,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.TwitterUtilOperation - def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do + def show_subscribe_form(conn, %{"nickname" => nick}) do with %User{} = user <- User.get_cached_by_nickname(nick), avatar = User.avatar_url(user) do conn @@ -54,11 +63,52 @@ def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do render(conn, "subscribe.html", %{ nickname: nick, avatar: nil, - error: "Could not find user" + error: + Pleroma.Web.Gettext.dpgettext( + "static_pages", + "remote follow error message - user not found", + "Could not find user" + ) }) end end + def show_subscribe_form(conn, %{"status_id" => id}) do + with %Activity{} = activity <- Activity.get_by_id(id), + {:ok, ap_id} <- get_ap_id(activity), + %User{} = user <- User.get_cached_by_ap_id(activity.actor), + avatar = User.avatar_url(user) do + conn + |> render("status_interact.html", %{ + status_link: ap_id, + status_id: id, + nickname: user.nickname, + avatar: avatar, + error: false + }) + else + _e -> + render(conn, "status_interact.html", %{ + status_id: id, + avatar: nil, + error: + Pleroma.Web.Gettext.dpgettext( + "static_pages", + "status interact error message - status not found", + "Could not find status" + ) + }) + end + end + + def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do + show_subscribe_form(conn, %{"nickname" => nick}) + end + + def remote_subscribe(conn, %{"status_id" => id, "profile" => _}) do + show_subscribe_form(conn, %{"status_id" => id}) + end + def remote_subscribe(conn, %{"user" => %{"nickname" => nick, "profile" => profile}}) do with {:ok, %{"subscribe_address" => template}} <- WebFinger.finger(profile), %User{ap_id: ap_id} <- User.get_cached_by_nickname(nick) do @@ -69,7 +119,33 @@ def remote_subscribe(conn, %{"user" => %{"nickname" => nick, "profile" => profil render(conn, "subscribe.html", %{ nickname: nick, avatar: nil, - error: "Something went wrong." + error: + Pleroma.Web.Gettext.dpgettext( + "static_pages", + "remote follow error message - unknown error", + "Something went wrong." + ) + }) + end + end + + def remote_subscribe(conn, %{"status" => %{"status_id" => id, "profile" => profile}}) do + with {:ok, %{"subscribe_address" => template}} <- WebFinger.finger(profile), + %Activity{} = activity <- Activity.get_by_id(id), + {:ok, ap_id} <- get_ap_id(activity) do + conn + |> Phoenix.Controller.redirect(external: String.replace(template, "{uri}", ap_id)) + else + _e -> + render(conn, "status_interact.html", %{ + status_id: id, + avatar: nil, + error: + Pleroma.Web.Gettext.dpgettext( + "static_pages", + "status interact error message - unknown error", + "Something went wrong." + ) }) end end @@ -83,6 +159,15 @@ def remote_interaction(%{body_params: %{ap_id: ap_id, profile: profile}} = conn, end end + defp get_ap_id(activity) do + object = Pleroma.Object.normalize(activity, fetch: false) + + case object do + %{data: %{"id" => ap_id}} -> {:ok, ap_id} + _ -> {:no_ap_id, nil} + end + end + def frontend_configurations(conn, _params) do render(conn, "frontend_configurations.json") end diff --git a/lib/pleroma/web/twitter_api/views/util_view.ex b/lib/pleroma/web/twitter_api/views/util_view.ex index a03020290..6ed74ee80 100644 --- a/lib/pleroma/web/twitter_api/views/util_view.ex +++ b/lib/pleroma/web/twitter_api/views/util_view.ex @@ -4,7 +4,9 @@ defmodule Pleroma.Web.TwitterAPI.UtilView do use Pleroma.Web, :view + import Phoenix.HTML import Phoenix.HTML.Form + import Phoenix.HTML.Link alias Pleroma.Config alias Pleroma.Web.Endpoint alias Pleroma.Web.Gettext diff --git a/test/pleroma/web/twitter_api/util_controller_test.exs b/test/pleroma/web/twitter_api/util_controller_test.exs index fb7da93f8..d669cd0fe 100644 --- a/test/pleroma/web/twitter_api/util_controller_test.exs +++ b/test/pleroma/web/twitter_api/util_controller_test.exs @@ -233,6 +233,102 @@ test "it renders form with error when user not found", %{conn: conn} do end end + describe "POST /main/ostatus - remote_subscribe/2 - with statuses" do + setup do: clear_config([:instance, :federating], true) + + test "renders subscribe form", %{conn: conn} do + user = insert(:user) + status = insert(:note_activity, %{user: user}) + status_id = status.id + + assert is_binary(status_id) + + response = + conn + |> post("/main/ostatus", %{"status_id" => status_id, "profile" => ""}) + |> response(:ok) + + refute response =~ "Could not find status" + assert response =~ "Interacting with" + end + + test "renders subscribe form with error when status not found", %{conn: conn} do + response = + conn + |> post("/main/ostatus", %{"status_id" => "somerandomid", "profile" => ""}) + |> response(:ok) + + assert response =~ "Could not find status" + refute response =~ "Interacting with" + end + + test "it redirect to webfinger url", %{conn: conn} do + user = insert(:user) + status = insert(:note_activity, %{user: user}) + status_id = status.id + status_ap_id = status.data["object"] + + assert is_binary(status_id) + assert is_binary(status_ap_id) + + user2 = insert(:user, ap_id: "shp@social.heldscal.la") + + conn = + conn + |> post("/main/ostatus", %{ + "status" => %{"status_id" => status_id, "profile" => user2.ap_id} + }) + + assert redirected_to(conn) == + "https://social.heldscal.la/main/ostatussub?profile=#{status_ap_id}" + end + + test "it renders form with error when status not found", %{conn: conn} do + user2 = insert(:user, ap_id: "shp@social.heldscal.la") + + response = + conn + |> post("/main/ostatus", %{ + "status" => %{"status_id" => "somerandomid", "profile" => user2.ap_id} + }) + |> response(:ok) + + assert response =~ "Something went wrong." + end + end + + describe "GET /main/ostatus - show_subscribe_form/2" do + setup do: clear_config([:instance, :federating], true) + + test "it works with users", %{conn: conn} do + user = insert(:user) + + response = + conn + |> get("/main/ostatus", %{"nickname" => user.nickname}) + |> response(:ok) + + refute response =~ "Could not find user" + assert response =~ "Remotely follow #{user.nickname}" + end + + test "it works with statuses", %{conn: conn} do + user = insert(:user) + status = insert(:note_activity, %{user: user}) + status_id = status.id + + assert is_binary(status_id) + + response = + conn + |> get("/main/ostatus", %{"status_id" => status_id}) + |> response(:ok) + + refute response =~ "Could not find status" + assert response =~ "Interacting with" + end + end + test "it returns new captcha", %{conn: conn} do with_mock Pleroma.Captcha, new: fn -> "test_captcha" end do From b8190f19dced7e8f23d7d4cce44a79c81f1e0cdf Mon Sep 17 00:00:00 2001 From: floatingghost Date: Sat, 10 Sep 2022 14:44:17 +0000 Subject: [PATCH 43/44] 2022.09 stable release chores (#206) Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/206 --- CHANGELOG.md | 4 +- SIGNING_KEY.pub | 2 + .../installation/verifying_otp_releases.md | 66 +++++++++++++++++++ mix.exs | 2 +- 4 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 SIGNING_KEY.pub create mode 100644 docs/docs/installation/verifying_otp_releases.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f63fa540c..5cd48b07c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## [Unreleased] +## 2022.09 ### Added - support for fedibird-fe, and non-breaking API parity for it to function @@ -13,9 +13,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - the ability to obfuscate domains in your MRF descriptions - automatic translation of statuses via DeepL or LibreTranslate - ability to edit posts +- ability to react with remote emoji ### Changed - MFM parsing is now done on the backend by a modified version of ilja's parser -> https://akkoma.dev/AkkomaGang/mfm-parser +- InlineQuotePolicy is now on by default ### Fixed - Compatibility with latest meilisearch diff --git a/SIGNING_KEY.pub b/SIGNING_KEY.pub new file mode 100644 index 000000000..7d8b48da8 --- /dev/null +++ b/SIGNING_KEY.pub @@ -0,0 +1,2 @@ +untrusted comment: Akkoma Signing Key public key +RWQRlw8Ex/uTbvo1wB1yK75tQ5nXKilB/vrKdkL41bgZHL9aKP+7fSS5 diff --git a/docs/docs/installation/verifying_otp_releases.md b/docs/docs/installation/verifying_otp_releases.md new file mode 100644 index 000000000..86dacfec2 --- /dev/null +++ b/docs/docs/installation/verifying_otp_releases.md @@ -0,0 +1,66 @@ +# Verifying OTP release integrity + +All stable OTP releases are cryptographically signed, to allow +you to verify the integrity if you choose to. + +Releases are signed with [Signify](https://man.openbsd.org/signify.1), +with [the public key in the main repository](https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/SIGNING_KEY.pub) + +Release URLs will always be of the form + +``` +https://akkoma-updates.s3-website.fr-par.scw.cloud/{branch}/akkoma-{flavour}.zip +``` + +Where branch is usually `stable` or `develop`, and `flavour` is +the one [that you detect on install](../otp_en/#detecting-flavour). + +So, for an AMD64 stable install, your update URL will be + +``` +https://akkoma-updates.s3-website.fr-par.scw.cloud/stable/akkoma-amd64.zip +``` + +To verify the integrity of this file, we have two helper files + +``` +# Checksums +https://akkoma-updates.s3-website.fr-par.scw.cloud/{branch}/akkoma-{flavour}.zip.sha256 + +# Signify signature of the hashes +https://akkoma-updates.s3-website.fr-par.scw.cloud/{branch}/akkoma-{flavour}.zip.sha256.sig +``` + +Thus, to upgrade manually, with integrity checking, consider the following script: + +```bash +#!/bin/bash +set -eo pipefail + +export FLAVOUR=amd64 +export BRANCH=stable + +# Fetch signing key +curl --silent https://akkoma.dev/AkkomaGang/akkoma/raw/branch/$BRANCH/SIGNING_KEY.pub -o AKKOMA_SIGNING_KEY.pub + +# Download zip file and sig files +wget -q https://akkoma-updates.s3-website.fr-par.scw.cloud/$BRANCH/akkoma-$FLAVOUR{.zip,.zip.sha256,.zip.sha256.sig} + +# Verify zip file's sha256 integrity +sha256sum --check akkoma-$FLAVOUR.zip.sha256 + +# Verify hash file's integrity +# Signify might be under the `signify` command, depending on your distribution +signify-openbsd -V -p AKKOMA_SIGNING_KEY.pub -m akkoma-$FLAVOUR.zip.sha256 + +# We're good, use that URL +echo "Update URL contents verified" +echo "use" +echo "./bin/pleroma_ctl update --zip-url https://akkoma-updates.s3-website.fr-par.scw.cloud/$BRANCH/akkoma-$FLAVOUR" +echo "to update your instance" + +# Clean up +rm akkoma-$FLAVOUR.zip +rm akkoma-$FLAVOUR.zip.sha256 +rm akkoma-$FLAVOUR.zip.sha256.sig +``` diff --git a/mix.exs b/mix.exs index ef038ce74..19e6fd045 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do def project do [ app: :pleroma, - version: version("3.1.0"), + version: version("3.2.0"), elixir: "~> 1.12", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(), From cad2745734fb8a0192b0ec9ecd3ff2dc3eaedde8 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sat, 10 Sep 2022 16:48:46 +0100 Subject: [PATCH 44/44] tmp: use akkoma build image --- .woodpecker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 32db2f1c5..955bbe7fd 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -112,7 +112,7 @@ pipeline: - /bin/sh /entrypoint.sh debian-bullseye: - image: elixir:1.13.4 + image: akkoma/debian <<: *on-release environment: MIX_ENV: prod