From 574f010bc8f6d5e6de48e0cc5a58d49cb44fa10f Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 10 Oct 2022 15:55:58 +0100 Subject: [PATCH 01/57] Extract deactivated users query to a join --- lib/pleroma/activity.ex | 5 ++--- lib/pleroma/web/activity_pub/activity_pub.ex | 1 + mix.exs | 2 +- mix.lock | 8 ++++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 01c9df53b..fef9aa874 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -368,9 +368,8 @@ defmodule Pleroma.Activity do end def restrict_deactivated_users(query) do - deactivated_users_query = from(u in User.Query.build(%{deactivated: true}), select: u.ap_id) - - from(activity in query, where: activity.actor not in subquery(deactivated_users_query)) + query + |> join(:inner, [activity], user in User, as: :user, on: activity.actor == user.ap_id and user.is_active == true) end defdelegate search(user, query, options \\ []), to: Pleroma.Search.DatabaseSearch diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index dcdc7085f..7ed617823 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -507,6 +507,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp fetch_paginated_optimized(query, opts, pagination) do # Note: tag-filtering funcs may apply "ORDER BY objects.id DESC", # and extra sorting on "activities.id DESC NULLS LAST" would worse the query plan + IO.inspect(Repo.to_sql(:all, query)) opts = Map.put(opts, :skip_extra_order, true) Pagination.fetch_paginated(query, opts, pagination) diff --git a/mix.exs b/mix.exs index c7e66b158..ea3c1b922 100644 --- a/mix.exs +++ b/mix.exs @@ -120,7 +120,7 @@ defmodule Pleroma.Mixfile do {:phoenix_pubsub, "~> 2.1"}, {:phoenix_ecto, "~> 4.4"}, {:ecto_enum, "~> 1.4"}, - {:ecto_sql, "~> 3.8.3"}, + {:ecto_sql, "~> 3.9.0"}, {:postgrex, ">= 0.16.3"}, {:oban, "~> 2.12.1"}, {:gettext, diff --git a/mix.lock b/mix.lock index 7eeb5c138..d0d20f7d3 100644 --- a/mix.lock +++ b/mix.lock @@ -26,10 +26,10 @@ "earmark": {:hex, :earmark, "1.4.26", "f0e3c3d5c278a6d448ad8c27ab0ecdec9c57a7710553138c56af220a6330a4fd", [:mix], [{:earmark_parser, "~> 1.4.26", [hex: :earmark_parser, repo: "hexpm", optional: false]}], "hexpm", "e1231882b56bece0692af33f0959f06c9cd580c2dc2ecb1dc9f16f2750fa78c5"}, "earmark_parser": {:hex, :earmark_parser, "1.4.26", "f4291134583f373c7d8755566122908eb9662df4c4b63caa66a0eabe06569b0a", [:mix], [], "hexpm", "48d460899f8a0c52c5470676611c01f64f3337bad0b26ddab43648428d94aabc"}, "eblurhash": {:hex, :eblurhash, "1.2.2", "7da4255aaea984b31bb71155f673257353b0e0554d0d30dcf859547e74602582", [:rebar3], [], "hexpm", "8c20ca00904de023a835a9dcb7b7762fed32264c85a80c3cafa85288e405044c"}, - "ecto": {:hex, :ecto, "3.8.4", "e06b8b87e62b27fea17fd2ff6041572ddd10339fd16cdf58446e402c6c90a74b", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f9244288b8d42db40515463a008cf3f4e0e564bb9c249fe87bf28a6d79fe82d4"}, + "ecto": {:hex, :ecto, "3.9.1", "67173b1687afeb68ce805ee7420b4261649d5e2deed8fe5550df23bab0bc4396", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c80bb3d736648df790f7f92f81b36c922d9dd3203ca65be4ff01d067f54eb304"}, "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"}, "ecto_psql_extras": {:hex, :ecto_psql_extras, "0.7.4", "5d43fd088d39a158c860b17e8d210669587f63ec89ea122a4654861c8c6e2db4", [:mix], [{:ecto_sql, "~> 3.4", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:postgrex, ">= 0.15.7", [hex: :postgrex, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1", [hex: :table_rex, repo: "hexpm", optional: false]}], "hexpm", "311db02f1b772e3d0dc7f56a05044b5e1499d78ed6abf38885e1ca70059449e5"}, - "ecto_sql": {:hex, :ecto_sql, "3.8.3", "a7d22c624202546a39d615ed7a6b784580391e65723f2d24f65941b4dd73d471", [:mix], [{:db_connection, "~> 2.5 or ~> 2.4.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.8.4", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0 or ~> 0.16.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "348cb17fb9e6daf6f251a87049eafcb57805e2892e5e6a0f5dea0985d367329b"}, + "ecto_sql": {:hex, :ecto_sql, "3.9.0", "2bb21210a2a13317e098a420a8c1cc58b0c3421ab8e3acfa96417dab7817918c", [:mix], [{:db_connection, "~> 2.5 or ~> 2.4.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.9.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a8f3f720073b8b1ac4c978be25fa7960ed7fd44997420c304a4a2e200b596453"}, "elasticsearch": {:git, "https://akkoma.dev/AkkomaGang/elasticsearch-elixir.git", "6cd946f75f6ab9042521a009d1d32d29a90113ca", [ref: "main"]}, "elixir_make": {:hex, :elixir_make, "0.6.3", "bc07d53221216838d79e03a8019d0839786703129599e9619f4ab74c8c096eac", [:mix], [], "hexpm", "f5cbd651c5678bcaabdbb7857658ee106b12509cd976c2c2fca99688e1daf716"}, "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"}, @@ -56,7 +56,7 @@ "httpoison": {:hex, :httpoison, "1.8.1", "df030d96de89dad2e9983f92b0c506a642d4b1f4a819c96ff77d12796189c63e", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "35156a6d678d6d516b9229e208942c405cf21232edd632327ecfaf4fd03e79e0"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "inet_cidr": {:hex, :inet_cidr, "1.0.4", "a05744ab7c221ca8e395c926c3919a821eb512e8f36547c062f62c4ca0cf3d6e", [:mix], [], "hexpm", "64a2d30189704ae41ca7dbdd587f5291db5d1dda1414e0774c29ffc81088c1bc"}, - "jason": {:hex, :jason, "1.3.0", "fa6b82a934feb176263ad2df0dbd91bf633d4a46ebfdffea0c8ae82953714946", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "53fc1f51255390e0ec7e50f9cb41e751c260d065dcba2bf0d08dc51a4002c2ac"}, + "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"}, "joken": {:hex, :joken, "2.5.0", "09be497d804b8115eb6f07615cef2e60c2a1008fb89dc0aef0d4c4b4609b99aa", [:mix], [{:jose, "~> 1.11.2", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "22b25c89617c5ed8ca7b31026340a25ea0f9ca7160f9706b79be9ed81fdf74e7"}, "jose": {:hex, :jose, "1.11.2", "f4c018ccf4fdce22c71e44d471f15f723cb3efab5d909ab2ba202b5bf35557b3", [:mix, :rebar3], [], "hexpm", "98143fbc48d55f3a18daba82d34fe48959d44538e9697c08f34200fa5f0947d2"}, "jumper": {:hex, :jumper, "1.0.1", "3c00542ef1a83532b72269fab9f0f0c82bf23a35e27d278bfd9ed0865cecabff", [:mix], [], "hexpm", "318c59078ac220e966d27af3646026db9b5a5e6703cb2aa3e26bcfaba65b7433"}, @@ -94,7 +94,7 @@ "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"}, "poison": {:hex, :poison, "3.1.0", "d9eb636610e096f86f25d9a46f35a9facac35609a7591b3be3326e99a0484665", [:mix], [], "hexpm", "fec8660eb7733ee4117b85f55799fd3833eb769a6df71ccf8903e8dc5447cfce"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, - "postgrex": {:hex, :postgrex, "0.16.3", "fac79a81a9a234b11c44235a4494d8565303fa4b9147acf57e48978a074971db", [:mix], [{:connection, "~> 1.1", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "aeaae1d2d1322da4e5fe90d241b0a564ce03a3add09d7270fb85362166194590"}, + "postgrex": {:hex, :postgrex, "0.16.5", "fcc4035cc90e23933c5d69a9cd686e329469446ef7abba2cf70f08e2c4b69810", [:mix], [{:connection, "~> 1.1", [hex: :connection, repo: "hexpm", optional: false]}, {:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "edead639dc6e882618c01d8fc891214c481ab9a3788dfe38dd5e37fd1d5fb2e8"}, "pot": {:hex, :pot, "1.0.2", "13abb849139fdc04ab8154986abbcb63bdee5de6ed2ba7e1713527e33df923dd", [:rebar3], [], "hexpm", "78fe127f5a4f5f919d6ea5a2a671827bd53eb9d37e5b4128c0ad3df99856c2e0"}, "quack": {:hex, :quack, "0.1.1", "cca7b4da1a233757fdb44b3334fce80c94785b3ad5a602053b7a002b5a8967bf", [:mix], [{:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}, {:tesla, "~> 1.2.0", [hex: :tesla, repo: "hexpm", optional: false]}], "hexpm", "d736bfa7444112eb840027bb887832a0e403a4a3437f48028c3b29a2dbbd2543"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, From ca9e6ffc554f7bf82cfb2c2093a0fcd283349f46 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 10 Oct 2022 16:45:02 +0100 Subject: [PATCH 02/57] Use inner lateral join to not get dropped in :total --- lib/pleroma/activity.ex | 2 +- lib/pleroma/web/activity_pub/activity_pub.ex | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index fef9aa874..12601b4af 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -369,7 +369,7 @@ defmodule Pleroma.Activity do def restrict_deactivated_users(query) do query - |> join(:inner, [activity], user in User, as: :user, on: activity.actor == user.ap_id and user.is_active == true) + |> join(:inner_lateral, [activity], active in fragment("SELECT is_active from users WHERE ap_id = ? AND is_active = TRUE", activity.actor)) end defdelegate search(user, query, options \\ []), to: Pleroma.Search.DatabaseSearch diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 7ed617823..dcdc7085f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -507,7 +507,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp fetch_paginated_optimized(query, opts, pagination) do # Note: tag-filtering funcs may apply "ORDER BY objects.id DESC", # and extra sorting on "activities.id DESC NULLS LAST" would worse the query plan - IO.inspect(Repo.to_sql(:all, query)) opts = Map.put(opts, :skip_extra_order, true) Pagination.fetch_paginated(query, opts, pagination) From 8af50dea366be35c5c29faa9023e9f9edc8693ae Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 10 Oct 2022 17:13:42 +0100 Subject: [PATCH 03/57] format --- lib/pleroma/activity.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 12601b4af..b01a838d8 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -369,7 +369,14 @@ defmodule Pleroma.Activity do def restrict_deactivated_users(query) do query - |> join(:inner_lateral, [activity], active in fragment("SELECT is_active from users WHERE ap_id = ? AND is_active = TRUE", activity.actor)) + |> join( + :inner_lateral, + [activity], + active in fragment( + "SELECT is_active from users WHERE ap_id = ? AND is_active = TRUE", + activity.actor + ) + ) end defdelegate search(user, query, options \\ []), to: Pleroma.Search.DatabaseSearch From cb9b0d3720cdfe5e9987d70d0822032fef7a3d8a Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Tue, 11 Oct 2022 11:40:43 +0100 Subject: [PATCH 04/57] optimise notifications query --- CHANGELOG.md | 5 ++++ lib/pleroma/following_relationship.ex | 24 -------------------- lib/pleroma/notification.ex | 19 +++++++++++++++- lib/pleroma/web/mastodon_api/mastodon_api.ex | 15 ++++++++---- test/pleroma/notification_test.exs | 12 ---------- 5 files changed, 33 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ece6af0d2..8a675a32e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## Unreleased + +## Changes +- Follows no longer override domain blocks, a domain block is final + ## 2022.10 ### Added diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index b101b9ee7..42db9463d 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -240,30 +240,6 @@ defmodule Pleroma.FollowingRelationship do end) end - @doc """ - For a query with joined activity, - keeps rows where activity's actor is followed by user -or- is NOT domain-blocked by user. - """ - def keep_following_or_not_domain_blocked(query, user) do - where( - query, - [_, activity], - fragment( - # "(actor's domain NOT in domain_blocks) OR (actor IS in followed AP IDs)" - """ - NOT (substring(? from '.*://([^/]*)') = ANY(?)) OR - ? = ANY(SELECT ap_id FROM users AS u INNER JOIN following_relationships AS fr - ON u.id = fr.following_id WHERE fr.follower_id = ? AND fr.state = ?) - """, - activity.actor, - ^user.domain_blocks, - activity.actor, - ^User.binary_id(user.id), - ^accept_state_code() - ) - ) - end - defp validate_not_self_relationship(%Changeset{} = changeset) do changeset |> validate_follower_id_following_id_inequality() diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 593448713..3995be01f 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -138,7 +138,24 @@ defmodule Pleroma.Notification do query |> where([n, a], a.actor not in ^blocked_ap_ids) - |> FollowingRelationship.keep_following_or_not_domain_blocked(user) + |> restrict_domain_blocked(user) + end + + defp restrict_domain_blocked(query, user) do + where( + query, + [_, activity], + fragment( + # "(actor's domain NOT in domain_blocks)" + """ + NOT ( + substring(? from '.*://([^/]*)') = ANY(?) + ) + """, + activity.actor, + ^user.domain_blocks + ) + ) end defp exclude_blockers(query, user) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index 23846b36a..69bc2f0d6 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -63,11 +63,16 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do def get_notifications(user, params \\ %{}) do options = cast_params(params) - user - |> Notification.for_user_query(options) - |> restrict(:include_types, options) - |> restrict(:exclude_types, options) - |> restrict(:account_ap_id, options) + query = + user + |> Notification.for_user_query(options) + |> restrict(:include_types, options) + |> restrict(:exclude_types, options) + |> restrict(:account_ap_id, options) + + IO.inspect(Pleroma.Repo.to_sql(:all, query)) + + query |> Pagination.fetch_paginated(params) end diff --git a/test/pleroma/notification_test.exs b/test/pleroma/notification_test.exs index 68330465b..721836a2c 100644 --- a/test/pleroma/notification_test.exs +++ b/test/pleroma/notification_test.exs @@ -1149,18 +1149,6 @@ defmodule Pleroma.NotificationTest do assert Notification.for_user(user) == [] end - test "it returns notifications for domain-blocked but followed user" do - user = insert(:user) - blocked = insert(:user, ap_id: "http://some-domain.com") - - {:ok, user} = User.block_domain(user, "some-domain.com") - {:ok, _, _} = User.follow(user, blocked) - - {:ok, _activity} = CommonAPI.post(blocked, %{status: "hey @#{user.nickname}"}) - - assert length(Notification.for_user(user)) == 1 - end - test "it doesn't return notifications for muted thread", %{user: user} do another_user = insert(:user) From 856c57208bbf0a4a88f33f922a3a96b037285d89 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Tue, 11 Oct 2022 14:30:08 +0100 Subject: [PATCH 05/57] Ensure deletes are handled after everything else --- CHANGELOG.md | 1 + config/config.exs | 5 ++++- lib/mix/tasks/pleroma/user.ex | 6 ++++++ lib/pleroma/web/activity_pub/side_effects.ex | 2 -- lib/pleroma/web/federator.ex | 15 +++++++++++---- lib/pleroma/web/mastodon_api/mastodon_api.ex | 15 +++++---------- 6 files changed, 27 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a675a32e..cbe09d598 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Changes - Follows no longer override domain blocks, a domain block is final +- Deletes are now the lowest priority to publish and will be handled after creates ## 2022.10 diff --git a/config/config.exs b/config/config.exs index d7005770e..5eb82cd33 100644 --- a/config/config.exs +++ b/config/config.exs @@ -569,7 +569,10 @@ config :pleroma, Oban, mute_expire: 5, search_indexing: 10 ], - plugins: [Oban.Plugins.Pruner], + plugins: [ + Oban.Plugins.Pruner, + {Oban.Plugins.Reindexer, schedule: "@weekly"} + ], crontab: [ {"0 0 * * 0", Pleroma.Workers.Cron.DigestEmailsWorker}, {"0 0 * * *", Pleroma.Workers.Cron.NewUsersDigestWorker} diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index f420d68bb..50c3fd7ce 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -538,6 +538,12 @@ defmodule Mix.Tasks.Pleroma.User do end end + def run(["convert_id", id]) do + {:ok, uuid} = FlakeId.Ecto.Type.dump(id) + {:ok, raw_id} = Ecto.UUID.load(uuid) + shell_info(raw_id) + end + defp refetch_public_keys(query) do query |> Pleroma.Repo.chunk_stream(50, :batches) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 43b1b089b..c3258c75b 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -323,8 +323,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end if result == :ok do - Notification.create_notifications(object) - # Only remove from index when deleting actual objects, not users or anything else with %Pleroma.Object{} <- deleted_object do Pleroma.Search.remove_from_index(deleted_object) diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index bc61130f1..770044de2 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -53,12 +53,19 @@ defmodule Pleroma.Web.Federator do @impl true def publish(%{data: %{"object" => object}} = activity) when is_map(object) or is_list(object) do - PublisherWorker.enqueue("publish", %{ - "activity_id" => activity.id, - "object_data" => Jason.encode!(object) - }) + PublisherWorker.enqueue( + "publish", + %{ + "activity_id" => activity.id, + "object_data" => Jason.encode!(object) + }, + priority: publish_priority(activity) + ) end + defp publish_priority(%{type: "Delete"}), do: 3 + defp publish_priority(_), do: 0 + # Job Worker Callbacks @spec perform(atom(), module(), any()) :: {:ok, any()} | {:error, any()} diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index 69bc2f0d6..23846b36a 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -63,16 +63,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do def get_notifications(user, params \\ %{}) do options = cast_params(params) - query = - user - |> Notification.for_user_query(options) - |> restrict(:include_types, options) - |> restrict(:exclude_types, options) - |> restrict(:account_ap_id, options) - - IO.inspect(Pleroma.Repo.to_sql(:all, query)) - - query + user + |> Notification.for_user_query(options) + |> restrict(:include_types, options) + |> restrict(:exclude_types, options) + |> restrict(:account_ap_id, options) |> Pagination.fetch_paginated(params) end From 03662501c3d1aea06526d76177c002e6b8c72766 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 14 Oct 2022 11:48:32 +0100 Subject: [PATCH 06/57] Check that the signature matches the creator --- lib/pleroma/web/activity_pub/publisher.ex | 4 +- .../mapped_signature_to_identity_plug.ex | 54 +++++++++++++++++-- .../activity_pub_controller_test.exs | 35 ++++++++++++ ...mapped_signature_to_identity_plug_test.exs | 22 ++++++++ 4 files changed, 110 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index ed67a060d..b187d3a48 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -108,8 +108,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do Config.get([:mrf_simple, :reject], []) end - defp should_federate?(inbox) do - %{host: host} = URI.parse(inbox) + def should_federate?(url) do + %{host: host} = URI.parse(url) quarantined_instances = blocked_instances() diff --git a/lib/pleroma/web/plugs/mapped_signature_to_identity_plug.ex b/lib/pleroma/web/plugs/mapped_signature_to_identity_plug.ex index 58cb0316a..a73def682 100644 --- a/lib/pleroma/web/plugs/mapped_signature_to_identity_plug.ex +++ b/lib/pleroma/web/plugs/mapped_signature_to_identity_plug.ex @@ -19,6 +19,7 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do def call(%{assigns: %{valid_signature: true}, params: %{"actor" => actor}} = conn, _opts) do with actor_id <- Utils.get_ap_id(actor), {:user, %User{} = user} <- {:user, user_from_key_id(conn)}, + {:federate, true} <- {:federate, should_federate?(user)}, {:user_match, true} <- {:user_match, user.ap_id == actor_id} do conn |> assign(:user, user) @@ -27,33 +28,70 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do {:user_match, false} -> Logger.debug("Failed to map identity from signature (payload actor mismatch)") Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}, actor=#{inspect(actor)}") - assign(conn, :valid_signature, false) + + conn + |> assign(:valid_signature, false) # remove me once testsuite uses mapped capabilities instead of what we do now {:user, nil} -> Logger.debug("Failed to map identity from signature (lookup failure)") Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}, actor=#{actor}") + conn + |> assign(:valid_signature, false) + + {:federate, false} -> + Logger.debug("Identity from signature is instance blocked") + Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}, actor=#{actor}") + + conn + |> assign(:valid_signature, false) end end # no payload, probably a signed fetch def call(%{assigns: %{valid_signature: true}} = conn, _opts) do - with %User{} = user <- user_from_key_id(conn) do + with %User{} = user <- user_from_key_id(conn), + {:federate, true} <- {:federate, should_federate?(user)} do conn |> assign(:user, user) |> AuthHelper.skip_oauth() else + {:federate, false} -> + Logger.debug("Identity from signature is instance blocked") + Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}") + + conn + |> assign(:valid_signature, false) + + nil -> + Logger.debug("Failed to map identity from signature (lookup failure)") + Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}") + + only_permit_user_routes(conn) + _ -> Logger.debug("Failed to map identity from signature (no payload actor mismatch)") Logger.debug("key_id=#{inspect(key_id_from_conn(conn))}") - assign(conn, :valid_signature, false) + + conn + |> assign(:valid_signature, false) end end # no signature at all def call(conn, _opts), do: conn + defp only_permit_user_routes(%{path_info: ["users", _]} = conn) do + conn + |> assign(:limited_ap, true) + end + + defp only_permit_user_routes(conn) do + conn + |> assign(:valid_signature, false) + end + defp key_id_from_conn(conn) do with %{"keyId" => key_id} <- HTTPSignatures.signature_for_conn(conn), {:ok, ap_id} <- Signature.key_id_to_actor_id(key_id) do @@ -73,4 +111,14 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do nil end end + + defp should_federate?(%User{ap_id: ap_id}), do: should_federate?(ap_id) + + defp should_federate?(ap_id) do + if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do + Pleroma.Web.ActivityPub.Publisher.should_federate?(ap_id) + else + true + end + end end diff --git a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs index e209bb46b..424b87b20 100644 --- a/test/pleroma/web/activity_pub/activity_pub_controller_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_controller_test.exs @@ -559,6 +559,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn = conn |> assign(:valid_signature, true) + |> put_req_header( + "signature", + "keyId=\"http://mastodon.example.org/users/admin/main-key\"" + ) |> put_req_header("content-type", "application/activity+json") |> post("/inbox", data) @@ -589,6 +593,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn = conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{user.ap_id}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/inbox", data) @@ -602,12 +607,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do data = File.read!("test/fixtures/mastodon-post-activity.json") |> Jason.decode!() sender_url = data["actor"] + sender = insert(:user, ap_id: data["actor"]) + Instances.set_consistently_unreachable(sender_url) refute Instances.reachable?(sender_url) conn = conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{sender.ap_id}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/inbox", data) @@ -632,6 +640,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert "ok" == conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{followed_relay.ap_id}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/inbox", accept) |> json_response(200) @@ -698,6 +707,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do actor = "https://example.com/users/lain" + insert(:user, + ap_id: actor, + featured_address: "https://example.com/users/lain/collections/featured" + ) + Tesla.Mock.mock(fn %{ method: :get, @@ -743,6 +757,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert "ok" == conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{actor}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/inbox", data) |> json_response(200) @@ -750,6 +765,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) assert Activity.get_by_ap_id(data["id"]) user = User.get_cached_by_ap_id(data["actor"]) + assert user.pinned_objects[data["object"]] data = %{ @@ -764,6 +780,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert "ok" == conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{actor}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/inbox", data) |> json_response(200) @@ -790,6 +807,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do actor = "https://example.com/users/lain" + sender = + insert(:user, + ap_id: actor, + featured_address: "https://example.com/users/lain/collections/featured" + ) + Tesla.Mock.mock(fn %{ method: :get, @@ -844,6 +867,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert "ok" == conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{sender.ap_id}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/inbox", data) |> json_response(200) @@ -863,6 +887,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert "ok" == conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{actor}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/inbox", data) |> json_response(200) @@ -894,6 +919,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn = conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{data["actor"]}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/users/#{user.nickname}/inbox", data) @@ -915,6 +941,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn = conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{data["actor"]}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/users/#{user.nickname}/inbox", data) @@ -936,6 +963,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn = conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{data["actor"]}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/users/#{user.nickname}/inbox", data) @@ -960,6 +988,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn = conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{data["actor"]}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/users/#{user.nickname}/inbox", data) @@ -987,6 +1016,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn = conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{announcer.ap_id}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/users/#{user.nickname}/inbox", data) @@ -1017,6 +1047,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn = conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{actor.ap_id}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/users/#{recipient.nickname}/inbox", data) @@ -1063,6 +1094,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn = conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{data["actor"]}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/users/#{user.nickname}/inbox", data) @@ -1101,6 +1133,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{actor.ap_id}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/users/#{recipient.nickname}/inbox", data) |> json_response(200) @@ -1193,6 +1226,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{actor.ap_id}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/users/#{reported_user.nickname}/inbox", data) |> json_response(200) @@ -1248,6 +1282,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do conn |> assign(:valid_signature, true) + |> put_req_header("signature", "keyId=\"#{remote_actor}/main-key\"") |> put_req_header("content-type", "application/activity+json") |> post("/users/#{reported_user.nickname}/inbox", data) |> json_response(200) diff --git a/test/pleroma/web/plugs/mapped_signature_to_identity_plug_test.exs b/test/pleroma/web/plugs/mapped_signature_to_identity_plug_test.exs index 00ce6492d..21c574ba3 100644 --- a/test/pleroma/web/plugs/mapped_signature_to_identity_plug_test.exs +++ b/test/pleroma/web/plugs/mapped_signature_to_identity_plug_test.exs @@ -9,6 +9,8 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlugTest do import Tesla.Mock import Plug.Conn + import Pleroma.Tests.Helpers, only: [clear_config: 2] + setup do mock(fn env -> apply(HttpRequestMock, :request, [env]) end) :ok @@ -47,6 +49,26 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlugTest do assert %{valid_signature: false} == conn.assigns end + test "it considers a mapped identity to be invalid when the associated instance is blocked" do + clear_config([:activitypub, :authorized_fetch_mode], true) + + clear_config([:mrf_simple, :reject], [ + {"mastodon.example.org", "anime is banned"} + ]) + + on_exit(fn -> + Pleroma.Config.put([:activitypub, :authorized_fetch_mode], false) + Pleroma.Config.put([:mrf_simple, :reject], []) + end) + + conn = + build_conn(:post, "/doesntmattter", %{"actor" => "http://mastodon.example.org/users/admin"}) + |> set_signature("http://mastodon.example.org/users/admin") + |> MappedSignatureToIdentityPlug.call(%{}) + + assert %{valid_signature: false} == conn.assigns + end + @tag skip: "known breakage; the testsuite presently depends on it" test "it considers a mapped identity to be invalid when the identity cannot be found" do conn = From 60b3c8d17be1ae27bc1f5b4b61f6f1be69c688bf Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 14 Oct 2022 12:49:35 +0100 Subject: [PATCH 07/57] bump version --- CHANGELOG.md | 1 + mix.exs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbe09d598..44d518528 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - Emoji updated to latest 15.0 draft - **Breaking**: `/api/v1/pleroma/backups` endpoints now requires `read:backups` scope instead of `read:accounts` +- Verify that the signature on posts is not domain blocked, and belongs to the correct user ### Fixed - OAuthPlug no longer joins with the database every call and uses the user cache diff --git a/mix.exs b/mix.exs index ea3c1b922..108930f2c 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do def project do [ app: :pleroma, - version: version("3.3.0"), + version: version("3.3.1"), elixir: "~> 1.12", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(), From 66f913355abb744c17f1b56d539bfd04b005070d Mon Sep 17 00:00:00 2001 From: floatingghost Date: Sun, 16 Oct 2022 19:25:54 +0000 Subject: [PATCH 08/57] Docker builds (#231) Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/231 --- .dockerignore | 6 +- .gitignore | 10 ++ CHANGELOG.md | 3 + Dockerfile | 52 +++----- config/docker.exs | 16 +-- docker-compose.yml | 61 +++++++++ docker-entrypoint.sh | 4 +- docker-resources/Caddyfile.example | 14 ++ docker-resources/build.sh | 4 + docker-resources/database/Dockerfile | 10 ++ docker-resources/env.example | 4 + docker-resources/manage.sh | 3 + docs/docs/installation/docker_en.md | 160 +++++++++++++++++++++++ docs/docs/installation/frontends.include | 6 + 14 files changed, 306 insertions(+), 47 deletions(-) create mode 100644 docker-compose.yml create mode 100644 docker-resources/Caddyfile.example create mode 100755 docker-resources/build.sh create mode 100644 docker-resources/database/Dockerfile create mode 100644 docker-resources/env.example create mode 100755 docker-resources/manage.sh create mode 100644 docs/docs/installation/docker_en.md diff --git a/.dockerignore b/.dockerignore index 2b5f1abff..1d4f80bdf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,12 +6,12 @@ COPYING *file elixir_buildpack.config test/ -instance/ -_build -deps test benchmarks docs/site +docker-db +uploads +instance # Required to get version !.git diff --git a/.gitignore b/.gitignore index 8fa79b68f..f9de4ed49 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,13 @@ secret /instance /priv/ssh_keys vm.args +.cache/ +.hex/ +.mix/ +.psql_history +docker-resources/Dockerfile +docker-resources/Caddyfile +pgdata # Prevent committing custom emojis /priv/static/emoji/custom/* @@ -65,3 +72,6 @@ pleroma.iml # Generated documentation docs/site + +# docker stuff +docker-db diff --git a/CHANGELOG.md b/CHANGELOG.md index 44d518528..f1c2e4460 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +## Added +- Officially supported docker release + ## Changes - Follows no longer override domain blocks, a domain block is final - Deletes are now the lowest priority to publish and will be handled after creates diff --git a/Dockerfile b/Dockerfile index 42ba9616b..730730ce3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,9 @@ -FROM elixir:1.13.4-alpine as build - -COPY . . +FROM hexpm/elixir:1.13.4-erlang-24.3.4.5-alpine-3.15.6 ENV MIX_ENV=prod -RUN apk add git gcc g++ musl-dev make cmake file-dev &&\ - echo "import Config" > config/prod.secret.exs &&\ - mix local.hex --force &&\ - mix local.rebar --force &&\ - mix deps.get --only prod &&\ - mkdir release &&\ - mix release --path release - -FROM alpine:3.16 - -ARG BUILD_DATE -ARG VCS_REF +ARG HOME=/opt/akkoma +ARG DATA=/var/lib/akkoma LABEL org.opencontainers.image.title="akkoma" \ org.opencontainers.image.description="Akkoma for Docker" \ @@ -26,25 +14,21 @@ LABEL org.opencontainers.image.title="akkoma" \ org.opencontainers.image.revision=$VCS_REF \ org.opencontainers.image.created=$BUILD_DATE -ARG HOME=/opt/akkoma -ARG DATA=/var/lib/akkoma - -RUN apk update &&\ - apk add exiftool ffmpeg imagemagick libmagic ncurses postgresql-client &&\ - adduser --system --shell /bin/false --home ${HOME} akkoma &&\ - mkdir -p ${DATA}/uploads &&\ - mkdir -p ${DATA}/static &&\ - chown -R akkoma ${DATA} &&\ - mkdir -p /etc/akkoma &&\ - chown -R akkoma /etc/akkoma - -USER akkoma - -COPY --from=build --chown=akkoma:0 /release ${HOME} - -COPY ./config/docker.exs /etc/akkoma/config.exs -COPY ./docker-entrypoint.sh ${HOME} +RUN apk add git gcc g++ musl-dev make cmake file-dev exiftool ffmpeg imagemagick libmagic ncurses postgresql-client EXPOSE 4000 -ENTRYPOINT ["/opt/akkoma/docker-entrypoint.sh"] +ARG UID=1000 +ARG GID=1000 +ARG UNAME=akkoma + +RUN addgroup -g $GID $UNAME +RUN adduser -u $UID -G $UNAME -D -h $HOME $UNAME + +WORKDIR /opt/akkoma + +USER $UNAME +RUN mix local.hex --force &&\ + mix local.rebar --force + +CMD ["/opt/akkoma/docker-entrypoint.sh"] diff --git a/config/docker.exs b/config/docker.exs index f9f27d141..fc24a4d67 100644 --- a/config/docker.exs +++ b/config/docker.exs @@ -24,11 +24,11 @@ config :pleroma, Pleroma.Repo, config :web_push_encryption, :vapid_details, subject: "mailto:#{System.get_env("NOTIFY_EMAIL")}" config :pleroma, :database, rum_enabled: false -config :pleroma, :instance, static_dir: "/var/lib/pleroma/static" -config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/pleroma/uploads" +config :pleroma, :instance, static_dir: "/var/lib/akkoma/static" +config :pleroma, Pleroma.Uploaders.Local, uploads: "/var/lib/akkoma/uploads" # We can't store the secrets in this file, since this is baked into the docker image -if not File.exists?("/var/lib/pleroma/secret.exs") do +if not File.exists?("/var/lib/akkoma/secret.exs") do secret = :crypto.strong_rand_bytes(64) |> Base.encode64() |> binary_part(0, 64) signing_salt = :crypto.strong_rand_bytes(8) |> Base.encode64() |> binary_part(0, 8) {web_push_public_key, web_push_private_key} = :crypto.generate_key(:ecdh, :prime256v1) @@ -52,16 +52,16 @@ if not File.exists?("/var/lib/pleroma/secret.exs") do web_push_private_key: Base.url_encode64(web_push_private_key, padding: false) ) - File.write("/var/lib/pleroma/secret.exs", secret_file) + File.write("/var/lib/akkoma/secret.exs", secret_file) end -import_config("/var/lib/pleroma/secret.exs") +import_config("/var/lib/akkoma/secret.exs") # For additional user config -if File.exists?("/var/lib/pleroma/config.exs"), - do: import_config("/var/lib/pleroma/config.exs"), +if File.exists?("/var/lib/akkoma/config.exs"), + do: import_config("/var/lib/akkoma/config.exs"), else: - File.write("/var/lib/pleroma/config.exs", """ + File.write("/var/lib/akkoma/config.exs", """ import Config # For additional configuration outside of environmental variables diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..b8da67a22 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,61 @@ +version: "3.7" + +services: + db: + image: akkoma-db:latest + build: ./docker-resources/database + restart: unless-stopped + user: ${DOCKER_USER} + environment: { + # This might seem insecure but is usually not a problem. + # You should leave this at the "akkoma" default. + # The DB is only reachable by containers in the same docker network, + # and is not exposed to the open internet. + # + # If you do change this, remember to update "config.exs". + POSTGRES_DB: akkoma, + POSTGRES_USER: akkoma, + POSTGRES_PASSWORD: akkoma, + } + env_file: + - .env + volumes: + - type: bind + source: ./pgdata + target: /var/lib/postgresql/data + + akkoma: + image: akkoma:latest + build: . + restart: unless-stopped + env_file: + - .env + links: + - db + ports: [ + # Uncomment/Change port mappings below as needed. + # The left side is your host machine, the right one is the akkoma container. + # You can prefix the left side with an ip. + + # Webserver (for reverse-proxies outside of docker) + # If you use a dockerized proxy, you can leave this commented + # and use a container link instead. + "127.0.0.1:4000:4000", + ] + volumes: + - .:/opt/akkoma + + # Uncomment the following if you want to use a reverse proxy + #proxy: + # image: caddy:2-alpine + # restart: unless-stopped + # links: + # - akkoma + # ports: [ + # "443:443", + # "80:80" + # ] + # volumes: + # - ./docker-resources/Caddyfile:/etc/caddy/Caddyfile + # - ./caddy-data:/data + # - ./caddy-config:/config \ No newline at end of file diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index f56f8c50a..778ef08e2 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -8,7 +8,7 @@ while ! pg_isready -U ${DB_USER:-pleroma} -d postgres://${DB_HOST:-db}:5432/${DB done echo "-- Running migrations..." -$HOME/bin/pleroma_ctl migrate +mix ecto.migrate echo "-- Starting!" -exec $HOME/bin/pleroma start +mix phx.server diff --git a/docker-resources/Caddyfile.example b/docker-resources/Caddyfile.example new file mode 100644 index 000000000..47b8c7c74 --- /dev/null +++ b/docker-resources/Caddyfile.example @@ -0,0 +1,14 @@ +# default docker Caddyfile config for Akkoma +# +# Simple installation instructions: +# 1. Replace 'example.tld' with your instance's domain wherever it appears. + +example.tld { + log { + output file /var/log/caddy/akkoma.log + } + + encode gzip + + reverse_proxy akkoma:4000 +} diff --git a/docker-resources/build.sh b/docker-resources/build.sh new file mode 100755 index 000000000..daa653da6 --- /dev/null +++ b/docker-resources/build.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +docker-compose build --build-arg UID=$(id -u) --build-arg GID=$(id -g) akkoma +docker-compose build --build-arg UID=$(id -u) --build-arg GID=$(id -g) db diff --git a/docker-resources/database/Dockerfile b/docker-resources/database/Dockerfile new file mode 100644 index 000000000..2a38dd16b --- /dev/null +++ b/docker-resources/database/Dockerfile @@ -0,0 +1,10 @@ +FROM postgres:14-alpine + +ARG UID=1000 +ARG GID=1000 +ARG UNAME=akkoma + +RUN addgroup -g $GID $UNAME +RUN adduser -u $UID -G $UNAME -D -h $HOME $UNAME + +USER akkoma diff --git a/docker-resources/env.example b/docker-resources/env.example new file mode 100644 index 000000000..d6cf0c7b8 --- /dev/null +++ b/docker-resources/env.example @@ -0,0 +1,4 @@ +MIX_ENV=prod +DB_NAME=akkoma +DB_USER=akkoma +DB_PASS=akkoma diff --git a/docker-resources/manage.sh b/docker-resources/manage.sh new file mode 100755 index 000000000..944f5e2e2 --- /dev/null +++ b/docker-resources/manage.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +docker-compose run --rm akkoma $@ diff --git a/docs/docs/installation/docker_en.md b/docs/docs/installation/docker_en.md new file mode 100644 index 000000000..ecb927649 --- /dev/null +++ b/docs/docs/installation/docker_en.md @@ -0,0 +1,160 @@ +# Installing in Docker + +## Installation + +This guide will show you how to get akkoma working in a docker container, +if you want isolation, or if you run a distribution not supported by the OTP +releases. + +### Prepare the system + +* Install docker and docker-compose + * [Docker](https://docs.docker.com/engine/install/) + * [Docker-compose](https://docs.docker.com/compose/install/) + * This will usually just be a repository installation and a package manager invocation. +* Clone the akkoma repository + * `git clone https://akkoma.dev/AkkomaGang/akkoma.git -b stable` + * `cd akkoma` + +### Set up basic configuration + +```bash +cp docker-resources/env.example .env +echo "DOCKER_USER=$(id -u):$(id -g)" >> .env +``` + +This probably won't need to be changed, it's only there to set basic environment +variables for the docker-compose file. + +### Building the container + +The container provided is a thin wrapper around akkoma's dependencies, +it does not contain the code itself. This is to allow for easy updates +and debugging if required. + +```bash +./docker-resources/build.sh +``` + +This will generate a container called `akkoma` which we can use +in our compose environment. + +### Generating your instance + +```bash +mkdir pgdata +# if you want to use caddy +mkdir caddy-data +mkdir caddy-config +./docker-resources/manage.sh mix deps.get +./docker-resources/manage.sh mix compile +./docker-resources/manage.sh mix pleroma.instance gen +``` + +This will ask you a few questions - the defaults are fine for most things, +the database hostname is `db`, and you will want to set the ip to `0.0.0.0`. + +Now we'll want to copy over the config it just created + +```bash +cp config/generated_config.exs config/prod.secret.exs +``` + +### Setting up the database + +We need to run a few commands on the database container, this isn't too bad + +```bash +docker-compose run --rm --user akkoma -d db +# Note down the name it gives here, it will be something like akkoma_db_run +docker-compose run --rm akkoma psql -h db -U akkoma -f config/setup_db.psql +docker stop akkoma_db_run # Replace with the name you noted down +``` + +Now we can actually run our migrations + +```bash +./docker-resources/manage.sh mix ecto.migrate +# this will recompile your files at the same time, since we changed the config +``` + +### Start the server + +We're going to run it in the foreground on the first run, just to make sure +everything start up. + +```bash +docker-compose up +``` + +If everything went well, you should be able to access your instance at http://localhost:4000 + +You can `ctrl-c` out of the docker-compose now to shutdown the server. + +### Running in the background + +```bash +docker-compose up -d +``` + +### Create your first user + +If your instance is up and running, you can create your first user with administrative rights with the following task: + +```shell +./docker-resources/manage.sh mix pleroma.user new MY_USERNAME MY_EMAIL@SOMEWHERE --admin +``` + +And follow the prompts + +### Reverse proxies + +This is a tad more complex in docker than on the host itself. It + +You've got two options. + +#### Running caddy in a container + +This is by far the easiest option. It'll handle HTTPS and all that for you. + +```bash +cp docker-resources/Caddyfile.example docker-resources/Caddyfile +``` + +Then edit the TLD in your caddyfile to the domain you're serving on. + +Uncomment the `caddy` section in the docker-compose file, +then run `docker-compose up -d` again. + +#### Running a reverse proxy on the host + +If you want, you can also run the reverse proxy on the host. This is a bit more complex, but it's also more flexible. + +Follow the guides for source install for your distribution of choice, or adapt +as needed. Your standard setup can be found in the [Debian Guide](../debian_based_en/#nginx) + +### You're done! + +All that's left is to set up your frontends. + +The standard from-source commands will apply to you, just make sure you +prefix them with `./docker-resources/manage.sh`! + +{! installation/frontends.include !} + +### Updating Docker Installs + +```bash +git pull +./docker-resources/build.sh +./docker-resources/manage.sh mix deps.get +./docker-resources/manage.sh mix compile +./docker-resources/manage.sh mix ecto.migrate +docker-compose restart akkoma +``` + +#### Further reading + +{! installation/further_reading.include !} + +{! support.include !} diff --git a/docs/docs/installation/frontends.include b/docs/docs/installation/frontends.include index 585be71ae..6da4018a9 100644 --- a/docs/docs/installation/frontends.include +++ b/docs/docs/installation/frontends.include @@ -21,5 +21,11 @@ For most installations, the following will suffice: mix pleroma.frontend install admin-fe --ref stable ``` +=== "Docker" + ```sh + ./docker-resources/manage.sh mix pleroma.frontend install pleroma-fe --ref stable + ./docker-resources/manage.sh mix pleroma.frontend install admin-fe --ref stable + ``` + For more customised installations, refer to [Frontend Management](../../configuration/frontend_management) From deba1d25f5a953fad86abe88c03c729a251d3b22 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 17 Oct 2022 16:29:36 +0100 Subject: [PATCH 09/57] add DB restart to docker file --- docs/docs/installation/docker_en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/installation/docker_en.md b/docs/docs/installation/docker_en.md index ecb927649..bf67f4d86 100644 --- a/docs/docs/installation/docker_en.md +++ b/docs/docs/installation/docker_en.md @@ -150,7 +150,7 @@ git pull ./docker-resources/manage.sh mix deps.get ./docker-resources/manage.sh mix compile ./docker-resources/manage.sh mix ecto.migrate -docker-compose restart akkoma +docker-compose restart akkoma db ``` #### Further reading From 5231d436d11a459cae087d18e03c8411ef775bc1 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Tue, 18 Oct 2022 16:16:55 +0100 Subject: [PATCH 10/57] Add docker migration guide --- docker-compose.yml | 2 +- docs/docs/installation/docker_en.md | 7 +- .../installation/migrating_to_docker_en.md | 158 ++++++++++++++++++ 3 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 docs/docs/installation/migrating_to_docker_en.md diff --git a/docker-compose.yml b/docker-compose.yml index b8da67a22..0dedbc87e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,4 +58,4 @@ services: # volumes: # - ./docker-resources/Caddyfile:/etc/caddy/Caddyfile # - ./caddy-data:/data - # - ./caddy-config:/config \ No newline at end of file + # - ./caddy-config:/config diff --git a/docs/docs/installation/docker_en.md b/docs/docs/installation/docker_en.md index bf67f4d86..64169852f 100644 --- a/docs/docs/installation/docker_en.md +++ b/docs/docs/installation/docker_en.md @@ -6,6 +6,8 @@ This guide will show you how to get akkoma working in a docker container, if you want isolation, or if you run a distribution not supported by the OTP releases. +If you want to migrate from or OTP to docker, check out [the migration guide](./migrating_to_docker_en.md). + ### Prepare the system * Install docker and docker-compose @@ -43,9 +45,6 @@ in our compose environment. ```bash mkdir pgdata -# if you want to use caddy -mkdir caddy-data -mkdir caddy-config ./docker-resources/manage.sh mix deps.get ./docker-resources/manage.sh mix compile ./docker-resources/manage.sh mix pleroma.instance gen @@ -118,6 +117,8 @@ You've got two options. This is by far the easiest option. It'll handle HTTPS and all that for you. ```bash +mkdir caddy-data +mkdir caddy-config cp docker-resources/Caddyfile.example docker-resources/Caddyfile ``` diff --git a/docs/docs/installation/migrating_to_docker_en.md b/docs/docs/installation/migrating_to_docker_en.md new file mode 100644 index 000000000..945f43090 --- /dev/null +++ b/docs/docs/installation/migrating_to_docker_en.md @@ -0,0 +1,158 @@ +# Migrating to a Docker Installation + +If you for any reason wish to migrate a source or OTP install to a docker one, +this guide is for you. + +You have a few options - your major one will be whether you want to keep your +reverse-proxy setup from before. + +You probably should, in the first instance. + +### Prepare the system + +* Install docker and docker-compose + * [Docker](https://docs.docker.com/engine/install/) + * [Docker-compose](https://docs.docker.com/compose/install/) + * This will usually just be a repository installation and a package manager invocation. + +=== "Source" +```bash +git pull +``` + +=== "OTP" +Clone the akkoma repository + +```bash +git clone https://akkoma.dev/AkkomaGang/akkoma.git -b stable +cd akkoma +``` + +### Back up your old database + +Change the database name as needed + +```bash +pg_dump -d akkoma_prod --format c > akkoma_backup.sql +``` + +### Getting your static files in the right place + +This will vary by every installation. Copy your `instance` directory to `instance/` in +the akkoma source directory - this is where the docker container will look for it. + +For *most* from-source installs it'll already be there. + +And the same with `uploads`, make sure your uploads (if you have them on disk) are +located at `uploads/` in the akkoma source directory. + +If you have them on a different disk, you will need to mount that disk into the docker-compose file, +with an entry that looks like this: + +```yaml +akkoma: + volumes: + - .:/opt/akkoma # This should already be there + - type: bind + source: /path/to/your/uploads + target: /opt/akkoma/uploads +``` + +### Set up basic configuration + +```bash +cp docker-resources/env.example .env +echo "DOCKER_USER=$(id -u):$(id -g)" >> .env +``` + +This probably won't need to be changed, it's only there to set basic environment +variables for the docker-compose file. + +=== "From source" + +You probably won't need to change your config. Provided your `config/prod.secret.exs` file +is still there, you're all good. + +=== "OTP" +```bash +cp /etc/akkoma/config.exs config/prod.secret.exs +``` + +**BOTH** + +Set the following config in `config/prod.secret.exs`: +```elixir +config :pleroma, Pleroma.Web.Endpoint, + ..., + http: [ip: {0, 0, 0, 0}, port: 4000] + +config :pleroma, Pleroma.Repo, + ..., + username: "akkoma", + password: "akkoma", + database: "akkoma", + hostname: "db" +``` + +### Building the container + +The container provided is a thin wrapper around akkoma's dependencies, +it does not contain the code itself. This is to allow for easy updates +and debugging if required. + +```bash +./docker-resources/build.sh +``` + +This will generate a container called `akkoma` which we can use +in our compose environment. + +### Setting up the docker resources + +```bash +# These won't exist if you're migrating from OTP +rm -rf deps +rm -rf _build +``` + +```bash +mkdir pgdata +./docker-resources/manage.sh mix deps.get +./docker-resources/manage.sh mix compile +``` + +### Setting up the database + +Now we can import our database to the container. + +```bash +docker-compose run --rm --user akkoma -d db +docker-compose run --rm akkoma pg_restore -v -U akkoma -j $(grep -c ^processor /proc/cpuinfo) -d akkoma -h db akkoma_backup.sql +``` + +### Reverse proxies + +If you're just reusing your old proxy, you may have to uncomment the line in +the docker-compose file under `ports`. You'll find it. + +Otherwise, you can use the same setup as the [docker installation guide](./docker_en.md#reverse-proxies). + +### Let's go + +```bash +docker-compose up -d +``` + +You should now be at the same point as you were before, but with a docker install. + +{! installation/frontends.include !} + +See the [docker installation guide](./docker_en.md) for more information on how to +update. + +#### Further reading + +{! installation/further_reading.include !} + +{! support.include !} + From f36d14818dadda78c6a5e8cb2977105cb0152bd3 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Wed, 19 Oct 2022 10:01:14 +0000 Subject: [PATCH 11/57] Unilateral remove from followers (#232) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit from https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3647/ Co-authored-by: marcin mikołajczak Co-authored-by: Tusooa Zhu Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/232 --- CHANGELOG.md | 1 + .../api_spec/operations/account_operation.ex | 16 +++++++ .../controllers/account_controller.ex | 21 +++++++-- lib/pleroma/web/router.ex | 1 + test/pleroma/user_test.exs | 6 +-- .../web/activity_pub/activity_pub_test.exs | 2 +- .../controllers/account_controller_test.exs | 44 +++++++++++++++++++ 7 files changed, 84 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1c2e4460..b3e4f371f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Added - Officially supported docker release +- Ability to remove followers unilaterally without a block ## Changes - Follows no longer override domain blocks, a domain block is final diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index b1f4932ee..b305dc1ea 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -334,6 +334,22 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do } end + def remove_from_followers_operation do + %Operation{ + tags: ["Account actions"], + summary: "Remove from followers", + operationId: "AccountController.remove_from_followers", + security: [%{"oAuth" => ["follow", "write:follows"]}], + description: "Remove the given account from followers", + parameters: [%Reference{"$ref": "#/components/parameters/accountIdOrNickname"}], + responses: %{ + 200 => Operation.response("Relationship", "application/json", AccountRelationship), + 400 => Operation.response("Error", "application/json", ApiError), + 404 => Operation.response("Error", "application/json", ApiError) + } + } + end + def note_operation do %Operation{ tags: ["Account actions"], diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index ed7fb802a..946c8544f 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -76,15 +76,16 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do plug( OAuthScopesPlug, - %{scopes: ["follow", "write:follows"]} when action in [:follow_by_uri, :follow, :unfollow] + %{scopes: ["follow", "write:follows"]} + when action in [:follow_by_uri, :follow, :unfollow, :remove_from_followers] ) plug(OAuthScopesPlug, %{scopes: ["follow", "read:mutes"]} when action == :mutes) plug(OAuthScopesPlug, %{scopes: ["follow", "write:mutes"]} when action in [:mute, :unmute]) - @relationship_actions [:follow, :unfollow] - @needs_account ~W(followers following lists follow unfollow mute unmute block unblock note)a + @relationship_actions [:follow, :unfollow, :remove_from_followers] + @needs_account ~W(followers following lists follow unfollow mute unmute block unblock note remove_from_followers)a plug( RateLimiter, @@ -447,6 +448,20 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do end end + @doc "POST /api/v1/accounts/:id/remove_from_followers" + def remove_from_followers(%{assigns: %{user: %{id: id}, account: %{id: id}}}, _params) do + {:error, "Can not unfollow yourself"} + end + + def remove_from_followers(%{assigns: %{user: followed, account: follower}} = conn, _params) do + with {:ok, follower} <- CommonAPI.reject_follow_request(follower, followed) do + render(conn, "relationship.json", user: followed, target: follower) + else + nil -> + render_error(conn, :not_found, "Record not found") + end + end + @doc "POST /api/v1/follows" def follow_by_uri(%{body_params: %{uri: uri}} = conn, _) do case User.get_cached_by_nickname(uri) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 838599c4d..71a9e4d29 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -509,6 +509,7 @@ defmodule Pleroma.Web.Router do post("/accounts/:id/mute", AccountController, :mute) post("/accounts/:id/unmute", AccountController, :unmute) post("/accounts/:id/note", AccountController, :note) + post("/accounts/:id/remove_from_followers", AccountController, :remove_from_followers) get("/conversations", ConversationController, :index) post("/conversations/:id/read", ConversationController, :mark_as_read) diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index 0272e3142..f11aec136 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -311,7 +311,7 @@ defmodule Pleroma.UserTest do describe "unfollow/2" do setup do: clear_config([:instance, :external_user_synchronization]) - test "unfollow with syncronizes external user" do + test "unfollow with synchronizes external user" do clear_config([:instance, :external_user_synchronization], true) followed = @@ -2260,7 +2260,7 @@ defmodule Pleroma.UserTest do assert other_user.follower_count == 1 end - test "syncronizes the counters with the remote instance for the followed when enabled" do + test "synchronizes the counters with the remote instance for the followed when enabled" do clear_config([:instance, :external_user_synchronization], false) user = insert(:user) @@ -2282,7 +2282,7 @@ defmodule Pleroma.UserTest do assert other_user.follower_count == 437 end - test "syncronizes the counters with the remote instance for the follower when enabled" do + test "synchronizes the counters with the remote instance for the follower when enabled" do clear_config([:instance, :external_user_synchronization], false) user = insert(:user) diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index ec562ac7b..c7b3334f3 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -1632,7 +1632,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do end describe "fetch_follow_information_for_user" do - test "syncronizes following/followers counters" do + test "synchronizes following/followers counters" do user = insert(:user, local: false, diff --git a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs index dcdff6c09..5b4f12402 100644 --- a/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/account_controller_test.exs @@ -1921,4 +1921,48 @@ defmodule Pleroma.Web.MastodonAPI.AccountControllerTest do |> get("/api/v1/accounts/relationships?id=#{other_user.id}") |> json_response_and_validate_schema(200) end + + describe "remove from followers" do + setup do: oauth_access(["follow"]) + + test "removing user from followers", %{conn: conn, user: user} do + %{id: other_user_id} = other_user = insert(:user) + + CommonAPI.follow(other_user, user) + + assert %{"id" => ^other_user_id, "followed_by" => false} = + conn + |> post("/api/v1/accounts/#{other_user_id}/remove_from_followers") + |> json_response_and_validate_schema(200) + + refute User.following?(other_user, user) + end + + test "removing remote user from followers", %{conn: conn, user: user} do + %{id: other_user_id} = other_user = insert(:user, local: false) + + CommonAPI.follow(other_user, user) + + assert User.following?(other_user, user) + + assert %{"id" => ^other_user_id, "followed_by" => false} = + conn + |> post("/api/v1/accounts/#{other_user_id}/remove_from_followers") + |> json_response_and_validate_schema(200) + + refute User.following?(other_user, user) + end + + test "removing user from followers errors", %{user: user, conn: conn} do + # self remove + conn_res = post(conn, "/api/v1/accounts/#{user.id}/remove_from_followers") + + assert %{"error" => "Can not unfollow yourself"} = + json_response_and_validate_schema(conn_res, 400) + + # remove non existing user + conn_res = post(conn, "/api/v1/accounts/doesntexist/remove_from_followers") + assert %{"error" => "Record not found"} = json_response_and_validate_schema(conn_res, 404) + end + end end From 7bb6df2d5b90786b099edbe936f14ecfbca79da3 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Thu, 20 Oct 2022 13:26:00 +0100 Subject: [PATCH 12/57] Remove unused DATA arg --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 730730ce3..6ba7a2269 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,6 @@ FROM hexpm/elixir:1.13.4-erlang-24.3.4.5-alpine-3.15.6 ENV MIX_ENV=prod ARG HOME=/opt/akkoma -ARG DATA=/var/lib/akkoma LABEL org.opencontainers.image.title="akkoma" \ org.opencontainers.image.description="Akkoma for Docker" \ From 16a31872fe5d0b9c7376d4b9a8600d544efbb397 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 21 Oct 2022 10:23:07 +0100 Subject: [PATCH 13/57] document local_bubble --- docs/docs/configuration/cheatsheet.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/docs/configuration/cheatsheet.md b/docs/docs/configuration/cheatsheet.md index 52062eaa0..ec8bea0cc 100644 --- a/docs/docs/configuration/cheatsheet.md +++ b/docs/docs/configuration/cheatsheet.md @@ -59,6 +59,7 @@ To add configuration to your config file, you can copy it from the base config. * `cleanup_attachments`: Remove attachments along with statuses. Does not affect duplicate files and attachments without status. Enabling this will increase load to database when deleting statuses on larger instances. * `show_reactions`: Let favourites and emoji reactions be viewed through the API (default: `true`). * `password_reset_token_validity`: The time after which reset tokens aren't accepted anymore, in seconds (default: one day). +* `local_bubble`: Array of domains representing instances closely related to yours. Used to populate the `bubble` timeline. e.g `['example.com']`, (default: `[]`) ## :database * `improved_hashtag_timeline`: Setting to force toggle / force disable improved hashtags timeline. `:enabled` forces hashtags to be fetched from `hashtags` table for hashtags timeline. `:disabled` forces object-embedded hashtags to be used (slower). Keep it `:auto` for automatic behaviour (it is auto-set to `:enabled` [unless overridden] when HashtagsTableMigrator completes). From e6ceea35534100f54eebfbbf8167ebcdf557bdb8 Mon Sep 17 00:00:00 2001 From: ilja Date: Sun, 23 Oct 2022 12:33:31 +0200 Subject: [PATCH 14/57] fix flaky participation_test.exs It was tested if the updated_at after marking as "read" was equal as the updated_at at insertion, but that seems wrong. Firstly, if a record is updated, you expect the updated_at to also update. Secondly, the insert and update happen almost at the same time, so it's flaky regardless. Here I make sure it has a much older updated_at during insert so we can clealy see the effect after update. I also check that the updated_at is actually updated because I expect that this is the expected behaviour and it's also the current behaviour. --- test/pleroma/conversation/participation_test.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pleroma/conversation/participation_test.exs b/test/pleroma/conversation/participation_test.exs index a25e17c95..2bf57f539 100644 --- a/test/pleroma/conversation/participation_test.exs +++ b/test/pleroma/conversation/participation_test.exs @@ -122,11 +122,11 @@ defmodule Pleroma.Conversation.ParticipationTest do end test "it marks a participation as read" do - participation = insert(:participation, %{read: false}) + participation = insert(:participation, %{updated_at: ~N[2017-07-17 17:09:58], read: false}) {:ok, updated_participation} = Participation.mark_as_read(participation) assert updated_participation.read - assert updated_participation.updated_at == participation.updated_at + assert :gt = NaiveDateTime.compare(updated_participation.updated_at, participation.updated_at) end test "it marks a participation as unread" do From a59d3109829e37057fb84eef7580605c74983bbd Mon Sep 17 00:00:00 2001 From: Ilja <672-ilja@users.noreply.git.pleroma.social> Date: Wed, 24 Aug 2022 15:24:57 +0000 Subject: [PATCH 15/57] fix flaky test filter_controller_test.exs:200 --- .../controllers/filter_controller_test.exs | 59 ++++++++----------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/test/pleroma/web/mastodon_api/controllers/filter_controller_test.exs b/test/pleroma/web/mastodon_api/controllers/filter_controller_test.exs index 66f7ed579..99f037483 100644 --- a/test/pleroma/web/mastodon_api/controllers/filter_controller_test.exs +++ b/test/pleroma/web/mastodon_api/controllers/filter_controller_test.exs @@ -3,9 +3,10 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do - use Pleroma.Web.ConnCase, async: true + use Pleroma.Web.ConnCase, async: false use Oban.Testing, repo: Pleroma.Repo + import Mock import Pleroma.Factory alias Pleroma.Filter @@ -53,25 +54,20 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do in_seconds = 600 response = - conn - |> put_req_header("content-type", "application/json") - |> post("/api/v1/filters", %{ - "phrase" => "knights", - context: ["home"], - expires_in: in_seconds - }) - |> json_response_and_validate_schema(200) + with_mock NaiveDateTime, [:passthrough], utc_now: fn -> ~N[2017-03-17 17:09:58] end do + conn + |> put_req_header("content-type", "application/json") + |> post("/api/v1/filters", %{ + "phrase" => "knights", + context: ["home"], + expires_in: in_seconds + }) + |> json_response_and_validate_schema(200) + end assert response["irreversible"] == false - expires_at = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(in_seconds) - - assert NaiveDateTime.diff( - NaiveDateTime.from_iso8601!(response["expires_at"]), - expires_at - ) < 5 + assert response["expires_at"] == "2017-03-17T17:19:58.000Z" filter = Filter.get(response["id"], user) @@ -183,26 +179,21 @@ defmodule Pleroma.Web.MastodonAPI.FilterControllerTest do in_seconds = 600 response = - conn - |> put_req_header("content-type", "application/json") - |> put("/api/v1/filters/#{filter.filter_id}", %{ - phrase: "nii", - context: ["public"], - expires_in: in_seconds, - irreversible: true - }) - |> json_response_and_validate_schema(200) + with_mock NaiveDateTime, [:passthrough], utc_now: fn -> ~N[2017-03-17 17:09:58] end do + conn + |> put_req_header("content-type", "application/json") + |> put("/api/v1/filters/#{filter.filter_id}", %{ + phrase: "nii", + context: ["public"], + expires_in: in_seconds, + irreversible: true + }) + |> json_response_and_validate_schema(200) + end assert response["irreversible"] == true - expected_time = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(in_seconds) - - assert NaiveDateTime.diff( - NaiveDateTime.from_iso8601!(response["expires_at"]), - expected_time - ) < 5 + assert response["expires_at"] == "2017-03-17T17:19:58.000Z" filter = Filter.get(response["id"], user) From 3562eaeedc14d6b410da86651c0fce0e3032ff66 Mon Sep 17 00:00:00 2001 From: ilja Date: Sun, 23 Oct 2022 13:31:01 +0200 Subject: [PATCH 16/57] fix flaky test_user_relationship_test.exs:81 The problem was double. On the one hand, the function didn't actually return what was in the DB. On the other hand the test was flaky because it used NaiveDateTime.utc_now() so test could fail or pass depending on a difference of microseconds. Both are fixed now. --- lib/pleroma/user_relationship.ex | 5 +++-- test/pleroma/user_relationship_test.exs | 10 ++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex index a467e9b65..2f77697d4 100644 --- a/lib/pleroma/user_relationship.ex +++ b/lib/pleroma/user_relationship.ex @@ -67,8 +67,9 @@ defmodule Pleroma.UserRelationship do target_id: target.id }) |> Repo.insert( - on_conflict: {:replace_all_except, [:id]}, - conflict_target: [:source_id, :relationship_type, :target_id] + on_conflict: {:replace_all_except, [:id, :inserted_at]}, + conflict_target: [:source_id, :relationship_type, :target_id], + returning: true ) end diff --git a/test/pleroma/user_relationship_test.exs b/test/pleroma/user_relationship_test.exs index b2b074607..b3a0dbe37 100644 --- a/test/pleroma/user_relationship_test.exs +++ b/test/pleroma/user_relationship_test.exs @@ -5,8 +5,9 @@ defmodule Pleroma.UserRelationshipTest do alias Pleroma.UserRelationship - use Pleroma.DataCase, async: true + use Pleroma.DataCase, async: false + import Mock import Pleroma.Factory describe "*_exists?/2" do @@ -79,7 +80,12 @@ defmodule Pleroma.UserRelationshipTest do end test "if record already exists, returns it", %{users: [user1, user2]} do - user_block = UserRelationship.create_block(user1, user2) + user_block = + with_mock NaiveDateTime, [:passthrough], utc_now: fn -> ~N[2017-03-17 17:09:58] end do + {:ok, %{inserted_at: ~N[2017-03-17 17:09:58]}} = + UserRelationship.create_block(user1, user2) + end + assert user_block == UserRelationship.create_block(user1, user2) end end From 6486211064027179a89478bf2b520ad144947a0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 1 Oct 2022 23:28:02 +0200 Subject: [PATCH 17/57] Push.Impl: support edits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/push/impl.ex | 12 +++++++++++- test/pleroma/web/push/impl_test.exs | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index c30a39e94..e103cafc2 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Web.Push.Impl do require Logger import Ecto.Query - @types ["Create", "Follow", "Announce", "Like", "Move", "EmojiReact"] + @types ["Create", "Follow", "Announce", "Like", "Move", "EmojiReact", "Update"] @doc "Performs sending notifications for user subscriptions" @spec perform(Notification.t()) :: list(any) | :error | {:error, :unknown_type} @@ -167,6 +167,15 @@ defmodule Pleroma.Web.Push.Impl do end end + def format_body( + %{activity: %{data: %{"type" => "Update"}}}, + actor, + _object, + _mastodon_type + ) do + "@#{actor.nickname} edited a status" + end + def format_title(activity, mastodon_type \\ nil) def format_title(%{activity: %{data: %{"directMessage" => true}}}, _mastodon_type) do @@ -180,6 +189,7 @@ defmodule Pleroma.Web.Push.Impl do "follow_request" -> "New Follow Request" "reblog" -> "New Repeat" "favourite" -> "New Favorite" + "update" -> "New Update" "pleroma:emoji_reaction" -> "New Reaction" type -> "New #{String.capitalize(type || "event")}" end diff --git a/test/pleroma/web/push/impl_test.exs b/test/pleroma/web/push/impl_test.exs index 9100433ae..326872ccd 100644 --- a/test/pleroma/web/push/impl_test.exs +++ b/test/pleroma/web/push/impl_test.exs @@ -200,6 +200,21 @@ defmodule Pleroma.Web.Push.ImplTest do "New Reaction" end + test "renders title and body for update activity" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "lorem ipsum"}) + + {:ok, activity} = CommonAPI.update(user, activity, %{status: "edited status"}) + object = Object.normalize(activity, fetch: false) + + assert Impl.format_body(%{activity: activity, type: "update"}, user, object) == + "@#{user.nickname} edited a status" + + assert Impl.format_title(%{activity: activity, type: "update"}) == + "New Update" + end + test "renders title for create activity with direct visibility" do user = insert(:user, nickname: "Bob") From 4d9ca8909d75cf7e3b2d361b23fc0ef30bb25829 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sat, 29 Oct 2022 21:57:50 +0100 Subject: [PATCH 18/57] Add StopGifs to description --- config/description.exs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/config/description.exs b/config/description.exs index a17897b98..d7bece75b 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1389,6 +1389,12 @@ config :pleroma, :config_description, [ label: "Render misskey markdown", type: :boolean, description: "Whether to render Misskey-flavoured markdown" + }, + %{ + key: :stopGifs, + label: "Pause animated images until you hover on them", + type: :boolean, + description: "Whether to play gifs by default" } ] }, From d782140e2b1a7cedaaafa6c55713928d7117bddc Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sat, 29 Oct 2022 22:08:18 +0100 Subject: [PATCH 19/57] Reword stop gifs --- config/description.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/description.exs b/config/description.exs index d7bece75b..1ff0a582b 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1392,9 +1392,9 @@ config :pleroma, :config_description, [ }, %{ key: :stopGifs, - label: "Pause animated images until you hover on them", + label: "Stop Gifs", type: :boolean, - description: "Whether to play gifs by default" + description: "Whether to pause animated images until they're hovered on" } ] }, From cbc693f83213b389b4fb7c7d892deff8edc2da7d Mon Sep 17 00:00:00 2001 From: nullobsi Date: Tue, 1 Nov 2022 14:17:55 +0000 Subject: [PATCH 20/57] Fix LDAP user registration (#229) Simple fix for LDAP user registration. I'm not sure what changed but I managed to get Akkoma running in a debug session and figured out it was missing a match for an extra value at the end. I don't know Elixir all that well so I'm not sure if this was the correct way to do it... but it works. :) Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/229 Co-authored-by: nullobsi Co-committed-by: nullobsi --- lib/pleroma/web/auth/ldap_authenticator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index f77e8d203..ccf4b4104 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -102,7 +102,7 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do {:scope, :eldap.wholeSubtree()}, {:timeout, @search_timeout} ]) do - {:ok, {:eldap_search_result, [{:eldap_entry, _, attributes}], _}} -> + {:ok, {:eldap_search_result, [{:eldap_entry, _, attributes}], _, _}} -> params = %{ name: name, nickname: name, From 1bb8b763118c4aff169cebf75b1eb628143d3ab7 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Tue, 1 Nov 2022 14:21:35 +0000 Subject: [PATCH 21/57] Fix tests in ldap registration --- CHANGELOG.md | 3 +++ test/pleroma/web/o_auth/ldap_authorization_test.exs | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b3e4f371f..34c09cd65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Follows no longer override domain blocks, a domain block is final - Deletes are now the lowest priority to publish and will be handled after creates +## Fixed +- Registrations via ldap are now compatible with the latest OTP24 + ## 2022.10 ### Added diff --git a/test/pleroma/web/o_auth/ldap_authorization_test.exs b/test/pleroma/web/o_auth/ldap_authorization_test.exs index 61b9ce6b7..c8a1d65ab 100644 --- a/test/pleroma/web/o_auth/ldap_authorization_test.exs +++ b/test/pleroma/web/o_auth/ldap_authorization_test.exs @@ -71,7 +71,7 @@ defmodule Pleroma.Web.OAuth.LDAPAuthorizationTest do equalityMatch: fn _type, _value -> :ok end, wholeSubtree: fn -> :ok end, search: fn _connection, _options -> - {:ok, {:eldap_search_result, [{:eldap_entry, '', []}], []}} + {:ok, {:eldap_search_result, [{:eldap_entry, '', []}], [], []}} end, close: fn _connection -> send(self(), :close_connection) From f1dfd76b988eeba78404a267a96ae3ad31e25040 Mon Sep 17 00:00:00 2001 From: ilja Date: Tue, 1 Nov 2022 14:25:54 +0000 Subject: [PATCH 22/57] Fix rate_limiter_test.exs test "it restricts based on config values" (#233) Fixes one of the 'erratic' tests It used a timer to sleep. But time also goes on when doing other things, so depending on hardware, the timings could be off. I slightly changed the tests so we still test what we functionally want. Instead of waiting until the cache expires I now have a function to expire the test and use that. That means we're not testing any more if the cache really expires after a certain amount of time, but that's the responsability of the dependency imo, so shouldn't be a problem. I also changed `Pleroma.Web.Endpoint, :http, :ip` in the tests to `127.0.0.1` Currently it was set to 8.8.8.8, but I see no reason for that and, while I assume that no calls are made to it, it may come over as weird or suspicious to people. Co-authored-by: Ilja Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/233 Co-authored-by: ilja Co-committed-by: ilja --- test/pleroma/web/plugs/rate_limiter_test.exs | 44 +++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/test/pleroma/web/plugs/rate_limiter_test.exs b/test/pleroma/web/plugs/rate_limiter_test.exs index b7cfde1f7..83a47ea30 100644 --- a/test/pleroma/web/plugs/rate_limiter_test.exs +++ b/test/pleroma/web/plugs/rate_limiter_test.exs @@ -48,38 +48,42 @@ defmodule Pleroma.Web.Plugs.RateLimiterTest do refute RateLimiter.disabled?(build_conn()) end - @tag :erratic test "it restricts based on config values" do limiter_name = :test_plug_opts scale = 80 limit = 5 - clear_config([Pleroma.Web.Endpoint, :http, :ip], {8, 8, 8, 8}) + clear_config([Pleroma.Web.Endpoint, :http, :ip], {127, 0, 0, 1}) clear_config([:rate_limit, limiter_name], {scale, limit}) plug_opts = RateLimiter.init(name: limiter_name) conn = build_conn(:get, "/") - for i <- 1..5 do - conn = RateLimiter.call(conn, plug_opts) - assert {^i, _} = RateLimiter.inspect_bucket(conn, limiter_name, plug_opts) - Process.sleep(10) + for _ <- 1..5 do + conn_limited = RateLimiter.call(conn, plug_opts) + + refute conn_limited.status == Conn.Status.code(:too_many_requests) + refute conn_limited.resp_body + refute conn_limited.halted end - conn = RateLimiter.call(conn, plug_opts) - assert %{"error" => "Throttled"} = ConnTest.json_response(conn, :too_many_requests) - assert conn.halted + conn_limited = RateLimiter.call(conn, plug_opts) + assert %{"error" => "Throttled"} = ConnTest.json_response(conn_limited, :too_many_requests) + assert conn_limited.halted - Process.sleep(50) + expire_ttl(conn, limiter_name) - conn = build_conn(:get, "/") + for _ <- 1..5 do + conn_limited = RateLimiter.call(conn, plug_opts) - conn = RateLimiter.call(conn, plug_opts) - assert {1, 4} = RateLimiter.inspect_bucket(conn, limiter_name, plug_opts) + refute conn_limited.status == Conn.Status.code(:too_many_requests) + refute conn_limited.resp_body + refute conn_limited.halted + end - refute conn.status == Conn.Status.code(:too_many_requests) - refute conn.resp_body - refute conn.halted + conn_limited = RateLimiter.call(conn, plug_opts) + assert %{"error" => "Throttled"} = ConnTest.json_response(conn_limited, :too_many_requests) + assert conn_limited.halted end describe "options" do @@ -263,4 +267,12 @@ defmodule Pleroma.Web.Plugs.RateLimiterTest do refute {:err, :not_found} == RateLimiter.inspect_bucket(conn, limiter_name, opts) end + + def expire_ttl(%{remote_ip: remote_ip} = _conn, bucket_name_root) do + bucket_name = "anon:#{bucket_name_root}" |> String.to_atom() + key_name = "ip::#{remote_ip |> Tuple.to_list() |> Enum.join(".")}" + + {:ok, bucket_value} = Cachex.get(bucket_name, key_name) + Cachex.put(bucket_name, key_name, bucket_value, ttl: -1) + end end From be5044f785e562c4978121b74bce03f131b827e3 Mon Sep 17 00:00:00 2001 From: ilja Date: Tue, 1 Nov 2022 14:31:29 +0000 Subject: [PATCH 23/57] fix_flaky_transfer_task_test.exs (#237) There were async calls happening, so they weren't always finished when assert happened. I also fixed some bugs in the erratic tests that were introduced when removing :shout.:shout is a key where restart is needed, and was changed in the test to use :rate_limit (which also requires a restart). But there was a bug in the syntax that didn't get caught because the test was tagged as erratic and therefor didn't fail. Here I fixed it. During compilation, we had a warning `:logger is used by the current application but the current application does not depend on :logger` which is now fixed as well (see commit message for complete stacktrace). Co-authored-by: Ilja Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/237 Co-authored-by: ilja Co-committed-by: ilja --- restarter/lib/pleroma.ex | 12 ++++ restarter/mix.exs | 3 +- test/pleroma/config/transfer_task_test.exs | 69 ++++++++++++++++++---- 3 files changed, 70 insertions(+), 14 deletions(-) diff --git a/restarter/lib/pleroma.ex b/restarter/lib/pleroma.ex index 149a569ce..a7186cec4 100644 --- a/restarter/lib/pleroma.ex +++ b/restarter/lib/pleroma.ex @@ -61,6 +61,12 @@ defmodule Restarter.Pleroma do {:noreply, @init_state} end + # Don't actually restart during tests. + # We just check if the correct call has been done. + # If we actually restart, we get errors during the tests like + # (RuntimeError) could not lookup Ecto repo Pleroma.Repo because it was not started or + # it does not exist + # See tests in Pleroma.Config.TransferTaskTest def handle_cast({:restart, :test, _}, state) do Logger.debug("pleroma manually restarted") {:noreply, Map.put(state, :need_reboot, false)} @@ -74,6 +80,12 @@ defmodule Restarter.Pleroma do def handle_cast({:after_boot, _}, %{after_boot: true} = state), do: {:noreply, state} + # Don't actually restart during tests. + # We just check if the correct call has been done. + # If we actually restart, we get errors during the tests like + # (RuntimeError) could not lookup Ecto repo Pleroma.Repo because it was not started or + # it does not exist + # See tests in Pleroma.Config.TransferTaskTest def handle_cast({:after_boot, :test}, state) do Logger.debug("pleroma restarted after boot") state = %{state | after_boot: true, rebooted: true} diff --git a/restarter/mix.exs b/restarter/mix.exs index b0908aece..9f26f5f64 100644 --- a/restarter/mix.exs +++ b/restarter/mix.exs @@ -13,7 +13,8 @@ defmodule Restarter.MixProject do def application do [ - mod: {Restarter, []} + mod: {Restarter, []}, + extra_applications: [:logger] ] end diff --git a/test/pleroma/config/transfer_task_test.exs b/test/pleroma/config/transfer_task_test.exs index 30cb92fa7..988214eb1 100644 --- a/test/pleroma/config/transfer_task_test.exs +++ b/test/pleroma/config/transfer_task_test.exs @@ -119,44 +119,87 @@ defmodule Pleroma.Config.TransferTaskTest do describe "pleroma restart" do setup do - on_exit(fn -> Restarter.Pleroma.refresh() end) + on_exit(fn -> + Restarter.Pleroma.refresh() + + # Restarter.Pleroma.refresh/0 is an asynchronous call. + # A GenServer will first finish the previous call before starting a new one. + # Here we do a synchronous call. + # That way we are sure that the previous call has finished before we continue. + # See https://stackoverflow.com/questions/51361856/how-to-use-task-await-with-genserver + Restarter.Pleroma.rebooted?() + end) end - @tag :erratic test "don't restart if no reboot time settings were changed" do clear_config(:emoji) insert(:config, key: :emoji, value: [groups: [a: 1, b: 2]]) refute String.contains?( - capture_log(fn -> TransferTask.start_link([]) end), + capture_log(fn -> + TransferTask.start_link([]) + + # TransferTask.start_link/1 is an asynchronous call. + # A GenServer will first finish the previous call before starting a new one. + # Here we do a synchronous call. + # That way we are sure that the previous call has finished before we continue. + Restarter.Pleroma.rebooted?() + end), "pleroma restarted" ) end - @tag :erratic test "on reboot time key" do - clear_config([:pleroma, :rate_limit]) - insert(:config, key: {:pleroma, :rate_limit}, value: [enabled: false]) - assert capture_log(fn -> TransferTask.start_link([]) end) =~ "pleroma restarted" + clear_config(:rate_limit) + insert(:config, key: :rate_limit, value: [enabled: false]) + + # Note that we don't actually restart Pleroma. + # See module Restarter.Pleroma + assert capture_log(fn -> + TransferTask.start_link([]) + + # TransferTask.start_link/1 is an asynchronous call. + # A GenServer will first finish the previous call before starting a new one. + # Here we do a synchronous call. + # That way we are sure that the previous call has finished before we continue. + Restarter.Pleroma.rebooted?() + end) =~ "pleroma restarted" end - @tag :erratic test "on reboot time subkey" do clear_config(Pleroma.Captcha) insert(:config, key: Pleroma.Captcha, value: [seconds_valid: 60]) - assert capture_log(fn -> TransferTask.start_link([]) end) =~ "pleroma restarted" + + # Note that we don't actually restart Pleroma. + # See module Restarter.Pleroma + assert capture_log(fn -> + TransferTask.start_link([]) + + # TransferTask.start_link/1 is an asynchronous call. + # A GenServer will first finish the previous call before starting a new one. + # Here we do a synchronous call. + # That way we are sure that the previous call has finished before we continue. + Restarter.Pleroma.rebooted?() + end) =~ "pleroma restarted" end - @tag :erratic test "don't restart pleroma on reboot time key and subkey if there is false flag" do - clear_config([:pleroma, :rate_limit]) + clear_config(:rate_limit) clear_config(Pleroma.Captcha) - insert(:config, key: {:pleroma, :rate_limit}, value: [enabled: false]) + insert(:config, key: :rate_limit, value: [enabled: false]) insert(:config, key: Pleroma.Captcha, value: [seconds_valid: 60]) refute String.contains?( - capture_log(fn -> TransferTask.load_and_update_env([], false) end), + capture_log(fn -> + TransferTask.load_and_update_env([], false) + + # TransferTask.start_link/1 is an asynchronous call. + # A GenServer will first finish the previous call before starting a new one. + # Here we do a synchronous call. + # That way we are sure that the previous call has finished before we continue. + Restarter.Pleroma.rebooted?() + end), "pleroma restarted" ) end From 9682ec4c5f7e29e937c7d23ae11e274b46b8e80e Mon Sep 17 00:00:00 2001 From: Norm Date: Tue, 1 Nov 2022 20:52:17 +0000 Subject: [PATCH 24/57] Change default instance name to "Akkoma" This was left at "Pleroma" for some reason. --- config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 5eb82cd33..1af037051 100644 --- a/config/config.exs +++ b/config/config.exs @@ -185,7 +185,7 @@ config :pleroma, :http, adapter: [] config :pleroma, :instance, - name: "Pleroma", + name: "Akkoma", email: "example@example.com", notify_email: "noreply@example.com", description: "Akkoma: The cooler fediverse server", From 7cfce562a9b43562bdc59effcc44d0396f618aee Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Wed, 2 Nov 2022 22:38:02 +0000 Subject: [PATCH 25/57] Add default favicon Fixes pleroma-fe#185 --- priv/static/favicon.png | Bin 0 -> 7132 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 priv/static/favicon.png diff --git a/priv/static/favicon.png b/priv/static/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..287c75bfafa195f66b769066d3b13a9dba88e40e GIT binary patch literal 7132 zcmV<28zbb2P)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D02p*dSaefwW^{L9 za%BK;VQFr3E^cLXAT%y8E;VI^GGzb&8%;?>K~#8N?c5ElWmRBsRe8UX^l}EQ>~g%O;c!1X=_t!f%G(sH_g>z;MqzUQ6$HS-<_c#}2vzH|54XYc>|um4(mpL1tvjb1M) zc4o0p7rXBTFL+_YcG+dOX3tk1+#GuFDb1#h7dKnacyF`q><={;ocrPCP1}C6dCP@2 zjoDk@{M(&rxA}y($GLm$c|`NFm;6HWiaic$_I~w-=D__=Z`L1iQFH8ucZ3~(>{ZPh zj(b;U{$`stUOBf7^`URq*>|6l;`%@PGdne9e(b(tA1=18*wglUaj^vX`^92F1_q=P zkAGWp)|u~b-hBQ?n~N^Eu`_?a>AV}73(xyV7~mau?2J?1-5kCC+yHp@m+u#5vWFjf zK}@~|BtaPg`_5IhKKQRc`qFUnZoBRkL@INAqS%*;omgy5UoR}Sq1dg(I*>6BP-mWY zZGeca0b&8O2DcBg&!_(4RRz#9q7Tjj_Q4086_czdJ^~gShe28ktFM3b`b#3ZcHU{1 zWcfFXA>5k0URmtwV)qv7KtASm=Z~@2dmr31z}HL`1sDuu6X~;fEttDKmZ1BLB^NQk8!=Wr5`Ne{U&@MjaqwI$#Zcw4E6_+KlwQ?Z3#WaIW? zPwH&l-bV#sDFs8o)|T4LF*H3TgWjbkKN~@=%^b`{LT{@SW#K{<3Vq&IjPOU8s1(lL z@_~ie3}^{3*=#_vxdF1F!9N?#Tu34PZUI*cD~o5oguuncLZ5X&V=YAH(PozIjz8{~ zLz|y#Z0_5DoG6E9Rwsadd$FjUfn;HVrECb;+EO;o-TE6*X*eI)y#8DSxi<3?wIUdH z#pJn(FSJ*2JX9?7_2iST7_oE?*+`(3bO#>rhOW}ED0!{}&63^b+)4>$7<^%L2Q)TH zgmVL3+Gy(>Ni=1{AOPJT*mKp)RfW}-f~vO_uau65cm&Xr+PsNS8xmvF$hjiI1)mRW z?wdKwOE4&Svay${Loy7F9s_!gM3`DG5xRid`T3yFlWnvxlPFs$8GFf?QFQ>Tla4un z#%N~ilx&PPp+;rh^Al(`0^6#x-Kgqb*74T!k<7@zBGOR;Py@pKsss;VSj z*6-HwQAd-3d6Oa8_$knn4Gwf$qyt)scT=$^QQa~ccyra!Z9C3c3H0#ApKq2D3L9Xn zJRY{vIrz1kW8Y$4pt0W~CzIh%fnI4Nu&<1I8`$^Cl;~o}(0%7tm(alKEbpEkjUL^MWv)5RvK@Qo^2c&T5@2Jt zRgUCHd+o}R2b)cOzyWQyFL&;8!LpW<=>zLpk`CDtQJifYY5sQH_xcA z__ZWRZLY=n1=rZvPTN+@^AGl|qdIXH2FPjtHc1lz#_9t-3OfcIASJl_yynnm!}=qe z3(kIH^VSQuHS1ozceC4zUes(pI^k zTzkc@G`D@`6V2cM?O!zi@h^YZ{P6yNiocIK^pF7c>MdtC*PVNQyaOEgK3{R#8O>#z zPmaHjf9*ldfv?)L+2f^q#5`OB2zlMLV)`_&^JS0Zaas;!a-w1}NGFa4H{AK6wmjL< zBLP4@V88vFw_SWu^SMvl(tPKeUu*vNzrNQr|NVoe`O(A6_QQwbdu=iKy=R}-Ty@5l z0CcaHzO1?Agp5{nl4 zUF-DfuweNPx&$I(Fd4{y{-r-`9{IO>$AE1fePG-Q_)k3eug&}4U7S<^#$cdgx~M#I z-97=nV^|JspMSM=%00H@}hTFB9aXm%Hczw#p;J4lt|X{bX}WhJ;KK zK6T5D%OGQNAKtzWJO8BFbkq?YyD|6YcHTLRq$H~Wp5~agJ_DV$8v{*Wj@;D& zd~UF_EL_c{xITcby5K~s*G(Ia2{s3V0X+mP>odGKgno+#Oa>nL0G-McQ8MXYya4T zAteAN55N{B9KeY{oCJ0nY0(DDRWATv4RWOtm@iN#y(GwoF?+S-!@x~IsEQVSpLE>D z=AOU)$_VHJS`F@)6$XF%uCGQ_@;7(<(Xvy#7rXv_*96!Y3hcuUJfQjb%{N4QjCXt% z;G`yHh>injM+uhEL5fGPu6)^Yy&V{`^^SXZ6iWIbM=#%X*U(cna{xCBXja;g8Bq{& zlTy=yo=Gy94eUF<@R=}R0}y+MK(38L^uckfQR3#g`Wity{&H)~&E6*@#~t_XIM6gv zm;ytcoXi{pwr_2l-}>Nn%{g1PG~fN!UCUq(Xtt!tk)(<2h@O;^exe*Q?JSJOwgBG@7^}DU0^Wt7WC&=hJB*XaLI$SD2ao{-H1DJ% z7!Bx5q&~N-rvIXbGcDaM;p6)Pb}o2mFaU?Z9WqiPQj8yA%x7`pjq;Ve*FN*tz5HCm z4EH$rmx zo@yjbTje78BLUmDA2b3R7Am_yxqW$N8XAc6NJ*{RH6}yzo@9xi2rt-n zVB1!`3Hsab#gTkiY3hT9VulF~KBtTsuIpus#_?_!N5UWG_9Cf#TQ);ZKs0&s zQm!8p;wT^_7=Vd-JQ-UXkQtEsQd70T@Oz5Ba~!}`L&hVbLRG>MtdhGzy=yXfKHc9G}&T70^pDfa=-^%xNont!PuD(wxsaedyDj49Y;Q{MCWUeKG^W4P1a=LDDgXL^qDba>XwA;u} z7Q1?>{UbztdDCUKCZ$t22xtaWmsOZeQnI!ND7LV?E8dlA?XtWEYF+~jsqjAZeK@Tx zAyZN~9lMig0FU#@TxsXCAcnduwg+v@>cR2e(AV?TCdv?$CazBwTc4L2WA zJS(qpgzu~R`p#nOmfSV>&<(J&5}73<+GLprIFg`-O|Y?AW;Z}Oh_!&lAfbrSs|s{& zLv8vdiVEA2!1&BTlBCS_w$E%?0R@NM|f?5}!)1N52&FIGG13_uc39b0vV0 zBSu-)g(VN~^>Wxdt)qFC%O*yvF$trEbFgd(h)D*>WF}eg-bxRhO^bHfM*C^<)B=tK zKmoXfiP;jW(hu0)fh~H?Y1Uo|{Ho0-KU{LxYu!>lk?NsJOl8yLBs)@l@U#KcU}xE9 zfCj9>FnIt=Vxf)m0u87t^d&?3pVf+xCrvaMBfWF3@3i6IxSsPTU9%-ZRxIR&q|kJ8 z0=78<+uzteT}npPT(uwK>9TFbZe8*WUjpu7O61CIXS+>a?Lo(s~SdF&*uDD(qo9n1pFq;$Uv*-l2 zxh1oGo_2rIG1$SV0IUj8Uw^*X;{kR)H0c3%zdXR&m1W?v$8wdJUI(`Kx*EykxR7MD zOcvVMCA?KLoCe5`+;DUA2cNsGx#Lg&viZi{-)g@7&-XXq`~H764?psFn9q*+PMhET z_^0DM;Bx}dV?1r7jF~aEq!2CSBDWlJBTcksE<#AS^rZyYy^`j}05iPe{%d}<698zYS8WV56RWWWGeV$%1fO(z6!zZ z(2y8R-o>#v6QN@{5XU&cV1;E&pq|i>l{WNT$lEZwut>lcKJ@F&H$M4$@&1l;wuUT- z6zs;@n$GBu;yljNul~u4B)9@_pNrA`T1Jou*g!_8Nm2srN%26<`k~OdVjk3*;S-)MjA?*krrGXd~Iggj5AJuzf2Nrjq~x#Pyf{Qu7a={r%?O zZ~t=hz#rch=5L_8<{LL17r@i))NFvJrEF-UEr9j;XE%Se`OX(V8}E)gUou=lhVnPL zsS{)kHZsXoR>u9kl8pYg3JKFD{~MFLm>0(Le=mf}M3Wpbf^Ld?z_#&%{ln zwDKKB1C4_KngHK-+vhv0k0sjy$gsXPU^sr```#76a@74_{$ke$Cw}#oj|JeayJCz* zx3ly!rasu_XWk-Nv^dQ(wvSc29=P89hPp2+tB?4GlnBS+@ zNy1dUF2`v!UB_qu0s=5K(6!Z|v1|)ahPnMVb-V1tbzIMNm5WiI=Qs=42p}?Y1zCfg zBqOEdbCf7YE9#UGVCPTKkrvP}od`Co6DB~j$j5F2k#sOv(y<^gm|V;#pJuTdgD?ye z`=HD+ZKv7LIceF(b!mCXNmeK=3CzhnkS8mU1Un-e2AL@RaxV9-hk#~N!Dax@A1v`C z_PQudlStIB7>vmV=R1taU9=S-r}r>rDFB^!48UhRho+R8a5s3K5BNNz%V2$W zve?&bK&wy^X#e#GpxF>+wJpulC{0_XAh4B2N~C(AHhF*-Az2T8niUqAm<^}3l#`m# z33N!I@%QDffQ>jLf+lA*0&GBA#92-I{|&KuUJT>r!MyC&PRD+xC&YpkDJ9IH4IblJ zJ%LHqwi79`zuI@T0oNBJRiwk(V63*Tt;_*gEOwNBSPg8S)zu8pc1K~d2w9ihrCF5S z3U)tjmRLG~bFeLVV+m)Lm2*@twl-HA@a)6#U2Y&*DyIN?Xnf_k=;RPmVF8n1t2(w9 zXl<8h&O+=iVWTZ}=Tl3O3|k{57CY+j(C#7j3|Vm)Spqn%lXcv+#4Ys62H}<8{}|U1ZbGSc*#P=3eg`!4TsEya#d*$fxZb#gES$( z(p7i*9Cy|i035?@&E}S<&$pSjkz7V1=04 zD=e0H%VyI;>~1@-s*GiPkrXVC=Rpd{`V(qEw~Sv1@K%OIjF6YeB(*FH$;&LO>u7So zc{3MTtc=lh%>&p#uB#7o9r{05rWu^_hWe5fpPec<3>NiRGF+W0GxL1*r6y0n$HI3W z9S3*;#yO(JofWKn^zN@U5B}9x0?0H{k)M6XwA0@C@uXDt*}qP0Wo$@+)i5C&v>Mn} zixQ)o0OmU{*eM@yXz$#No?dle5q77!u=D&E8T236{!c)EyI44(06vzOW=%W|6RY-; zE`|bf$VhE;Sha^eA&4V1wL&{n{oTuLaJ2$!)Wp{1PQ>Exf z5=pOehNZ0I^c_d6Kw2OZv#yfo7XH-Y9k8Bhra9( zhKxWxOB&z+YN&&0=Db|+&^Dj5FBj%v8|{|*DvoaBXI_@Cg|$HEY7rL3?!TZ`>gByw zVCQz~JPHu=F5j!z;A#~r(Jt9IvNEzN!ahk)XG5J#Go1t01^WGDNb$7)z8^y*E-a74 z^-O?HUdlC%L2?(}fHp?bCrJW0Wh85DH&S9wQwV9Uq%`(Tiq_M7eJrImrRRW=zzuaY z&GI62+kpLDGa&vh{pPJU6H=|XW7B~SgduQN%Q&@MhrX*)ETS-4L;_l_>4TkHs+lCo zE%w%Puji;A?Z}7nQ6J7a0!zHBj@Ia|}dPf&oKilE0Y1k@~xh$tpHrb&`xS$oPCYFHtPW%=ZilC4Xm^MbA&18iD*z8t}saf7)y7Z@be zIJ^6W1uX9pIFWhywv^ZpyDL1W4r&5sZm`lNaaLU}xmlc`eWJcSLg;vukNv z6;>?kcY+yI3i4XQ(C9>W$1 z7JCdjpLH8*8;&TTORAn$^+2SjPdzvZbbz}n6jh33*9V&e3gD!MWMH%i$>;r2rvZm` zB6ixgvazT3CQB2e8W(djb~#FJ(l2=$VszS4e&7wPFsQ6hYB@=H=nrm^nM^Wv)~=%O zLMMx3JQ$ge20!}P|Hb3JoPfDqS=a}>I6)%CEaT;*`bk!Q1D{j@aZ(;(=9xqh1#(yd zn4rTbbz6QCDcUAc6Vuw98(?QAJSl92VJ1!;t4;tn4Y0nMOomq`+ldowJACd*1F<4(JnXIYF%Ov?= z9oT@y=5}W)K=-BN!bTWQ)455etr9X4$H0oRw%R!-Y~B*AOrUDO$wk!PtyX)tiR5Mj zpfNhnVLp>2ea^FfFh=u4FnDB?Cupevq{LoNfL?L~ypps-7GV1a^<`PhR$v2H1IKlq zNrAVgg<#*bOZ_7e)0Vl+L}hi3?K9M8>;$;lIrO_EP4WTgl%E>tH0Oz!O!h<9W+{~_ zDccFaHuoBAm}H`L<7HA!-M0WcPX;ic8DLYZ(ivHshJF`z&I5SLLu&V|T}8*4YuFw1 z3xfpDJ1{vx;0S1IX)ht|mw zaNZdJbDWqyiPg1MN8dA8oofTj@ys7e$$piQ1SKK$!pFO|zVluKe$Cc%n*Rs+Hy#kR SmkNXc0000 Date: Fri, 4 Nov 2022 09:43:13 +0100 Subject: [PATCH 26/57] Fix typo in CSP Report-To header name The header name was Report-To, not Reply-To. In any case, that's now being changed to the Reporting-Endpoints HTTP Response Header. https://w3c.github.io/reporting/#header https://github.com/w3c/reporting/issues/177 CanIUse says the Report-To header is still supported by current Chrome and friends. https://caniuse.com/mdn-http_headers_report-to It doesn't have any data for the Reporting-Endpoints HTTP header, but this article says Chrome 96 supports it. https://web.dev/reporting-api/ (Even though that's come out one year ago, that's not compatible with Network Error Logging which's still using the Report-To version of the API) Signed-off-by: Thomas Citharel --- lib/pleroma/web/plugs/http_security_plug.ex | 2 +- test/pleroma/web/plugs/http_security_plug_test.exs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index d1e6cc9d3..3e8e931d1 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -68,7 +68,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do ] } - [{"reply-to", Jason.encode!(report_group)} | headers] + [{"report-to", Jason.encode!(report_group)} | headers] else headers end diff --git a/test/pleroma/web/plugs/http_security_plug_test.exs b/test/pleroma/web/plugs/http_security_plug_test.exs index 4e7befdd5..eb94cd665 100644 --- a/test/pleroma/web/plugs/http_security_plug_test.exs +++ b/test/pleroma/web/plugs/http_security_plug_test.exs @@ -59,9 +59,9 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do assert csp =~ ~r|report-uri https://endpoint.com;report-to csp-endpoint;| - [reply_to] = Conn.get_resp_header(conn, "reply-to") + [report_to] = Conn.get_resp_header(conn, "report-to") - assert reply_to == + assert report_to == "{\"endpoints\":[{\"url\":\"https://endpoint.com\"}],\"group\":\"csp-endpoint\",\"max-age\":10886400}" end From ccdf55acff7ff7d29682aab3ccf283b59ad33d26 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 4 Nov 2022 18:42:12 +0000 Subject: [PATCH 27/57] Fix instance name in email test --- test/pleroma/emails/user_email_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pleroma/emails/user_email_test.exs b/test/pleroma/emails/user_email_test.exs index 771a9a490..564552004 100644 --- a/test/pleroma/emails/user_email_test.exs +++ b/test/pleroma/emails/user_email_test.exs @@ -27,7 +27,7 @@ defmodule Pleroma.Emails.UserEmailTest do token = %Pleroma.UserInviteToken{token: "test-token"} email = UserEmail.user_invitation_email(user, token, "test@test.com", "Jonh") assert email.from == {config[:name], config[:notify_email]} - assert email.subject == "Invitation to Pleroma" + assert email.subject == "Invitation to Akkoma" assert email.to == [{"Jonh", "test@test.com"}] assert email.html_body =~ From b7e8ce235073ab45db24dfc6c27bc6998b43a200 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Sun, 6 Nov 2022 22:49:39 +0000 Subject: [PATCH 28/57] Scrape instance nodeinfo (#251) Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/251 --- config/config.exs | 6 +- config/description.exs | 13 + config/test.exs | 2 + lib/pleroma/application.ex | 3 +- lib/pleroma/instances/instance.ex | 203 +++++++++++---- lib/pleroma/web/activity_pub/side_effects.ex | 9 +- .../web/mastodon_api/views/account_view.ex | 34 ++- .../workers/nodeinfo_fetcher_worker.ex | 18 ++ .../20221020135943_add_nodeinfo.exs | 17 ++ test/pleroma/instances/instance_test.exs | 238 +++++++++++++++--- .../web/activity_pub/side_effects_test.exs | 29 +++ .../mastodon_api/views/account_view_test.exs | 100 +++++++- test/pleroma/web/streamer_test.exs | 2 +- test/support/factory.ex | 16 +- 14 files changed, 575 insertions(+), 115 deletions(-) create mode 100644 lib/pleroma/workers/nodeinfo_fetcher_worker.ex create mode 100644 priv/repo/migrations/20221020135943_add_nodeinfo.exs diff --git a/config/config.exs b/config/config.exs index 1af037051..644155aeb 100644 --- a/config/config.exs +++ b/config/config.exs @@ -567,7 +567,8 @@ config :pleroma, Oban, attachments_cleanup: 1, new_users_digest: 1, mute_expire: 5, - search_indexing: 10 + search_indexing: 10, + nodeinfo_fetcher: 1 ], plugins: [ Oban.Plugins.Pruner, @@ -806,7 +807,8 @@ config :ex_aws, http_client: Pleroma.HTTP.ExAws config :web_push_encryption, http_client: Pleroma.HTTP.WebPush -config :pleroma, :instances_favicons, enabled: false +config :pleroma, :instances_favicons, enabled: true +config :pleroma, :instances_nodeinfo, enabled: true config :floki, :html_parser, Floki.HTMLParser.FastHtml diff --git a/config/description.exs b/config/description.exs index 1ff0a582b..4843c0aae 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3047,6 +3047,19 @@ config :pleroma, :config_description, [ } ] }, + %{ + group: :pleroma, + key: :instances_nodeinfo, + type: :group, + description: "Control favicons for instances", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "Allow/disallow getting instance nodeinfo" + } + ] + }, %{ group: :ex_aws, key: :s3, diff --git a/config/test.exs b/config/test.exs index a5edb1149..3056dbd03 100644 --- a/config/test.exs +++ b/config/test.exs @@ -139,6 +139,8 @@ config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", priv # Reduce recompilation time # https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects config :phoenix, :plug_init_mode, :runtime +config :pleroma, :instances_favicons, enabled: false +config :pleroma, :instances_nodeinfo, enabled: false if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index adccd7c5d..a78924dfa 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -156,7 +156,8 @@ defmodule Pleroma.Application do build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("failed_proxy_url", limit: 2500), build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000), - build_cachex("translations", default_ttl: :timer.hours(24 * 30), limit: 2500) + build_cachex("translations", default_ttl: :timer.hours(24 * 30), limit: 2500), + build_cachex("instances", default_ttl: :timer.hours(24), limit: 2500) ] end diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 533dbbb82..fcf3181bf 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -5,6 +5,8 @@ defmodule Pleroma.Instances.Instance do @moduledoc "Instance." + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + alias Pleroma.Instances alias Pleroma.Instances.Instance alias Pleroma.Repo @@ -22,7 +24,8 @@ defmodule Pleroma.Instances.Instance do field(:host, :string) field(:unreachable_since, :naive_datetime_usec) field(:favicon, :string) - field(:favicon_updated_at, :naive_datetime) + field(:metadata_updated_at, :naive_datetime) + field(:nodeinfo, :map, default: %{}) timestamps() end @@ -31,7 +34,7 @@ defmodule Pleroma.Instances.Instance do def changeset(struct, params \\ %{}) do struct - |> cast(params, [:host, :unreachable_since, :favicon, :favicon_updated_at]) + |> cast(params, [:host, :unreachable_since, :favicon, :nodeinfo, :metadata_updated_at]) |> validate_required([:host]) |> unique_constraint(:host) end @@ -138,63 +141,144 @@ defmodule Pleroma.Instances.Instance do defp parse_datetime(datetime), do: datetime - def get_or_update_favicon(%URI{host: host} = instance_uri) do - existing_record = Repo.get_by(Instance, %{host: host}) - now = NaiveDateTime.utc_now() + def needs_update(nil), do: true - if existing_record && existing_record.favicon_updated_at && - NaiveDateTime.diff(now, existing_record.favicon_updated_at) < 86_400 do + def needs_update(%Instance{metadata_updated_at: nil}), do: true + + def needs_update(%Instance{metadata_updated_at: metadata_updated_at}) do + now = NaiveDateTime.utc_now() + NaiveDateTime.diff(now, metadata_updated_at) > 86_400 + end + + def local do + %Instance{ + host: Pleroma.Web.Endpoint.host(), + favicon: Pleroma.Web.Endpoint.url() <> "/favicon.png", + nodeinfo: Pleroma.Web.Nodeinfo.NodeinfoController.raw_nodeinfo() + } + end + + def update_metadata(%URI{host: host} = uri) do + Logger.info("Checking metadata for #{host}") + existing_record = Repo.get_by(Instance, %{host: host}) + + if reachable?(host) do + do_update_metadata(uri, existing_record) + else + {:discard, :unreachable} + end + end + + defp do_update_metadata(%URI{host: host} = uri, existing_record) do + if existing_record do + if needs_update(existing_record) do + Logger.info("Updating metadata for #{host}") + favicon = scrape_favicon(uri) + nodeinfo = scrape_nodeinfo(uri) + + {:ok, instance} = + existing_record + |> changeset(%{ + host: host, + favicon: favicon, + nodeinfo: nodeinfo, + metadata_updated_at: NaiveDateTime.utc_now() + }) + |> Repo.update() + + @cachex.put(:instances_cache, "instances:#{host}", instance) + else + {:discard, "Does not require update"} + end + else + favicon = scrape_favicon(uri) + nodeinfo = scrape_nodeinfo(uri) + + Logger.info("Creating metadata for #{host}") + + {:ok, instance} = + %Instance{} + |> changeset(%{ + host: host, + favicon: favicon, + nodeinfo: nodeinfo, + metadata_updated_at: NaiveDateTime.utc_now() + }) + |> Repo.insert() + + @cachex.put(:instances_cache, "instances:#{host}", instance) + end + end + + def get_favicon(%URI{host: host}) do + existing_record = Repo.get_by(Instance, %{host: host}) + + if existing_record do existing_record.favicon else - favicon = scrape_favicon(instance_uri) - - if existing_record do - existing_record - |> changeset(%{favicon: favicon, favicon_updated_at: now}) - |> Repo.update() - else - %Instance{} - |> changeset(%{host: host, favicon: favicon, favicon_updated_at: now}) - |> Repo.insert() - end - - favicon - end - rescue - e -> - Logger.warn("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}") nil + end + end + + defp scrape_nodeinfo(%URI{} = instance_uri) do + with true <- Pleroma.Config.get([:instances_nodeinfo, :enabled]), + {_, true} <- {:reachable, reachable?(instance_uri.host)}, + {:ok, %Tesla.Env{status: 200, body: body}} <- + Tesla.get( + "https://#{instance_uri.host}/.well-known/nodeinfo", + headers: [{"Accept", "application/json"}] + ), + {:ok, json} <- Jason.decode(body), + {:ok, %{"links" => links}} <- {:ok, json}, + {:ok, %{"href" => href}} <- + {:ok, + Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0"))}, + {:ok, %Tesla.Env{body: data}} <- + Pleroma.HTTP.get(href, [{"accept", "application/json"}], []), + {:length, true} <- {:length, String.length(data) < 50_000}, + {:ok, nodeinfo} <- Jason.decode(data) do + nodeinfo + else + {:reachable, false} -> + Logger.debug( + "Instance.scrape_nodeinfo(\"#{to_string(instance_uri)}\") ignored unreachable host" + ) + + nil + + {:length, false} -> + Logger.debug( + "Instance.scrape_nodeinfo(\"#{to_string(instance_uri)}\") ignored too long body" + ) + + nil + + _ -> + nil + end end defp scrape_favicon(%URI{} = instance_uri) do - try do - with {_, true} <- {:reachable, reachable?(instance_uri.host)}, - {:ok, %Tesla.Env{body: html}} <- - Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], []), - {_, [favicon_rel | _]} when is_binary(favicon_rel) <- - {:parse, - html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")}, - {_, favicon} when is_binary(favicon) <- - {:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do - favicon - else - {:reachable, false} -> - Logger.debug( - "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") ignored unreachable host" - ) - - nil - - _ -> - nil - end - rescue - e -> - Logger.warn( - "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}" + with true <- Pleroma.Config.get([:instances_favicons, :enabled]), + {_, true} <- {:reachable, reachable?(instance_uri.host)}, + {:ok, %Tesla.Env{body: html}} <- + Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], []), + {_, [favicon_rel | _]} when is_binary(favicon_rel) <- + {:parse, html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")}, + {_, favicon} when is_binary(favicon) <- + {:merge, URI.merge(instance_uri, favicon_rel) |> to_string()}, + {:length, true} <- {:length, String.length(favicon) < 255} do + favicon + else + {:reachable, false} -> + Logger.debug( + "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") ignored unreachable host" ) nil + + _ -> + nil end end @@ -217,4 +301,25 @@ defmodule Pleroma.Instances.Instance do end) |> Stream.run() end + + def get_by_url(url_or_host) do + url = host(url_or_host) + Repo.get_by(Instance, host: url) + end + + def get_cached_by_url(url_or_host) do + url = host(url_or_host) + + if url == Pleroma.Web.Endpoint.host() do + {:ok, local()} + else + @cachex.fetch!(:instances_cache, "instances:#{url}", fn _ -> + with %Instance{} = instance <- get_by_url(url) do + {:commit, {:ok, instance}} + else + _ -> {:ignore, nil} + end + end) + end + end end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index c3258c75b..18643662e 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -192,6 +192,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # - Increase the user note count # - Increase the reply count # - Increase replies count + # - Ask for scraping of nodeinfo # - Set up ActivityExpiration # - Set up notifications # - Index incoming posts for search (if needed) @@ -209,6 +210,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do reply_depth = (meta[:depth] || 0) + 1 + Pleroma.Workers.NodeInfoFetcherWorker.enqueue("process", %{ + "source_url" => activity.data["actor"] + }) + # FIXME: Force inReplyTo to replies if Pleroma.Web.Federator.allowed_thread_distance?(reply_depth) and object.data["replies"] != nil do @@ -234,7 +239,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do {:ok, activity, meta} else - e -> Repo.rollback(e) + e -> + Logger.error(inspect(e)) + Repo.rollback(e) end end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 06acf0a26..cbb57aee6 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -186,6 +186,16 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do render_many(targets, AccountView, "relationship.json", render_opts) end + def render("instance.json", %{instance: %Pleroma.Instances.Instance{} = instance}) do + %{ + name: instance.host, + favicon: instance.favicon |> MediaProxy.url(), + nodeinfo: instance.nodeinfo + } + end + + def render("instance.json", _), do: nil + defp do_render("show.json", %{user: user} = opts) do user = User.sanitize_html(user, User.html_filter_policy(opts[:for])) display_name = user.name || user.nickname @@ -230,16 +240,20 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do %{} end - favicon = - if Pleroma.Config.get([:instances_favicons, :enabled]) do - user - |> Map.get(:ap_id, "") - |> URI.parse() - |> URI.merge("/") - |> Pleroma.Instances.Instance.get_or_update_favicon() - |> MediaProxy.url() + instance = + with {:ok, instance} <- Pleroma.Instances.Instance.get_cached_by_url(user.ap_id) do + instance else + _ -> + nil + end + + favicon = + if is_nil(instance) do nil + else + instance.favicon + |> MediaProxy.url() end %{ @@ -271,7 +285,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do } }, last_status_at: user.last_status_at, - + akkoma: %{ + instance: render("instance.json", %{instance: instance}) + }, # Pleroma extensions # Note: it's insecure to output :email but fully-qualified nickname may serve as safe stub fqn: User.full_nickname(user), diff --git a/lib/pleroma/workers/nodeinfo_fetcher_worker.ex b/lib/pleroma/workers/nodeinfo_fetcher_worker.ex new file mode 100644 index 000000000..27492e1e3 --- /dev/null +++ b/lib/pleroma/workers/nodeinfo_fetcher_worker.ex @@ -0,0 +1,18 @@ +defmodule Pleroma.Workers.NodeInfoFetcherWorker do + use Pleroma.Workers.WorkerHelper, queue: "nodeinfo_fetcher" + + alias Oban.Job + alias Pleroma.Instances.Instance + + @impl Oban.Worker + def perform(%Job{ + args: %{"op" => "process", "source_url" => domain} + }) do + uri = + domain + |> URI.parse() + |> URI.merge("/") + + Instance.update_metadata(uri) + end +end diff --git a/priv/repo/migrations/20221020135943_add_nodeinfo.exs b/priv/repo/migrations/20221020135943_add_nodeinfo.exs new file mode 100644 index 000000000..17707f3f7 --- /dev/null +++ b/priv/repo/migrations/20221020135943_add_nodeinfo.exs @@ -0,0 +1,17 @@ +defmodule Pleroma.Repo.Migrations.AddNodeinfo do + use Ecto.Migration + + def up do + alter table(:instances) do + add_if_not_exists(:nodeinfo, :map, default: %{}) + add_if_not_exists(:metadata_updated_at, :naive_datetime) + end + end + + def down do + alter table(:instances) do + remove_if_exists(:nodeinfo, :map) + remove_if_exists(:metadata_updated_at, :naive_datetime) + end + end +end diff --git a/test/pleroma/instances/instance_test.exs b/test/pleroma/instances/instance_test.exs index e49922724..adc847da5 100644 --- a/test/pleroma/instances/instance_test.exs +++ b/test/pleroma/instances/instance_test.exs @@ -9,12 +9,16 @@ defmodule Pleroma.Instances.InstanceTest do alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI - use Pleroma.DataCase + use Pleroma.DataCase, async: true import ExUnit.CaptureLog import Pleroma.Factory - setup_all do: clear_config([:instance, :federation_reachability_timeout_days], 1) + setup_all do + clear_config([:instance, :federation_reachability_timeout_days], 1) + clear_config([:instances_nodeinfo, :enabled], true) + clear_config([:instances_favicons, :enabled], true) + end describe "set_reachable/1" do test "clears `unreachable_since` of existing matching Instance record having non-nil `unreachable_since`" do @@ -102,62 +106,220 @@ defmodule Pleroma.Instances.InstanceTest do end end - describe "get_or_update_favicon/1" do - test "Scrapes favicon URLs" do - Tesla.Mock.mock(fn %{url: "https://favicon.example.org/"} -> - %Tesla.Env{ - status: 200, - body: ~s[] - } + describe "update_metadata/1" do + test "Scrapes favicon URLs and nodeinfo" do + Tesla.Mock.mock(fn + %{url: "https://favicon.example.org/"} -> + %Tesla.Env{ + status: 200, + body: ~s[] + } + + %{url: "https://favicon.example.org/.well-known/nodeinfo"} -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + links: [ + %{ + rel: "http://nodeinfo.diaspora.software/ns/schema/2.0", + href: "https://favicon.example.org/nodeinfo/2.0" + } + ] + }) + } + + %{url: "https://favicon.example.org/nodeinfo/2.0"} -> + %Tesla.Env{ + status: 200, + body: Jason.encode!(%{version: "2.0", software: %{name: "Akkoma"}}) + } end) - assert "https://favicon.example.org/favicon.png" == - Instance.get_or_update_favicon(URI.parse("https://favicon.example.org/")) + assert {:ok, true} == + Instance.update_metadata(URI.parse("https://favicon.example.org/")) + + {:ok, instance} = Instance.get_cached_by_url("https://favicon.example.org/") + assert instance.favicon == "https://favicon.example.org/favicon.png" + assert instance.nodeinfo == %{"version" => "2.0", "software" => %{"name" => "Akkoma"}} end - test "Returns nil on too long favicon URLs" do + test "Does not retain favicons that are too long" do long_favicon_url = "https://Lorem.ipsum.dolor.sit.amet/consecteturadipiscingelit/Praesentpharetrapurusutaliquamtempus/Mauriseulaoreetarcu/atfacilisisorci/Nullamporttitor/nequesedfeugiatmollis/dolormagnaefficiturlorem/nonpretiumsapienorcieurisus/Nullamveleratsem/Maecenassedaccumsanexnam/favicon.png" - Tesla.Mock.mock(fn %{url: "https://long-favicon.example.org/"} -> - %Tesla.Env{ - status: 200, - body: - ~s[] - } + Tesla.Mock.mock(fn + %{url: "https://long-favicon.example.org/"} -> + %Tesla.Env{ + status: 200, + body: + ~s[] + } + + %{url: "https://long-favicon.example.org/.well-known/nodeinfo"} -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + links: [ + %{ + rel: "http://nodeinfo.diaspora.software/ns/schema/2.0", + href: "https://long-favicon.example.org/nodeinfo/2.0" + } + ] + }) + } + + %{url: "https://long-favicon.example.org/nodeinfo/2.0"} -> + %Tesla.Env{ + status: 200, + body: Jason.encode!(%{version: "2.0", software: %{name: "Akkoma"}}) + } end) - assert capture_log(fn -> - assert nil == - Instance.get_or_update_favicon( - URI.parse("https://long-favicon.example.org/") - ) - end) =~ - "Instance.get_or_update_favicon(\"long-favicon.example.org\") error: %Postgrex.Error{" + assert {:ok, true} == + Instance.update_metadata(URI.parse("https://long-favicon.example.org/")) + + {:ok, instance} = Instance.get_cached_by_url("https://long-favicon.example.org/") + assert instance.favicon == nil end test "Handles not getting a favicon URL properly" do - Tesla.Mock.mock(fn %{url: "https://no-favicon.example.org/"} -> - %Tesla.Env{ - status: 200, - body: ~s[

I wil look down and whisper "GNO.."

] - } + Tesla.Mock.mock(fn + %{url: "https://no-favicon.example.org/"} -> + %Tesla.Env{ + status: 200, + body: ~s[

I wil look down and whisper "GNO.."

] + } + + %{url: "https://no-favicon.example.org/.well-known/nodeinfo"} -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + links: [ + %{ + rel: "http://nodeinfo.diaspora.software/ns/schema/2.0", + href: "https://no-favicon.example.org/nodeinfo/2.0" + } + ] + }) + } + + %{url: "https://no-favicon.example.org/nodeinfo/2.0"} -> + %Tesla.Env{ + status: 200, + body: Jason.encode!(%{version: "2.0", software: %{name: "Akkoma"}}) + } end) refute capture_log(fn -> - assert nil == - Instance.get_or_update_favicon( - URI.parse("https://no-favicon.example.org/") - ) - end) =~ "Instance.scrape_favicon(\"https://no-favicon.example.org/\") error: " + assert {:ok, true} = + Instance.update_metadata(URI.parse("https://no-favicon.example.org/")) + end) =~ "Instance.update_metadata(\"https://no-favicon.example.org/\") error: " end - test "Doesn't scrapes unreachable instances" do + test "Doesn't scrape unreachable instances" do instance = insert(:instance, unreachable_since: Instances.reachability_datetime_threshold()) url = "https://" <> instance.host - assert capture_log(fn -> assert nil == Instance.get_or_update_favicon(URI.parse(url)) end) =~ - "Instance.scrape_favicon(\"#{url}\") ignored unreachable host" + assert {:discard, :unreachable} == Instance.update_metadata(URI.parse(url)) + end + + test "doesn't continue scraping nodeinfo if we can't find a link" do + Tesla.Mock.mock(fn + %{url: "https://bad-nodeinfo.example.org/"} -> + %Tesla.Env{ + status: 200, + body: ~s[

I wil look down and whisper "GNO.."

] + } + + %{url: "https://bad-nodeinfo.example.org/.well-known/nodeinfo"} -> + %Tesla.Env{ + status: 200, + body: "oepsie woepsie de nodeinfo is kapotie uwu" + } + end) + + assert {:ok, true} == + Instance.update_metadata(URI.parse("https://bad-nodeinfo.example.org/")) + + {:ok, instance} = Instance.get_cached_by_url("https://bad-nodeinfo.example.org/") + assert instance.nodeinfo == nil + end + + test "doesn't store bad json in the nodeinfo" do + Tesla.Mock.mock(fn + %{url: "https://bad-nodeinfo.example.org/"} -> + %Tesla.Env{ + status: 200, + body: ~s[

I wil look down and whisper "GNO.."

] + } + + %{url: "https://bad-nodeinfo.example.org/.well-known/nodeinfo"} -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + links: [ + %{ + rel: "http://nodeinfo.diaspora.software/ns/schema/2.0", + href: "https://bad-nodeinfo.example.org/nodeinfo/2.0" + } + ] + }) + } + + %{url: "https://bad-nodeinfo.example.org/nodeinfo/2.0"} -> + %Tesla.Env{ + status: 200, + body: "oepsie woepsie de json might be bad uwu" + } + end) + + assert {:ok, true} == + Instance.update_metadata(URI.parse("https://bad-nodeinfo.example.org/")) + + {:ok, instance} = Instance.get_cached_by_url("https://bad-nodeinfo.example.org/") + assert instance.nodeinfo == nil + end + + test "doesn't store incredibly long json nodeinfo" do + too_long = String.duplicate("a", 50_000) + + Tesla.Mock.mock(fn + %{url: "https://bad-nodeinfo.example.org/"} -> + %Tesla.Env{ + status: 200, + body: ~s[

I wil look down and whisper "GNO.."

] + } + + %{url: "https://bad-nodeinfo.example.org/.well-known/nodeinfo"} -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + links: [ + %{ + rel: "http://nodeinfo.diaspora.software/ns/schema/2.0", + href: "https://bad-nodeinfo.example.org/nodeinfo/2.0" + } + ] + }) + } + + %{url: "https://bad-nodeinfo.example.org/nodeinfo/2.0"} -> + %Tesla.Env{ + status: 200, + body: Jason.encode!(%{version: "2.0", software: %{name: too_long}}) + } + end) + + assert {:ok, true} == + Instance.update_metadata(URI.parse("https://bad-nodeinfo.example.org/")) + + {:ok, instance} = Instance.get_cached_by_url("https://bad-nodeinfo.example.org/") + assert instance.nodeinfo == nil end end diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index fa8171eab..ee664bb8f 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -21,6 +21,35 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do import Mock import Pleroma.Factory + describe "handle" do + test "it queues a fetch of instance information" do + author = insert(:user, local: false, ap_id: "https://wowee.example.com/users/1") + recipient = insert(:user, local: true) + + {:ok, note_data, _meta} = + Builder.note(%Pleroma.Web.CommonAPI.ActivityDraft{ + user: author, + to: [recipient.ap_id], + mentions: [recipient], + content_html: "hey", + extra: %{"id" => "https://wowee.example.com/notes/1"} + }) + + {:ok, create_activity_data, _meta} = + Builder.create(author, note_data["id"], [recipient.ap_id]) + + {:ok, create_activity, _meta} = ActivityPub.persist(create_activity_data, local: false) + + {:ok, _create_activity, _meta} = + SideEffects.handle(create_activity, local: false, object_data: note_data) + + assert_enqueued( + worker: Pleroma.Workers.NodeInfoFetcherWorker, + args: %{"op" => "process", "source_url" => "https://wowee.example.com/users/1"} + ) + end + end + describe "handle_after_transaction" do test "it streams out notifications and streams" do author = insert(:user, local: true) diff --git a/test/pleroma/web/mastodon_api/views/account_view_test.exs b/test/pleroma/web/mastodon_api/views/account_view_test.exs index 8db887137..d1903af80 100644 --- a/test/pleroma/web/mastodon_api/views/account_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/account_view_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.AccountViewTest do - use Pleroma.DataCase + use Pleroma.DataCase, async: false alias Pleroma.User alias Pleroma.UserRelationship @@ -12,6 +12,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do import Pleroma.Factory import Tesla.Mock + import Mock setup do mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -25,6 +26,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do user = insert(:user, %{ + ap_id: "https://example.com/users/chikichikibanban", follower_count: 3, note_count: 5, background: background_image, @@ -38,6 +40,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do also_known_as: ["https://shitposter.zone/users/shp"] }) + insert(:instance, %{host: "example.com", nodeinfo: %{version: "2.1"}}) + expected = %{ id: to_string(user.id), username: "shp", @@ -50,6 +54,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do statuses_count: 5, note: "valid html. a
b
c
d
f '&<>"", url: user.ap_id, + akkoma: %{ + instance: %{ + name: "example.com", + nodeinfo: %{ + "version" => "2.1" + }, + favicon: nil + } + }, avatar: "http://localhost:4001/images/avi.png", avatar_static: "http://localhost:4001/images/avi.png", header: "http://localhost:4001/images/banner.png", @@ -98,9 +111,57 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do assert expected == AccountView.render("show.json", %{user: user, skip_visibility_check: true}) end + describe "nodeinfo" do + setup do + [ + user: insert(:user, ap_id: "https://somewhere.example.com/users/chikichikibanban"), + instance: + insert(:instance, %{ + host: "somewhere.example.com", + favicon: "https://example.com/favicon.ico" + }) + ] + end + + test "is embedded in the account view", %{user: user} do + assert %{ + akkoma: %{ + instance: %{ + name: "somewhere.example.com", + nodeinfo: %{ + "version" => "2.0" + }, + favicon: "https://example.com/favicon.ico" + } + } + } = AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + end + + test "uses local nodeinfo for local users" do + user = insert(:user) + + assert %{ + akkoma: %{ + instance: %{ + name: "localhost", + nodeinfo: %{ + software: %{ + name: "akkoma" + } + } + } + } + } = AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + end + end + describe "favicon" do setup do - [user: insert(:user)] + [ + user: insert(:user, ap_id: "https://example.com/users/chikichikibanban"), + instance: + insert(:instance, %{host: "example.com", favicon: "https://example.com/favicon.ico"}) + ] end test "is parsed when :instance_favicons is enabled", %{user: user} do @@ -108,13 +169,14 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do assert %{ pleroma: %{ - favicon: - "https://shitposter.club/plugins/Qvitter/img/gnusocial-favicons/favicon-16x16.png" + favicon: "https://example.com/favicon.ico" } } = AccountView.render("show.json", %{user: user, skip_visibility_check: true}) end - test "is nil when :instances_favicons is disabled", %{user: user} do + test "is nil when we have no instance", %{user: user} do + user = %{user | ap_id: "https://wowee.example.com/users/2"} + assert %{pleroma: %{favicon: nil}} = AccountView.render("show.json", %{user: user, skip_visibility_check: true}) end @@ -176,11 +238,18 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do }, fqn: "shp@shitposter.club", last_status_at: nil, + akkoma: %{ + instance: %{ + name: "localhost", + favicon: "http://localhost:4001/favicon.png", + nodeinfo: %{version: "2.0"} + } + }, pleroma: %{ ap_id: user.ap_id, also_known_as: [], background_image: nil, - favicon: nil, + favicon: "http://localhost:4001/favicon.png", is_confirmed: true, tags: [], is_admin: false, @@ -196,7 +265,13 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do } } - assert expected == AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + with_mock( + Pleroma.Web.Nodeinfo.NodeinfoController, + raw_nodeinfo: fn -> %{version: "2.0"} end + ) do + assert expected == + AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + end end test "Represent a Funkwhale channel" do @@ -578,6 +653,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do emoji: %{"joker_smile" => "https://evil.website/society.png"} ) + insert(:instance, %{host: "localhost", favicon: "https://evil.website/favicon.png"}) + with media_preview_enabled <- [false, true] do clear_config([:media_preview_proxy, :enabled], media_preview_enabled) @@ -586,6 +663,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do {key, url} when key in [:avatar, :avatar_static, :header, :header_static] -> String.starts_with?(url, Pleroma.Web.Endpoint.url()) + {:akkoma, %{instance: %{favicon: favicon_url}}} -> + String.starts_with?(favicon_url, Pleroma.Web.Endpoint.url()) + {:emojis, emojis} -> Enum.all?(emojis, fn %{url: url, static_url: static_url} -> String.starts_with?(url, Pleroma.Web.Endpoint.url()) && @@ -598,4 +678,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do |> assert() end end + + test "returns nil in the instance field when no instance is held locally" do + user = insert(:user, ap_id: "https://example.com/users/1") + view = AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + assert view[:akkoma][:instance] == nil + end end diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs index 8e2ab5016..a9db5a015 100644 --- a/test/pleroma/web/streamer_test.exs +++ b/test/pleroma/web/streamer_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.StreamerTest do - use Pleroma.DataCase + use Pleroma.DataCase, async: false import Pleroma.Factory diff --git a/test/support/factory.ex b/test/support/factory.ex index 54d385bc4..bd9d7fe42 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -36,6 +36,15 @@ defmodule Pleroma.Factory do } end + def instance_factory(attrs \\ %{}) do + %Pleroma.Instances.Instance{ + host: attrs[:domain] || "example.com", + nodeinfo: %{version: "2.0", openRegistrations: true}, + unreachable_since: nil + } + |> Map.merge(attrs) + end + def user_factory(attrs \\ %{}) do pem = Enum.random(@rsa_keys) @@ -522,13 +531,6 @@ defmodule Pleroma.Factory do } end - def instance_factory do - %Pleroma.Instances.Instance{ - host: "domain.com", - unreachable_since: nil - } - end - def oauth_token_factory(attrs \\ %{}) do scopes = Map.get(attrs, :scopes, ["read"]) oauth_app = Map.get_lazy(attrs, :app, fn -> insert(:oauth_app, scopes: scopes) end) From bbedbaaf5cf1b9a76c656e7ef26df4dec2cc91bf Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sun, 6 Nov 2022 22:50:11 +0000 Subject: [PATCH 29/57] Changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34c09cd65..6bc87ada1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Added - Officially supported docker release - Ability to remove followers unilaterally without a block +- Scraping of nodeinfo from remote instances to display instance info ## Changes - Follows no longer override domain blocks, a domain block is final From 5123b3a5dd9683441f5b4e904deda89fd40098ac Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sun, 6 Nov 2022 22:55:26 +0000 Subject: [PATCH 30/57] Add enabled check on /translation/languages --- .../web/akkoma_api/controllers/translation_controller.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/akkoma_api/controllers/translation_controller.ex b/lib/pleroma/web/akkoma_api/controllers/translation_controller.ex index 9983a7e39..ca9b4b64a 100644 --- a/lib/pleroma/web/akkoma_api/controllers/translation_controller.ex +++ b/lib/pleroma/web/akkoma_api/controllers/translation_controller.ex @@ -21,10 +21,12 @@ defmodule Pleroma.Web.AkkomaAPI.TranslationController do @doc "GET /api/v1/akkoma/translation/languages" def languages(conn, _params) do - with {:ok, source_languages, dest_languages} <- get_languages() do + with {:enabled, true} <- {:enabled, Pleroma.Config.get([:translator, :enabled])}, + {:ok, source_languages, dest_languages} <- get_languages() do conn |> json(%{source: source_languages, target: dest_languages}) else + {:enabled, false} -> json(conn, %{}) e -> IO.inspect(e) end end From 31ad09010e03f30643b9e921c4fd85c113071a4d Mon Sep 17 00:00:00 2001 From: floatingghost Date: Sun, 6 Nov 2022 23:50:32 +0000 Subject: [PATCH 31/57] Fix regex usage in MRF (#254) fixes #235 fixes #228 Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/254 --- CHANGELOG.md | 4 ++ lib/pleroma/web/activity_pub/mrf.ex | 11 ++- test/pleroma/user_test.exs | 29 ++++---- .../activity_pub/mrf/simple_policy_test.exs | 68 +++++++++---------- test/pleroma/web/activity_pub/mrf_test.exs | 16 ++--- 5 files changed, 72 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6bc87ada1..05d24a821 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased +## UPGRADE NOTES +- Change your instance blocks to remove any `*.` prefixes. `example.com` will block `*.example.com` by default now + ## Added - Officially supported docker release - Ability to remove followers unilaterally without a block @@ -14,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Changes - Follows no longer override domain blocks, a domain block is final - Deletes are now the lowest priority to publish and will be handled after creates +- Domain blocks are now subdomain-matches by default ## Fixed - Registrations via ldap are now compatible with the latest OTP24 diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 4df226e80..7b7f44646 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -149,9 +149,18 @@ defmodule Pleroma.Web.ActivityPub.MRF do defp get_policies(policies) when is_list(policies), do: policies defp get_policies(_), do: [] + # Matches the following: + # - https://baddomain.net + # - https://extra.baddomain.net/ + # Does NOT match the following: + # - https://maybebaddomain.net/ + def subdomain_regex(domain) do + ~r/^(.+\.)?#{Regex.escape(domain)}$/i + end + @spec subdomains_regex([String.t()]) :: [Regex.t()] def subdomains_regex(domains) when is_list(domains) do - for domain <- domains, do: ~r(^#{String.replace(domain, "*.", "(.*\\.)*")}$)i + Enum.map(domains, &subdomain_regex/1) end @spec subdomain_match?([Regex.t()], String.t()) :: boolean() diff --git a/test/pleroma/user_test.exs b/test/pleroma/user_test.exs index f11aec136..195df2a03 100644 --- a/test/pleroma/user_test.exs +++ b/test/pleroma/user_test.exs @@ -444,17 +444,20 @@ defmodule Pleroma.UserTest do end setup do: - clear_config(:mrf_simple, - media_removal: [], - media_nsfw: [], - federated_timeline_removal: [], - report_removal: [], - reject: [], - followers_only: [], - accept: [], - avatar_removal: [], - banner_removal: [], - reject_deletes: [] + clear_config( + [:mrf_simple], + %{ + media_removal: [], + media_nsfw: [], + federated_timeline_removal: [], + report_removal: [], + reject: [], + followers_only: [], + accept: [], + avatar_removal: [], + banner_removal: [], + reject_deletes: [] + } ) setup do: @@ -1324,7 +1327,7 @@ defmodule Pleroma.UserTest do collateral_user = insert(:user, %{ap_id: "https://another-awful-and-rude-instance.com/user/bully"}) - {:ok, user} = User.block_domain(user, "*.awful-and-rude-instance.com") + {:ok, user} = User.block_domain(user, "awful-and-rude-instance.com") refute User.blocks?(user, collateral_user) end @@ -1342,7 +1345,7 @@ defmodule Pleroma.UserTest do user_domain = insert(:user, %{ap_id: "https://awful-and-rude-instance.com/user/bully"}) - {:ok, user} = User.block_domain(user, "*.awful-and-rude-instance.com") + {:ok, user} = User.block_domain(user, "awful-and-rude-instance.com") assert User.blocks?(user, user_from_subdomain) assert User.blocks?(user, user_with_two_subdomains) diff --git a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs index 0569bfed3..5f80c1629 100644 --- a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs @@ -46,8 +46,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "match with wildcard domain" do - clear_config([:mrf_simple, :media_removal], [{"*.remote.instance", "Whatever reason"}]) - media_message = build_media_message() + clear_config([:mrf_simple, :media_removal], [{"remote.instance", "Whatever reason"}]) + media_message = build_media_message("sub.remote.instance") local_message = build_local_message() assert SimplePolicy.filter(media_message) == @@ -81,8 +81,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "match with wildcard domain" do - clear_config([:mrf_simple, :media_nsfw], [{"*.remote.instance", "yeah yeah"}]) - media_message = build_media_message() + clear_config([:mrf_simple, :media_nsfw], [{"remote.instance", "yeah yeah"}]) + media_message = build_media_message("sub.remote.instance") local_message = build_local_message() assert SimplePolicy.filter(media_message) == @@ -92,9 +92,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end end - defp build_media_message do + defp build_media_message(domain \\ "remote.instance") do %{ - "actor" => "https://remote.instance/users/bob", + "actor" => "https://#{domain}/users/bob", "type" => "Create", "object" => %{ "attachment" => [%{}], @@ -124,8 +124,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "match with wildcard domain" do - clear_config([:mrf_simple, :report_removal], [{"*.remote.instance", "suya"}]) - report_message = build_report_message() + clear_config([:mrf_simple, :report_removal], [{"remote.instance", "suya"}]) + report_message = build_report_message("sub.remote.instance") local_message = build_local_message() assert {:reject, _} = SimplePolicy.filter(report_message) @@ -133,9 +133,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end end - defp build_report_message do + defp build_report_message(domain \\ "remote.instance") do %{ - "actor" => "https://remote.instance/users/bob", + "actor" => "https://#{domain}/users/bob", "type" => "Flag" } end @@ -143,7 +143,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do describe "when :federated_timeline_removal" do test "is empty" do clear_config([:mrf_simple, :federated_timeline_removal], []) - {_, ftl_message} = build_ftl_actor_and_message() + {_, ftl_message} = build_ftl_actor_and_message("https://remote.instance/users/bob") local_message = build_local_message() assert SimplePolicy.filter(ftl_message) == {:ok, ftl_message} @@ -151,7 +151,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "has a matching host" do - {actor, ftl_message} = build_ftl_actor_and_message() + {actor, ftl_message} = build_ftl_actor_and_message("https://remote.instance/users/bob") ftl_message_actor_host = ftl_message @@ -172,7 +172,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "match with wildcard domain" do - {actor, ftl_message} = build_ftl_actor_and_message() + {actor, ftl_message} = build_ftl_actor_and_message("https://sub.remote.instance/users/bob") ftl_message_actor_host = ftl_message @@ -181,7 +181,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do |> Map.fetch!(:host) clear_config([:mrf_simple, :federated_timeline_removal], [ - {"*." <> ftl_message_actor_host, "owo"} + {ftl_message_actor_host, "owo"} ]) local_message = build_local_message() @@ -196,7 +196,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "has a matching host but only as:Public in to" do - {_actor, ftl_message} = build_ftl_actor_and_message() + {_actor, ftl_message} = build_ftl_actor_and_message("https://remote.instance/users/bob") ftl_message_actor_host = ftl_message @@ -253,8 +253,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end end - defp build_ftl_actor_and_message do - actor = insert(:user) + defp build_ftl_actor_and_message(ap_id) do + actor = insert(:user, ap_id: ap_id) {actor, %{ @@ -282,9 +282,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "activity matches with wildcard domain" do - clear_config([:mrf_simple, :reject], [{"*.remote.instance", ""}]) + clear_config([:mrf_simple, :reject], [{"remote.instance", ""}]) - remote_message = build_remote_message() + remote_message = build_remote_message("sub.remote.instance") assert {:reject, _} = SimplePolicy.filter(remote_message) end @@ -325,7 +325,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do describe "when :followers_only" do test "is empty" do clear_config([:mrf_simple, :followers_only], []) - {_, ftl_message} = build_ftl_actor_and_message() + {_, ftl_message} = build_ftl_actor_and_message("https://remote.instance/users/alice") local_message = build_local_message() assert SimplePolicy.filter(ftl_message) == {:ok, ftl_message} @@ -412,10 +412,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "activity matches with wildcard domain" do - clear_config([:mrf_simple, :accept], [{"*.remote.instance", ""}]) + clear_config([:mrf_simple, :accept], [{"remote.instance", ""}]) local_message = build_local_message() - remote_message = build_remote_message() + remote_message = build_remote_message("sub.remote.instance") assert SimplePolicy.filter(local_message) == {:ok, local_message} assert SimplePolicy.filter(remote_message) == {:ok, remote_message} @@ -457,9 +457,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "match with wildcard domain" do - clear_config([:mrf_simple, :avatar_removal], [{"*.remote.instance", ""}]) + clear_config([:mrf_simple, :avatar_removal], [{"remote.instance", ""}]) - remote_user = build_remote_user() + remote_user = build_remote_user("sub.remote.instance") {:ok, filtered} = SimplePolicy.filter(remote_user) refute filtered["icon"] @@ -493,9 +493,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "match with wildcard domain" do - clear_config([:mrf_simple, :banner_removal], [{"*.remote.instance", ""}]) + clear_config([:mrf_simple, :banner_removal], [{"remote.instance", ""}]) - remote_user = build_remote_user() + remote_user = build_remote_user("sub.remote.instance") {:ok, filtered} = SimplePolicy.filter(remote_user) refute filtered["image"] @@ -553,10 +553,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end describe "when :reject_deletes match with wildcard domain" do - setup do: clear_config([:mrf_simple, :reject_deletes], [{"*.remote.instance", ""}]) + setup do: clear_config([:mrf_simple, :reject_deletes], [{"remote.instance", ""}]) test "it rejects the deletion" do - deletion_message = build_remote_deletion_message() + deletion_message = build_remote_deletion_message("sub.remote.instance") assert {:reject, _} = SimplePolicy.filter(deletion_message) end @@ -570,13 +570,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do } end - defp build_remote_message do - %{"actor" => "https://remote.instance/users/bob"} + defp build_remote_message(domain \\ "remote.instance") do + %{"actor" => "https://#{domain}/users/bob"} end - defp build_remote_user do + defp build_remote_user(domain \\ "remote.instance") do %{ - "id" => "https://remote.instance/users/bob", + "id" => "https://#{domain}/users/bob", "icon" => %{ "url" => "http://example.com/image.jpg", "type" => "Image" @@ -589,10 +589,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do } end - defp build_remote_deletion_message do + defp build_remote_deletion_message(domain \\ "remote.instance") do %{ "type" => "Delete", - "actor" => "https://remote.instance/users/bob" + "actor" => "https://#{domain}/users/bob" } end end diff --git a/test/pleroma/web/activity_pub/mrf_test.exs b/test/pleroma/web/activity_pub/mrf_test.exs index ed3233758..ec4dab30f 100644 --- a/test/pleroma/web/activity_pub/mrf_test.exs +++ b/test/pleroma/web/activity_pub/mrf_test.exs @@ -9,8 +9,8 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "subdomains_regex/1" do assert MRF.subdomains_regex(["unsafe.tld", "*.unsafe.tld"]) == [ - ~r/^unsafe.tld$/i, - ~r/^(.*\.)*unsafe.tld$/i + ~r/^(.+\.)?unsafe\.tld$/i, + ~r/^(.+\.)?\*\.unsafe\.tld$/i ] end @@ -18,7 +18,7 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "common domains" do regexes = MRF.subdomains_regex(["unsafe.tld", "unsafe2.tld"]) - assert regexes == [~r/^unsafe.tld$/i, ~r/^unsafe2.tld$/i] + assert regexes == [~r/^(.+\.)?unsafe\.tld$/i, ~r/^(.+\.)?unsafe2\.tld$/i] assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "unsafe2.tld") @@ -27,9 +27,9 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do end test "wildcard domains with one subdomain" do - regexes = MRF.subdomains_regex(["*.unsafe.tld"]) + regexes = MRF.subdomains_regex(["unsafe.tld"]) - assert regexes == [~r/^(.*\.)*unsafe.tld$/i] + assert regexes == [~r/^(.+\.)?unsafe\.tld$/i] assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "sub.unsafe.tld") @@ -38,9 +38,9 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do end test "wildcard domains with two subdomains" do - regexes = MRF.subdomains_regex(["*.unsafe.tld"]) + regexes = MRF.subdomains_regex(["unsafe.tld"]) - assert regexes == [~r/^(.*\.)*unsafe.tld$/i] + assert regexes == [~r/^(.+\.)?unsafe\.tld$/i] assert MRF.subdomain_match?(regexes, "unsafe.tld") assert MRF.subdomain_match?(regexes, "sub.sub.unsafe.tld") @@ -51,7 +51,7 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "matches are case-insensitive" do regexes = MRF.subdomains_regex(["UnSafe.TLD", "UnSAFE2.Tld"]) - assert regexes == [~r/^UnSafe.TLD$/i, ~r/^UnSAFE2.Tld$/i] + assert regexes == [~r/^(.+\.)?UnSafe\.TLD$/i, ~r/^(.+\.)?UnSAFE2\.Tld$/i] assert MRF.subdomain_match?(regexes, "UNSAFE.TLD") assert MRF.subdomain_match?(regexes, "UNSAFE2.TLD") From 48309c141e8f555c277136122fe339534f88d39a Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sun, 6 Nov 2022 23:57:43 +0000 Subject: [PATCH 32/57] Add "differences" in readme --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index c3ead7fc1..74b8919e4 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,27 @@ For clients it supports the [Mastodon client API](https://docs.joinmastodon.org/ - [Client Applications for Akkoma](https://docs.akkoma.dev/stable/clients/) +## Differences with Pleroma + +Akkoma is a faster-paced fork, it has a varied and potentially experimental feature set tailored specifically to the corner of the fediverse inhabited by the project +creator and contributors. + +This should not be considered a one-for-one match with pleroma; it is more opinionated in many ways, and has a smaller community (which is good or +bad depending on your view) + +For example, Akkoma has: +- Custom Emoji reactions (compatible with misskey) +- Misskey-flavoured markdown support +- Elasticsearch and Meilisearch support for search +- Mastodon frontend (Glitch-Soc and Fedibird flavours) support +- Automatic post translation via DeepL or LibreTranslate +- A multitude of heavy modifications to the Pleroma Frontend (Pleroma-FE) +- The "bubble" concept, in which instance administrators can choose closely-related instances to make a "community of communities", so to say + +And takes a more opinionated stance on issues like Domain blocks, which are enforced far more on Akkoma. + +Take a look at the Changelog if you want a full list of recent changes, everything since 3.0 has been Akkoma. + ## Installation ### OTP releases (Recommended) From e0032e47992bca069e61a8208c322c0eb8a0c9e3 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 7 Nov 2022 00:08:20 +0000 Subject: [PATCH 33/57] Add rollbacks for associated_object_id --- docs/docs/installation/migrating_to_akkoma.md | 9 + ...2322_add_associated_object_id_function.exs | 37 +++++ ...0_switch_to_associated_object_id_index.exs | 37 +++++ ..._visibility_to_use_new_object_id_index.exs | 156 ++++++++++++++++++ 4 files changed, 239 insertions(+) create mode 100644 priv/repo/optional_migrations/pleroma_develop_rollbacks/20220711182322_add_associated_object_id_function.exs create mode 100644 priv/repo/optional_migrations/pleroma_develop_rollbacks/20220711192750_switch_to_associated_object_id_index.exs create mode 100644 priv/repo/optional_migrations/pleroma_develop_rollbacks/20220821004840_change_thread_visibility_to_use_new_object_id_index.exs diff --git a/docs/docs/installation/migrating_to_akkoma.md b/docs/docs/installation/migrating_to_akkoma.md index d8ea0ea25..b64cdd056 100644 --- a/docs/docs/installation/migrating_to_akkoma.md +++ b/docs/docs/installation/migrating_to_akkoma.md @@ -34,6 +34,15 @@ git pull -r # to run "git merge stable" instead (or develop if you want) ``` +### WARNING - Migrating from Pleroma Develop +If you are on pleroma develop, and have updated since 2022-08, you may have issues with database migrations. + +Please roll back the given migrations: + +```bash +MIX_ENV=prod mix ecto.rollback --migrations-path priv/repo/optional_migrations/pleroma_develop_rollbacks -n3 +``` + Then compile, migrate and restart as usual. ## From OTP diff --git a/priv/repo/optional_migrations/pleroma_develop_rollbacks/20220711182322_add_associated_object_id_function.exs b/priv/repo/optional_migrations/pleroma_develop_rollbacks/20220711182322_add_associated_object_id_function.exs new file mode 100644 index 000000000..76348f31a --- /dev/null +++ b/priv/repo/optional_migrations/pleroma_develop_rollbacks/20220711182322_add_associated_object_id_function.exs @@ -0,0 +1,37 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.AddAssociatedObjectIdFunction do + use Ecto.Migration + + def up do + statement = """ + CREATE OR REPLACE FUNCTION associated_object_id(data jsonb) RETURNS varchar AS $$ + DECLARE + object_data jsonb; + BEGIN + IF jsonb_typeof(data->'object') = 'array' THEN + object_data := data->'object'->0; + ELSE + object_data := data->'object'; + END IF; + + IF jsonb_typeof(object_data->'id') = 'string' THEN + RETURN object_data->>'id'; + ELSIF jsonb_typeof(object_data) = 'string' THEN + RETURN object_data#>>'{}'; + ELSE + RETURN NULL; + END IF; + END; + $$ LANGUAGE plpgsql IMMUTABLE; + """ + + execute(statement) + end + + def down do + execute("DROP FUNCTION IF EXISTS associated_object_id(data jsonb)") + end +end diff --git a/priv/repo/optional_migrations/pleroma_develop_rollbacks/20220711192750_switch_to_associated_object_id_index.exs b/priv/repo/optional_migrations/pleroma_develop_rollbacks/20220711192750_switch_to_associated_object_id_index.exs new file mode 100644 index 000000000..75c1cd40b --- /dev/null +++ b/priv/repo/optional_migrations/pleroma_develop_rollbacks/20220711192750_switch_to_associated_object_id_index.exs @@ -0,0 +1,37 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.SwitchToAssociatedObjectIdIndex do + use Ecto.Migration + @disable_ddl_transaction true + @disable_migration_lock true + + def up do + drop_if_exists( + index(:activities, ["(coalesce(data->'object'->>'id', data->>'object'))"], + name: :activities_create_objects_index + ) + ) + + create( + index(:activities, ["associated_object_id(data)"], + name: :activities_create_objects_index, + concurrently: true + ) + ) + end + + def down do + drop_if_exists( + index(:activities, ["associated_object_id(data)"], name: :activities_create_objects_index) + ) + + create( + index(:activities, ["(coalesce(data->'object'->>'id', data->>'object'))"], + name: :activities_create_objects_index, + concurrently: true + ) + ) + end +end diff --git a/priv/repo/optional_migrations/pleroma_develop_rollbacks/20220821004840_change_thread_visibility_to_use_new_object_id_index.exs b/priv/repo/optional_migrations/pleroma_develop_rollbacks/20220821004840_change_thread_visibility_to_use_new_object_id_index.exs new file mode 100644 index 000000000..bb56843cb --- /dev/null +++ b/priv/repo/optional_migrations/pleroma_develop_rollbacks/20220821004840_change_thread_visibility_to_use_new_object_id_index.exs @@ -0,0 +1,156 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.ChangeThreadVisibilityToUseNewObjectIdIndex do + use Ecto.Migration + + def up do + execute(update_thread_visibility()) + end + + def down do + execute(restore_thread_visibility()) + end + + def update_thread_visibility do + """ + CREATE OR REPLACE FUNCTION thread_visibility(actor varchar, activity_id varchar, local_public varchar default '') RETURNS boolean AS $$ + DECLARE + public varchar := 'https://www.w3.org/ns/activitystreams#Public'; + child objects%ROWTYPE; + activity activities%ROWTYPE; + author_fa varchar; + valid_recipients varchar[]; + actor_user_following varchar[]; + BEGIN + --- Fetch actor following + SELECT array_agg(following.follower_address) INTO actor_user_following FROM following_relationships + JOIN users ON users.id = following_relationships.follower_id + JOIN users AS following ON following.id = following_relationships.following_id + WHERE users.ap_id = actor; + + --- Fetch our initial activity. + SELECT * INTO activity FROM activities WHERE activities.data->>'id' = activity_id; + + LOOP + --- Ensure that we have an activity before continuing. + --- If we don't, the thread is not satisfiable. + IF activity IS NULL THEN + RETURN false; + END IF; + + --- We only care about Create activities. + IF activity.data->>'type' != 'Create' THEN + RETURN true; + END IF; + + --- Normalize the child object into child. + SELECT * INTO child FROM objects + INNER JOIN activities ON associated_object_id(activities.data) = objects.data->>'id' + WHERE associated_object_id(activity.data) = objects.data->>'id'; + + --- Fetch the author's AS2 following collection. + SELECT COALESCE(users.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor; + + --- Prepare valid recipients array. + valid_recipients := ARRAY[actor, public]; + --- If we specified local public, add it. + IF local_public <> '' THEN + valid_recipients := valid_recipients || local_public; + END IF; + IF ARRAY[author_fa] && actor_user_following THEN + valid_recipients := valid_recipients || author_fa; + END IF; + + --- Check visibility. + IF NOT valid_recipients && activity.recipients THEN + --- activity not visible, break out of the loop + RETURN false; + END IF; + + --- If there's a parent, load it and do this all over again. + IF (child.data->'inReplyTo' IS NOT NULL) AND (child.data->'inReplyTo' != 'null'::jsonb) THEN + SELECT * INTO activity FROM activities + INNER JOIN objects ON associated_object_id(activities.data) = objects.data->>'id' + WHERE child.data->>'inReplyTo' = objects.data->>'id'; + ELSE + RETURN true; + END IF; + END LOOP; + END; + $$ LANGUAGE plpgsql IMMUTABLE; + """ + end + + # priv/repo/migrations/20220509180452_change_thread_visibility_to_be_local_only_aware.exs + def restore_thread_visibility do + """ + CREATE OR REPLACE FUNCTION thread_visibility(actor varchar, activity_id varchar, local_public varchar default '') RETURNS boolean AS $$ + DECLARE + public varchar := 'https://www.w3.org/ns/activitystreams#Public'; + child objects%ROWTYPE; + activity activities%ROWTYPE; + author_fa varchar; + valid_recipients varchar[]; + actor_user_following varchar[]; + BEGIN + --- Fetch actor following + SELECT array_agg(following.follower_address) INTO actor_user_following FROM following_relationships + JOIN users ON users.id = following_relationships.follower_id + JOIN users AS following ON following.id = following_relationships.following_id + WHERE users.ap_id = actor; + + --- Fetch our initial activity. + SELECT * INTO activity FROM activities WHERE activities.data->>'id' = activity_id; + + LOOP + --- Ensure that we have an activity before continuing. + --- If we don't, the thread is not satisfiable. + IF activity IS NULL THEN + RETURN false; + END IF; + + --- We only care about Create activities. + IF activity.data->>'type' != 'Create' THEN + RETURN true; + END IF; + + --- Normalize the child object into child. + SELECT * INTO child FROM objects + INNER JOIN activities ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' + WHERE COALESCE(activity.data->'object'->>'id', activity.data->>'object') = objects.data->>'id'; + + --- Fetch the author's AS2 following collection. + SELECT COALESCE(users.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor; + + --- Prepare valid recipients array. + valid_recipients := ARRAY[actor, public]; + --- If we specified local public, add it. + IF local_public <> '' THEN + valid_recipients := valid_recipients || local_public; + END IF; + IF ARRAY[author_fa] && actor_user_following THEN + valid_recipients := valid_recipients || author_fa; + END IF; + + --- Check visibility. + IF NOT valid_recipients && activity.recipients THEN + --- activity not visible, break out of the loop + RETURN false; + END IF; + + --- If there's a parent, load it and do this all over again. + IF (child.data->'inReplyTo' IS NOT NULL) AND (child.data->'inReplyTo' != 'null'::jsonb) THEN + SELECT * INTO activity FROM activities + INNER JOIN objects ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' + WHERE child.data->>'inReplyTo' = objects.data->>'id'; + ELSE + RETURN true; + END IF; + END LOOP; + END; + $$ LANGUAGE plpgsql IMMUTABLE; + """ + end +end From c0eecb55bf67afe0e199482a52d38d4d976e4bd6 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 7 Nov 2022 13:32:34 +0000 Subject: [PATCH 34/57] Update finch --- mix.exs | 2 +- mix.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mix.exs b/mix.exs index 108930f2c..d95cc1778 100644 --- a/mix.exs +++ b/mix.exs @@ -139,7 +139,7 @@ defmodule Pleroma.Mixfile do {:castore, "~> 0.1"}, {:cowlib, "~> 2.9", override: true}, {:gun, "~> 2.0.0-rc.1", override: true}, - {:finch, "~> 0.10.0"}, + {:finch, "~> 0.13.0"}, {:jason, "~> 1.2"}, {:mogrify, "~> 0.9.1"}, {:ex_aws, "~> 2.1.6"}, diff --git a/mix.lock b/mix.lock index d0d20f7d3..d3400da7a 100644 --- a/mix.lock +++ b/mix.lock @@ -7,7 +7,7 @@ "cachex": {:hex, :cachex, "3.4.0", "868b2959ea4aeb328c6b60ff66c8d5123c083466ad3c33d3d8b5f142e13101fb", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "370123b1ab4fba4d2965fb18f87fd758325709787c8c5fce35b3fe80645ccbe5"}, "calendar": {:hex, :calendar, "1.0.0", "f52073a708528482ec33d0a171954ca610fe2bd28f1e871f247dc7f1565fa807", [:mix], [{:tzdata, "~> 0.5.20 or ~> 0.1.201603 or ~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "990e9581920c82912a5ee50e62ff5ef96da6b15949a2ee4734f935fdef0f0a6f"}, "captcha": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", "e0f16822d578866e186a0974d65ad58cddc1e2ab", [ref: "e0f16822d578866e186a0974d65ad58cddc1e2ab"]}, - "castore": {:hex, :castore, "0.1.17", "ba672681de4e51ed8ec1f74ed624d104c0db72742ea1a5e74edbc770c815182f", [:mix], [], "hexpm", "d9844227ed52d26e7519224525cb6868650c272d4a3d327ce3ca5570c12163f9"}, + "castore": {:hex, :castore, "0.1.18", "deb5b9ab02400561b6f5708f3e7660fc35ca2d51bfc6a940d2f513f89c2975fc", [:mix], [], "hexpm", "61bbaf6452b782ef80b33cdb45701afbcf0a918a45ebe7e73f1130d661e66a06"}, "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "comeonin": {:hex, :comeonin, "5.3.3", "2c564dac95a35650e9b6acfe6d2952083d8a08e4a89b93a481acb552b325892e", [:mix], [], "hexpm", "3e38c9c2cb080828116597ca8807bb482618a315bfafd98c90bc22a821cc84df"}, @@ -43,14 +43,14 @@ "fast_html": {:hex, :fast_html, "2.0.5", "c61760340606c1077ff1f196f17834056cb1dd3d5cb92a9f2cabf28bc6221c3c", [:make, :mix], [{:elixir_make, "~> 0.4", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}], "hexpm", "605f4f4829443c14127694ebabb681778712ceecb4470ec32aa31012330e6506"}, "fast_sanitize": {:hex, :fast_sanitize, "0.2.3", "67b93dfb34e302bef49fec3aaab74951e0f0602fd9fa99085987af05bd91c7a5", [:mix], [{:fast_html, "~> 2.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "e8ad286d10d0386e15d67d0ee125245ebcfbc7d7290b08712ba9013c8c5e56e2"}, "file_system": {:hex, :file_system, "0.2.10", "fb082005a9cd1711c05b5248710f8826b02d7d1784e7c3451f9c1231d4fc162d", [:mix], [], "hexpm", "41195edbfb562a593726eda3b3e8b103a309b733ad25f3d642ba49696bf715dc"}, - "finch": {:hex, :finch, "0.10.2", "9ad27d68270d879f73f26604bb2e573d40f29bf0e907064a9a337f90a16a0312", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dd8b11b282072cec2ef30852283949c248bd5d2820c88d8acc89402b81db7550"}, + "finch": {:hex, :finch, "0.13.0", "c881e5460ec563bf02d4f4584079e62201db676ed4c0ef3e59189331c4eddf7b", [:mix], [{:castore, "~> 0.1", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "49957dcde10dcdc042a123a507a9c5ec5a803f53646d451db2f7dea696fba6cc"}, "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "31fc8090fde1acd267c07c36ea7365b8604055f897d3a53dd967658c691bd827"}, "floki": {:hex, :floki, "0.33.1", "f20f1eb471e726342b45ccb68edb9486729e7df94da403936ea94a794f072781", [:mix], [{:html_entities, "~> 0.5.0", [hex: :html_entities, repo: "hexpm", optional: false]}], "hexpm", "461035fd125f13fdf30f243c85a0b1e50afbec876cbf1ceefe6fddd2e6d712c6"}, "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm", "29bd14a88030980849c7ed2447b8db6d6c9278a28b11a44cafe41b791205440f"}, "gettext": {:git, "https://github.com/tusooa/gettext.git", "72fb2496b6c5280ed911bdc3756890e7f38a4808", [ref: "72fb2496b6c5280ed911bdc3756890e7f38a4808"]}, "gun": {:hex, :gun, "2.0.0-rc.2", "7c489a32dedccb77b6e82d1f3c5a7dadfbfa004ec14e322cdb5e579c438632d2", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}], "hexpm", "6b9d1eae146410d727140dbf8b404b9631302ecc2066d1d12f22097ad7d254fc"}, "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~>2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"}, - "hpax": {:hex, :hpax, "0.1.1", "2396c313683ada39e98c20a75a82911592b47e5c24391363343bde74f82396ca", [:mix], [], "hexpm", "0ae7d5a0b04a8a60caf7a39fcf3ec476f35cc2cc16c05abea730d3ce6ac6c826"}, + "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"}, "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"}, "http_signatures": {:hex, :http_signatures, "0.1.1", "ca7ebc1b61542b163644c8c3b1f0e0f41037d35f2395940d3c6c7deceab41fd8", [:mix], [], "hexpm", "cc3b8a007322cc7b624c0c15eec49ee58ac977254ff529a3c482f681465942a3"}, "httpoison": {:hex, :httpoison, "1.8.1", "df030d96de89dad2e9983f92b0c506a642d4b1f4a819c96ff77d12796189c63e", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "35156a6d678d6d516b9229e208942c405cf21232edd632327ecfaf4fd03e79e0"}, From 7bbaa8f8e07044726c5b97b9ab8126ca41f2fd78 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 7 Nov 2022 22:33:18 +0000 Subject: [PATCH 35/57] automatically trim loading *. prefixes on domain blocks --- CHANGELOG.md | 3 - lib/pleroma/web/activity_pub/mrf.ex | 2 + .../activity_pub/mrf/simple_policy_test.exs | 68 +++++++++---------- test/pleroma/web/activity_pub/mrf_test.exs | 2 +- 4 files changed, 37 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05d24a821..15472eda6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Unreleased -## UPGRADE NOTES -- Change your instance blocks to remove any `*.` prefixes. `example.com` will block `*.example.com` by default now - ## Added - Officially supported docker release - Ability to remove followers unilaterally without a block diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 7b7f44646..0d4986e68 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -154,6 +154,8 @@ defmodule Pleroma.Web.ActivityPub.MRF do # - https://extra.baddomain.net/ # Does NOT match the following: # - https://maybebaddomain.net/ + def subdomain_regex("*." <> domain), do: subdomain_regex(domain) + def subdomain_regex(domain) do ~r/^(.+\.)?#{Regex.escape(domain)}$/i end diff --git a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs index 5f80c1629..0569bfed3 100644 --- a/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/pleroma/web/activity_pub/mrf/simple_policy_test.exs @@ -46,8 +46,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "match with wildcard domain" do - clear_config([:mrf_simple, :media_removal], [{"remote.instance", "Whatever reason"}]) - media_message = build_media_message("sub.remote.instance") + clear_config([:mrf_simple, :media_removal], [{"*.remote.instance", "Whatever reason"}]) + media_message = build_media_message() local_message = build_local_message() assert SimplePolicy.filter(media_message) == @@ -81,8 +81,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "match with wildcard domain" do - clear_config([:mrf_simple, :media_nsfw], [{"remote.instance", "yeah yeah"}]) - media_message = build_media_message("sub.remote.instance") + clear_config([:mrf_simple, :media_nsfw], [{"*.remote.instance", "yeah yeah"}]) + media_message = build_media_message() local_message = build_local_message() assert SimplePolicy.filter(media_message) == @@ -92,9 +92,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end end - defp build_media_message(domain \\ "remote.instance") do + defp build_media_message do %{ - "actor" => "https://#{domain}/users/bob", + "actor" => "https://remote.instance/users/bob", "type" => "Create", "object" => %{ "attachment" => [%{}], @@ -124,8 +124,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "match with wildcard domain" do - clear_config([:mrf_simple, :report_removal], [{"remote.instance", "suya"}]) - report_message = build_report_message("sub.remote.instance") + clear_config([:mrf_simple, :report_removal], [{"*.remote.instance", "suya"}]) + report_message = build_report_message() local_message = build_local_message() assert {:reject, _} = SimplePolicy.filter(report_message) @@ -133,9 +133,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end end - defp build_report_message(domain \\ "remote.instance") do + defp build_report_message do %{ - "actor" => "https://#{domain}/users/bob", + "actor" => "https://remote.instance/users/bob", "type" => "Flag" } end @@ -143,7 +143,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do describe "when :federated_timeline_removal" do test "is empty" do clear_config([:mrf_simple, :federated_timeline_removal], []) - {_, ftl_message} = build_ftl_actor_and_message("https://remote.instance/users/bob") + {_, ftl_message} = build_ftl_actor_and_message() local_message = build_local_message() assert SimplePolicy.filter(ftl_message) == {:ok, ftl_message} @@ -151,7 +151,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "has a matching host" do - {actor, ftl_message} = build_ftl_actor_and_message("https://remote.instance/users/bob") + {actor, ftl_message} = build_ftl_actor_and_message() ftl_message_actor_host = ftl_message @@ -172,7 +172,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "match with wildcard domain" do - {actor, ftl_message} = build_ftl_actor_and_message("https://sub.remote.instance/users/bob") + {actor, ftl_message} = build_ftl_actor_and_message() ftl_message_actor_host = ftl_message @@ -181,7 +181,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do |> Map.fetch!(:host) clear_config([:mrf_simple, :federated_timeline_removal], [ - {ftl_message_actor_host, "owo"} + {"*." <> ftl_message_actor_host, "owo"} ]) local_message = build_local_message() @@ -196,7 +196,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "has a matching host but only as:Public in to" do - {_actor, ftl_message} = build_ftl_actor_and_message("https://remote.instance/users/bob") + {_actor, ftl_message} = build_ftl_actor_and_message() ftl_message_actor_host = ftl_message @@ -253,8 +253,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end end - defp build_ftl_actor_and_message(ap_id) do - actor = insert(:user, ap_id: ap_id) + defp build_ftl_actor_and_message do + actor = insert(:user) {actor, %{ @@ -282,9 +282,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "activity matches with wildcard domain" do - clear_config([:mrf_simple, :reject], [{"remote.instance", ""}]) + clear_config([:mrf_simple, :reject], [{"*.remote.instance", ""}]) - remote_message = build_remote_message("sub.remote.instance") + remote_message = build_remote_message() assert {:reject, _} = SimplePolicy.filter(remote_message) end @@ -325,7 +325,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do describe "when :followers_only" do test "is empty" do clear_config([:mrf_simple, :followers_only], []) - {_, ftl_message} = build_ftl_actor_and_message("https://remote.instance/users/alice") + {_, ftl_message} = build_ftl_actor_and_message() local_message = build_local_message() assert SimplePolicy.filter(ftl_message) == {:ok, ftl_message} @@ -412,10 +412,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "activity matches with wildcard domain" do - clear_config([:mrf_simple, :accept], [{"remote.instance", ""}]) + clear_config([:mrf_simple, :accept], [{"*.remote.instance", ""}]) local_message = build_local_message() - remote_message = build_remote_message("sub.remote.instance") + remote_message = build_remote_message() assert SimplePolicy.filter(local_message) == {:ok, local_message} assert SimplePolicy.filter(remote_message) == {:ok, remote_message} @@ -457,9 +457,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "match with wildcard domain" do - clear_config([:mrf_simple, :avatar_removal], [{"remote.instance", ""}]) + clear_config([:mrf_simple, :avatar_removal], [{"*.remote.instance", ""}]) - remote_user = build_remote_user("sub.remote.instance") + remote_user = build_remote_user() {:ok, filtered} = SimplePolicy.filter(remote_user) refute filtered["icon"] @@ -493,9 +493,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end test "match with wildcard domain" do - clear_config([:mrf_simple, :banner_removal], [{"remote.instance", ""}]) + clear_config([:mrf_simple, :banner_removal], [{"*.remote.instance", ""}]) - remote_user = build_remote_user("sub.remote.instance") + remote_user = build_remote_user() {:ok, filtered} = SimplePolicy.filter(remote_user) refute filtered["image"] @@ -553,10 +553,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do end describe "when :reject_deletes match with wildcard domain" do - setup do: clear_config([:mrf_simple, :reject_deletes], [{"remote.instance", ""}]) + setup do: clear_config([:mrf_simple, :reject_deletes], [{"*.remote.instance", ""}]) test "it rejects the deletion" do - deletion_message = build_remote_deletion_message("sub.remote.instance") + deletion_message = build_remote_deletion_message() assert {:reject, _} = SimplePolicy.filter(deletion_message) end @@ -570,13 +570,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do } end - defp build_remote_message(domain \\ "remote.instance") do - %{"actor" => "https://#{domain}/users/bob"} + defp build_remote_message do + %{"actor" => "https://remote.instance/users/bob"} end - defp build_remote_user(domain \\ "remote.instance") do + defp build_remote_user do %{ - "id" => "https://#{domain}/users/bob", + "id" => "https://remote.instance/users/bob", "icon" => %{ "url" => "http://example.com/image.jpg", "type" => "Image" @@ -589,10 +589,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do } end - defp build_remote_deletion_message(domain \\ "remote.instance") do + defp build_remote_deletion_message do %{ "type" => "Delete", - "actor" => "https://#{domain}/users/bob" + "actor" => "https://remote.instance/users/bob" } end end diff --git a/test/pleroma/web/activity_pub/mrf_test.exs b/test/pleroma/web/activity_pub/mrf_test.exs index ec4dab30f..f04c69afb 100644 --- a/test/pleroma/web/activity_pub/mrf_test.exs +++ b/test/pleroma/web/activity_pub/mrf_test.exs @@ -10,7 +10,7 @@ defmodule Pleroma.Web.ActivityPub.MRFTest do test "subdomains_regex/1" do assert MRF.subdomains_regex(["unsafe.tld", "*.unsafe.tld"]) == [ ~r/^(.+\.)?unsafe\.tld$/i, - ~r/^(.+\.)?\*\.unsafe\.tld$/i + ~r/^(.+\.)?unsafe\.tld$/i ] end From a0b8e3c84280c11ab43b7660414b64b55c1b7259 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Tue, 8 Nov 2022 10:39:01 +0000 Subject: [PATCH 36/57] Don't mess with the cache on metadata update --- lib/pleroma/application.ex | 2 +- lib/pleroma/instances/instance.ex | 21 +++++++------------ test/pleroma/instances/instance_test.exs | 12 +++++------ .../web/admin_api/views/report_view_test.exs | 2 +- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index a78924dfa..b9bcad40c 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -157,7 +157,7 @@ defmodule Pleroma.Application do build_cachex("failed_proxy_url", limit: 2500), build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000), build_cachex("translations", default_ttl: :timer.hours(24 * 30), limit: 2500), - build_cachex("instances", default_ttl: :timer.hours(24), limit: 2500) + build_cachex("instances", default_ttl: :timer.hours(24), ttl_interval: 1000, limit: 2500) ] end diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index fcf3181bf..5ee408f21 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -176,17 +176,14 @@ defmodule Pleroma.Instances.Instance do favicon = scrape_favicon(uri) nodeinfo = scrape_nodeinfo(uri) - {:ok, instance} = - existing_record - |> changeset(%{ - host: host, - favicon: favicon, - nodeinfo: nodeinfo, - metadata_updated_at: NaiveDateTime.utc_now() - }) - |> Repo.update() - - @cachex.put(:instances_cache, "instances:#{host}", instance) + existing_record + |> changeset(%{ + host: host, + favicon: favicon, + nodeinfo: nodeinfo, + metadata_updated_at: NaiveDateTime.utc_now() + }) + |> Repo.update() else {:discard, "Does not require update"} end @@ -205,8 +202,6 @@ defmodule Pleroma.Instances.Instance do metadata_updated_at: NaiveDateTime.utc_now() }) |> Repo.insert() - - @cachex.put(:instances_cache, "instances:#{host}", instance) end end diff --git a/test/pleroma/instances/instance_test.exs b/test/pleroma/instances/instance_test.exs index adc847da5..6ec55c7b7 100644 --- a/test/pleroma/instances/instance_test.exs +++ b/test/pleroma/instances/instance_test.exs @@ -136,7 +136,7 @@ defmodule Pleroma.Instances.InstanceTest do } end) - assert {:ok, true} == + assert {:ok, %Instance{host: "favicon.example.org"}} = Instance.update_metadata(URI.parse("https://favicon.example.org/")) {:ok, instance} = Instance.get_cached_by_url("https://favicon.example.org/") @@ -177,7 +177,7 @@ defmodule Pleroma.Instances.InstanceTest do } end) - assert {:ok, true} == + assert {:ok, %Instance{host: "long-favicon.example.org"}} = Instance.update_metadata(URI.parse("https://long-favicon.example.org/")) {:ok, instance} = Instance.get_cached_by_url("https://long-favicon.example.org/") @@ -214,7 +214,7 @@ defmodule Pleroma.Instances.InstanceTest do end) refute capture_log(fn -> - assert {:ok, true} = + assert {:ok, %Instance{host: "no-favicon.example.org"}} = Instance.update_metadata(URI.parse("https://no-favicon.example.org/")) end) =~ "Instance.update_metadata(\"https://no-favicon.example.org/\") error: " end @@ -241,7 +241,7 @@ defmodule Pleroma.Instances.InstanceTest do } end) - assert {:ok, true} == + assert {:ok, %Instance{host: "bad-nodeinfo.example.org"}} = Instance.update_metadata(URI.parse("https://bad-nodeinfo.example.org/")) {:ok, instance} = Instance.get_cached_by_url("https://bad-nodeinfo.example.org/") @@ -277,7 +277,7 @@ defmodule Pleroma.Instances.InstanceTest do } end) - assert {:ok, true} == + assert {:ok, %Instance{host: "bad-nodeinfo.example.org"}} = Instance.update_metadata(URI.parse("https://bad-nodeinfo.example.org/")) {:ok, instance} = Instance.get_cached_by_url("https://bad-nodeinfo.example.org/") @@ -315,7 +315,7 @@ defmodule Pleroma.Instances.InstanceTest do } end) - assert {:ok, true} == + assert {:ok, %Instance{host: "bad-nodeinfo.example.org"}} = Instance.update_metadata(URI.parse("https://bad-nodeinfo.example.org/")) {:ok, instance} = Instance.get_cached_by_url("https://bad-nodeinfo.example.org/") diff --git a/test/pleroma/web/admin_api/views/report_view_test.exs b/test/pleroma/web/admin_api/views/report_view_test.exs index 093e2d95d..1c115528a 100644 --- a/test/pleroma/web/admin_api/views/report_view_test.exs +++ b/test/pleroma/web/admin_api/views/report_view_test.exs @@ -45,7 +45,7 @@ defmodule Pleroma.Web.AdminAPI.ReportViewTest do ReportView.render("show.json", Report.extract_report_info(activity)) |> Map.delete(:created_at) - assert result == expected + assert Jason.encode!(result) == Jason.encode!(expected) end test "includes reported statuses" do From 479aacb1b66349390c65372e142b4cb477f60c65 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Tue, 8 Nov 2022 11:01:47 +0000 Subject: [PATCH 37/57] Add fallback for reports that don't have attached activities --- .../web/mastodon_api/views/status_view.ex | 364 +++++++++--------- .../web/admin_api/views/report_view_test.exs | 2 +- 2 files changed, 184 insertions(+), 182 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index b3a35526e..929641b84 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -209,212 +209,214 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do - object = Object.normalize(activity, fetch: false) + with %Object{} = object <- Object.normalize(activity, fetch: false) do + user = CommonAPI.get_user(activity.data["actor"]) + user_follower_address = user.follower_address - user = CommonAPI.get_user(activity.data["actor"]) - user_follower_address = user.follower_address + like_count = object.data["like_count"] || 0 + announcement_count = object.data["announcement_count"] || 0 - like_count = object.data["like_count"] || 0 - announcement_count = object.data["announcement_count"] || 0 + hashtags = Object.hashtags(object) + sensitive = object.data["sensitive"] || Enum.member?(hashtags, "nsfw") - hashtags = Object.hashtags(object) - sensitive = object.data["sensitive"] || Enum.member?(hashtags, "nsfw") + tags = Object.tags(object) - tags = Object.tags(object) + tag_mentions = + tags + |> Enum.filter(fn tag -> is_map(tag) and tag["type"] == "Mention" end) + |> Enum.map(fn tag -> tag["href"] end) - tag_mentions = - tags - |> Enum.filter(fn tag -> is_map(tag) and tag["type"] == "Mention" end) - |> Enum.map(fn tag -> tag["href"] end) + mentions = + (object.data["to"] ++ tag_mentions) + |> Enum.uniq() + |> Enum.map(fn + Pleroma.Constants.as_public() -> nil + ^user_follower_address -> nil + ap_id -> User.get_cached_by_ap_id(ap_id) + end) + |> Enum.filter(& &1) + |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end) - mentions = - (object.data["to"] ++ tag_mentions) - |> Enum.uniq() - |> Enum.map(fn - Pleroma.Constants.as_public() -> nil - ^user_follower_address -> nil - ap_id -> User.get_cached_by_ap_id(ap_id) - end) - |> Enum.filter(& &1) - |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end) + favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || []) - favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || []) + bookmarked = Activity.get_bookmark(activity, opts[:for]) != nil - bookmarked = Activity.get_bookmark(activity, opts[:for]) != nil + client_posted_this_activity = opts[:for] && user.id == opts[:for].id - client_posted_this_activity = opts[:for] && user.id == opts[:for].id + expires_at = + with true <- client_posted_this_activity, + %Oban.Job{scheduled_at: scheduled_at} <- + Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id) do + scheduled_at + else + _ -> nil + end - expires_at = - with true <- client_posted_this_activity, - %Oban.Job{scheduled_at: scheduled_at} <- - Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id) do - scheduled_at - else - _ -> nil - end + thread_muted? = + cond do + is_nil(opts[:for]) -> false + is_boolean(activity.thread_muted?) -> activity.thread_muted? + true -> CommonAPI.thread_muted?(opts[:for], activity) + end - thread_muted? = - cond do - is_nil(opts[:for]) -> false - is_boolean(activity.thread_muted?) -> activity.thread_muted? - true -> CommonAPI.thread_muted?(opts[:for], activity) - end + attachment_data = object.data["attachment"] || [] + attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment) - attachment_data = object.data["attachment"] || [] - attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment) + created_at = Utils.to_masto_date(object.data["published"]) - created_at = Utils.to_masto_date(object.data["published"]) + edited_at = + with %{"updated" => updated} <- object.data, + date <- Utils.to_masto_date(updated), + true <- date != "" do + date + else + _ -> + nil + end - edited_at = - with %{"updated" => updated} <- object.data, - date <- Utils.to_masto_date(updated), - true <- date != "" do - date - else - _ -> - nil - end + reply_to = get_reply_to(activity, opts) - reply_to = get_reply_to(activity, opts) + reply_to_user = reply_to && CommonAPI.get_user(reply_to.data["actor"]) - reply_to_user = reply_to && CommonAPI.get_user(reply_to.data["actor"]) + history_len = + 1 + + (Object.Updater.history_for(object.data) + |> Map.get("orderedItems") + |> length()) - history_len = - 1 + - (Object.Updater.history_for(object.data) - |> Map.get("orderedItems") - |> length()) + # See render("history.json", ...) for more details + # Here the implicit index of the current content is 0 + chrono_order = history_len - 1 - # See render("history.json", ...) for more details - # Here the implicit index of the current content is 0 - chrono_order = history_len - 1 + content = + object + |> render_content() - content = - object - |> render_content() - - content_html = - content - |> Activity.HTML.get_cached_scrubbed_html_for_activity( - User.html_filter_policy(opts[:for]), - activity, - "mastoapi:content:#{chrono_order}" - ) - - content_plaintext = - content - |> Activity.HTML.get_cached_stripped_html_for_activity( - activity, - "mastoapi:content:#{chrono_order}" - ) - - summary = object.data["summary"] || "" - - card = render("card.json", Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)) - - url = - if user.local do - Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity) - else - object.data["url"] || object.data["external_url"] || object.data["id"] - end - - direct_conversation_id = - with {_, nil} <- {:direct_conversation_id, opts[:direct_conversation_id]}, - {_, true} <- {:include_id, opts[:with_direct_conversation_id]}, - {_, %User{} = for_user} <- {:for_user, opts[:for]} do - Activity.direct_conversation_id(activity, for_user) - else - {:direct_conversation_id, participation_id} when is_integer(participation_id) -> - participation_id - - _e -> - nil - end - - emoji_reactions = - object.data - |> Map.get("reactions", []) - |> EmojiReactionController.filter_allowed_users( - opts[:for], - Map.get(opts, :with_muted, false) - ) - |> Stream.map(fn {emoji, users, url} -> - build_emoji_map(emoji, users, url, opts[:for]) - end) - |> Enum.to_list() - - # Status muted state (would do 1 request per status unless user mutes are preloaded) - muted = - thread_muted? || - UserRelationship.exists?( - get_in(opts, [:relationships, :user_relationships]), - :mute, - opts[:for], - user, - fn for_user, user -> User.mutes?(for_user, user) end + content_html = + content + |> Activity.HTML.get_cached_scrubbed_html_for_activity( + User.html_filter_policy(opts[:for]), + activity, + "mastoapi:content:#{chrono_order}" ) - {pinned?, pinned_at} = pin_data(object, user) + content_plaintext = + content + |> Activity.HTML.get_cached_stripped_html_for_activity( + activity, + "mastoapi:content:#{chrono_order}" + ) - quote = Activity.get_quoted_activity_from_object(object) + summary = object.data["summary"] || "" - %{ - id: to_string(activity.id), - uri: object.data["id"], - url: url, - account: - AccountView.render("show.json", %{ - user: user, - for: opts[:for] - }), - in_reply_to_id: reply_to && to_string(reply_to.id), - in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), - reblog: nil, - card: card, - content: content_html, - text: opts[:with_source] && get_source_text(object.data["source"]), - created_at: created_at, - edited_at: edited_at, - reblogs_count: announcement_count, - replies_count: object.data["repliesCount"] || 0, - favourites_count: like_count, - reblogged: reblogged?(activity, opts[:for]), - favourited: present?(favorited), - bookmarked: present?(bookmarked), - muted: muted, - pinned: pinned?, - sensitive: sensitive, - spoiler_text: summary, - visibility: get_visibility(object), - media_attachments: attachments, - poll: render(PollView, "show.json", object: object, for: opts[:for]), - mentions: mentions, - tags: build_tags(tags), - application: build_application(object.data["generator"]), - language: nil, - emojis: build_emojis(object.data["emoji"]), - quote_id: if(quote, do: quote.id, else: nil), - quote: maybe_render_quote(quote, opts), - emoji_reactions: emoji_reactions, - pleroma: %{ - local: activity.local, - conversation_id: get_context_id(activity), - context: object.data["context"], - in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, - content: %{"text/plain" => content_plaintext}, - spoiler_text: %{"text/plain" => summary}, - expires_at: expires_at, - direct_conversation_id: direct_conversation_id, - thread_muted: thread_muted?, + card = render("card.json", Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)) + + url = + if user.local do + Pleroma.Web.Router.Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, activity) + else + object.data["url"] || object.data["external_url"] || object.data["id"] + end + + direct_conversation_id = + with {_, nil} <- {:direct_conversation_id, opts[:direct_conversation_id]}, + {_, true} <- {:include_id, opts[:with_direct_conversation_id]}, + {_, %User{} = for_user} <- {:for_user, opts[:for]} do + Activity.direct_conversation_id(activity, for_user) + else + {:direct_conversation_id, participation_id} when is_integer(participation_id) -> + participation_id + + _e -> + nil + end + + emoji_reactions = + object.data + |> Map.get("reactions", []) + |> EmojiReactionController.filter_allowed_users( + opts[:for], + Map.get(opts, :with_muted, false) + ) + |> Stream.map(fn {emoji, users, url} -> + build_emoji_map(emoji, users, url, opts[:for]) + end) + |> Enum.to_list() + + # Status muted state (would do 1 request per status unless user mutes are preloaded) + muted = + thread_muted? || + UserRelationship.exists?( + get_in(opts, [:relationships, :user_relationships]), + :mute, + opts[:for], + user, + fn for_user, user -> User.mutes?(for_user, user) end + ) + + {pinned?, pinned_at} = pin_data(object, user) + + quote = Activity.get_quoted_activity_from_object(object) + + %{ + id: to_string(activity.id), + uri: object.data["id"], + url: url, + account: + AccountView.render("show.json", %{ + user: user, + for: opts[:for] + }), + in_reply_to_id: reply_to && to_string(reply_to.id), + in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id), + reblog: nil, + card: card, + content: content_html, + text: opts[:with_source] && get_source_text(object.data["source"]), + created_at: created_at, + edited_at: edited_at, + reblogs_count: announcement_count, + replies_count: object.data["repliesCount"] || 0, + favourites_count: like_count, + reblogged: reblogged?(activity, opts[:for]), + favourited: present?(favorited), + bookmarked: present?(bookmarked), + muted: muted, + pinned: pinned?, + sensitive: sensitive, + spoiler_text: summary, + visibility: get_visibility(object), + media_attachments: attachments, + poll: render(PollView, "show.json", object: object, for: opts[:for]), + mentions: mentions, + tags: build_tags(tags), + application: build_application(object.data["generator"]), + language: nil, + emojis: build_emojis(object.data["emoji"]), + quote_id: if(quote, do: quote.id, else: nil), + quote: maybe_render_quote(quote, opts), emoji_reactions: emoji_reactions, - parent_visible: visible_for_user?(reply_to, opts[:for]), - pinned_at: pinned_at - }, - akkoma: %{ - source: object.data["source"] + pleroma: %{ + local: activity.local, + conversation_id: get_context_id(activity), + context: object.data["context"], + in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, + content: %{"text/plain" => content_plaintext}, + spoiler_text: %{"text/plain" => summary}, + expires_at: expires_at, + direct_conversation_id: direct_conversation_id, + thread_muted: thread_muted?, + emoji_reactions: emoji_reactions, + parent_visible: visible_for_user?(reply_to, opts[:for]), + pinned_at: pinned_at + }, + akkoma: %{ + source: object.data["source"] + } } - } + else + nil -> nil + end end def render("show.json", _) do diff --git a/test/pleroma/web/admin_api/views/report_view_test.exs b/test/pleroma/web/admin_api/views/report_view_test.exs index 1c115528a..d9ff48dde 100644 --- a/test/pleroma/web/admin_api/views/report_view_test.exs +++ b/test/pleroma/web/admin_api/views/report_view_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.AdminAPI.ReportViewTest do - use Pleroma.DataCase, async: true + use Pleroma.DataCase, async: false import Pleroma.Factory From 2e895b6c0262e4a77e42225b2a1b1386edbcb4cd Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Tue, 8 Nov 2022 11:03:43 +0000 Subject: [PATCH 38/57] make metdata check a debug log --- lib/pleroma/instances/instance.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 5ee408f21..aed0cd2d0 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -159,7 +159,7 @@ defmodule Pleroma.Instances.Instance do end def update_metadata(%URI{host: host} = uri) do - Logger.info("Checking metadata for #{host}") + Logger.debug("Checking metadata for #{host}") existing_record = Repo.get_by(Instance, %{host: host}) if reachable?(host) do From 4e8ab0deeb16e942bb677c5f0fc7dc6ba1508801 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Tue, 8 Nov 2022 13:50:04 +0000 Subject: [PATCH 39/57] fix count of poll voters --- lib/pleroma/web/mastodon_api/views/poll_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/views/poll_view.ex b/lib/pleroma/web/mastodon_api/views/poll_view.ex index 71bc8b949..aa6443754 100644 --- a/lib/pleroma/web/mastodon_api/views/poll_view.ex +++ b/lib/pleroma/web/mastodon_api/views/poll_view.ex @@ -68,7 +68,7 @@ defmodule Pleroma.Web.MastodonAPI.PollView do end) end - defp voters_count(%{data: %{"voters" => [_ | _] = voters}}) do + defp voters_count(%{data: %{"voters" => voters}}) when is_list(voters) do length(voters) end From 0681a26dbb58b420105275fdaca11ec627257837 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Tue, 8 Nov 2022 13:54:43 +0000 Subject: [PATCH 40/57] Remove unused pattern --- lib/pleroma/instances/instance.ex | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index aed0cd2d0..27dbf7661 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -193,15 +193,14 @@ defmodule Pleroma.Instances.Instance do Logger.info("Creating metadata for #{host}") - {:ok, instance} = - %Instance{} - |> changeset(%{ - host: host, - favicon: favicon, - nodeinfo: nodeinfo, - metadata_updated_at: NaiveDateTime.utc_now() - }) - |> Repo.insert() + %Instance{} + |> changeset(%{ + host: host, + favicon: favicon, + nodeinfo: nodeinfo, + metadata_updated_at: NaiveDateTime.utc_now() + }) + |> Repo.insert() end end From 53fbe26c802e58325783fe005811747c652c6a47 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Wed, 9 Nov 2022 13:22:44 +0000 Subject: [PATCH 41/57] reference "stable" in all URLs --- docs/docs/installation/migrating_from_source_otp_en.md | 2 +- docs/docs/installation/otp_en.md | 2 +- docs/docs/installation/verifying_otp_releases.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/docs/installation/migrating_from_source_otp_en.md b/docs/docs/installation/migrating_from_source_otp_en.md index 148564d9a..505f3cd6d 100644 --- a/docs/docs/installation/migrating_from_source_otp_en.md +++ b/docs/docs/installation/migrating_from_source_otp_en.md @@ -87,7 +87,7 @@ export FLAVOUR="amd64-musl" # Clone the release build into a temporary directory and unpack it # Replace `stable` with `unstable` if you want to run the unstable branch su akkoma -s $SHELL -lc " -curl 'https://akkoma-updates.s3-website.fr-par.scw.cloud/develop/akkoma-$FLAVOUR.zip' -o /tmp/akkoma.zip +curl 'https://akkoma-updates.s3-website.fr-par.scw.cloud/stable/akkoma-$FLAVOUR.zip' -o /tmp/akkoma.zip unzip /tmp/akkoma.zip -d /tmp/ " diff --git a/docs/docs/installation/otp_en.md b/docs/docs/installation/otp_en.md index 3e00d3262..3d5d2152b 100644 --- a/docs/docs/installation/otp_en.md +++ b/docs/docs/installation/otp_en.md @@ -123,7 +123,7 @@ export FLAVOUR="amd64-musl" # Clone the release build into a temporary directory and unpack it su akkoma -s $SHELL -lc " -curl 'https://akkoma-updates.s3-website.fr-par.scw.cloud/develop/akkoma-$FLAVOUR.zip' -o /tmp/akkoma.zip +curl 'https://akkoma-updates.s3-website.fr-par.scw.cloud/stable/akkoma-$FLAVOUR.zip' -o /tmp/akkoma.zip unzip /tmp/akkoma.zip -d /tmp/ " diff --git a/docs/docs/installation/verifying_otp_releases.md b/docs/docs/installation/verifying_otp_releases.md index 86dacfec2..5f1ac6949 100644 --- a/docs/docs/installation/verifying_otp_releases.md +++ b/docs/docs/installation/verifying_otp_releases.md @@ -4,7 +4,7 @@ All stable OTP releases are cryptographically signed, to allow you to verify the integrity if you choose to. Releases are signed with [Signify](https://man.openbsd.org/signify.1), -with [the public key in the main repository](https://akkoma.dev/AkkomaGang/akkoma/src/branch/develop/SIGNING_KEY.pub) +with [the public key in the main repository](https://akkoma.dev/AkkomaGang/akkoma/src/branch/stable/SIGNING_KEY.pub) Release URLs will always be of the form From cc6a0762022644ea751767ed6b7673e069aca855 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Thu, 10 Nov 2022 03:16:32 +0000 Subject: [PATCH 42/57] Include requested_by in relationship (#260) Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/260 --- .../web/mastodon_api/views/account_view.ex | 13 +++++++------ .../mastodon_api/views/account_view_test.exs | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index cbb57aee6..a04ffaaf3 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -94,12 +94,12 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do followed_by = if following_relationships do - case FollowingRelationship.find(following_relationships, target, reading_user) do - %{state: :follow_accept} -> true - _ -> false - end + target_to_user_following_relation = + FollowingRelationship.find(following_relationships, target, reading_user) + + User.get_follow_state(target, reading_user, target_to_user_following_relation) else - User.following?(target, reading_user) + User.get_follow_state(target, reading_user) end subscribing = @@ -115,7 +115,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do %{ id: to_string(target.id), following: follow_state == :follow_accept, - followed_by: followed_by, + followed_by: followed_by == :follow_accept, blocking: UserRelationship.exists?( user_relationships, @@ -151,6 +151,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do subscribing: subscribing, notifying: subscribing, requested: follow_state == :follow_pending, + requested_by: followed_by == :follow_pending, domain_blocking: User.blocks_domain?(reading_user, target), showing_reblogs: not UserRelationship.exists?( diff --git a/test/pleroma/web/mastodon_api/views/account_view_test.exs b/test/pleroma/web/mastodon_api/views/account_view_test.exs index d1903af80..f4a5f4d50 100644 --- a/test/pleroma/web/mastodon_api/views/account_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/account_view_test.exs @@ -347,6 +347,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do subscribing: false, notifying: false, requested: false, + requested_by: false, domain_blocking: false, showing_reblogs: true, endorsed: false, @@ -432,6 +433,24 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do end end + test "represent a relationship for a user with an inbound pending follow request" do + follower = insert(:user) + followed = insert(:user, is_locked: true) + + {:ok, follower, followed, _} = CommonAPI.follow(follower, followed) + + follower = User.get_cached_by_id(follower.id) + followed = User.get_cached_by_id(followed.id) + + expected = + Map.merge( + @blank_response, + %{requested_by: true, followed_by: false, id: to_string(follower.id)} + ) + + test_relationship_rendering(followed, follower, expected) + end + test "returns the settings store if the requesting user is the represented user and it's requested specifically" do user = insert(:user, pleroma_settings_store: %{fe: "test"}) From f7c1e15d08eaec176f5e7c33bfe9f8783c2caa49 Mon Sep 17 00:00:00 2001 From: Weblate Date: Fri, 19 Aug 2022 09:25:17 +0000 Subject: [PATCH 43/57] Translated using Weblate (Spanish) Currently translated at 21.6% (23 of 106 strings) Co-authored-by: mint Translate-URL: http://translate.akkoma.dev/projects/akkoma/akkoma-backend-errors/es/ Translation: Pleroma fe/Akkoma Backend (Errors) --- priv/gettext/es/LC_MESSAGES/errors.po | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/priv/gettext/es/LC_MESSAGES/errors.po b/priv/gettext/es/LC_MESSAGES/errors.po index 0a6fceaad..166e2fb98 100644 --- a/priv/gettext/es/LC_MESSAGES/errors.po +++ b/priv/gettext/es/LC_MESSAGES/errors.po @@ -3,16 +3,16 @@ msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" "POT-Creation-Date: 2020-09-09 09:49+0000\n" -"PO-Revision-Date: 2020-09-11 21:26+0000\n" -"Last-Translator: tarteka \n" -"Language-Team: Spanish \n" +"PO-Revision-Date: 2022-08-19 09:25+0000\n" +"Last-Translator: mint \n" +"Language-Team: Spanish \n" "Language: es\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=n != 1;\n" -"X-Generator: Weblate 4.0.4\n" +"X-Generator: Weblate 4.13.1\n" ## This file is a PO Template file. ## @@ -66,8 +66,8 @@ msgstr[1] "debe tener %{count} caracteres" msgid "should have %{count} item(s)" msgid_plural "should have %{count} item(s)" -msgstr[0] "" -msgstr[1] "" +msgstr[0] "debería tener %{count} item" +msgstr[1] "debería tener %{count} items" msgid "should be at least %{count} character(s)" msgid_plural "should be at least %{count} character(s)" From f2c6749b57af4967d8a1e796d00f825ca269ca14 Mon Sep 17 00:00:00 2001 From: Weblate Date: Fri, 19 Aug 2022 09:25:17 +0000 Subject: [PATCH 44/57] Update translation files Updated by "Squash Git commits" hook in Weblate. Translation: Pleroma fe/Akkoma Backend (Errors) Translate-URL: http://translate.akkoma.dev/projects/akkoma/akkoma-backend-errors/ --- priv/gettext/nl/LC_MESSAGES/posix_errors.po | 163 ++++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 priv/gettext/nl/LC_MESSAGES/posix_errors.po diff --git a/priv/gettext/nl/LC_MESSAGES/posix_errors.po b/priv/gettext/nl/LC_MESSAGES/posix_errors.po new file mode 100644 index 000000000..d94b0c8e2 --- /dev/null +++ b/priv/gettext/nl/LC_MESSAGES/posix_errors.po @@ -0,0 +1,163 @@ +msgid "" +msgstr "" +"Project-Id-Version: PACKAGE VERSION\n" +"Report-Msgid-Bugs-To: \n" +"POT-Creation-Date: 2022-08-16 10:49+0000\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: Automatically generated\n" +"Language-Team: none\n" +"Language: nl\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"X-Generator: Translate Toolkit 3.7.1\n" + +## This file is a PO Template file. +## +## `msgid`s here are often extracted from source code. +## Add new translations manually only if they're dynamic +## translations that can't be statically extracted. +## +## Run `mix gettext.extract` to bring this file up to +## date. Leave `msgstr`s empty as changing them here as no +## effect: edit them in PO (`.po`) files instead. +msgid "eperm" +msgstr "" + +msgid "eacces" +msgstr "" + +msgid "eagain" +msgstr "" + +msgid "ebadf" +msgstr "" + +msgid "ebadmsg" +msgstr "" + +msgid "ebusy" +msgstr "" + +msgid "edeadlk" +msgstr "" + +msgid "edeadlock" +msgstr "" + +msgid "edquot" +msgstr "" + +msgid "eexist" +msgstr "" + +msgid "efault" +msgstr "" + +msgid "efbig" +msgstr "" + +msgid "eftype" +msgstr "" + +msgid "eintr" +msgstr "" + +msgid "einval" +msgstr "" + +msgid "eio" +msgstr "" + +msgid "eisdir" +msgstr "" + +msgid "eloop" +msgstr "" + +msgid "emfile" +msgstr "" + +msgid "emlink" +msgstr "" + +msgid "emultihop" +msgstr "" + +msgid "enametoolong" +msgstr "" + +msgid "enfile" +msgstr "" + +msgid "enobufs" +msgstr "" + +msgid "enodev" +msgstr "" + +msgid "enolck" +msgstr "" + +msgid "enolink" +msgstr "" + +msgid "enoent" +msgstr "" + +msgid "enomem" +msgstr "" + +msgid "enospc" +msgstr "" + +msgid "enosr" +msgstr "" + +msgid "enostr" +msgstr "" + +msgid "enosys" +msgstr "" + +msgid "enotblk" +msgstr "" + +msgid "enotdir" +msgstr "" + +msgid "enotsup" +msgstr "" + +msgid "enxio" +msgstr "" + +msgid "eopnotsupp" +msgstr "" + +msgid "eoverflow" +msgstr "" + +msgid "epipe" +msgstr "" + +msgid "erange" +msgstr "" + +msgid "erofs" +msgstr "" + +msgid "espipe" +msgstr "" + +msgid "esrch" +msgstr "" + +msgid "estale" +msgstr "" + +msgid "etxtbsy" +msgstr "" + +msgid "exdev" +msgstr "" From c5b6cb746f2ae6f54f9e3d84c49c11b86548ecf3 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Thu, 10 Nov 2022 03:17:00 +0000 Subject: [PATCH 45/57] add requested_by changelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 15472eda6..0e71b2127 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Officially supported docker release - Ability to remove followers unilaterally without a block - Scraping of nodeinfo from remote instances to display instance info +- `requested_by` in relationships when the user has requested to follow you ## Changes - Follows no longer override domain blocks, a domain block is final From 66eb844bd2e7d7f07dcb07624d695672dfedd89d Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Thu, 10 Nov 2022 03:38:10 +0000 Subject: [PATCH 46/57] Update documentation builder --- docs/Pipfile.lock | 95 +++++++++++++++++++++++++++++++---------------- 1 file changed, 63 insertions(+), 32 deletions(-) diff --git a/docs/Pipfile.lock b/docs/Pipfile.lock index ae39a2776..c7b8f50db 100644 --- a/docs/Pipfile.lock +++ b/docs/Pipfile.lock @@ -14,6 +14,22 @@ ] }, "default": { + "certifi": { + "hashes": [ + "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14", + "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382" + ], + "markers": "python_version >= '3.6'", + "version": "==2022.9.24" + }, + "charset-normalizer": { + "hashes": [ + "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", + "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" + ], + "markers": "python_version >= '3.6'", + "version": "==2.1.1" + }, "click": { "hashes": [ "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", @@ -29,13 +45,13 @@ ], "version": "==2.1.0" }, - "importlib-metadata": { + "idna": { "hashes": [ - "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670", - "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23" + "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", + "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" ], - "markers": "python_version >= '3.7'", - "version": "==4.12.0" + "markers": "python_version >= '3.5'", + "version": "==3.4" }, "jinja2": { "hashes": [ @@ -55,10 +71,10 @@ }, "markdown-include": { "hashes": [ - "sha256:6f5d680e36f7780c7f0f61dca53ca581bd50d1b56137ddcd6353efafa0c3e4a2" + "sha256:a06183b7c7225e73112737acdc6fe0ac0686c39457234eeb5ede23881fed001d" ], "index": "pypi", - "version": "==0.6.0" + "version": "==0.7.0" }, "markupsafe": { "hashes": [ @@ -116,27 +132,27 @@ }, "mkdocs": { "hashes": [ - "sha256:26bd2b03d739ac57a3e6eed0b7bcc86168703b719c27b99ad6ca91dc439aacde", - "sha256:b504405b04da38795fec9b2e5e28f6aa3a73bb0960cb6d5d27ead28952bd35ea" + "sha256:8947af423a6d0facf41ea1195b8e1e8c85ad94ac95ae307fe11232e0424b11c5", + "sha256:c8856a832c1e56702577023cd64cc5f84948280c1c0fcc6af4cd39006ea6aa8c" ], - "markers": "python_version >= '3.6'", - "version": "==1.3.0" + "markers": "python_version >= '3.7'", + "version": "==1.4.2" }, "mkdocs-material": { "hashes": [ - "sha256:263f2721f3abe533b61f7c8bed435a0462620912742c919821ac2d698b4bfe67", - "sha256:dc82b667d2a83f0de581b46a6d0949732ab77e7638b87ea35b770b33bc02e75a" + "sha256:143ea55843b3747b640e1110824d91e8a4c670352380e166e64959f9abe98862", + "sha256:45eeabb23d2caba8fa3b85c91d9ec8e8b22add716e9bba8faf16d56af8aa5622" ], "index": "pypi", - "version": "==8.3.9" + "version": "==8.5.9" }, "mkdocs-material-extensions": { "hashes": [ - "sha256:a82b70e533ce060b2a5d9eb2bc2e1be201cf61f901f93704b4acf6e3d5983a44", - "sha256:bfd24dfdef7b41c312ede42648f9eb83476ea168ec163b613f9abd12bbfddba2" + "sha256:96ca979dae66d65c2099eefe189b49d5ac62f76afb59c38e069ffc7cf3c131ec", + "sha256:bcc2e5fc70c0ec50e59703ee6e639d87c7e664c0c441c014ea84461a90f1e902" ], - "markers": "python_version >= '3.6'", - "version": "==1.0.3" + "markers": "python_version >= '3.7'", + "version": "==1.1" }, "packaging": { "hashes": [ @@ -148,19 +164,19 @@ }, "pygments": { "hashes": [ - "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb", - "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519" + "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1", + "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42" ], "markers": "python_version >= '3.6'", - "version": "==2.12.0" + "version": "==2.13.0" }, "pymdown-extensions": { "hashes": [ - "sha256:3ef2d998c0d5fa7eb09291926d90d69391283561cf6306f85cd588a5eb5befa0", - "sha256:ec141c0f4983755349f0c8710416348d1a13753976c028186ed14f190c8061c4" + "sha256:1bd4a173095ef8c433b831af1f3cb13c10883be0c100ae613560668e594651f7", + "sha256:8e62688a8b1128acd42fa823f3d429d22f4284b5e6dd4d3cd56721559a5a211b" ], "markers": "python_version >= '3.7'", - "version": "==9.5" + "version": "==9.8" }, "pyparsing": { "hashes": [ @@ -180,6 +196,7 @@ }, "pyyaml": { "hashes": [ + "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf", "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293", "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57", @@ -191,26 +208,32 @@ "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513", "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", + "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782", "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92", "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f", "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc", + "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1", "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86", "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4", "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c", "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b", + "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d", "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c", "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb", + "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7", "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737", "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d", + "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358", "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53", "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78", "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a", + "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f", "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5" ], @@ -225,6 +248,14 @@ "markers": "python_version >= '3.6'", "version": "==0.1" }, + "requests": { + "hashes": [ + "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", + "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" + ], + "markers": "python_version >= '3.7' and python_version < '4'", + "version": "==2.28.1" + }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", @@ -233,6 +264,14 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, + "urllib3": { + "hashes": [ + "sha256:3fa96cf423e6987997fc326ae8df396db2a8b7c667747d47ddd8ecba91f4a74e", + "sha256:b930dd878d5a8afb066a637fbb35144fe7901e3b209d1cd4f524bd0e9deee997" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", + "version": "==1.26.12" + }, "watchdog": { "hashes": [ "sha256:083171652584e1b8829581f965b9b7723ca5f9a2cd7e20271edf264cfd7c1412", @@ -263,14 +302,6 @@ ], "markers": "python_version >= '3.6'", "version": "==2.1.9" - }, - "zipp": { - "hashes": [ - "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad", - "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099" - ], - "markers": "python_version >= '3.7'", - "version": "==3.8.0" } }, "develop": {} From 539c6d6666aa7ec30a80f1a20f63c8908da2b53b Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Thu, 10 Nov 2022 03:40:36 +0000 Subject: [PATCH 47/57] update requirements.txt --- docs/requirements.txt | 174 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 155 insertions(+), 19 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 70fbd9dc4..3167d24b5 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,22 +1,158 @@ -click==8.1.3 -ghp-import==2.1.0 -importlib-metadata==4.12.0 -Jinja2==3.1.2 -Markdown==3.3.7 -markdown-include==0.6.0 -MarkupSafe==2.1.1 -mergedeep==1.3.4 -mkdocs==1.3.0 -mkdocs-bootswatch==1.1 -mkdocs-material==8.1.8 -mkdocs-material-extensions==1.0.3 +appdirs==1.4.4 +arrow==1.2.2 +astroid==2.8.5 +attrs==21.2.0 +aws-sam-translator==1.40.0 +aws-xray-sdk==2.8.0 +awscli-local==0.17 +backports.entry-points-selectable==1.1.0 +bcrypt==3.2.0 +binaryornot==0.4.4 +black==19.10b0 +bleach==4.1.0 +blurhash==1.1.4 +boto==2.49.0 +boto3==1.20.7 +botocore==1.23.7 +certifi==2021.5.30 +cffi==1.15.0 +cfn-lint==0.56.1 +chardet==4.0.0 +charset-normalizer==2.0.7 +Cheetah3==3.2.6.post1 +cli-routed==1.0.0 +click==8.0.3 +colorama==0.4.4 +colored==1.4.3 +confpy==0.11.0 +cookiecutter==1.7.3 +cosmos-release==1.0 +coverage==6.1.2 +cryptography==35.0.0 +decorator==5.1.1 +distlib==0.3.3 +distro==1.7.0 +docker==5.0.3 +docker-compose==1.29.2 +dockerpty==0.4.1 +docopt==0.6.2 +docutils==0.18 +ecdsa==0.14.1 +emoji==1.7.0 +entrypoints==0.3 +execnet==1.9.0 +filelock==3.3.0 +flake8==3.7.8 +Flask==2.0.2 +Flask-Cors==3.0.10 +freezegun==1.1.0 +future==0.18.2 +humanize==4.0.0 +idna==3.3 +importlib-metadata==4.8.2 +inflection==0.3.1 +iniconfig==1.1.1 +isort==5.10.1 +itsdangerous==2.0.1 +jedi==0.17.2 +jeepney==0.7.1 +Jinja2==3.0.3 +jinja2-time==0.2.0 +jmespath==0.10.0 +jschema-to-python==1.2.3 +jsondiff==1.3.0 +jsonpatch==1.32 +jsonpickle==2.0.0 +jsonpointer==2.2 +jsonschema==3.2.0 +junit-xml==1.9 +keyring==23.2.1 +lambda-repository-cli==0.6.0 +lazy-object-proxy==1.6.0 +localstack-client==1.26 +lxml==4.6.4 +MarkupSafe==2.0.1 +Mastodon.py==1.5.0 +mccabe==0.6.1 +mock==4.0.3 +mutagen==1.45.1 +networkx==2.6.3 +ordereddict==1.1 packaging==21.3 -Pygments==2.11.2 -pymdown-extensions==9.1 -pyparsing==3.0.9 +paramiko==2.10.3 +parso==0.7.1 +pathspec==0.9.0 +pbr==5.8.0 +pipenv==2021.5.29 +pkginfo==1.7.1 +platformdirs==2.4.0 +pluggy==1.0.0 +poyo==0.5.0 +prompt-toolkit==3.0.22 +psutil==5.9.0 +py==1.11.0 +pyaml==21.10.1 +pyasn1==0.4.8 +pycodestyle==2.5.0 +pycparser==2.21 +pycryptodomex==3.14.1 +pycurl==7.44.1 +pyflakes==2.1.1 +Pygments==2.10.0 +pylint==2.11.1 +pymongo==4.0.1 +PyNaCl==1.5.0 +pyparsing==3.0.6 +pyrsistent==0.18.0 +pytest==6.2.5 +pytest-cov==3.0.0 +pytest-forked==1.3.0 +pytest-xdist==2.4.0 python-dateutil==2.8.2 -PyYAML==6.0 -pyyaml_env_tag==0.1 +python-dotenv==0.20.0 +python-jose==3.3.0 +python-jsonrpc-server==0.4.0 +python-language-server==0.36.2 +python-magic==0.4.25 +python-slugify==6.1.2 +pytz==2021.3 +PyYAML==5.4.1 +readme-renderer==30.0 +regex==2019.11.1 +requests==2.26.0 +requests-toolbelt==0.9.1 +requirements-parser==0.3.1 +responses==0.16.0 +rfc3986==1.5.0 +rpmvenv==0.25.0 +rsa==4.7.2 +s3transfer==0.5.0 +sarif-om==1.0.4 +SecretStorage==3.3.1 +semver==2.9.1 six==1.16.0 -watchdog==2.1.9 -zipp==3.8.0 +sshpubkeys==3.3.1 +sure==2.0.0 +text-unidecode==1.3 +texttable==1.6.4 +toml==0.10.2 +tomli==1.2.2 +tootstream==0.3.8.1 +tqdm==4.62.3 +twine==3.6.0 +typed-ast==1.5.0 +types-setuptools==57.4.7 +ujson==4.2.0 +urllib3==1.26.7 +venvctrl==0.4.2 +virtualenv==20.8.1 +virtualenv-clone==0.5.7 +wcwidth==0.2.5 +webencodings==0.5.1 +websocket-client==0.59.0 +websockets==10.2 +Werkzeug==2.0.2 +wrapt==1.13.3 +xmltodict==0.12.0 +yt-dlp==2022.2.4 +zipp==3.6.0 From f8b4e360a0d11a9c875455ed83122ffa39ee9eee Mon Sep 17 00:00:00 2001 From: Eloy Degen Date: Thu, 10 Nov 2022 10:00:39 +0100 Subject: [PATCH 48/57] Fix typo in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 74b8919e4..212f99c60 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ This is a fork of Pleroma, which is a microblogging server software that can federate (= exchange messages with) other servers that support ActivityPub. What that means is that you can host a server for yourself or your friends and stay in control of your online identity, but still exchange messages with people on larger servers. Akkoma will federate with all servers that implement ActivityPub, like Friendica, GNU Social, Hubzilla, Mastodon, Misskey, Peertube, and Pixelfed. -Akkoma is written in Elixir and uses PostgresSQL for data storage. +Akkoma is written in Elixir and uses PostgreSQL for data storage. For clients it supports the [Mastodon client API](https://docs.joinmastodon.org/api/guidelines/) with Pleroma extensions (see the API section on ). From dcc36df8cf97122308aa1616285064910cbdf7c5 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Thu, 10 Nov 2022 10:55:57 +0000 Subject: [PATCH 49/57] add manual deploy for docs --- .gitignore | 1 + docs/Makefile | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index f9de4ed49..14373fb8c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # App artifacts docs/site +*.zip *.sw* secret /_build diff --git a/docs/Makefile b/docs/Makefile index 022459cf0..85b6dee65 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,9 +1,14 @@ all: install pipenv run mkdocs build +branch := $(shell git rev-parse --abbrev-ref HEAD) install: pipenv install clean: rm -rf site serve: pipenv run python3 -m http.server -d site +zip: + zip -r docs.zip site/* +deploy: + cd site && rclone copy . scaleway:akkoma-docs/$(branch) From bab1ab5b6c74de5f02d9ddd06f0026efa547c2a5 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Thu, 10 Nov 2022 11:54:12 +0000 Subject: [PATCH 50/57] strip \r and \r from content-disposition filenames --- lib/pleroma/web/plugs/uploaded_media.ex | 9 ++++++++- test/pleroma/web/plugs/uploaded_media_plug_test.exs | 11 +++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/plugs/uploaded_media.ex b/lib/pleroma/web/plugs/uploaded_media.ex index 7b87d8f17..72f20e8de 100644 --- a/lib/pleroma/web/plugs/uploaded_media.ex +++ b/lib/pleroma/web/plugs/uploaded_media.ex @@ -35,7 +35,7 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do conn = case fetch_query_params(conn) do %{query_params: %{"name" => name}} = conn -> - name = String.replace(name, "\"", "\\\"") + name = escape_header_value(name) put_resp_header(conn, "content-disposition", "filename=\"#{name}\"") @@ -98,4 +98,11 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do |> send_resp(:internal_server_error, dgettext("errors", "Internal Error")) |> halt() end + + defp escape_header_value(value) do + value + |> String.replace("\"", "\\\"") + |> String.replace("\\r", "") + |> String.replace("\\n", "") + end end diff --git a/test/pleroma/web/plugs/uploaded_media_plug_test.exs b/test/pleroma/web/plugs/uploaded_media_plug_test.exs index 75f313282..c71a7e789 100644 --- a/test/pleroma/web/plugs/uploaded_media_plug_test.exs +++ b/test/pleroma/web/plugs/uploaded_media_plug_test.exs @@ -40,4 +40,15 @@ defmodule Pleroma.Web.Plugs.UploadedMediaPlugTest do &(&1 == {"content-disposition", "filename=\"\\\"cofe\\\".gif\""}) ) end + + test "removes control characters from the Content-Disposition header", %{ + attachment_url: attachment_url + } do + conn = get(build_conn(), attachment_url <> "?name=\"cofe\".gif\\r\\n") + + assert Enum.any?( + conn.resp_headers, + &(&1 == {"content-disposition", "filename=\"\\\"cofe\\\".gif\""}) + ) + end end From ac0c00cdee239e6bc37c0df2bfdb9f0ec1d24606 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Thu, 10 Nov 2022 17:26:51 +0000 Subject: [PATCH 51/57] Add media sources to connect-src if media proxy is enabled --- lib/pleroma/web/plugs/http_security_plug.ex | 14 ++++++-------- test/pleroma/web/plugs/http_security_plug_test.exs | 8 ++++++++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index 3e8e931d1..43b075447 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -104,14 +104,12 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do {[img_src, " https:"], [media_src, " https:"]} end - connect_src = ["connect-src 'self' blob: ", static_url, ?\s, websocket_url] - - connect_src = - if Config.get(:env) == :dev do - [connect_src, " http://localhost:3035/"] - else - connect_src - end + connect_src = if Config.get([:media_proxy, :enabled]) do + sources = build_csp_multimedia_source_list() + ["connect-src 'self' blob: ", static_url, ?\s, websocket_url, ?\s, sources] + else + ["connect-src 'self' blob: ", static_url, ?\s, websocket_url] + end script_src = if Config.get(:env) == :dev do diff --git a/test/pleroma/web/plugs/http_security_plug_test.exs b/test/pleroma/web/plugs/http_security_plug_test.exs index eb94cd665..7f85f4a11 100644 --- a/test/pleroma/web/plugs/http_security_plug_test.exs +++ b/test/pleroma/web/plugs/http_security_plug_test.exs @@ -100,12 +100,14 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do url = "https://example.com" clear_config([:media_proxy, :base_url], url) assert_media_img_src(conn, url) + assert_connect_src(conn, url) end test "upload with base url", %{conn: conn} do url = "https://example2.com" clear_config([Pleroma.Upload, :base_url], url) assert_media_img_src(conn, url) + assert_connect_src(conn, url) end test "with S3 public endpoint", %{conn: conn} do @@ -138,6 +140,12 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlugTest do assert csp =~ "img-src 'self' data: blob: #{url};" end + defp assert_connect_src(conn, url) do + conn = get(conn, "/api/v1/instance") + [csp] = Conn.get_resp_header(conn, "content-security-policy") + assert csp =~ ~r/connect-src 'self' blob: [^;]+ #{url}/ + end + test "it does not send CSP headers when disabled", %{conn: conn} do clear_config([:http_security, :enabled], false) From 634463ff64b0e32bd0b247c0f9b011928acf2b72 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 11 Nov 2022 16:07:07 +0000 Subject: [PATCH 52/57] fix requirements --- docs/requirements.txt | 178 ++++++------------------------------------ 1 file changed, 23 insertions(+), 155 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 3167d24b5..d67bbf65f 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,158 +1,26 @@ -appdirs==1.4.4 -arrow==1.2.2 -astroid==2.8.5 -attrs==21.2.0 -aws-sam-translator==1.40.0 -aws-xray-sdk==2.8.0 -awscli-local==0.17 -backports.entry-points-selectable==1.1.0 -bcrypt==3.2.0 -binaryornot==0.4.4 -black==19.10b0 -bleach==4.1.0 -blurhash==1.1.4 -boto==2.49.0 -boto3==1.20.7 -botocore==1.23.7 -certifi==2021.5.30 -cffi==1.15.0 -cfn-lint==0.56.1 -chardet==4.0.0 -charset-normalizer==2.0.7 -Cheetah3==3.2.6.post1 -cli-routed==1.0.0 -click==8.0.3 -colorama==0.4.4 -colored==1.4.3 -confpy==0.11.0 -cookiecutter==1.7.3 -cosmos-release==1.0 -coverage==6.1.2 -cryptography==35.0.0 -decorator==5.1.1 -distlib==0.3.3 -distro==1.7.0 -docker==5.0.3 -docker-compose==1.29.2 -dockerpty==0.4.1 -docopt==0.6.2 -docutils==0.18 -ecdsa==0.14.1 -emoji==1.7.0 -entrypoints==0.3 -execnet==1.9.0 -filelock==3.3.0 -flake8==3.7.8 -Flask==2.0.2 -Flask-Cors==3.0.10 -freezegun==1.1.0 -future==0.18.2 -humanize==4.0.0 -idna==3.3 -importlib-metadata==4.8.2 -inflection==0.3.1 -iniconfig==1.1.1 -isort==5.10.1 -itsdangerous==2.0.1 -jedi==0.17.2 -jeepney==0.7.1 -Jinja2==3.0.3 -jinja2-time==0.2.0 -jmespath==0.10.0 -jschema-to-python==1.2.3 -jsondiff==1.3.0 -jsonpatch==1.32 -jsonpickle==2.0.0 -jsonpointer==2.2 -jsonschema==3.2.0 -junit-xml==1.9 -keyring==23.2.1 -lambda-repository-cli==0.6.0 -lazy-object-proxy==1.6.0 -localstack-client==1.26 -lxml==4.6.4 -MarkupSafe==2.0.1 -Mastodon.py==1.5.0 -mccabe==0.6.1 -mock==4.0.3 -mutagen==1.45.1 -networkx==2.6.3 -ordereddict==1.1 +certifi==2022.9.24 +charset-normalizer==2.1.1 +click==8.1.3 +ghp-import==2.1.0 +idna==3.4 +importlib-metadata==4.12.0 +Jinja2==3.1.2 +Markdown==3.3.7 +markdown-include==0.7.0 +MarkupSafe==2.1.1 +mergedeep==1.3.4 +mkdocs==1.4.2 +mkdocs-material==8.5.9 +mkdocs-material-extensions==1.1 packaging==21.3 -paramiko==2.10.3 -parso==0.7.1 -pathspec==0.9.0 -pbr==5.8.0 -pipenv==2021.5.29 -pkginfo==1.7.1 -platformdirs==2.4.0 -pluggy==1.0.0 -poyo==0.5.0 -prompt-toolkit==3.0.22 -psutil==5.9.0 -py==1.11.0 -pyaml==21.10.1 -pyasn1==0.4.8 -pycodestyle==2.5.0 -pycparser==2.21 -pycryptodomex==3.14.1 -pycurl==7.44.1 -pyflakes==2.1.1 -Pygments==2.10.0 -pylint==2.11.1 -pymongo==4.0.1 -PyNaCl==1.5.0 -pyparsing==3.0.6 -pyrsistent==0.18.0 -pytest==6.2.5 -pytest-cov==3.0.0 -pytest-forked==1.3.0 -pytest-xdist==2.4.0 +Pygments==2.13.0 +pymdown-extensions==9.8 +pyparsing==3.0.9 python-dateutil==2.8.2 -python-dotenv==0.20.0 -python-jose==3.3.0 -python-jsonrpc-server==0.4.0 -python-language-server==0.36.2 -python-magic==0.4.25 -python-slugify==6.1.2 -pytz==2021.3 -PyYAML==5.4.1 -readme-renderer==30.0 -regex==2019.11.1 -requests==2.26.0 -requests-toolbelt==0.9.1 -requirements-parser==0.3.1 -responses==0.16.0 -rfc3986==1.5.0 -rpmvenv==0.25.0 -rsa==4.7.2 -s3transfer==0.5.0 -sarif-om==1.0.4 -SecretStorage==3.3.1 -semver==2.9.1 +PyYAML==6.0 +pyyaml_env_tag==0.1 +requests==2.28.1 six==1.16.0 -sshpubkeys==3.3.1 -sure==2.0.0 -text-unidecode==1.3 -texttable==1.6.4 -toml==0.10.2 -tomli==1.2.2 -tootstream==0.3.8.1 -tqdm==4.62.3 -twine==3.6.0 -typed-ast==1.5.0 -types-setuptools==57.4.7 -ujson==4.2.0 -urllib3==1.26.7 -venvctrl==0.4.2 -virtualenv==20.8.1 -virtualenv-clone==0.5.7 -wcwidth==0.2.5 -webencodings==0.5.1 -websocket-client==0.59.0 -websockets==10.2 -Werkzeug==2.0.2 -wrapt==1.13.3 -xmltodict==0.12.0 -yt-dlp==2022.2.4 -zipp==3.6.0 +urllib3==1.26.12 +watchdog==2.1.9 +zipp==3.8.0 From 89dbc7177b55c7f49632e75915fdcc77f737f20e Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 11 Nov 2022 16:12:04 +0000 Subject: [PATCH 53/57] Chores for 2022.11 --- CHANGELOG.md | 7 ++++++- README.md | 5 ++--- lib/pleroma/web/plugs/http_security_plug.ex | 13 +++++++------ mix.exs | 2 +- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e71b2127..ac6695c36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## Unreleased +## 2022.11 ## Added - Officially supported docker release @@ -20,6 +20,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## Fixed - Registrations via ldap are now compatible with the latest OTP24 +## Update notes +- If you use LDAP and run from source, please update your elixir/erlang + to the latest. The changes in OTP24.3 are breaking. +- You can now remove the leading `*.` from domain blocks, but you do not have to. + ## 2022.10 ### Added diff --git a/README.md b/README.md index 212f99c60..8d35212aa 100644 --- a/README.md +++ b/README.md @@ -46,15 +46,13 @@ If your platform is not supported, or you just want to be able to edit the sourc - [Alpine Linux](https://docs.akkoma.dev/stable/installation/alpine_linux_en/) - [Arch Linux](https://docs.akkoma.dev/stable/installation/arch_linux_en/) - [Debian-based](https://docs.akkoma.dev/stable/installation/debian_based_en/) -- [Debian-based (jp)](https://docs.akkoma.dev/stable/installation/debian_based_jp/) - [FreeBSD](https://docs.akkoma.dev/stable/installation/freebsd_en/) - [Gentoo Linux](https://docs.akkoma.dev/stable/installation/gentoo_en/) - [NetBSD](https://docs.akkoma.dev/stable/installation/netbsd_en/) - [OpenBSD](https://docs.akkoma.dev/stable/installation/openbsd_en/) -- [OpenBSD (fi)](https://docs.akkoma.dev/stable/installation/openbsd_fi/) ### Docker -While we don’t provide docker files, other people have written very good ones. Take a look at or . +Docker installation is supported via [this setup](https://docs.akkoma.dev/stable/installation/docker_en/) ### Compilation Troubleshooting If you ever encounter compilation issues during the updating of Akkoma, you can try these commands and see if they fix things: @@ -66,3 +64,4 @@ If you ever encounter compilation issues during the updating of Akkoma, you can ## Documentation - https://docs.akkoma.dev/stable +- https://docs.akkoma.dev/develop diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index 43b075447..fc2f7b268 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -104,12 +104,13 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do {[img_src, " https:"], [media_src, " https:"]} end - connect_src = if Config.get([:media_proxy, :enabled]) do - sources = build_csp_multimedia_source_list() - ["connect-src 'self' blob: ", static_url, ?\s, websocket_url, ?\s, sources] - else - ["connect-src 'self' blob: ", static_url, ?\s, websocket_url] - end + connect_src = + if Config.get([:media_proxy, :enabled]) do + sources = build_csp_multimedia_source_list() + ["connect-src 'self' blob: ", static_url, ?\s, websocket_url, ?\s, sources] + else + ["connect-src 'self' blob: ", static_url, ?\s, websocket_url] + end script_src = if Config.get(:env) == :dev do diff --git a/mix.exs b/mix.exs index d95cc1778..201299aa5 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Pleroma.Mixfile do def project do [ app: :pleroma, - version: version("3.3.1"), + version: version("3.4.0"), elixir: "~> 1.12", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(), From 35cddd7cf7c8a95dcdc5c97466eac082f9eca939 Mon Sep 17 00:00:00 2001 From: nocebo Date: Sun, 13 Nov 2022 08:43:12 +0000 Subject: [PATCH 54/57] change default redirectRootNoLogin to /main/public close #268 --- config/config.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.exs b/config/config.exs index 644155aeb..fd470ed93 100644 --- a/config/config.exs +++ b/config/config.exs @@ -317,7 +317,7 @@ config :pleroma, :frontend_configurations, nsfwCensorImage: "", postContentType: "text/plain", redirectRootLogin: "/main/friends", - redirectRootNoLogin: "/main/all", + redirectRootNoLogin: "/main/public", scopeCopy: true, sidebarRight: false, showFeaturesPanel: true, From c1127e321b151a98709072c1789a04c98bcf8c91 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Sun, 13 Nov 2022 23:55:51 +0000 Subject: [PATCH 55/57] Add configurable timeline per oban job (#273) Heavily inspired by https://git.pleroma.social/pleroma/pleroma/-/merge_requests/3777 Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/273 --- config/config.exs | 21 +++++++++++++++ config/description.exs | 26 +++++++++++++++++++ lib/pleroma/workers/backup_worker.ex | 6 +++++ lib/pleroma/workers/purge_expired_activity.ex | 5 ++++ lib/pleroma/workers/purge_expired_filter.ex | 5 ++++ lib/pleroma/workers/purge_expired_token.ex | 5 ++++ lib/pleroma/workers/worker_helper.ex | 6 +++++ .../workers/purge_expired_activity_test.exs | 5 ++++ .../scheduled_activity_worker_test.exs | 5 ++++ 9 files changed, 84 insertions(+) diff --git a/config/config.exs b/config/config.exs index fd470ed93..ba77d8b02 100644 --- a/config/config.exs +++ b/config/config.exs @@ -584,6 +584,27 @@ config :pleroma, :workers, federator_incoming: 5, federator_outgoing: 5, search_indexing: 2 + ], + timeout: [ + activity_expiration: :timer.seconds(5), + token_expiration: :timer.seconds(5), + filter_expiration: :timer.seconds(5), + backup: :timer.seconds(900), + federator_incoming: :timer.seconds(10), + federator_outgoing: :timer.seconds(10), + ingestion_queue: :timer.seconds(5), + web_push: :timer.seconds(5), + mailer: :timer.seconds(5), + transmogrifier: :timer.seconds(5), + scheduled_activities: :timer.seconds(5), + poll_notifications: :timer.seconds(5), + background: :timer.seconds(5), + remote_fetcher: :timer.seconds(10), + attachments_cleanup: :timer.seconds(900), + new_users_digest: :timer.seconds(10), + mute_expire: :timer.seconds(5), + search_indexing: :timer.seconds(5), + nodeinfo_fetcher: :timer.seconds(10) ] config :pleroma, Pleroma.Formatter, diff --git a/config/description.exs b/config/description.exs index 4843c0aae..287abb747 100644 --- a/config/description.exs +++ b/config/description.exs @@ -1979,6 +1979,32 @@ config :pleroma, :config_description, [ federator_incoming: 5, federator_outgoing: 5 ] + }, + %{ + key: :timeout, + type: {:keyword, :integer}, + description: "Timeout for jobs, per `Oban` queue, in ms", + suggestions: [ + activity_expiration: :timer.seconds(5), + token_expiration: :timer.seconds(5), + filter_expiration: :timer.seconds(5), + backup: :timer.seconds(900), + federator_incoming: :timer.seconds(10), + federator_outgoing: :timer.seconds(10), + ingestion_queue: :timer.seconds(5), + web_push: :timer.seconds(5), + mailer: :timer.seconds(5), + transmogrifier: :timer.seconds(5), + scheduled_activities: :timer.seconds(5), + poll_notifications: :timer.seconds(5), + background: :timer.seconds(5), + remote_fetcher: :timer.seconds(10), + attachments_cleanup: :timer.seconds(900), + new_users_digest: :timer.seconds(10), + mute_expire: :timer.seconds(5), + search_indexing: :timer.seconds(5), + nodeinfo_fetcher: :timer.seconds(10) + ] } ] }, diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index 66c5c3591..4ab08706e 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -14,6 +14,11 @@ defmodule Pleroma.Workers.BackupWorker do |> Oban.insert() end + @impl Oban.Worker + def timeout(_job) do + Pleroma.Config.get([:workers, :timeout, :backup]) || :timer.minutes(1) + end + def schedule_deletion(backup) do days = Pleroma.Config.get([Backup, :purge_after_days]) time = 60 * 60 * 24 * days @@ -30,6 +35,7 @@ defmodule Pleroma.Workers.BackupWorker do |> Oban.insert() end + @impl true def perform(%Job{ args: %{"op" => "process", "backup_id" => backup_id, "admin_user_id" => admin_user_id} }) do diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex index 027171c1e..ece84df37 100644 --- a/lib/pleroma/workers/purge_expired_activity.ex +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -27,6 +27,11 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do end end + @impl Oban.Worker + def timeout(_job) do + Pleroma.Config.get([:workers, :timeout, :activity_expiration]) || :timer.minutes(1) + end + @impl true def perform(%Oban.Job{args: %{"activity_id" => id}}) do with %Activity{} = activity <- find_activity(id), diff --git a/lib/pleroma/workers/purge_expired_filter.ex b/lib/pleroma/workers/purge_expired_filter.ex index 4740d52e9..dd6a22a68 100644 --- a/lib/pleroma/workers/purge_expired_filter.ex +++ b/lib/pleroma/workers/purge_expired_filter.ex @@ -24,6 +24,11 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do |> Oban.insert() end + @impl Oban.Worker + def timeout(_job) do + Pleroma.Config.get([:workers, :timeout, :filter_expiration]) || :timer.minutes(1) + end + @impl true def perform(%Job{args: %{"filter_id" => id}}) do Pleroma.Filter diff --git a/lib/pleroma/workers/purge_expired_token.ex b/lib/pleroma/workers/purge_expired_token.ex index cfdf5c6dc..1773aeff9 100644 --- a/lib/pleroma/workers/purge_expired_token.ex +++ b/lib/pleroma/workers/purge_expired_token.ex @@ -19,6 +19,11 @@ defmodule Pleroma.Workers.PurgeExpiredToken do |> Oban.insert() end + @impl Oban.Worker + def timeout(_job) do + Pleroma.Config.get([:workers, :timeout, :token_expiration]) || :timer.minutes(1) + end + @impl true def perform(%Oban.Job{args: %{"token_id" => id, "mod" => module}}) do module diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex index 4befbeb3b..97c0e4e5c 100644 --- a/lib/pleroma/workers/worker_helper.ex +++ b/lib/pleroma/workers/worker_helper.ex @@ -43,6 +43,12 @@ defmodule Pleroma.Workers.WorkerHelper do |> apply(:new, [params, worker_args]) |> Oban.insert() end + + @impl Oban.Worker + def timeout(_job) do + queue_atom = String.to_atom(unquote(queue)) + Config.get([:workers, :timeout, queue_atom]) || :timer.minutes(1) + end end end end diff --git a/test/pleroma/workers/purge_expired_activity_test.exs b/test/pleroma/workers/purge_expired_activity_test.exs index 98f30f61f..6285dca3e 100644 --- a/test/pleroma/workers/purge_expired_activity_test.exs +++ b/test/pleroma/workers/purge_expired_activity_test.exs @@ -56,4 +56,9 @@ defmodule Pleroma.Workers.PurgeExpiredActivityTest do assert {:error, :activity_not_found} = perform_job(Pleroma.Workers.PurgeExpiredActivity, %{activity_id: "some_if"}) end + + test "has a timeout" do + clear_config([:workers, :timeout, :activity_expiration], 50) + assert Pleroma.Workers.PurgeExpiredActivity.timeout(%Oban.Job{}) == 50 + end end diff --git a/test/pleroma/workers/scheduled_activity_worker_test.exs b/test/pleroma/workers/scheduled_activity_worker_test.exs index 5558d5b5f..9f5f1b687 100644 --- a/test/pleroma/workers/scheduled_activity_worker_test.exs +++ b/test/pleroma/workers/scheduled_activity_worker_test.exs @@ -49,4 +49,9 @@ defmodule Pleroma.Workers.ScheduledActivityWorkerTest do ScheduledActivityWorker.perform(%Oban.Job{args: %{"activity_id" => 42}}) end) =~ "Couldn't find scheduled activity: 42" end + + test "has a timeout" do + clear_config([:workers, :timeout, :scheduled_activities], :timer.minutes(5)) + assert ScheduledActivityWorker.timeout(nil) == :timer.minutes(5) + end end From 893bfde66fdfe37f20e15cb26989d2ddf36d4964 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Mon, 14 Nov 2022 00:01:31 +0000 Subject: [PATCH 56/57] Remove references to soykaf Fixes #271 --- .../docs/development/API/differences_in_mastoapi_responses.md | 2 +- docs/docs/development/API/prometheus.md | 2 +- docs/docs/index.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/docs/development/API/differences_in_mastoapi_responses.md b/docs/docs/development/API/differences_in_mastoapi_responses.md index 752be1762..b41561c45 100644 --- a/docs/docs/development/API/differences_in_mastoapi_responses.md +++ b/docs/docs/development/API/differences_in_mastoapi_responses.md @@ -195,7 +195,7 @@ Additional parameters can be added to the JSON body/Form data: - `preview`: boolean, if set to `true` the post won't be actually posted, but the status entity would still be rendered back. This could be useful for previewing rich text/custom emoji, for example. - `content_type`: string, contain the MIME type of the status, it is transformed into HTML by the backend. You can get the list of the supported MIME types with the nodeinfo endpoint. -- `to`: A list of nicknames (like `lain@soykaf.club` or `lain` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for post visibility are not affected by this and will still apply. +- `to`: A list of nicknames (like `admin@otp.akkoma.dev` or `admin` on the local server) that will be used to determine who is going to be addressed by this post. Using this will disable the implicit addressing by mentioned names in the `status` body, only the people in the `to` list will be addressed. The normal rules for post visibility are not affected by this and will still apply. - `visibility`: string, besides standard MastoAPI values (`direct`, `private`, `unlisted`, `local` or `public`) it can be used to address a List by setting it to `list:LIST_ID`. - `expires_in`: The number of seconds the posted activity should expire in. When a posted activity expires it will be deleted from the server, and a delete request for it will be federated. This needs to be longer than an hour. - `in_reply_to_conversation_id`: Will reply to a given conversation, addressing only the people who are part of the recipient set of that conversation. Sets the visibility to `direct`. diff --git a/docs/docs/development/API/prometheus.md b/docs/docs/development/API/prometheus.md index 0c23a404d..39ecc5d38 100644 --- a/docs/docs/development/API/prometheus.md +++ b/docs/docs/development/API/prometheus.md @@ -40,5 +40,5 @@ The following is a config example to use with [Grafana](https://grafana.com) metrics_path: /api/pleroma/app_metrics scheme: https static_configs: - - targets: ['pleroma.soykaf.com'] + - targets: ['otp.akkoma.dev'] ``` diff --git a/docs/docs/index.md b/docs/docs/index.md index 1018e9c2b..f5b00b457 100644 --- a/docs/docs/index.md +++ b/docs/docs/index.md @@ -29,14 +29,14 @@ If you don't feel like joining an existing instance, but instead prefer to deplo Installation instructions can be found in the installation section of these docs. ## I got an account, now what? -Great! Now you can explore the fediverse! Open the login page for your Akkoma instance (e.g. ) and login with your username and password. (If you don't have an account yet, click on Register) +Great! Now you can explore the fediverse! Open the login page for your Akkoma instance (e.g. ) and login with your username and password. (If you don't have an account yet, click on Register) ### Pleroma-FE The default front-end used by Akkoma is Pleroma-FE. You can find more information on what it is and how to use it in the [Introduction to Pleroma-FE](https://docs-fe.akkoma.dev/stable/). ### Mastodon interface If the Pleroma-FE interface isn't your thing, or you're just trying something new but you want to keep using the familiar Mastodon interface, we got that too! -Just add a "/web" after your instance url (e.g. ) and you'll end on the Mastodon web interface, but with a Akkoma backend! MAGIC! +Just add a "/web" after your instance url (e.g. ) and you'll end on the Mastodon web interface, but with a Akkoma backend! MAGIC! The Mastodon interface is from the Glitch-soc fork. For more information on the Mastodon interface you can check the [Mastodon](https://docs.joinmastodon.org/) and [Glitch-soc](https://glitch-soc.github.io/docs/) documentation. Remember, what you see is only the frontend part of Mastodon, the backend is still Akkoma. From 2a1f17e3ede2054f31c35b5dc240f51d0d5509d1 Mon Sep 17 00:00:00 2001 From: floatingghost Date: Mon, 14 Nov 2022 15:07:26 +0000 Subject: [PATCH 57/57] and i yoink (#275) Co-authored-by: Mark Felder Co-authored-by: FloatingGhost Reviewed-on: https://akkoma.dev/AkkomaGang/akkoma/pulls/275 --- lib/pleroma/web/federator.ex | 6 ++- lib/pleroma/workers/backup_worker.ex | 2 +- lib/pleroma/workers/purge_expired_activity.ex | 2 +- lib/pleroma/workers/purge_expired_filter.ex | 2 +- lib/pleroma/workers/purge_expired_token.ex | 2 +- lib/pleroma/workers/worker_helper.ex | 2 +- .../pleroma/workers/publisher_worker_test.exs | 52 +++++++++++++++++++ 7 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 test/pleroma/workers/publisher_worker_test.exs diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 770044de2..3a00424c6 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -48,7 +48,9 @@ defmodule Pleroma.Web.Federator do @impl true def publish(%{data: %{"object" => object}} = activity) when is_binary(object) do - PublisherWorker.enqueue("publish", %{"activity_id" => activity.id, "object_data" => nil}) + PublisherWorker.enqueue("publish", %{"activity_id" => activity.id, "object_data" => nil}, + priority: publish_priority(activity) + ) end @impl true @@ -63,7 +65,7 @@ defmodule Pleroma.Web.Federator do ) end - defp publish_priority(%{type: "Delete"}), do: 3 + defp publish_priority(%{data: %{"type" => "Delete"}}), do: 3 defp publish_priority(_), do: 0 # Job Worker Callbacks diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index 4ab08706e..cf78f1cb9 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Workers.BackupWorker do @impl Oban.Worker def timeout(_job) do - Pleroma.Config.get([:workers, :timeout, :backup]) || :timer.minutes(1) + Pleroma.Config.get([:workers, :timeout, :backup], :timer.minutes(1)) end def schedule_deletion(backup) do diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex index ece84df37..652e1d6b1 100644 --- a/lib/pleroma/workers/purge_expired_activity.ex +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -29,7 +29,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do @impl Oban.Worker def timeout(_job) do - Pleroma.Config.get([:workers, :timeout, :activity_expiration]) || :timer.minutes(1) + Pleroma.Config.get([:workers, :timeout, :activity_expiration], :timer.minutes(1)) end @impl true diff --git a/lib/pleroma/workers/purge_expired_filter.ex b/lib/pleroma/workers/purge_expired_filter.ex index dd6a22a68..593380d13 100644 --- a/lib/pleroma/workers/purge_expired_filter.ex +++ b/lib/pleroma/workers/purge_expired_filter.ex @@ -26,7 +26,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do @impl Oban.Worker def timeout(_job) do - Pleroma.Config.get([:workers, :timeout, :filter_expiration]) || :timer.minutes(1) + Pleroma.Config.get([:workers, :timeout, :filter_expiration], :timer.minutes(1)) end @impl true diff --git a/lib/pleroma/workers/purge_expired_token.ex b/lib/pleroma/workers/purge_expired_token.ex index 1773aeff9..b4db84f4e 100644 --- a/lib/pleroma/workers/purge_expired_token.ex +++ b/lib/pleroma/workers/purge_expired_token.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Workers.PurgeExpiredToken do @impl Oban.Worker def timeout(_job) do - Pleroma.Config.get([:workers, :timeout, :token_expiration]) || :timer.minutes(1) + Pleroma.Config.get([:workers, :timeout, :token_expiration], :timer.minutes(1)) end @impl true diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex index 97c0e4e5c..4c0a55774 100644 --- a/lib/pleroma/workers/worker_helper.ex +++ b/lib/pleroma/workers/worker_helper.ex @@ -47,7 +47,7 @@ defmodule Pleroma.Workers.WorkerHelper do @impl Oban.Worker def timeout(_job) do queue_atom = String.to_atom(unquote(queue)) - Config.get([:workers, :timeout, queue_atom]) || :timer.minutes(1) + Config.get([:workers, :timeout, queue_atom], :timer.minutes(1)) end end end diff --git a/test/pleroma/workers/publisher_worker_test.exs b/test/pleroma/workers/publisher_worker_test.exs new file mode 100644 index 000000000..cf0ac0ccb --- /dev/null +++ b/test/pleroma/workers/publisher_worker_test.exs @@ -0,0 +1,52 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.PublisherWorkerTest do + use Pleroma.DataCase, async: true + use Oban.Testing, repo: Pleroma.Repo + + import Pleroma.Factory + + alias Pleroma.Object + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Builder + alias Pleroma.Web.CommonAPI + alias Pleroma.Web.Federator + + describe "Oban job priority:" do + setup do + user = insert(:user) + + {:ok, post} = CommonAPI.post(user, %{status: "Regrettable post"}) + object = Object.normalize(post, fetch: false) + {:ok, delete_data, _meta} = Builder.delete(user, object.data["id"]) + {:ok, delete, _meta} = ActivityPub.persist(delete_data, local: true) + + %{ + post: post, + delete: delete + } + end + + test "Deletions are lower priority", %{delete: delete} do + assert {:ok, %Oban.Job{priority: 3}} = Federator.publish(delete) + end + + test "Creates are normal priority", %{post: post} do + assert {:ok, %Oban.Job{priority: 0}} = Federator.publish(post) + end + end + + describe "Oban job timeout" do + test "should have a timeout" do + clear_config([:workers, :timeout, :federator_outgoing], :timer.minutes(2)) + assert Pleroma.Workers.PublisherWorker.timeout(nil) == :timer.minutes(2) + end + + test "should use a default timeout if none specified" do + clear_config([:workers, :timeout, :federator_outgoing]) + assert Pleroma.Workers.PublisherWorker.timeout(nil) == :timer.seconds(10) + end + end +end