From e272430ef409c2e6dfaab8c2851ecce10f422680 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 21 Oct 2022 10:21:08 +0100 Subject: [PATCH 01/13] Add nodeinfo tasker --- config/config.exs | 3 +- lib/pleroma/instances/instance.ex | 127 +++++++++++------- .../workers/nodeinfo_fetcher_worker.ex | 19 +++ .../20221020135943_add_nodeinfo.exs | 15 +++ 4 files changed, 115 insertions(+), 49 deletions(-) create mode 100644 lib/pleroma/workers/nodeinfo_fetcher_worker.ex create mode 100644 priv/repo/migrations/20221020135943_add_nodeinfo.exs diff --git a/config/config.exs b/config/config.exs index 5eb82cd33..be4c065a3 100644 --- a/config/config.exs +++ b/config/config.exs @@ -567,7 +567,8 @@ config :pleroma, Oban, attachments_cleanup: 1, new_users_digest: 1, mute_expire: 5, - search_indexing: 10 + search_indexing: 10, + nodeinfo_fetcher: 1 ], plugins: [ Oban.Plugins.Pruner, diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 533dbbb82..7fdf35bcc 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -22,7 +22,8 @@ defmodule Pleroma.Instances.Instance do field(:host, :string) field(:unreachable_since, :naive_datetime_usec) field(:favicon, :string) - field(:favicon_updated_at, :naive_datetime) + field(:metadata_updated_at, :naive_datetime) + field(:nodeinfo, :map, default: %{}) timestamps() end @@ -138,63 +139,93 @@ defmodule Pleroma.Instances.Instance do defp parse_datetime(datetime), do: datetime - def get_or_update_favicon(%URI{host: host} = instance_uri) do - existing_record = Repo.get_by(Instance, %{host: host}) - now = NaiveDateTime.utc_now() + def needs_update(nil), do: true - if existing_record && existing_record.favicon_updated_at && - NaiveDateTime.diff(now, existing_record.favicon_updated_at) < 86_400 do + def needs_update(%Instance{metadata_updated_at: metadata_updated_at}) do + now = NaiveDateTime.utc_now() + NaiveDateTime.diff(now, metadata_updated_at) > 86_400 + end + + def update_metadata(%URI{host: host} = uri) do + existing_record = Repo.get_by(Instance, %{host: host}) + + if existing_record do + if needs_update(existing_record) do + favicon = scrape_favicon(uri) + nodeinfo = scrape_nodeinfo(uri) + + %Instance{} + |> changeset(%{host: host, favicon: favicon, nodeinfo: nodeinfo}) + |> Repo.update() + end + else + favicon = scrape_favicon(uri) + nodeinfo = scrape_nodeinfo(uri) + + %Instance{} + |> changeset(%{host: host, favicon: favicon, nodeinfo: nodeinfo}) + |> Repo.insert() + end + end + + def get_favicon(%URI{host: host} = instance_uri) do + existing_record = Repo.get_by(Instance, %{host: host}) + + if existing_record do existing_record.favicon else - favicon = scrape_favicon(instance_uri) - - if existing_record do - existing_record - |> changeset(%{favicon: favicon, favicon_updated_at: now}) - |> Repo.update() - else - %Instance{} - |> changeset(%{host: host, favicon: favicon, favicon_updated_at: now}) - |> Repo.insert() - end - - favicon - end - rescue - e -> - Logger.warn("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}") nil + end + end + + defp scrape_nodeinfo(%URI{} = instance_uri) do + with {_, true} <- {:reachable, reachable?(instance_uri.host)}, + {:ok, %Tesla.Env{status: 200, body: body}} <- + Tesla.get( + "https://#{instance_uri.host}/.well-known/nodeinfo", + headers: [{"Accept", "application/json"}] + ), + {:ok, json} <- Jason.decode(body), + {:ok, %{"links" => links}} <- {:ok, json}, + {:ok, %{"href" => href}} <- + {:ok, + Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0"))}, + {:ok, %Tesla.Env{body: data}} <- + Pleroma.HTTP.get(ref, [{"accept", "application/json"}], []), + {:ok, nodeinfo} <- Jason.decode(data) do + nodeinfo + else + {:reachable, false} -> + Logger.debug( + "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") ignored unreachable host" + ) + + nil + + _ -> + nil + end end defp scrape_favicon(%URI{} = instance_uri) do - try do - with {_, true} <- {:reachable, reachable?(instance_uri.host)}, - {:ok, %Tesla.Env{body: html}} <- - Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], []), - {_, [favicon_rel | _]} when is_binary(favicon_rel) <- - {:parse, - html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")}, - {_, favicon} when is_binary(favicon) <- - {:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do - favicon - else - {:reachable, false} -> - Logger.debug( - "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") ignored unreachable host" - ) - - nil - - _ -> - nil - end - rescue - e -> - Logger.warn( - "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}" + with {_, true} <- {:reachable, reachable?(instance_uri.host)}, + {:ok, %Tesla.Env{body: html}} <- + Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], []), + {_, [favicon_rel | _]} when is_binary(favicon_rel) <- + {:parse, html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")}, + {_, favicon} when is_binary(favicon) <- + {:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do + favicon + else + {:reachable, false} -> + Logger.debug( + "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") ignored unreachable host" ) nil + + _ -> + nil end end diff --git a/lib/pleroma/workers/nodeinfo_fetcher_worker.ex b/lib/pleroma/workers/nodeinfo_fetcher_worker.ex new file mode 100644 index 000000000..019a3d5c2 --- /dev/null +++ b/lib/pleroma/workers/nodeinfo_fetcher_worker.ex @@ -0,0 +1,19 @@ +defmodule Pleroma.Workers.NodeInfoFetcherWorker do + use Oban.Worker, queue: :backup, max_attempts: 1 + + alias Oban.Job + alias Pleroma.Instance + + def process(domain) do + %{"op" => "process", "domain" => domain} + |> new() + |> Oban.insert() + end + + def perform(%Job{ + args: %{"op" => "process", "domain" => domain} + }) do + uri = URI.parse(domain) + Instance.get_or_update_favicon(uri) + end +end diff --git a/priv/repo/migrations/20221020135943_add_nodeinfo.exs b/priv/repo/migrations/20221020135943_add_nodeinfo.exs new file mode 100644 index 000000000..6eab3b884 --- /dev/null +++ b/priv/repo/migrations/20221020135943_add_nodeinfo.exs @@ -0,0 +1,15 @@ +defmodule Pleroma.Repo.Migrations.AddNodeinfo do + use Ecto.Migration + + def up do + alter table(:instances) do + add_if_not_exists(:nodeinfo, :map, default: %{}) + end + end + + def down do + alter table(:instances) do + remove_if_exists(:nodeinfo, :map) + end + end +end -- 2.34.1 From 881dbea92ecdd9d024ceb90578c682f3683c963e Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sat, 29 Oct 2022 21:51:45 +0100 Subject: [PATCH 02/13] Add tasker to get nodeinfo --- lib/pleroma/instances/instance.ex | 19 +++++++++++++------ lib/pleroma/web/activity_pub/side_effects.ex | 2 ++ .../workers/nodeinfo_fetcher_worker.ex | 18 ++++++++---------- lib/pleroma/workers/worker_helper.ex | 1 + .../20221020135943_add_nodeinfo.exs | 2 ++ 5 files changed, 26 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 7fdf35bcc..7b3ba79c9 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -32,7 +32,7 @@ defmodule Pleroma.Instances.Instance do def changeset(struct, params \\ %{}) do struct - |> cast(params, [:host, :unreachable_since, :favicon, :favicon_updated_at]) + |> cast(params, [:host, :unreachable_since, :favicon, :nodeinfo, :metadata_updated_at]) |> validate_required([:host]) |> unique_constraint(:host) end @@ -141,34 +141,41 @@ defmodule Pleroma.Instances.Instance do def needs_update(nil), do: true + def needs_update(%Instance{metadata_updated_at: nil}), do: true + def needs_update(%Instance{metadata_updated_at: metadata_updated_at}) do now = NaiveDateTime.utc_now() NaiveDateTime.diff(now, metadata_updated_at) > 86_400 end def update_metadata(%URI{host: host} = uri) do + Logger.info("Checking metadata for #{host}") existing_record = Repo.get_by(Instance, %{host: host}) if existing_record do if needs_update(existing_record) do + Logger.info("Updating metadata for #{host}") favicon = scrape_favicon(uri) nodeinfo = scrape_nodeinfo(uri) - %Instance{} - |> changeset(%{host: host, favicon: favicon, nodeinfo: nodeinfo}) + existing_record + |> changeset(%{host: host, favicon: favicon, nodeinfo: nodeinfo, metadata_updated_at: NaiveDateTime.utc_now()}) |> Repo.update() + else + {:discard, "Does not require update"} end else favicon = scrape_favicon(uri) nodeinfo = scrape_nodeinfo(uri) + Logger.info("Creating metadata for #{host}") %Instance{} - |> changeset(%{host: host, favicon: favicon, nodeinfo: nodeinfo}) + |> changeset(%{host: host, favicon: favicon, nodeinfo: nodeinfo, metadata_updated_at: NaiveDateTime.utc_now()}) |> Repo.insert() end end - def get_favicon(%URI{host: host} = instance_uri) do + def get_favicon(%URI{host: host}) do existing_record = Repo.get_by(Instance, %{host: host}) if existing_record do @@ -191,7 +198,7 @@ defmodule Pleroma.Instances.Instance do {:ok, Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0"))}, {:ok, %Tesla.Env{body: data}} <- - Pleroma.HTTP.get(ref, [{"accept", "application/json"}], []), + Pleroma.HTTP.get(href, [{"accept", "application/json"}], []), {:ok, nodeinfo} <- Jason.decode(data) do nodeinfo else diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index c3258c75b..a3830e408 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -209,6 +209,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do reply_depth = (meta[:depth] || 0) + 1 + IO.puts("QUEUE!") + Pleroma.Workers.NodeInfoFetcherWorker.enqueue("process", %{"domain" => activity.data["actor"]}) # FIXME: Force inReplyTo to replies if Pleroma.Web.Federator.allowed_thread_distance?(reply_depth) and object.data["replies"] != nil do diff --git a/lib/pleroma/workers/nodeinfo_fetcher_worker.ex b/lib/pleroma/workers/nodeinfo_fetcher_worker.ex index 019a3d5c2..5a58de268 100644 --- a/lib/pleroma/workers/nodeinfo_fetcher_worker.ex +++ b/lib/pleroma/workers/nodeinfo_fetcher_worker.ex @@ -1,19 +1,17 @@ defmodule Pleroma.Workers.NodeInfoFetcherWorker do - use Oban.Worker, queue: :backup, max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "nodeinfo_fetcher" alias Oban.Job - alias Pleroma.Instance - - def process(domain) do - %{"op" => "process", "domain" => domain} - |> new() - |> Oban.insert() - end + alias Pleroma.Instances.Instance + @impl Oban.Worker def perform(%Job{ args: %{"op" => "process", "domain" => domain} }) do - uri = URI.parse(domain) - Instance.get_or_update_favicon(uri) + uri = domain + |> URI.parse() + |> URI.merge("/") + + Instance.update_metadata(uri) end end diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex index 4befbeb3b..47afc9dd7 100644 --- a/lib/pleroma/workers/worker_helper.ex +++ b/lib/pleroma/workers/worker_helper.ex @@ -42,6 +42,7 @@ defmodule Pleroma.Workers.WorkerHelper do unquote(caller_module) |> apply(:new, [params, worker_args]) |> Oban.insert() + |> IO.inspect() end end end diff --git a/priv/repo/migrations/20221020135943_add_nodeinfo.exs b/priv/repo/migrations/20221020135943_add_nodeinfo.exs index 6eab3b884..17707f3f7 100644 --- a/priv/repo/migrations/20221020135943_add_nodeinfo.exs +++ b/priv/repo/migrations/20221020135943_add_nodeinfo.exs @@ -4,12 +4,14 @@ defmodule Pleroma.Repo.Migrations.AddNodeinfo do def up do alter table(:instances) do add_if_not_exists(:nodeinfo, :map, default: %{}) + add_if_not_exists(:metadata_updated_at, :naive_datetime) end end def down do alter table(:instances) do remove_if_exists(:nodeinfo, :map) + remove_if_exists(:metadata_updated_at, :naive_datetime) end end end -- 2.34.1 From d736e55746413099919af49b50d2d1680928bcb2 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Wed, 2 Nov 2022 22:37:44 +0000 Subject: [PATCH 03/13] Add instance rendering in user json --- lib/pleroma/application.ex | 3 +- lib/pleroma/instances/instance.ex | 48 ++++++++++++++++--- lib/pleroma/web/activity_pub/side_effects.ex | 6 ++- .../web/mastodon_api/views/account_view.ex | 38 ++++++++++----- .../web/mastodon_api/views/status_view.ex | 1 + .../workers/nodeinfo_fetcher_worker.ex | 7 +-- 6 files changed, 80 insertions(+), 23 deletions(-) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index adccd7c5d..a78924dfa 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -156,7 +156,8 @@ defmodule Pleroma.Application do build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("failed_proxy_url", limit: 2500), build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000), - build_cachex("translations", default_ttl: :timer.hours(24 * 30), limit: 2500) + build_cachex("translations", default_ttl: :timer.hours(24 * 30), limit: 2500), + build_cachex("instances", default_ttl: :timer.hours(24), limit: 2500) ] end diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 7b3ba79c9..0627aec64 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -5,6 +5,8 @@ defmodule Pleroma.Instances.Instance do @moduledoc "Instance." + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + alias Pleroma.Instances alias Pleroma.Instances.Instance alias Pleroma.Repo @@ -158,9 +160,17 @@ defmodule Pleroma.Instances.Instance do favicon = scrape_favicon(uri) nodeinfo = scrape_nodeinfo(uri) - existing_record - |> changeset(%{host: host, favicon: favicon, nodeinfo: nodeinfo, metadata_updated_at: NaiveDateTime.utc_now()}) - |> Repo.update() + {:ok, instance} = + existing_record + |> changeset(%{ + host: host, + favicon: favicon, + nodeinfo: nodeinfo, + metadata_updated_at: NaiveDateTime.utc_now() + }) + |> Repo.update() + + @cachex.put(:instances_cache, "instances:#{host}", instance) else {:discard, "Does not require update"} end @@ -169,9 +179,18 @@ defmodule Pleroma.Instances.Instance do nodeinfo = scrape_nodeinfo(uri) Logger.info("Creating metadata for #{host}") - %Instance{} - |> changeset(%{host: host, favicon: favicon, nodeinfo: nodeinfo, metadata_updated_at: NaiveDateTime.utc_now()}) - |> Repo.insert() + + {:ok, instance} = + %Instance{} + |> changeset(%{ + host: host, + favicon: favicon, + nodeinfo: nodeinfo, + metadata_updated_at: NaiveDateTime.utc_now() + }) + |> Repo.insert() + + @cachex.put(:instances_cache, "instances:#{host}", instance) end end @@ -255,4 +274,21 @@ defmodule Pleroma.Instances.Instance do end) |> Stream.run() end + + def get_by_url(url_or_host) do + url = host(url_or_host) + Repo.get_by(Instance, host: url) + end + + def get_cached_by_url(url_or_host) do + url = host(url_or_host) + + @cachex.fetch!(:instances_cache, "instances:#{url}", fn _ -> + with %Instance{} = instance <- get_by_url(url) do + {:ok, instance} + else + _ -> {:ignore, nil} + end + end) + end end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index a3830e408..46434ad5b 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -210,7 +210,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do reply_depth = (meta[:depth] || 0) + 1 IO.puts("QUEUE!") - Pleroma.Workers.NodeInfoFetcherWorker.enqueue("process", %{"domain" => activity.data["actor"]}) + + Pleroma.Workers.NodeInfoFetcherWorker.enqueue("process", %{ + "domain" => activity.data["actor"] + }) + # FIXME: Force inReplyTo to replies if Pleroma.Web.Federator.allowed_thread_distance?(reply_depth) and object.data["replies"] != nil do diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 06acf0a26..8172a81fb 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -186,6 +186,16 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do render_many(targets, AccountView, "relationship.json", render_opts) end + def render("instance.json", %{instance: %Pleroma.Instances.Instance{} = instance}) do + %{ + name: instance.host, + favicon: instance.favicon |> MediaProxy.url(), + nodeinfo: instance.nodeinfo + } + end + + def render("instance.json", _), do: nil + defp do_render("show.json", %{user: user} = opts) do user = User.sanitize_html(user, User.html_filter_policy(opts[:for])) display_name = user.name || user.nickname @@ -230,17 +240,19 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do %{} end - favicon = - if Pleroma.Config.get([:instances_favicons, :enabled]) do - user - |> Map.get(:ap_id, "") - |> URI.parse() - |> URI.merge("/") - |> Pleroma.Instances.Instance.get_or_update_favicon() - |> MediaProxy.url() - else - nil - end + instance = with {:ok, instance} <- Pleroma.Instances.Instance.get_cached_by_url(user.ap_id) do + instance + else + _ -> + nil + end + + favicon = if is_nil(instance) do + nil + else + instance.favicon + |> MediaProxy.url() + end %{ id: to_string(user.id), @@ -271,7 +283,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do } }, last_status_at: user.last_status_at, - + akkoma: %{ + instance: render("instance.json", %{instance: instance}) + }, # Pleroma extensions # Note: it's insecure to output :email but fully-qualified nickname may serve as safe stub fqn: User.full_nickname(user), diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index b3a35526e..8bb192bad 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -421,6 +421,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do nil end + def render("history.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do object = Object.normalize(activity, fetch: false) diff --git a/lib/pleroma/workers/nodeinfo_fetcher_worker.ex b/lib/pleroma/workers/nodeinfo_fetcher_worker.ex index 5a58de268..0ac395cdc 100644 --- a/lib/pleroma/workers/nodeinfo_fetcher_worker.ex +++ b/lib/pleroma/workers/nodeinfo_fetcher_worker.ex @@ -8,9 +8,10 @@ defmodule Pleroma.Workers.NodeInfoFetcherWorker do def perform(%Job{ args: %{"op" => "process", "domain" => domain} }) do - uri = domain - |> URI.parse() - |> URI.merge("/") + uri = + domain + |> URI.parse() + |> URI.merge("/") Instance.update_metadata(uri) end -- 2.34.1 From 525434bc9cbed8e7083f3209f9b0699ba55edfab Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 4 Nov 2022 03:46:55 +0000 Subject: [PATCH 04/13] Add test for queueing of incoming activities --- lib/pleroma/web/activity_pub/side_effects.ex | 9 ++++--- .../workers/nodeinfo_fetcher_worker.ex | 2 +- lib/pleroma/workers/worker_helper.ex | 3 +-- .../web/activity_pub/side_effects_test.exs | 26 +++++++++++++++++++ 4 files changed, 33 insertions(+), 7 deletions(-) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 46434ad5b..18643662e 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -192,6 +192,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # - Increase the user note count # - Increase the reply count # - Increase replies count + # - Ask for scraping of nodeinfo # - Set up ActivityExpiration # - Set up notifications # - Index incoming posts for search (if needed) @@ -209,10 +210,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do reply_depth = (meta[:depth] || 0) + 1 - IO.puts("QUEUE!") - Pleroma.Workers.NodeInfoFetcherWorker.enqueue("process", %{ - "domain" => activity.data["actor"] + "source_url" => activity.data["actor"] }) # FIXME: Force inReplyTo to replies @@ -240,7 +239,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do {:ok, activity, meta} else - e -> Repo.rollback(e) + e -> + Logger.error(inspect(e)) + Repo.rollback(e) end end diff --git a/lib/pleroma/workers/nodeinfo_fetcher_worker.ex b/lib/pleroma/workers/nodeinfo_fetcher_worker.ex index 0ac395cdc..27492e1e3 100644 --- a/lib/pleroma/workers/nodeinfo_fetcher_worker.ex +++ b/lib/pleroma/workers/nodeinfo_fetcher_worker.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Workers.NodeInfoFetcherWorker do @impl Oban.Worker def perform(%Job{ - args: %{"op" => "process", "domain" => domain} + args: %{"op" => "process", "source_url" => domain} }) do uri = domain diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex index 47afc9dd7..ed13a96a9 100644 --- a/lib/pleroma/workers/worker_helper.ex +++ b/lib/pleroma/workers/worker_helper.ex @@ -41,8 +41,7 @@ defmodule Pleroma.Workers.WorkerHelper do unquote(caller_module) |> apply(:new, [params, worker_args]) - |> Oban.insert() - |> IO.inspect() + |> Oban.insert()\ end end end diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index fa8171eab..94c0836f2 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -21,6 +21,32 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do import Mock import Pleroma.Factory + describe "handle" do + test "it queues a fetch of instance information" do + author = insert(:user, local: false, ap_id: "https://wowee.example.com/users/1") + recipient = insert(:user, local: true) + + {:ok, note_data, _meta} = + Builder.note(%Pleroma.Web.CommonAPI.ActivityDraft{ + user: author, + to: [recipient.ap_id], + mentions: [recipient], + content_html: "hey", + extra: %{"id" => "https://wowee.example.com/notes/1"} + }) + + {:ok, create_activity_data, _meta} = + Builder.create(author, note_data["id"], [recipient.ap_id]) + + {:ok, create_activity, _meta} = ActivityPub.persist(create_activity_data, local: false) + + {:ok, _create_activity, _meta} = + SideEffects.handle(create_activity, local: false, object_data: note_data) + + assert_enqueued(worker: Pleroma.Workers.NodeInfoFetcherWorker, args: %{"op" => "process", "source_url" => "https://wowee.example.com/users/1"}) + end + end + describe "handle_after_transaction" do test "it streams out notifications and streams" do author = insert(:user, local: true) -- 2.34.1 From 3d52295727ecb5c1fe2894161489761d19a5fe9c Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 4 Nov 2022 03:48:24 +0000 Subject: [PATCH 05/13] mix format --- .../web/mastodon_api/views/account_view.ex | 24 ++++++++++--------- .../web/mastodon_api/views/status_view.ex | 1 - lib/pleroma/workers/worker_helper.ex | 2 +- .../web/activity_pub/side_effects_test.exs | 5 +++- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 8172a81fb..cbb57aee6 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -240,19 +240,21 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do %{} end - instance = with {:ok, instance} <- Pleroma.Instances.Instance.get_cached_by_url(user.ap_id) do - instance - else - _ -> + instance = + with {:ok, instance} <- Pleroma.Instances.Instance.get_cached_by_url(user.ap_id) do + instance + else + _ -> nil - end + end - favicon = if is_nil(instance) do - nil - else - instance.favicon - |> MediaProxy.url() - end + favicon = + if is_nil(instance) do + nil + else + instance.favicon + |> MediaProxy.url() + end %{ id: to_string(user.id), diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 8bb192bad..b3a35526e 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -421,7 +421,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do nil end - def render("history.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do object = Object.normalize(activity, fetch: false) diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex index ed13a96a9..4befbeb3b 100644 --- a/lib/pleroma/workers/worker_helper.ex +++ b/lib/pleroma/workers/worker_helper.ex @@ -41,7 +41,7 @@ defmodule Pleroma.Workers.WorkerHelper do unquote(caller_module) |> apply(:new, [params, worker_args]) - |> Oban.insert()\ + |> Oban.insert() end end end diff --git a/test/pleroma/web/activity_pub/side_effects_test.exs b/test/pleroma/web/activity_pub/side_effects_test.exs index 94c0836f2..ee664bb8f 100644 --- a/test/pleroma/web/activity_pub/side_effects_test.exs +++ b/test/pleroma/web/activity_pub/side_effects_test.exs @@ -43,7 +43,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffectsTest do {:ok, _create_activity, _meta} = SideEffects.handle(create_activity, local: false, object_data: note_data) - assert_enqueued(worker: Pleroma.Workers.NodeInfoFetcherWorker, args: %{"op" => "process", "source_url" => "https://wowee.example.com/users/1"}) + assert_enqueued( + worker: Pleroma.Workers.NodeInfoFetcherWorker, + args: %{"op" => "process", "source_url" => "https://wowee.example.com/users/1"} + ) end end -- 2.34.1 From c7cdc00af8d342a6cb798f4a834f9eb9577f9b9b Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 4 Nov 2022 04:35:02 +0000 Subject: [PATCH 06/13] add instance rendering test --- .../mastodon_api/views/account_view_test.exs | 18 ++++++++++++++++++ test/support/factory.ex | 16 +++++++++------- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/account_view_test.exs b/test/pleroma/web/mastodon_api/views/account_view_test.exs index 8db887137..19224f126 100644 --- a/test/pleroma/web/mastodon_api/views/account_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/account_view_test.exs @@ -25,6 +25,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do user = insert(:user, %{ + ap_id: "https://example.com/users/chikichikibanban", follower_count: 3, note_count: 5, background: background_image, @@ -38,6 +39,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do also_known_as: ["https://shitposter.zone/users/shp"] }) + insert(:instance, %{host: "example.com", nodeinfo: %{version: "2.1"}}) + expected = %{ id: to_string(user.id), username: "shp", @@ -50,6 +53,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do statuses_count: 5, note: "valid html. a
b
c
d
f '&<>"", url: user.ap_id, + akkoma: %{ + instance: %{ + name: "example.com", + nodeinfo: %{ + "version" => "2.1" + }, + favicon: nil + } + }, avatar: "http://localhost:4001/images/avi.png", avatar_static: "http://localhost:4001/images/avi.png", header: "http://localhost:4001/images/banner.png", @@ -598,4 +610,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do |> assert() end end + + test "returns nil in the instance field when no instance is held locally" do + user = insert(:user) + view = AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + assert view[:akkoma][:instance] == nil + end end diff --git a/test/support/factory.ex b/test/support/factory.ex index 54d385bc4..bd9d7fe42 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -36,6 +36,15 @@ defmodule Pleroma.Factory do } end + def instance_factory(attrs \\ %{}) do + %Pleroma.Instances.Instance{ + host: attrs[:domain] || "example.com", + nodeinfo: %{version: "2.0", openRegistrations: true}, + unreachable_since: nil + } + |> Map.merge(attrs) + end + def user_factory(attrs \\ %{}) do pem = Enum.random(@rsa_keys) @@ -522,13 +531,6 @@ defmodule Pleroma.Factory do } end - def instance_factory do - %Pleroma.Instances.Instance{ - host: "domain.com", - unreachable_since: nil - } - end - def oauth_token_factory(attrs \\ %{}) do scopes = Map.get(attrs, :scopes, ["read"]) oauth_app = Map.get_lazy(attrs, :app, fn -> insert(:oauth_app, scopes: scopes) end) -- 2.34.1 From 139dc54a95d62e6a3ddccac5e715ae93a3f30925 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 4 Nov 2022 05:09:49 +0000 Subject: [PATCH 07/13] add favicon tests --- .../web/mastodon_api/views/account_view_test.exs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/account_view_test.exs b/test/pleroma/web/mastodon_api/views/account_view_test.exs index 19224f126..541525520 100644 --- a/test/pleroma/web/mastodon_api/views/account_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/account_view_test.exs @@ -112,7 +112,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do describe "favicon" do setup do - [user: insert(:user)] + [user: insert(:user), instance: insert(:instance, %{host: "localhost", favicon: "https://example.com/favicon.ico"})] end test "is parsed when :instance_favicons is enabled", %{user: user} do @@ -121,12 +121,13 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do assert %{ pleroma: %{ favicon: - "https://shitposter.club/plugins/Qvitter/img/gnusocial-favicons/favicon-16x16.png" + "https://example.com/favicon.ico" } } = AccountView.render("show.json", %{user: user, skip_visibility_check: true}) end - test "is nil when :instances_favicons is disabled", %{user: user} do + test "is nil when we have no instance", %{user: user} do + user = %{user | ap_id: "https://wowee.example.com/users/2"} assert %{pleroma: %{favicon: nil}} = AccountView.render("show.json", %{user: user, skip_visibility_check: true}) end @@ -188,6 +189,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do }, fqn: "shp@shitposter.club", last_status_at: nil, + akkoma: %{instance: nil}, pleroma: %{ ap_id: user.ap_id, also_known_as: [], @@ -590,6 +592,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do emoji: %{"joker_smile" => "https://evil.website/society.png"} ) + insert(:instance, %{host: "localhost", favicon: "https://evil.website/favicon.png"}) + with media_preview_enabled <- [false, true] do clear_config([:media_preview_proxy, :enabled], media_preview_enabled) @@ -598,6 +602,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do {key, url} when key in [:avatar, :avatar_static, :header, :header_static] -> String.starts_with?(url, Pleroma.Web.Endpoint.url()) + {:akkoma, %{instance: %{favicon: favicon_url}}} -> + String.starts_with?(favicon_url, Pleroma.Web.Endpoint.url()) + {:emojis, emojis} -> Enum.all?(emojis, fn %{url: url, static_url: static_url} -> String.starts_with?(url, Pleroma.Web.Endpoint.url()) && -- 2.34.1 From 167253581ee87e62763079bab4d42ae23c3d8ef6 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 4 Nov 2022 18:27:48 +0000 Subject: [PATCH 08/13] Fix tests and description --- config/config.exs | 3 +- config/description.exs | 13 + config/test.exs | 2 + lib/pleroma/instances/instance.ex | 30 ++- test/pleroma/instances/instance_test.exs | 235 +++++++++++++++--- .../mastodon_api/views/account_view_test.exs | 13 +- 6 files changed, 246 insertions(+), 50 deletions(-) diff --git a/config/config.exs b/config/config.exs index edeed7da2..644155aeb 100644 --- a/config/config.exs +++ b/config/config.exs @@ -807,7 +807,8 @@ config :ex_aws, http_client: Pleroma.HTTP.ExAws config :web_push_encryption, http_client: Pleroma.HTTP.WebPush -config :pleroma, :instances_favicons, enabled: false +config :pleroma, :instances_favicons, enabled: true +config :pleroma, :instances_nodeinfo, enabled: true config :floki, :html_parser, Floki.HTMLParser.FastHtml diff --git a/config/description.exs b/config/description.exs index 1ff0a582b..1e81c3905 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3047,6 +3047,19 @@ config :pleroma, :config_description, [ } ] }, + %{ + group: :pleroma, + key: :instances_nodeinfo, + type: :group, + description: "Control favicons for instances", + children: [ + %{ + key: :enabled, + type: :boolean, + description: "Allow/disallow getting instance nodeinfo" + } + ] + }, %{ group: :ex_aws, key: :s3, diff --git a/config/test.exs b/config/test.exs index a5edb1149..3056dbd03 100644 --- a/config/test.exs +++ b/config/test.exs @@ -139,6 +139,8 @@ config :pleroma, Pleroma.Search.Meilisearch, url: "http://127.0.0.1:7700/", priv # Reduce recompilation time # https://dashbit.co/blog/speeding-up-re-compilation-of-elixir-projects config :phoenix, :plug_init_mode, :runtime +config :pleroma, :instances_favicons, enabled: false +config :pleroma, :instances_nodeinfo, enabled: false if File.exists?("./config/test.secret.exs") do import_config "test.secret.exs" diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 0627aec64..65062135f 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -154,6 +154,14 @@ defmodule Pleroma.Instances.Instance do Logger.info("Checking metadata for #{host}") existing_record = Repo.get_by(Instance, %{host: host}) + if reachable?(host) do + do_update_metadata(uri, existing_record) + else + {:discard, :unreachable} + end + end + + defp do_update_metadata(%URI{host: host} = uri, existing_record) do if existing_record do if needs_update(existing_record) do Logger.info("Updating metadata for #{host}") @@ -205,7 +213,8 @@ defmodule Pleroma.Instances.Instance do end defp scrape_nodeinfo(%URI{} = instance_uri) do - with {_, true} <- {:reachable, reachable?(instance_uri.host)}, + with true <- Pleroma.Config.get([:instances_nodeinfo, :enabled]), + {_, true} <- {:reachable, reachable?(instance_uri.host)}, {:ok, %Tesla.Env{status: 200, body: body}} <- Tesla.get( "https://#{instance_uri.host}/.well-known/nodeinfo", @@ -218,12 +227,20 @@ defmodule Pleroma.Instances.Instance do Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0"))}, {:ok, %Tesla.Env{body: data}} <- Pleroma.HTTP.get(href, [{"accept", "application/json"}], []), + {:length, true} <- {:length, String.length(data) < 50_000}, {:ok, nodeinfo} <- Jason.decode(data) do nodeinfo else {:reachable, false} -> Logger.debug( - "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") ignored unreachable host" + "Instance.scrape_nodeinfo(\"#{to_string(instance_uri)}\") ignored unreachable host" + ) + + nil + + {:length, false} -> + Logger.debug( + "Instance.scrape_nodeinfo(\"#{to_string(instance_uri)}\") ignored too long body" ) nil @@ -234,13 +251,15 @@ defmodule Pleroma.Instances.Instance do end defp scrape_favicon(%URI{} = instance_uri) do - with {_, true} <- {:reachable, reachable?(instance_uri.host)}, + with true <- Pleroma.Config.get([:instances_favicons, :enabled]), + {_, true} <- {:reachable, reachable?(instance_uri.host)}, {:ok, %Tesla.Env{body: html}} <- Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], []), {_, [favicon_rel | _]} when is_binary(favicon_rel) <- {:parse, html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")}, {_, favicon} when is_binary(favicon) <- - {:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do + {:merge, URI.merge(instance_uri, favicon_rel) |> to_string()}, + {:length, true} <- {:length, String.length(favicon) < 255} do favicon else {:reachable, false} -> @@ -282,10 +301,9 @@ defmodule Pleroma.Instances.Instance do def get_cached_by_url(url_or_host) do url = host(url_or_host) - @cachex.fetch!(:instances_cache, "instances:#{url}", fn _ -> with %Instance{} = instance <- get_by_url(url) do - {:ok, instance} + {:commit, {:ok, instance}} else _ -> {:ignore, nil} end diff --git a/test/pleroma/instances/instance_test.exs b/test/pleroma/instances/instance_test.exs index e49922724..a2ca9ca00 100644 --- a/test/pleroma/instances/instance_test.exs +++ b/test/pleroma/instances/instance_test.exs @@ -9,12 +9,16 @@ defmodule Pleroma.Instances.InstanceTest do alias Pleroma.Tests.ObanHelpers alias Pleroma.Web.CommonAPI - use Pleroma.DataCase + use Pleroma.DataCase, async: true import ExUnit.CaptureLog import Pleroma.Factory - setup_all do: clear_config([:instance, :federation_reachability_timeout_days], 1) + setup_all do + clear_config([:instance, :federation_reachability_timeout_days], 1) + clear_config([:instances_nodeinfo, :enabled], true) + clear_config([:instances_favicons, :enabled], true) + end describe "set_reachable/1" do test "clears `unreachable_since` of existing matching Instance record having non-nil `unreachable_since`" do @@ -102,62 +106,217 @@ defmodule Pleroma.Instances.InstanceTest do end end - describe "get_or_update_favicon/1" do - test "Scrapes favicon URLs" do - Tesla.Mock.mock(fn %{url: "https://favicon.example.org/"} -> - %Tesla.Env{ - status: 200, - body: ~s[] - } + describe "update_metadata/1" do + test "Scrapes favicon URLs and nodeinfo" do + Tesla.Mock.mock(fn + %{url: "https://favicon.example.org/"} -> + %Tesla.Env{ + status: 200, + body: ~s[] + } + + %{url: "https://favicon.example.org/.well-known/nodeinfo"} -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + links: [ + %{ + rel: "http://nodeinfo.diaspora.software/ns/schema/2.0", + href: "https://favicon.example.org/nodeinfo/2.0" + } + ] + }) + } + + %{url: "https://favicon.example.org/nodeinfo/2.0"} -> + %Tesla.Env{ + status: 200, + body: Jason.encode!(%{version: "2.0", software: %{name: "Akkoma"}}) + } end) - assert "https://favicon.example.org/favicon.png" == - Instance.get_or_update_favicon(URI.parse("https://favicon.example.org/")) + assert {:ok, true} == + Instance.update_metadata(URI.parse("https://favicon.example.org/")) + + {:ok, instance} = Instance.get_cached_by_url("https://favicon.example.org/") + assert instance.favicon == "https://favicon.example.org/favicon.png" + assert instance.nodeinfo == %{"version" => "2.0", "software" => %{"name" => "Akkoma"}} end - test "Returns nil on too long favicon URLs" do + test "Does not retain favicons that are too long" do long_favicon_url = "https://Lorem.ipsum.dolor.sit.amet/consecteturadipiscingelit/Praesentpharetrapurusutaliquamtempus/Mauriseulaoreetarcu/atfacilisisorci/Nullamporttitor/nequesedfeugiatmollis/dolormagnaefficiturlorem/nonpretiumsapienorcieurisus/Nullamveleratsem/Maecenassedaccumsanexnam/favicon.png" - Tesla.Mock.mock(fn %{url: "https://long-favicon.example.org/"} -> - %Tesla.Env{ - status: 200, - body: - ~s[] - } + Tesla.Mock.mock(fn + %{url: "https://long-favicon.example.org/"} -> + %Tesla.Env{ + status: 200, + body: + ~s[] + } + + %{url: "https://long-favicon.example.org/.well-known/nodeinfo"} -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + links: [ + %{ + rel: "http://nodeinfo.diaspora.software/ns/schema/2.0", + href: "https://long-favicon.example.org/nodeinfo/2.0" + } + ] + }) + } + + %{url: "https://long-favicon.example.org/nodeinfo/2.0"} -> + %Tesla.Env{ + status: 200, + body: Jason.encode!(%{version: "2.0", software: %{name: "Akkoma"}}) + } end) - assert capture_log(fn -> - assert nil == - Instance.get_or_update_favicon( - URI.parse("https://long-favicon.example.org/") - ) - end) =~ - "Instance.get_or_update_favicon(\"long-favicon.example.org\") error: %Postgrex.Error{" + assert {:ok, true} == + Instance.update_metadata(URI.parse("https://long-favicon.example.org/")) + + {:ok, instance} = Instance.get_cached_by_url("https://long-favicon.example.org/") + assert instance.favicon == nil end test "Handles not getting a favicon URL properly" do - Tesla.Mock.mock(fn %{url: "https://no-favicon.example.org/"} -> - %Tesla.Env{ - status: 200, - body: ~s[

I wil look down and whisper "GNO.."

] - } + Tesla.Mock.mock(fn + %{url: "https://no-favicon.example.org/"} -> + %Tesla.Env{ + status: 200, + body: ~s[

I wil look down and whisper "GNO.."

] + } + + %{url: "https://no-favicon.example.org/.well-known/nodeinfo"} -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + links: [ + %{ + rel: "http://nodeinfo.diaspora.software/ns/schema/2.0", + href: "https://no-favicon.example.org/nodeinfo/2.0" + } + ] + }) + } + + %{url: "https://no-favicon.example.org/nodeinfo/2.0"} -> + %Tesla.Env{ + status: 200, + body: Jason.encode!(%{version: "2.0", software: %{name: "Akkoma"}}) + } end) refute capture_log(fn -> - assert nil == - Instance.get_or_update_favicon( - URI.parse("https://no-favicon.example.org/") - ) - end) =~ "Instance.scrape_favicon(\"https://no-favicon.example.org/\") error: " + assert {:ok, true} = + Instance.update_metadata(URI.parse("https://no-favicon.example.org/")) + end) =~ "Instance.update_metadata(\"https://no-favicon.example.org/\") error: " end - test "Doesn't scrapes unreachable instances" do + test "Doesn't scrape unreachable instances" do instance = insert(:instance, unreachable_since: Instances.reachability_datetime_threshold()) url = "https://" <> instance.host - assert capture_log(fn -> assert nil == Instance.get_or_update_favicon(URI.parse(url)) end) =~ - "Instance.scrape_favicon(\"#{url}\") ignored unreachable host" + assert {:discard, :unreachable} == Instance.update_metadata(URI.parse(url)) + end + + test "doesn't continue scraping nodeinfo if we can't find a link" do + Tesla.Mock.mock(fn + %{url: "https://bad-nodeinfo.example.org/"} -> + %Tesla.Env{ + status: 200, + body: ~s[

I wil look down and whisper "GNO.."

] + } + + %{url: "https://bad-nodeinfo.example.org/.well-known/nodeinfo"} -> + %Tesla.Env{ + status: 200, + body: + "oepsie woepsie de nodeinfo is kapotie uwu" + } + end) + + assert {:ok, true} == + Instance.update_metadata(URI.parse("https://bad-nodeinfo.example.org/")) + {:ok, instance} = Instance.get_cached_by_url("https://bad-nodeinfo.example.org/") + assert instance.nodeinfo == nil + end + + test "doesn't store bad json in the nodeinfo" do + Tesla.Mock.mock(fn + %{url: "https://bad-nodeinfo.example.org/"} -> + %Tesla.Env{ + status: 200, + body: ~s[

I wil look down and whisper "GNO.."

] + } + + %{url: "https://bad-nodeinfo.example.org/.well-known/nodeinfo"} -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + links: [ + %{ + rel: "http://nodeinfo.diaspora.software/ns/schema/2.0", + href: "https://bad-nodeinfo.example.org/nodeinfo/2.0" + } + ] + }) + } + + %{url: "https://bad-nodeinfo.example.org/nodeinfo/2.0"} -> + %Tesla.Env{ + status: 200, + body: "oepsie woepsie de json might be bad uwu" + } + end) + + assert {:ok, true} == + Instance.update_metadata(URI.parse("https://bad-nodeinfo.example.org/")) + {:ok, instance} = Instance.get_cached_by_url("https://bad-nodeinfo.example.org/") + assert instance.nodeinfo == nil + end + + test "doesn't store incredibly long json nodeinfo" do + too_long = String.duplicate("a", 50_000) + Tesla.Mock.mock(fn + %{url: "https://bad-nodeinfo.example.org/"} -> + %Tesla.Env{ + status: 200, + body: ~s[

I wil look down and whisper "GNO.."

] + } + + %{url: "https://bad-nodeinfo.example.org/.well-known/nodeinfo"} -> + %Tesla.Env{ + status: 200, + body: + Jason.encode!(%{ + links: [ + %{ + rel: "http://nodeinfo.diaspora.software/ns/schema/2.0", + href: "https://bad-nodeinfo.example.org/nodeinfo/2.0" + } + ] + }) + } + + %{url: "https://bad-nodeinfo.example.org/nodeinfo/2.0"} -> + %Tesla.Env{ + status: 200, + body: Jason.encode!(%{version: "2.0", software: %{name: too_long}}) + } + end) + + assert {:ok, true} == + Instance.update_metadata(URI.parse("https://bad-nodeinfo.example.org/")) + {:ok, instance} = Instance.get_cached_by_url("https://bad-nodeinfo.example.org/") + assert instance.nodeinfo == nil end end diff --git a/test/pleroma/web/mastodon_api/views/account_view_test.exs b/test/pleroma/web/mastodon_api/views/account_view_test.exs index 541525520..4badfb63b 100644 --- a/test/pleroma/web/mastodon_api/views/account_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/account_view_test.exs @@ -112,22 +112,25 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do describe "favicon" do setup do - [user: insert(:user), instance: insert(:instance, %{host: "localhost", favicon: "https://example.com/favicon.ico"})] + [ + user: insert(:user), + instance: + insert(:instance, %{host: "localhost", favicon: "https://example.com/favicon.ico"}) + ] end - test "is parsed when :instance_favicons is enabled", %{user: user} do + test "is parsed when :instance_favicons is enabled", %{user: user, instance: instance} do clear_config([:instances_favicons, :enabled], true) - assert %{ pleroma: %{ - favicon: - "https://example.com/favicon.ico" + favicon: "https://example.com/favicon.ico" } } = AccountView.render("show.json", %{user: user, skip_visibility_check: true}) end test "is nil when we have no instance", %{user: user} do user = %{user | ap_id: "https://wowee.example.com/users/2"} + assert %{pleroma: %{favicon: nil}} = AccountView.render("show.json", %{user: user, skip_visibility_check: true}) end -- 2.34.1 From 053771900cf2e6cad1a357c01675f81dd6517331 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 4 Nov 2022 18:44:30 +0000 Subject: [PATCH 09/13] mix format --- config/description.exs | 4 ++-- lib/pleroma/instances/instance.ex | 1 + test/pleroma/instances/instance_test.exs | 7 +++++-- test/pleroma/web/mastodon_api/views/account_view_test.exs | 1 + 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/config/description.exs b/config/description.exs index 1e81c3905..4843c0aae 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3047,13 +3047,13 @@ config :pleroma, :config_description, [ } ] }, - %{ + %{ group: :pleroma, key: :instances_nodeinfo, type: :group, description: "Control favicons for instances", children: [ - %{ + %{ key: :enabled, type: :boolean, description: "Allow/disallow getting instance nodeinfo" diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 65062135f..a5989392e 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -301,6 +301,7 @@ defmodule Pleroma.Instances.Instance do def get_cached_by_url(url_or_host) do url = host(url_or_host) + @cachex.fetch!(:instances_cache, "instances:#{url}", fn _ -> with %Instance{} = instance <- get_by_url(url) do {:commit, {:ok, instance}} diff --git a/test/pleroma/instances/instance_test.exs b/test/pleroma/instances/instance_test.exs index a2ca9ca00..adc847da5 100644 --- a/test/pleroma/instances/instance_test.exs +++ b/test/pleroma/instances/instance_test.exs @@ -237,13 +237,13 @@ defmodule Pleroma.Instances.InstanceTest do %{url: "https://bad-nodeinfo.example.org/.well-known/nodeinfo"} -> %Tesla.Env{ status: 200, - body: - "oepsie woepsie de nodeinfo is kapotie uwu" + body: "oepsie woepsie de nodeinfo is kapotie uwu" } end) assert {:ok, true} == Instance.update_metadata(URI.parse("https://bad-nodeinfo.example.org/")) + {:ok, instance} = Instance.get_cached_by_url("https://bad-nodeinfo.example.org/") assert instance.nodeinfo == nil end @@ -279,12 +279,14 @@ defmodule Pleroma.Instances.InstanceTest do assert {:ok, true} == Instance.update_metadata(URI.parse("https://bad-nodeinfo.example.org/")) + {:ok, instance} = Instance.get_cached_by_url("https://bad-nodeinfo.example.org/") assert instance.nodeinfo == nil end test "doesn't store incredibly long json nodeinfo" do too_long = String.duplicate("a", 50_000) + Tesla.Mock.mock(fn %{url: "https://bad-nodeinfo.example.org/"} -> %Tesla.Env{ @@ -315,6 +317,7 @@ defmodule Pleroma.Instances.InstanceTest do assert {:ok, true} == Instance.update_metadata(URI.parse("https://bad-nodeinfo.example.org/")) + {:ok, instance} = Instance.get_cached_by_url("https://bad-nodeinfo.example.org/") assert instance.nodeinfo == nil end diff --git a/test/pleroma/web/mastodon_api/views/account_view_test.exs b/test/pleroma/web/mastodon_api/views/account_view_test.exs index 4badfb63b..9ca897662 100644 --- a/test/pleroma/web/mastodon_api/views/account_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/account_view_test.exs @@ -121,6 +121,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do test "is parsed when :instance_favicons is enabled", %{user: user, instance: instance} do clear_config([:instances_favicons, :enabled], true) + assert %{ pleroma: %{ favicon: "https://example.com/favicon.ico" -- 2.34.1 From fa5e6c98824734314e4e02d755b198c3ce5a33ab Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 4 Nov 2022 19:21:02 +0000 Subject: [PATCH 10/13] Revert "Merge pull request 'Push.Impl: support edits' (#244) from norm/akkoma:push-support-edits into develop" This reverts commit 9038da01ccfa8cb474fa0f44037b5eb04996b77f, reversing changes made to e44e147b541ab0d3c702c56c7cb4a04e8107e6d9. --- lib/pleroma/web/push/impl.ex | 12 +----------- test/pleroma/web/push/impl_test.exs | 15 --------------- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index e103cafc2..c30a39e94 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Web.Push.Impl do require Logger import Ecto.Query - @types ["Create", "Follow", "Announce", "Like", "Move", "EmojiReact", "Update"] + @types ["Create", "Follow", "Announce", "Like", "Move", "EmojiReact"] @doc "Performs sending notifications for user subscriptions" @spec perform(Notification.t()) :: list(any) | :error | {:error, :unknown_type} @@ -167,15 +167,6 @@ defmodule Pleroma.Web.Push.Impl do end end - def format_body( - %{activity: %{data: %{"type" => "Update"}}}, - actor, - _object, - _mastodon_type - ) do - "@#{actor.nickname} edited a status" - end - def format_title(activity, mastodon_type \\ nil) def format_title(%{activity: %{data: %{"directMessage" => true}}}, _mastodon_type) do @@ -189,7 +180,6 @@ defmodule Pleroma.Web.Push.Impl do "follow_request" -> "New Follow Request" "reblog" -> "New Repeat" "favourite" -> "New Favorite" - "update" -> "New Update" "pleroma:emoji_reaction" -> "New Reaction" type -> "New #{String.capitalize(type || "event")}" end diff --git a/test/pleroma/web/push/impl_test.exs b/test/pleroma/web/push/impl_test.exs index 326872ccd..9100433ae 100644 --- a/test/pleroma/web/push/impl_test.exs +++ b/test/pleroma/web/push/impl_test.exs @@ -200,21 +200,6 @@ defmodule Pleroma.Web.Push.ImplTest do "New Reaction" end - test "renders title and body for update activity" do - user = insert(:user) - - {:ok, activity} = CommonAPI.post(user, %{status: "lorem ipsum"}) - - {:ok, activity} = CommonAPI.update(user, activity, %{status: "edited status"}) - object = Object.normalize(activity, fetch: false) - - assert Impl.format_body(%{activity: activity, type: "update"}, user, object) == - "@#{user.nickname} edited a status" - - assert Impl.format_title(%{activity: activity, type: "update"}) == - "New Update" - end - test "renders title for create activity with direct visibility" do user = insert(:user, nickname: "Bob") -- 2.34.1 From cf0394738c52ec23dd0cad4259be6dff6b2fc5cf Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 4 Nov 2022 19:22:31 +0000 Subject: [PATCH 11/13] Revert "Revert "Merge pull request 'Push.Impl: support edits' (#244) from norm/akkoma:push-support-edits into develop"" This reverts commit fa5e6c98824734314e4e02d755b198c3ce5a33ab. --- lib/pleroma/web/push/impl.ex | 12 +++++++++++- test/pleroma/web/push/impl_test.exs | 15 +++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index c30a39e94..e103cafc2 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Web.Push.Impl do require Logger import Ecto.Query - @types ["Create", "Follow", "Announce", "Like", "Move", "EmojiReact"] + @types ["Create", "Follow", "Announce", "Like", "Move", "EmojiReact", "Update"] @doc "Performs sending notifications for user subscriptions" @spec perform(Notification.t()) :: list(any) | :error | {:error, :unknown_type} @@ -167,6 +167,15 @@ defmodule Pleroma.Web.Push.Impl do end end + def format_body( + %{activity: %{data: %{"type" => "Update"}}}, + actor, + _object, + _mastodon_type + ) do + "@#{actor.nickname} edited a status" + end + def format_title(activity, mastodon_type \\ nil) def format_title(%{activity: %{data: %{"directMessage" => true}}}, _mastodon_type) do @@ -180,6 +189,7 @@ defmodule Pleroma.Web.Push.Impl do "follow_request" -> "New Follow Request" "reblog" -> "New Repeat" "favourite" -> "New Favorite" + "update" -> "New Update" "pleroma:emoji_reaction" -> "New Reaction" type -> "New #{String.capitalize(type || "event")}" end diff --git a/test/pleroma/web/push/impl_test.exs b/test/pleroma/web/push/impl_test.exs index 9100433ae..326872ccd 100644 --- a/test/pleroma/web/push/impl_test.exs +++ b/test/pleroma/web/push/impl_test.exs @@ -200,6 +200,21 @@ defmodule Pleroma.Web.Push.ImplTest do "New Reaction" end + test "renders title and body for update activity" do + user = insert(:user) + + {:ok, activity} = CommonAPI.post(user, %{status: "lorem ipsum"}) + + {:ok, activity} = CommonAPI.update(user, activity, %{status: "edited status"}) + object = Object.normalize(activity, fetch: false) + + assert Impl.format_body(%{activity: activity, type: "update"}, user, object) == + "@#{user.nickname} edited a status" + + assert Impl.format_title(%{activity: activity, type: "update"}) == + "New Update" + end + test "renders title for create activity with direct visibility" do user = insert(:user, nickname: "Bob") -- 2.34.1 From 57b112b6b25b890218803cae5c507ebac36ab17c Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Fri, 4 Nov 2022 19:32:20 +0000 Subject: [PATCH 12/13] make streamer tests somewhat less flimsy --- test/pleroma/web/mastodon_api/views/account_view_test.exs | 2 +- test/pleroma/web/streamer_test.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pleroma/web/mastodon_api/views/account_view_test.exs b/test/pleroma/web/mastodon_api/views/account_view_test.exs index 9ca897662..552e37a9c 100644 --- a/test/pleroma/web/mastodon_api/views/account_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/account_view_test.exs @@ -119,7 +119,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do ] end - test "is parsed when :instance_favicons is enabled", %{user: user, instance: instance} do + test "is parsed when :instance_favicons is enabled", %{user: user} do clear_config([:instances_favicons, :enabled], true) assert %{ diff --git a/test/pleroma/web/streamer_test.exs b/test/pleroma/web/streamer_test.exs index 8e2ab5016..a9db5a015 100644 --- a/test/pleroma/web/streamer_test.exs +++ b/test/pleroma/web/streamer_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.StreamerTest do - use Pleroma.DataCase + use Pleroma.DataCase, async: false import Pleroma.Factory -- 2.34.1 From 8509742b77e64a282414f9e9a0b907d11ce27299 Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sun, 6 Nov 2022 22:47:42 +0000 Subject: [PATCH 13/13] Add local nodeinfo check --- lib/pleroma/instances/instance.ex | 26 +++++-- .../mastodon_api/views/account_view_test.exs | 71 +++++++++++++++++-- 2 files changed, 83 insertions(+), 14 deletions(-) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index a5989392e..fcf3181bf 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -150,6 +150,14 @@ defmodule Pleroma.Instances.Instance do NaiveDateTime.diff(now, metadata_updated_at) > 86_400 end + def local do + %Instance{ + host: Pleroma.Web.Endpoint.host(), + favicon: Pleroma.Web.Endpoint.url() <> "/favicon.png", + nodeinfo: Pleroma.Web.Nodeinfo.NodeinfoController.raw_nodeinfo() + } + end + def update_metadata(%URI{host: host} = uri) do Logger.info("Checking metadata for #{host}") existing_record = Repo.get_by(Instance, %{host: host}) @@ -302,12 +310,16 @@ defmodule Pleroma.Instances.Instance do def get_cached_by_url(url_or_host) do url = host(url_or_host) - @cachex.fetch!(:instances_cache, "instances:#{url}", fn _ -> - with %Instance{} = instance <- get_by_url(url) do - {:commit, {:ok, instance}} - else - _ -> {:ignore, nil} - end - end) + if url == Pleroma.Web.Endpoint.host() do + {:ok, local()} + else + @cachex.fetch!(:instances_cache, "instances:#{url}", fn _ -> + with %Instance{} = instance <- get_by_url(url) do + {:commit, {:ok, instance}} + else + _ -> {:ignore, nil} + end + end) + end end end diff --git a/test/pleroma/web/mastodon_api/views/account_view_test.exs b/test/pleroma/web/mastodon_api/views/account_view_test.exs index 552e37a9c..d1903af80 100644 --- a/test/pleroma/web/mastodon_api/views/account_view_test.exs +++ b/test/pleroma/web/mastodon_api/views/account_view_test.exs @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.AccountViewTest do - use Pleroma.DataCase + use Pleroma.DataCase, async: false alias Pleroma.User alias Pleroma.UserRelationship @@ -12,6 +12,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do import Pleroma.Factory import Tesla.Mock + import Mock setup do mock(fn env -> apply(HttpRequestMock, :request, [env]) end) @@ -110,12 +111,56 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do assert expected == AccountView.render("show.json", %{user: user, skip_visibility_check: true}) end + describe "nodeinfo" do + setup do + [ + user: insert(:user, ap_id: "https://somewhere.example.com/users/chikichikibanban"), + instance: + insert(:instance, %{ + host: "somewhere.example.com", + favicon: "https://example.com/favicon.ico" + }) + ] + end + + test "is embedded in the account view", %{user: user} do + assert %{ + akkoma: %{ + instance: %{ + name: "somewhere.example.com", + nodeinfo: %{ + "version" => "2.0" + }, + favicon: "https://example.com/favicon.ico" + } + } + } = AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + end + + test "uses local nodeinfo for local users" do + user = insert(:user) + + assert %{ + akkoma: %{ + instance: %{ + name: "localhost", + nodeinfo: %{ + software: %{ + name: "akkoma" + } + } + } + } + } = AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + end + end + describe "favicon" do setup do [ - user: insert(:user), + user: insert(:user, ap_id: "https://example.com/users/chikichikibanban"), instance: - insert(:instance, %{host: "localhost", favicon: "https://example.com/favicon.ico"}) + insert(:instance, %{host: "example.com", favicon: "https://example.com/favicon.ico"}) ] end @@ -193,12 +238,18 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do }, fqn: "shp@shitposter.club", last_status_at: nil, - akkoma: %{instance: nil}, + akkoma: %{ + instance: %{ + name: "localhost", + favicon: "http://localhost:4001/favicon.png", + nodeinfo: %{version: "2.0"} + } + }, pleroma: %{ ap_id: user.ap_id, also_known_as: [], background_image: nil, - favicon: nil, + favicon: "http://localhost:4001/favicon.png", is_confirmed: true, tags: [], is_admin: false, @@ -214,7 +265,13 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do } } - assert expected == AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + with_mock( + Pleroma.Web.Nodeinfo.NodeinfoController, + raw_nodeinfo: fn -> %{version: "2.0"} end + ) do + assert expected == + AccountView.render("show.json", %{user: user, skip_visibility_check: true}) + end end test "Represent a Funkwhale channel" do @@ -623,7 +680,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do end test "returns nil in the instance field when no instance is held locally" do - user = insert(:user) + user = insert(:user, ap_id: "https://example.com/users/1") view = AccountView.render("show.json", %{user: user, skip_visibility_check: true}) assert view[:akkoma][:instance] == nil end -- 2.34.1