From 881dbea92ecdd9d024ceb90578c682f3683c963e Mon Sep 17 00:00:00 2001 From: FloatingGhost Date: Sat, 29 Oct 2022 21:51:45 +0100 Subject: [PATCH] Add tasker to get nodeinfo --- lib/pleroma/instances/instance.ex | 19 +++++++++++++------ lib/pleroma/web/activity_pub/side_effects.ex | 2 ++ .../workers/nodeinfo_fetcher_worker.ex | 18 ++++++++---------- lib/pleroma/workers/worker_helper.ex | 1 + .../20221020135943_add_nodeinfo.exs | 2 ++ 5 files changed, 26 insertions(+), 16 deletions(-) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 7fdf35bcc..7b3ba79c9 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -32,7 +32,7 @@ defmodule Pleroma.Instances.Instance do def changeset(struct, params \\ %{}) do struct - |> cast(params, [:host, :unreachable_since, :favicon, :favicon_updated_at]) + |> cast(params, [:host, :unreachable_since, :favicon, :nodeinfo, :metadata_updated_at]) |> validate_required([:host]) |> unique_constraint(:host) end @@ -141,34 +141,41 @@ defmodule Pleroma.Instances.Instance do def needs_update(nil), do: true + def needs_update(%Instance{metadata_updated_at: nil}), do: true + def needs_update(%Instance{metadata_updated_at: metadata_updated_at}) do now = NaiveDateTime.utc_now() NaiveDateTime.diff(now, metadata_updated_at) > 86_400 end def update_metadata(%URI{host: host} = uri) do + Logger.info("Checking metadata for #{host}") existing_record = Repo.get_by(Instance, %{host: host}) if existing_record do if needs_update(existing_record) do + Logger.info("Updating metadata for #{host}") favicon = scrape_favicon(uri) nodeinfo = scrape_nodeinfo(uri) - %Instance{} - |> changeset(%{host: host, favicon: favicon, nodeinfo: nodeinfo}) + existing_record + |> changeset(%{host: host, favicon: favicon, nodeinfo: nodeinfo, metadata_updated_at: NaiveDateTime.utc_now()}) |> Repo.update() + else + {:discard, "Does not require update"} end else favicon = scrape_favicon(uri) nodeinfo = scrape_nodeinfo(uri) + Logger.info("Creating metadata for #{host}") %Instance{} - |> changeset(%{host: host, favicon: favicon, nodeinfo: nodeinfo}) + |> changeset(%{host: host, favicon: favicon, nodeinfo: nodeinfo, metadata_updated_at: NaiveDateTime.utc_now()}) |> Repo.insert() end end - def get_favicon(%URI{host: host} = instance_uri) do + def get_favicon(%URI{host: host}) do existing_record = Repo.get_by(Instance, %{host: host}) if existing_record do @@ -191,7 +198,7 @@ defmodule Pleroma.Instances.Instance do {:ok, Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0"))}, {:ok, %Tesla.Env{body: data}} <- - Pleroma.HTTP.get(ref, [{"accept", "application/json"}], []), + Pleroma.HTTP.get(href, [{"accept", "application/json"}], []), {:ok, nodeinfo} <- Jason.decode(data) do nodeinfo else diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index c3258c75b..a3830e408 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -209,6 +209,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do reply_depth = (meta[:depth] || 0) + 1 + IO.puts("QUEUE!") + Pleroma.Workers.NodeInfoFetcherWorker.enqueue("process", %{"domain" => activity.data["actor"]}) # FIXME: Force inReplyTo to replies if Pleroma.Web.Federator.allowed_thread_distance?(reply_depth) and object.data["replies"] != nil do diff --git a/lib/pleroma/workers/nodeinfo_fetcher_worker.ex b/lib/pleroma/workers/nodeinfo_fetcher_worker.ex index 019a3d5c2..5a58de268 100644 --- a/lib/pleroma/workers/nodeinfo_fetcher_worker.ex +++ b/lib/pleroma/workers/nodeinfo_fetcher_worker.ex @@ -1,19 +1,17 @@ defmodule Pleroma.Workers.NodeInfoFetcherWorker do - use Oban.Worker, queue: :backup, max_attempts: 1 + use Pleroma.Workers.WorkerHelper, queue: "nodeinfo_fetcher" alias Oban.Job - alias Pleroma.Instance - - def process(domain) do - %{"op" => "process", "domain" => domain} - |> new() - |> Oban.insert() - end + alias Pleroma.Instances.Instance + @impl Oban.Worker def perform(%Job{ args: %{"op" => "process", "domain" => domain} }) do - uri = URI.parse(domain) - Instance.get_or_update_favicon(uri) + uri = domain + |> URI.parse() + |> URI.merge("/") + + Instance.update_metadata(uri) end end diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex index 4befbeb3b..47afc9dd7 100644 --- a/lib/pleroma/workers/worker_helper.ex +++ b/lib/pleroma/workers/worker_helper.ex @@ -42,6 +42,7 @@ defmodule Pleroma.Workers.WorkerHelper do unquote(caller_module) |> apply(:new, [params, worker_args]) |> Oban.insert() + |> IO.inspect() end end end diff --git a/priv/repo/migrations/20221020135943_add_nodeinfo.exs b/priv/repo/migrations/20221020135943_add_nodeinfo.exs index 6eab3b884..17707f3f7 100644 --- a/priv/repo/migrations/20221020135943_add_nodeinfo.exs +++ b/priv/repo/migrations/20221020135943_add_nodeinfo.exs @@ -4,12 +4,14 @@ defmodule Pleroma.Repo.Migrations.AddNodeinfo do def up do alter table(:instances) do add_if_not_exists(:nodeinfo, :map, default: %{}) + add_if_not_exists(:metadata_updated_at, :naive_datetime) end end def down do alter table(:instances) do remove_if_exists(:nodeinfo, :map) + remove_if_exists(:metadata_updated_at, :naive_datetime) end end end