Merge pull request 'pool timeouts/rich media cherry-picks' (#796) from pool-timeouts into develop

Reviewed-on: #796

CHANGELOG.md

@@ -16,6 +16,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 ## Fixed
 - Meilisearch: order of results returned from our REST API now actually matches how Meilisearch ranks results
 
+## Changed
+- Refactored Rich Media to cache the content in the database. Fetching operations that could block status rendering have been eliminated.
+
 ## 2024.04.1 (Security)
 
 ## Fixed

config/config.exs

@@ -189,8 +189,10 @@
   receive_timeout: :timer.seconds(15),
   proxy_url: nil,
   user_agent: :default,
-  pool_size: 50,
-  adapter: []
+  pool_size: 10,
+  adapter: [],
+  # see: https://hexdocs.pm/finch/Finch.html#start_link/1
+  pool_max_idle_time: :timer.seconds(30)
 
 config :pleroma, :instance,
   name: "Akkoma",
@@ -437,8 +439,12 @@
     Pleroma.Web.RichMedia.Parsers.TwitterCard,
     Pleroma.Web.RichMedia.Parsers.OEmbed
   ],
-  failure_backoff: :timer.minutes(20),
-  ttl_setters: [Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl]
+  failure_backoff: 60_000,
+  ttl_setters: [
+    Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl,
+    Pleroma.Web.RichMedia.Parser.TTL.Opengraph
+  ],
+  max_body: 5_000_000
 
 config :pleroma, :media_proxy,
   enabled: false,
@@ -576,7 +582,9 @@
     mute_expire: 5,
     search_indexing: 10,
     nodeinfo_fetcher: 1,
-    database_prune: 1
+    database_prune: 1,
+    rich_media_backfill: 2,
+    rich_media_expiration: 2
   ],
   plugins: [
     Oban.Plugins.Pruner,
@@ -592,7 +600,8 @@
   retries: [
     federator_incoming: 5,
     federator_outgoing: 5,
-    search_indexing: 2
+    search_indexing: 2,
+    rich_media_backfill: 3
   ],
   timeout: [
     activity_expiration: :timer.seconds(5),
@@ -614,7 +623,8 @@
     mute_expire: :timer.seconds(5),
     search_indexing: :timer.seconds(5),
     nodeinfo_fetcher: :timer.seconds(10),
-    database_prune: :timer.minutes(10)
+    database_prune: :timer.minutes(10),
+    rich_media_backfill: :timer.seconds(30)
   ]
 
 config :pleroma, Pleroma.Formatter,

config/description.exs

@@ -2717,8 +2717,8 @@
       %{
         key: :pool_size,
         type: :integer,
-        description: "Number of concurrent outbound HTTP requests to allow. Default 50.",
-        suggestions: [50]
+        description: "Number of concurrent outbound HTTP requests to allow PER HOST. Default 10.",
+        suggestions: [10]
       },
       %{
         key: :adapter,
@@ -2741,6 +2741,13 @@
             ]
           }
         ]
+      },
+      %{
+        key: :pool_max_idle_time,
+        type: :integer,
+        description:
+          "Number of seconds to retain an HTTP pool; pool will remain if actively in use. Default 30 seconds (in ms).",
+        suggestions: [30_000]
       }
     ]
   },

config/test.exs

@@ -63,7 +63,8 @@
 config :pleroma, :rich_media,
   enabled: false,
   ignore_hosts: [],
-  ignore_tld: ["local", "localdomain", "lan"]
+  ignore_tld: ["local", "localdomain", "lan"],
+  max_body: 2_000_000
 
 config :pleroma, :instance,
   multi_factor_authentication: [
@@ -141,6 +142,8 @@
 config :pleroma, :instances_favicons, enabled: false
 config :pleroma, :instances_nodeinfo, enabled: false
 
+config :pleroma, Pleroma.Web.RichMedia.Backfill, provider: Pleroma.Web.RichMedia.Backfill
+
 if File.exists?("./config/test.secret.exs") do
   import_config "test.secret.exs"
 else

diagnostic-tools/binary-leak-checker.sh

@@ -0,0 +1,10 @@
+if [ "$#" -ne 2 ]; then
+	echo "Usage: binary-leak-checker.sh <nodename> <erlang cookie>"
+	exit 1
+fi
+
+echo "The command you want to run is:
+:recon.bin_leak(10)
+"
+
+iex --sname debug --remsh $1 --erl "-setcookie $2" 

lib/pleroma/activity/html.ex

@@ -28,7 +28,7 @@ defp get_cache_keys_for(activity_id) do
     end
   end
 
-  defp add_cache_key_for(activity_id, additional_key) do
+  def add_cache_key_for(activity_id, additional_key) do
     current = get_cache_keys_for(activity_id)
 
     unless additional_key in current do

lib/pleroma/application.ex

@@ -264,7 +264,9 @@ def limiters_setup do
   defp http_children do
     proxy_url = Config.get([:http, :proxy_url])
     proxy = Pleroma.HTTP.AdapterHelper.format_proxy(proxy_url)
-    pool_size = Config.get([:http, :pool_size])
+    pool_size = Config.get([:http, :pool_size], 10)
+    pool_timeout = Config.get([:http, :pool_timeout], 60_000)
+    connection_timeout = Config.get([:http, :conn_max_idle_time], 10_000)
 
     :public_key.cacerts_load()
 
@@ -274,6 +276,8 @@ defp http_children do
       |> Pleroma.HTTP.AdapterHelper.add_pool_size(pool_size)
       |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy_pool(proxy)
       |> Pleroma.HTTP.AdapterHelper.ensure_ipv6()
+      |> Pleroma.HTTP.AdapterHelper.add_default_conn_max_idle_time(connection_timeout)
+      |> Pleroma.HTTP.AdapterHelper.add_default_pool_max_idle_time(pool_timeout)
       |> Keyword.put(:name, MyFinch)
 
     [{Finch, config}]

lib/pleroma/html.ex

@@ -6,8 +6,6 @@ defmodule Pleroma.HTML do
   # Scrubbers are compiled on boot so they can be configured in OTP releases
   #  @on_load :compile_scrubbers
 
-  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
-
   def compile_scrubbers do
     dir = Path.join(:code.priv_dir(:pleroma), "scrubbers")
 
@@ -67,22 +65,9 @@ def ensure_scrubbed_html(
     end
   end
 
-  def extract_first_external_url_from_object(%{data: %{"content" => content}} = object)
+  @spec extract_first_external_url_from_object(Pleroma.Object.t()) :: String.t() | nil
+  def extract_first_external_url_from_object(%{data: %{"content" => content}})
       when is_binary(content) do
-    unless object.data["fake"] do
-      key = "URL|#{object.id}"
-
-      @cachex.fetch!(:scrubber_cache, key, fn _key ->
-        {:commit, {:ok, extract_first_external_url(content)}}
-      end)
-    else
-      {:ok, extract_first_external_url(content)}
-    end
-  end
-
-  def extract_first_external_url_from_object(_), do: {:error, :no_content}
-
-  def extract_first_external_url(content) do
     content
     |> Floki.parse_fragment!()
     |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])")
@@ -90,4 +75,6 @@ def extract_first_external_url(content) do
     |> Floki.attribute("href")
     |> Enum.at(0)
   end
+
+  def extract_first_external_url_from_object(_), do: nil
 end

lib/pleroma/http.ex

@@ -74,6 +74,7 @@ def request(method, url, body, headers, options) when is_binary(url) do
     request = build_request(method, headers, options, url, body, params)
     client = Tesla.client([Tesla.Middleware.FollowRedirects, Tesla.Middleware.Telemetry])
 
+    Logger.debug("Outbound: #{method} #{url}")
     request(client, request)
   rescue
     e ->

lib/pleroma/http/adapter_helper.ex

@@ -116,6 +116,20 @@ defp maybe_add_transport_opts(opts) do
     put_in(opts, [:pools, :default, :conn_opts, :transport_opts, :inet6], true)
   end
 
+  def add_default_pool_max_idle_time(opts, pool_timeout) do
+    opts
+    |> maybe_add_pools()
+    |> maybe_add_default_pool()
+    |> put_in([:pools, :default, :pool_max_idle_time], pool_timeout)
+  end
+
+  def add_default_conn_max_idle_time(opts, connection_timeout) do
+    opts
+    |> maybe_add_pools()
+    |> maybe_add_default_pool()
+    |> put_in([:pools, :default, :conn_max_idle_time], connection_timeout)
+  end
+
   @doc """
   Merge default connection & adapter options with received ones.
   """

lib/pleroma/web/activity_pub/activity_pub.ex

@@ -155,9 +155,7 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when
       # Splice in the child object if we have one.
       activity = Maps.put_if_present(activity, :object, object)
 
-      ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn ->
-        Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
-      end)
+      Pleroma.Web.RichMedia.Card.get_by_activity(activity)
 
       # Add local posts to search index
       if local, do: Pleroma.Search.add_to_index(activity)
@@ -185,7 +183,7 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when
           id: "pleroma:fakeid"
         }
 
-        Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+        Pleroma.Web.RichMedia.Card.get_by_activity(activity)
         {:ok, activity}
 
       {:remote_limit_pass, _} ->

lib/pleroma/web/activity_pub/side_effects.ex

@@ -225,9 +225,7 @@ def handle(%{data: %{"type" => "Create"}} = activity, meta) do
         end
       end
 
-      ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn ->
-        Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end)
-      end)
+      Pleroma.Web.RichMedia.Card.get_by_activity(activity)
 
       Pleroma.Search.add_to_index(Map.put(activity, :object, object))
 

lib/pleroma/web/mastodon_api/views/status_view.ex

@@ -22,17 +22,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   alias Pleroma.Web.MediaProxy
   alias Pleroma.Web.PleromaAPI.EmojiReactionController
   require Logger
+  alias Pleroma.Web.RichMedia.Card
 
   import Pleroma.Web.ActivityPub.Visibility, only: [get_visibility: 1, visible_for_user?: 2]
 
-  # This is a naive way to do this, just spawning a process per activity
-  # to fetch the preview. However it should be fine considering
-  # pagination is restricted to 40 activities at a time
   defp fetch_rich_media_for_activities(activities) do
     Enum.each(activities, fn activity ->
-      spawn(fn ->
-        Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-      end)
+      Card.get_by_activity(activity)
     end)
   end
 
@@ -93,9 +89,7 @@ def render("index.json", opts) do
     # To do: check AdminAPIControllerTest on the reasons behind nil activities in the list
     activities = Enum.filter(opts.activities, & &1)
 
-    # Start fetching rich media before doing anything else, so that later calls to get the cards
-    # only block for timeout in the worst case, as opposed to
-    # length(activities_with_links) * timeout
+    # Start prefetching rich media before doing anything else
     fetch_rich_media_for_activities(activities)
     replied_to_activities = get_replied_to_activities(activities)
 
@@ -309,6 +303,12 @@ def render("show.json", %{activity: %{id: id, data: %{"object" => _object}} = ac
           "mastoapi:content:#{chrono_order}"
         )
 
+      card =
+        case Card.get_by_activity(activity) do
+          %Card{} = result -> render("card.json", result)
+          _ -> nil
+        end
+
       content_plaintext =
         content
         |> Activity.HTML.get_cached_stripped_html_for_activity(
@@ -318,8 +318,6 @@ def render("show.json", %{activity: %{id: id, data: %{"object" => _object}} = ac
 
       summary = object.data["summary"] || ""
 
-      card = render("card.json", Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity))
-
       url =
         if user.local do
           url(~p[/notice/#{activity}])
@@ -528,37 +526,30 @@ def render("source.json", %{activity: %{data: %{"object" => _object}} = activity
     }
   end
 
-  def render("card.json", %{rich_media: rich_media, page_url: page_url}) do
-    page_url_data = URI.parse(page_url)
-
-    page_url_data =
-      if is_binary(rich_media["url"]) do
-        URI.merge(page_url_data, URI.parse(rich_media["url"]))
-      else
-        page_url_data
-      end
+  def render("card.json", %Card{fields: rich_media}) do
+    page_url_data = URI.parse(rich_media["url"])
 
     page_url = page_url_data |> to_string
 
-    image_url_data =
-      if is_binary(rich_media["image"]) do
-        URI.parse(rich_media["image"])
-      else
-        nil
-      end
-
-    image_url = build_image_url(image_url_data, page_url_data)
+    image_url = proxied_url(rich_media["image"], page_url_data)
+    audio_url = proxied_url(rich_media["audio"], page_url_data)
+    video_url = proxied_url(rich_media["video"], page_url_data)
 
     %{
       type: "link",
       provider_name: page_url_data.host,
       provider_url: page_url_data.scheme <> "://" <> page_url_data.host,
       url: page_url,
-      image: image_url |> MediaProxy.url(),
+      image: image_url,
+      image_description: rich_media["image:alt"] || "",
       title: rich_media["title"] || "",
       description: rich_media["description"] || "",
       pleroma: %{
-        opengraph: rich_media
+        opengraph:
+          rich_media
+          |> Maps.put_if_present("image", image_url)
+          |> Maps.put_if_present("audio", audio_url)
+          |> Maps.put_if_present("video", video_url)
       }
     }
   end
@@ -636,6 +627,14 @@ def render("context.json", %{activity: activity, activities: activities, user: u
     }
   end
 
+  defp proxied_url(url, page_url_data) do
+    if is_binary(url) do
+      build_image_url(URI.parse(url), page_url_data) |> MediaProxy.url()
+    else
+      nil
+    end
+  end
+
   def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do
     object = Object.normalize(activity, fetch: false)
 
@@ -740,19 +739,7 @@ defp build_application(%{"type" => _type, "name" => name, "url" => url}),
 
   defp build_application(_), do: nil
 
-  # Workaround for Elixir issue #10771
-  # Avoid applying URI.merge unless necessary
-  # TODO: revert to always attempting URI.merge(image_url_data, page_url_data)
-  # when Elixir 1.12 is the minimum supported version
   @spec build_image_url(struct() | nil, struct()) :: String.t() | nil
-  defp build_image_url(
-         %URI{scheme: image_scheme, host: image_host} = image_url_data,
-         %URI{} = _page_url_data
-       )
-       when not is_nil(image_scheme) and not is_nil(image_host) do
-    image_url_data |> to_string
-  end
-
   defp build_image_url(%URI{} = image_url_data, %URI{} = page_url_data) do
     URI.merge(page_url_data, image_url_data) |> to_string
   end

lib/pleroma/web/rich_media/backfill.ex

@@ -0,0 +1,98 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.Backfill do
+  use Pleroma.Workers.WorkerHelper,
+    queue: "rich_media_backfill",
+    unique: [period: 300, states: Oban.Job.states(), keys: [:op, :url_hash]]
+
+  alias Pleroma.Web.RichMedia.Card
+  alias Pleroma.Web.RichMedia.Parser
+  alias Pleroma.Web.RichMedia.Parser.TTL
+  alias Pleroma.Workers.RichMediaExpirationWorker
+
+  require Logger
+
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+
+  def start(%{url: url} = args) when is_binary(url) do
+    url_hash = Card.url_to_hash(url)
+
+    args =
+      args
+      |> Map.put(:url_hash, url_hash)
+
+    __MODULE__.enqueue("rich_media_backfill", args)
+  end
+
+  def perform(%Oban.Job{args: %{"op" => "rich_media_backfill", "url" => url} = args})
+      when is_binary(url) do
+    run(args)
+  end
+
+  def run(%{"url" => url, "url_hash" => url_hash} = args) do
+    case Parser.parse(url) do
+      {:ok, fields} ->
+        {:ok, card} = Card.create(url, fields)
+
+        maybe_schedule_expiration(url, fields)
+
+        if Map.has_key?(args, "activity_id") do
+          stream_update(args)
+        end
+
+        warm_cache(url_hash, card)
+        :ok
+
+      {:error, {:invalid_metadata, fields}} ->
+        Logger.debug("Rich media incomplete or invalid metadata for #{url}: #{inspect(fields)}")
+        negative_cache(url_hash, :timer.minutes(30))
+
+      {:error, :body_too_large} ->
+        Logger.error("Rich media error for #{url}: :body_too_large")
+        negative_cache(url_hash, :timer.minutes(30))
+
+      {:error, {:content_type, type}} ->
+        Logger.debug("Rich media error for #{url}: :content_type is #{type}")
+        negative_cache(url_hash, :timer.minutes(30))
+
+      e ->
+        Logger.debug("Rich media error for #{url}: #{inspect(e)}")
+        {:error, e}
+    end
+  end
+
+  def run(e) do
+    Logger.error("Rich media failure - invalid args: #{inspect(e)}")
+    {:discard, :invalid}
+  end
+
+  defp maybe_schedule_expiration(url, fields) do
+    case TTL.process(fields, url) do
+      {:ok, ttl} when is_number(ttl) ->
+        timestamp = DateTime.from_unix!(ttl)
+
+        RichMediaExpirationWorker.new(%{"url" => url}, scheduled_at: timestamp)
+        |> Oban.insert()
+
+      _ ->
+        :ok
+    end
+  end
+
+  defp stream_update(%{"activity_id" => activity_id}) do
+    Logger.info("Rich media backfill: streaming update for activity #{activity_id}")
+
+    Pleroma.Activity.get_by_id(activity_id)
+    |> Pleroma.Activity.normalize()
+    |> Pleroma.Web.ActivityPub.ActivityPub.stream_out()
+  end
+
+  defp warm_cache(key, val), do: @cachex.put(:rich_media_cache, key, val)
+
+  def negative_cache(key, ttl \\ :timer.minutes(30)) do
+    @cachex.put(:rich_media_cache, key, nil, ttl: ttl)
+    {:discard, :error}
+  end
+end

lib/pleroma/web/rich_media/card.ex

@@ -0,0 +1,149 @@
+defmodule Pleroma.Web.RichMedia.Card do
+  use Ecto.Schema
+  import Ecto.Changeset
+  import Ecto.Query
+
+  alias Pleroma.Activity
+  alias Pleroma.HTML
+  alias Pleroma.Object
+  alias Pleroma.Repo
+  alias Pleroma.Web.RichMedia.Backfill
+  alias Pleroma.Web.RichMedia.Parser
+
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+  @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
+
+  @type t :: %__MODULE__{}
+
+  schema "rich_media_card" do
+    field(:url_hash, :binary)
+    field(:fields, :map)
+
+    timestamps()
+  end
+
+  @doc false
+  def changeset(card, attrs) do
+    card
+    |> cast(attrs, [:url_hash, :fields])
+    |> validate_required([:url_hash, :fields])
+    |> unique_constraint(:url_hash)
+  end
+
+  @spec create(String.t(), map()) :: {:ok, t()}
+  def create(url, fields) do
+    url_hash = url_to_hash(url)
+
+    fields = Map.put_new(fields, "url", url)
+
+    %__MODULE__{}
+    |> changeset(%{url_hash: url_hash, fields: fields})
+    |> Repo.insert(on_conflict: {:replace, [:fields]}, conflict_target: :url_hash)
+  end
+
+  @spec delete(String.t()) :: {:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t()} | :ok
+  def delete(url) do
+    url_hash = url_to_hash(url)
+    @cachex.del(:rich_media_cache, url_hash)
+
+    case get_by_url(url) do
+      %__MODULE__{} = card -> Repo.delete(card)
+      nil -> :ok
+    end
+  end
+
+  @spec get_by_url(String.t() | nil) :: t() | nil | :error
+  def get_by_url(url) when is_binary(url) do
+    if @config_impl.get([:rich_media, :enabled]) do
+      url_hash = url_to_hash(url)
+
+      @cachex.fetch!(:rich_media_cache, url_hash, fn _ ->
+        result =
+          __MODULE__
+          |> where(url_hash: ^url_hash)
+          |> Repo.one()
+
+        case result do
+          %__MODULE__{} = card -> {:commit, card}
+          _ -> {:ignore, nil}
+        end
+      end)
+    else
+      :error
+    end
+  end
+
+  def get_by_url(nil), do: nil
+
+  @spec get_or_backfill_by_url(String.t(), map()) :: t() | nil
+  def get_or_backfill_by_url(url, backfill_opts \\ %{}) do
+    case get_by_url(url) do
+      %__MODULE__{} = card ->
+        card
+
+      nil ->
+        backfill_opts = Map.put(backfill_opts, :url, url)
+
+        Backfill.start(backfill_opts)
+
+        nil
+
+      :error ->
+        nil
+    end
+  end
+
+  @spec get_by_activity(Activity.t()) :: t() | nil | :error
+  # Fake/Draft activity
+  def get_by_activity(%Activity{id: "pleroma:fakeid"} = activity) do
+    with %Object{} = object <- Object.normalize(activity, fetch: false),
+         url when not is_nil(url) <- HTML.extract_first_external_url_from_object(object) do
+      case get_by_url(url) do
+        # Cache hit
+        %__MODULE__{} = card ->
+          card
+
+        # Cache miss, but fetch for rendering the Draft
+        _ ->
+          with {:ok, fields} <- Parser.parse(url),
+               {:ok, card} <- create(url, fields) do
+            card
+          else
+            _ -> nil
+          end
+      end
+    else
+      _ ->
+        nil
+    end
+  end
+
+  def get_by_activity(activity) do
+    with %Object{} = object <- Object.normalize(activity, fetch: false),
+         {_, nil} <- {:cached, get_cached_url(object, activity.id)} do
+      nil
+    else
+      {:cached, url} ->
+        get_or_backfill_by_url(url, %{activity_id: activity.id})
+
+      _ ->
+        :error
+    end
+  end
+
+  @spec url_to_hash(String.t()) :: String.t()
+  def url_to_hash(url) do
+    :crypto.hash(:sha256, url) |> Base.encode16(case: :lower)
+  end
+
+  defp get_cached_url(object, activity_id) do
+    key = "URL|#{activity_id}"
+
+    @cachex.fetch!(:scrubber_cache, key, fn _ ->
+      url = HTML.extract_first_external_url_from_object(object)
+      Activity.HTML.add_cache_key_for(activity_id, key)
+
+      {:commit, url}
+    end)
+  end
+end

lib/pleroma/web/rich_media/helpers.ex

@@ -3,85 +3,13 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Helpers do
-  alias Pleroma.Activity
   alias Pleroma.Config
-  alias Pleroma.HTML
-  alias Pleroma.Object
-  alias Pleroma.Web.RichMedia.Parser
-
-  @options [
-    max_body: 2_000_000,
-    receive_timeout: 2_000
-  ]
-
-  @spec validate_page_url(URI.t() | binary()) :: :ok | :error
-  defp validate_page_url(page_url) when is_binary(page_url) do
-    validate_tld = Config.get([Pleroma.Formatter, :validate_tld])
-
-    page_url
-    |> Linkify.Parser.url?(validate_tld: validate_tld)
-    |> parse_uri(page_url)
-  end
-
-  defp validate_page_url(%URI{host: host, scheme: "https", authority: authority})
-       when is_binary(authority) do
-    cond do
-      host in Config.get([:rich_media, :ignore_hosts], []) ->
-        :error
-
-      get_tld(host) in Config.get([:rich_media, :ignore_tld], []) ->
-        :error
-
-      true ->
-        :ok
-    end
-  end
-
-  defp validate_page_url(_), do: :error
-
-  defp parse_uri(true, url) do
-    url
-    |> URI.parse()
-    |> validate_page_url
-  end
-
-  defp parse_uri(_, _), do: :error
-
-  defp get_tld(host) do
-    host
-    |> String.split(".")
-    |> Enum.reverse()
-    |> hd
-  end
-
-  def fetch_data_for_object(object) do
-    with true <- Config.get([:rich_media, :enabled]),
-         {:ok, page_url} <-
-           HTML.extract_first_external_url_from_object(object),
-         :ok <- validate_page_url(page_url),
-         {:ok, rich_media} <- Parser.parse(page_url) do
-      %{page_url: page_url, rich_media: rich_media}
-    else
-      _ -> %{}
-    end
-  end
-
-  def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do
-    with true <- Config.get([:rich_media, :enabled]),
-         %Object{} = object <- Object.normalize(activity, fetch: false) do
-      fetch_data_for_object(object)
-    else
-      _ -> %{}
-    end
-  end
-
-  def fetch_data_for_activity(_), do: %{}
 
   def rich_media_get(url) do
     headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
 
     head_check =
-      case Pleroma.HTTP.head(url, headers, @options) do
+      case Pleroma.HTTP.head(url, headers, http_options()) do
         # If the HEAD request didn't reach the server for whatever reason,
         # we assume the GET that comes right after won't either
         {:error, _} = e ->
@@ -96,7 +24,7 @@ def rich_media_get(url) do
           :ok
       end
 
-    with :ok <- head_check, do: Pleroma.HTTP.get(url, headers, @options)
+    with :ok <- head_check, do: Pleroma.HTTP.get(url, headers, http_options())
   end
 
   defp check_content_type(headers) do
@@ -112,12 +40,13 @@ defp check_content_type(headers) do
     end
   end
 
-  @max_body @options[:max_body]
   defp check_content_length(headers) do
+    max_body = Keyword.get(http_options(), :max_body)
+
     case List.keyfind(headers, "content-length", 0) do
       {_, maybe_content_length} ->
         case Integer.parse(maybe_content_length) do
-          {content_length, ""} when content_length <= @max_body -> :ok
+          {content_length, ""} when content_length <= max_body -> :ok
           {_, ""} -> {:error, :body_too_large}
           _ -> :ok
         end
@@ -126,4 +55,11 @@ defp check_content_length(headers) do
         :ok
     end
   end
+
+  defp http_options do
+    [
+      pool: :media,
+      max_body: Config.get([:rich_media, :max_body], 5_000_000)
+    ]
+  end
 end

lib/pleroma/web/rich_media/parser.ex

@@ -1,161 +1,41 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Parser do
   require Logger
 
-  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+  @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config)
 
   defp parsers do
     Pleroma.Config.get([:rich_media, :parsers])
   end
 
-  def parse(nil), do: {:error, "No URL provided"}
+  def parse(nil), do: nil
 
-  if Pleroma.Config.get(:env) == :test do
-    @spec parse(String.t()) :: {:ok, map()} | {:error, any()}
-    def parse(url), do: parse_with_timeout(url)
-  else
-    @spec parse(String.t()) :: {:ok, map()} | {:error, any()}
-    def parse(url) do
-      with {:ok, data} <- get_cached_or_parse(url),
-           {:ok, _} <- set_ttl_based_on_image(data, url) do
-        {:ok, data}
-      end
-    end
-
-    defp get_cached_or_parse(url) do
-      case @cachex.fetch(:rich_media_cache, url, fn ->
-             case parse_with_timeout(url) do
-               {:ok, _} = res ->
-                 {:commit, res}
-
-               {:error, reason} = e ->
-                 # Unfortunately we have to log errors here, instead of doing that
-                 # along with ttl setting at the bottom. Otherwise we can get log spam
-                 # if more than one process was waiting for the rich media card
-                 # while it was generated. Ideally we would set ttl here as well,
-                 # so we don't override it number_of_waiters_on_generation
-                 # times, but one, obviously, can't set ttl for not-yet-created entry
-                 # and Cachex doesn't support returning ttl from the fetch callback.
-                 log_error(url, reason)
-                 {:commit, e}
-             end
-           end) do
-        {action, res} when action in [:commit, :ok] ->
-          case res do
-            {:ok, _data} = res ->
-              res
-
-            {:error, reason} = e ->
-              if action == :commit, do: set_error_ttl(url, reason)
-              e
-          end
-
-        {:error, e} ->
-          {:error, {:cachex_error, e}}
-      end
-    end
-
-    defp set_error_ttl(_url, :body_too_large), do: :ok
-    defp set_error_ttl(_url, {:content_type, _}), do: :ok
-
-    # The TTL is not set for the errors above, since they are unlikely to change
-    # with time
-
-    defp set_error_ttl(url, _reason) do
-      ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
-      @cachex.expire(:rich_media_cache, url, ttl)
-      :ok
-    end
-
-    defp log_error(url, {:invalid_metadata, data}) do
-      Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end)
-    end
-
-    defp log_error(url, reason) do
-      Logger.warning(fn -> "Rich media error for #{url}: #{inspect(reason)}" end)
+  @spec parse(String.t()) :: {:ok, map()} | {:error, any()}
+  def parse(url) do
+    with {_, true} <- {:config, @config_impl.get([:rich_media, :enabled])},
+         :ok <- validate_page_url(url),
+         {:ok, data} <- parse_url(url) do
+      data = Map.put(data, "url", url)
+      {:ok, data}
+    else
+      {:config, _} -> {:error, :rich_media_disabled}
+      e -> e
     end
   end
 
-  @doc """
-  Set the rich media cache based on the expiration time of image.
-
-  Adopt behaviour `Pleroma.Web.RichMedia.Parser.TTL`
-
-  ## Example
-
-      defmodule MyModule do
-        @behaviour Pleroma.Web.RichMedia.Parser.TTL
-        def ttl(data, url) do
-          image_url = Map.get(data, :image)
-          # do some parsing in the url and get the ttl of the image
-          # and return ttl is unix time
-          parse_ttl_from_url(image_url)
-        end
-      end
-
-  Define the module in the config
-
-      config :pleroma, :rich_media,
-        ttl_setters: [MyModule]
-  """
-  @spec set_ttl_based_on_image(map(), String.t()) ::
-          {:ok, Integer.t() | :noop} | {:error, :no_key}
-  def set_ttl_based_on_image(data, url) do
-    case get_ttl_from_image(data, url) do
-      {:ok, ttl} when is_number(ttl) ->
-        ttl = ttl * 1000
-
-        case @cachex.expire_at(:rich_media_cache, url, ttl) do
-          {:ok, true} -> {:ok, ttl}
-          {:ok, false} -> {:error, :no_key}
-        end
-
-      _ ->
-        {:ok, :noop}
-    end
-  end
-
-  defp get_ttl_from_image(data, url) do
-    [:rich_media, :ttl_setters]
-    |> Pleroma.Config.get()
-    |> Enum.reduce({:ok, nil}, fn
-      module, {:ok, _ttl} ->
-        module.ttl(data, url)
-
-      _, error ->
-        error
-    end)
-  end
-
-  def parse_url(url) do
+  defp parse_url(url) do
     with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url),
          {:ok, html} <- Floki.parse_document(html) do
       html
       |> maybe_parse()
-      |> Map.put("url", url)
       |> clean_parsed_data()
       |> check_parsed_data()
     end
   end
 
-  def parse_with_timeout(url) do
-    try do
-      task =
-        Task.Supervisor.async_nolink(Pleroma.TaskSupervisor, fn ->
-          parse_url(url)
-        end)
-
-      Task.await(task, 5000)
-    catch
-      :exit, {:timeout, _} ->
-        Logger.warning("Timeout while fetching rich media for #{url}")
-        {:error, :timeout}
-    end
-  end
-
   defp maybe_parse(html) do
     Enum.reduce_while(parsers(), %{}, fn parser, acc ->
       case parser.parse(html, acc) do
@@ -181,4 +61,46 @@ defp clean_parsed_data(data) do
     end)
     |> Map.new()
   end
+
+  @spec validate_page_url(URI.t() | binary()) :: :ok | :error
+  defp validate_page_url(page_url) when is_binary(page_url) do
+    validate_tld = @config_impl.get([Pleroma.Formatter, :validate_tld])
+
+    page_url
+    |> Linkify.Parser.url?(validate_tld: validate_tld)
+    |> parse_uri(page_url)
+  end
+
+  defp validate_page_url(%URI{host: host, scheme: "https"}) do
+    cond do
+      Linkify.Parser.ip?(host) ->
+        :error
+
+      host in @config_impl.get([:rich_media, :ignore_hosts], []) ->
+        :error
+
+      get_tld(host) in @config_impl.get([:rich_media, :ignore_tld], []) ->
+        :error
+
+      true ->
+        :ok
+    end
+  end
+
+  defp validate_page_url(_), do: :error
+
+  defp parse_uri(true, url) do
+    url
+    |> URI.parse()
+    |> validate_page_url
+  end
+
+  defp parse_uri(_, _), do: :error
+
+  defp get_tld(host) do
+    host
+    |> String.split(".")
+    |> Enum.reverse()
+    |> hd
+  end
 end

lib/pleroma/web/rich_media/parser/ttl.ex

@@ -3,5 +3,18 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Parser.TTL do
-  @callback ttl(Map.t(), String.t()) :: Integer.t() | nil
+  @callback ttl(map(), String.t()) :: integer() | nil
+
+  @spec process(map(), String.t()) :: {:ok, integer() | nil}
+  def process(data, url) do
+    [:rich_media, :ttl_setters]
+    |> Pleroma.Config.get()
+    |> Enum.reduce_while({:ok, nil}, fn
+      module, acc ->
+        case module.ttl(data, url) do
+          ttl when is_number(ttl) -> {:halt, {:ok, ttl}}
+          _ -> {:cont, acc}
+        end
+    end)
+  end
 end

lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
@@ -7,25 +7,26 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
 
   @impl true
   def ttl(data, _url) do
-    image = Map.get(data, :image)
+    image = Map.get(data, "image")
 
-    if is_aws_signed_url(image) do
+    if aws_signed_url?(image) do
       image
       |> parse_query_params()
       |> format_query_params()
       |> get_expiration_timestamp()
     else
-      {:error, "Not aws signed url #{inspect(image)}"}
+      nil
     end
   end
 
-  defp is_aws_signed_url(image) when is_binary(image) and image != "" do
+  defp aws_signed_url?(image) when is_binary(image) and image != "" do
     %URI{host: host, query: query} = URI.parse(image)
 
-    String.contains?(host, "amazonaws.com") and String.contains?(query, "X-Amz-Expires")
+    is_binary(host) and String.contains?(host, "amazonaws.com") and
+      is_binary(query) and String.contains?(query, "X-Amz-Expires")
   end
 
-  defp is_aws_signed_url(_), do: nil
+  defp aws_signed_url?(_), do: nil
 
   defp parse_query_params(image) do
     %URI{query: query} = URI.parse(image)
@@ -45,6 +46,6 @@ defp get_expiration_timestamp(params) when is_map(params) do
       |> Map.get("X-Amz-Date")
       |> Timex.parse("{ISO:Basic:Z}")
 
-    {:ok, Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires"))}
+    Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires"))
   end
 end

lib/pleroma/web/rich_media/parser/ttl/opengraph.ex

@@ -0,0 +1,20 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.Parser.TTL.Opengraph do
+  @behaviour Pleroma.Web.RichMedia.Parser.TTL
+
+  @impl true
+  def ttl(%{"ttl" => ttl_string}, _url) when is_binary(ttl_string) do
+    try do
+      ttl = String.to_integer(ttl_string)
+      now = DateTime.utc_now() |> DateTime.to_unix()
+      now + ttl
+    rescue
+      _ -> nil
+    end
+  end
+
+  def ttl(_, _), do: nil
+end

lib/pleroma/web/rich_media/parsers/o_embed.ex

@@ -1,5 +1,5 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do

lib/pleroma/workers/rich_media_expiration_worker.ex

@@ -0,0 +1,15 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Workers.RichMediaExpirationWorker do
+  alias Pleroma.Web.RichMedia.Card
+
+  use Oban.Worker,
+    queue: :rich_media_expiration
+
+  @impl Oban.Worker
+  def perform(%Job{args: %{"url" => url} = _args}) do
+    Card.delete(url)
+  end
+end

mix.exs

@@ -78,7 +78,8 @@ def application do
         :comeonin,
         :fast_sanitize,
         :os_mon,
-        :ssl
+        :ssl,
+        :recon
       ],
       included_applications: [:ex_syslogger]
     ]
@@ -136,7 +137,7 @@ defp deps do
       {:tesla, "~> 1.7"},
       {:castore, "~> 1.0"},
       {:cowlib, "~> 2.12"},
-      {:finch, "~> 0.16.0"},
+      {:finch, "~> 0.18.0"},
       {:jason, "~> 1.4"},
       {:trailing_format_plug, "~> 0.0.7"},
       {:mogrify, "~> 0.9"},
@@ -157,10 +158,8 @@ defp deps do
       {:floki, "~> 0.34"},
       {:timex, "~> 3.7"},
       {:ueberauth, "== 0.10.5"},
-      {:linkify, git: "https://akkoma.dev/AkkomaGang/linkify.git"},
-      {:http_signatures,
-       git: "https://akkoma.dev/AkkomaGang/http_signatures.git",
-       ref: "6640ce7d24c783ac2ef56e27d00d12e8dc85f396"},
+      {:linkify, "~> 0.5.3"},
+      {:http_signatures, "~> 0.1.2"},
       {:telemetry, "~> 1.2"},
       {:telemetry_poller, "~> 1.0"},
       {:telemetry_metrics, "~> 0.6"},

mix.lock

@@ -3,12 +3,12 @@
   "base62": {:hex, :base62, "1.2.2", "85c6627eb609317b70f555294045895ffaaeb1758666ab9ef9ca38865b11e629", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "d41336bda8eaa5be197f1e4592400513ee60518e5b9f4dcf38f4b4dae6f377bb"},
   "bbcode_pleroma": {:hex, :bbcode_pleroma, "0.2.0", "d36f5bca6e2f62261c45be30fa9b92725c0655ad45c99025cb1c3e28e25803ef", [:mix], [{:nimble_parsec, "~> 0.5", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "19851074419a5fedb4ef49e1f01b30df504bb5dbb6d6adfc135238063bebd1c3"},
   "bcrypt_elixir": {:hex, :bcrypt_elixir, "3.0.1", "9be815469e6bfefec40fa74658ecbbe6897acfb57614df1416eeccd4903f602c", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "486bb95efb645d1efc6794c1ddd776a186a9a713abf06f45708a6ce324fb96cf"},
-  "benchee": {:hex, :benchee, "1.3.0", "f64e3b64ad3563fa9838146ddefb2d2f94cf5b473bdfd63f5ca4d0657bf96694", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "34f4294068c11b2bd2ebf2c59aac9c7da26ffa0068afdf3419f1b176e16c5f81"},
+  "benchee": {:hex, :benchee, "1.3.1", "c786e6a76321121a44229dde3988fc772bca73ea75170a73fd5f4ddf1af95ccf", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "76224c58ea1d0391c8309a8ecbfe27d71062878f59bd41a390266bf4ac1cc56d"},
   "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
   "cachex": {:hex, :cachex, "3.6.0", "14a1bfbeee060dd9bec25a5b6f4e4691e3670ebda28c8ba2884b12fe30b36bf8", [:mix], [{:eternal, "~> 1.2", [hex: :eternal, repo: "hexpm", optional: false]}, {:jumper, "~> 1.0", [hex: :jumper, repo: "hexpm", optional: false]}, {:sleeplocks, "~> 1.1", [hex: :sleeplocks, repo: "hexpm", optional: false]}, {:unsafe, "~> 1.0", [hex: :unsafe, repo: "hexpm", optional: false]}], "hexpm", "ebf24e373883bc8e0c8d894a63bbe102ae13d918f790121f5cfe6e485cc8e2e2"},
   "calendar": {:hex, :calendar, "1.0.0", "f52073a708528482ec33d0a171954ca610fe2bd28f1e871f247dc7f1565fa807", [:mix], [{:tzdata, "~> 0.1.201603 or ~> 0.5.20 or ~> 1.0", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "990e9581920c82912a5ee50e62ff5ef96da6b15949a2ee4734f935fdef0f0a6f"},
   "captcha": {:git, "https://git.pleroma.social/pleroma/elixir-libraries/elixir-captcha.git", "90f6ce7672f70f56708792a98d98bd05176c9176", [ref: "90f6ce7672f70f56708792a98d98bd05176c9176"]},
-  "castore": {:hex, :castore, "1.0.6", "ffc42f110ebfdafab0ea159cd43d31365fa0af0ce4a02ecebf1707ae619ee727", [:mix], [], "hexpm", "374c6e7ca752296be3d6780a6d5b922854ffcc74123da90f2f328996b962d33a"},
+  "castore": {:hex, :castore, "1.0.7", "b651241514e5f6956028147fe6637f7ac13802537e895a724f90bf3e36ddd1dd", [:mix], [], "hexpm", "da7785a4b0d2a021cd1292a60875a784b6caef71e76bf4917bdee1f390455cf5"},
   "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"},
   "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
   "comeonin": {:hex, :comeonin, "5.4.0", "246a56ca3f41d404380fc6465650ddaa532c7f98be4bda1b4656b3a37cc13abe", [:mix], [], "hexpm", "796393a9e50d01999d56b7b8420ab0481a7538d0caf80919da493b4a6e51faf1"},
@@ -18,7 +18,7 @@
   "cowboy": {:hex, :cowboy, "2.12.0", "f276d521a1ff88b2b9b4c54d0e753da6c66dd7be6c9fca3d9418b561828a3731", [:make, :rebar3], [{:cowlib, "2.13.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "8a7abe6d183372ceb21caa2709bec928ab2b72e18a3911aa1771639bef82651e"},
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
   "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"},
-  "credo": {:hex, :credo, "1.7.5", "643213503b1c766ec0496d828c90c424471ea54da77c8a168c725686377b9545", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "f799e9b5cd1891577d8c773d245668aa74a2fcd15eb277f51a0131690ebfb3fd"},
+  "credo": {:hex, :credo, "1.7.6", "b8f14011a5443f2839b04def0b252300842ce7388f3af177157c86da18dfbeea", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "146f347fb9f8cbc5f7e39e3f22f70acbef51d441baa6d10169dd604bfbc55296"},
   "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"},
   "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"},
   "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
@@ -29,18 +29,18 @@
   "eblurhash": {:hex, :eblurhash, "1.2.2", "7da4255aaea984b31bb71155f673257353b0e0554d0d30dcf859547e74602582", [:rebar3], [], "hexpm", "8c20ca00904de023a835a9dcb7b7762fed32264c85a80c3cafa85288e405044c"},
   "ecto": {:hex, :ecto, "3.10.3", "eb2ae2eecd210b4eb8bece1217b297ad4ff824b4384c0e3fdd28aaf96edd6135", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "44bec74e2364d491d70f7e42cd0d690922659d329f6465e89feb8a34e8cd3433"},
   "ecto_enum": {:hex, :ecto_enum, "1.4.0", "d14b00e04b974afc69c251632d1e49594d899067ee2b376277efd8233027aec8", [:mix], [{:ecto, ">= 3.0.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "> 3.0.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:mariaex, ">= 0.0.0", [hex: :mariaex, repo: "hexpm", optional: true]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "8fb55c087181c2b15eee406519dc22578fa60dd82c088be376d0010172764ee4"},
-  "ecto_psql_extras": {:hex, :ecto_psql_extras, "0.7.15", "0fc29dbae0e444a29bd6abeee4cf3c4c037e692a272478a234a1cc765077dbb1", [:mix], [{:ecto_sql, "~> 3.7", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0", [hex: :postgrex, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1 or ~> 4.0.0", [hex: :table_rex, repo: "hexpm", optional: false]}], "hexpm", "b6127f3a5c6fc3d84895e4768cc7c199f22b48b67d6c99b13fbf4a374e73f039"},
+  "ecto_psql_extras": {:hex, :ecto_psql_extras, "0.8.0", "440719cd74f09b3f01c84455707a2c3972b725c513808e68eb6c5b0ab82bf523", [:mix], [{:ecto_sql, "~> 3.7", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 0.18.0", [hex: :postgrex, repo: "hexpm", optional: false]}, {:table_rex, "~> 3.1.1 or ~> 4.0.0", [hex: :table_rex, repo: "hexpm", optional: false]}], "hexpm", "f1512812dc196bcb932a96c82e55f69b543dc125e9d39f5e3631a9c4ec65ef12"},
   "ecto_sql": {:hex, :ecto_sql, "3.10.2", "6b98b46534b5c2f8b8b5f03f126e75e2a73c64f3c071149d32987a5378b0fdbd", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.10.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16.0 or ~> 0.17.0 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "68c018debca57cb9235e3889affdaec7a10616a4e3a80c99fa1d01fdafaa9007"},
   "elasticsearch": {:git, "https://akkoma.dev/AkkomaGang/elasticsearch-elixir.git", "6cd946f75f6ab9042521a009d1d32d29a90113ca", [ref: "main"]},
-  "elixir_make": {:hex, :elixir_make, "0.8.3", "d38d7ee1578d722d89b4d452a3e36bcfdc644c618f0d063b874661876e708683", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "5c99a18571a756d4af7a4d89ca75c28ac899e6103af6f223982f09ce44942cc9"},
+  "elixir_make": {:hex, :elixir_make, "0.8.4", "4960a03ce79081dee8fe119d80ad372c4e7badb84c493cc75983f9d3bc8bde0f", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "6e7f1d619b5f61dfabd0a20aa268e575572b542ac31723293a4c1a567d5ef040"},
   "elixir_xml_to_map": {:hex, :elixir_xml_to_map, "3.1.0", "4d6260486a8cce59e4bf3575fe2dd2a24766546ceeef9f93fcec6f7c62a2827a", [:mix], [{:erlsom, "~> 1.4", [hex: :erlsom, repo: "hexpm", optional: false]}], "hexpm", "8fe5f2e75f90bab07ee2161120c2dc038ebcae8135554f5582990f1c8c21f911"},
   "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"},
   "erlsom": {:hex, :erlsom, "1.5.1", "c8fe2babd33ff0846403f6522328b8ab676f896b793634cfe7ef181c05316c03", [:rebar3], [], "hexpm", "7965485494c5844dd127656ac40f141aadfa174839ec1be1074e7edf5b4239eb"},
   "eternal": {:hex, :eternal, "1.2.2", "d1641c86368de99375b98d183042dd6c2b234262b8d08dfd72b9eeaafc2a1abd", [:mix], [], "hexpm", "2c9fe32b9c3726703ba5e1d43a1d255a4f3f2d8f8f9bc19f094c7cb1a7a9e782"},
-  "ex_aws": {:hex, :ex_aws, "2.5.3", "9c2d05ba0c057395b12c7b5ca6267d14cdaec1d8e65bdf6481fe1fd245accfb4", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "67115f1d399d7ec4d191812ee565c6106cb4b1bbf19a9d4db06f265fd87da97e"},
+  "ex_aws": {:hex, :ex_aws, "2.5.4", "86c5bb870a49e0ab6f5aa5dd58cf505f09d2624ebe17530db3c1b61c88a673af", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e82bd0091bb9a5bb190139599f922ff3fc7aebcca4374d65c99c4e23aa6d1625"},
   "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.3", "422468e5c3e1a4da5298e66c3468b465cfd354b842e512cb1f6fbbe4e2f5bdaf", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "4f09dd372cc386550e484808c5ac5027766c8d0cd8271ccc578b82ee6ef4f3b8"},
-  "ex_const": {:hex, :ex_const, "0.2.4", "d06e540c9d834865b012a17407761455efa71d0ce91e5831e86881b9c9d82448", [:mix], [], "hexpm", "96fd346610cc992b8f896ed26a98be82ac4efb065a0578f334a32d60a3ba9767"},
-  "ex_doc": {:hex, :ex_doc, "0.32.0", "896afb57b1e00030f6ec8b2e19d3ca99a197afb23858d49d94aea673dc222f12", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.1", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "ed2c3e42c558f49bda3ff37e05713432006e1719a6c4a3320c7e4735787374e7"},
+  "ex_const": {:hex, :ex_const, "0.3.0", "9d79516679991baf540ef445438eef1455ca91cf1a3c2680d8fb9e5bea2fe4de", [:mix], [], "hexpm", "76546322abb9e40ee4a2f454cf1c8a5b25c3672fa79bed1ea52c31e0d2428ca9"},
+  "ex_doc": {:hex, :ex_doc, "0.34.0", "ab95e0775db3df71d30cf8d78728dd9261c355c81382bcd4cefdc74610bef13e", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "60734fb4c1353f270c3286df4a0d51e65a2c1d9fba66af3940847cc65a8066d7"},
   "ex_machina": {:hex, :ex_machina, "2.7.0", "b792cc3127fd0680fecdb6299235b4727a4944a09ff0fa904cc639272cd92dc7", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "419aa7a39bde11894c87a615c4ecaa52d8f107bbdd81d810465186f783245bf8"},
   "ex_syslogger": {:hex, :ex_syslogger, "2.0.0", "de6de5c5472a9c4fdafb28fa6610e381ae79ebc17da6490b81d785d68bd124c9", [:mix], [{:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: true]}, {:syslog, "~> 1.1.0", [hex: :syslog, repo: "hexpm", optional: false]}], "hexpm", "a52b2fe71764e9e6ecd149ab66635812f68e39279cbeee27c52c0e35e8b8019e"},
   "excoveralls": {:hex, :excoveralls, "0.16.1", "0bd42ed05c7d2f4d180331a20113ec537be509da31fed5c8f7047ce59ee5a7c5", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "dae763468e2008cf7075a64cb1249c97cb4bc71e236c5c2b5e5cdf1cfa2bf138"},
@@ -49,55 +49,55 @@
   "fast_sanitize": {:hex, :fast_sanitize, "0.2.3", "67b93dfb34e302bef49fec3aaab74951e0f0602fd9fa99085987af05bd91c7a5", [:mix], [{:fast_html, "~> 2.0", [hex: :fast_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "e8ad286d10d0386e15d67d0ee125245ebcfbc7d7290b08712ba9013c8c5e56e2"},
   "file_ex": {:git, "https://akkoma.dev/AkkomaGang/file_ex.git", "cc7067c7d446c2526e9ecf91d40896b088851569", [ref: "cc7067c7d446c2526e9ecf91d40896b088851569"]},
   "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"},
-  "finch": {:hex, :finch, "0.16.0", "40733f02c89f94a112518071c0a91fe86069560f5dbdb39f9150042f44dcfb1a", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f660174c4d519e5fec629016054d60edd822cdfe2b7270836739ac2f97735ec5"},
+  "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"},
   "flake_id": {:hex, :flake_id, "0.1.0", "7716b086d2e405d09b647121a166498a0d93d1a623bead243e1f74216079ccb3", [:mix], [{:base62, "~> 1.2", [hex: :base62, repo: "hexpm", optional: false]}, {:ecto, ">= 2.0.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "31fc8090fde1acd267c07c36ea7365b8604055f897d3a53dd967658c691bd827"},
-  "floki": {:hex, :floki, "0.36.1", "712b7f2ba19a4d5a47dfe3e74d81876c95bbcbee44fe551f0af3d2a388abb3da", [:mix], [], "hexpm", "21ba57abb8204bcc70c439b423fc0dd9f0286de67dc82773a14b0200ada0995f"},
+  "floki": {:hex, :floki, "0.36.2", "a7da0193538c93f937714a6704369711998a51a6164a222d710ebd54020aa7a3", [:mix], [], "hexpm", "a8766c0bc92f074e5cb36c4f9961982eda84c5d2b8e979ca67f5c268ec8ed580"},
   "gen_smtp": {:hex, :gen_smtp, "1.2.0", "9cfc75c72a8821588b9b9fe947ae5ab2aed95a052b81237e0928633a13276fd3", [:rebar3], [{:ranch, ">= 1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "5ee0375680bca8f20c4d85f58c2894441443a743355430ff33a783fe03296779"},
   "gettext": {:hex, :gettext, "0.22.3", "c8273e78db4a0bb6fba7e9f0fd881112f349a3117f7f7c598fa18c66c888e524", [:mix], [{:expo, "~> 0.4.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "935f23447713954a6866f1bb28c3a878c4c011e802bcd68a726f5e558e4b64bd"},
   "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},
   "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"},
   "html_entities": {:hex, :html_entities, "0.5.2", "9e47e70598da7de2a9ff6af8758399251db6dbb7eebe2b013f2bbd2515895c3c", [:mix], [], "hexpm", "c53ba390403485615623b9531e97696f076ed415e8d8058b1dbaa28181f4fdcc"},
-  "http_signatures": {:git, "https://akkoma.dev/AkkomaGang/http_signatures.git", "6640ce7d24c783ac2ef56e27d00d12e8dc85f396", [ref: "6640ce7d24c783ac2ef56e27d00d12e8dc85f396"]},
+  "http_signatures": {:hex, :http_signatures, "0.1.2", "ed1cc7043abcf5bb4f30d68fb7bad9d618ec1a45c4ff6c023664e78b67d9c406", [:mix], [], "hexpm", "f08aa9ac121829dae109d608d83c84b940ef2f183ae50f2dd1e9a8bc619d8be7"},
   "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
   "inet_cidr": {:hex, :inet_cidr, "1.0.8", "d26bb7bdbdf21ae401ead2092bf2bb4bf57fe44a62f5eaa5025280720ace8a40", [:mix], [], "hexpm", "d5b26da66603bb56c933c65214c72152f0de9a6ea53618b56d63302a68f6a90e"},
   "jason": {:hex, :jason, "1.4.1", "af1504e35f629ddcdd6addb3513c3853991f694921b1b9368b0bd32beb9f1b63", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fbb01ecdfd565b56261302f7e1fcc27c4fb8f32d56eab74db621fc154604a7a1"},
   "joken": {:hex, :joken, "2.6.1", "2ca3d8d7f83bf7196296a3d9b2ecda421a404634bfc618159981a960020480a1", [:mix], [{:jose, "~> 1.11.9", [hex: :jose, repo: "hexpm", optional: false]}], "hexpm", "ab26122c400b3d254ce7d86ed066d6afad27e70416df947cdcb01e13a7382e68"},
-  "jose": {:hex, :jose, "1.11.9", "c861eb99d9e9f62acd071dc5a49ffbeab9014e44490cd85ea3e49e3d36184777", [:mix, :rebar3], [], "hexpm", "b5ccc3749d2e1638c26bed806259df5bc9e438797fe60dc71e9fa0716133899b"},
+  "jose": {:hex, :jose, "1.11.10", "a903f5227417bd2a08c8a00a0cbcc458118be84480955e8d251297a425723f83", [:mix, :rebar3], [], "hexpm", "0d6cd36ff8ba174db29148fc112b5842186b68a90ce9fc2b3ec3afe76593e614"},
   "jumper": {:hex, :jumper, "1.0.2", "68cdcd84472a00ac596b4e6459a41b3062d4427cbd4f1e8c8793c5b54f1406a7", [:mix], [], "hexpm", "9b7782409021e01ab3c08270e26f36eb62976a38c1aa64b2eaf6348422f165e1"},
-  "linkify": {:git, "https://akkoma.dev/AkkomaGang/linkify.git", "2567e2c1073fa371fd26fd66dfa5bc77b6919c16", []},
+  "linkify": {:hex, :linkify, "0.5.3", "5f8143d8f61f5ff08d3aeeff47ef6509492b4948d8f08007fbf66e4d2246a7f2", [:mix], [], "hexpm", "3ef35a1377d47c25506e07c1c005ea9d38d700699d92ee92825f024434258177"},
   "mail": {:hex, :mail, "0.3.1", "cb0a14e4ed8904e4e5a08214e686ccf6f9099346885db17d8c309381f865cc5c", [:mix], [], "hexpm", "1db701e89865c1d5fa296b2b57b1cd587587cca8d8a1a22892b35ef5a8e352a6"},
   "majic": {:git, "https://akkoma.dev/AkkomaGang/majic.git", "80540b36939ec83f48e76c61e5000e0fd67706f0", [ref: "80540b36939ec83f48e76c61e5000e0fd67706f0"]},
-  "makeup": {:hex, :makeup, "1.1.1", "fa0bc768698053b2b3869fa8a62616501ff9d11a562f3ce39580d60860c3a55e", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5dc62fbdd0de44de194898b6710692490be74baa02d9d108bc29f007783b0b48"},
+  "makeup": {:hex, :makeup, "1.1.2", "9ba8837913bdf757787e71c1581c21f9d2455f4dd04cfca785c70bbfff1a76a3", [:mix], [{:nimble_parsec, "~> 1.2.2 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cce1566b81fbcbd21eca8ffe808f33b221f9eee2cbc7a1706fc3da9ff18e6cac"},
   "makeup_elixir": {:hex, :makeup_elixir, "0.16.2", "627e84b8e8bf22e60a2579dad15067c755531fea049ae26ef1020cad58fe9578", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "41193978704763f6bbe6cc2758b84909e62984c7752b3784bd3c218bb341706b"},
-  "makeup_erlang": {:hex, :makeup_erlang, "0.1.5", "e0ff5a7c708dda34311f7522a8758e23bfcd7d8d8068dc312b5eb41c6fd76eba", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "94d2e986428585a21516d7d7149781480013c56e30c6a233534bedf38867a59a"},
+  "makeup_erlang": {:hex, :makeup_erlang, "1.0.0", "6f0eff9c9c489f26b69b61440bf1b238d95badae49adac77973cbacae87e3c2e", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "ea7a9307de9d1548d2a72d299058d1fd2339e3d398560a0e46c27dab4891e4d2"},
   "meck": {:hex, :meck, "0.9.2", "85ccbab053f1db86c7ca240e9fc718170ee5bda03810a6292b5306bf31bae5f5", [:rebar3], [], "hexpm", "81344f561357dc40a8344afa53767c32669153355b626ea9fcbc8da6b3045826"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
   "mfm_parser": {:git, "https://akkoma.dev/AkkomaGang/mfm-parser.git", "b21ab7754024af096f2d14247574f55f0063295b", [ref: "b21ab7754024af096f2d14247574f55f0063295b"]},
   "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"},
-  "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
+  "mimerl": {:hex, :mimerl, "1.3.0", "d0cd9fc04b9061f82490f6581e0128379830e78535e017f7780f37fea7545726", [:rebar3], [], "hexpm", "a1e15a50d1887217de95f0b9b0793e32853f7c258a5cd227650889b38839fe9d"},
   "mint": {:hex, :mint, "1.5.2", "4805e059f96028948870d23d7783613b7e6b0e2fb4e98d720383852a760067fd", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "d77d9e9ce4eb35941907f1d3df38d8f750c357865353e21d335bdcdf6d892a02"},
   "mock": {:hex, :mock, "0.3.8", "7046a306b71db2488ef54395eeb74df0a7f335a7caca4a3d3875d1fc81c884dd", [:mix], [{:meck, "~> 0.9.2", [hex: :meck, repo: "hexpm", optional: false]}], "hexpm", "7fa82364c97617d79bb7d15571193fc0c4fe5afd0c932cef09426b3ee6fe2022"},
   "mogrify": {:hex, :mogrify, "0.9.3", "238c782f00271dace01369ad35ae2e9dd020feee3443b9299ea5ea6bed559841", [:mix], [], "hexpm", "0189b1e1de27455f2b9ae8cf88239cefd23d38de9276eb5add7159aea51731e6"},
   "mox": {:hex, :mox, "1.1.0", "0f5e399649ce9ab7602f72e718305c0f9cdc351190f72844599545e4996af73c", [:mix], [], "hexpm", "d44474c50be02d5b72131070281a5d3895c0e7a95c780e90bc0cfe712f633a13"},
-  "nimble_options": {:hex, :nimble_options, "1.1.0", "3b31a57ede9cb1502071fade751ab0c7b8dbe75a9a4c2b5bbb0943a690b63172", [:mix], [], "hexpm", "8bbbb3941af3ca9acc7835f5655ea062111c9c27bcac53e004460dfd19008a99"},
+  "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"},
   "nimble_parsec": {:hex, :nimble_parsec, "1.4.0", "51f9b613ea62cfa97b25ccc2c1b4216e81df970acd8e16e8d1bdc58fef21370d", [:mix], [], "hexpm", "9c565862810fb383e9838c1dd2d7d2c437b3d13b267414ba6af33e50d2d1cf28"},
   "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"},
-  "oban": {:hex, :oban, "2.17.8", "7fd7c8e82c7819afc1b5b5ed8d6d92bf0ecdd7ba170328fb043301eb06d32521", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a2165bf93843b7bcb68182c82725ddd4cb43c0c3719f114e7aa3b6c99c4b6129"},
-  "open_api_spex": {:hex, :open_api_spex, "3.18.3", "fefb84fe323cacfc92afdd0ecb9e89bc0261ae00b7e3167ffc2028ce3944de42", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "c0cfc31570199ce7e7520b494a591027da609af45f6bf9adce51e2469b1609fb"},
+  "oban": {:hex, :oban, "2.17.10", "c3e5bd739b5c3fdc38eba1d43ab270a8c6ca4463bb779b7705c69400b0d87678", [:mix], [{:ecto_sql, "~> 3.10", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, "~> 0.9", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.16", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "4afd027b8e2bc3c399b54318b4f46ee8c40251fb55a285cb4e38b5363f0ee7c4"},
+  "open_api_spex": {:hex, :open_api_spex, "3.19.1", "65ccb5d06e3d664d1eec7c5ea2af2289bd2f37897094a74d7219fb03fc2b5994", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:poison, "~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :poison, repo: "hexpm", optional: true]}, {:ymlr, "~> 2.0 or ~> 3.0 or ~> 4.0 or ~> 5.0", [hex: :ymlr, repo: "hexpm", optional: true]}], "hexpm", "392895827ce2984a3459c91a484e70708132d8c2c6c5363972b4b91d6bbac3dd"},
   "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"},
   "phoenix": {:hex, :phoenix, "1.7.12", "1cc589e0eab99f593a8aa38ec45f15d25297dd6187ee801c8de8947090b5a9d3", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "d646192fbade9f485b01bc9920c139bfdd19d0f8df3d73fd8eaf2dfbe0d2837c"},
-  "phoenix_ecto": {:hex, :phoenix_ecto, "4.5.1", "6fdbc334ea53620e71655664df6f33f670747b3a7a6c4041cdda3e2c32df6257", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "ebe43aa580db129e54408e719fb9659b7f9e0d52b965c5be26cdca416ecead28"},
-  "phoenix_html": {:hex, :phoenix_html, "3.3.3", "380b8fb45912b5638d2f1d925a3771b4516b9a78587249cabe394e0a5d579dc9", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "923ebe6fec6e2e3b3e569dfbdc6560de932cd54b000ada0208b5f45024bdd76c"},
+  "phoenix_ecto": {:hex, :phoenix_ecto, "4.6.1", "96798325fab2fed5a824ca204e877b81f9afd2e480f581e81f7b4b64a5a477f2", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.17", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "0ae544ff99f3c482b0807c5cec2c8289e810ecacabc04959d82c3337f4703391"},
+  "phoenix_html": {:hex, :phoenix_html, "3.3.4", "42a09fc443bbc1da37e372a5c8e6755d046f22b9b11343bf885067357da21cb3", [:mix], [{:plug, "~> 1.5", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0249d3abec3714aff3415e7ee3d9786cb325be3151e6c4b3021502c585bf53fb"},
   "phoenix_live_dashboard": {:hex, :phoenix_live_dashboard, "0.7.2", "97cc4ff2dba1ebe504db72cb45098cb8e91f11160528b980bd282cc45c73b29c", [:mix], [{:ecto, "~> 3.6.2 or ~> 3.7", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_mysql_extras, "~> 0.5", [hex: :ecto_mysql_extras, repo: "hexpm", optional: true]}, {:ecto_psql_extras, "~> 0.7", [hex: :ecto_psql_extras, repo: "hexpm", optional: true]}, {:mime, "~> 1.6 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 0.18.3", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6 or ~> 1.0", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "0e5fdf063c7a3b620c566a30fcf68b7ee02e5e46fe48ee46a6ec3ba382dc05b7"},
   "phoenix_live_view": {:hex, :phoenix_live_view, "0.18.18", "1f38fbd7c363723f19aad1a04b5490ff3a178e37daaf6999594d5f34796c47fc", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6.15 or ~> 1.7.0", [hex: :phoenix, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 3.3", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.2 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a5810d0472f3189ede6d2a95bda7f31c6113156b91784a3426cb0ab6a6d85214"},
   "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"},
   "phoenix_swoosh": {:hex, :phoenix_swoosh, "1.2.1", "b74ccaa8046fbc388a62134360ee7d9742d5a8ae74063f34eb050279de7a99e1", [:mix], [{:finch, "~> 0.8", [hex: :finch, repo: "hexpm", optional: true]}, {:hackney, "~> 1.10", [hex: :hackney, repo: "hexpm", optional: true]}, {:phoenix, "~> 1.6", [hex: :phoenix, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_view, "~> 1.0 or ~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: false]}, {:swoosh, "~> 1.5", [hex: :swoosh, repo: "hexpm", optional: false]}], "hexpm", "4000eeba3f9d7d1a6bf56d2bd56733d5cadf41a7f0d8ffe5bb67e7d667e204a2"},
   "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"},
   "phoenix_view": {:hex, :phoenix_view, "2.0.3", "4d32c4817fce933693741deeb99ef1392619f942633dde834a5163124813aad3", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "cd34049af41be2c627df99cd4eaa71fc52a328c0c3d8e7d4aa28f880c30e7f64"},
-  "plug": {:hex, :plug, "1.15.3", "712976f504418f6dff0a3e554c40d705a9bcf89a7ccef92fc6a5ef8f16a30a97", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cc4365a3c010a56af402e0809208873d113e9c38c401cabd88027ef4f5c01fd2"},
+  "plug": {:hex, :plug, "1.16.0", "1d07d50cb9bb05097fdf187b31cf087c7297aafc3fed8299aac79c128a707e47", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cbf53aa1f5c4d758a7559c0bd6d59e286c2be0c6a1fac8cc3eee2f638243b93e"},
   "plug_cowboy": {:hex, :plug_cowboy, "2.7.1", "87677ffe3b765bc96a89be7960f81703223fe2e21efa42c125fcd0127dd9d6b2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "02dbd5f9ab571b864ae39418db7811618506256f6d13b4a45037e5fe78dc5de3"},
-  "plug_crypto": {:hex, :plug_crypto, "2.0.0", "77515cc10af06645abbfb5e6ad7a3e9714f805ae118fa1a70205f80d2d70fe73", [:mix], [], "hexpm", "53695bae57cc4e54566d993eb01074e4d894b65a3766f1c43e2c61a1b0f45ea9"},
+  "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"},
   "plug_static_index_html": {:hex, :plug_static_index_html, "1.0.0", "840123d4d3975585133485ea86af73cb2600afd7f2a976f9f5fd8b3808e636a0", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "79fd4fcf34d110605c26560cbae8f23c603ec4158c08298bd4360fdea90bb5cf"},
   "poison": {:hex, :poison, "5.0.0", "d2b54589ab4157bbb82ec2050757779bfed724463a544b6e20d79855a9e43b24", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "11dc6117c501b80c62a7594f941d043982a1bd05a1184280c0d9166eb4d8d3fc"},
   "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"},
@@ -107,7 +107,7 @@
   "recon": {:hex, :recon, "2.5.5", "c108a4c406fa301a529151a3bb53158cadc4064ec0c5f99b03ddb8c0e4281bdf", [:mix, :rebar3], [], "hexpm", "632a6f447df7ccc1a4a10bdcfce71514412b16660fe59deca0fcf0aa3c054404"},
   "remote_ip": {:hex, :remote_ip, "1.1.0", "cb308841595d15df3f9073b7c39243a1dd6ca56e5020295cb012c76fbec50f2d", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "616ffdf66aaad6a72fc546dabf42eed87e2a99e97b09cbd92b10cc180d02ed74"},
   "search_parser": {:git, "https://github.com/FloatingGhost/pleroma-contrib-search-parser.git", "08971a81e68686f9ac465cfb6661d51c5e4e1e7f", [ref: "08971a81e68686f9ac465cfb6661d51c5e4e1e7f"]},
-  "sleeplocks": {:hex, :sleeplocks, "1.1.2", "d45aa1c5513da48c888715e3381211c859af34bee9b8290490e10c90bb6ff0ca", [:rebar3], [], "hexpm", "9fe5d048c5b781d6305c1a3a0f40bb3dfc06f49bf40571f3d2d0c57eaa7f59a5"},
+  "sleeplocks": {:hex, :sleeplocks, "1.1.3", "96a86460cc33b435c7310dbd27ec82ca2c1f24ae38e34f8edde97f756503441a", [:rebar3], [], "hexpm", "d3b3958552e6eb16f463921e70ae7c767519ef8f5be46d7696cc1ed649421321"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"},
   "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"},
   "sweet_xml": {:hex, :sweet_xml, "0.7.4", "a8b7e1ce7ecd775c7e8a65d501bc2cd933bff3a9c41ab763f5105688ef485d08", [:mix], [], "hexpm", "e7c4b0bdbf460c928234951def54fe87edf1a170f6896675443279e2dbeba167"},

priv/repo/migrations/20240207035927_create_rich_media_card.exs

@@ -0,0 +1,14 @@
+defmodule Pleroma.Repo.Migrations.CreateRichMediaCard do
+  use Ecto.Migration
+
+  def change do
+    create table(:rich_media_card) do
+      add(:url_hash, :bytea)
+      add(:fields, :map)
+
+      timestamps()
+    end
+
+    create(unique_index(:rich_media_card, [:url_hash]))
+  end
+end

test/fixtures/rich_media/google.html

@@ -0,0 +1,12 @@
+<meta property="og:url" content="https://google.com">
+<meta property="og:type" content="website">
+<meta property="og:title" content="Google">
+<meta property="og:description" content="Search the world's information, including webpages, images, videos and more. Google has many special features to help you find exactly what you're looking for.">
+<meta property="og:image" content="">
+
+<meta name="twitter:card" content="summary_large_image">
+<meta property="twitter:domain" content="google.com">
+<meta property="twitter:url" content="https://google.com">
+<meta name="twitter:title" content="Google">
+<meta name="twitter:description" content="Search the world's information, including webpages, images, videos and more. Google has many special features to help you find exactly what you're looking for.">
+<meta name="twitter:image" content="">

test/fixtures/rich_media/oembed.html

@@ -1,3 +1,3 @@
 <link rel="alternate" type="application/json+oembed"
-  href="http://example.com/oembed.json"
+  href="https://example.com/oembed.json"
   title="Bacon Lollys oEmbed Profile" />

test/fixtures/rich_media/yahoo.html

@@ -0,0 +1,12 @@
+<meta property="og:url" content="https://yahoo.com">
+<meta property="og:type" content="website">
+<meta property="og:title" content="Yahoo | Mail, Weather, Search, Politics, News, Finance, Sports & Videos">
+<meta property="og:description" content="Latest news coverage, email, free stock quotes, live scores and video are just the beginning. Discover more every day at Yahoo!">
+<meta property="og:image" content="https://s.yimg.com/cv/apiv2/social/images/yahoo_default_logo.png">
+
+<meta name="twitter:card" content="summary_large_image">
+<meta property="twitter:domain" content="yahoo.com">
+<meta property="twitter:url" content="https://yahoo.com">
+<meta name="twitter:title" content="Yahoo | Mail, Weather, Search, Politics, News, Finance, Sports & Videos">
+<meta name="twitter:description" content="Latest news coverage, email, free stock quotes, live scores and video are just the beginning. Discover more every day at Yahoo!">
+<meta name="twitter:image" content="https://s.yimg.com/cv/apiv2/social/images/yahoo_default_logo.png">

test/mix/tasks/pleroma/database_test.exs

@@ -398,6 +398,7 @@ test "We don't have unexpected tables which may contain objects that are referen
                ["push_subscriptions"],
                ["registrations"],
                ["report_notes"],
+               ["rich_media_card"],
                ["scheduled_activities"],
                ["schema_migrations"],
                ["thread_mutes"],

test/pleroma/html_test.exs

@@ -176,7 +176,7 @@ test "extracts the url" do
         })
 
       object = Object.normalize(activity, fetch: false)
-      {:ok, url} = HTML.extract_first_external_url_from_object(object)
+      url = HTML.extract_first_external_url_from_object(object)
       assert url == "https://github.com/komeiji-satori/Dress"
     end
 
@@ -191,7 +191,7 @@ test "skips mentions" do
         })
 
       object = Object.normalize(activity, fetch: false)
-      {:ok, url} = HTML.extract_first_external_url_from_object(object)
+      url = HTML.extract_first_external_url_from_object(object)
 
       assert url == "https://github.com/syuilo/misskey/blob/develop/docs/setup.en.md"
 
@@ -207,7 +207,7 @@ test "skips hashtags" do
         })
 
       object = Object.normalize(activity, fetch: false)
-      {:ok, url} = HTML.extract_first_external_url_from_object(object)
+      url = HTML.extract_first_external_url_from_object(object)
 
       assert url == "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72255140"
     end
@@ -223,7 +223,7 @@ test "skips microformats hashtags" do
         })
 
       object = Object.normalize(activity, fetch: false)
-      {:ok, url} = HTML.extract_first_external_url_from_object(object)
+      url = HTML.extract_first_external_url_from_object(object)
 
       assert url == "https://www.pixiv.net/member_illust.php?mode=medium&illust_id=72255140"
     end
@@ -235,7 +235,7 @@ test "does not crash when there is an HTML entity in a link" do
 
       object = Object.normalize(activity, fetch: false)
 
-      assert {:ok, nil} = HTML.extract_first_external_url_from_object(object)
+      assert nil == HTML.extract_first_external_url_from_object(object)
     end
 
     test "skips attachment links" do
@@ -249,7 +249,7 @@ test "skips attachment links" do
 
       object = Object.normalize(activity, fetch: false)
 
-      assert {:ok, nil} = HTML.extract_first_external_url_from_object(object)
+      assert nil == HTML.extract_first_external_url_from_object(object)
     end
   end
 end

test/pleroma/http/adapter_helper_test.exs

@@ -73,4 +73,18 @@ test "should get set" do
       assert options[:pools][:default][:size] == 50
     end
   end
+
+  describe "pool idle time setting" do
+    test "should get set" do
+      options = AdapterHelper.add_default_pool_max_idle_time([], 50)
+      assert options[:pools][:default][:pool_max_idle_time] == 50
+    end
+  end
+
+  describe "connection timeout setting" do
+    test "should get set" do
+      options = AdapterHelper.add_default_conn_max_idle_time([], 50)
+      assert options[:pools][:default][:conn_max_idle_time] == 50
+    end
+  end
 end

test/pleroma/web/mastodon_api/controllers/status_controller_test.exs

@@ -364,59 +364,6 @@ test "posting a fake status", %{conn: conn} do
       assert real_status == fake_status
     end
 
-    test "fake statuses' preview card is not cached", %{conn: conn} do
-      clear_config([:rich_media, :enabled], true)
-
-      Tesla.Mock.mock_global(fn
-        %{
-          method: :get,
-          url: "https://example.com/twitter-card"
-        } ->
-          %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/twitter_card.html")}
-
-        env ->
-          apply(HttpRequestMock, :request, [env])
-      end)
-
-      conn1 =
-        conn
-        |> put_req_header("content-type", "application/json")
-        |> post("/api/v1/statuses", %{
-          "status" => "https://example.com/ogp",
-          "preview" => true
-        })
-
-      conn2 =
-        conn
-        |> put_req_header("content-type", "application/json")
-        |> post("/api/v1/statuses", %{
-          "status" => "https://example.com/twitter-card",
-          "preview" => true
-        })
-
-      assert %{"card" => %{"title" => "The Rock"}} = json_response_and_validate_schema(conn1, 200)
-
-      assert %{"card" => %{"title" => "Small Island Developing States Photo Submission"}} =
-               json_response_and_validate_schema(conn2, 200)
-    end
-
-    test "posting a status with OGP link preview", %{conn: conn} do
-      Tesla.Mock.mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
-      clear_config([:rich_media, :enabled], true)
-
-      conn =
-        conn
-        |> put_req_header("content-type", "application/json")
-        |> post("/api/v1/statuses", %{
-          "status" => "https://example.com/ogp"
-        })
-
-      assert %{"id" => id, "card" => %{"title" => "The Rock"}} =
-               json_response_and_validate_schema(conn, 200)
-
-      assert Activity.get_by_id(id)
-    end
-
     test "posting a direct status", %{conn: conn} do
       user2 = insert(:user)
       content = "direct cofe @#{user2.nickname}"

test/pleroma/web/mastodon_api/views/status_view_test.exs

@@ -16,10 +16,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusViewTest do
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.AccountView
   alias Pleroma.Web.MastodonAPI.StatusView
+  alias Pleroma.Web.RichMedia.Card
+  alias Pleroma.UnstubbedConfigMock, as: ConfigMock
 
   import Pleroma.Factory
   import Tesla.Mock
   import OpenApiSpex.TestAssertions
+  import Mox
 
   setup do
     mock(fn env -> apply(HttpRequestMock, :request, [env]) end)
@@ -677,56 +680,88 @@ test "it returns a a dictionary tags" do
 
   describe "rich media cards" do
     test "a rich media card without a site name renders correctly" do
-      page_url = "http://example.com"
+      page_url = "https://example.com"
 
-      card = %{
-        url: page_url,
-        image: page_url <> "/example.jpg",
-        title: "Example website"
-      }
+      {:ok, card} =
+        Card.create(page_url, %{image: page_url <> "/example.jpg", title: "Example website"})
 
-      %{provider_name: "example.com"} =
-        StatusView.render("card.json", %{page_url: page_url, rich_media: card})
+      %{provider_name: "example.com"} = StatusView.render("card.json", card)
     end
 
     test "a rich media card without a site name or image renders correctly" do
-      page_url = "http://example.com"
+      page_url = "https://example.com"
 
-      card = %{
-        url: page_url,
-        title: "Example website"
+      fields = %{
+        "url" => page_url,
+        "title" => "Example website"
       }
 
-      %{provider_name: "example.com"} =
-        StatusView.render("card.json", %{page_url: page_url, rich_media: card})
+      {:ok, card} = Card.create(page_url, fields)
+
+      %{provider_name: "example.com"} = StatusView.render("card.json", card)
     end
 
     test "a rich media card without an image renders correctly" do
-      page_url = "http://example.com"
+      page_url = "https://example.com"
 
-      card = %{
-        url: page_url,
-        site_name: "Example site name",
-        title: "Example website"
+      fields = %{
+        "url" => page_url,
+        "site_name" => "Example site name",
+        "title" => "Example website"
       }
 
-      %{provider_name: "example.com"} =
-        StatusView.render("card.json", %{page_url: page_url, rich_media: card})
+      {:ok, card} = Card.create(page_url, fields)
+
+      %{provider_name: "example.com"} = StatusView.render("card.json", card)
     end
 
     test "a rich media card with all relevant data renders correctly" do
-      page_url = "http://example.com"
+      page_url = "https://example.com"
 
-      card = %{
-        url: page_url,
-        site_name: "Example site name",
-        title: "Example website",
-        image: page_url <> "/example.jpg",
-        description: "Example description"
+      fields = %{
+        "url" => page_url,
+        "site_name" => "Example site name",
+        "title" => "Example website",
+        "image" => page_url <> "/example.jpg",
+        "description" => "Example description"
       }
 
-      %{provider_name: "example.com"} =
-        StatusView.render("card.json", %{page_url: page_url, rich_media: card})
+      {:ok, card} = Card.create(page_url, fields)
+
+      %{provider_name: "example.com"} = StatusView.render("card.json", card)
+    end
+
+    test "a rich media card has all media proxied" do
+      clear_config([:media_proxy, :enabled], true)
+      clear_config([:media_preview_proxy, :enabled])
+
+      ConfigMock
+      |> stub_with(Pleroma.Test.StaticConfig)
+
+      page_url = "https://example.com"
+
+      fields = %{
+        "url" => page_url,
+        "site_name" => "Example site name",
+        "title" => "Example website",
+        "image" => page_url <> "/example.jpg",
+        "audio" => page_url <> "/example.ogg",
+        "video" => page_url <> "/example.mp4",
+        "description" => "Example description"
+      }
+
+      {:ok, card} = Card.create(page_url, fields)
+
+      %{
+        provider_name: "example.com",
+        image: image,
+        pleroma: %{opengraph: og}
+      } = StatusView.render("card.json", card)
+
+      assert String.match?(image, ~r/\/proxy\//)
+      assert String.match?(og["image"], ~r/\/proxy\//)
+      assert String.match?(og["audio"], ~r/\/proxy\//)
+      assert String.match?(og["video"], ~r/\/proxy\//)
     end
   end
 

test/pleroma/web/rich_media/card_test.exs

@@ -0,0 +1,78 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.CardTest do
+  use Pleroma.DataCase, async: true
+
+  alias Pleroma.UnstubbedConfigMock, as: ConfigMock
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.RichMedia.Card
+
+  import Mox
+  import Pleroma.Factory
+  import Tesla.Mock
+
+  setup do
+    mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
+
+    ConfigMock
+    |> stub_with(Pleroma.Test.StaticConfig)
+
+    :ok
+  end
+
+  setup do: clear_config([:rich_media, :enabled], true)
+
+  test "crawls URL in activity" do
+    user = insert(:user)
+
+    url = "https://example.com/ogp"
+    url_hash = Card.url_to_hash(url)
+
+    {:ok, activity} =
+      CommonAPI.post(user, %{
+        status: "[test](#{url})",
+        content_type: "text/markdown"
+      })
+
+    # wait for oban
+    Pleroma.Tests.ObanHelpers.perform_all()
+
+    assert %Card{url_hash: ^url_hash, fields: _} = Card.get_by_activity(activity)
+  end
+
+  test "recrawls URLs on status edits/updates" do
+    original_url = "https://google.com/"
+    original_url_hash = Card.url_to_hash(original_url)
+    updated_url = "https://yahoo.com/"
+    updated_url_hash = Card.url_to_hash(updated_url)
+
+    user = insert(:user)
+    {:ok, activity} = CommonAPI.post(user, %{status: "I like this site #{original_url}"})
+
+    # Force a backfill
+    Card.get_by_activity(activity)
+    # wait for oban
+    Pleroma.Tests.ObanHelpers.perform_all()
+
+    assert match?(
+             %Card{url_hash: ^original_url_hash, fields: _},
+             Card.get_by_activity(activity)
+           )
+
+    {:ok, _} = CommonAPI.update(user, activity, %{status: "I like this site #{updated_url}"})
+
+    activity = Pleroma.Activity.get_by_id(activity.id)
+
+    # Force a backfill
+    Card.get_by_activity(activity)
+    # wait for oban
+    Pleroma.Tests.ObanHelpers.perform_all()
+
+    assert match?(
+             %Card{url_hash: ^updated_url_hash, fields: _},
+             Card.get_by_activity(activity)
+           )
+  end
+end

test/pleroma/web/rich_media/helpers_test.exs

@@ -1,91 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.RichMedia.HelpersTest do
-  use Pleroma.DataCase
-
-  alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.RichMedia.Helpers
-
-  import Pleroma.Factory
-  import Tesla.Mock
-
-  setup do
-    mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
-
-    :ok
-  end
-
-  setup do: clear_config([:rich_media, :enabled])
-
-  test "refuses to crawl incomplete URLs" do
-    user = insert(:user)
-
-    {:ok, activity} =
-      CommonAPI.post(user, %{
-        status: "[test](example.com/ogp)",
-        content_type: "text/markdown"
-      })
-
-    clear_config([:rich_media, :enabled], true)
-
-    assert %{} == Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-  end
-
-  test "refuses to crawl malformed URLs" do
-    user = insert(:user)
-
-    {:ok, activity} =
-      CommonAPI.post(user, %{
-        status: "[test](example.com[]/ogp)",
-        content_type: "text/markdown"
-      })
-
-    clear_config([:rich_media, :enabled], true)
-
-    assert %{} == Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-  end
-
-  test "crawls valid, complete URLs" do
-    user = insert(:user)
-
-    {:ok, activity} =
-      CommonAPI.post(user, %{
-        status: "[test](https://example.com/ogp)",
-        content_type: "text/markdown"
-      })
-
-    clear_config([:rich_media, :enabled], true)
-
-    assert %{page_url: "https://example.com/ogp", rich_media: _} =
-             Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
-  end
-
-  test "refuses to crawl URLs of private network from posts" do
-    user = insert(:user)
-
-    {:ok, activity} =
-      CommonAPI.post(user, %{status: "http://127.0.0.1:4000/notice/9kCP7VNyPJXFOXDrgO"})
-
-    {:ok, activity2} = CommonAPI.post(user, %{status: "https://10.111.10.1/notice/9kCP7V"})
-    {:ok, activity3} = CommonAPI.post(user, %{status: "https://172.16.32.40/notice/9kCP7V"})
-    {:ok, activity4} = CommonAPI.post(user, %{status: "https://192.168.10.40/notice/9kCP7V"})
-    {:ok, activity5} = CommonAPI.post(user, %{status: "https://pleroma.local/notice/9kCP7V"})
-
-    clear_config([:rich_media, :enabled], true)
-
-    assert %{} = Helpers.fetch_data_for_activity(activity)
-    assert %{} = Helpers.fetch_data_for_activity(activity2)
-    assert %{} = Helpers.fetch_data_for_activity(activity3)
-    assert %{} = Helpers.fetch_data_for_activity(activity4)
-    assert %{} = Helpers.fetch_data_for_activity(activity5)
-  end
-
-  test "catches errors in fetching" do
-    Tesla.Mock.mock(fn _ -> raise ArgumentError end)
-
-    assert {:error, :fetch_error} ==
-             Helpers.rich_media_get("wp-json/oembed/1.0/embed?url=http:%252F%252F")
-  end
-end

test/pleroma/web/rich_media/parser/ttl/aws_signed_url_test.exs

@@ -1,10 +1,25 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrlTest do
-  # Relies on Cachex, needs to be synchronous
-  use Pleroma.DataCase
+  use Pleroma.DataCase, async: false
+  use Oban.Testing, repo: Pleroma.Repo
+
+  import Mox
+
+  alias Pleroma.UnstubbedConfigMock, as: ConfigMock
+  alias Pleroma.Web.RichMedia.Card
+  alias Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl
+
+  setup do
+    ConfigMock
+    |> stub_with(Pleroma.Test.StaticConfig)
+
+    clear_config([:rich_media, :enabled], true)
+
+    :ok
+  end
 
   test "s3 signed url is parsed correct for expiration time" do
     url = "https://pleroma.social/amz"
@@ -22,7 +37,7 @@ test "s3 signed url is parsed correct for expiration time" do
     expire_time =
       Timex.parse!(timestamp, "{ISO:Basic:Z}") |> Timex.to_unix() |> Kernel.+(valid_till)
 
-    assert {:ok, expire_time} == Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.ttl(metadata, url)
+    assert expire_time == Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl.ttl(metadata, url)
   end
 
   test "s3 signed url is parsed and correct ttl is set for rich media" do
@@ -43,26 +58,37 @@ test "s3 signed url is parsed and correct ttl is set for rich media" do
     <meta name="twitter:site" content="Pleroma" />
     <meta name="twitter:title" content="Pleroma" />
     <meta name="twitter:description" content="Pleroma" />
-    <meta name="twitter:image" content="#{Map.get(metadata, :image)}" />
+    <meta name="twitter:image" content="#{Map.get(metadata, "image")}" />
     """
 
     Tesla.Mock.mock(fn
       %{
         method: :get,
-        url: "https://pleroma.social/amz"
+        url: ^url
       } ->
         %Tesla.Env{status: 200, body: body}
+
+      %{method: :head} ->
+        %Tesla.Env{status: 200}
     end)
 
-    Cachex.put(:rich_media_cache, url, metadata)
+    Card.get_or_backfill_by_url(url)
+    # wait for oban
+    Pleroma.Tests.ObanHelpers.perform_all()
 
-    Pleroma.Web.RichMedia.Parser.set_ttl_based_on_image(metadata, url)
+    assert_enqueued(worker: Pleroma.Workers.RichMediaExpirationWorker, args: %{"url" => url})
 
-    {:ok, cache_ttl} = Cachex.ttl(:rich_media_cache, url)
+    [%Oban.Job{scheduled_at: scheduled_at}] = all_enqueued()
 
-    # as there is delay in setting and pulling the data from cache we ignore 1 second
-    # make it 2 seconds for flakyness
-    assert_in_delta(valid_till * 1000, cache_ttl, 2000)
+    timestamp_dt = Timex.parse!(timestamp, "{ISO:Basic:Z}")
+
+    assert DateTime.diff(scheduled_at, timestamp_dt) == valid_till
+  end
+
+  test "AWS URL for an image without expiration works" do
+    og_data = %{"image" => "https://amazonaws.com/image.png"}
+
+    assert is_nil(AwsSignedUrl.ttl(og_data, ""))
   end
 
   defp construct_s3_url(timestamp, valid_till) do
@@ -71,11 +97,11 @@ defp construct_s3_url(timestamp, valid_till) do
 
   defp construct_metadata(timestamp, valid_till, url) do
     %{
-      image: construct_s3_url(timestamp, valid_till),
-      site: "Pleroma",
-      title: "Pleroma",
-      description: "Pleroma",
-      url: url
+      "image" => construct_s3_url(timestamp, valid_till),
+      "site" => "Pleroma",
+      "title" => "Pleroma",
+      "description" => "Pleroma",
+      "url" => url
     }
   end
 end

test/pleroma/web/rich_media/parser/ttl/opengraph_test.exs

@@ -0,0 +1,43 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.RichMedia.Parser.TTL.OpengraphTest do
+  use Pleroma.DataCase
+  use Oban.Testing, repo: Pleroma.Repo
+
+  import Mox
+
+  alias Pleroma.UnstubbedConfigMock, as: ConfigMock
+  alias Pleroma.Web.RichMedia.Card
+
+  setup do
+    ConfigMock
+    |> stub_with(Pleroma.Test.StaticConfig)
+
+    clear_config([:rich_media, :enabled], true)
+
+    :ok
+  end
+
+  test "OpenGraph TTL value is honored" do
+    url = "https://reddit.com/r/somepost"
+
+    Tesla.Mock.mock(fn
+      %{
+        method: :get,
+        url: ^url
+      } ->
+        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/reddit.html")}
+
+      %{method: :head} ->
+        %Tesla.Env{status: 200}
+    end)
+
+    Card.get_or_backfill_by_url(url)
+    # wait for oban
+    Pleroma.Tests.ObanHelpers.perform_all()
+
+    assert_enqueued(worker: Pleroma.Workers.RichMediaExpirationWorker, args: %{"url" => url})
+  end
+end

test/pleroma/web/rich_media/parser_test.exs

@@ -1,97 +1,30 @@
 # Pleroma: A lightweight social networking server
-# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.ParserTest do
-  use ExUnit.Case
+  use Pleroma.DataCase
 
   alias Pleroma.Web.RichMedia.Parser
 
+  import Tesla.Mock
+
   setup do
-    Tesla.Mock.mock_global(fn
-      %{
-        method: :get,
-        url: "http://example.com/ogp"
-      } ->
-        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/ogp.html")}
-
-      %{
-        method: :get,
-        url: "http://example.com/non-ogp"
-      } ->
-        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/non_ogp_embed.html")}
-
-      %{
-        method: :get,
-        url: "http://example.com/ogp-missing-title"
-      } ->
-        %Tesla.Env{
-          status: 200,
-          body: File.read!("test/fixtures/rich_media/ogp-missing-title.html")
-        }
-
-      %{
-        method: :get,
-        url: "http://example.com/twitter-card"
-      } ->
-        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/twitter_card.html")}
-
-      %{
-        method: :get,
-        url: "http://example.com/oembed"
-      } ->
-        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/oembed.html")}
-
-      %{
-        method: :get,
-        url: "http://example.com/oembed.json"
-      } ->
-        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/oembed.json")}
-
-      %{method: :get, url: "http://example.com/empty"} ->
-        %Tesla.Env{status: 200, body: "hello"}
-
-      %{method: :get, url: "http://example.com/malformed"} ->
-        %Tesla.Env{status: 200, body: File.read!("test/fixtures/rich_media/malformed-data.html")}
-
-      %{method: :get, url: "http://example.com/error"} ->
-        {:error, :overload}
-
-      %{
-        method: :head,
-        url: "http://example.com/huge-page"
-      } ->
-        %Tesla.Env{
-          status: 200,
-          headers: [{"content-length", "2000001"}, {"content-type", "text/html"}]
-        }
-
-      %{
-        method: :head,
-        url: "http://example.com/pdf-file"
-      } ->
-        %Tesla.Env{
-          status: 200,
-          headers: [{"content-length", "1000000"}, {"content-type", "application/pdf"}]
-        }
-
-      %{method: :head} ->
-        %Tesla.Env{status: 404, body: "", headers: []}
-    end)
-
-    :ok
+    mock_global(fn env -> apply(HttpRequestMock, :request, [env]) end)
   end
 
+  setup_all do: clear_config([:rich_media, :enabled], true)
+
   test "returns error when no metadata present" do
-    assert {:error, _} = Parser.parse("http://example.com/empty")
+    assert {:error, _} = Parser.parse("https://example.com/empty")
   end
 
   test "doesn't just add a title" do
-    assert {:error, {:invalid_metadata, _}} = Parser.parse("http://example.com/non-ogp")
+    assert {:error, {:invalid_metadata, _}} = Parser.parse("https://example.com/non-ogp")
   end
 
   test "parses ogp" do
-    assert Parser.parse("http://example.com/ogp") ==
+    assert Parser.parse("https://example.com/ogp") ==
              {:ok,
               %{
                 "image" => "http://ia.media-imdb.com/images/rock.jpg",
@@ -99,12 +32,12 @@ test "parses ogp" do
                 "description" =>
                   "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.",
                 "type" => "video.movie",
-                "url" => "http://example.com/ogp"
+                "url" => "https://example.com/ogp"
               }}
   end
 
   test "falls back to <title> when ogp:title is missing" do
-    assert Parser.parse("http://example.com/ogp-missing-title") ==
+    assert Parser.parse("https://example.com/ogp-missing-title") ==
              {:ok,
               %{
                 "image" => "http://ia.media-imdb.com/images/rock.jpg",
@@ -112,12 +45,12 @@ test "falls back to <title> when ogp:title is missing" do
                 "description" =>
                   "Directed by Michael Bay. With Sean Connery, Nicolas Cage, Ed Harris, John Spencer.",
                 "type" => "video.movie",
-                "url" => "http://example.com/ogp-missing-title"
+                "url" => "https://example.com/ogp-missing-title"
               }}
   end
 
   test "parses twitter card" do
-    assert Parser.parse("http://example.com/twitter-card") ==
+    assert Parser.parse("https://example.com/twitter-card") ==
              {:ok,
               %{
                 "card" => "summary",
@@ -125,12 +58,12 @@ test "parses twitter card" do
                 "image" => "https://farm6.staticflickr.com/5510/14338202952_93595258ff_z.jpg",
                 "title" => "Small Island Developing States Photo Submission",
                 "description" => "View the album on Flickr.",
-                "url" => "http://example.com/twitter-card"
+                "url" => "https://example.com/twitter-card"
               }}
   end
 
   test "parses OEmbed and filters HTML tags" do
-    assert Parser.parse("http://example.com/oembed") ==
+    assert Parser.parse("https://example.com/oembed") ==
              {:ok,
               %{
                 "author_name" => "\u202E\u202D\u202Cbees\u202C",
@@ -150,7 +83,7 @@ test "parses OEmbed and filters HTML tags" do
                 "thumbnail_width" => 150,
                 "title" => "Bacon Lollys",
                 "type" => "photo",
-                "url" => "http://example.com/oembed",
+                "url" => "https://example.com/oembed",
                 "version" => "1.0",
                 "web_page" => "https://www.flickr.com/photos/bees/2362225867/",
                 "web_page_short_url" => "https://flic.kr/p/4AK2sc",
@@ -159,18 +92,47 @@ test "parses OEmbed and filters HTML tags" do
   end
 
   test "rejects invalid OGP data" do
-    assert {:error, _} = Parser.parse("http://example.com/malformed")
+    assert {:error, _} = Parser.parse("https://example.com/malformed")
   end
 
   test "returns error if getting page was not successful" do
-    assert {:error, :overload} = Parser.parse("http://example.com/error")
+    assert {:error, :overload} = Parser.parse("https://example.com/error")
   end
 
   test "does a HEAD request to check if the body is too large" do
-    assert {:error, :body_too_large} = Parser.parse("http://example.com/huge-page")
+    assert {:error, :body_too_large} = Parser.parse("https://example.com/huge-page")
   end
 
   test "does a HEAD request to check if the body is html" do
-    assert {:error, {:content_type, _}} = Parser.parse("http://example.com/pdf-file")
+    assert {:error, {:content_type, _}} = Parser.parse("https://example.com/pdf-file")
+  end
+
+  test "refuses to crawl incomplete URLs" do
+    url = "example.com/ogp"
+    assert :error == Parser.parse(url)
+  end
+
+  test "refuses to crawl malformed URLs" do
+    url = "example.com[]/ogp"
+    assert :error == Parser.parse(url)
+  end
+
+  test "refuses to crawl URLs of private network from posts" do
+    [
+      "http://127.0.0.1:4000/notice/9kCP7VNyPJXFOXDrgO",
+      "https://10.111.10.1/notice/9kCP7V",
+      "https://172.16.32.40/notice/9kCP7V",
+      "https://192.168.10.40/notice/9kCP7V",
+      "https://pleroma.local/notice/9kCP7V"
+    ]
+    |> Enum.each(fn url ->
+      assert :error == Parser.parse(url)
+    end)
+  end
+
+  test "returns error when disabled" do
+    clear_config([:rich_media, :enabled], false)
+
+    assert match?({:error, :rich_media_disabled}, Parser.parse("https://example.com/ogp"))
   end
 end

test/support/http_request_mock.ex

@@ -1708,14 +1708,41 @@ def post(url, query, body, headers) do
 
   # Most of the rich media mocks are missing HEAD requests, so we just return 404.
   @rich_media_mocks [
+    "https://example.com/empty",
+    "https://example.com/error",
+    "https://example.com/malformed",
+    "https://example.com/non-ogp",
+    "https://example.com/oembed",
+    "https://example.com/oembed.json",
     "https://example.com/ogp",
     "https://example.com/ogp-missing-data",
-    "https://example.com/twitter-card"
+    "https://example.com/ogp-missing-title",
+    "https://example.com/twitter-card",
+    "https://google.com/",
+    "https://pleroma.local/notice/9kCP7V",
+    "https://yahoo.com/"
   ]
+
   def head(url, _query, _body, _headers) when url in @rich_media_mocks do
     {:ok, %Tesla.Env{status: 404, body: ""}}
   end
 
+  def head("https://example.com/pdf-file", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       headers: [{"content-length", "1000000"}, {"content-type", "application/pdf"}]
+     }}
+  end
+
+  def head("https://example.com/huge-page", _, _, _) do
+    {:ok,
+     %Tesla.Env{
+       status: 200,
+       headers: [{"content-length", "2000001"}, {"content-type", "text/html"}]
+     }}
+  end
+
   def head(url, query, body, headers) do
     {:error,
      "Mock response not implemented for HEAD #{inspect(url)}, #{query}, #{inspect(body)}, #{inspect(headers)}"}

test/support/mocks.ex

@@ -26,5 +26,6 @@
 Mox.defmock(Pleroma.Web.FederatorMock, for: Pleroma.Web.Federator.Publishing)
 
 Mox.defmock(Pleroma.ConfigMock, for: Pleroma.Config.Getting)
+Mox.defmock(Pleroma.UnstubbedConfigMock, for: Pleroma.Config.Getting)
 
 Mox.defmock(Pleroma.LoggerMock, for: Pleroma.Logging)

test/test_helper.exs

@@ -17,3 +17,16 @@
   uploads = Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads], "test/uploads")
   File.rm_rf!(uploads)
 end)
+
+defmodule Pleroma.Test.StaticConfig do
+  @moduledoc """
+  This module provides a Config that is completely static, built at startup time from the environment. It's safe to use in testing as it will not modify any state.
+  """
+
+  @behaviour Pleroma.Config.Getting
+  @config Application.get_all_env(:pleroma)
+
+  def get(path, default \\ nil) do
+    get_in(@config, path) || default
+  end
+end