Use instances table instead of Cachex

This commit is contained in:
Haelwenn (lanodan) Monnier 2020-07-07 11:13:38 +02:00
parent 6a679d80c9
commit 013e2c5057
No known key found for this signature in database
GPG key ID: D5B7A8E43C997DEE
5 changed files with 65 additions and 32 deletions

View file

@ -150,7 +150,6 @@ defp cachex_children do
build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10),
build_cachex("failed_proxy_url", limit: 2500), build_cachex("failed_proxy_url", limit: 2500),
build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000)
build_cachex("instances", default_ttl: 25_000, ttl_interval: 1000, limit: 2500)
] ]
end end

View file

@ -37,32 +37,4 @@ def host(url_or_host) when is_binary(url_or_host) do
url_or_host url_or_host
end end
end end
def get_cached_favicon(instance_url) when is_binary(instance_url) do
Cachex.fetch!(:instances_cache, instance_url, fn _ -> get_favicon(instance_url) end)
end
def get_cached_favicon(_instance_url) do
nil
end
def get_favicon(instance_url) when is_binary(instance_url) do
try do
with {:ok, %Tesla.Env{body: html}} <-
Pleroma.HTTP.get(instance_url, [{:Accept, "text/html"}]),
favicon_rel <-
html
|> Floki.parse_document!()
|> Floki.attribute("link[rel=icon]", "href")
|> List.first(),
favicon_url <- URI.merge(URI.parse(instance_url), favicon_rel) |> to_string(),
true <- is_binary(favicon_url) do
favicon_url
else
_ -> nil
end
rescue
_ -> nil
end
end
end end

View file

@ -17,6 +17,8 @@ defmodule Pleroma.Instances.Instance do
schema "instances" do schema "instances" do
field(:host, :string) field(:host, :string)
field(:unreachable_since, :naive_datetime_usec) field(:unreachable_since, :naive_datetime_usec)
field(:favicon, :string)
field(:favicon_updated_at, :naive_datetime)
timestamps() timestamps()
end end
@ -25,7 +27,7 @@ defmodule Pleroma.Instances.Instance do
def changeset(struct, params \\ %{}) do def changeset(struct, params \\ %{}) do
struct struct
|> cast(params, [:host, :unreachable_since]) |> cast(params, [:host, :unreachable_since, :favicon, :favicon_updated_at])
|> validate_required([:host]) |> validate_required([:host])
|> unique_constraint(:host) |> unique_constraint(:host)
end end
@ -120,4 +122,55 @@ defp parse_datetime(datetime) when is_binary(datetime) do
end end
defp parse_datetime(datetime), do: datetime defp parse_datetime(datetime), do: datetime
def get_or_update_favicon(%URI{host: host} = instance_uri) do
existing_record = Repo.get_by(Instance, %{host: host})
now = NaiveDateTime.utc_now()
if existing_record && existing_record.favicon &&
NaiveDateTime.diff(now, existing_record.favicon_updated_at) < 86_400 do
existing_record.favicon
else
favicon = scrape_favicon(instance_uri)
cond do
is_binary(favicon) && existing_record ->
existing_record
|> changeset(%{favicon: favicon, favicon_updated_at: now})
|> Repo.update()
favicon
is_binary(favicon) ->
%Instance{}
|> changeset(%{host: host, favicon: favicon, favicon_updated_at: now})
|> Repo.insert()
favicon
true ->
nil
end
end
end
defp scrape_favicon(%URI{} = instance_uri) do
try do
with {:ok, %Tesla.Env{body: html}} <-
Pleroma.HTTP.get(to_string(instance_uri), [{:Accept, "text/html"}]),
favicon_rel <-
html
|> Floki.parse_document!()
|> Floki.attribute("link[rel=icon]", "href")
|> List.first(),
favicon <- URI.merge(instance_uri, favicon_rel) |> to_string(),
true <- is_binary(favicon) do
favicon
else
_ -> nil
end
rescue
_ -> nil
end
end
end end

View file

@ -209,8 +209,7 @@ defp do_render("show.json", %{user: user} = opts) do
|> Map.get(:ap_id, "") |> Map.get(:ap_id, "")
|> URI.parse() |> URI.parse()
|> URI.merge("/") |> URI.merge("/")
|> to_string() |> Pleroma.Instances.Instance.get_or_update_favicon()
|> Pleroma.Instances.get_cached_favicon()
|> MediaProxy.url() |> MediaProxy.url()
%{ %{

View file

@ -0,0 +1,10 @@
defmodule Pleroma.Repo.Migrations.InstancesAddFavicon do
use Ecto.Migration
def change do
alter table(:instances) do
add(:favicon, :string)
add(:favicon_updated_at, :naive_datetime)
end
end
end