Refactor Follows/Followers counter syncronization

- Actually sync counters in the database instead of info cache (which got
overriden after user update was finished anyway)
- Add following count field to user info
- Set hide_followers/hide_follows for remote users based on http status
codes for the first collection page
This commit is contained in:
rinpatch 2019-07-13 19:17:57 +03:00
parent 02cdedbf9f
commit e8fa477793
7 changed files with 73 additions and 61 deletions

View file

@ -29,7 +29,8 @@ config :pleroma, :instance,
email: "admin@example.com",
notify_email: "noreply@example.com",
skip_thread_containment: false,
federating: false
federating: false,
external_user_synchronization: false
# Configure your database
config :pleroma, Pleroma.Repo,

View file

@ -76,7 +76,7 @@ defmodule Pleroma.Object.Fetcher do
end
end
def fetch_and_contain_remote_object_from_id(id) do
def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
Logger.info("Fetching object #{id} via AP")
with true <- String.starts_with?(id, "http"),
@ -96,4 +96,8 @@ defmodule Pleroma.Object.Fetcher do
{:error, e}
end
end
def fetch_and_contain_remote_object_from_id(_id) do
{:error, "id must be a string"}
end
end

View file

@ -114,7 +114,9 @@ defmodule Pleroma.User do
def user_info(%User{} = user, args \\ %{}) do
following_count =
if args[:following_count], do: args[:following_count], else: following_count(user)
if args[:following_count],
do: args[:following_count],
else: user.info.following_count || following_count(user)
follower_count =
if args[:follower_count], do: args[:follower_count], else: user.info.follower_count

View file

@ -16,6 +16,7 @@ defmodule Pleroma.User.Info do
field(:source_data, :map, default: %{})
field(:note_count, :integer, default: 0)
field(:follower_count, :integer, default: 0)
field(:following_count, :integer, default: nil)
field(:locked, :boolean, default: false)
field(:confirmation_pending, :boolean, default: false)
field(:confirmation_token, :string, default: nil)
@ -195,7 +196,11 @@ defmodule Pleroma.User.Info do
:uri,
:hub,
:topic,
:salmon
:salmon,
:hide_followers,
:hide_follows,
:follower_count,
:following_count
])
end
@ -206,7 +211,11 @@ defmodule Pleroma.User.Info do
:source_data,
:banner,
:locked,
:magic_key
:magic_key,
:follower_count,
:following_count,
:hide_follows,
:hide_followers
])
end

View file

@ -1013,6 +1013,56 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
{:ok, user_data}
end
defp maybe_update_follow_information(data) do
with {:enabled, true} <-
{:enabled, Pleroma.Config.get([:instance, :external_user_synchronization])},
{:ok, following_data} <-
Fetcher.fetch_and_contain_remote_object_from_id(data.following_address),
following_count <- following_data["totalItems"],
hide_follows <- collection_private?(following_data),
{:ok, followers_data} <-
Fetcher.fetch_and_contain_remote_object_from_id(data.follower_address),
followers_count <- followers_data["totalItems"],
hide_followers <- collection_private?(followers_data) do
info = %{
"hide_follows" => hide_follows,
"follower_count" => followers_count,
"following_count" => following_count,
"hide_followers" => hide_followers
}
info = Map.merge(data.info, info)
Map.put(data, :info, info)
else
{:enabled, false} ->
data
e ->
Logger.error(
"Follower/Following counter update for #{data.ap_id} failed.\n" <> inspect(e)
)
data
end
end
defp collection_private?(data) do
if is_map(data["first"]) and
data["first"]["type"] in ["CollectionPage", "OrderedCollectionPage"] do
false
else
with {:ok, _data} <- Fetcher.fetch_and_contain_remote_object_from_id(data["first"]) do
false
else
{:error, {:ok, %{status: code}}} when code in [401, 403] ->
true
_e ->
false
end
end
end
def user_data_from_user_object(data) do
with {:ok, data} <- MRF.filter(data),
{:ok, data} <- object_to_user_data(data) do
@ -1024,7 +1074,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
def fetch_and_prepare_user_from_ap_id(ap_id) do
with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
{:ok, data} <- user_data_from_user_object(data) do
{:ok, data} <- user_data_from_user_object(data),
data <- maybe_update_follow_information(data) do
{:ok, data}
else
e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")

View file

@ -1087,10 +1087,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
PleromaJobQueue.enqueue(:transmogrifier, __MODULE__, [:user_upgrade, user])
end
if Pleroma.Config.get([:instance, :external_user_synchronization]) do
update_following_followers_counters(user)
end
{:ok, user}
else
%User{} = user -> {:ok, user}
@ -1123,27 +1119,4 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
data
|> maybe_fix_user_url
end
def update_following_followers_counters(user) do
info = %{}
following = fetch_counter(user.following_address)
info = if following, do: Map.put(info, :following_count, following), else: info
followers = fetch_counter(user.follower_address)
info = if followers, do: Map.put(info, :follower_count, followers), else: info
User.set_info_cache(user, info)
end
defp fetch_counter(url) do
with {:ok, %{body: body, status: code}} when code in 200..299 <-
Pleroma.HTTP.get(
url,
[{:Accept, "application/activity+json"}]
),
{:ok, data} <- Jason.decode(body) do
data["totalItems"]
end
end
end

View file

@ -1359,32 +1359,4 @@ defmodule Pleroma.Web.ActivityPub.TransmogrifierTest do
refute recipient.follower_address in fixed_object["to"]
end
end
test "update_following_followers_counters/1" do
user1 =
insert(:user,
local: false,
follower_address: "http://localhost:4001/users/masto_closed/followers",
following_address: "http://localhost:4001/users/masto_closed/following"
)
user2 =
insert(:user,
local: false,
follower_address: "http://localhost:4001/users/fuser2/followers",
following_address: "http://localhost:4001/users/fuser2/following"
)
Transmogrifier.update_following_followers_counters(user1)
Transmogrifier.update_following_followers_counters(user2)
%{follower_count: followers, following_count: following} = User.get_cached_user_info(user1)
assert followers == 437
assert following == 152
%{follower_count: followers, following_count: following} = User.get_cached_user_info(user2)
assert followers == 527
assert following == 267
end
end