Merge branch 'status-visibility-count-stats' into 'develop'

Status count by visibility scope (`GET /api/pleroma/admin/stats` )

See merge request pleroma/pleroma!2097
This commit is contained in:
Haelwenn 2020-02-24 19:21:29 +00:00
commit 908b6ee49a
11 changed files with 321 additions and 1 deletions

View file

@ -76,6 +76,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`). - OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`).
- Add an option `authorized_fetch_mode` to require HTTP signatures for AP fetches. - Add an option `authorized_fetch_mode` to require HTTP signatures for AP fetches.
- ActivityPub: support for `replies` collection (output for outgoing federation & fetching on incoming federation). - ActivityPub: support for `replies` collection (output for outgoing federation & fetching on incoming federation).
- Mix task to refresh counter cache (`mix pleroma.refresh_counter_cache`)
<details> <details>
<summary>API Changes</summary> <summary>API Changes</summary>
@ -119,6 +120,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Mastodon API: Add `reacted` property to `emoji_reactions` - Mastodon API: Add `reacted` property to `emoji_reactions`
- Pleroma API: Add reactions for a single emoji. - Pleroma API: Add reactions for a single emoji.
- ActivityPub: `[:activitypub, :note_replies_output_limit]` setting sets the number of note self-replies to output on outgoing federation. - ActivityPub: `[:activitypub, :note_replies_output_limit]` setting sets the number of note self-replies to output on outgoing federation.
- Admin API: `GET /api/pleroma/admin/stats` to get status count by visibility scope
</details> </details>
### Fixed ### Fixed

View file

@ -939,3 +939,20 @@ Loads json generated from `config/descriptions.exs`.
- Params: - Params:
- `nicknames` - `nicknames`
- Response: Array of user nicknames - Response: Array of user nicknames
## `GET /api/pleroma/admin/stats`
### Stats
- Response:
```json
{
"status_visibility": {
"direct": 739,
"private": 9,
"public": 17,
"unlisted": 14
}
}
```

View file

@ -0,0 +1,46 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.RefreshCounterCache do
@shortdoc "Refreshes counter cache"
use Mix.Task
alias Pleroma.Activity
alias Pleroma.CounterCache
alias Pleroma.Repo
require Logger
import Ecto.Query
def run([]) do
Mix.Pleroma.start_pleroma()
["public", "unlisted", "private", "direct"]
|> Enum.each(fn visibility ->
count = status_visibility_count_query(visibility)
name = "status_visibility_#{visibility}"
CounterCache.set(name, count)
Mix.Pleroma.shell_info("Set #{name} to #{count}")
end)
Mix.Pleroma.shell_info("Done")
end
defp status_visibility_count_query(visibility) do
Activity
|> where(
[a],
fragment(
"activity_visibility(?, ?, ?) = ?",
a.actor,
a.recipients,
a.data,
^visibility
)
)
|> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
|> Repo.aggregate(:count, :id, timeout: :timer.minutes(30))
end
end

View file

@ -0,0 +1,41 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.CounterCache do
alias Pleroma.CounterCache
alias Pleroma.Repo
use Ecto.Schema
import Ecto.Changeset
import Ecto.Query
schema "counter_cache" do
field(:name, :string)
field(:count, :integer)
end
def changeset(struct, params) do
struct
|> cast(params, [:name, :count])
|> validate_required([:name])
|> unique_constraint(:name)
end
def get_as_map(names) when is_list(names) do
CounterCache
|> where([cc], cc.name in ^names)
|> Repo.all()
|> Enum.group_by(& &1.name, & &1.count)
|> Map.new(fn {k, v} -> {k, hd(v)} end)
end
def set(name, count) do
%CounterCache{}
|> changeset(%{"name" => name, "count" => count})
|> Repo.insert(
on_conflict: [set: [count: count]],
returning: true,
conflict_target: :name
)
end
end

View file

@ -4,6 +4,7 @@
defmodule Pleroma.Stats do defmodule Pleroma.Stats do
import Ecto.Query import Ecto.Query
alias Pleroma.CounterCache
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.User alias Pleroma.User
@ -96,4 +97,21 @@ defmodule Pleroma.Stats do
} }
} }
end end
def get_status_visibility_count do
counter_cache =
CounterCache.get_as_map([
"status_visibility_public",
"status_visibility_private",
"status_visibility_unlisted",
"status_visibility_direct"
])
%{
public: counter_cache["status_visibility_public"] || 0,
unlisted: counter_cache["status_visibility_unlisted"] || 0,
private: counter_cache["status_visibility_private"] || 0,
direct: counter_cache["status_visibility_direct"] || 0
}
end
end end

View file

@ -13,6 +13,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
alias Pleroma.ModerationLog alias Pleroma.ModerationLog
alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.Plugs.OAuthScopesPlug
alias Pleroma.ReportNote alias Pleroma.ReportNote
alias Pleroma.Stats
alias Pleroma.User alias Pleroma.User
alias Pleroma.UserInviteToken alias Pleroma.UserInviteToken
alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.ActivityPub
@ -98,7 +99,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
plug( plug(
OAuthScopesPlug, OAuthScopesPlug,
%{scopes: ["read"], admin: true} %{scopes: ["read"], admin: true}
when action in [:config_show, :list_log] when action in [:config_show, :list_log, :stats]
) )
plug( plug(
@ -953,6 +954,13 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
conn |> json("") conn |> json("")
end end
def stats(conn, _) do
count = Stats.get_status_visibility_count()
conn
|> json(%{"status_visibility" => count})
end
def errors(conn, {:error, :not_found}) do def errors(conn, {:error, :not_found}) do
conn conn
|> put_status(:not_found) |> put_status(:not_found)

View file

@ -201,6 +201,7 @@ defmodule Pleroma.Web.Router do
get("/moderation_log", AdminAPIController, :list_log) get("/moderation_log", AdminAPIController, :list_log)
post("/reload_emoji", AdminAPIController, :reload_emoji) post("/reload_emoji", AdminAPIController, :reload_emoji)
get("/stats", AdminAPIController, :stats)
end end
scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do

View file

@ -0,0 +1,55 @@
defmodule Pleroma.Repo.Migrations.AddCounterCacheTable do
use Ecto.Migration
def up do
create_if_not_exists table(:counter_cache) do
add(:name, :string, null: false)
add(:count, :bigint, null: false, default: 0)
end
create_if_not_exists(unique_index(:counter_cache, [:name]))
"""
CREATE OR REPLACE FUNCTION update_status_visibility_counter_cache()
RETURNS TRIGGER AS
$$
DECLARE
BEGIN
IF TG_OP = 'INSERT' THEN
IF NEW.data->>'type' = 'Create' THEN
EXECUTE 'INSERT INTO counter_cache (name, count) VALUES (''status_visibility_' || activity_visibility(NEW.actor, NEW.recipients, NEW.data) || ''', 1) ON CONFLICT (name) DO UPDATE SET count = counter_cache.count + 1';
END IF;
RETURN NEW;
ELSIF TG_OP = 'UPDATE' THEN
IF (NEW.data->>'type' = 'Create') and (OLD.data->>'type' = 'Create') and activity_visibility(NEW.actor, NEW.recipients, NEW.data) != activity_visibility(OLD.actor, OLD.recipients, OLD.data) THEN
EXECUTE 'INSERT INTO counter_cache (name, count) VALUES (''status_visibility_' || activity_visibility(NEW.actor, NEW.recipients, NEW.data) || ''', 1) ON CONFLICT (name) DO UPDATE SET count = counter_cache.count + 1';
EXECUTE 'update counter_cache SET count = counter_cache.count - 1 where count > 0 and name = ''status_visibility_' || activity_visibility(OLD.actor, OLD.recipients, OLD.data) || ''';';
END IF;
RETURN NEW;
ELSIF TG_OP = 'DELETE' THEN
IF OLD.data->>'type' = 'Create' THEN
EXECUTE 'update counter_cache SET count = counter_cache.count - 1 where count > 0 and name = ''status_visibility_' || activity_visibility(OLD.actor, OLD.recipients, OLD.data) || ''';';
END IF;
RETURN OLD;
END IF;
END;
$$
LANGUAGE 'plpgsql';
"""
|> execute()
"""
CREATE TRIGGER status_visibility_counter_cache_trigger BEFORE INSERT OR UPDATE of recipients, data OR DELETE ON activities
FOR EACH ROW
EXECUTE PROCEDURE update_status_visibility_counter_cache();
"""
|> execute()
end
def down do
execute("drop trigger if exists status_visibility_counter_cache_trigger on activities")
execute("drop function if exists update_status_visibility_counter_cache()")
drop_if_exists(unique_index(:counter_cache, [:name]))
drop_if_exists(table(:counter_cache))
end
end

70
test/stat_test.exs Normal file
View file

@ -0,0 +1,70 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Pleroma.StateTest do
use Pleroma.DataCase
import Pleroma.Factory
alias Pleroma.Web.CommonAPI
describe "status visibility count" do
test "on new status" do
user = insert(:user)
other_user = insert(:user)
CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
Enum.each(0..1, fn _ ->
CommonAPI.post(user, %{
"visibility" => "unlisted",
"status" => "hey"
})
end)
Enum.each(0..2, fn _ ->
CommonAPI.post(user, %{
"visibility" => "direct",
"status" => "hey @#{other_user.nickname}"
})
end)
Enum.each(0..3, fn _ ->
CommonAPI.post(user, %{
"visibility" => "private",
"status" => "hey"
})
end)
assert %{direct: 3, private: 4, public: 1, unlisted: 2} =
Pleroma.Stats.get_status_visibility_count()
end
test "on status delete" do
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
assert %{public: 1} = Pleroma.Stats.get_status_visibility_count()
CommonAPI.delete(activity.id, user)
assert %{public: 0} = Pleroma.Stats.get_status_visibility_count()
end
test "on status visibility update" do
user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
assert %{public: 1, private: 0} = Pleroma.Stats.get_status_visibility_count()
{:ok, _} = CommonAPI.update_activity_scope(activity.id, %{"visibility" => "private"})
assert %{public: 0, private: 1} = Pleroma.Stats.get_status_visibility_count()
end
test "doesn't count unrelated activities" do
user = insert(:user)
other_user = insert(:user)
{:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
_ = CommonAPI.follow(user, other_user)
CommonAPI.favorite(activity.id, other_user)
CommonAPI.repeat(activity.id, other_user)
assert %{direct: 0, private: 0, public: 1, unlisted: 0} =
Pleroma.Stats.get_status_visibility_count()
end
end
end

View file

@ -0,0 +1,43 @@
# Pleroma: A lightweight social networking server
# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
# SPDX-License-Identifier: AGPL-3.0-only
defmodule Mix.Tasks.Pleroma.RefreshCounterCacheTest do
use Pleroma.DataCase
alias Pleroma.Web.CommonAPI
import ExUnit.CaptureIO, only: [capture_io: 1]
import Pleroma.Factory
test "counts statuses" do
user = insert(:user)
other_user = insert(:user)
CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
Enum.each(0..1, fn _ ->
CommonAPI.post(user, %{
"visibility" => "unlisted",
"status" => "hey"
})
end)
Enum.each(0..2, fn _ ->
CommonAPI.post(user, %{
"visibility" => "direct",
"status" => "hey @#{other_user.nickname}"
})
end)
Enum.each(0..3, fn _ ->
CommonAPI.post(user, %{
"visibility" => "private",
"status" => "hey"
})
end)
assert capture_io(fn -> Mix.Tasks.Pleroma.RefreshCounterCache.run([]) end) =~ "Done\n"
assert %{direct: 3, private: 4, public: 1, unlisted: 2} =
Pleroma.Stats.get_status_visibility_count()
end
end

View file

@ -3545,6 +3545,25 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
assert String.starts_with?(child["group"], ":") assert String.starts_with?(child["group"], ":")
assert child["description"] assert child["description"]
end end
describe "/api/pleroma/admin/stats" do
test "status visibility count", %{conn: conn} do
admin = insert(:user, is_admin: true)
user = insert(:user)
CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
CommonAPI.post(user, %{"visibility" => "unlisted", "status" => "hey"})
CommonAPI.post(user, %{"visibility" => "unlisted", "status" => "hey"})
response =
conn
|> assign(:user, admin)
|> get("/api/pleroma/admin/stats")
|> json_response(200)
assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 2} =
response["status_visibility"]
end
end
end end
# Needed for testing # Needed for testing