Merge branch 'status-visibility-count-stats' into 'develop'

Status count by visibility scope (`GET /api/pleroma/admin/stats` )

See merge request pleroma/pleroma!2097

CHANGELOG.md

@@ -76,6 +76,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - OAuth: admin scopes support (relevant setting: `[:auth, :enforce_oauth_admin_scope_usage]`).
 - Add an option `authorized_fetch_mode` to require HTTP signatures for AP fetches.
 - ActivityPub: support for `replies` collection (output for outgoing federation & fetching on incoming federation).
+- Mix task to refresh counter cache (`mix pleroma.refresh_counter_cache`)
 <details>
   <summary>API Changes</summary>
 
@@ -119,6 +120,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 - Mastodon API: Add `reacted` property to `emoji_reactions`
 - Pleroma API: Add reactions for a single emoji.
 - ActivityPub: `[:activitypub, :note_replies_output_limit]` setting sets the number of note self-replies to output on outgoing federation.
+- Admin API: `GET /api/pleroma/admin/stats` to get status count by visibility scope
 </details>
 
 ### Fixed

docs/API/admin_api.md

@@ -939,3 +939,20 @@ Loads json generated from `config/descriptions.exs`.
 - Params:
   - `nicknames`
 - Response: Array of user nicknames
+
+## `GET /api/pleroma/admin/stats`
+
+### Stats
+
+- Response:
+
+```json
+{
+  "status_visibility": {
+    "direct": 739,
+    "private": 9,
+    "public": 17,
+    "unlisted": 14
+  }
+}
+```

lib/mix/tasks/pleroma/refresh_counter_cache.ex

@@ -0,0 +1,46 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.RefreshCounterCache do
+  @shortdoc "Refreshes counter cache"
+
+  use Mix.Task
+
+  alias Pleroma.Activity
+  alias Pleroma.CounterCache
+  alias Pleroma.Repo
+
+  require Logger
+  import Ecto.Query
+
+  def run([]) do
+    Mix.Pleroma.start_pleroma()
+
+    ["public", "unlisted", "private", "direct"]
+    |> Enum.each(fn visibility ->
+      count = status_visibility_count_query(visibility)
+      name = "status_visibility_#{visibility}"
+      CounterCache.set(name, count)
+      Mix.Pleroma.shell_info("Set #{name} to #{count}")
+    end)
+
+    Mix.Pleroma.shell_info("Done")
+  end
+
+  defp status_visibility_count_query(visibility) do
+    Activity
+    |> where(
+      [a],
+      fragment(
+        "activity_visibility(?, ?, ?) = ?",
+        a.actor,
+        a.recipients,
+        a.data,
+        ^visibility
+      )
+    )
+    |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
+    |> Repo.aggregate(:count, :id, timeout: :timer.minutes(30))
+  end
+end

lib/pleroma/counter_cache.ex

@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.CounterCache do
+  alias Pleroma.CounterCache
+  alias Pleroma.Repo
+  use Ecto.Schema
+  import Ecto.Changeset
+  import Ecto.Query
+
+  schema "counter_cache" do
+    field(:name, :string)
+    field(:count, :integer)
+  end
+
+  def changeset(struct, params) do
+    struct
+    |> cast(params, [:name, :count])
+    |> validate_required([:name])
+    |> unique_constraint(:name)
+  end
+
+  def get_as_map(names) when is_list(names) do
+    CounterCache
+    |> where([cc], cc.name in ^names)
+    |> Repo.all()
+    |> Enum.group_by(& &1.name, & &1.count)
+    |> Map.new(fn {k, v} -> {k, hd(v)} end)
+  end
+
+  def set(name, count) do
+    %CounterCache{}
+    |> changeset(%{"name" => name, "count" => count})
+    |> Repo.insert(
+      on_conflict: [set: [count: count]],
+      returning: true,
+      conflict_target: :name
+    )
+  end
+end

lib/pleroma/stats.ex

@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Stats do
   import Ecto.Query
+  alias Pleroma.CounterCache
   alias Pleroma.Repo
   alias Pleroma.User
 
@@ -96,4 +97,21 @@ defp get_stat_data do
       }
     }
   end
+
+  def get_status_visibility_count do
+    counter_cache =
+      CounterCache.get_as_map([
+        "status_visibility_public",
+        "status_visibility_private",
+        "status_visibility_unlisted",
+        "status_visibility_direct"
+      ])
+
+    %{
+      public: counter_cache["status_visibility_public"] || 0,
+      unlisted: counter_cache["status_visibility_unlisted"] || 0,
+      private: counter_cache["status_visibility_private"] || 0,
+      direct: counter_cache["status_visibility_direct"] || 0
+    }
+  end
 end

lib/pleroma/web/admin_api/admin_api_controller.ex

@@ -13,6 +13,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   alias Pleroma.ModerationLog
   alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.ReportNote
+  alias Pleroma.Stats
   alias Pleroma.User
   alias Pleroma.UserInviteToken
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -98,7 +99,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   plug(
     OAuthScopesPlug,
     %{scopes: ["read"], admin: true}
-    when action in [:config_show, :list_log]
+    when action in [:config_show, :list_log, :stats]
   )
 
   plug(
@@ -953,6 +954,13 @@ def resend_confirmation_email(%{assigns: %{user: admin}} = conn, %{"nicknames" =
     conn |> json("")
   end
 
+  def stats(conn, _) do
+    count = Stats.get_status_visibility_count()
+
+    conn
+    |> json(%{"status_visibility" => count})
+  end
+
   def errors(conn, {:error, :not_found}) do
     conn
     |> put_status(:not_found)

lib/pleroma/web/router.ex

@@ -201,6 +201,7 @@ defmodule Pleroma.Web.Router do
     get("/moderation_log", AdminAPIController, :list_log)
 
     post("/reload_emoji", AdminAPIController, :reload_emoji)
+    get("/stats", AdminAPIController, :stats)
   end
 
   scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do

priv/repo/migrations/20200109123126_add_counter_cache_table.exs

@@ -0,0 +1,55 @@
+defmodule Pleroma.Repo.Migrations.AddCounterCacheTable do
+  use Ecto.Migration
+
+  def up do
+    create_if_not_exists table(:counter_cache) do
+      add(:name, :string, null: false)
+      add(:count, :bigint, null: false, default: 0)
+    end
+
+    create_if_not_exists(unique_index(:counter_cache, [:name]))
+
+    """
+    CREATE OR REPLACE FUNCTION update_status_visibility_counter_cache()
+    RETURNS TRIGGER AS
+    $$
+      DECLARE
+      BEGIN
+      IF TG_OP = 'INSERT' THEN
+          IF NEW.data->>'type' = 'Create' THEN
+            EXECUTE 'INSERT INTO counter_cache (name, count) VALUES (''status_visibility_' || activity_visibility(NEW.actor, NEW.recipients, NEW.data) || ''', 1) ON CONFLICT (name) DO UPDATE SET count = counter_cache.count + 1';
+          END IF;
+          RETURN NEW;
+      ELSIF TG_OP = 'UPDATE' THEN
+          IF (NEW.data->>'type' = 'Create') and (OLD.data->>'type' = 'Create') and activity_visibility(NEW.actor, NEW.recipients, NEW.data) != activity_visibility(OLD.actor, OLD.recipients, OLD.data) THEN
+             EXECUTE 'INSERT INTO counter_cache (name, count) VALUES (''status_visibility_' || activity_visibility(NEW.actor, NEW.recipients, NEW.data) || ''', 1) ON CONFLICT (name) DO UPDATE SET count = counter_cache.count + 1';
+             EXECUTE 'update counter_cache SET count = counter_cache.count - 1 where count > 0 and name = ''status_visibility_' || activity_visibility(OLD.actor, OLD.recipients, OLD.data) || ''';';
+          END IF;
+          RETURN NEW;
+      ELSIF TG_OP = 'DELETE' THEN
+          IF OLD.data->>'type' = 'Create' THEN
+            EXECUTE 'update counter_cache SET count = counter_cache.count - 1 where count > 0 and name = ''status_visibility_' || activity_visibility(OLD.actor, OLD.recipients, OLD.data) || ''';';
+          END IF;
+          RETURN OLD;
+      END IF;
+      END;
+    $$
+    LANGUAGE 'plpgsql';
+    """
+    |> execute()
+
+    """
+    CREATE TRIGGER status_visibility_counter_cache_trigger BEFORE INSERT OR UPDATE of recipients, data OR DELETE ON activities
+    FOR EACH ROW
+    EXECUTE PROCEDURE update_status_visibility_counter_cache();
+    """
+    |> execute()
+  end
+
+  def down do
+    execute("drop trigger if exists status_visibility_counter_cache_trigger on activities")
+    execute("drop function if exists update_status_visibility_counter_cache()")
+    drop_if_exists(unique_index(:counter_cache, [:name]))
+    drop_if_exists(table(:counter_cache))
+  end
+end

test/stat_test.exs

@@ -0,0 +1,70 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.StateTest do
+  use Pleroma.DataCase
+  import Pleroma.Factory
+  alias Pleroma.Web.CommonAPI
+
+  describe "status visibility count" do
+    test "on new status" do
+      user = insert(:user)
+      other_user = insert(:user)
+
+      CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+
+      Enum.each(0..1, fn _ ->
+        CommonAPI.post(user, %{
+          "visibility" => "unlisted",
+          "status" => "hey"
+        })
+      end)
+
+      Enum.each(0..2, fn _ ->
+        CommonAPI.post(user, %{
+          "visibility" => "direct",
+          "status" => "hey @#{other_user.nickname}"
+        })
+      end)
+
+      Enum.each(0..3, fn _ ->
+        CommonAPI.post(user, %{
+          "visibility" => "private",
+          "status" => "hey"
+        })
+      end)
+
+      assert %{direct: 3, private: 4, public: 1, unlisted: 2} =
+               Pleroma.Stats.get_status_visibility_count()
+    end
+
+    test "on status delete" do
+      user = insert(:user)
+      {:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+      assert %{public: 1} = Pleroma.Stats.get_status_visibility_count()
+      CommonAPI.delete(activity.id, user)
+      assert %{public: 0} = Pleroma.Stats.get_status_visibility_count()
+    end
+
+    test "on status visibility update" do
+      user = insert(:user)
+      {:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+      assert %{public: 1, private: 0} = Pleroma.Stats.get_status_visibility_count()
+      {:ok, _} = CommonAPI.update_activity_scope(activity.id, %{"visibility" => "private"})
+      assert %{public: 0, private: 1} = Pleroma.Stats.get_status_visibility_count()
+    end
+
+    test "doesn't count unrelated activities" do
+      user = insert(:user)
+      other_user = insert(:user)
+      {:ok, activity} = CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+      _ = CommonAPI.follow(user, other_user)
+      CommonAPI.favorite(activity.id, other_user)
+      CommonAPI.repeat(activity.id, other_user)
+
+      assert %{direct: 0, private: 0, public: 1, unlisted: 0} =
+               Pleroma.Stats.get_status_visibility_count()
+    end
+  end
+end

test/tasks/refresh_counter_cache_test.exs

@@ -0,0 +1,43 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.RefreshCounterCacheTest do
+  use Pleroma.DataCase
+  alias Pleroma.Web.CommonAPI
+  import ExUnit.CaptureIO, only: [capture_io: 1]
+  import Pleroma.Factory
+
+  test "counts statuses" do
+    user = insert(:user)
+    other_user = insert(:user)
+
+    CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+
+    Enum.each(0..1, fn _ ->
+      CommonAPI.post(user, %{
+        "visibility" => "unlisted",
+        "status" => "hey"
+      })
+    end)
+
+    Enum.each(0..2, fn _ ->
+      CommonAPI.post(user, %{
+        "visibility" => "direct",
+        "status" => "hey @#{other_user.nickname}"
+      })
+    end)
+
+    Enum.each(0..3, fn _ ->
+      CommonAPI.post(user, %{
+        "visibility" => "private",
+        "status" => "hey"
+      })
+    end)
+
+    assert capture_io(fn -> Mix.Tasks.Pleroma.RefreshCounterCache.run([]) end) =~ "Done\n"
+
+    assert %{direct: 3, private: 4, public: 1, unlisted: 2} =
+             Pleroma.Stats.get_status_visibility_count()
+  end
+end

test/web/admin_api/admin_api_controller_test.exs

@@ -3545,6 +3545,25 @@ test "GET /api/pleroma/admin/config/descriptions", %{conn: conn} do
     assert String.starts_with?(child["group"], ":")
     assert child["description"]
   end
+
+  describe "/api/pleroma/admin/stats" do
+    test "status visibility count", %{conn: conn} do
+      admin = insert(:user, is_admin: true)
+      user = insert(:user)
+      CommonAPI.post(user, %{"visibility" => "public", "status" => "hey"})
+      CommonAPI.post(user, %{"visibility" => "unlisted", "status" => "hey"})
+      CommonAPI.post(user, %{"visibility" => "unlisted", "status" => "hey"})
+
+      response =
+        conn
+        |> assign(:user, admin)
+        |> get("/api/pleroma/admin/stats")
+        |> json_response(200)
+
+      assert %{"direct" => 0, "private" => 0, "public" => 1, "unlisted" => 2} =
+               response["status_visibility"]
+    end
+  end
 end
 
 # Needed for testing