From db690bede92e9ba65b1afaa8605aa5ddb3c992a4 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 2 Aug 2019 21:33:12 +0300 Subject: [PATCH 01/36] temp commit --- config/benchmark.exs | 88 +++++++++++++++++++ lib/load_testing/fetcher.ex | 116 ++++++++++++++++++++++++++ lib/load_testing/generator.ex | 83 ++++++++++++++++++ lib/load_testing/helper.ex | 16 ++++ lib/mix/tasks/pleroma/load_testing.ex | 100 ++++++++++++++++++++++ 5 files changed, 403 insertions(+) create mode 100644 config/benchmark.exs create mode 100644 lib/load_testing/fetcher.ex create mode 100644 lib/load_testing/generator.ex create mode 100644 lib/load_testing/helper.ex create mode 100644 lib/mix/tasks/pleroma/load_testing.ex diff --git a/config/benchmark.exs b/config/benchmark.exs new file mode 100644 index 000000000..b4f5dbdbd --- /dev/null +++ b/config/benchmark.exs @@ -0,0 +1,88 @@ +use Mix.Config + +# We don't run a server during test. If one is required, +# you can enable the server option below. +config :pleroma, Pleroma.Web.Endpoint, + http: [port: 4001], + url: [port: 4001], + server: true + +# Disable captha for tests +config :pleroma, Pleroma.Captcha, + # It should not be enabled for automatic tests + enabled: false, + # A fake captcha service for tests + method: Pleroma.Captcha.Mock + +# Print only warnings and errors during test +config :logger, level: :warn + +config :pleroma, :auth, oauth_consumer_strategies: [] + +config :pleroma, Pleroma.Upload, filters: [], link_name: false + +config :pleroma, Pleroma.Uploaders.Local, uploads: "test/uploads" + +config :pleroma, Pleroma.Emails.Mailer, adapter: Swoosh.Adapters.Test, enabled: true + +config :pleroma, :instance, + email: "admin@example.com", + notify_email: "noreply@example.com", + skip_thread_containment: false, + federating: false + +config :pleroma, :activitypub, sign_object_fetches: false + +# Configure your database +config :pleroma, Pleroma.Repo, + adapter: Ecto.Adapters.Postgres, + username: System.get_env("DB_USER") || "postgres", + database: System.get_env("DB_DATABASE") || "pleroma_test", + hostname: System.get_env("DB_HOST") || "localhost", + # username: "pleroma", + password: + System.get_env("DB_PASS") || + "cAUrGezwXjRwd/lIPzZAcwjb/hiZiGi3FIaSGy9l/XsTcGA61FMy7eCBiRcg1DyQ", + # password: "", + pool_size: 10, + timeout: 180_000 + +# Reduce hash rounds for testing +config :pbkdf2_elixir, rounds: 1 + +config :tesla, adapter: Tesla.Mock + +config :pleroma, :rich_media, + enabled: false, + ignore_hosts: [], + ignore_tld: ["local", "localdomain", "lan"] + +config :web_push_encryption, :vapid_details, + subject: "mailto:administrator@example.com", + public_key: + "BLH1qVhJItRGCfxgTtONfsOKDc9VRAraXw-3NsmjMngWSh7NxOizN6bkuRA7iLTMPS82PjwJAr3UoK9EC1IFrz4", + private_key: "_-XZ0iebPrRfZ_o0-IatTdszYa8VCH1yLN-JauK7HHA" + +config :web_push_encryption, :http_client, Pleroma.Web.WebPushHttpClientMock + +config :pleroma_job_queue, disabled: true + +config :pleroma, Pleroma.ScheduledActivity, + daily_user_limit: 2, + total_user_limit: 3, + enabled: false + +config :pleroma, :rate_limit, + search: [{1000, 30}, {1000, 30}], + app_account_creation: {10_000, 5}, + password_reset: {1000, 30} + +config :pleroma, :http_security, report_uri: "https://endpoint.com" + +config :pleroma, :http, send_user_agent: false + +rum_enabled = System.get_env("RUM_ENABLED") == "true" +config :pleroma, :database, rum_enabled: rum_enabled +IO.puts("RUM enabled: #{rum_enabled}") + +config :pleroma, Pleroma.ReverseProxy.Client, Pleroma.ReverseProxy.ClientMock diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex new file mode 100644 index 000000000..70c0fcd0c --- /dev/null +++ b/lib/load_testing/fetcher.ex @@ -0,0 +1,116 @@ +defmodule Pleroma.LoadTesting.Fetcher do + use Pleroma.LoadTesting.Helper + + def fetch_user(user) do + IO.puts("=================================") + + {time, _value} = :timer.tc(fn -> Repo.get_by(User, id: user.id) end) + + IO.puts("Query user by id: #{to_sec(time)} sec.") + + {time, _value} = + :timer.tc(fn -> + Repo.get_by(User, ap_id: user.ap_id) + end) + + IO.puts("Query user by ap_id: #{to_sec(time)} sec.") + + {time, _value} = + :timer.tc(fn -> + Repo.get_by(User, email: user.email) + end) + + IO.puts("Query user by email: #{to_sec(time)} sec.") + + {time, _value} = :timer.tc(fn -> Repo.get_by(User, nickname: user.nickname) end) + + IO.puts("Query user by nickname: #{to_sec(time)} sec.") + end + + def query_timelines(user) do + IO.puts("\n=================================") + + params = %{ + "count" => 20, + "with_muted" => true, + "type" => ["Create", "Announce"], + "blocking_user" => user, + "muting_user" => user, + "user" => user + } + + {time, _} = + :timer.tc(fn -> + ActivityPub.ActivityPub.fetch_activities([user.ap_id | user.following], params) + end) + + IO.puts("Query user home timeline: #{to_sec(time)} sec.") + + params = %{ + "count" => 20, + "local_only" => true, + "only_media" => "false", + "type" => ["Create", "Announce"], + "with_muted" => "true", + "blocking_user" => user, + "muting_user" => user + } + + {time, _} = + :timer.tc(fn -> + ActivityPub.ActivityPub.fetch_public_activities(params) + end) + + IO.puts("Query user mastodon public timeline: #{to_sec(time)} sec.") + + params = %{ + "count" => 20, + "only_media" => "false", + "type" => ["Create", "Announce"], + "with_muted" => "true", + "blocking_user" => user, + "muting_user" => user + } + + {time, _} = + :timer.tc(fn -> + ActivityPub.ActivityPub.fetch_public_activities(params) + end) + + IO.puts("Query user mastodon federated public timeline: #{to_sec(time)} sec.") + end + + def query_notifications(user) do + IO.puts("\n=================================") + params = %{"count" => "20", "with_muted" => "false"} + + {time, _} = + :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end) + + IO.puts("Query user notifications with out muted: #{to_sec(time)} sec.") + + params = %{"count" => "20", "with_muted" => "true"} + + {time, _} = + :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end) + + IO.puts("Query user notifications with muted: #{to_sec(time)} sec.") + end + + def query_long_thread(user, activity) do + IO.puts("\n=================================") + + {time, replies} = + :timer.tc(fn -> + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context( + activity.data["context"], + %{ + "blocking_user" => user, + "user" => user + } + ) + end) + + IO.puts("Query long thread with #{length(replies)} replies: #{to_sec(time)} sec.") + end +end diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex new file mode 100644 index 000000000..a9016b9e8 --- /dev/null +++ b/lib/load_testing/generator.ex @@ -0,0 +1,83 @@ +defmodule Pleroma.LoadTesting.Generator do + use Pleroma.LoadTesting.Helper + + def generate_users(opts) do + IO.puts("Starting generating #{opts[:users_max]} users...") + {time, _} = :timer.tc(fn -> do_generate_users(opts) end) + IO.puts("Inserting users take #{to_sec(time)} sec.\n") + end + + defp do_generate_users(opts) do + min = Keyword.get(opts, :users_min, 1) + max = Keyword.get(opts, :users_max) + + query = + "INSERT INTO \"users\" (\"ap_id\",\"bio\",\"email\",\"follower_address\",\"following\",\"following_address\",\"info\", + \"local\",\"name\",\"nickname\",\"password_hash\",\"tags\",\"id\",\"inserted_at\",\"updated_at\") VALUES \n" + + users = + Task.async_stream( + min..max, + &generate_user_data(&1), + max_concurrency: 10, + timeout: 30_000 + ) + |> Enum.reduce("", fn {:ok, data}, acc -> acc <> data <> ", \n" end) + + query = query <> String.replace_trailing(users, ", \n", ";") + + Ecto.Adapters.SQL.query!(Repo, query) + end + + defp generate_user_data(i) do + user = %User{ + name: "Test テスト User #{i}", + email: "user#{i}@example.com", + nickname: "nick#{i}", + password_hash: Comeonin.Pbkdf2.hashpwsalt("test"), + bio: "Tester Number #{i}", + info: %{} + } + + user = %{ + user + | ap_id: User.ap_id(user), + follower_address: User.ap_followers(user), + following_address: User.ap_following(user), + following: [User.ap_id(user)] + } + + "('#{user.ap_id}', '#{user.bio}', '#{user.email}', '#{user.follower_address}', '{#{ + user.following + }}', '#{user.following_address}', '#{Jason.encode!(user.info)}', '#{user.local}', '#{ + user.name + }', '#{user.nickname}', '#{user.password_hash}', '{#{user.tags}}', uuid_generate_v4(), NOW(), NOW())" + end + + def generate_activities(users, opts) do + IO.puts("Starting generating #{opts[:activities_max]} activities...") + {time, _} = :timer.tc(fn -> do_generate_activities(users, opts) end) + IO.puts("Inserting activities take #{to_sec(time)} sec.\n") + end + + defp do_generate_activities(users, opts) do + Task.async_stream( + 1..opts[:activities_max], + fn _ -> + do_generate_activity(users, opts) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end + + defp do_generate_activity(users, opts) do + status = + if opts[:mention], + do: "some status with @#{opts[:mention].nickname}", + else: "some status" + + Pleroma.Web.CommonAPI.post(Enum.random(users), %{"status" => status}) + end +end diff --git a/lib/load_testing/helper.ex b/lib/load_testing/helper.ex new file mode 100644 index 000000000..338dba323 --- /dev/null +++ b/lib/load_testing/helper.ex @@ -0,0 +1,16 @@ +defmodule Pleroma.LoadTesting.Helper do + defmacro __using__(_) do + quote do + import Ecto.Query + alias Pleroma.Activity + alias Pleroma.Notification + alias Pleroma.Object + alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.Web.ActivityPub + alias Pleroma.Web.CommonAPI + + defp to_sec(microseconds), do: microseconds / 1_000_000 + end + end +end diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex new file mode 100644 index 000000000..9ed30db7e --- /dev/null +++ b/lib/mix/tasks/pleroma/load_testing.ex @@ -0,0 +1,100 @@ +defmodule Mix.Tasks.Pleroma.LoadTesting do + use Mix.Task + use Pleroma.LoadTesting.Helper + import Mix.Pleroma + import Pleroma.LoadTesting.Generator + import Pleroma.LoadTesting.Fetcher + + # tODO: remove autovacuum worker until generation is not ended + @shortdoc "Factory for generation data" + @moduledoc """ + Generates data like: + - users + - activities with notifications + + ## Generate data + MIX_ENV=test mix pleroma.load_testing --users 10000 --activities 20000 + MIX_ENV=test mix pleroma.load_testing -u 10000 -a 20000 + + Options: + - `--users NUMBER` - number of users to generate (default: 10000) + - `--activities NUMBER` - number of activities to generate (default: 20000) + """ + + @aliases [u: :users, a: :activities, d: :delete] + @switches [users: :integer, activities: :integer, delete: :boolean] + @users_default 20_000 + @activities_default 50_000 + + def run(args) do + {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) + start_pleroma() + + current_max = Keyword.get(opts, :users, @users_default) + activities_max = Keyword.get(opts, :activities, @activities_default) + + {users_min, users_max} = + if opts[:delete] do + clean_tables() + {1, current_max} + else + current_count = Repo.aggregate(from(u in User), :count, :id) + 1 + {current_count, current_max + current_count} + end + + opts = + Keyword.put(opts, :users_min, users_min) + |> Keyword.put(:users_max, users_max) + |> Keyword.put(:activities_max, activities_max) + + generate_users(opts) + + # main user for queries + IO.puts("Fetching main user...") + + {time, user} = + :timer.tc(fn -> Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1)) end) + + IO.puts("Fetching main user take #{to_sec(time)} sec.\n") + + IO.puts("Fetching users...") + + {time, users} = + :timer.tc(fn -> + Repo.all( + from(u in User, + where: u.id != ^user.id, + order_by: fragment("RANDOM()"), + limit: 10 + ) + ) + end) + + IO.puts("Fetching users take #{to_sec(time)} sec.\n") + + generate_activities(users, opts) + + generate_activities(users, Keyword.put(opts, :mention, user)) + + # generate_replies(user, users, activities) + + # activity = Enum.random(activities) + # generate_long_thread(user, users, activity) + + IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}") + IO.puts("Activities in DB: #{Repo.aggregate(from(a in Activity), :count, :id)}") + IO.puts("Objects in DB: #{Repo.aggregate(from(o in Object), :count, :id)}") + IO.puts("Notifications in DB: #{Repo.aggregate(from(n in Notification), :count, :id)}") + + query_timelines(user) + query_notifications(user) + # query_long_thread(user, activity) + end + + defp clean_tables do + IO.puts("\n\nDeleting old data...\n") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;") + Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;") + end +end From 79dde5804427ab24c5bc9aa1b5a44d2259cecc6d Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 4 Sep 2019 20:18:11 +0300 Subject: [PATCH 02/36] one more temp commit --- lib/load_testing/fetcher.ex | 120 +++++++++--------- lib/load_testing/generator.ex | 127 +++++++++++++++---- lib/mix/tasks/pleroma/load_testing.ex | 46 ++++--- lib/pleroma/web/activity_pub/activity_pub.ex | 5 + 4 files changed, 202 insertions(+), 96 deletions(-) diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex index 70c0fcd0c..ed744db9b 100644 --- a/lib/load_testing/fetcher.ex +++ b/lib/load_testing/fetcher.ex @@ -4,33 +4,18 @@ defmodule Pleroma.LoadTesting.Fetcher do def fetch_user(user) do IO.puts("=================================") - {time, _value} = :timer.tc(fn -> Repo.get_by(User, id: user.id) end) - - IO.puts("Query user by id: #{to_sec(time)} sec.") - - {time, _value} = - :timer.tc(fn -> - Repo.get_by(User, ap_id: user.ap_id) - end) - - IO.puts("Query user by ap_id: #{to_sec(time)} sec.") - - {time, _value} = - :timer.tc(fn -> - Repo.get_by(User, email: user.email) - end) - - IO.puts("Query user by email: #{to_sec(time)} sec.") - - {time, _value} = :timer.tc(fn -> Repo.get_by(User, nickname: user.nickname) end) - - IO.puts("Query user by nickname: #{to_sec(time)} sec.") + Benchee.run(%{ + "By id" => fn -> Repo.get_by(User, id: user.id) end, + "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end, + "By email" => fn -> Repo.get_by(User, email: user.email) end, + "By nickname" => fn -> Repo.get_by(User, nickname: user.nickname) end + }) end def query_timelines(user) do IO.puts("\n=================================") - params = %{ + home_timeline_params = %{ "count" => 20, "with_muted" => true, "type" => ["Create", "Announce"], @@ -39,14 +24,7 @@ def query_timelines(user) do "user" => user } - {time, _} = - :timer.tc(fn -> - ActivityPub.ActivityPub.fetch_activities([user.ap_id | user.following], params) - end) - - IO.puts("Query user home timeline: #{to_sec(time)} sec.") - - params = %{ + mastodon_public_timeline_params = %{ "count" => 20, "local_only" => true, "only_media" => "false", @@ -56,14 +34,7 @@ def query_timelines(user) do "muting_user" => user } - {time, _} = - :timer.tc(fn -> - ActivityPub.ActivityPub.fetch_public_activities(params) - end) - - IO.puts("Query user mastodon public timeline: #{to_sec(time)} sec.") - - params = %{ + mastodon_federated_timeline_params = %{ "count" => 20, "only_media" => "false", "type" => ["Create", "Announce"], @@ -72,45 +43,76 @@ def query_timelines(user) do "muting_user" => user } - {time, _} = - :timer.tc(fn -> - ActivityPub.ActivityPub.fetch_public_activities(params) - end) - - IO.puts("Query user mastodon federated public timeline: #{to_sec(time)} sec.") + Benchee.run(%{ + "User home timeline" => fn -> + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities( + [user.ap_id | user.following], + home_timeline_params + ) + end, + "User mastodon public timeline" => fn -> + ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params) + end, + "User mastodon federated public timeline" => fn -> + ActivityPub.ActivityPub.fetch_public_activities(mastodon_federated_timeline_params) + end + }) end def query_notifications(user) do IO.puts("\n=================================") - params = %{"count" => "20", "with_muted" => "false"} + without_muted_params = %{"count" => "20", "with_muted" => "false"} + with_muted_params = %{"count" => "20", "with_muted" => "true"} - {time, _} = - :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end) + Benchee.run(%{ + "Notifications without muted" => fn -> + Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params) + end, + "Notifications with muted" => fn -> + Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params) + end + }) + end - IO.puts("Query user notifications with out muted: #{to_sec(time)} sec.") + def query_dms(user) do + IO.puts("\n=================================") - params = %{"count" => "20", "with_muted" => "true"} + params = %{ + "count" => "20", + "with_muted" => "true", + "type" => "Create", + "blocking_user" => user, + "user" => user, + visibility: "direct" + } - {time, _} = - :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end) - - IO.puts("Query user notifications with muted: #{to_sec(time)} sec.") + Benchee.run(%{ + "Direct messages with muted" => fn -> + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params) + |> Pleroma.Pagination.fetch_paginated(params) + end, + "Direct messages without muted" => fn -> + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params) + |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false)) + end + }) end def query_long_thread(user, activity) do IO.puts("\n=================================") - {time, replies} = - :timer.tc(fn -> + Benchee.run(%{ + "Fetch main post" => fn -> Activity.get_by_id_with_object(activity.id) end, + "Fetch context of main post" => fn -> Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context( activity.data["context"], %{ "blocking_user" => user, - "user" => user + "user" => user, + "exclude_id" => activity.id } ) - end) - - IO.puts("Query long thread with #{length(replies)} replies: #{to_sec(time)} sec.") + end + }) end end diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex index a9016b9e8..7f50ee68e 100644 --- a/lib/load_testing/generator.ex +++ b/lib/load_testing/generator.ex @@ -8,25 +8,15 @@ def generate_users(opts) do end defp do_generate_users(opts) do - min = Keyword.get(opts, :users_min, 1) max = Keyword.get(opts, :users_max) - query = - "INSERT INTO \"users\" (\"ap_id\",\"bio\",\"email\",\"follower_address\",\"following\",\"following_address\",\"info\", - \"local\",\"name\",\"nickname\",\"password_hash\",\"tags\",\"id\",\"inserted_at\",\"updated_at\") VALUES \n" - - users = - Task.async_stream( - min..max, - &generate_user_data(&1), - max_concurrency: 10, - timeout: 30_000 - ) - |> Enum.reduce("", fn {:ok, data}, acc -> acc <> data <> ", \n" end) - - query = query <> String.replace_trailing(users, ", \n", ";") - - Ecto.Adapters.SQL.query!(Repo, query) + Task.async_stream( + 1..max, + &generate_user_data(&1), + max_concurrency: 10, + timeout: 30_000 + ) + |> Enum.to_list() end defp generate_user_data(i) do @@ -47,11 +37,7 @@ defp generate_user_data(i) do following: [User.ap_id(user)] } - "('#{user.ap_id}', '#{user.bio}', '#{user.email}', '#{user.follower_address}', '{#{ - user.following - }}', '#{user.following_address}', '#{Jason.encode!(user.info)}', '#{user.local}', '#{ - user.name - }', '#{user.nickname}', '#{user.password_hash}', '{#{user.tags}}', uuid_generate_v4(), NOW(), NOW())" + Pleroma.Repo.insert!(user) end def generate_activities(users, opts) do @@ -80,4 +66,101 @@ defp do_generate_activity(users, opts) do Pleroma.Web.CommonAPI.post(Enum.random(users), %{"status" => status}) end + + def generate_dms(user, users, opts) do + IO.puts("Starting generating #{opts[:dms_max]} DMs") + {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end) + IO.puts("Inserting dms take #{to_sec(time)} sec.\n") + end + + defp do_generate_dms(user, users, opts) do + Task.async_stream( + 1..opts[:dms_max], + fn _ -> + do_generate_dm(user, users) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end + + defp do_generate_dm(user, users) do + post = %{ + "status" => "@#{user.nickname} some direct message", + "visibility" => "direct" + } + + Pleroma.Web.CommonAPI.post(Enum.random(users), post) + end + + def generate_long_thread(user, users, opts) do + IO.puts("Starting generating long thread with #{opts[:long_thread_length]} replies") + {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end) + IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n") + {:ok, activity} + end + + defp do_generate_long_thread(user, users, opts) do + {:ok, %{id: id} = activity} = + Pleroma.Web.CommonAPI.post(user, %{"status" => "Start of long thread"}) + + Task.async_stream( + 1..opts[:long_thread_length], + fn _ -> do_generate_thread(users, id) end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + + activity + end + + defp do_generate_thread(users, activity_id) do + Pleroma.Web.CommonAPI.post(Enum.random(users), %{ + "status" => "reply to main post", + "in_reply_to_status_id" => activity_id + }) + end + + def generate_private_thread(users, opts) do + IO.puts("Starting generating long thread with #{opts[:non_visible_posts_max]} replies") + {time, _} = :timer.tc(fn -> do_generate_non_visible_posts(users, opts) end) + IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n") + end + + defp do_generate_non_visible_posts(users, opts) do + [user1, user2] = Enum.take(users, 2) + {:ok, user1} = Pleroma.User.follow(user1, user2) + {:ok, user2} = Pleroma.User.follow(user2, user1) + + {:ok, activity} = + Pleroma.Web.CommonAPI.post(user1, %{ + "status" => "Some private post", + "visibility" => "private" + }) + + {:ok, activity_public} = + Pleroma.Web.CommonAPI.post(user2, %{ + "status" => "Some public reply", + "in_reply_to_status_id" => activity.id + }) + + Task.async_stream( + 1..opts[:non_visible_posts_max], + fn _ -> do_generate_non_visible_post(users, activity_public) end, + max_concurrency: 10, + timeout: 30_000 + ) + end + + defp do_generate_non_visible_post(users, activity) do + visibility = Enum.random(["private", "public"]) + + Pleroma.Web.CommonAPI.post(Enum.random(users), %{ + "visibility" => visibility, + "status" => "Some #{visibility} reply", + "in_reply_to_status_id" => activity.id + }) + end end diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex index 9ed30db7e..d17cf0b3e 100644 --- a/lib/mix/tasks/pleroma/load_testing.ex +++ b/lib/mix/tasks/pleroma/load_testing.ex @@ -21,31 +21,38 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do - `--activities NUMBER` - number of activities to generate (default: 20000) """ - @aliases [u: :users, a: :activities, d: :delete] - @switches [users: :integer, activities: :integer, delete: :boolean] + @aliases [u: :users, a: :activities] + @switches [ + users: :integer, + activities: :integer, + dms: :integer, + thread_length: :integer, + non_visible_posts: :integer + ] @users_default 20_000 @activities_default 50_000 + @dms_default 50_000 + @thread_length_default 2_000 + @non_visible_posts_default 2_000 def run(args) do - {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) start_pleroma() + {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) - current_max = Keyword.get(opts, :users, @users_default) + users_max = Keyword.get(opts, :users, @users_default) activities_max = Keyword.get(opts, :activities, @activities_default) + dms_max = Keyword.get(opts, :dms, @dms_default) + long_thread_length = Keyword.get(opts, :thread_length, @thread_length_default) + non_visible_posts = Keyword.get(opts, :non_visible_posts, @non_visible_posts_default) - {users_min, users_max} = - if opts[:delete] do - clean_tables() - {1, current_max} - else - current_count = Repo.aggregate(from(u in User), :count, :id) + 1 - {current_count, current_max + current_count} - end + clean_tables() opts = - Keyword.put(opts, :users_min, users_min) - |> Keyword.put(:users_max, users_max) + Keyword.put(opts, :users_max, users_max) |> Keyword.put(:activities_max, activities_max) + |> Keyword.put(:dms_max, dms_max) + |> Keyword.put(:long_thread_length, long_thread_length) + |> Keyword.put(:non_visible_posts_max, non_visible_posts) generate_users(opts) @@ -76,6 +83,12 @@ def run(args) do generate_activities(users, Keyword.put(opts, :mention, user)) + generate_dms(user, users, opts) + + {:ok, activity} = generate_long_thread(user, users, opts) + + generate_private_thread(users, opts) + # generate_replies(user, users, activities) # activity = Enum.random(activities) @@ -86,9 +99,12 @@ def run(args) do IO.puts("Objects in DB: #{Repo.aggregate(from(o in Object), :count, :id)}") IO.puts("Notifications in DB: #{Repo.aggregate(from(n in Notification), :count, :id)}") + fetch_user(user) query_timelines(user) query_notifications(user) - # query_long_thread(user, activity) + query_dms(user) + query_long_thread(user, activity) + query_timelines(user) end defp clean_tables do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index eeb826814..f2b322314 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -249,6 +249,7 @@ def create(%{to: to, actor: actor, context: context, object: object} = params, f # only accept false as false value local = !(params[:local] == false) published = params[:published] + quick_insert? = Pleroma.Config.get([:env]) == :benchmark with create_data <- make_create_data( @@ -259,12 +260,16 @@ def create(%{to: to, actor: actor, context: context, object: object} = params, f {:fake, false, activity} <- {:fake, fake, activity}, _ <- increase_replies_count_if_reply(create_data), _ <- increase_poll_votes_if_vote(create_data), + {:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity}, # Changing note count prior to enqueuing federation task in order to avoid # race conditions on updating user.info {:ok, _actor} <- increase_note_count_if_public(actor, activity), :ok <- maybe_federate(activity) do {:ok, activity} else + {:quick_insert, true, activity} -> + {:ok, activity} + {:fake, true, activity} -> {:ok, activity} From a1125bd5644a5819d1cfbbc8f6b3bb2dadbbc38a Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 5 Sep 2019 16:01:52 +0300 Subject: [PATCH 03/36] generatoin and fetching --- lib/load_testing/fetcher.ex | 126 +++++++++++++++- lib/load_testing/generator.ex | 203 +++++++++++++++++++------- lib/load_testing/helper.ex | 5 - lib/mix/tasks/pleroma/load_testing.ex | 49 +++---- 4 files changed, 290 insertions(+), 93 deletions(-) diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex index ed744db9b..825f921e6 100644 --- a/lib/load_testing/fetcher.ex +++ b/lib/load_testing/fetcher.ex @@ -51,10 +51,52 @@ def query_timelines(user) do ) end, "User mastodon public timeline" => fn -> - ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params) + Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities( + mastodon_public_timeline_params + ) end, "User mastodon federated public timeline" => fn -> - ActivityPub.ActivityPub.fetch_public_activities(mastodon_federated_timeline_params) + Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities( + mastodon_federated_timeline_params + ) + end + }) + + home_activities = + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities( + [user.ap_id | user.following], + home_timeline_params + ) + + public_activities = + Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params) + + public_federated_activities = + Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities( + mastodon_federated_timeline_params + ) + + Benchee.run(%{ + "Rendering home timeline" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: home_activities, + for: user, + as: :activity + }) + end, + "Rendering public timeline" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: public_activities, + for: user, + as: :activity + }) + end, + "Rendering public federated timeline" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: public_federated_activities, + for: user, + as: :activity + }) end }) end @@ -72,6 +114,27 @@ def query_notifications(user) do Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params) end }) + + without_muted_notifications = + Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params) + + with_muted_notifications = + Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params) + + Benchee.run(%{ + "Render notifications without muted" => fn -> + Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{ + notifications: without_muted_notifications, + for: user + }) + end, + "Render notifications with muted" => fn -> + Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{ + notifications: with_muted_notifications, + for: user + }) + end + }) end def query_dms(user) do @@ -96,13 +159,40 @@ def query_dms(user) do |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false)) end }) + + dms_with_muted = + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params) + |> Pleroma.Pagination.fetch_paginated(params) + + dms_without_muted = + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params) + |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false)) + + Benchee.run(%{ + "Rendering dms with muted" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: dms_with_muted, + for: user, + as: :activity + }) + end, + "Rendering dms without muted" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ + activities: dms_without_muted, + for: user, + as: :activity + }) + end + }) end def query_long_thread(user, activity) do IO.puts("\n=================================") Benchee.run(%{ - "Fetch main post" => fn -> Activity.get_by_id_with_object(activity.id) end, + "Fetch main post" => fn -> + Pleroma.Activity.get_by_id_with_object(activity.id) + end, "Fetch context of main post" => fn -> Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context( activity.data["context"], @@ -114,5 +204,35 @@ def query_long_thread(user, activity) do ) end }) + + activity = Pleroma.Activity.get_by_id_with_object(activity.id) + + context = + Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context( + activity.data["context"], + %{ + "blocking_user" => user, + "user" => user, + "exclude_id" => activity.id + } + ) + + Benchee.run(%{ + "Render status" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{ + activity: activity, + for: user + }) + end, + "Render context ancestors" => fn -> + Pleroma.Web.MastodonAPI.StatusView.render( + "index.json", + for: user, + activities: context, + as: :activity + ) + |> Enum.reverse() + end + }) end end diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex index 7f50ee68e..6df518aba 100644 --- a/lib/load_testing/generator.ex +++ b/lib/load_testing/generator.ex @@ -1,9 +1,11 @@ defmodule Pleroma.LoadTesting.Generator do use Pleroma.LoadTesting.Helper + alias Pleroma.Web.CommonAPI def generate_users(opts) do IO.puts("Starting generating #{opts[:users_max]} users...") {time, _} = :timer.tc(fn -> do_generate_users(opts) end) + IO.puts("Inserting users take #{to_sec(time)} sec.\n") end @@ -37,34 +39,111 @@ defp generate_user_data(i) do following: [User.ap_id(user)] } - Pleroma.Repo.insert!(user) + Repo.insert!(user) end - def generate_activities(users, opts) do - IO.puts("Starting generating #{opts[:activities_max]} activities...") - {time, _} = :timer.tc(fn -> do_generate_activities(users, opts) end) - IO.puts("Inserting activities take #{to_sec(time)} sec.\n") + def generate_activities(user, users) do + do_generate_activities(user, users) end - defp do_generate_activities(users, opts) do - Task.async_stream( - 1..opts[:activities_max], - fn _ -> - do_generate_activity(users, opts) - end, - max_concurrency: 10, - timeout: 30_000 - ) - |> Stream.run() + defp do_generate_activities(user, users) do + IO.puts("Starting generating 20000 common activities...") + + {time, _} = + :timer.tc(fn -> + Task.async_stream( + 1..20_000, + fn _ -> + do_generate_activity([user | users]) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end) + + IO.puts("Inserting common activities take #{to_sec(time)} sec.\n") + + IO.puts("Starting generating 20000 activities with mentions...") + + {time, _} = + :timer.tc(fn -> + Task.async_stream( + 1..20_000, + fn _ -> + do_generate_activity_with_mention(user, users) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end) + + IO.puts("Inserting activities with menthions take #{to_sec(time)} sec.\n") + + IO.puts("Starting generating 10000 activities with threads...") + + {time, _} = + :timer.tc(fn -> + Task.async_stream( + 1..10_000, + fn _ -> + do_generate_threads([user | users]) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end) + + IO.puts("Inserting activities with threads take #{to_sec(time)} sec.\n") end - defp do_generate_activity(users, opts) do - status = - if opts[:mention], - do: "some status with @#{opts[:mention].nickname}", - else: "some status" + defp do_generate_activity(users) do + post = %{ + "status" => "Some status without mention with random user" + } - Pleroma.Web.CommonAPI.post(Enum.random(users), %{"status" => status}) + CommonAPI.post(Enum.random(users), post) + end + + defp do_generate_activity_with_mention(user, users) do + mentions_cnt = Enum.random([2, 3, 4, 5]) + with_user = Enum.random([true, false]) + users = Enum.shuffle(users) + mentions_users = Enum.take(users, mentions_cnt) + mentions_users = if with_user, do: [user | mentions_users], else: mentions_users + + mentions_str = + Enum.map(mentions_users, fn user -> "@" <> user.nickname end) |> Enum.join(", ") + + post = %{ + "status" => mentions_str <> "some status with mentions random users" + } + + CommonAPI.post(Enum.random(users), post) + end + + defp do_generate_threads(users) do + thread_length = Enum.random([2, 3, 4, 5]) + actor = Enum.random(users) + + post = %{ + "status" => "Start of the thread" + } + + {:ok, activity} = CommonAPI.post(actor, post) + + Enum.each(1..thread_length, fn _ -> + user = Enum.random(users) + + post = %{ + "status" => "@#{actor.nickname} reply to thread", + "in_reply_to_status_id" => activity.id + } + + CommonAPI.post(user, post) + end) end def generate_dms(user, users, opts) do @@ -91,22 +170,21 @@ defp do_generate_dm(user, users) do "visibility" => "direct" } - Pleroma.Web.CommonAPI.post(Enum.random(users), post) + CommonAPI.post(Enum.random(users), post) end def generate_long_thread(user, users, opts) do - IO.puts("Starting generating long thread with #{opts[:long_thread_length]} replies") + IO.puts("Starting generating long thread with #{opts[:thread_length]} replies") {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end) IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n") {:ok, activity} end defp do_generate_long_thread(user, users, opts) do - {:ok, %{id: id} = activity} = - Pleroma.Web.CommonAPI.post(user, %{"status" => "Start of long thread"}) + {:ok, %{id: id} = activity} = CommonAPI.post(user, %{"status" => "Start of long thread"}) Task.async_stream( - 1..opts[:long_thread_length], + 1..opts[:thread_length], fn _ -> do_generate_thread(users, id) end, max_concurrency: 10, timeout: 30_000 @@ -117,50 +195,63 @@ defp do_generate_long_thread(user, users, opts) do end defp do_generate_thread(users, activity_id) do - Pleroma.Web.CommonAPI.post(Enum.random(users), %{ + CommonAPI.post(Enum.random(users), %{ "status" => "reply to main post", "in_reply_to_status_id" => activity_id }) end - def generate_private_thread(users, opts) do - IO.puts("Starting generating long thread with #{opts[:non_visible_posts_max]} replies") - {time, _} = :timer.tc(fn -> do_generate_non_visible_posts(users, opts) end) - IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n") + def generate_non_visible_message(user, users) do + IO.puts("Starting generating 1000 non visible posts") + + {time, _} = + :timer.tc(fn -> + do_generate_non_visible_posts(user, users) + end) + + IO.puts("Inserting non visible posts take #{to_sec(time)} sec.\n") end - defp do_generate_non_visible_posts(users, opts) do - [user1, user2] = Enum.take(users, 2) - {:ok, user1} = Pleroma.User.follow(user1, user2) - {:ok, user2} = Pleroma.User.follow(user2, user1) + defp do_generate_non_visible_posts(user, users) do + [not_friend | users] = users - {:ok, activity} = - Pleroma.Web.CommonAPI.post(user1, %{ - "status" => "Some private post", - "visibility" => "private" - }) + make_friends(user, users) - {:ok, activity_public} = - Pleroma.Web.CommonAPI.post(user2, %{ - "status" => "Some public reply", - "in_reply_to_status_id" => activity.id - }) - - Task.async_stream( - 1..opts[:non_visible_posts_max], - fn _ -> do_generate_non_visible_post(users, activity_public) end, + Task.async_stream(1..1000, fn _ -> do_generate_non_visible_post(not_friend, users) end, max_concurrency: 10, timeout: 30_000 ) + |> Stream.run() end - defp do_generate_non_visible_post(users, activity) do - visibility = Enum.random(["private", "public"]) + defp make_friends(_user, []), do: nil - Pleroma.Web.CommonAPI.post(Enum.random(users), %{ - "visibility" => visibility, - "status" => "Some #{visibility} reply", - "in_reply_to_status_id" => activity.id - }) + defp make_friends(user, [friend | users]) do + {:ok, _} = User.follow(user, friend) + {:ok, _} = User.follow(friend, user) + make_friends(user, users) + end + + defp do_generate_non_visible_post(not_friend, users) do + post = %{ + "status" => "some non visible post", + "visibility" => "private" + } + + {:ok, activity} = CommonAPI.post(not_friend, post) + + thread_length = Enum.random([2, 3, 4, 5]) + + Enum.each(1..thread_length, fn _ -> + user = Enum.random(users) + + post = %{ + "status" => "@#{not_friend.nickname} reply to non visible post", + "in_reply_to_status_id" => activity.id, + "visibility" => "private" + } + + CommonAPI.post(user, post) + end) end end diff --git a/lib/load_testing/helper.ex b/lib/load_testing/helper.ex index 338dba323..47b25c65f 100644 --- a/lib/load_testing/helper.ex +++ b/lib/load_testing/helper.ex @@ -2,13 +2,8 @@ defmodule Pleroma.LoadTesting.Helper do defmacro __using__(_) do quote do import Ecto.Query - alias Pleroma.Activity - alias Pleroma.Notification - alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User - alias Pleroma.Web.ActivityPub - alias Pleroma.Web.CommonAPI defp to_sec(microseconds), do: microseconds / 1_000_000 end diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex index d17cf0b3e..83e531abf 100644 --- a/lib/mix/tasks/pleroma/load_testing.ex +++ b/lib/mix/tasks/pleroma/load_testing.ex @@ -13,46 +13,37 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do - activities with notifications ## Generate data - MIX_ENV=test mix pleroma.load_testing --users 10000 --activities 20000 - MIX_ENV=test mix pleroma.load_testing -u 10000 -a 20000 + MIX_ENV=benchmark mix pleroma.load_testing --users 10000 + MIX_ENV=benchmark mix pleroma.load_testing -u 10000 Options: - `--users NUMBER` - number of users to generate (default: 10000) - - `--activities NUMBER` - number of activities to generate (default: 20000) """ - @aliases [u: :users, a: :activities] + @aliases [u: :users, d: :dms, t: :thread_length] @switches [ users: :integer, - activities: :integer, dms: :integer, - thread_length: :integer, - non_visible_posts: :integer + thread_length: :integer ] @users_default 20_000 - @activities_default 50_000 - @dms_default 50_000 + @dms_default 20_000 @thread_length_default 2_000 - @non_visible_posts_default 2_000 def run(args) do start_pleroma() {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) users_max = Keyword.get(opts, :users, @users_default) - activities_max = Keyword.get(opts, :activities, @activities_default) dms_max = Keyword.get(opts, :dms, @dms_default) - long_thread_length = Keyword.get(opts, :thread_length, @thread_length_default) - non_visible_posts = Keyword.get(opts, :non_visible_posts, @non_visible_posts_default) + thread_length = Keyword.get(opts, :thread_length, @thread_length_default) clean_tables() opts = Keyword.put(opts, :users_max, users_max) - |> Keyword.put(:activities_max, activities_max) |> Keyword.put(:dms_max, dms_max) - |> Keyword.put(:long_thread_length, long_thread_length) - |> Keyword.put(:non_visible_posts_max, non_visible_posts) + |> Keyword.put(:thread_length, thread_length) generate_users(opts) @@ -60,7 +51,9 @@ def run(args) do IO.puts("Fetching main user...") {time, user} = - :timer.tc(fn -> Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1)) end) + :timer.tc(fn -> + Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1)) + end) IO.puts("Fetching main user take #{to_sec(time)} sec.\n") @@ -79,25 +72,23 @@ def run(args) do IO.puts("Fetching users take #{to_sec(time)} sec.\n") - generate_activities(users, opts) - - generate_activities(users, Keyword.put(opts, :mention, user)) + generate_activities(user, users) generate_dms(user, users, opts) {:ok, activity} = generate_long_thread(user, users, opts) - generate_private_thread(users, opts) - - # generate_replies(user, users, activities) - - # activity = Enum.random(activities) - # generate_long_thread(user, users, activity) + generate_non_visible_message(user, users) IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}") - IO.puts("Activities in DB: #{Repo.aggregate(from(a in Activity), :count, :id)}") - IO.puts("Objects in DB: #{Repo.aggregate(from(o in Object), :count, :id)}") - IO.puts("Notifications in DB: #{Repo.aggregate(from(n in Notification), :count, :id)}") + + IO.puts("Activities in DB: #{Repo.aggregate(from(a in Pleroma.Activity), :count, :id)}") + + IO.puts("Objects in DB: #{Repo.aggregate(from(o in Pleroma.Object), :count, :id)}") + + IO.puts( + "Notifications in DB: #{Repo.aggregate(from(n in Pleroma.Notification), :count, :id)}" + ) fetch_user(user) query_timelines(user) From 252e5db45c5d9d63116e5a24cc35c26f91ec7de4 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 6 Sep 2019 16:37:18 +0300 Subject: [PATCH 04/36] docs fixes --- lib/load_testing/fetcher.ex | 9 --------- lib/load_testing/generator.ex | 3 ++- lib/mix/tasks/pleroma/load_testing.ex | 16 +++++++++++----- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex index 825f921e6..b92b6e04f 100644 --- a/lib/load_testing/fetcher.ex +++ b/lib/load_testing/fetcher.ex @@ -2,8 +2,6 @@ defmodule Pleroma.LoadTesting.Fetcher do use Pleroma.LoadTesting.Helper def fetch_user(user) do - IO.puts("=================================") - Benchee.run(%{ "By id" => fn -> Repo.get_by(User, id: user.id) end, "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end, @@ -13,8 +11,6 @@ def fetch_user(user) do end def query_timelines(user) do - IO.puts("\n=================================") - home_timeline_params = %{ "count" => 20, "with_muted" => true, @@ -102,7 +98,6 @@ def query_timelines(user) do end def query_notifications(user) do - IO.puts("\n=================================") without_muted_params = %{"count" => "20", "with_muted" => "false"} with_muted_params = %{"count" => "20", "with_muted" => "true"} @@ -138,8 +133,6 @@ def query_notifications(user) do end def query_dms(user) do - IO.puts("\n=================================") - params = %{ "count" => "20", "with_muted" => "true", @@ -187,8 +180,6 @@ def query_dms(user) do end def query_long_thread(user, activity) do - IO.puts("\n=================================") - Benchee.run(%{ "Fetch main post" => fn -> Pleroma.Activity.get_by_id_with_object(activity.id) diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex index 6df518aba..7ad68dcc1 100644 --- a/lib/load_testing/generator.ex +++ b/lib/load_testing/generator.ex @@ -28,7 +28,8 @@ defp generate_user_data(i) do nickname: "nick#{i}", password_hash: Comeonin.Pbkdf2.hashpwsalt("test"), bio: "Tester Number #{i}", - info: %{} + info: %{}, + local: Enum.random([true, false]) } user = %{ diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex index 83e531abf..4fed6bf74 100644 --- a/lib/mix/tasks/pleroma/load_testing.ex +++ b/lib/mix/tasks/pleroma/load_testing.ex @@ -5,19 +5,23 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do import Pleroma.LoadTesting.Generator import Pleroma.LoadTesting.Fetcher - # tODO: remove autovacuum worker until generation is not ended @shortdoc "Factory for generation data" @moduledoc """ Generates data like: - users - activities with notifications + - direct messages + - long thread + - non visible posts ## Generate data - MIX_ENV=benchmark mix pleroma.load_testing --users 10000 - MIX_ENV=benchmark mix pleroma.load_testing -u 10000 + MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --dms 20000 --thread_length 2000 + MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -d 20000 -t 2000 Options: - - `--users NUMBER` - number of users to generate (default: 10000) + - `--users NUMBER` - number of users to generate. Defaults to: 20000. Alias: `-u` + - `--dms NUMBER` - number of direct messages to generate. Defaults to: 20000. Alias `-d` + - `--thread_length` - number of messages in thread. Defaults to: 2000. ALias `-t` """ @aliases [u: :users, d: :dms, t: :thread_length] @@ -32,6 +36,7 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do def run(args) do start_pleroma() + Pleroma.Config.put([:instance, :skip_thread_containment], true) {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases) users_max = Keyword.get(opts, :users, @users_default) @@ -95,11 +100,12 @@ def run(args) do query_notifications(user) query_dms(user) query_long_thread(user, activity) + Pleroma.Config.put([:instance, :skip_thread_containment], false) query_timelines(user) end defp clean_tables do - IO.puts("\n\nDeleting old data...\n") + IO.puts("Deleting old data...\n") Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;") Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;") Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;") From b3f6f6a4091c048813e367f6a9414eba6fce5109 Mon Sep 17 00:00:00 2001 From: Alex S Date: Fri, 6 Sep 2019 20:14:02 +0300 Subject: [PATCH 05/36] generating remote users --- lib/load_testing/fetcher.ex | 2 +- lib/load_testing/generator.ex | 34 ++++++++++++++++++++++++++-------- lib/pleroma/user.ex | 4 +++- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex index b92b6e04f..0ff2f28c0 100644 --- a/lib/load_testing/fetcher.ex +++ b/lib/load_testing/fetcher.ex @@ -215,7 +215,7 @@ def query_long_thread(user, activity) do for: user }) end, - "Render context ancestors" => fn -> + "Render context" => fn -> Pleroma.Web.MastodonAPI.StatusView.render( "index.json", for: user, diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex index 7ad68dcc1..61e3d8686 100644 --- a/lib/load_testing/generator.ex +++ b/lib/load_testing/generator.ex @@ -22,6 +22,8 @@ defp do_generate_users(opts) do end defp generate_user_data(i) do + remote = Enum.random([true, false]) + user = %User{ name: "Test テスト User #{i}", email: "user#{i}@example.com", @@ -29,16 +31,32 @@ defp generate_user_data(i) do password_hash: Comeonin.Pbkdf2.hashpwsalt("test"), bio: "Tester Number #{i}", info: %{}, - local: Enum.random([true, false]) + local: remote } - user = %{ - user - | ap_id: User.ap_id(user), - follower_address: User.ap_followers(user), - following_address: User.ap_following(user), - following: [User.ap_id(user)] - } + user_urls = + if remote do + base_url = + Enum.random(["https://domain1.com", "https://domain2.com", "https://domain3.com"]) + + ap_id = "#{base_url}/users/#{user.nickname}" + + %{ + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following", + following: [ap_id] + } + else + %{ + ap_id: User.ap_id(user), + follower_address: User.ap_followers(user), + following_address: User.ap_following(user), + following: [User.ap_id(user)] + } + end + + user = Map.merge(user, user_urls) Repo.insert!(user) end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3aa245f2a..37e59a651 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -429,7 +429,9 @@ def follow(%User{} = follower, %User{info: info} = followed) do {:error, "Could not follow user: #{followed.nickname} blocked you."} true -> - if !followed.local && follower.local && !ap_enabled?(followed) do + benchmark? = Pleroma.Config.get([:env]) == :benchmark + + if !followed.local && follower.local && !ap_enabled?(followed) && !benchmark? do Websub.subscribe(follower, followed) end From 924d7e6aa60095e27c15b4a5f473a192ab2b42ef Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 19 Sep 2019 12:59:36 +0300 Subject: [PATCH 06/36] generating remote activities --- lib/load_testing/generator.ex | 78 ++++++++++++++++++++++++++- lib/mix/tasks/pleroma/load_testing.ex | 29 ++++++++-- 2 files changed, 102 insertions(+), 5 deletions(-) diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex index 61e3d8686..5c5a5c122 100644 --- a/lib/load_testing/generator.ex +++ b/lib/load_testing/generator.ex @@ -28,7 +28,8 @@ defp generate_user_data(i) do name: "Test テスト User #{i}", email: "user#{i}@example.com", nickname: "nick#{i}", - password_hash: Comeonin.Pbkdf2.hashpwsalt("test"), + password_hash: + "$pbkdf2-sha512$160000$bU.OSFI7H/yqWb5DPEqyjw$uKp/2rmXw12QqnRRTqTtuk2DTwZfF8VR4MYW2xMeIlqPR/UX1nT1CEKVUx2CowFMZ5JON8aDvURrZpJjSgqXrg", bio: "Tester Number #{i}", info: %{}, local: remote @@ -165,6 +166,81 @@ defp do_generate_threads(users) do end) end + def generate_remote_activities(user, users) do + do_generate_remote_activities(user, users) + end + + defp do_generate_remote_activities(user, users) do + IO.puts("Starting generating 10000 remote activities...") + + {time, _} = + :timer.tc(fn -> + Task.async_stream( + 1..10_000, + fn i -> + do_generate_remote_activity(i, user, users) + end, + max_concurrency: 10, + timeout: 30_000 + ) + |> Stream.run() + end) + + IO.puts("Inserting remote activities take #{to_sec(time)} sec.\n") + end + + defp do_generate_remote_activity(i, user, users) do + actor = Enum.random(users) + %{host: host} = URI.parse(actor.ap_id) + date = Date.utc_today() + datetime = DateTime.utc_now() + + map = %{ + "actor" => actor.ap_id, + "cc" => [actor.follower_address, user.ap_id], + "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation", + "id" => actor.ap_id <> "/statuses/#{i}/activity", + "object" => %{ + "actor" => actor.ap_id, + "atomUri" => actor.ap_id <> "/statuses/#{i}", + "attachment" => [], + "attributedTo" => actor.ap_id, + "bcc" => [], + "bto" => [], + "cc" => [actor.follower_address, user.ap_id], + "content" => + "

+ user.ap_id <> + "\" class=\"u-url mention\">@" <> user.nickname <> "

", + "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation", + "conversation" => + "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation", + "emoji" => %{}, + "id" => actor.ap_id <> "/statuses/#{i}", + "inReplyTo" => nil, + "inReplyToAtomUri" => nil, + "published" => datetime, + "sensitive" => true, + "summary" => "cw", + "tag" => [ + %{ + "href" => user.ap_id, + "name" => "@#{user.nickname}@#{host}", + "type" => "Mention" + } + ], + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "type" => "Note", + "url" => "http://#{host}/@#{actor.nickname}/#{i}" + }, + "published" => datetime, + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "type" => "Create" + } + + Pleroma.Web.ActivityPub.ActivityPub.insert(map, false) + end + def generate_dms(user, users, opts) do IO.puts("Starting generating #{opts[:dms_max]} DMs") {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end) diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex index 4fed6bf74..a7057395d 100644 --- a/lib/mix/tasks/pleroma/load_testing.ex +++ b/lib/mix/tasks/pleroma/load_testing.ex @@ -53,32 +53,53 @@ def run(args) do generate_users(opts) # main user for queries - IO.puts("Fetching main user...") + IO.puts("Fetching local main user...") {time, user} = :timer.tc(fn -> - Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1)) + Repo.one( + from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1) + ) end) IO.puts("Fetching main user take #{to_sec(time)} sec.\n") - IO.puts("Fetching users...") + IO.puts("Fetching local users...") {time, users} = :timer.tc(fn -> Repo.all( from(u in User, where: u.id != ^user.id, + where: u.local == true, order_by: fragment("RANDOM()"), limit: 10 ) ) end) - IO.puts("Fetching users take #{to_sec(time)} sec.\n") + IO.puts("Fetching local users take #{to_sec(time)} sec.\n") + + IO.puts("Fetching remote users...") + + {time, remote_users} = + :timer.tc(fn -> + Repo.all( + from(u in User, + where: u.id != ^user.id, + where: u.local == false, + order_by: fragment("RANDOM()"), + limit: 10 + ) + ) + end) + + IO.puts("Fetching remote users take #{to_sec(time)} sec.\n") generate_activities(user, users) + generate_remote_activities(user, remote_users) + generate_dms(user, users, opts) {:ok, activity} = generate_long_thread(user, users, opts) From 1d285e6fada841b6959b6516aca1c84d1a2c9b93 Mon Sep 17 00:00:00 2001 From: Alex S Date: Thu, 19 Sep 2019 14:02:27 +0300 Subject: [PATCH 07/36] moving to separate dir --- {lib => benchmarks}/load_testing/fetcher.ex | 0 {lib => benchmarks}/load_testing/generator.ex | 0 {lib => benchmarks}/load_testing/helper.ex | 0 {lib => benchmarks}/mix/tasks/pleroma/load_testing.ex | 4 ++-- mix.exs | 1 + 5 files changed, 3 insertions(+), 2 deletions(-) rename {lib => benchmarks}/load_testing/fetcher.ex (100%) rename {lib => benchmarks}/load_testing/generator.ex (100%) rename {lib => benchmarks}/load_testing/helper.ex (100%) rename {lib => benchmarks}/mix/tasks/pleroma/load_testing.ex (98%) diff --git a/lib/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex similarity index 100% rename from lib/load_testing/fetcher.ex rename to benchmarks/load_testing/fetcher.ex diff --git a/lib/load_testing/generator.ex b/benchmarks/load_testing/generator.ex similarity index 100% rename from lib/load_testing/generator.ex rename to benchmarks/load_testing/generator.ex diff --git a/lib/load_testing/helper.ex b/benchmarks/load_testing/helper.ex similarity index 100% rename from lib/load_testing/helper.ex rename to benchmarks/load_testing/helper.ex diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/benchmarks/mix/tasks/pleroma/load_testing.ex similarity index 98% rename from lib/mix/tasks/pleroma/load_testing.ex rename to benchmarks/mix/tasks/pleroma/load_testing.ex index a7057395d..7b2293acc 100644 --- a/lib/mix/tasks/pleroma/load_testing.ex +++ b/benchmarks/mix/tasks/pleroma/load_testing.ex @@ -8,8 +8,8 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do @shortdoc "Factory for generation data" @moduledoc """ Generates data like: - - users - - activities with notifications + - local/remote users + - local/remote activities with notifications - direct messages - long thread - non visible posts diff --git a/mix.exs b/mix.exs index 3170d6f2d..b0f2c4afb 100644 --- a/mix.exs +++ b/mix.exs @@ -69,6 +69,7 @@ def application do end # Specifies which paths to compile per environment. + defp elixirc_paths(:benchmark), do: ["lib", "benchmarks"] defp elixirc_paths(:test), do: ["lib", "test/support"] defp elixirc_paths(_), do: ["lib"] From ad42837244ba4c945b76c5addaffe47353cf62a8 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Wed, 9 Oct 2019 17:03:54 +0300 Subject: [PATCH 08/36] Ability to toggle activation status and permission group for a group of users --- CHANGELOG.md | 3 + docs/API/admin_api.md | 49 +++++-- lib/pleroma/moderation_log.ex | 60 ++++----- lib/pleroma/user.ex | 16 ++- .../web/admin_api/admin_api_controller.ex | 95 ++++++------- .../web/admin_api/views/account_view.ex | 6 + lib/pleroma/web/router.ex | 14 +- test/moderation_log_test.exs | 4 +- .../admin_api/admin_api_controller_test.exs | 125 +++++++----------- 9 files changed, 189 insertions(+), 183 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b24db7f4..584c917f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) - **Breaking:** Admin API: Return link alongside with token on password reset +- **Breaking:** Admin API: `/users/:nickname/toggle_activation` endpoint was split into two: `/users/activate`, `/users/deactivate`, both accept `nicknames` array +- **Breaking:** Admin API: `POST /users/permission_group/:permission_group` / `DELETE /users/permission_group/:permission_group` now accept `nicknames` array - Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) - Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler - Admin API: Return `total` when querying for reports @@ -40,6 +42,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Removed - **Breaking:** GNU Social API with Qvitter extensions support +- **Breaking:** Admin API: `/users/:nickname/activation_status` was removed in favor of `/users/activate`, `/users/deactivate` - Emoji: Remove longfox emojis. - Remove `Reply-To` header from report emails for admins. diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index ee9e68cb1..55f8749e1 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -154,31 +154,62 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret } ``` +## `POST /api/pleroma/admin/users/permission_group/:permission_group` + ### Add user in permission group -- Method: `POST` -- Params: none +- Params: + - `nicknames`: nicknames array - Response: - On failure: `{"error": "…"}` - On success: JSON of the `user.info` +## `DELETE /api/pleroma/admin/users/permission_group/:permission_group` + ### Remove user from permission group -- Method: `DELETE` -- Params: none +- Params: + - `nicknames`: nicknames array - Response: - On failure: `{"error": "…"}` - On success: JSON of the `user.info` - Note: An admin cannot revoke their own admin status. -## `/api/pleroma/admin/users/:nickname/activation_status` +## `PATCH /api/pleroma/admin/users/activate` -### Active or deactivate a user +### Activate user -- Method: `PUT` - Params: - - `nickname` - - `status` BOOLEAN field, false value means deactivation. + - `nicknames`: nicknames array +- Response: + +```json +{ + users: [ + { + // user object + } + ] +} +``` + +## `PATCH /api/pleroma/admin/users/deactivate` + +### Deactivate user + +- Params: + - `nicknames`: nicknames array +- Response: + +```json +{ + users: [ + { + // user object + } + ] +} +``` ## `/api/pleroma/admin/users/:nickname_or_id` diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex index 352cad433..42649ff02 100644 --- a/lib/pleroma/moderation_log.ex +++ b/lib/pleroma/moderation_log.ex @@ -86,18 +86,18 @@ defp parse_datetime(datetime) do parsed_datetime end - @spec insert_log(%{actor: User, subject: User, action: String.t(), permission: String.t()}) :: + @spec insert_log(%{actor: User, subject: [User], action: String.t(), permission: String.t()}) :: {:ok, ModerationLog} | {:error, any} def insert_log(%{ actor: %User{} = actor, - subject: %User{} = subject, + subject: subjects, action: action, permission: permission }) do %ModerationLog{ data: %{ "actor" => user_to_map(actor), - "subject" => user_to_map(subject), + "subject" => user_to_map(subjects), "action" => action, "permission" => permission, "message" => "" @@ -303,13 +303,16 @@ def insert_log(%{ end @spec insert_log_entry_with_message(ModerationLog) :: {:ok, ModerationLog} | {:error, any} - defp insert_log_entry_with_message(entry) do entry.data["message"] |> put_in(get_log_entry_message(entry)) |> Repo.insert() end + defp user_to_map(users) when is_list(users) do + users |> Enum.map(&user_to_map/1) + end + defp user_to_map(%User{} = user) do user |> Map.from_struct() @@ -363,12 +366,7 @@ def get_log_entry_message(%ModerationLog{ "subjects" => subjects } }) do - nicknames = - subjects - |> Enum.map(&"@#{&1["nickname"]}") - |> Enum.join(", ") - - "@#{actor_nickname} created users: #{nicknames}" + "@#{actor_nickname} created users: #{users_to_nicknames_string(subjects)}" end @spec get_log_entry_message(ModerationLog) :: String.t() @@ -376,10 +374,10 @@ def get_log_entry_message(%ModerationLog{ data: %{ "actor" => %{"nickname" => actor_nickname}, "action" => "activate", - "subject" => %{"nickname" => subject_nickname, "type" => "user"} + "subject" => users } }) do - "@#{actor_nickname} activated user @#{subject_nickname}" + "@#{actor_nickname} activated users: #{users_to_nicknames_string(users)}" end @spec get_log_entry_message(ModerationLog) :: String.t() @@ -387,10 +385,10 @@ def get_log_entry_message(%ModerationLog{ data: %{ "actor" => %{"nickname" => actor_nickname}, "action" => "deactivate", - "subject" => %{"nickname" => subject_nickname, "type" => "user"} + "subject" => users } }) do - "@#{actor_nickname} deactivated user @#{subject_nickname}" + "@#{actor_nickname} deactivated users: #{users_to_nicknames_string(users)}" end @spec get_log_entry_message(ModerationLog) :: String.t() @@ -402,14 +400,9 @@ def get_log_entry_message(%ModerationLog{ "action" => "tag" } }) do - nicknames_string = - nicknames - |> Enum.map(&"@#{&1}") - |> Enum.join(", ") - tags_string = tags |> Enum.join(", ") - "@#{actor_nickname} added tags: #{tags_string} to users: #{nicknames_string}" + "@#{actor_nickname} added tags: #{tags_string} to users: #{nicknames_to_string(nicknames)}" end @spec get_log_entry_message(ModerationLog) :: String.t() @@ -421,14 +414,9 @@ def get_log_entry_message(%ModerationLog{ "action" => "untag" } }) do - nicknames_string = - nicknames - |> Enum.map(&"@#{&1}") - |> Enum.join(", ") - tags_string = tags |> Enum.join(", ") - "@#{actor_nickname} removed tags: #{tags_string} from users: #{nicknames_string}" + "@#{actor_nickname} removed tags: #{tags_string} from users: #{nicknames_to_string(nicknames)}" end @spec get_log_entry_message(ModerationLog) :: String.t() @@ -436,11 +424,11 @@ def get_log_entry_message(%ModerationLog{ data: %{ "actor" => %{"nickname" => actor_nickname}, "action" => "grant", - "subject" => %{"nickname" => subject_nickname}, + "subject" => users, "permission" => permission } }) do - "@#{actor_nickname} made @#{subject_nickname} #{permission}" + "@#{actor_nickname} made #{users_to_nicknames_string(users)} #{permission}" end @spec get_log_entry_message(ModerationLog) :: String.t() @@ -448,11 +436,11 @@ def get_log_entry_message(%ModerationLog{ data: %{ "actor" => %{"nickname" => actor_nickname}, "action" => "revoke", - "subject" => %{"nickname" => subject_nickname}, + "subject" => users, "permission" => permission } }) do - "@#{actor_nickname} revoked #{permission} role from @#{subject_nickname}" + "@#{actor_nickname} revoked #{permission} role from #{users_to_nicknames_string(users)}" end @spec get_log_entry_message(ModerationLog) :: String.t() @@ -551,4 +539,16 @@ def get_log_entry_message(%ModerationLog{ }) do "@#{actor_nickname} deleted status ##{subject_id}" end + + defp nicknames_to_string(nicknames) do + nicknames + |> Enum.map(&"@#{&1}") + |> Enum.join(", ") + end + + defp users_to_nicknames_string(users) do + users + |> Enum.map(&"@#{&1["nickname"]}") + |> Enum.join(", ") + end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 2cfb13a8c..a76a5ad70 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1059,7 +1059,15 @@ def deactivate_async(user, status \\ true) do BackgroundWorker.enqueue("deactivate_user", %{"user_id" => user.id, "status" => status}) end - def deactivate(%User{} = user, status \\ true) do + def deactivate(user, status \\ true) + + def deactivate(users, status) when is_list(users) do + Repo.transaction(fn -> + for user <- users, do: deactivate(user, status) + end) + end + + def deactivate(%User{} = user, status) do with {:ok, user} <- update_info(user, &User.Info.set_activation_status(&1, status)) do Enum.each(get_followers(user), &invalidate_cache/1) Enum.each(get_friends(user), &update_follower_count/1) @@ -1625,6 +1633,12 @@ def change_info(user, fun) do `fun` is called with the `user.info`. """ + def update_info(users, fun) when is_list(users) do + Repo.transaction(fn -> + for user <- users, do: update_info(user, fun) + end) + end + def update_info(user, fun) do user |> change_info(fun) diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 513bae800..d825a5d28 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -231,22 +231,34 @@ def list_user_statuses(conn, %{"nickname" => nickname} = params) do end end - def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do - user = User.get_cached_by_nickname(nickname) - - {:ok, updated_user} = User.deactivate(user, !user.info.deactivated) - - action = if user.info.deactivated, do: "activate", else: "deactivate" + def user_activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do + users = Enum.map(nicknames, &User.get_cached_by_nickname/1) + {:ok, updated_users} = User.deactivate(users, false) ModerationLog.insert_log(%{ actor: admin, - subject: user, - action: action + subject: users, + action: "activate" }) conn |> put_view(AccountView) - |> render("show.json", %{user: updated_user}) + |> render("index.json", %{users: Keyword.values(updated_users)}) + end + + def user_deactivate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do + users = Enum.map(nicknames, &User.get_cached_by_nickname/1) + {:ok, updated_users} = User.deactivate(users, true) + + ModerationLog.insert_log(%{ + actor: admin, + subject: users, + action: "deactivate" + }) + + conn + |> put_view(AccountView) + |> render("index.json", %{users: Keyword.values(updated_users)}) end def tag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "tags" => tags}) do @@ -315,20 +327,19 @@ defp maybe_parse_filters(filters) do def right_add(%{assigns: %{user: admin}} = conn, %{ "permission_group" => permission_group, - "nickname" => nickname + "nicknames" => nicknames }) when permission_group in ["moderator", "admin"] do info = Map.put(%{}, "is_" <> permission_group, true) - {:ok, user} = - nickname - |> User.get_cached_by_nickname() - |> User.update_info(&User.Info.admin_api_update(&1, info)) + users = nicknames |> Enum.map(&User.get_cached_by_nickname/1) + + User.update_info(users, &User.Info.admin_api_update(&1, info)) ModerationLog.insert_log(%{ action: "grant", actor: admin, - subject: user, + subject: users, permission: permission_group }) @@ -349,58 +360,38 @@ def right_get(conn, %{"nickname" => nickname}) do }) end - def right_delete(%{assigns: %{user: %{nickname: nickname}}} = conn, %{"nickname" => nickname}) do - render_error(conn, :forbidden, "You can't revoke your own admin status.") - end - def right_delete( - %{assigns: %{user: admin}} = conn, + %{assigns: %{user: %{nickname: admin_nickname} = admin}} = conn, %{ "permission_group" => permission_group, - "nickname" => nickname + "nicknames" => nicknames } ) when permission_group in ["moderator", "admin"] do - info = Map.put(%{}, "is_" <> permission_group, false) + with false <- Enum.member?(nicknames, admin_nickname) do + info = Map.put(%{}, "is_" <> permission_group, false) - {:ok, user} = - nickname - |> User.get_cached_by_nickname() - |> User.update_info(&User.Info.admin_api_update(&1, info)) + users = nicknames |> Enum.map(&User.get_cached_by_nickname/1) - ModerationLog.insert_log(%{ - action: "revoke", - actor: admin, - subject: user, - permission: permission_group - }) + User.update_info(users, &User.Info.admin_api_update(&1, info)) - json(conn, info) + ModerationLog.insert_log(%{ + action: "revoke", + actor: admin, + subject: users, + permission: permission_group + }) + + json(conn, info) + else + _ -> render_error(conn, :forbidden, "You can't revoke your own admin/moderator status.") + end end def right_delete(conn, _) do render_error(conn, :not_found, "No such permission_group") end - def set_activation_status(%{assigns: %{user: admin}} = conn, %{ - "nickname" => nickname, - "status" => status - }) do - with {:ok, status} <- Ecto.Type.cast(:boolean, status), - %User{} = user <- User.get_cached_by_nickname(nickname), - {:ok, _} <- User.deactivate(user, !status) do - action = if(user.info.deactivated, do: "activate", else: "deactivate") - - ModerationLog.insert_log(%{ - actor: admin, - subject: user, - action: action - }) - - json_response(conn, :no_content, "") - end - end - def relay_follow(%{assigns: %{user: admin}} = conn, %{"relay_url" => target}) do with {:ok, _message} <- Relay.follow(target) do ModerationLog.insert_log(%{ diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex index a96affd40..441269162 100644 --- a/lib/pleroma/web/admin_api/views/account_view.ex +++ b/lib/pleroma/web/admin_api/views/account_view.ex @@ -19,6 +19,12 @@ def render("index.json", %{users: users, count: count, page_size: page_size}) do } end + def render("index.json", %{users: users}) do + %{ + users: render_many(users, AccountView, "show.json", as: :user) + } + end + def render("show.json", %{user: user}) do avatar = User.avatar_url(user) |> MediaProxy.url() display_name = HTML.strip_tags(user.name || user.nickname) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index ae799b8ac..894375357 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -136,21 +136,15 @@ defmodule Pleroma.Web.Router do delete("/users", AdminAPIController, :user_delete) post("/users", AdminAPIController, :users_create) - patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation) + patch("/users/activate", AdminAPIController, :user_activate) + patch("/users/deactivate", AdminAPIController, :user_deactivate) put("/users/tag", AdminAPIController, :tag_users) delete("/users/tag", AdminAPIController, :untag_users) get("/users/:nickname/permission_group", AdminAPIController, :right_get) get("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_get) - post("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_add) - - delete( - "/users/:nickname/permission_group/:permission_group", - AdminAPIController, - :right_delete - ) - - put("/users/:nickname/activation_status", AdminAPIController, :set_activation_status) + post("/users/permission_group/:permission_group", AdminAPIController, :right_add) + delete("/users/permission_group/:permission_group", AdminAPIController, :right_delete) post("/relay", AdminAPIController, :relay_follow) delete("/relay", AdminAPIController, :relay_unfollow) diff --git a/test/moderation_log_test.exs b/test/moderation_log_test.exs index a39a00e02..ead97e948 100644 --- a/test/moderation_log_test.exs +++ b/test/moderation_log_test.exs @@ -128,7 +128,7 @@ test "logging user grant by moderator", %{moderator: moderator, subject1: subjec {:ok, _} = ModerationLog.insert_log(%{ actor: moderator, - subject: subject1, + subject: [subject1], action: "grant", permission: "moderator" }) @@ -142,7 +142,7 @@ test "logging user revoke by moderator", %{moderator: moderator, subject1: subje {:ok, _} = ModerationLog.insert_log(%{ actor: moderator, - subject: subject1, + subject: [subject1], action: "revoke", permission: "moderator" }) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index b5c355e66..c57c71203 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -386,13 +386,16 @@ test "GET is giving user_info" do test "/:right POST, can add to a permission group" do admin = insert(:user, info: %{is_admin: true}) - user = insert(:user) + user_one = insert(:user) + user_two = insert(:user) conn = build_conn() |> assign(:user, admin) |> put_req_header("accept", "application/json") - |> post("/api/pleroma/admin/users/#{user.nickname}/permission_group/admin") + |> post("/api/pleroma/admin/users/permission_group/admin", %{ + nicknames: [user_one.nickname, user_two.nickname] + }) assert json_response(conn, 200) == %{ "is_admin" => true @@ -401,18 +404,21 @@ test "/:right POST, can add to a permission group" do log_entry = Repo.one(ModerationLog) assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} made @#{user.nickname} admin" + "@#{admin.nickname} made @#{user_one.nickname}, @#{user_two.nickname} admin" end test "/:right DELETE, can remove from a permission group" do admin = insert(:user, info: %{is_admin: true}) - user = insert(:user, info: %{is_admin: true}) + user_one = insert(:user, info: %{is_admin: true}) + user_two = insert(:user, info: %{is_admin: true}) conn = build_conn() |> assign(:user, admin) |> put_req_header("accept", "application/json") - |> delete("/api/pleroma/admin/users/#{user.nickname}/permission_group/admin") + |> delete("/api/pleroma/admin/users/permission_group/admin", %{ + nicknames: [user_one.nickname, user_two.nickname] + }) assert json_response(conn, 200) == %{ "is_admin" => false @@ -421,65 +427,9 @@ test "/:right DELETE, can remove from a permission group" do log_entry = Repo.one(ModerationLog) assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} revoked admin role from @#{user.nickname}" - end - end - - describe "PUT /api/pleroma/admin/users/:nickname/activation_status" do - setup %{conn: conn} do - admin = insert(:user, info: %{is_admin: true}) - - conn = - conn - |> assign(:user, admin) - |> put_req_header("accept", "application/json") - - %{conn: conn, admin: admin} - end - - test "deactivates the user", %{conn: conn, admin: admin} do - user = insert(:user) - - conn = - conn - |> put("/api/pleroma/admin/users/#{user.nickname}/activation_status", %{status: false}) - - user = User.get_cached_by_id(user.id) - assert user.info.deactivated == true - assert json_response(conn, :no_content) - - log_entry = Repo.one(ModerationLog) - - assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} deactivated user @#{user.nickname}" - end - - test "activates the user", %{conn: conn, admin: admin} do - user = insert(:user, info: %{deactivated: true}) - - conn = - conn - |> put("/api/pleroma/admin/users/#{user.nickname}/activation_status", %{status: true}) - - user = User.get_cached_by_id(user.id) - assert user.info.deactivated == false - assert json_response(conn, :no_content) - - log_entry = Repo.one(ModerationLog) - - assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} activated user @#{user.nickname}" - end - - test "returns 403 when requested by a non-admin", %{conn: conn} do - user = insert(:user) - - conn = - conn - |> assign(:user, user) - |> put("/api/pleroma/admin/users/#{user.nickname}/activation_status", %{status: false}) - - assert json_response(conn, :forbidden) + "@#{admin.nickname} revoked admin role from @#{user_one.nickname}, @#{ + user_two.nickname + }" end end @@ -1029,31 +979,48 @@ test "it works with multiple filters" do end end - test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation" do + test "PATCH /api/pleroma/admin/users/activate" do admin = insert(:user, info: %{is_admin: true}) - user = insert(:user) + user_one = insert(:user, info: %{deactivated: true}) + user_two = insert(:user, info: %{deactivated: true}) conn = build_conn() |> assign(:user, admin) - |> patch("/api/pleroma/admin/users/#{user.nickname}/toggle_activation") + |> patch( + "/api/pleroma/admin/users/activate", + %{nicknames: [user_one.nickname, user_two.nickname]} + ) - assert json_response(conn, 200) == - %{ - "deactivated" => !user.info.deactivated, - "id" => user.id, - "nickname" => user.nickname, - "roles" => %{"admin" => false, "moderator" => false}, - "local" => true, - "tags" => [], - "avatar" => User.avatar_url(user) |> MediaProxy.url(), - "display_name" => HTML.strip_tags(user.name || user.nickname) - } + response = json_response(conn, 200) + assert Enum.map(response["users"], & &1["deactivated"]) == [false, false] log_entry = Repo.one(ModerationLog) assert ModerationLog.get_log_entry_message(log_entry) == - "@#{admin.nickname} deactivated user @#{user.nickname}" + "@#{admin.nickname} activated users: @#{user_one.nickname}, @#{user_two.nickname}" + end + + test "PATCH /api/pleroma/admin/users/deactivate" do + admin = insert(:user, info: %{is_admin: true}) + user_one = insert(:user, info: %{deactivated: false}) + user_two = insert(:user, info: %{deactivated: false}) + + conn = + build_conn() + |> assign(:user, admin) + |> patch( + "/api/pleroma/admin/users/deactivate", + %{nicknames: [user_one.nickname, user_two.nickname]} + ) + + response = json_response(conn, 200) + assert Enum.map(response["users"], & &1["deactivated"]) == [true, true] + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} deactivated users: @#{user_one.nickname}, @#{user_two.nickname}" end describe "POST /api/pleroma/admin/users/invite_token" do From 02f8e2a8ab65c3e8497bab4576ce4e75f8df3217 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 10 Oct 2019 14:24:54 +0200 Subject: [PATCH 09/36] Gitlab: Run benchmark in CI. --- .gitlab-ci.yml | 14 ++++++++++++++ config/benchmark.exs | 3 ++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d0c540b16..09684df02 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,6 +15,7 @@ cache: stages: - build - test + - benchmark - deploy - release @@ -44,6 +45,19 @@ docs-build: paths: - priv/static/doc +benchmark: + stage: benchmark + variables: + MIX_ENV: benchmark + services: + - name: lainsoykaf/postgres-with-rum + alias: postgres + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + script: + - mix deps.get + - mix ecto.create + - mix ecto.migrate + - mix pleroma.benchmark unit-testing: stage: test diff --git a/config/benchmark.exs b/config/benchmark.exs index b4f5dbdbd..62ba42e28 100644 --- a/config/benchmark.exs +++ b/config/benchmark.exs @@ -29,7 +29,8 @@ email: "admin@example.com", notify_email: "noreply@example.com", skip_thread_containment: false, - federating: false + federating: false, + external_user_synchronization: false config :pleroma, :activitypub, sign_object_fetches: false From 2629493804bb5522903434cef7fe70510725f0c1 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 10 Oct 2019 15:01:06 +0200 Subject: [PATCH 10/36] Benchmark config: Database adjustments. --- config/benchmark.exs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/config/benchmark.exs b/config/benchmark.exs index 62ba42e28..289a0e317 100644 --- a/config/benchmark.exs +++ b/config/benchmark.exs @@ -37,16 +37,11 @@ # Configure your database config :pleroma, Pleroma.Repo, adapter: Ecto.Adapters.Postgres, - username: System.get_env("DB_USER") || "postgres", - database: System.get_env("DB_DATABASE") || "pleroma_test", + username: "postgres", + password: "postgres", + database: "pleroma_test", hostname: System.get_env("DB_HOST") || "localhost", - # username: "pleroma", - password: - System.get_env("DB_PASS") || - "cAUrGezwXjRwd/lIPzZAcwjb/hiZiGi3FIaSGy9l/XsTcGA61FMy7eCBiRcg1DyQ", - # password: "", - pool_size: 10, - timeout: 180_000 + pool: Ecto.Adapters.SQL.Sandbox # Reduce hash rounds for testing config :pbkdf2_elixir, rounds: 1 From b9b00e6e96b458d9c951ce6505ae903fe47ffa57 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 10 Oct 2019 15:31:32 +0200 Subject: [PATCH 11/36] Gitlab CI: Run correct benchmark task. --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 460c1311f..7c3ca5ee0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -57,7 +57,7 @@ benchmark: - mix deps.get - mix ecto.create - mix ecto.migrate - - mix pleroma.benchmark + - mix pleroma.load_testing unit-testing: stage: test From ecd7ac855bfc6a8e2f95c099b442d9f0f9117e6b Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 10 Oct 2019 15:42:54 +0200 Subject: [PATCH 12/36] Benchmark config: Don't use the sql sandbox. --- config/benchmark.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/benchmark.exs b/config/benchmark.exs index 289a0e317..dd99cf5fd 100644 --- a/config/benchmark.exs +++ b/config/benchmark.exs @@ -41,7 +41,7 @@ password: "postgres", database: "pleroma_test", hostname: System.get_env("DB_HOST") || "localhost", - pool: Ecto.Adapters.SQL.Sandbox + pool_size: 10 # Reduce hash rounds for testing config :pbkdf2_elixir, rounds: 1 From 29647dfd09a78fa66a605ff97119a21779d15020 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 10 Oct 2019 17:17:33 +0200 Subject: [PATCH 13/36] Transmogrifier: Save correct ids for incoming deletes. --- lib/pleroma/web/activity_pub/activity_pub.ex | 8 ++++++-- lib/pleroma/web/activity_pub/transmogrifier.ex | 4 ++-- test/web/activity_pub/transmogrifier_test.exs | 3 ++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 9f29087df..3a0c382ca 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -409,7 +409,10 @@ def delete(%User{ap_id: ap_id, follower_address: follower_address} = user) do end end - def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ true) do + def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, options \\ []) do + local = Keyword.get(options, :local, true) + activity_id = Keyword.get(options, :activity_id, nil) + user = User.get_cached_by_ap_id(actor) to = (object.data["to"] || []) ++ (object.data["cc"] || []) @@ -420,7 +423,8 @@ def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ tru "object" => id, "to" => to, "deleted_activity_id" => activity && activity.id - }, + } + |> maybe_put("id", activity_id), {:ok, activity} <- insert(data, local, false), stream_out_participations(object, user), _ <- decrease_replies_count_if_reply(object), diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 872ed0eb2..620b54d3b 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -637,7 +637,7 @@ def handle_incoming( # an error or a tombstone. This would allow us to verify that a deletion actually took # place. def handle_incoming( - %{"type" => "Delete", "object" => object_id, "actor" => actor, "id" => _id} = data, + %{"type" => "Delete", "object" => object_id, "actor" => actor, "id" => id} = data, _options ) do object_id = Utils.get_ap_id(object_id) @@ -646,7 +646,7 @@ def handle_incoming( {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id), :ok <- Containment.contain_origin(actor.ap_id, object.data), - {:ok, activity} <- ActivityPub.delete(object, false) do + {:ok, activity} <- ActivityPub.delete(object, local: false, activity_id: id) do {:ok, activity} else nil -> diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 50c0bfb84..4f9c4af33 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -696,8 +696,9 @@ test "it works for incoming deletes" do |> Map.put("object", object) |> Map.put("actor", activity.data["actor"]) - {:ok, %Activity{local: false}} = Transmogrifier.handle_incoming(data) + {:ok, %Activity{local: false, data: %{"id" => id}}} = Transmogrifier.handle_incoming(data) + assert id == data["id"] refute Activity.get_by_id(activity.id) end From f5104f36bbec7d49d4ff5acee4b9d28223c6474d Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 11 Oct 2019 00:24:31 +0300 Subject: [PATCH 14/36] Deprecate /api/pleroma/admin/users/:nickname/toggle_activation instead of deleting it --- CHANGELOG.md | 3 +-- docs/API/admin_api.md | 8 ++++++ .../web/admin_api/admin_api_controller.ex | 20 ++++++++++++++ lib/pleroma/web/router.ex | 1 + .../admin_api/admin_api_controller_test.exs | 27 +++++++++++++++++++ 5 files changed, 57 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 584c917f4..c996e7476 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,11 +16,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - OAuth: support for hierarchical permissions / [Mastodon 2.4.3 OAuth permissions](https://docs.joinmastodon.org/api/permissions/) - Authentication: Added rate limit for password-authorized actions / login existence checks - Metadata Link: Atom syndication Feed +- Admin API: `/users/:nickname/toggle_activation` endpoint is now deprecated in favor of: `/users/activate`, `/users/deactivate`, both accept `nicknames` array ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) - **Breaking:** Admin API: Return link alongside with token on password reset -- **Breaking:** Admin API: `/users/:nickname/toggle_activation` endpoint was split into two: `/users/activate`, `/users/deactivate`, both accept `nicknames` array - **Breaking:** Admin API: `POST /users/permission_group/:permission_group` / `DELETE /users/permission_group/:permission_group` now accept `nicknames` array - Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) - Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler @@ -42,7 +42,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Removed - **Breaking:** GNU Social API with Qvitter extensions support -- **Breaking:** Admin API: `/users/:nickname/activation_status` was removed in favor of `/users/activate`, `/users/deactivate` - Emoji: Remove longfox emojis. - Remove `Reply-To` header from report emails for admins. diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index 55f8749e1..b0b827960 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -211,6 +211,14 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret } ``` +## DEPRECATED `PATCH /api/pleroma/admin/users/:nickname/activation_status` + +### Active or deactivate a user + +- Params: + - `nickname` + - `status` BOOLEAN field, false value means deactivation. + ## `/api/pleroma/admin/users/:nickname_or_id` ### Retrive the details of a user diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index d825a5d28..5b513bd7c 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -46,6 +46,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do :user_delete, :users_create, :user_toggle_activation, + :user_activate, + :user_deactivate, :tag_users, :untag_users, :right_add, @@ -231,6 +233,24 @@ def list_user_statuses(conn, %{"nickname" => nickname} = params) do end end + def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do + user = User.get_cached_by_nickname(nickname) + + {:ok, updated_user} = User.deactivate(user, !user.info.deactivated) + + action = if user.info.deactivated, do: "activate", else: "deactivate" + + ModerationLog.insert_log(%{ + actor: admin, + subject: [user], + action: action + }) + + conn + |> put_view(AccountView) + |> render("show.json", %{user: updated_user}) + end + def user_activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do users = Enum.map(nicknames, &User.get_cached_by_nickname/1) {:ok, updated_users} = User.deactivate(users, false) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 894375357..a79df51a2 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -136,6 +136,7 @@ defmodule Pleroma.Web.Router do delete("/users", AdminAPIController, :user_delete) post("/users", AdminAPIController, :users_create) + patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation) patch("/users/activate", AdminAPIController, :user_activate) patch("/users/deactivate", AdminAPIController, :user_deactivate) put("/users/tag", AdminAPIController, :tag_users) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index c57c71203..c1b325a24 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -1023,6 +1023,33 @@ test "PATCH /api/pleroma/admin/users/deactivate" do "@#{admin.nickname} deactivated users: @#{user_one.nickname}, @#{user_two.nickname}" end + test "PATCH /api/pleroma/admin/users/:nickname/toggle_activation" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user) + + conn = + build_conn() + |> assign(:user, admin) + |> patch("/api/pleroma/admin/users/#{user.nickname}/toggle_activation") + + assert json_response(conn, 200) == + %{ + "deactivated" => !user.info.deactivated, + "id" => user.id, + "nickname" => user.nickname, + "roles" => %{"admin" => false, "moderator" => false}, + "local" => true, + "tags" => [], + "avatar" => User.avatar_url(user) |> MediaProxy.url(), + "display_name" => HTML.strip_tags(user.name || user.nickname) + } + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} deactivated users: @#{user.nickname}" + end + describe "POST /api/pleroma/admin/users/invite_token" do setup do admin = insert(:user, info: %{is_admin: true}) From 9b963064eb4c8cb740ffa128f491f2ee581fdb8b Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 11 Oct 2019 11:25:45 +0200 Subject: [PATCH 15/36] Transmogrifier: Actually store who deleted a note. --- lib/pleroma/web/activity_pub/activity_pub.ex | 18 ++++++++++-------- lib/pleroma/web/activity_pub/transmogrifier.ex | 3 ++- test/web/activity_pub/transmogrifier_test.exs | 7 +++++-- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 3a0c382ca..f78574455 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -412,19 +412,21 @@ def delete(%User{ap_id: ap_id, follower_address: follower_address} = user) do def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, options \\ []) do local = Keyword.get(options, :local, true) activity_id = Keyword.get(options, :activity_id, nil) + actor = Keyword.get(options, :actor, actor) user = User.get_cached_by_ap_id(actor) to = (object.data["to"] || []) ++ (object.data["cc"] || []) with {:ok, object, activity} <- Object.delete(object), - data <- %{ - "type" => "Delete", - "actor" => actor, - "object" => id, - "to" => to, - "deleted_activity_id" => activity && activity.id - } - |> maybe_put("id", activity_id), + data <- + %{ + "type" => "Delete", + "actor" => actor, + "object" => id, + "to" => to, + "deleted_activity_id" => activity && activity.id + } + |> maybe_put("id", activity_id), {:ok, activity} <- insert(data, local, false), stream_out_participations(object, user), _ <- decrease_replies_count_if_reply(object), diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 620b54d3b..64fbb9fa4 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -646,7 +646,8 @@ def handle_incoming( {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor), {:ok, object} <- get_obj_helper(object_id), :ok <- Containment.contain_origin(actor.ap_id, object.data), - {:ok, activity} <- ActivityPub.delete(object, local: false, activity_id: id) do + {:ok, activity} <- + ActivityPub.delete(object, local: false, activity_id: id, actor: actor.ap_id) do {:ok, activity} else nil -> diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 4f9c4af33..9f98ec708 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -682,6 +682,7 @@ test "it works for incoming update activities which lock the account" do test "it works for incoming deletes" do activity = insert(:note_activity) + deleting_user = insert(:user) data = File.read!("test/fixtures/mastodon-delete.json") @@ -694,12 +695,14 @@ test "it works for incoming deletes" do data = data |> Map.put("object", object) - |> Map.put("actor", activity.data["actor"]) + |> Map.put("actor", deleting_user.ap_id) - {:ok, %Activity{local: false, data: %{"id" => id}}} = Transmogrifier.handle_incoming(data) + {:ok, %Activity{actor: actor, local: false, data: %{"id" => id}}} = + Transmogrifier.handle_incoming(data) assert id == data["id"] refute Activity.get_by_id(activity.id) + assert actor == deleting_user.ap_id end test "it fails for incoming deletes with spoofed origin" do From 37812740c4c03f0a79d17ff186db644a60414ff7 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 11 Oct 2019 11:48:58 +0200 Subject: [PATCH 16/36] Transmogrifier: Correctly save incoming ids for Accept/Reject. --- lib/pleroma/web/activity_pub/activity_pub.ex | 25 +++++++++---------- .../web/activity_pub/transmogrifier.ex | 10 +++++--- test/web/activity_pub/transmogrifier_test.exs | 3 +++ 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index f78574455..364452b5d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -269,22 +269,21 @@ def listen(%{to: to, actor: actor, context: context, object: object} = params) d end end - def accept(%{to: to, actor: actor, object: object} = params) do - # only accept false as false value - local = !(params[:local] == false) - - with data <- %{"to" => to, "type" => "Accept", "actor" => actor.ap_id, "object" => object}, - {:ok, activity} <- insert(data, local), - :ok <- maybe_federate(activity) do - {:ok, activity} - end + def accept(params) do + accept_or_reject("Accept", params) end - def reject(%{to: to, actor: actor, object: object} = params) do - # only accept false as false value - local = !(params[:local] == false) + def reject(params) do + accept_or_reject("Reject", params) + end - with data <- %{"to" => to, "type" => "Reject", "actor" => actor.ap_id, "object" => object}, + def accept_or_reject(type, %{to: to, actor: actor, object: object} = params) do + local = Map.get(params, :local, true) + activity_id = Map.get(params, :activity_id, nil) + + with data <- + %{"to" => to, "type" => type, "actor" => actor.ap_id, "object" => object} + |> Utils.maybe_put("id", activity_id), {:ok, activity} <- insert(data, local), :ok <- maybe_federate(activity) do {:ok, activity} diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 64fbb9fa4..b56343beb 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -514,7 +514,7 @@ def handle_incoming( end def handle_incoming( - %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data, + %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => id} = data, _options ) do with actor <- Containment.get_actor(data), @@ -528,7 +528,8 @@ def handle_incoming( type: "Accept", actor: followed, object: follow_activity.data["id"], - local: false + local: false, + activity_id: id }) else _e -> :error @@ -536,7 +537,7 @@ def handle_incoming( end def handle_incoming( - %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data, + %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => id} = data, _options ) do with actor <- Containment.get_actor(data), @@ -550,7 +551,8 @@ def handle_incoming( type: "Reject", actor: followed, object: follow_activity.data["id"], - local: false + local: false, + activity_id: id }) do User.unfollow(follower, followed) diff --git a/test/web/activity_pub/transmogrifier_test.exs b/test/web/activity_pub/transmogrifier_test.exs index 9f98ec708..6c35a6f4d 100644 --- a/test/web/activity_pub/transmogrifier_test.exs +++ b/test/web/activity_pub/transmogrifier_test.exs @@ -909,6 +909,8 @@ test "it works for incoming accepts which were pre-accepted" do assert activity.data["object"] == follow_activity.data["id"] + assert activity.data["id"] == accept_data["id"] + follower = User.get_cached_by_id(follower.id) assert User.following?(follower, followed) == true @@ -1013,6 +1015,7 @@ test "it works for incoming rejects which are orphaned" do {:ok, activity} = Transmogrifier.handle_incoming(reject_data) refute activity.local + assert activity.data["id"] == reject_data["id"] follower = User.get_cached_by_id(follower.id) From b750135acd9d1f8444f85ea5e0345483599daed9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 11 Oct 2019 13:22:21 +0300 Subject: [PATCH 17/36] pleroma_ctl: Make it possible to specify branches/flavours when updating --- rel/files/bin/pleroma_ctl | 94 +++++++++++++++++++++++++++------------ 1 file changed, 65 insertions(+), 29 deletions(-) diff --git a/rel/files/bin/pleroma_ctl b/rel/files/bin/pleroma_ctl index e731d20eb..f767fe134 100755 --- a/rel/files/bin/pleroma_ctl +++ b/rel/files/bin/pleroma_ctl @@ -37,29 +37,60 @@ detect_branch() { elif [ "$branch" = "" ]; then echo "master" else - # Note: branch name in version is of SemVer format and may only contain [0-9a-zA-Z-] symbols — - # if supporting releases for more branches, need to ensure they contain only these symbols. + # Note: branch name in version is of SemVer format and may only contain [0-9a-zA-Z-] symbols — + # if supporting releases for more branches, need to ensure they contain only these symbols. echo "Releases are built only for master and develop branches" >&2 exit 1 fi } update() { set -e + NO_RM=false + + while echo "$1" | grep "^-" >/dev/null; do + case "$1" in + --zip-url) + FULL_URI="$2" + shift 2 + ;; + --no-rm) + NO_RM=true + shift + ;; + --flavour) + FLAVOUR="$2" + shift 2 + ;; + --branch) + BRANCH="$2" + shift 2 + ;; + --tmp-dir) + TMP_DIR="$2" + shift 2 + ;; + -*) + echo "invalid option: $1" 1>&2 + shift + ;; + esac + done + RELEASE_ROOT=$(dirname "$SCRIPTPATH") - uri="${PLEROMA_CTL_URI:-https://git.pleroma.social}" - project_id="${PLEROMA_CTL_PROJECT_ID:-2}" - project_branch="$(detect_branch)" - flavour="${PLEROMA_CTL_FLAVOUR:-$(detect_flavour)}" - echo "Detected flavour: $flavour" - tmp="${PLEROMA_CTL_TMP_DIR:-/tmp}" + uri="https://git.pleroma.social" + project_id="2" + project_branch="${BRANCH:-$(detect_branch)}" + flavour="${FLAVOUR:-$(detect_flavour)}" + tmp="${TMP_DIR:-/tmp}" artifact="$tmp/pleroma.zip" - full_uri="${uri}/api/v4/projects/${project_id}/jobs/artifacts/${project_branch}/download?job=${flavour}" + full_uri="${FULL_URI:-${uri}/api/v4/projects/${project_id}/jobs/artifacts/${project_branch}/download?job=${flavour}}" echo "Downloading the artifact from ${full_uri} to ${artifact}" curl "$full_uri" -o "${artifact}" echo "Unpacking ${artifact} to ${tmp}" unzip -q "$artifact" -d "$tmp" echo "Copying files over to $RELEASE_ROOT" - if [ "$1" != "--no-rm" ]; then + if [ "$NO_RM" = false ]; then + echo "Removing files from the previous release" rm -r "${RELEASE_ROOT:-?}"/* fi cp -rf "$tmp/release"/* "$RELEASE_ROOT" @@ -86,36 +117,41 @@ if [ -z "$1" ] || [ "$1" = "help" ]; then Rollback database migrations (needs to be done before downgrading) update [OPTIONS] - Update the instance using the latest CI artifact for the current branch. - - The only supported option is --no-rm, when set the script won't delete the whole directory, but - just force copy over files from the new release. This wastes more space, but may be useful if - some files are stored inside of the release directories (although you really shouldn't store them - there), or if you want to be able to quickly revert a broken update. - - The script will try to detect your architecture and ABI and set a flavour automatically, - but if it is wrong, you can overwrite it by setting PLEROMA_CTL_FLAVOUR to the desired flavour. - - By default the artifact will be downloaded from https://git.pleroma.social for pleroma/pleroma (project id: 2) - to /tmp/, you can overwrite these settings by setting PLEROMA_CTL_URI, PLEROMA_CTL_PROJECT_ID and PLEROMA_CTL_TMP_DIR - respectively. + Update the instance. + Options: + --branch Update to a specified branch, instead of the latest version of the current one. + --flavour Update to a specified flavour (CPU architecture+libc), instead of the current one. + --zip-url Get the release from a specified url. If set, renders the previous 2 options inactive. + --no-rm Do not erase previous release's files. + --tmp-dir Download the temporary files to a specified directory. and any mix tasks under Pleroma namespace, for example \`mix pleroma.user COMMAND\` is equivalent to \`$(basename "$0") user COMMAND\` By default pleroma_ctl will try calling into a running instance to execute non migration-related commands, - if for some reason this is undesired, set PLEROMA_CTL_RPC_DISABLED environment variable + if for some reason this is undesired, set PLEROMA_CTL_RPC_DISABLED environment variable. + " else SCRIPT=$(readlink -f "$0") SCRIPTPATH=$(dirname "$SCRIPT") - if [ "$1" = "update" ]; then - update "$2" - elif [ "$1" = "migrate" ] || [ "$1" = "rollback" ] || [ "$1" = "create" ] || [ "$1 $2" = "instance gen" ] || [ -n "$PLEROMA_CTL_RPC_DISABLED" ]; then - "$SCRIPTPATH"/pleroma eval 'Pleroma.ReleaseTasks.run("'"$*"'")' + FULL_ARGS="$*" + + ACTION="$1" + shift + + if [ "$(echo \"$1\" | grep \"^-\" >/dev/null)" = false ]; then + SUBACTION="$1" + shift + fi + + if [ "$ACTION" = "update" ]; then + update "$@" + elif [ "$ACTION" = "migrate" ] || [ "$ACTION" = "rollback" ] || [ "$ACTION" = "create" ] || [ "$ACTION $SUBACTION" = "instance gen" ] || [ "$PLEROMA_CTL_RPC_DISABLED" = true ]; then + "$SCRIPTPATH"/pleroma eval 'Pleroma.ReleaseTasks.run("'"$FULL_ARGS"'")' else - "$SCRIPTPATH"/pleroma rpc 'Pleroma.ReleaseTasks.run("'"$*"'")' + "$SCRIPTPATH"/pleroma rpc 'Pleroma.ReleaseTasks.run("'"$FULL_ARGS"'")' fi fi From cb8492962e2a765090dbce82b8fc9f000d49bfa1 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 11 Oct 2019 12:41:44 +0200 Subject: [PATCH 18/36] SearchController: Fix test. Turns out you can't actually find the user with this. --- .../mastodon_api/controllers/search_controller_test.exs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/web/mastodon_api/controllers/search_controller_test.exs b/test/web/mastodon_api/controllers/search_controller_test.exs index 0ca896e01..ee413eef7 100644 --- a/test/web/mastodon_api/controllers/search_controller_test.exs +++ b/test/web/mastodon_api/controllers/search_controller_test.exs @@ -40,9 +40,9 @@ test "it returns empty result if user or status search return undefined error", test "search", %{conn: conn} do user = insert(:user) user_two = insert(:user, %{nickname: "shp@shitposter.club"}) - user_three = insert(:user, %{nickname: "shp@heldscal.la", name: "I love 2hu 天子"}) + user_three = insert(:user, %{nickname: "shp@heldscal.la", name: "I love 2hu"}) - {:ok, activity} = CommonAPI.post(user, %{"status" => "This is about 2hu private"}) + {:ok, activity} = CommonAPI.post(user, %{"status" => "This is about 2hu private 天子"}) {:ok, _activity} = CommonAPI.post(user, %{ @@ -70,8 +70,8 @@ test "search", %{conn: conn} do get(conn, "/api/v2/search", %{"q" => "天子"}) |> json_response(200) - [account] == results["accounts"] - assert account["id"] == to_string(user_three.id) + [status] = results["statuses"] + assert status["id"] == to_string(activity.id) end end From 422aa6befee0cbcbf71d3cacd96d90c1be3263ba Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 11 Oct 2019 12:53:09 +0200 Subject: [PATCH 19/36] Ostatus DeleteHandler: Fix for new option format. --- lib/pleroma/web/ostatus/handlers/delete_handler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/ostatus/handlers/delete_handler.ex b/lib/pleroma/web/ostatus/handlers/delete_handler.ex index b2f9f3946..ac2dc115c 100644 --- a/lib/pleroma/web/ostatus/handlers/delete_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/delete_handler.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.OStatus.DeleteHandler do def handle_delete(entry, _doc \\ nil) do with id <- XML.string_from_xpath("//id", entry), %Object{} = object <- Object.normalize(id), - {:ok, delete} <- ActivityPub.delete(object, false) do + {:ok, delete} <- ActivityPub.delete(object, local: false) do delete end end From aaa4252f416fbad099f95232de4cf6eab11dd7d2 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Fri, 11 Oct 2019 15:58:45 +0300 Subject: [PATCH 20/36] Deprecate POST/DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group instead of deleting it --- CHANGELOG.md | 3 +- docs/API/admin_api.md | 23 ++++++- .../web/admin_api/admin_api_controller.ex | 61 ++++++++++++++++++- lib/pleroma/web/router.ex | 18 +++++- .../admin_api/admin_api_controller_test.exs | 40 ++++++++++++ 5 files changed, 137 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c996e7476..f06ad365d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,11 +17,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Authentication: Added rate limit for password-authorized actions / login existence checks - Metadata Link: Atom syndication Feed - Admin API: `/users/:nickname/toggle_activation` endpoint is now deprecated in favor of: `/users/activate`, `/users/deactivate`, both accept `nicknames` array +- Admin API: `POST /api/pleroma/admin/users/:nickname/permission_group/:permission_group` / `DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group` are deprecated in favor of: `POST /api/pleroma/admin/users/permission_group/:permission_group` / `DELETE /api/pleroma/admin/users/permission_group/:permission_group` (both accept `nicknames` array) + ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) - **Breaking:** Admin API: Return link alongside with token on password reset -- **Breaking:** Admin API: `POST /users/permission_group/:permission_group` / `DELETE /users/permission_group/:permission_group` now accept `nicknames` array - Replaced [pleroma_job_queue](https://git.pleroma.social/pleroma/pleroma_job_queue) and `Pleroma.Web.Federator.RetryQueue` with [Oban](https://github.com/sorentwo/oban) (see [`docs/config.md`](docs/config.md) on migrating customized worker / retry settings) - Introduced [quantum](https://github.com/quantum-elixir/quantum-core) job scheduler - Admin API: Return `total` when querying for reports diff --git a/docs/API/admin_api.md b/docs/API/admin_api.md index b0b827960..2c8237b57 100644 --- a/docs/API/admin_api.md +++ b/docs/API/admin_api.md @@ -154,9 +154,18 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret } ``` +## DEPRECATED `POST /api/pleroma/admin/users/:nickname/permission_group/:permission_group` + +### Add user to permission group + +- Params: none +- Response: + - On failure: `{"error": "…"}` + - On success: JSON of the `user.info` + ## `POST /api/pleroma/admin/users/permission_group/:permission_group` -### Add user in permission group +### Add users to permission group - Params: - `nicknames`: nicknames array @@ -164,10 +173,20 @@ Note: Available `:permission_group` is currently moderator and admin. 404 is ret - On failure: `{"error": "…"}` - On success: JSON of the `user.info` -## `DELETE /api/pleroma/admin/users/permission_group/:permission_group` +## DEPRECATED `DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group` ### Remove user from permission group +- Params: none +- Response: + - On failure: `{"error": "…"}` + - On success: JSON of the `user.info` +- Note: An admin cannot revoke their own admin status. + +## `DELETE /api/pleroma/admin/users/permission_group/:permission_group` + +### Remove users from permission group + - Params: - `nicknames`: nicknames array - Response: diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex index 5b513bd7c..33e2180ec 100644 --- a/lib/pleroma/web/admin_api/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/admin_api_controller.ex @@ -345,7 +345,7 @@ defp maybe_parse_filters(filters) do |> Enum.into(%{}, &{&1, true}) end - def right_add(%{assigns: %{user: admin}} = conn, %{ + def right_add_multiple(%{assigns: %{user: admin}} = conn, %{ "permission_group" => permission_group, "nicknames" => nicknames }) @@ -366,6 +366,32 @@ def right_add(%{assigns: %{user: admin}} = conn, %{ json(conn, info) end + def right_add_multiple(conn, _) do + render_error(conn, :not_found, "No such permission_group") + end + + def right_add(%{assigns: %{user: admin}} = conn, %{ + "permission_group" => permission_group, + "nickname" => nickname + }) + when permission_group in ["moderator", "admin"] do + info = Map.put(%{}, "is_" <> permission_group, true) + + {:ok, user} = + nickname + |> User.get_cached_by_nickname() + |> User.update_info(&User.Info.admin_api_update(&1, info)) + + ModerationLog.insert_log(%{ + action: "grant", + actor: admin, + subject: [user], + permission: permission_group + }) + + json(conn, info) + end + def right_add(conn, _) do render_error(conn, :not_found, "No such permission_group") end @@ -380,7 +406,7 @@ def right_get(conn, %{"nickname" => nickname}) do }) end - def right_delete( + def right_delete_multiple( %{assigns: %{user: %{nickname: admin_nickname} = admin}} = conn, %{ "permission_group" => permission_group, @@ -408,10 +434,39 @@ def right_delete( end end - def right_delete(conn, _) do + def right_delete_multiple(conn, _) do render_error(conn, :not_found, "No such permission_group") end + def right_delete( + %{assigns: %{user: admin}} = conn, + %{ + "permission_group" => permission_group, + "nickname" => nickname + } + ) + when permission_group in ["moderator", "admin"] do + info = Map.put(%{}, "is_" <> permission_group, false) + + {:ok, user} = + nickname + |> User.get_cached_by_nickname() + |> User.update_info(&User.Info.admin_api_update(&1, info)) + + ModerationLog.insert_log(%{ + action: "revoke", + actor: admin, + subject: [user], + permission: permission_group + }) + + json(conn, info) + end + + def right_delete(%{assigns: %{user: %{nickname: nickname}}} = conn, %{"nickname" => nickname}) do + render_error(conn, :forbidden, "You can't revoke your own admin status.") + end + def relay_follow(%{assigns: %{user: admin}} = conn, %{"relay_url" => target}) do with {:ok, _message} <- Relay.follow(target) do ModerationLog.insert_log(%{ diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index a79df51a2..80651f3ff 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -144,8 +144,22 @@ defmodule Pleroma.Web.Router do get("/users/:nickname/permission_group", AdminAPIController, :right_get) get("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_get) - post("/users/permission_group/:permission_group", AdminAPIController, :right_add) - delete("/users/permission_group/:permission_group", AdminAPIController, :right_delete) + + post("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_add) + + delete( + "/users/:nickname/permission_group/:permission_group", + AdminAPIController, + :right_delete + ) + + post("/users/permission_group/:permission_group", AdminAPIController, :right_add_multiple) + + delete( + "/users/permission_group/:permission_group", + AdminAPIController, + :right_delete_multiple + ) post("/relay", AdminAPIController, :relay_follow) delete("/relay", AdminAPIController, :relay_unfollow) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index c1b325a24..81dc5d101 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -385,6 +385,26 @@ test "GET is giving user_info" do end test "/:right POST, can add to a permission group" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user) + + conn = + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> post("/api/pleroma/admin/users/#{user.nickname}/permission_group/admin") + + assert json_response(conn, 200) == %{ + "is_admin" => true + } + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} made @#{user.nickname} admin" + end + + test "/:right POST, can add to a permission group (multiple)" do admin = insert(:user, info: %{is_admin: true}) user_one = insert(:user) user_two = insert(:user) @@ -408,6 +428,26 @@ test "/:right POST, can add to a permission group" do end test "/:right DELETE, can remove from a permission group" do + admin = insert(:user, info: %{is_admin: true}) + user = insert(:user, info: %{is_admin: true}) + + conn = + build_conn() + |> assign(:user, admin) + |> put_req_header("accept", "application/json") + |> delete("/api/pleroma/admin/users/#{user.nickname}/permission_group/admin") + + assert json_response(conn, 200) == %{ + "is_admin" => false + } + + log_entry = Repo.one(ModerationLog) + + assert ModerationLog.get_log_entry_message(log_entry) == + "@#{admin.nickname} revoked admin role from @#{user.nickname}" + end + + test "/:right DELETE, can remove from a permission group (multiple)" do admin = insert(:user, info: %{is_admin: true}) user_one = insert(:user, info: %{is_admin: true}) user_two = insert(:user, info: %{is_admin: true}) From 9bdbf0811b362d8eb9d2456e9c87c9dde1b35d9b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 11 Oct 2019 22:52:38 +0300 Subject: [PATCH 21/36] Make MediaProxy failure tracking less brutal The current failure tracking mechanism will never request anything that didn't respond with a success, 403, 404, or 5xx codes. This is causing issues when using in real fediverse because of weird status codes some software has and timeouts being frequent. This patch changes failure tracking mechanism to only never request the url again if it responded with 400, 204, or the body is too large, otherwise it can be re-requested in 60 seconds. --- lib/pleroma/reverse_proxy/reverse_proxy.ex | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 78144cae3..2ed719315 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -401,11 +401,9 @@ defp increase_read_duration(_) do defp client, do: Pleroma.ReverseProxy.Client - defp track_failed_url(url, code, opts) do - code = to_string(code) - + defp track_failed_url(url, error, opts) do ttl = - if code in ["403", "404"] or String.starts_with?(code, "5") do + unless error in [:body_too_large, 400, 204] do Keyword.get(opts, :failed_request_ttl, @failed_request_ttl) else nil From a97b642289659a5fccb5943c54caa1ecdce5fd2f Mon Sep 17 00:00:00 2001 From: eugenijm Date: Tue, 8 Oct 2019 23:05:57 +0300 Subject: [PATCH 22/36] Mastodon API: Add `exclude_visibilities` parameter to the timeline and notification endpoints --- CHANGELOG.md | 1 + docs/API/differences_in_mastoapi_responses.md | 7 ++ lib/pleroma/notification.ex | 108 +++++++++++++----- lib/pleroma/web/activity_pub/activity_pub.ex | 44 +++++++ lib/pleroma/web/mastodon_api/mastodon_api.ex | 1 + test/web/activity_pub/activity_pub_test.exs | 60 ++++++++++ .../controllers/account_controller_test.exs | 14 +++ .../notification_controller_test.exs | 51 +++++++++ .../controllers/timeline_controller_test.exs | 55 ++++++--- 9 files changed, 297 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd06ec866..e3839391b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Authentication: Added rate limit for password-authorized actions / login existence checks - Metadata Link: Atom syndication Feed - Mix task to re-count statuses for all users (`mix pleroma.count_statuses`) +- Mastodon API: Add `exclude_visibilities` parameter to the timeline and notification endpoints ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) diff --git a/docs/API/differences_in_mastoapi_responses.md b/docs/API/differences_in_mastoapi_responses.md index 21b297529..aca0f5e0e 100644 --- a/docs/API/differences_in_mastoapi_responses.md +++ b/docs/API/differences_in_mastoapi_responses.md @@ -13,6 +13,7 @@ Some apps operate under the assumption that no more than 4 attachments can be re ## Timelines Adding the parameter `with_muted=true` to the timeline queries will also return activities by muted (not by blocked!) users. +Adding the parameter `exclude_visibilities` to the timeline queries will exclude the statuses with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`), e.g., `exclude_visibilities[]=direct&exclude_visibilities[]=private`. ## Statuses @@ -84,6 +85,12 @@ Has these additional fields under the `pleroma` object: - `is_seen`: true if the notification was read by the user +## GET `/api/v1/notifications` + +Accepts additional parameters: + +- `exclude_visibilities`: will exclude the notifications for activities with the given visibilities. The parameter accepts an array of visibility types (`public`, `unlisted`, `private`, `direct`). Usage example: `GET /api/v1/notifications?exclude_visibilities[]=direct&exclude_visibilities[]=private`. + ## POST `/api/v1/statuses` Additional parameters can be added to the JSON body/Form data: diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index d94ae5971..d145f8d5b 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -17,6 +17,7 @@ defmodule Pleroma.Notification do import Ecto.Query import Ecto.Changeset + require Logger @type t :: %__MODULE__{} @@ -34,43 +35,92 @@ def changeset(%Notification{} = notification, attrs) do end def for_user_query(user, opts \\ []) do - query = - Notification - |> where(user_id: ^user.id) + Notification + |> where(user_id: ^user.id) + |> where( + [n, a], + fragment( + "? not in (SELECT ap_id FROM users WHERE info->'deactivated' @> 'true')", + a.actor + ) + ) + |> join(:inner, [n], activity in assoc(n, :activity)) + |> join(:left, [n, a], object in Object, + on: + fragment( + "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + object.data, + a.data + ) + ) + |> preload([n, a, o], activity: {a, object: o}) + |> exclude_muted(user, opts) + |> exclude_visibility(opts) + end + + defp exclude_muted(query, _, %{with_muted: true}) do + query + end + + defp exclude_muted(query, user, _opts) do + query + |> where([n, a], a.actor not in ^user.info.muted_notifications) + |> where([n, a], a.actor not in ^user.info.blocks) + |> where( + [n, a], + fragment("substring(? from '.*://([^/]*)')", a.actor) not in ^user.info.domain_blocks + ) + |> join(:left, [n, a], tm in Pleroma.ThreadMute, + on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data) + ) + |> where([n, a, o, tm], is_nil(tm.user_id)) + end + + @valid_visibilities ~w[direct unlisted public private] + + defp exclude_visibility(query, %{exclude_visibilities: visibility}) + when is_list(visibility) do + if Enum.all?(visibility, &(&1 in @valid_visibilities)) do + query |> where( [n, a], - fragment( - "? not in (SELECT ap_id FROM users WHERE info->'deactivated' @> 'true')", - a.actor + not fragment( + "activity_visibility(?, ?, ?) = ANY (?)", + a.actor, + a.recipients, + a.data, + ^visibility ) ) - |> join(:inner, [n], activity in assoc(n, :activity)) - |> join(:left, [n, a], object in Object, - on: - fragment( - "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", - object.data, - a.data - ) - ) - |> preload([n, a, o], activity: {a, object: o}) - - if opts[:with_muted] do - query else - where(query, [n, a], a.actor not in ^user.info.muted_notifications) - |> where([n, a], a.actor not in ^user.info.blocks) - |> where( - [n, a], - fragment("substring(? from '.*://([^/]*)')", a.actor) not in ^user.info.domain_blocks - ) - |> join(:left, [n, a], tm in Pleroma.ThreadMute, - on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data) - ) - |> where([n, a, o, tm], is_nil(tm.user_id)) + Logger.error("Could not exclude visibility to #{visibility}") + query end end + defp exclude_visibility(query, %{exclude_visibilities: visibility}) + when visibility in @valid_visibilities do + query + |> where( + [n, a], + not fragment( + "activity_visibility(?, ?, ?) = (?)", + a.actor, + a.recipients, + a.data, + ^visibility + ) + ) + end + + defp exclude_visibility(query, %{exclude_visibilities: visibility}) + when visibility not in @valid_visibilities do + Logger.error("Could not exclude visibility to #{visibility}") + query + end + + defp exclude_visibility(query, _visibility), do: query + def for_user(user, opts \\ %{}) do user |> for_user_query(opts) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 364452b5d..1d34c4d7e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -596,6 +596,49 @@ defp restrict_visibility(_query, %{visibility: visibility}) defp restrict_visibility(query, _visibility), do: query + defp exclude_visibility(query, %{"exclude_visibilities" => visibility}) + when is_list(visibility) do + if Enum.all?(visibility, &(&1 in @valid_visibilities)) do + from( + a in query, + where: + not fragment( + "activity_visibility(?, ?, ?) = ANY (?)", + a.actor, + a.recipients, + a.data, + ^visibility + ) + ) + else + Logger.error("Could not exclude visibility to #{visibility}") + query + end + end + + defp exclude_visibility(query, %{"exclude_visibilities" => visibility}) + when visibility in @valid_visibilities do + from( + a in query, + where: + not fragment( + "activity_visibility(?, ?, ?) = ?", + a.actor, + a.recipients, + a.data, + ^visibility + ) + ) + end + + defp exclude_visibility(query, %{"exclude_visibilities" => visibility}) + when visibility not in @valid_visibilities do + Logger.error("Could not exclude visibility to #{visibility}") + query + end + + defp exclude_visibility(query, _visibility), do: query + defp restrict_thread_visibility(query, _, %{skip_thread_containment: true} = _), do: query @@ -960,6 +1003,7 @@ def fetch_activities_query(recipients, opts \\ %{}) do |> restrict_muted_reblogs(opts) |> Activity.restrict_deactivated_users() |> exclude_poll_votes(opts) + |> exclude_visibility(opts) end def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index ac01d1ff3..d875a5788 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -71,6 +71,7 @@ def get_scheduled_activities(user, params \\ %{}) do defp cast_params(params) do param_types = %{ exclude_types: {:array, :string}, + exclude_visibilities: {:array, :string}, reblogs: :boolean, with_muted: :boolean } diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index c9f2a92e7..3a5a2f984 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -87,6 +87,66 @@ test "it restricts by the appropriate visibility" do end end + describe "fetching excluded by visibility" do + test "it excludes by the appropriate visibility" do + user = insert(:user) + + {:ok, public_activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "public"}) + + {:ok, direct_activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "direct"}) + + {:ok, unlisted_activity} = + CommonAPI.post(user, %{"status" => ".", "visibility" => "unlisted"}) + + {:ok, private_activity} = + CommonAPI.post(user, %{"status" => ".", "visibility" => "private"}) + + activities = + ActivityPub.fetch_activities([], %{ + "exclude_visibilities" => "direct", + "actor_id" => user.ap_id + }) + + assert public_activity in activities + assert unlisted_activity in activities + assert private_activity in activities + refute direct_activity in activities + + activities = + ActivityPub.fetch_activities([], %{ + "exclude_visibilities" => "unlisted", + "actor_id" => user.ap_id + }) + + assert public_activity in activities + refute unlisted_activity in activities + assert private_activity in activities + assert direct_activity in activities + + activities = + ActivityPub.fetch_activities([], %{ + "exclude_visibilities" => "private", + "actor_id" => user.ap_id + }) + + assert public_activity in activities + assert unlisted_activity in activities + refute private_activity in activities + assert direct_activity in activities + + activities = + ActivityPub.fetch_activities([], %{ + "exclude_visibilities" => "public", + "actor_id" => user.ap_id + }) + + refute public_activity in activities + assert unlisted_activity in activities + assert private_activity in activities + assert direct_activity in activities + end + end + describe "building a user from his ap id" do test "it returns a user" do user_id = "http://mastodon.example.org/users/admin" diff --git a/test/web/mastodon_api/controllers/account_controller_test.exs b/test/web/mastodon_api/controllers/account_controller_test.exs index 6a59c3d94..745383757 100644 --- a/test/web/mastodon_api/controllers/account_controller_test.exs +++ b/test/web/mastodon_api/controllers/account_controller_test.exs @@ -237,6 +237,20 @@ test "filters user's statuses by a hashtag", %{conn: conn} do assert [%{"id" => id}] = json_response(conn, 200) assert id == to_string(post.id) end + + test "the user views their own timelines and excludes direct messages", %{conn: conn} do + user = insert(:user) + {:ok, public_activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "public"}) + {:ok, _direct_activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "direct"}) + + conn = + conn + |> assign(:user, user) + |> get("/api/v1/accounts/#{user.id}/statuses", %{"exclude_visibilities" => ["direct"]}) + + assert [%{"id" => id}] = json_response(conn, 200) + assert id == to_string(public_activity.id) + end end describe "followers" do diff --git a/test/web/mastodon_api/controllers/notification_controller_test.exs b/test/web/mastodon_api/controllers/notification_controller_test.exs index e4137e92c..fa55a7cf9 100644 --- a/test/web/mastodon_api/controllers/notification_controller_test.exs +++ b/test/web/mastodon_api/controllers/notification_controller_test.exs @@ -137,6 +137,57 @@ test "paginates notifications using min_id, since_id, max_id, and limit", %{conn assert [%{"id" => ^notification3_id}, %{"id" => ^notification2_id}] = result end + test "filters notifications using exclude_visibilities", %{conn: conn} do + user = insert(:user) + other_user = insert(:user) + + {:ok, public_activity} = + CommonAPI.post(other_user, %{"status" => "@#{user.nickname}", "visibility" => "public"}) + + {:ok, direct_activity} = + CommonAPI.post(other_user, %{"status" => "@#{user.nickname}", "visibility" => "direct"}) + + {:ok, unlisted_activity} = + CommonAPI.post(other_user, %{"status" => "@#{user.nickname}", "visibility" => "unlisted"}) + + {:ok, private_activity} = + CommonAPI.post(other_user, %{"status" => "@#{user.nickname}", "visibility" => "private"}) + + conn = assign(conn, :user, user) + + conn_res = + get(conn, "/api/v1/notifications", %{ + exclude_visibilities: ["public", "unlisted", "private"] + }) + + assert [%{"status" => %{"id" => id}}] = json_response(conn_res, 200) + assert id == direct_activity.id + + conn_res = + get(conn, "/api/v1/notifications", %{ + exclude_visibilities: ["public", "unlisted", "direct"] + }) + + assert [%{"status" => %{"id" => id}}] = json_response(conn_res, 200) + assert id == private_activity.id + + conn_res = + get(conn, "/api/v1/notifications", %{ + exclude_visibilities: ["public", "private", "direct"] + }) + + assert [%{"status" => %{"id" => id}}] = json_response(conn_res, 200) + assert id == unlisted_activity.id + + conn_res = + get(conn, "/api/v1/notifications", %{ + exclude_visibilities: ["unlisted", "private", "direct"] + }) + + assert [%{"status" => %{"id" => id}}] = json_response(conn_res, 200) + assert id == public_activity.id + end + test "filters notifications using exclude_types", %{conn: conn} do user = insert(:user) other_user = insert(:user) diff --git a/test/web/mastodon_api/controllers/timeline_controller_test.exs b/test/web/mastodon_api/controllers/timeline_controller_test.exs index d3652d964..fc45c25de 100644 --- a/test/web/mastodon_api/controllers/timeline_controller_test.exs +++ b/test/web/mastodon_api/controllers/timeline_controller_test.exs @@ -20,27 +20,52 @@ defmodule Pleroma.Web.MastodonAPI.TimelineControllerTest do :ok end - test "the home timeline", %{conn: conn} do - user = insert(:user) - following = insert(:user) + describe "home" do + test "the home timeline", %{conn: conn} do + user = insert(:user) + following = insert(:user) - {:ok, _activity} = CommonAPI.post(following, %{"status" => "test"}) + {:ok, _activity} = CommonAPI.post(following, %{"status" => "test"}) - conn = - conn - |> assign(:user, user) - |> get("/api/v1/timelines/home") + conn = + conn + |> assign(:user, user) + |> get("/api/v1/timelines/home") - assert Enum.empty?(json_response(conn, :ok)) + assert Enum.empty?(json_response(conn, :ok)) - {:ok, user} = User.follow(user, following) + {:ok, user} = User.follow(user, following) - conn = - build_conn() - |> assign(:user, user) - |> get("/api/v1/timelines/home") + conn = + build_conn() + |> assign(:user, user) + |> get("/api/v1/timelines/home") - assert [%{"content" => "test"}] = json_response(conn, :ok) + assert [%{"content" => "test"}] = json_response(conn, :ok) + end + + test "the home timeline when the direct messages are excluded", %{conn: conn} do + user = insert(:user) + {:ok, public_activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "public"}) + {:ok, direct_activity} = CommonAPI.post(user, %{"status" => ".", "visibility" => "direct"}) + + {:ok, unlisted_activity} = + CommonAPI.post(user, %{"status" => ".", "visibility" => "unlisted"}) + + {:ok, private_activity} = + CommonAPI.post(user, %{"status" => ".", "visibility" => "private"}) + + conn = + conn + |> assign(:user, user) + |> get("/api/v1/timelines/home", %{"exclude_visibilities" => ["direct"]}) + + assert status_ids = json_response(conn, :ok) |> Enum.map(& &1["id"]) + assert public_activity.id in status_ids + assert unlisted_activity.id in status_ids + assert private_activity.id in status_ids + refute direct_activity.id in status_ids + end end describe "public" do From 0e9243e8a28e957f2a9401eb153cc2b27723d0a3 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 14 Oct 2019 18:38:51 +0300 Subject: [PATCH 23/36] Preparations for renaming `master` to `stable` --- .gitlab-ci.yml | 5 +++-- docs/installation/alpine_linux_en.md | 2 +- docs/installation/arch_linux_en.md | 2 +- docs/installation/centos7_en.md | 2 +- docs/installation/debian_based_en.md | 2 +- docs/installation/debian_based_jp.md | 2 +- docs/installation/gentoo_en.md | 2 +- docs/installation/migrating_from_source_otp_en.md | 4 ++-- docs/installation/netbsd_en.md | 2 +- docs/installation/openbsd_en.md | 2 +- docs/installation/openbsd_fi.md | 2 +- docs/installation/otp_en.md | 2 +- mix.exs | 5 ++++- 13 files changed, 19 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 748bec74a..36137b38e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -70,7 +70,7 @@ docs-deploy: stage: deploy image: alpine:latest only: - - master@pleroma/pleroma + - stable@pleroma/pleroma - develop@pleroma/pleroma before_script: - apk add curl @@ -127,9 +127,10 @@ amd64: # TODO: Replace with upstream image when 1.9.0 comes out image: rinpatch/elixir:1.9.0-rc.0 only: &release-only - - master@pleroma/pleroma + - stable@pleroma/pleroma - develop@pleroma/pleroma - /^maint/.*$/@pleroma/pleroma + - /^release/.*$/@pleroma/pleroma artifacts: &release-artifacts name: "pleroma-$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA-$CI_JOB_NAME" paths: diff --git a/docs/installation/alpine_linux_en.md b/docs/installation/alpine_linux_en.md index f5d1fade1..2a9b8f6ff 100644 --- a/docs/installation/alpine_linux_en.md +++ b/docs/installation/alpine_linux_en.md @@ -91,7 +91,7 @@ sudo adduser -S -s /bin/false -h /opt/pleroma -H -G pleroma pleroma ```shell sudo mkdir -p /opt/pleroma sudo chown -R pleroma:pleroma /opt/pleroma -sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma +sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma ``` * Change to the new directory: diff --git a/docs/installation/arch_linux_en.md b/docs/installation/arch_linux_en.md index 58a8d023f..8370986ad 100644 --- a/docs/installation/arch_linux_en.md +++ b/docs/installation/arch_linux_en.md @@ -66,7 +66,7 @@ sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma ```shell sudo mkdir -p /opt/pleroma sudo chown -R pleroma:pleroma /opt/pleroma -sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma +sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma ``` * Change to the new directory: diff --git a/docs/installation/centos7_en.md b/docs/installation/centos7_en.md index fe26339b5..ad4f58dc1 100644 --- a/docs/installation/centos7_en.md +++ b/docs/installation/centos7_en.md @@ -143,7 +143,7 @@ sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma ```shell sudo mkdir -p /opt/pleroma sudo chown -R pleroma:pleroma /opt/pleroma -sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma +sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma ``` * Change to the new directory: diff --git a/docs/installation/debian_based_en.md b/docs/installation/debian_based_en.md index e5f8dd670..fe2dbb92d 100644 --- a/docs/installation/debian_based_en.md +++ b/docs/installation/debian_based_en.md @@ -68,7 +68,7 @@ sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma ```shell sudo mkdir -p /opt/pleroma sudo chown -R pleroma:pleroma /opt/pleroma -sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma +sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma ``` * Change to the new directory: diff --git a/docs/installation/debian_based_jp.md b/docs/installation/debian_based_jp.md index e7d834b17..7aa0bcc24 100644 --- a/docs/installation/debian_based_jp.md +++ b/docs/installation/debian_based_jp.md @@ -68,7 +68,7 @@ sudo useradd -r -s /bin/false -m -d /var/lib/pleroma -U pleroma ``` sudo mkdir -p /opt/pleroma sudo chown -R pleroma:pleroma /opt/pleroma -sudo -Hu pleroma git clone -b master https://git.pleroma.social/pleroma/pleroma /opt/pleroma +sudo -Hu pleroma git clone -b stable https://git.pleroma.social/pleroma/pleroma /opt/pleroma ``` * 新しいディレクトリに移動します。 diff --git a/docs/installation/gentoo_en.md b/docs/installation/gentoo_en.md index 55d677acf..1e61373cc 100644 --- a/docs/installation/gentoo_en.md +++ b/docs/installation/gentoo_en.md @@ -106,7 +106,7 @@ It is highly recommended you use your own fork for the `https://path/to/repo` pa ```shell pleroma$ cd ~ - pleroma$ git clone -b master https://path/to/repo + pleroma$ git clone -b stable https://path/to/repo ``` * Change to the new directory: diff --git a/docs/installation/migrating_from_source_otp_en.md b/docs/installation/migrating_from_source_otp_en.md index e204cb3ee..87568faad 100644 --- a/docs/installation/migrating_from_source_otp_en.md +++ b/docs/installation/migrating_from_source_otp_en.md @@ -96,9 +96,9 @@ rm -r ~pleroma/* export FLAVOUR="arm64-musl" # Clone the release build into a temporary directory and unpack it -# Replace `master` with `develop` if you want to run the develop branch +# Replace `stable` with `unstable` if you want to run the unstable branch su pleroma -s $SHELL -lc " -curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/master/download?job=$FLAVOUR' -o /tmp/pleroma.zip +curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/stable/download?job=$FLAVOUR' -o /tmp/pleroma.zip unzip /tmp/pleroma.zip -d /tmp/ " diff --git a/docs/installation/netbsd_en.md b/docs/installation/netbsd_en.md index a096d5354..6a922a27e 100644 --- a/docs/installation/netbsd_en.md +++ b/docs/installation/netbsd_en.md @@ -58,7 +58,7 @@ Clone the repository: ``` $ cd /home/pleroma -$ git clone -b master https://git.pleroma.social/pleroma/pleroma.git +$ git clone -b stable https://git.pleroma.social/pleroma/pleroma.git ``` Configure Pleroma. Note that you need a domain name at this point: diff --git a/docs/installation/openbsd_en.md b/docs/installation/openbsd_en.md index fcba38b2c..3585a326b 100644 --- a/docs/installation/openbsd_en.md +++ b/docs/installation/openbsd_en.md @@ -29,7 +29,7 @@ This creates a "pleroma" login class and sets higher values than default for dat Create the \_pleroma user, assign it the pleroma login class and create its home directory (/home/\_pleroma/): `useradd -m -L pleroma _pleroma` #### Clone pleroma's directory -Enter a shell as the \_pleroma user. As root, run `su _pleroma -;cd`. Then clone the repository with `git clone -b master https://git.pleroma.social/pleroma/pleroma.git`. Pleroma is now installed in /home/\_pleroma/pleroma/, it will be configured and started at the end of this guide. +Enter a shell as the \_pleroma user. As root, run `su _pleroma -;cd`. Then clone the repository with `git clone -b stable https://git.pleroma.social/pleroma/pleroma.git`. Pleroma is now installed in /home/\_pleroma/pleroma/, it will be configured and started at the end of this guide. #### Postgresql Start a shell as the \_postgresql user (as root run `su _postgresql -` then run the `initdb` command to initialize postgresql: diff --git a/docs/installation/openbsd_fi.md b/docs/installation/openbsd_fi.md index 39819a8c8..272273cff 100644 --- a/docs/installation/openbsd_fi.md +++ b/docs/installation/openbsd_fi.md @@ -44,7 +44,7 @@ Vaihda pleroma-käyttäjään ja mene kotihakemistoosi: Lataa pleroman lähdekoodi: -`$ git clone -b master https://git.pleroma.social/pleroma/pleroma.git` +`$ git clone -b stable https://git.pleroma.social/pleroma/pleroma.git` `$ cd pleroma` diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index b4e5254cd..c028f4229 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -80,7 +80,7 @@ export FLAVOUR="arm64-musl" # Clone the release build into a temporary directory and unpack it su pleroma -s $SHELL -lc " -curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/master/download?job=$FLAVOUR' -o /tmp/pleroma.zip +curl 'https://git.pleroma.social/api/v4/projects/2/jobs/artifacts/stable/download?job=$FLAVOUR' -o /tmp/pleroma.zip unzip /tmp/pleroma.zip -d /tmp/ " diff --git a/mix.exs b/mix.exs index 3a605b455..0ae9c3f05 100644 --- a/mix.exs +++ b/mix.exs @@ -220,7 +220,10 @@ defp version(version) do with {branch_name, 0} <- System.cmd("git", ["rev-parse", "--abbrev-ref", "HEAD"]), branch_name <- String.trim(branch_name), branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name, - true <- branch_name not in ["master", "HEAD"] do + true <- + !Enum.all?(["master", "HEAD", "release/", "stable"], fn name -> + name != branch_name + end) do branch_name = branch_name |> String.trim() From 58fea88564c8dcece007a547b186965963e4dcb4 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 14 Oct 2019 18:39:56 +0300 Subject: [PATCH 24/36] Prepare pleroma_ctl for moving master to stable and relax the error message --- rel/files/bin/pleroma_ctl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rel/files/bin/pleroma_ctl b/rel/files/bin/pleroma_ctl index f767fe134..90f87a990 100755 --- a/rel/files/bin/pleroma_ctl +++ b/rel/files/bin/pleroma_ctl @@ -35,11 +35,11 @@ detect_branch() { if [ "$branch" = "develop" ]; then echo "develop" elif [ "$branch" = "" ]; then - echo "master" + echo "stable" else # Note: branch name in version is of SemVer format and may only contain [0-9a-zA-Z-] symbols — # if supporting releases for more branches, need to ensure they contain only these symbols. - echo "Releases are built only for master and develop branches" >&2 + echo "Can't detect the branch automatically, please specify it by using the --branch option." >&2 exit 1 fi } From d1e969842d9f78c262504adc0a80b8025f46c6a9 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 14 Oct 2019 18:42:30 +0300 Subject: [PATCH 25/36] Add a changelog entry for branch-moving --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd06ec866..a9f124a8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Report emails now include functional links to profiles of remote user accounts ## [1.1.0] - 2019-??-?? +**Breaking:** The stable branch has been changed from `master` to `stable`, `master` now points to `release/1.0` ### Security - Mastodon API: respect post privacy in `/api/v1/statuses/:id/{favourited,reblogged}_by` From 17b8896c40fa1c4aaab96f82fd07d5f5f2698400 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 14 Oct 2019 18:45:16 +0300 Subject: [PATCH 26/36] Correct version parser branch name detection --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 0ae9c3f05..329ce3d89 100644 --- a/mix.exs +++ b/mix.exs @@ -222,7 +222,7 @@ defp version(version) do branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name, true <- !Enum.all?(["master", "HEAD", "release/", "stable"], fn name -> - name != branch_name + String.starts_with?(name, branch_name) end) do branch_name = branch_name From 2f0ed5f6c4dcfb16de510851dbdf9886bc25f1d5 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Mon, 14 Oct 2019 22:13:18 +0300 Subject: [PATCH 27/36] Fix hiding branch name logic --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 329ce3d89..6cf766c52 100644 --- a/mix.exs +++ b/mix.exs @@ -221,7 +221,7 @@ defp version(version) do branch_name <- String.trim(branch_name), branch_name <- System.get_env("PLEROMA_BUILD_BRANCH") || branch_name, true <- - !Enum.all?(["master", "HEAD", "release/", "stable"], fn name -> + !Enum.any?(["master", "HEAD", "release/", "stable"], fn name -> String.starts_with?(name, branch_name) end) do branch_name = From 5bd0717de286a918e20c5ef2409e5f4cbcea8524 Mon Sep 17 00:00:00 2001 From: kPherox Date: Tue, 15 Oct 2019 19:10:22 +0900 Subject: [PATCH 28/36] Add `Sec-WebSocket-Protocol` to response header --- lib/pleroma/web/mastodon_api/websocket_handler.ex | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 3c26eb406..1a757363f 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -35,6 +35,13 @@ def init(%{qs: qs} = req, state) do {_, stream} <- List.keyfind(params, "stream", 0), {:ok, user} <- allow_request(stream, [access_token, sec_websocket]), topic when is_binary(topic) <- expand_topic(stream, params) do + req = + if sec_websocket != nil do + :cowboy_req.set_resp_header("sec-websocket-protocol", sec_websocket, req) + else + req + end + {:cowboy_websocket, req, %{user: user, topic: topic}, %{idle_timeout: @timeout}} else {:error, code} -> From e7bb762ec256bb5dc607797c042aa9432a9aa993 Mon Sep 17 00:00:00 2001 From: Alexander Date: Tue, 15 Oct 2019 15:16:17 +0300 Subject: [PATCH 29/36] don't stream in benchmark env --- lib/pleroma/web/streamer/streamer.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex index 8cf719277..2fc7ac8cf 100644 --- a/lib/pleroma/web/streamer/streamer.ex +++ b/lib/pleroma/web/streamer/streamer.ex @@ -49,7 +49,7 @@ defp handle_should_send(:test) do end end - defp handle_should_send(_) do - true - end + defp handle_should_send(:benchmark), do: false + + defp handle_should_send(_), do: true end From 8c0cfed825480ae6338fb1d183dee1440e0ee29c Mon Sep 17 00:00:00 2001 From: Alexander Date: Tue, 15 Oct 2019 16:26:04 +0300 Subject: [PATCH 30/36] some fixes --- benchmarks/load_testing/fetcher.ex | 2 +- benchmarks/mix/tasks/pleroma/load_testing.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index 0ff2f28c0..e378c51e7 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -210,7 +210,7 @@ def query_long_thread(user, activity) do Benchee.run(%{ "Render status" => fn -> - Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{ + Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{ activity: activity, for: user }) diff --git a/benchmarks/mix/tasks/pleroma/load_testing.ex b/benchmarks/mix/tasks/pleroma/load_testing.ex index 7b2293acc..4fa3eec49 100644 --- a/benchmarks/mix/tasks/pleroma/load_testing.ex +++ b/benchmarks/mix/tasks/pleroma/load_testing.ex @@ -31,7 +31,7 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do thread_length: :integer ] @users_default 20_000 - @dms_default 20_000 + @dms_default 1_000 @thread_length_default 2_000 def run(args) do From c10ce113d487d71c4daa6fabcc641a5caa0d04cb Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Oct 2019 12:52:47 +0300 Subject: [PATCH 31/36] User search: Remove trigram and refactor the module - Remove trigram as it tends to rank garbage results highly, resulting in it prioritized above fts, which gives actually decent results. ACKed by kaniini and lain on irc. - Remove a test for handling misspelled requests, since we no longer have trigram - Remove a test for searching users with `nil` display names, because it is unrealistic, we don't accept usernames that are not >1 char strings - Make rank boosting for followers/followees sane again, previous values resulted in garbage matches getting on top just because the users are followers/followees --- lib/pleroma/user/search.ex | 152 ++++++++++++------------------------- test/user_search_test.exs | 17 ----- 2 files changed, 50 insertions(+), 119 deletions(-) diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index 6fb2c2352..fb2f3fedb 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -4,7 +4,6 @@ defmodule Pleroma.User.Search do alias Pleroma.Pagination - alias Pleroma.Repo alias Pleroma.User import Ecto.Query @@ -23,18 +22,10 @@ def search(query_string, opts \\ []) do maybe_resolve(resolve, for_user, query_string) - {:ok, results} = - Repo.transaction(fn -> - Ecto.Adapters.SQL.query( - Repo, - "select set_limit(#{@similarity_threshold})", - [] - ) - - query_string - |> search_query(for_user, following) - |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset) - end) + results = + query_string + |> search_query(for_user, following) + |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset) results end @@ -56,15 +47,52 @@ defp search_query(query_string, for_user, following) do |> base_query(following) |> filter_blocked_user(for_user) |> filter_blocked_domains(for_user) - |> search_subqueries(query_string) - |> union_subqueries - |> distinct_query() - |> boost_search_rank_query(for_user) + |> fts_subquery(query_string) |> subquery() + |> where([u], u.search_rank > @similarity_threshold) + |> boost_search_rank(for_user) |> order_by(desc: :search_rank) |> maybe_restrict_local(for_user) end + @nickname_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~\-@]+$/ + defp fts_subquery(query, query_string) do + {nickname_weight, name_weight} = + if String.match?(query_string, @nickname_regex) do + {"A", "B"} + else + {"B", "A"} + end + + query_string = to_tsquery(query_string) + + from( + u in query, + select_merge: %{ + search_rank: + fragment( + """ + ts_rank_cd((setweight(to_tsvector('simple', ?), ?) || setweight(to_tsvector('simple', ?), ?)), to_tsquery('simple', ?)) + """, + u.name, + ^name_weight, + u.nickname, + ^nickname_weight, + ^query_string + ) + } + ) + end + + defp to_tsquery(query_string) do + String.trim_trailing(query_string, "@" <> local_domain()) + |> String.replace(~r/[!-\/|@|[-`|{-~|:-?]+/, " ") + |> String.trim() + |> String.split() + |> Enum.map(&(&1 <> ":*")) + |> Enum.join(" | ") + end + defp base_query(_user, false), do: User defp base_query(user, true), do: User.get_followers_query(user) @@ -87,21 +115,6 @@ defp filter_blocked_domains(query, %User{info: %{domain_blocks: domain_blocks}}) defp filter_blocked_domains(query, _), do: query - defp union_subqueries({fts_subquery, trigram_subquery}) do - from(s in trigram_subquery, union_all: ^fts_subquery) - end - - defp search_subqueries(base_query, query_string) do - { - fts_search_subquery(base_query, query_string), - trigram_search_subquery(base_query, query_string) - } - end - - defp distinct_query(q) do - from(s in subquery(q), order_by: s.search_type, distinct: s.id) - end - defp maybe_resolve(true, user, query) do case {limit(), user} do {:all, _} -> :noop @@ -126,9 +139,9 @@ defp limit, do: Pleroma.Config.get([:instance, :limit_to_local_content], :unauth defp restrict_local(q), do: where(q, [u], u.local == true) - defp boost_search_rank_query(query, nil), do: query + defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) - defp boost_search_rank_query(query, for_user) do + defp boost_search_rank(query, %User{} = for_user) do friends_ids = User.get_friends_ids(for_user) followers_ids = User.get_followers_ids(for_user) @@ -137,8 +150,8 @@ defp boost_search_rank_query(query, for_user) do search_rank: fragment( """ - CASE WHEN (?) THEN 0.5 + (?) * 1.3 - WHEN (?) THEN 0.5 + (?) * 1.2 + CASE WHEN (?) THEN (?) * 1.5 + WHEN (?) THEN (?) * 1.3 WHEN (?) THEN (?) * 1.1 ELSE (?) END """, @@ -154,70 +167,5 @@ defp boost_search_rank_query(query, for_user) do ) end - @spec fts_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t() - defp fts_search_subquery(query, term) do - processed_query = - String.trim_trailing(term, "@" <> local_domain()) - |> String.replace(~r/[!-\/|@|[-`|{-~|:-?]+/, " ") - |> String.trim() - |> String.split() - |> Enum.map(&(&1 <> ":*")) - |> Enum.join(" | ") - - from( - u in query, - select_merge: %{ - search_type: ^0, - search_rank: - fragment( - """ - ts_rank_cd( - setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') || - setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'), - to_tsquery('simple', ?), - 32 - ) - """, - u.nickname, - u.name, - ^processed_query - ) - }, - where: - fragment( - """ - (setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') || - setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')) @@ to_tsquery('simple', ?) - """, - u.nickname, - u.name, - ^processed_query - ) - ) - |> User.restrict_deactivated() - end - - @spec trigram_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t() - defp trigram_search_subquery(query, term) do - term = String.trim_trailing(term, "@" <> local_domain()) - - from( - u in query, - select_merge: %{ - # ^1 gives 'Postgrex expected a binary, got 1' for some weird reason - search_type: fragment("?", 1), - search_rank: - fragment( - "similarity(?, trim(? || ' ' || coalesce(?, '')))", - ^term, - u.nickname, - u.name - ) - }, - where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^term) - ) - |> User.restrict_deactivated() - end - - defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host]) + defp boost_search_rank(query, _for_user), do: query end diff --git a/test/user_search_test.exs b/test/user_search_test.exs index f7ab31287..e413f0893 100644 --- a/test/user_search_test.exs +++ b/test/user_search_test.exs @@ -74,12 +74,6 @@ test "finds users, ranking by similarity" do assert [u4.id, u3.id, u1.id] == Enum.map(User.search("lain@ple", for_user: u1), & &1.id) end - test "finds users, handling misspelled requests" do - u1 = insert(:user, %{name: "lain"}) - - assert [u1.id] == Enum.map(User.search("laiin"), & &1.id) - end - test "finds users, boosting ranks of friends and followers" do u1 = insert(:user) u2 = insert(:user, %{name: "Doe"}) @@ -163,17 +157,6 @@ test "find all users for unauthenticated users when `limit_to_local_content` is Pleroma.Config.put([:instance, :limit_to_local_content], :unauthenticated) end - test "finds a user whose name is nil" do - _user = insert(:user, %{name: "notamatch", nickname: "testuser@pleroma.amplifie.red"}) - user_two = insert(:user, %{name: nil, nickname: "lain@pleroma.soykaf.com"}) - - assert user_two == - User.search("lain@pleroma.soykaf.com") - |> List.first() - |> Map.put(:search_rank, nil) - |> Map.put(:search_type, nil) - end - test "does not yield false-positive matches" do insert(:user, %{name: "John Doe"}) From 0a5175ecbb796cf3c192a42dc561debd73640a54 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Oct 2019 13:49:33 +0300 Subject: [PATCH 32/36] Order fts results by trigram --- lib/pleroma/user/search.ex | 48 ++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index fb2f3fedb..0d697fe3d 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -7,7 +7,6 @@ defmodule Pleroma.User.Search do alias Pleroma.User import Ecto.Query - @similarity_threshold 0.25 @limit 20 def search(query_string, opts \\ []) do @@ -47,16 +46,16 @@ defp search_query(query_string, for_user, following) do |> base_query(following) |> filter_blocked_user(for_user) |> filter_blocked_domains(for_user) - |> fts_subquery(query_string) - |> subquery() - |> where([u], u.search_rank > @similarity_threshold) + |> fts_search(query_string) + |> trigram_rank(query_string) |> boost_search_rank(for_user) + |> subquery() |> order_by(desc: :search_rank) |> maybe_restrict_local(for_user) end @nickname_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~\-@]+$/ - defp fts_subquery(query, query_string) do + defp fts_search(query, query_string) do {nickname_weight, name_weight} = if String.match?(query_string, @nickname_regex) do {"A", "B"} @@ -68,19 +67,17 @@ defp fts_subquery(query, query_string) do from( u in query, - select_merge: %{ - search_rank: - fragment( - """ - ts_rank_cd((setweight(to_tsvector('simple', ?), ?) || setweight(to_tsvector('simple', ?), ?)), to_tsquery('simple', ?)) - """, - u.name, - ^name_weight, - u.nickname, - ^nickname_weight, - ^query_string - ) - } + where: + fragment( + """ + (setweight(to_tsvector('simple', ?), ?) || setweight(to_tsvector('simple', ?), ?)) @@ to_tsquery('simple', ?) + """, + u.name, + ^name_weight, + u.nickname, + ^nickname_weight, + ^query_string + ) ) end @@ -93,6 +90,21 @@ defp to_tsquery(query_string) do |> Enum.join(" | ") end + defp trigram_rank(query, query_string) do + from( + u in query, + select_merge: %{ + search_rank: + fragment( + "similarity(?, trim(? || ' ' || coalesce(?, '')))", + ^query_string, + u.nickname, + u.name + ) + } + ) + end + defp base_query(_user, false), do: User defp base_query(user, true), do: User.get_followers_query(user) From b8be6a4dc92ef067bc6844a89095986f07367fba Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 16 Oct 2019 15:09:39 +0300 Subject: [PATCH 33/36] Remove a failing search test due to it being unrealistic While the case tested (`lain@ple`) gives wrong ordering due to using only trigram to order, it almost never happens in reality. In reality it would be either `lain` (in which case it's fine to list the user with `lain` in display name first), or `@lain@pleroma.soykaf.com`/`lain@pleroma.soykaf.com` (which is handled fine as well) --- test/user_search_test.exs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/test/user_search_test.exs b/test/user_search_test.exs index e413f0893..78a02d536 100644 --- a/test/user_search_test.exs +++ b/test/user_search_test.exs @@ -65,15 +65,6 @@ test "finds users, considering density of matched tokens" do assert [u2.id, u1.id] == Enum.map(User.search("bar word"), & &1.id) end - test "finds users, ranking by similarity" do - u1 = insert(:user, %{name: "lain"}) - _u2 = insert(:user, %{name: "ean"}) - u3 = insert(:user, %{name: "ebn", nickname: "lain@mastodon.social"}) - u4 = insert(:user, %{nickname: "lain@pleroma.soykaf.com"}) - - assert [u4.id, u3.id, u1.id] == Enum.map(User.search("lain@ple", for_user: u1), & &1.id) - end - test "finds users, boosting ranks of friends and followers" do u1 = insert(:user) u2 = insert(:user, %{name: "Doe"}) From 359dd1890e6afcf80584021eaa2421336614dd07 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Thu, 17 Oct 2019 15:25:15 +0300 Subject: [PATCH 34/36] Mastodon API: Mark the conversation as read for the author when they send a new direct message --- CHANGELOG.md | 1 + lib/pleroma/conversation/participation.ex | 6 ++++ lib/pleroma/web/activity_pub/activity_pub.ex | 23 ++++++++---- test/conversation/participation_test.exs | 35 ++++++++++++++++++- test/web/activity_pub/activity_pub_test.exs | 21 +++++++++++ .../conversation_controller_test.exs | 14 +++++--- .../mastodon_api/views/account_view_test.exs | 4 +-- 7 files changed, 91 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24876d3f2..f9f84b056 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - MRF (Simple Policy): Also use `:accept`/`:reject` on the actors rather than only their activities - OStatus: Extract RSS functionality - Mastodon API: Add `pleroma.direct_conversation_id` to the status endpoint (`GET /api/v1/statuses/:id`) +- Mastodon API: Mark the direct conversation as read for the author when they send a new direct message ### Fixed - Mastodon API: Fix private and direct statuses not being filtered out from the public timeline for an authenticated user (`GET /api/v1/timelines/public`) diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index ab81f3217..e17f49e58 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -48,6 +48,12 @@ def read_cng(struct, params) do |> validate_required([:read]) end + def mark_as_read(%User{} = user, %Conversation{} = conversation) do + with %__MODULE__{} = participation <- for_user_and_conversation(user, conversation) do + mark_as_read(participation) + end + end + def mark_as_read(participation) do participation |> read_cng(%{read: true}) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 091ec2588..d391732a2 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Activity.Ir.Topics alias Pleroma.Config alias Pleroma.Conversation + alias Pleroma.Conversation.Participation alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Object.Containment @@ -153,11 +154,8 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when Notification.create_notifications(activity) - participations = - activity - |> Conversation.create_or_bump_for() - |> get_participations() - + conversation = create_or_bump_conversation(activity, map["actor"]) + participations = get_participations(conversation) stream_out(activity) stream_out_participations(participations) {:ok, activity} @@ -182,7 +180,20 @@ def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when end end - defp get_participations({:ok, %{participations: participations}}), do: participations + defp create_or_bump_conversation(activity, actor) do + with {:ok, conversation} <- Conversation.create_or_bump_for(activity), + %User{} = user <- User.get_cached_by_ap_id(actor), + Participation.mark_as_read(user, conversation) do + {:ok, conversation} + end + end + + defp get_participations({:ok, conversation}) do + conversation + |> Repo.preload(:participations, force: true) + |> Map.get(:participations) + end + defp get_participations(_), do: [] def stream_out_participations(participations) do diff --git a/test/conversation/participation_test.exs b/test/conversation/participation_test.exs index f430bdf75..a5af0d1b2 100644 --- a/test/conversation/participation_test.exs +++ b/test/conversation/participation_test.exs @@ -23,6 +23,39 @@ test "getting a participation will also preload things" do assert %Pleroma.Conversation{} = participation.conversation end + test "for a new conversation or a reply, it doesn't mark the author's participation as unread" do + user = insert(:user) + other_user = insert(:user) + + {:ok, _} = + CommonAPI.post(user, %{"status" => "Hey @#{other_user.nickname}.", "visibility" => "direct"}) + + user = User.get_cached_by_id(user.id) + other_user = User.get_cached_by_id(other_user.id) + + [%{read: true}] = Participation.for_user(user) + [%{read: false} = participation] = Participation.for_user(other_user) + + assert User.get_cached_by_id(user.id).info.unread_conversation_count == 0 + assert User.get_cached_by_id(other_user.id).info.unread_conversation_count == 1 + + {:ok, _} = + CommonAPI.post(other_user, %{ + "status" => "Hey @#{user.nickname}.", + "visibility" => "direct", + "in_reply_to_conversation_id" => participation.id + }) + + user = User.get_cached_by_id(user.id) + other_user = User.get_cached_by_id(other_user.id) + + [%{read: false}] = Participation.for_user(user) + [%{read: true}] = Participation.for_user(other_user) + + assert User.get_cached_by_id(user.id).info.unread_conversation_count == 1 + assert User.get_cached_by_id(other_user.id).info.unread_conversation_count == 0 + end + test "for a new conversation, it sets the recipents of the participation" do user = insert(:user) other_user = insert(:user) @@ -32,7 +65,7 @@ test "for a new conversation, it sets the recipents of the participation" do CommonAPI.post(user, %{"status" => "Hey @#{other_user.nickname}.", "visibility" => "direct"}) user = User.get_cached_by_id(user.id) - other_user = User.get_cached_by_id(user.id) + other_user = User.get_cached_by_id(other_user.id) [participation] = Participation.for_user(user) participation = Pleroma.Repo.preload(participation, :recipients) diff --git a/test/web/activity_pub/activity_pub_test.exs b/test/web/activity_pub/activity_pub_test.exs index 3a5a2f984..28a9b773c 100644 --- a/test/web/activity_pub/activity_pub_test.exs +++ b/test/web/activity_pub/activity_pub_test.exs @@ -41,6 +41,27 @@ test "it streams them out" do assert called(Pleroma.Web.Streamer.stream("participation", participations)) end end + + test "streams them out on activity creation" do + user_one = insert(:user) + user_two = insert(:user) + + with_mock Pleroma.Web.Streamer, + stream: fn _, _ -> nil end do + {:ok, activity} = + CommonAPI.post(user_one, %{ + "status" => "@#{user_two.nickname}", + "visibility" => "direct" + }) + + conversation = + activity.data["context"] + |> Pleroma.Conversation.get_for_ap_id() + |> Repo.preload(participations: :user) + + assert called(Pleroma.Web.Streamer.stream("participation", conversation.participations)) + end + end end describe "fetching restricted by visibility" do diff --git a/test/web/mastodon_api/controllers/conversation_controller_test.exs b/test/web/mastodon_api/controllers/conversation_controller_test.exs index a308a7620..d89a87179 100644 --- a/test/web/mastodon_api/controllers/conversation_controller_test.exs +++ b/test/web/mastodon_api/controllers/conversation_controller_test.exs @@ -54,9 +54,9 @@ test "returns a list of conversations", %{conn: conn} do assert user_two.id in account_ids assert user_three.id in account_ids assert is_binary(res_id) - assert unread == true + assert unread == false assert res_last_status["id"] == direct.id - assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 1 + assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 0 end test "updates the last_status on reply", %{conn: conn} do @@ -95,19 +95,23 @@ test "the user marks a conversation as read", %{conn: conn} do "visibility" => "direct" }) + assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 0 + assert User.get_cached_by_id(user_two.id).info.unread_conversation_count == 1 + [%{"id" => direct_conversation_id, "unread" => true}] = conn - |> assign(:user, user_one) + |> assign(:user, user_two) |> get("/api/v1/conversations") |> json_response(200) %{"unread" => false} = conn - |> assign(:user, user_one) + |> assign(:user, user_two) |> post("/api/v1/conversations/#{direct_conversation_id}/read") |> json_response(200) assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 0 + assert User.get_cached_by_id(user_two.id).info.unread_conversation_count == 0 # The conversation is marked as unread on reply {:ok, _} = @@ -124,6 +128,7 @@ test "the user marks a conversation as read", %{conn: conn} do |> json_response(200) assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 1 + assert User.get_cached_by_id(user_two.id).info.unread_conversation_count == 0 # A reply doesn't increment the user's unread_conversation_count if the conversation is unread {:ok, _} = @@ -134,6 +139,7 @@ test "the user marks a conversation as read", %{conn: conn} do }) assert User.get_cached_by_id(user_one.id).info.unread_conversation_count == 1 + assert User.get_cached_by_id(user_two.id).info.unread_conversation_count == 0 end test "(vanilla) Mastodon frontend behaviour", %{conn: conn} do diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs index b7a4938a6..ad209b4a3 100644 --- a/test/web/mastodon_api/views/account_view_test.exs +++ b/test/web/mastodon_api/views/account_view_test.exs @@ -424,8 +424,8 @@ test "shows unread_conversation_count only to the account owner" do other_user = insert(:user) {:ok, _activity} = - CommonAPI.post(user, %{ - "status" => "Hey @#{other_user.nickname}.", + CommonAPI.post(other_user, %{ + "status" => "Hey @#{user.nickname}.", "visibility" => "direct" }) From 733b73b71c2a13a2bfbf4c0a0d35ddfa140a1ce0 Mon Sep 17 00:00:00 2001 From: kaniini Date: Fri, 18 Oct 2019 04:36:37 +0000 Subject: [PATCH 35/36] Apply suggestion to lib/pleroma/web/mastodon_api/websocket_handler.ex --- lib/pleroma/web/mastodon_api/websocket_handler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 1a757363f..a400d1c8d 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -36,7 +36,7 @@ def init(%{qs: qs} = req, state) do {:ok, user} <- allow_request(stream, [access_token, sec_websocket]), topic when is_binary(topic) <- expand_topic(stream, params) do req = - if sec_websocket != nil do + if sec_websocket do :cowboy_req.set_resp_header("sec-websocket-protocol", sec_websocket, req) else req From 3c6fd0bb991c865aad2fb27bc798b2b3c3c248d2 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 18 Oct 2019 12:29:42 +0200 Subject: [PATCH 36/36] upload.ex: Remove deprecated configuration --- CHANGELOG.md | 3 +++ lib/pleroma/upload.ex | 33 +-------------------------------- 2 files changed, 4 insertions(+), 32 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9f84b056..ee6b6c3a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Admin API: `/users/:nickname/toggle_activation` endpoint is now deprecated in favor of: `/users/activate`, `/users/deactivate`, both accept `nicknames` array - Admin API: `POST/DELETE /api/pleroma/admin/users/:nickname/permission_group/:permission_group` are deprecated in favor of: `POST/DELETE /api/pleroma/admin/users/permission_group/:permission_group` (both accept `nicknames` array), `DELETE /api/pleroma/admin/users` (`nickname` query param or `nickname` sent in JSON body) is deprecated in favor of: `DELETE /api/pleroma/admin/users` (`nicknames` query array param or `nicknames` sent in JSON body). +### Removed +- **Breaking**: Removed 1.0+ deprecated configurations `Pleroma.Upload, :strip_exif` and `:instance, :dedupe_media` + ### Changed - **Breaking:** Elixir >=1.8 is now required (was >= 1.7) - **Breaking:** Admin API: Return link alongside with token on password reset diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 9f0adde5b..2e0986197 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -105,7 +105,7 @@ defp get_opts(opts) do {Pleroma.Config.get!([:instance, :upload_limit]), "Document"} end - opts = %{ + %{ activity_type: Keyword.get(opts, :activity_type, activity_type), size_limit: Keyword.get(opts, :size_limit, size_limit), uploader: Keyword.get(opts, :uploader, Pleroma.Config.get([__MODULE__, :uploader])), @@ -118,37 +118,6 @@ defp get_opts(opts) do Pleroma.Config.get([__MODULE__, :base_url], Pleroma.Web.base_url()) ) } - - # TODO: 1.0+ : remove old config compatibility - opts = - if Pleroma.Config.get([__MODULE__, :strip_exif]) == true && - !Enum.member?(opts.filters, Pleroma.Upload.Filter.Mogrify) do - Logger.warn(""" - Pleroma: configuration `:instance, :strip_exif` is deprecated, please instead set: - - :pleroma, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Mogrify]] - - :pleroma, Pleroma.Upload.Filter.Mogrify, args: ["strip", "auto-orient"] - """) - - Pleroma.Config.put([Pleroma.Upload.Filter.Mogrify], args: ["strip", "auto-orient"]) - Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Mogrify]) - else - opts - end - - if Pleroma.Config.get([:instance, :dedupe_media]) == true && - !Enum.member?(opts.filters, Pleroma.Upload.Filter.Dedupe) do - Logger.warn(""" - Pleroma: configuration `:instance, :dedupe_media` is deprecated, please instead set: - - :pleroma, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Dedupe]] - """) - - Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Dedupe]) - else - opts - end end defp prepare_upload(%Plug.Upload{} = file, opts) do