sync with develop

This commit is contained in:
Alexander Strizhakov 2020-02-18 12:19:10 +03:00
parent eb9744cade
commit 1f29ecdcd7
No known key found for this signature in database
GPG key ID: 022896A53AEF1381
6 changed files with 99 additions and 50 deletions

View file

@ -85,6 +85,48 @@ defmodule Pleroma.LoadTesting.Activities do
:ok :ok
end end
def generate_power_intervals(opts \\ []) do
count = Keyword.get(opts, :count, 20)
power = Keyword.get(opts, :power, 2)
IO.puts("Generating #{count} intervals for a power #{power} series...")
counts = Enum.map(1..count, fn n -> :math.pow(n, power) end)
sum = Enum.sum(counts)
densities =
Enum.map(counts, fn c ->
c / sum
end)
densities
|> Enum.reduce(0, fn density, acc ->
if acc == 0 do
[{0, density}]
else
[{_, lower} | _] = acc
[{lower, lower + density} | acc]
end
end)
|> Enum.reverse()
end
def generate_tagged_activities(opts \\ []) do
tag_count = Keyword.get(opts, :tag_count, 20)
users = Keyword.get(opts, :users, Repo.all(Pleroma.User))
activity_count = Keyword.get(opts, :count, 200_000)
intervals = generate_power_intervals(count: tag_count)
IO.puts(
"Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0"
)
Enum.each(1..activity_count, fn _ ->
random = :rand.uniform()
i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end)
CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"})
end)
end
defp generate_long_thread(visibility, user, friends, non_friends, _opts) do defp generate_long_thread(visibility, user, friends, non_friends, _opts) do
group = group =
if visibility == "public", if visibility == "public",

View file

@ -1,3 +1,14 @@
defmodule Pleroma.LoadTesting.Helper do defmodule Pleroma.LoadTesting.Helper do
alias Ecto.Adapters.SQL
alias Pleroma.Repo
def to_sec(microseconds), do: microseconds / 1_000_000 def to_sec(microseconds), do: microseconds / 1_000_000
def clean_tables do
IO.puts("Deleting old data...\n")
SQL.query!(Repo, "TRUNCATE users CASCADE;")
SQL.query!(Repo, "TRUNCATE activities CASCADE;")
SQL.query!(Repo, "TRUNCATE objects CASCADE;")
SQL.query!(Repo, "TRUNCATE oban_jobs CASCADE;")
end
end end

View file

@ -20,31 +20,31 @@ defmodule Pleroma.LoadTesting.Users do
def generate(opts \\ []) do def generate(opts \\ []) do
opts = Keyword.merge(@defaults, opts) opts = Keyword.merge(@defaults, opts)
IO.puts("Starting generating #{opts[:users]} users...") generate_users(opts[:users])
{time, _} = :timer.tc(fn -> generate_users(opts[:users]) end)
IO.puts("Generating users take #{to_sec(time)} sec.\n")
main_user = main_user =
Repo.one(from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)) Repo.one(from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1))
IO.puts("Starting making friends for #{opts[:friends]} users...") make_friends(main_user, opts[:friends])
{time, _} = :timer.tc(fn -> make_friends(main_user, opts[:friends]) end)
IO.puts("Making friends take #{to_sec(time)} sec.\n")
Repo.get(User, main_user.id) Repo.get(User, main_user.id)
end end
defp generate_users(max) do def generate_users(max) do
Task.async_stream( IO.puts("Starting generating #{opts[:users]} users...")
1..max,
&generate_user(&1), {time, _} =
max_concurrency: @max_concurrency, :timer.tc(fn ->
timeout: 30_000 Task.async_stream(
) 1..max,
|> Stream.run() &generate_user(&1),
max_concurrency: @max_concurrency,
timeout: 30_000
)
|> Stream.run()
end)
IO.puts("Generating users take #{to_sec(time)} sec.\n")
end end
defp generate_user(i) do defp generate_user(i) do
@ -86,18 +86,25 @@ defmodule Pleroma.LoadTesting.Users do
Map.merge(user, urls) Map.merge(user, urls)
end end
defp make_friends(main_user, max) when is_integer(max) do def make_friends(main_user, max) when is_integer(max) do
number_of_users = IO.puts("Starting making friends for #{opts[:friends]} users...")
(max / 2)
|> Kernel.trunc()
main_user {time, _} =
|> get_users(%{limit: number_of_users, local: :local}) :timer.tc(fn ->
|> run_stream(main_user) number_of_users =
(max / 2)
|> Kernel.trunc()
main_user main_user
|> get_users(%{limit: number_of_users, local: :external}) |> get_users(%{limit: number_of_users, local: :local})
|> run_stream(main_user) |> run_stream(main_user)
main_user
|> get_users(%{limit: number_of_users, local: :external})
|> run_stream(main_user)
end)
IO.puts("Making friends take #{to_sec(time)} sec.\n")
end end
defp make_friends(%User{} = main_user, %User{} = user) do defp make_friends(%User{} = main_user, %User{} = user) do

View file

@ -1,9 +1,12 @@
defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
use Mix.Task use Mix.Task
alias Pleroma.Repo
alias Pleroma.LoadTesting.Generator import Pleroma.LoadTesting.Helper, only: [clean_tables: 0]
import Ecto.Query import Ecto.Query
alias Pleroma.Repo
alias Pleroma.Web.MastodonAPI.TimelineController
def run(_args) do def run(_args) do
Mix.Pleroma.start_pleroma() Mix.Pleroma.start_pleroma()
activities_count = Repo.aggregate(from(a in Pleroma.Activity), :count, :id) activities_count = Repo.aggregate(from(a in Pleroma.Activity), :count, :id)
@ -11,8 +14,8 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
if activities_count == 0 do if activities_count == 0 do
IO.puts("Did not find any activities, cleaning and generating") IO.puts("Did not find any activities, cleaning and generating")
clean_tables() clean_tables()
Generator.generate_users(users_max: 10) Pleroma.LoadTesting.Users.generate_users(10)
Generator.generate_tagged_activities() Pleroma.LoadTesting.Activities.generate_tagged_activities()
else else
IO.puts("Found #{activities_count} activities, won't generate new ones") IO.puts("Found #{activities_count} activities, won't generate new ones")
end end
@ -34,7 +37,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
Benchee.run( Benchee.run(
%{ %{
"Hashtag fetching, any" => fn tags -> "Hashtag fetching, any" => fn tags ->
Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching( TimelineController.hashtag_fetching(
%{ %{
"any" => tags "any" => tags
}, },
@ -44,7 +47,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
end, end,
# Will always return zero results because no overlapping hashtags are generated. # Will always return zero results because no overlapping hashtags are generated.
"Hashtag fetching, all" => fn tags -> "Hashtag fetching, all" => fn tags ->
Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching( TimelineController.hashtag_fetching(
%{ %{
"all" => tags "all" => tags
}, },
@ -64,7 +67,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
Benchee.run( Benchee.run(
%{ %{
"Hashtag fetching" => fn tag -> "Hashtag fetching" => fn tag ->
Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching( TimelineController.hashtag_fetching(
%{ %{
"tag" => tag "tag" => tag
}, },
@ -77,11 +80,4 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
time: 5 time: 5
) )
end end
defp clean_tables do
IO.puts("Deleting old data...\n")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
end
end end

View file

@ -1,8 +1,8 @@
defmodule Mix.Tasks.Pleroma.LoadTesting do defmodule Mix.Tasks.Pleroma.LoadTesting do
use Mix.Task use Mix.Task
import Ecto.Query import Ecto.Query
import Pleroma.LoadTesting.Helper, only: [clean_tables: 0]
alias Ecto.Adapters.SQL
alias Pleroma.Repo alias Pleroma.Repo
alias Pleroma.User alias Pleroma.User
@ -63,12 +63,4 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
Pleroma.LoadTesting.Fetcher.run_benchmarks(user) Pleroma.LoadTesting.Fetcher.run_benchmarks(user)
end end
defp clean_tables do
IO.puts("Deleting old data...\n")
SQL.query!(Repo, "TRUNCATE users CASCADE;")
SQL.query!(Repo, "TRUNCATE activities CASCADE;")
SQL.query!(Repo, "TRUNCATE objects CASCADE;")
SQL.query!(Repo, "TRUNCATE oban_jobs CASCADE;")
end
end end

View file

@ -5,6 +5,7 @@
defmodule Mix.Pleroma do defmodule Mix.Pleroma do
@doc "Common functions to be reused in mix tasks" @doc "Common functions to be reused in mix tasks"
def start_pleroma do def start_pleroma do
Mix.Task.run("app.start")
Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
if Pleroma.Config.get(:env) != :test do if Pleroma.Config.get(:env) != :test do