Merge branch 'hashtag-benchmarks' into 'develop'

Hashtag benchmarks

See merge request 
This commit is contained in:
Haelwenn 2020-01-20 11:26:07 +00:00
commit e8759cb5ba
3 changed files with 139 additions and 6 deletions
benchmarks
load_testing
mix/tasks/pleroma/benchmarks
lib/pleroma/web/mastodon_api/controllers

View file

@ -9,7 +9,7 @@ def generate_like_activities(user, posts) do
{time, _} =
:timer.tc(fn ->
Task.async_stream(
Enum.take_random(posts, count_likes),
Enum.take_random(posts, count_likes),
fn post -> {:ok, _, _} = CommonAPI.favorite(post.id, user) end,
max_concurrency: 10,
timeout: 30_000
@ -142,6 +142,48 @@ defp do_generate_activity(users) do
CommonAPI.post(Enum.random(users), post)
end
def generate_power_intervals(opts \\ []) do
count = Keyword.get(opts, :count, 20)
power = Keyword.get(opts, :power, 2)
IO.puts("Generating #{count} intervals for a power #{power} series...")
counts = Enum.map(1..count, fn n -> :math.pow(n, power) end)
sum = Enum.sum(counts)
densities =
Enum.map(counts, fn c ->
c / sum
end)
densities
|> Enum.reduce(0, fn density, acc ->
if acc == 0 do
[{0, density}]
else
[{_, lower} | _] = acc
[{lower, lower + density} | acc]
end
end)
|> Enum.reverse()
end
def generate_tagged_activities(opts \\ []) do
tag_count = Keyword.get(opts, :tag_count, 20)
users = Keyword.get(opts, :users, Repo.all(User))
activity_count = Keyword.get(opts, :count, 200_000)
intervals = generate_power_intervals(count: tag_count)
IO.puts(
"Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0"
)
Enum.each(1..activity_count, fn _ ->
random = :rand.uniform()
i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end)
CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"})
end)
end
defp do_generate_activity_with_mention(user, users) do
mentions_cnt = Enum.random([2, 3, 4, 5])
with_user = Enum.random([true, false])

View file

@ -0,0 +1,87 @@
defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
use Mix.Task
alias Pleroma.Repo
alias Pleroma.LoadTesting.Generator
import Ecto.Query
def run(_args) do
Mix.Pleroma.start_pleroma()
activities_count = Repo.aggregate(from(a in Pleroma.Activity), :count, :id)
if activities_count == 0 do
IO.puts("Did not find any activities, cleaning and generating")
clean_tables()
Generator.generate_users(users_max: 10)
Generator.generate_tagged_activities()
else
IO.puts("Found #{activities_count} activities, won't generate new ones")
end
tags = Enum.map(0..20, fn i -> {"For #tag_#{i}", "tag_#{i}"} end)
Enum.each(tags, fn {_, tag} ->
query =
from(o in Pleroma.Object,
where: fragment("(?)->'tag' \\? (?)", o.data, ^tag)
)
count = Repo.aggregate(query, :count, :id)
IO.puts("Database contains #{count} posts tagged with #{tag}")
end)
user = Repo.all(Pleroma.User) |> List.first()
Benchee.run(
%{
"Hashtag fetching, any" => fn tags ->
Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
%{
"any" => tags
},
user,
false
)
end,
# Will always return zero results because no overlapping hashtags are generated.
"Hashtag fetching, all" => fn tags ->
Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
%{
"all" => tags
},
user,
false
)
end
},
inputs:
tags
|> Enum.map(fn {_, v} -> v end)
|> Enum.chunk_every(2)
|> Enum.map(fn tags -> {"For #{inspect(tags)}", tags} end),
time: 5
)
Benchee.run(
%{
"Hashtag fetching" => fn tag ->
Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
%{
"tag" => tag
},
user,
false
)
end
},
inputs: tags,
time: 5
)
end
defp clean_tables do
IO.puts("Deleting old data...\n")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
end
end

View file

@ -77,10 +77,7 @@ def public(%{assigns: %{user: user}} = conn, params) do
|> render("index.json", activities: activities, for: user, as: :activity)
end
# GET /api/v1/timelines/tag/:tag
def hashtag(%{assigns: %{user: user}} = conn, params) do
local_only = truthy_param?(params["local"])
def hashtag_fetching(params, user, local_only) do
tags =
[params["tag"], params["any"]]
|> List.flatten()
@ -98,7 +95,7 @@ def hashtag(%{assigns: %{user: user}} = conn, params) do
|> Map.get("none", [])
|> Enum.map(&String.downcase(&1))
activities =
_activities =
params
|> Map.put("type", "Create")
|> Map.put("local_only", local_only)
@ -109,6 +106,13 @@ def hashtag(%{assigns: %{user: user}} = conn, params) do
|> Map.put("tag_all", tag_all)
|> Map.put("tag_reject", tag_reject)
|> ActivityPub.fetch_public_activities()
end
# GET /api/v1/timelines/tag/:tag
def hashtag(%{assigns: %{user: user}} = conn, params) do
local_only = truthy_param?(params["local"])
activities = hashtag_fetching(params, user, local_only)
conn
|> add_link_headers(activities, %{"local" => local_only})