[#3213] rescue around potentially-raising Repo.insert_all/_ calls. Misc. improvements (docs etc.).

This commit is contained in:
Ivan Tashkinov 2021-02-13 22:01:11 +03:00
parent 5992382cf8
commit 349b8b0f4f
6 changed files with 51 additions and 22 deletions

View file

@ -33,7 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Admin API: Reports now ordered by newest - Admin API: Reports now ordered by newest
</details> </details>
- Extracted object hashtags into separate table in order to improve hashtag timeline performance (via background migration in `Pleroma.Migrators.HashtagsTableMigrator`). - Improved hashtag timeline performance (requires a background migration).
### Added ### Added

View file

@ -556,7 +556,6 @@ config :pleroma, Oban,
remote_fetcher: 2, remote_fetcher: 2,
attachments_cleanup: 1, attachments_cleanup: 1,
new_users_digest: 1, new_users_digest: 1,
hashtags_cleanup: 1,
mute_expire: 5 mute_expire: 5
], ],
plugins: [Oban.Plugins.Pruner], plugins: [Oban.Plugins.Pruner],

View file

@ -473,6 +473,20 @@ config :pleroma, :config_description, [
} }
] ]
}, },
%{
group: :pleroma,
key: :populate_hashtags_table,
type: :group,
description: "`populate_hashtags_table` background migration settings",
children: [
%{
key: :sleep_interval_ms,
type: :integer,
description:
"Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances)."
}
]
},
%{ %{
group: :pleroma, group: :pleroma,
key: :instance, key: :instance,

View file

@ -65,6 +65,12 @@ To add configuration to your config file, you can copy it from the base config.
* `show_reactions`: Let favourites and emoji reactions be viewed through the API (default: `true`). * `show_reactions`: Let favourites and emoji reactions be viewed through the API (default: `true`).
* `password_reset_token_validity`: The time after which reset tokens aren't accepted anymore, in seconds (default: one day). * `password_reset_token_validity`: The time after which reset tokens aren't accepted anymore, in seconds (default: one day).
## :database
* `improved_hashtag_timeline`: If `true`, hashtags will be fetched from `hashtags` table for hashtags timeline. When `false`, object-embedded hashtags will be used (slower). Is auto-set to `true` (unless overridden) when `HashtagsTableMigrator` completes.
## Background migrations
* `populate_hashtags_table/sleep_interval_ms`: Sleep interval between each chunk of processed records in order to decrease the load on the system (defaults to 0 and should be keep default on most instances).
## Welcome ## Welcome
* `direct_message`: - welcome message sent as a direct message. * `direct_message`: - welcome message sent as a direct message.
* `enabled`: Enables the send a direct message to a newly registered user. Defaults to `false`. * `enabled`: Enables the send a direct message to a newly registered user. Defaults to `false`.

View file

@ -47,16 +47,20 @@ defmodule Pleroma.Hashtag do
|> Map.merge(%{inserted_at: timestamp, updated_at: timestamp}) |> Map.merge(%{inserted_at: timestamp, updated_at: timestamp})
end) end)
with {:ok, %{query_op: hashtags}} <- try do
Multi.new() with {:ok, %{query_op: hashtags}} <-
|> Multi.insert_all(:insert_all_op, Hashtag, structs, on_conflict: :nothing) Multi.new()
|> Multi.run(:query_op, fn _repo, _changes -> |> Multi.insert_all(:insert_all_op, Hashtag, structs, on_conflict: :nothing)
{:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))} |> Multi.run(:query_op, fn _repo, _changes ->
end) {:ok, Repo.all(from(ht in Hashtag, where: ht.name in ^names))}
|> Repo.transaction() do end)
{:ok, hashtags} |> Repo.transaction() do
else {:ok, hashtags}
{:error, _name, value, _changes_so_far} -> {:error, value} else
{:error, _name, value, _changes_so_far} -> {:error, value}
end
rescue
e -> {:error, e}
end end
end end
@ -74,8 +78,9 @@ defmodule Pleroma.Hashtag do
where: hto.object_id == ^object_id, where: hto.object_id == ^object_id,
select: hto.hashtag_id select: hto.hashtag_id
) )
|> Repo.delete_all() do |> Repo.delete_all(),
delete_unreferenced(hashtag_ids) {:ok, unreferenced_count} <- delete_unreferenced(hashtag_ids) do
{:ok, length(hashtag_ids), unreferenced_count}
end end
end end

View file

@ -214,15 +214,20 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do
maps = Enum.map(hashtag_records, &%{hashtag_id: &1.id, object_id: object.id}) maps = Enum.map(hashtag_records, &%{hashtag_id: &1.id, object_id: object.id})
expected_rows = length(hashtag_records) expected_rows = length(hashtag_records)
with {^expected_rows, _} <- Repo.insert_all("hashtags_objects", maps) do base_error =
object.id "ERROR when inserting #{expected_rows} hashtags_objects for obj. #{object.id}"
else
e ->
error =
"ERROR when inserting #{expected_rows} hashtags_objects " <>
"for object #{object.id}: #{inspect(e)}"
Logger.error(error) try do
with {^expected_rows, _} <- Repo.insert_all("hashtags_objects", maps) do
object.id
else
e ->
Logger.error("#{base_error}: #{inspect(e)}")
Repo.rollback(object.id)
end
rescue
e ->
Logger.error("#{base_error}: #{inspect(e)}")
Repo.rollback(object.id) Repo.rollback(object.id)
end end
else else