[#3213] Removed PK from hashtags_objects table. Improved hashtags_transfer mix task (logging of failed ids).

This commit is contained in:
Ivan Tashkinov 2021-01-07 12:20:29 +03:00
parent 48e0f22ab1
commit 0d521022fe
3 changed files with 20 additions and 15 deletions

View file

@ -139,6 +139,7 @@ defmodule Mix.Tasks.Pleroma.Database do
Logger.info("Starting transferring object embedded hashtags to `hashtags` table...") Logger.info("Starting transferring object embedded hashtags to `hashtags` table...")
# Note: most objects have Mention-type AS2 tags and no hashtags (but we can't filter them out)
from( from(
object in Object, object in Object,
left_join: hashtag in assoc(object, :hashtags), left_join: hashtag in assoc(object, :hashtags),
@ -153,13 +154,10 @@ defmodule Mix.Tasks.Pleroma.Database do
|> Stream.each(fn objects -> |> Stream.each(fn objects ->
Logger.info("Processing #{length(objects)} objects starting from id #{hd(objects).id}...") Logger.info("Processing #{length(objects)} objects starting from id #{hd(objects).id}...")
Enum.map( failed_ids =
objects, objects
fn object -> |> Enum.map(fn object ->
hashtags = hashtags = Object.object_data_hashtags(%{"tag" => Jason.decode!(object.tag)})
object.tag
|> Jason.decode!()
|> Enum.filter(&is_bitstring(&1))
Repo.transaction(fn -> Repo.transaction(fn ->
with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do
@ -169,7 +167,7 @@ defmodule Mix.Tasks.Pleroma.Database do
"insert into hashtags_objects(hashtag_id, object_id) values ($1, $2);", "insert into hashtags_objects(hashtag_id, object_id) values ($1, $2);",
[hashtag_record.id, object.id] [hashtag_record.id, object.id]
) do ) do
:noop nil
else else
{:error, e} -> {:error, e} ->
error = error =
@ -177,18 +175,25 @@ defmodule Mix.Tasks.Pleroma.Database do
"#{hashtag_record.id}: #{inspect(e)}" "#{hashtag_record.id}: #{inspect(e)}"
Logger.error(error) Logger.error(error)
Repo.rollback(error) Repo.rollback(object.id)
end end
end end
object.id
else else
e -> e ->
error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}" error = "ERROR: could not create hashtags for object #{object.id}: #{inspect(e)}"
Logger.error(error) Logger.error(error)
Repo.rollback(error) Repo.rollback(object.id)
end end
end) end)
end)
|> Enum.filter(&(elem(&1, 0) == :error))
|> Enum.map(&elem(&1, 1))
if Enum.any?(failed_ids) do
Logger.error("ERROR: transfer_hashtags iteration failed for ids: #{inspect(failed_ids)}")
end end
)
end) end)
|> Stream.run() |> Stream.run()

View file

@ -404,7 +404,7 @@ defmodule Pleroma.Object do
defp embedded_hashtags(_), do: [] defp embedded_hashtags(_), do: []
defp object_data_hashtags(%{"tag" => tags}) when is_list(tags) do def object_data_hashtags(%{"tag" => tags}) when is_list(tags) do
tags tags
|> Enum.filter(fn |> Enum.filter(fn
%{"type" => "Hashtag"} = data -> Map.has_key?(data, "name") %{"type" => "Hashtag"} = data -> Map.has_key?(data, "name")
@ -419,5 +419,5 @@ defmodule Pleroma.Object do
|> Enum.uniq() |> Enum.uniq()
end end
defp object_data_hashtags(_), do: [] def object_data_hashtags(_), do: []
end end

View file

@ -2,7 +2,7 @@ defmodule Pleroma.Repo.Migrations.CreateHashtagsObjects do
use Ecto.Migration use Ecto.Migration
def change do def change do
create_if_not_exists table(:hashtags_objects) do create_if_not_exists table(:hashtags_objects, primary_key: false) do
add(:hashtag_id, references(:hashtags), null: false) add(:hashtag_id, references(:hashtags), null: false)
add(:object_id, references(:objects), null: false) add(:object_id, references(:objects), null: false)
end end