Only run the fix-up tasks once.

lib/pleroma/web/activity_pub/transmogrifier.ex

@@ -266,16 +266,19 @@ def upgrade_user_from_ap_id(ap_id, async \\ true) do
       data = data
       |> Map.put(:info, Map.merge(user.info, data[:info]))
 
+      already_ap = User.ap_enabled?(user)
       {:ok, user} = User.upgrade_changeset(user, data)
       |> Repo.update()
 
-      # This could potentially take a long time, do it in the background
-      if async do
-        Task.start(fn ->
+      if !already_ap do
+        # This could potentially take a long time, do it in the background
+        if async do
+          Task.start(fn ->
+            user_upgrade_task(user)
+          end)
+        else
           user_upgrade_task(user)
-        end)
-      else
-        user_upgrade_task(user)
+        end
       end
 
       {:ok, user}