diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 506fc3daf..8cc9ce425 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -164,7 +164,8 @@ defp cachex_children do
       build_cachex("request_signatures", default_ttl: :timer.hours(24 * 30), limit: 3000),
       build_cachex("rel_me", default_ttl: :timer.hours(24 * 30), limit: 300),
       build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000),
-      build_cachex("http_backoff", default_ttl: :timer.hours(24 * 30), limit: 10000)
+      build_cachex("http_backoff", default_ttl: :timer.hours(24 * 30), limit: 10000),
+      build_cachex("backfetch_timer", default_ttl: :timer.hours(24 * 30), limit: 10000)
     ]
   end
 
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 7bc7b4971..6a7385f5a 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -153,7 +153,6 @@ defmodule Pleroma.User do
     field(:inbox, :string)
     field(:shared_inbox, :string)
     field(:outbox, :string, default: nil)
-    field(:last_outbox_fetch, :naive_datetime, default: nil)
     field(:last_active_at, :naive_datetime)
     field(:disclose_client, :boolean, default: true)
     field(:pinned_objects, :map, default: %{})
@@ -2772,12 +2771,4 @@ def accepts_direct_messages?(%User{accepts_direct_messages_from: :everybody}, _)
 
   def accepts_direct_messages?(%User{accepts_direct_messages_from: :nobody}, _),
     do: false
-
-  def outbox_refreshed(%User{} = user) do
-    # now = NaiveDateTime.truncate(NaiveDateTime.utc_now(), :second)
-
-    user
-    |> cast(%{last_outbox_fetch: NaiveDateTime.utc_now()}, [:last_outbox_fetch])
-    |> update_and_set_cache()
-  end
 end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index c3bd69ae5..19de072e2 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1858,31 +1858,28 @@ def enqueue_pin_fetches(%{pinned_objects: pins}) do
 
   def enqueue_pin_fetches(_), do: nil
 
-  defp need_outbox_refresh?(last_fetch)
+  @cachex Pleroma.Config.get([:cachex, :provider], Cachex)
+  @timer :backfetch_timer_cache
 
-  defp need_outbox_refresh?(nil), do: true
+  def enqueue_outbox_fetches(%{outbox: outbox_address, local: false}) do
+    case @cachex.get(@timer, outbox_address) do
+      {:ok, nil} ->
+        @cachex.put(
+          @timer,
+          outbox_address,
+          true,
+          ttl: Config.get!([:activitypub, :outbox_refetch_cooldown])
+        )
 
-  defp need_outbox_refresh?(%NaiveDateTime{} = last_fetch) do
-    NaiveDateTime.diff(NaiveDateTime.utc_now(), last_fetch) >
-      Config.get!([:activitypub, :outbox_refetch_cooldown])
-  end
+        # enqueue a task to fetch the outbox
+        Logger.debug("Refetching outbox #{outbox_address}")
 
-  def enqueue_outbox_fetches(
-        %{outbox: outbox_address, last_outbox_fetch: last_fetch, local: false} = user
-      ) do
-    if need_outbox_refresh?(last_fetch) do
-      # enqueue a task to fetch the outbox
-      Logger.debug("Refetching outbox #{outbox_address}")
+        Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_outbox", %{
+          "id" => outbox_address
+        })
 
-      Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_outbox", %{
-        "id" => outbox_address
-      })
-
-      User.outbox_refreshed(user)
-
-      :ok
-    else
-      Logger.debug("Not refetching outbox (TTL not reached: #{last_fetch}, age #{NaiveDateTime.diff(NaiveDateTime.utc_now(), last_fetch)})")
+      a ->
+        Logger.debug("Not refetching outbox (TTL not reached)")
     end
 
     :ok
diff --git a/priv/repo/migrations/20241027040000_users_add_outboxes.exs b/priv/repo/migrations/20241027040000_users_add_outboxes.exs
index 10b60507a..cf4ab76cf 100644
--- a/priv/repo/migrations/20241027040000_users_add_outboxes.exs
+++ b/priv/repo/migrations/20241027040000_users_add_outboxes.exs
@@ -4,14 +4,12 @@ defmodule Pleroma.Repo.Migrations.UsersAddOutboxes do
   def up do
     alter table(:users) do
       add_if_not_exists(:outbox, :text)
-      add_if_not_exists(:last_outbox_fetch, :naive_datetime)
     end
   end
 
   def down do
     alter table(:users) do
       remove_if_exists(:outbox, :text)
-      remove_if_exists(:last_outbox_fetch, :naive_datetime)
     end
   end
 end