diff --git a/lib/pleroma/web/activity_pub/object_validators/accept_validator.ex b/lib/pleroma/web/activity_pub/object_validators/accept_validator.ex
index b81e078e3..6d0fa669a 100644
--- a/lib/pleroma/web/activity_pub/object_validators/accept_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/accept_validator.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AcceptValidator do
   use Ecto.Schema
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.Activity
 
   import Ecto.Changeset
   import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
@@ -31,7 +32,8 @@ def validate_data(cng) do
     |> validate_required([:id, :type, :actor, :to, :cc, :object])
     |> validate_inclusion(:type, ["Accept"])
     |> validate_actor_presence()
-    |> validate_object_presence()
+    |> validate_object_presence(allowed_types: ["Follow"])
+    |> validate_accept_rights()
   end
 
   def cast_and_validate(data) do
@@ -39,4 +41,16 @@ def cast_and_validate(data) do
     |> cast_data
     |> validate_data
   end
+
+  def validate_accept_rights(cng) do
+    with object_id when is_binary(object_id) <- get_field(cng, :object),
+         %Activity{data: %{"object" => followed_actor}} <- Activity.get_by_ap_id(object_id),
+         true <- followed_actor == get_field(cng, :actor) do
+      cng
+    else
+      _e ->
+        cng
+        |> add_error(:actor, "can't accept the given activity")
+    end
+  end
 end
diff --git a/test/web/activity_pub/object_validators/accept_validation_test.exs b/test/web/activity_pub/object_validators/accept_validation_test.exs
index 7f5dc14af..2d5d18046 100644
--- a/test/web/activity_pub/object_validators/accept_validation_test.exs
+++ b/test/web/activity_pub/object_validators/accept_validation_test.exs
@@ -41,4 +41,15 @@ test "it fails when the accepted activity doesn't exist", %{accept_data: accept_
 
     assert {:error, _} = ObjectValidator.validate(accept_data, [])
   end
+
+  test "for an accepted follow, it only validates if the actor of the accept is the followed actor",
+       %{accept_data: accept_data} do
+    stranger = insert(:user)
+
+    accept_data =
+      accept_data
+      |> Map.put("actor", stranger.ap_id)
+
+    assert {:error, _} = ObjectValidator.validate(accept_data, [])
+  end
 end