diff --git a/packages/backend/src/server/api/authenticate.ts b/packages/backend/src/server/api/authenticate.ts
index 89cbb4f9c..cfcdfcf36 100644
--- a/packages/backend/src/server/api/authenticate.ts
+++ b/packages/backend/src/server/api/authenticate.ts
@@ -3,7 +3,7 @@ import { Users, AccessTokens, Apps } from '@/models/index.js';
 import { AccessToken } from '@/models/entities/access-token.js';
 import { Cache } from '@/misc/cache.js';
 import { App } from '@/models/entities/app.js';
-import { localUserByIdCache, localUserByNativeTokenCache } from '@/services/user-cache.js';
+import { userByIdCache, localUserByNativeTokenCache } from '@/services/user-cache.js';
 import isNativeToken from './common/is-native-token.js';
 
 const appCache = new Cache<App>(Infinity);
@@ -64,11 +64,14 @@ export default async (authorization: string | null | undefined, bodyToken: strin
 			lastUsedAt: new Date(),
 		});
 
-		const user = await localUserByIdCache.fetch(accessToken.userId,
+		const user = await userByIdCache.fetch(accessToken.userId,
 			() => Users.findOneBy({
 				id: accessToken.userId,
 			}) as Promise<ILocalUser>);
 
+		// can't authorize remote users
+		if (!Users.isLocalUser(user)) return [null, null];
+
 		if (accessToken.appId) {
 			const app = await appCache.fetch(accessToken.appId,
 				() => Apps.findOneByOrFail({ id: accessToken.appId! }));
diff --git a/packages/backend/src/services/user-cache.ts b/packages/backend/src/services/user-cache.ts
index d95a8968a..ef939d12b 100644
--- a/packages/backend/src/services/user-cache.ts
+++ b/packages/backend/src/services/user-cache.ts
@@ -5,7 +5,6 @@ import { subscriber } from '@/db/redis.js';
 
 export const userByIdCache = new Cache<CacheableUser>(Infinity);
 export const localUserByNativeTokenCache = new Cache<CacheableLocalUser | null>(Infinity);
-export const localUserByIdCache = new Cache<CacheableLocalUser>(Infinity);
 export const uriPersonCache = new Cache<CacheableUser | null>(Infinity);
 
 subscriber.on('message', async (_, data) => {
@@ -27,7 +26,6 @@ subscriber.on('message', async (_, data) => {
 				}
 				if (Users.isLocalUser(user)) {
 					localUserByNativeTokenCache.set(user.token, user);
-					localUserByIdCache.set(user.id, user);
 				}
 				break;
 			}