feat: introduce intersection calculation of charts

packages/backend/migration/1644344266289-chart-v14.js

@@ -0,0 +1,47 @@
+const { MigrationInterface, QueryRunner } = require("typeorm");
+
+module.exports = class chartV141644344266289 {
+    name = 'chartV141644344266289'
+
+    async up(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "unique_temp___users"`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "___users"`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "unique_temp___notedUsers"`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "___notedUsers"`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "unique_temp___users"`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "___users"`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "unique_temp___notedUsers"`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "___notedUsers"`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "___readWrite" smallint NOT NULL DEFAULT '0'`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "unique_temp___read" character varying array NOT NULL DEFAULT '{}'`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "___read" smallint NOT NULL DEFAULT '0'`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "unique_temp___write" character varying array NOT NULL DEFAULT '{}'`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "___write" smallint NOT NULL DEFAULT '0'`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "___readWrite" smallint NOT NULL DEFAULT '0'`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "unique_temp___read" character varying array NOT NULL DEFAULT '{}'`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "___read" smallint NOT NULL DEFAULT '0'`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "unique_temp___write" character varying array NOT NULL DEFAULT '{}'`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "___write" smallint NOT NULL DEFAULT '0'`);
+    }
+
+    async down(queryRunner) {
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "___write"`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "unique_temp___write"`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "___read"`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "unique_temp___read"`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" DROP COLUMN "___readWrite"`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "___write"`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "unique_temp___write"`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "___read"`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "unique_temp___read"`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" DROP COLUMN "___readWrite"`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "___notedUsers" smallint NOT NULL DEFAULT '0'`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "unique_temp___notedUsers" character varying array NOT NULL DEFAULT '{}'`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "___users" integer NOT NULL DEFAULT '0'`);
+        await queryRunner.query(`ALTER TABLE "__chart_day__active_users" ADD "unique_temp___users" character varying array NOT NULL DEFAULT '{}'`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "___notedUsers" smallint NOT NULL DEFAULT '0'`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "unique_temp___notedUsers" character varying array NOT NULL DEFAULT '{}'`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "___users" integer NOT NULL DEFAULT '0'`);
+        await queryRunner.query(`ALTER TABLE "__chart__active_users" ADD "unique_temp___users" character varying array NOT NULL DEFAULT '{}'`);
+    }
+}

packages/backend/src/server/api/endpoints/channels/timeline.ts

@@ -80,7 +80,7 @@ export default define(meta, async (ps, user) => {
 
 	const timeline = await query.take(ps.limit!).getMany();
 
-	if (user) activeUsersChart.update(user);
+	if (user) activeUsersChart.read(user);
 
 	return await Notes.packMany(timeline, user);
 });

packages/backend/src/server/api/endpoints/notes/global-timeline.ts

@@ -96,7 +96,7 @@ export default define(meta, async (ps, user) => {
 
 	process.nextTick(() => {
 		if (user) {
-			activeUsersChart.update(user);
+			activeUsersChart.read(user);
 		}
 	});
 

packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts

@@ -153,7 +153,7 @@ export default define(meta, async (ps, user) => {
 
 	process.nextTick(() => {
 		if (user) {
-			activeUsersChart.update(user);
+			activeUsersChart.read(user);
 		}
 	});
 

packages/backend/src/server/api/endpoints/notes/local-timeline.ts

@@ -122,7 +122,7 @@ export default define(meta, async (ps, user) => {
 
 	process.nextTick(() => {
 		if (user) {
-			activeUsersChart.update(user);
+			activeUsersChart.read(user);
 		}
 	});
 

packages/backend/src/server/api/endpoints/notes/timeline.ts

@@ -145,7 +145,7 @@ export default define(meta, async (ps, user) => {
 
 	process.nextTick(() => {
 		if (user) {
-			activeUsersChart.update(user);
+			activeUsersChart.read(user);
 		}
 	});
 

packages/backend/src/server/api/endpoints/notes/user-list-timeline.ts

@@ -142,7 +142,7 @@ export default define(meta, async (ps, user) => {
 
 	const timeline = await query.take(ps.limit!).getMany();
 
-	activeUsersChart.update(user);
+	activeUsersChart.read(user);
 
 	return await Notes.packMany(timeline, user);
 });

packages/backend/src/services/chart/charts/active-users.ts

@@ -23,9 +23,9 @@ export default class ActiveUsersChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	public async update(user: { id: User['id'], host: null, createdAt: User['createdAt'] }): Promise<void> {
+	public async read(user: { id: User['id'], host: null, createdAt: User['createdAt'] }): Promise<void> {
 		await this.commit({
-			'users': [user.id],
+			'read': [user.id],
 			'registeredWithinWeek': (Date.now() - user.createdAt.getTime() < week) ? [user.id] : [],
 			'registeredWithinMonth': (Date.now() - user.createdAt.getTime() < month) ? [user.id] : [],
 			'registeredWithinYear': (Date.now() - user.createdAt.getTime() < year) ? [user.id] : [],
@@ -36,9 +36,9 @@ export default class ActiveUsersChart extends Chart<typeof schema> {
 	}
 
 	@autobind
-	public async noted(user: { id: User['id'], host: null, createdAt: User['createdAt'] }): Promise<void> {
+	public async write(user: { id: User['id'], host: null, createdAt: User['createdAt'] }): Promise<void> {
 		await this.commit({
-			'notedUsers': [user.id],
+			'write': [user.id],
 		});
 	}
 }

packages/backend/src/services/chart/charts/entities/active-users.ts

@@ -3,8 +3,9 @@ import Chart from '../../core';
 export const name = 'activeUsers';
 
 export const schema = {
-	'users': { uniqueIncrement: true },
+	'readWrite': { intersection: ['read', 'write'], range: 'small' },
-	'notedUsers': { uniqueIncrement: true, range: 'small' },
+	'read': { uniqueIncrement: true, range: 'small' },
+	'write': { uniqueIncrement: true, range: 'small' },
 	'registeredWithinWeek': { uniqueIncrement: true, range: 'small' },
 	'registeredWithinMonth': { uniqueIncrement: true, range: 'small' },
 	'registeredWithinYear': { uniqueIncrement: true, range: 'small' },

packages/backend/src/services/chart/charts/entities/test-intersection.ts

@@ -0,0 +1,11 @@
+import Chart from '../../core';
+
+export const name = 'testIntersection';
+
+export const schema = {
+	'a': { uniqueIncrement: true },
+	'b': { uniqueIncrement: true },
+	'aAndB': { intersection: ['a', 'b'] },
+} as const;
+
+export const entity = Chart.schemaToEntity(name, schema);

packages/backend/src/services/chart/charts/test-intersection.ts

@@ -0,0 +1,32 @@
+import autobind from 'autobind-decorator';
+import Chart, { KVs } from '../core';
+import { name, schema } from './entities/test-intersection';
+
+/**
+ * For testing
+ */
+// eslint-disable-next-line import/no-default-export
+export default class TestIntersectionChart extends Chart<typeof schema> {
+	constructor() {
+		super(name, schema);
+	}
+
+	@autobind
+	protected async queryCurrentState(): Promise<Partial<KVs<typeof schema>>> {
+		return {};
+	}
+
+	@autobind
+	public async addA(key: string): Promise<void> {
+		await this.commit({
+			a: [key],
+		});
+	}
+
+	@autobind
+	public async addB(key: string): Promise<void> {
+		await this.commit({
+			b: [key],
+		});
+	}
+}

packages/backend/src/services/chart/core.ts

@@ -46,6 +46,8 @@ const removeDuplicates = (array: any[]) => Array.from(new Set(array));
 type Schema = Record<string, {
 	uniqueIncrement?: boolean;
 
+	intersection?: string[] | ReadonlyArray<string>;
+
 	range?: 'big' | 'small' | 'medium';
 
 	// previousな値を引き継ぐかどうか
@@ -384,6 +386,33 @@ export default abstract class Chart<T extends Schema> {
 				}
 			}
 
+			// compute intersection
+			// TODO: intersectionに指定されたカラムがintersectionだった場合の対応
+			for (const [k, v] of Object.entries(this.schema)) {
+				const intersection = v.intersection;
+				if (intersection) {
+					const name = columnPrefix + k.replaceAll('.', columnDot);
+					const firstKey = intersection[0];
+					const firstTempColumnName = uniqueTempColumnPrefix + firstKey.replaceAll('.', columnDot);
+					const currentValuesForHour = new Set([...(finalDiffs[firstKey] ?? []), ...logHour[firstTempColumnName]]);
+					const currentValuesForDay = new Set([...(finalDiffs[firstKey] ?? []), ...logDay[firstTempColumnName]]);
+					for (let i = 1; i < intersection.length; i++) {
+						const targetKey = intersection[i];
+						const targetTempColumnName = uniqueTempColumnPrefix + targetKey.replaceAll('.', columnDot);
+						const targetValuesForHour = new Set([...(finalDiffs[targetKey] ?? []), ...logHour[targetTempColumnName]]);
+						const targetValuesForDay = new Set([...(finalDiffs[targetKey] ?? []), ...logDay[targetTempColumnName]]);
+						currentValuesForHour.forEach(v => {
+							if (!targetValuesForHour.has(v)) currentValuesForHour.delete(v);
+						});
+						currentValuesForDay.forEach(v => {
+							if (!targetValuesForDay.has(v)) currentValuesForDay.delete(v);
+						});
+					}
+					queryForHour[name] = currentValuesForHour.size;
+					queryForDay[name] = currentValuesForDay.size;
+				}
+			}
+
 			// ログ更新
 			await Promise.all([
 				this.repositoryForHour.createQueryBuilder()

packages/backend/src/services/note/create.ts

@@ -297,7 +297,7 @@ export default async (user: { id: User['id']; username: User['username']; host:
 	}
 
 	if (!silent) {
-		if (Users.isLocalUser(user)) activeUsersChart.noted(user);
+		if (Users.isLocalUser(user)) activeUsersChart.write(user);
 
 		// 未読通知を作成
 		if (data.visibility === 'specified') {

packages/backend/test/chart.ts

@@ -6,14 +6,17 @@ import { async, initTestDb } from './utils';
 import TestChart from '../src/services/chart/charts/test';
 import TestGroupedChart from '../src/services/chart/charts/test-grouped';
 import TestUniqueChart from '../src/services/chart/charts/test-unique';
+import TestIntersectionChart from '../src/services/chart/charts/test-intersection';
 import * as _TestChart from '../src/services/chart/charts/entities/test';
 import * as _TestGroupedChart from '../src/services/chart/charts/entities/test-grouped';
 import * as _TestUniqueChart from '../src/services/chart/charts/entities/test-unique';
+import * as _TestIntersectionChart from '../src/services/chart/charts/entities/test-intersection';
 
 describe('Chart', () => {
 	let testChart: TestChart;
 	let testGroupedChart: TestGroupedChart;
 	let testUniqueChart: TestUniqueChart;
+	let testIntersectionChart: TestIntersectionChart;
 	let clock: lolex.Clock;
 
 	beforeEach(async(async () => {
@@ -21,11 +24,13 @@ describe('Chart', () => {
 			_TestChart.entity.hour, _TestChart.entity.day,
 			_TestGroupedChart.entity.hour, _TestGroupedChart.entity.day,
 			_TestUniqueChart.entity.hour, _TestUniqueChart.entity.day,
+			_TestIntersectionChart.entity.hour, _TestIntersectionChart.entity.day,
 		]);
 
 		testChart = new TestChart();
 		testGroupedChart = new TestGroupedChart();
 		testUniqueChart = new TestUniqueChart();
+		testIntersectionChart = new TestIntersectionChart();
 
 		clock = lolex.install({
 			now: new Date(Date.UTC(2000, 0, 1, 0, 0, 0))
@@ -426,6 +431,45 @@ describe('Chart', () => {
 				foo: [2, 0, 0],
 			});
 		}));
+
+		describe('Intersection', () => {
+			it('条件が満たされていない場合はカウントされない', async(async () => {
+				await testIntersectionChart.addA('alice');
+				await testIntersectionChart.addA('bob');
+				await testIntersectionChart.addB('carol');
+				await testIntersectionChart.save();
+	
+				const chartHours = await testUniqueChart.getChart('hour', 3, null);
+				const chartDays = await testUniqueChart.getChart('day', 3, null);
+	
+				assert.deepStrictEqual(chartHours, {
+					aAndB: [0, 0, 0],
+				});
+	
+				assert.deepStrictEqual(chartDays, {
+					aAndB: [0, 0, 0],
+				});
+			}));
+
+			it('条件が満たされている場合にカウントされる', async(async () => {
+				await testIntersectionChart.addA('alice');
+				await testIntersectionChart.addA('bob');
+				await testIntersectionChart.addB('carol');
+				await testIntersectionChart.addB('alice');
+				await testIntersectionChart.save();
+	
+				const chartHours = await testUniqueChart.getChart('hour', 3, null);
+				const chartDays = await testUniqueChart.getChart('day', 3, null);
+	
+				assert.deepStrictEqual(chartHours, {
+					aAndB: [1, 0, 0],
+				});
+	
+				assert.deepStrictEqual(chartDays, {
+					aAndB: [1, 0, 0],
+				});
+			}));
+		});
 	});
 
 	describe('Resync', () => {

packages/client/src/components/chart.vue

@@ -69,6 +69,7 @@ const colors = {
 	yellow: '#FEB019',
 	red: '#FF4560',
 	purple: '#e300db',
+	orange: '#fe6919',
 };
 const colorSets = [colors.blue, colors.green, colors.yellow, colors.red, colors.purple];
 const getColor = (i) => {
@@ -518,15 +519,20 @@ export default defineComponent({
 			const raw = await os.api('charts/active-users', { limit: props.limit, span: props.span });
 			return {
 				series: [{
-					name: 'Active',
+					name: 'Read & Write',
 					type: 'area',
-					data: format(raw.users),
+					data: format(raw.readWrite),
-					color: '#888888',
+					color: colors.orange,
 				}, {
-					name: 'Noted',
+					name: 'Write',
 					type: 'area',
-					data: format(raw.notedUsers),
+					data: format(raw.write),
 					color: colors.blue,
+				}, {
+					name: 'Read',
+					type: 'area',
+					data: format(raw.read),
+					color: '#888888',
 				}, {
 					name: '< Week',
 					type: 'area',