EventIndexer: ensure we add initial checkpoints when the db is first opened (#31448)

* EventIndex: add some documentation

... because comments are not subject to rationing.

* EventIndex: rationalise addInitialCheckpoints logic

* EventIndex: improve logging

* use a single `logger` instance to do the prefixing
* use `JSON.stringify` on the checkpoints so that the rageshakes are useful
* distinguish between logger.warn and logger.debug
* emit some logs if the crawler loop fails

* Move check for empty database to `init`

The problem with checking if the database is empty in `onSync` is that, by the
time we get there, it won't be.

Instead let's remember if the db was empty in `init`, and then once a sync
completes, we can add the checkpoints.

* Some tests for EventIndexer

test/unit-tests/indexing/EventIndex-test.ts

@@ -0,0 +1,186 @@
+/*
+Copyright 2025 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { type Mocked } from "jest-mock";
+import {
+    Direction,
+    type MatrixClient,
+    type IEvent,
+    MatrixEvent,
+    type Room,
+    ClientEvent,
+    SyncState,
+} from "matrix-js-sdk/src/matrix";
+
+import EventIndex from "../../../src/indexing/EventIndex.ts";
+import { emitPromise, getMockClientWithEventEmitter, mockClientMethodsRooms, mockPlatformPeg } from "../../test-utils";
+import type BaseEventIndexManager from "../../../src/indexing/BaseEventIndexManager.ts";
+import { type ICrawlerCheckpoint } from "../../../src/indexing/BaseEventIndexManager.ts";
+import SettingsStore from "../../../src/settings/SettingsStore.ts";
+
+afterEach(() => {
+    jest.restoreAllMocks();
+});
+
+describe("EventIndex", () => {
+    it("crawls through the loaded checkpoints", async () => {
+        const mockIndexingManager = {
+            loadCheckpoints: jest.fn(),
+            removeCrawlerCheckpoint: jest.fn(),
+            isEventIndexEmpty: jest.fn().mockResolvedValue(false),
+        } as any as Mocked<BaseEventIndexManager>;
+        mockPlatformPeg({ getEventIndexingManager: () => mockIndexingManager });
+
+        const room1 = { roomId: "!room1:id" } as any as Room;
+        const room2 = { roomId: "!room2:id" } as any as Room;
+        const mockClient = getMockClientWithEventEmitter({
+            getEventMapper: () => (obj: Partial<IEvent>) => new MatrixEvent(obj),
+            createMessagesRequest: jest.fn(),
+            ...mockClientMethodsRooms([room1, room2]),
+        });
+
+        jest.spyOn(SettingsStore, "getValueAt").mockImplementation((_level, settingName): any => {
+            if (settingName === "crawlerSleepTime") return 0;
+            return undefined;
+        });
+
+        mockIndexingManager.loadCheckpoints.mockResolvedValue([
+            { roomId: "!room1:id", token: "token1", direction: Direction.Backward } as ICrawlerCheckpoint,
+            { roomId: "!room2:id", token: "token2", direction: Direction.Forward } as ICrawlerCheckpoint,
+        ]);
+
+        const indexer = new EventIndex();
+        await indexer.init();
+        let changedCheckpointPromise = emitPromise(indexer, "changedCheckpoint") as Promise<Room>;
+
+        indexer.startCrawler();
+
+        // Mock out the /messags request, and wait for the crawler to hit the first room
+        const mock1 = mockCreateMessagesRequest(mockClient);
+        let changedCheckpoint = await changedCheckpointPromise;
+        expect(changedCheckpoint.roomId).toEqual("!room1:id");
+
+        await mock1.called;
+        expect(mockClient.createMessagesRequest).toHaveBeenCalledWith("!room1:id", "token1", 100, "b");
+
+        // Continue, and wait for the crawler to hit the second room
+        changedCheckpointPromise = emitPromise(indexer, "changedCheckpoint") as Promise<Room>;
+        mock1.resolve({ chunk: [] });
+        changedCheckpoint = await changedCheckpointPromise;
+        expect(changedCheckpoint.roomId).toEqual("!room2:id");
+
+        // Mock out the /messages request again, and wait for it to be called
+        const mock2 = mockCreateMessagesRequest(mockClient);
+        await mock2.called;
+        expect(mockClient.createMessagesRequest).toHaveBeenCalledWith("!room2:id", "token2", 100, "f");
+    });
+
+    it("adds checkpoints for the encrypted rooms after the first sync", async () => {
+        const mockIndexingManager = {
+            loadCheckpoints: jest.fn().mockResolvedValue([]),
+            isEventIndexEmpty: jest.fn().mockResolvedValue(true),
+            addCrawlerCheckpoint: jest.fn(),
+            removeCrawlerCheckpoint: jest.fn(),
+            commitLiveEvents: jest.fn(),
+        } as any as Mocked<BaseEventIndexManager>;
+        mockPlatformPeg({ getEventIndexingManager: () => mockIndexingManager });
+
+        const room1 = {
+            roomId: "!room1:id",
+            getLiveTimeline: () => ({
+                getPaginationToken: () => "token1",
+            }),
+        } as any as Room;
+        const room2 = {
+            roomId: "!room2:id",
+            getLiveTimeline: () => ({
+                getPaginationToken: () => "token2",
+            }),
+        } as any as Room;
+        const mockCrypto = {
+            isEncryptionEnabledInRoom: jest.fn().mockResolvedValue(true),
+        };
+        const mockClient = getMockClientWithEventEmitter({
+            getEventMapper: () => (obj: Partial<IEvent>) => new MatrixEvent(obj),
+            createMessagesRequest: jest.fn(),
+            getCrypto: () => mockCrypto as any,
+            ...mockClientMethodsRooms([room1, room2]),
+        });
+
+        const commitLiveEventsCalled = Promise.withResolvers<void>();
+        mockIndexingManager.commitLiveEvents.mockImplementation(async () => {
+            commitLiveEventsCalled.resolve();
+        });
+
+        const indexer = new EventIndex();
+        await indexer.init();
+
+        // During the first sync, some events are added to the index, meaning that `isEventIndexEmpty` will now be false.
+        mockIndexingManager.isEventIndexEmpty.mockResolvedValue(false);
+
+        // The first sync completes:
+        mockClient.emit(ClientEvent.Sync, SyncState.Syncing, null, {});
+
+        // Wait for `commitLiveEvents` to be called, by which time the checkpoints should have been added.
+        await commitLiveEventsCalled.promise;
+        expect(mockIndexingManager.addCrawlerCheckpoint).toHaveBeenCalledTimes(4);
+        expect(mockIndexingManager.addCrawlerCheckpoint).toHaveBeenCalledWith({
+            roomId: "!room1:id",
+            token: "token1",
+            direction: Direction.Backward,
+            fullCrawl: true,
+        });
+        expect(mockIndexingManager.addCrawlerCheckpoint).toHaveBeenCalledWith({
+            roomId: "!room1:id",
+            token: "token1",
+            direction: Direction.Forward,
+        });
+        expect(mockIndexingManager.addCrawlerCheckpoint).toHaveBeenCalledWith({
+            roomId: "!room2:id",
+            token: "token2",
+            direction: Direction.Backward,
+            fullCrawl: true,
+        });
+        expect(mockIndexingManager.addCrawlerCheckpoint).toHaveBeenCalledWith({
+            roomId: "!room2:id",
+            token: "token2",
+            direction: Direction.Forward,
+        });
+    });
+});
+
+/**
+ * Mock out the `createMessagesRequest` method on the client, with an implementation that will block until a resolver is called.
+ *
+ * @returns An object with the following properties:
+ *  * `called`: A promise that resolves when `createMessagesRequest` is called.
+ *  * `resolve`: A function that can be called to allow `createMessagesRequest` to complete.
+ */
+function mockCreateMessagesRequest(mockClient: Mocked<MatrixClient>): {
+    called: Promise<void>;
+    resolve: (result: any) => void;
+} {
+    const messagesCalledPromise = Promise.withResolvers<void>();
+    const messagesResultPromise = Promise.withResolvers();
+    mockClient.createMessagesRequest.mockImplementationOnce(() => {
+        messagesCalledPromise.resolve();
+        return messagesResultPromise.promise as any;
+    });
+    return {
+        called: messagesCalledPromise.promise,
+        resolve: messagesResultPromise.resolve,
+    };
+}