Fix flaky playwright tests (#28957)

* Docs Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Avoid reusing user1234 Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Fix stale-screenshot-reporter.ts Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Clean up public rooms between tests on reused homeserver Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Deflake spotlight when homeserver is reused Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Deflake more tests using existing username Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Clean mailhog between tests Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Fix more flakes Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Fix missing _request Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Fix playwright flaky tests Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Wipe mailhog between test runs Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Delint Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * delint Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Deflake more tests Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Fix flaky tests Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Fix flaky tests Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Fix mas config Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Fix another flaky test Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --------- Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
2025-01-14 11:37:39 +00:00
parent 37f8d70d89
commit 5882b004f5
10 changed files with 448 additions and 206 deletions
--- a/playwright/testcontainers/HomeserverContainer.ts
+++ b/playwright/testcontainers/HomeserverContainer.ts
@@ -6,17 +6,19 @@ Please see LICENSE files in the repository root for full details.
 */

 import { AbstractStartedContainer, GenericContainer } from "testcontainers";
-import { APIRequestContext } from "@playwright/test";
+import { APIRequestContext, TestInfo } from "@playwright/test";

-import { StartedSynapseContainer } from "./synapse.ts";
 import { HomeserverInstance } from "../plugins/homeserver";
+import { StartedMatrixAuthenticationServiceContainer } from "./mas.ts";

 export interface HomeserverContainer<Config> extends GenericContainer {
    withConfigField(key: string, value: any): this;
    withConfig(config: Partial<Config>): this;
-    start(): Promise<StartedSynapseContainer>;
+    withMatrixAuthenticationService(mas?: StartedMatrixAuthenticationServiceContainer): this;
+    start(): Promise<StartedHomeserverContainer>;
 }

 export interface StartedHomeserverContainer extends AbstractStartedContainer, HomeserverInstance {
    setRequest(request: APIRequestContext): void;
+    onTestFinished(testInfo: TestInfo): Promise<void>;
 }
--- a/playwright/testcontainers/dendrite.ts
+++ b/playwright/testcontainers/dendrite.ts
@@ -13,6 +13,7 @@ import { randB64Bytes } from "../plugins/utils/rand.ts";
 import { StartedSynapseContainer } from "./synapse.ts";
 import { deepCopy } from "../plugins/utils/object.ts";
 import { HomeserverContainer } from "./HomeserverContainer.ts";
+import { StartedMatrixAuthenticationServiceContainer } from "./mas.ts";

 const DEFAULT_CONFIG = {
    version: 2,
@@ -235,7 +236,11 @@ export class DendriteContainer extends GenericContainer implements HomeserverCon
        return this;
    }

-    public override async start(): Promise<StartedSynapseContainer> {
+    public withMatrixAuthenticationService(mas?: StartedMatrixAuthenticationServiceContainer): this {
+        throw new Error("Dendrite does not support MAS.");
+    }
+
+    public override async start(): Promise<StartedDendriteContainer> {
        this.withCopyContentToContainer([
            {
                target: "/etc/dendrite/dendrite.yaml",
@@ -244,8 +249,7 @@ export class DendriteContainer extends GenericContainer implements HomeserverCon
        ]);

        const container = await super.start();
-        // Surprisingly, Dendrite implements the same register user Admin API Synapse, so we can just extend it
-        return new StartedSynapseContainer(
+        return new StartedDendriteContainer(
            container,
            `http://${container.getHost()}:${container.getMappedPort(8008)}`,
            this.config.client_api.registration_shared_secret,
@@ -258,3 +262,6 @@ export class PineconeContainer extends DendriteContainer {
        super("matrixdotorg/dendrite-demo-pinecone:main", "/usr/bin/dendrite-demo-pinecone");
    }
 }
+
+// Surprisingly, Dendrite implements the same register user Synapse Admin API, so we can just extend it
+export class StartedDendriteContainer extends StartedSynapseContainer {}
--- a/playwright/testcontainers/mas.ts
+++ b/playwright/testcontainers/mas.ts
@@ -5,12 +5,13 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */

-import { AbstractStartedContainer, GenericContainer, StartedTestContainer, Wait } from "testcontainers";
+import { AbstractStartedContainer, GenericContainer, StartedTestContainer, Wait, ExecResult } from "testcontainers";
 import { StartedPostgreSqlContainer } from "@testcontainers/postgresql";
 import * as YAML from "yaml";

 import { getFreePort } from "../plugins/utils/port.ts";
 import { deepCopy } from "../plugins/utils/object.ts";
+import { Credentials } from "../plugins/homeserver";

 const DEFAULT_CONFIG = {
    http: {
@@ -18,18 +19,10 @@ const DEFAULT_CONFIG = {
            {
                name: "web",
                resources: [
-                    {
-                        name: "discovery",
-                    },
-                    {
-                        name: "human",
-                    },
-                    {
-                        name: "oauth",
-                    },
-                    {
-                        name: "compat",
-                    },
+                    { name: "discovery" },
+                    { name: "human" },
+                    { name: "oauth" },
+                    { name: "compat" },
                    {
                        name: "graphql",
                        playground: true,
@@ -182,9 +175,12 @@ const DEFAULT_CONFIG = {

 export class MatrixAuthenticationServiceContainer extends GenericContainer {
    private config: typeof DEFAULT_CONFIG;
+    private readonly args = ["-c", "/config/config.yaml"];

    constructor(db: StartedPostgreSqlContainer) {
-        super("ghcr.io/element-hq/matrix-authentication-service:0.12.0");
+        // We rely on `mas-cli manage add-email` which isn't in a release yet
+        // https://github.com/element-hq/matrix-authentication-service/pull/3235
+        super("ghcr.io/element-hq/matrix-authentication-service:sha-0b90c33");

        this.config = deepCopy(DEFAULT_CONFIG);
        this.config.database.username = db.getUsername();
@@ -192,7 +188,7 @@ export class MatrixAuthenticationServiceContainer extends GenericContainer {

        this.withExposedPorts(8080, 8081)
            .withWaitStrategy(Wait.forHttp("/health", 8081))
-            .withCommand(["server", "--config", "/config/config.yaml"]);
+            .withCommand(["server", ...this.args]);
    }

    public withConfig(config: object): this {
@@ -220,15 +216,125 @@ export class MatrixAuthenticationServiceContainer extends GenericContainer {
            },
        ]);

-        return new StartedMatrixAuthenticationServiceContainer(await super.start(), `http://localhost:${port}`);
+        return new StartedMatrixAuthenticationServiceContainer(
+            await super.start(),
+            `http://localhost:${port}`,
+            this.args,
+        );
    }
 }

 export class StartedMatrixAuthenticationServiceContainer extends AbstractStartedContainer {
+    private adminTokenPromise?: Promise<string>;
+
    constructor(
        container: StartedTestContainer,
        public readonly baseUrl: string,
+        private readonly args: string[],
    ) {
        super(container);
    }
+
+    public async getAdminToken(): Promise<string> {
+        if (this.adminTokenPromise === undefined) {
+            this.adminTokenPromise = this.registerUserInternal(
+                "admin",
+                "totalyinsecureadminpassword",
+                undefined,
+                true,
+            ).then((res) => res.accessToken);
+        }
+        return this.adminTokenPromise;
+    }
+
+    private async manage(cmd: string, ...args: string[]): Promise<ExecResult> {
+        const result = await this.exec(["mas-cli", "manage", cmd, ...this.args, ...args]);
+        if (result.exitCode !== 0) {
+            throw new Error(`Failed mas-cli manage ${cmd}: ${result.output}`);
+        }
+        return result;
+    }
+
+    private async manageRegisterUser(
+        username: string,
+        password: string,
+        displayName?: string,
+        admin = false,
+    ): Promise<string> {
+        const args: string[] = [];
+        if (admin) args.push("-a");
+        const result = await this.manage(
+            "register-user",
+            ...args,
+            "-y",
+            "-p",
+            password,
+            "-d",
+            displayName ?? "",
+            username,
+        );
+
+        const registerLines = result.output.trim().split("\n");
+        const userId = registerLines
+            .find((line) => line.includes("Matrix ID: "))
+            ?.split(": ")
+            .pop();
+
+        if (!userId) {
+            throw new Error(`Failed to register user: ${result.output}`);
+        }
+
+        return userId;
+    }
+
+    private async manageIssueCompatibilityToken(
+        username: string,
+        admin = false,
+    ): Promise<{ accessToken: string; deviceId: string }> {
+        const args: string[] = [];
+        if (admin) args.push("--yes-i-want-to-grant-synapse-admin-privileges");
+        const result = await this.manage("issue-compatibility-token", ...args, username);
+
+        const parts = result.output.trim().split(/\s+/);
+        const accessToken = parts.find((part) => part.startsWith("mct_"));
+        const deviceId = parts.find((part) => part.startsWith("compat_session.device="))?.split("=")[1];
+
+        if (!accessToken || !deviceId) {
+            throw new Error(`Failed to issue compatibility token: ${result.output}`);
+        }
+
+        return { accessToken, deviceId };
+    }
+
+    private async registerUserInternal(
+        username: string,
+        password: string,
+        displayName?: string,
+        admin = false,
+    ): Promise<Credentials> {
+        const userId = await this.manageRegisterUser(username, password, displayName, admin);
+        const { deviceId, accessToken } = await this.manageIssueCompatibilityToken(username, admin);
+
+        return {
+            userId,
+            accessToken,
+            deviceId,
+            homeServer: userId.slice(1).split(":").slice(1).join(":"),
+            displayName,
+            username,
+            password,
+        };
+    }
+
+    public async registerUser(username: string, password: string, displayName?: string): Promise<Credentials> {
+        return this.registerUserInternal(username, password, displayName, false);
+    }
+
+    public async setThreepid(username: string, medium: string, address: string): Promise<void> {
+        if (medium !== "email") {
+            throw new Error("Only email threepids are supported by MAS");
+        }
+
+        await this.manage("add-email", username, address);
+    }
 }
--- a/playwright/testcontainers/synapse.ts
+++ b/playwright/testcontainers/synapse.ts
@@ -6,7 +6,7 @@ Please see LICENSE files in the repository root for full details.
 */

 import { AbstractStartedContainer, GenericContainer, RestartOptions, StartedTestContainer, Wait } from "testcontainers";
-import { APIRequestContext } from "@playwright/test";
+import { APIRequestContext, TestInfo } from "@playwright/test";
 import crypto from "node:crypto";
 import * as YAML from "yaml";
 import { set } from "lodash";
@@ -16,6 +16,8 @@ import { randB64Bytes } from "../plugins/utils/rand.ts";
 import { Credentials } from "../plugins/homeserver";
 import { deepCopy } from "../plugins/utils/object.ts";
 import { HomeserverContainer, StartedHomeserverContainer } from "./HomeserverContainer.ts";
+import { StartedMatrixAuthenticationServiceContainer } from "./mas.ts";
+import { Api, ClientServerApi, Verb } from "../plugins/utils/api.ts";

 const TAG = "develop@sha256:7be2e00da62dfbb2bad071c6d408fecb1fabf740a538d08768b9b3e0a8c45350";

@@ -142,6 +144,7 @@ export type SynapseConfig = Partial<typeof DEFAULT_CONFIG>;

 export class SynapseContainer extends GenericContainer implements HomeserverContainer<typeof DEFAULT_CONFIG> {
    private config: typeof DEFAULT_CONFIG;
+    private mas?: StartedMatrixAuthenticationServiceContainer;

    constructor() {
        super(`ghcr.io/element-hq/synapse:${TAG}`);
@@ -201,6 +204,11 @@ export class SynapseContainer extends GenericContainer implements HomeserverCont
        return this;
    }

+    public withMatrixAuthenticationService(mas?: StartedMatrixAuthenticationServiceContainer): this {
+        this.mas = mas;
+        return this;
+    }
+
    public override async start(): Promise<StartedSynapseContainer> {
        // Synapse config public_baseurl needs to know what URL it'll be accessed from, so we have to map the port manually
        const port = await getFreePort();
@@ -219,17 +227,25 @@ export class SynapseContainer extends GenericContainer implements HomeserverCont
                },
            ]);

-        return new StartedSynapseContainer(
-            await super.start(),
-            `http://localhost:${port}`,
-            this.config.registration_shared_secret,
-        );
+        const container = await super.start();
+        const baseUrl = `http://localhost:${port}`;
+        if (this.mas) {
+            return new StartedSynapseWithMasContainer(
+                container,
+                baseUrl,
+                this.config.registration_shared_secret,
+                this.mas,
+            );
+        }
+
+        return new StartedSynapseContainer(container, baseUrl, this.config.registration_shared_secret);
    }
 }

 export class StartedSynapseContainer extends AbstractStartedContainer implements StartedHomeserverContainer {
-    private adminToken?: string;
-    private request?: APIRequestContext;
+    protected adminTokenPromise?: Promise<string>;
+    protected readonly adminApi: Api;
+    public readonly csApi: ClientServerApi;

    constructor(
        container: StartedTestContainer,
@@ -237,15 +253,36 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements
        private readonly registrationSharedSecret: string,
    ) {
        super(container);
+        this.adminApi = new Api(`${this.baseUrl}/_synapse/admin`);
+        this.csApi = new ClientServerApi(this.baseUrl);
    }

    public restart(options?: Partial<RestartOptions>): Promise<void> {
-        this.adminToken = undefined;
+        this.adminTokenPromise = undefined;
        return super.restart(options);
    }

    public setRequest(request: APIRequestContext): void {
-        this.request = request;
+        this.csApi.setRequest(request);
+        this.adminApi.setRequest(request);
+    }
+
+    public async onTestFinished(testInfo: TestInfo): Promise<void> {
+        // Clean up the server to prevent rooms leaking between tests
+        await this.deletePublicRooms();
+    }
+
+    protected async deletePublicRooms(): Promise<void> {
+        const token = await this.getAdminToken();
+        // We hide the rooms from the room directory to save time between tests and for portability between homeservers
+        const { chunk: rooms } = await this.csApi.request<{
+            chunk: { room_id: string }[];
+        }>("GET", "/v3/publicRooms", token, {});
+        await Promise.all(
+            rooms.map((room) =>
+                this.csApi.request("PUT", `/v3/directory/list/room/${room.room_id}`, token, { visibility: "private" }),
+            ),
+        );
    }

    private async registerUserInternal(
@@ -254,28 +291,26 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements
        displayName?: string,
        admin = false,
    ): Promise<Credentials> {
-        const url = `${this.baseUrl}/_synapse/admin/v1/register`;
-        const { nonce } = await this.request.get(url).then((r) => r.json());
+        const path = "/v1/register";
+        const { nonce } = await this.adminApi.request<{ nonce: string }>("GET", path, undefined, {});
        const mac = crypto
            .createHmac("sha1", this.registrationSharedSecret)
            .update(`${nonce}\0${username}\0${password}\0${admin ? "" : "not"}admin`)
            .digest("hex");
-        const res = await this.request.post(url, {
-            data: {
-                nonce,
-                username,
-                password,
-                mac,
-                admin,
-                displayname: displayName,
-            },
+        const data = await this.adminApi.request<{
+            home_server: string;
+            access_token: string;
+            user_id: string;
+            device_id: string;
+        }>("POST", path, undefined, {
+            nonce,
+            username,
+            password,
+            mac,
+            admin,
+            displayname: displayName,
        });

-        if (!res.ok()) {
-            throw await res.json();
-        }
-
-        const data = await res.json();
        return {
            homeServer: data.home_server || data.user_id.split(":").slice(1).join(":"),
            accessToken: data.access_token,
@@ -287,57 +322,67 @@ export class StartedSynapseContainer extends AbstractStartedContainer implements
        };
    }

+    protected async getAdminToken(): Promise<string> {
+        if (this.adminTokenPromise === undefined) {
+            this.adminTokenPromise = this.registerUserInternal(
+                "admin",
+                "totalyinsecureadminpassword",
+                undefined,
+                true,
+            ).then((res) => res.accessToken);
+        }
+        return this.adminTokenPromise;
+    }
+
+    private async adminRequest<R extends {}>(verb: "GET", path: string, data?: never): Promise<R>;
+    private async adminRequest<R extends {}>(verb: Verb, path: string, data?: object): Promise<R>;
+    private async adminRequest<R extends {}>(verb: Verb, path: string, data?: object): Promise<R> {
+        const adminToken = await this.getAdminToken();
+        return this.adminApi.request(verb, path, adminToken, data);
+    }
+
    public registerUser(username: string, password: string, displayName?: string): Promise<Credentials> {
        return this.registerUserInternal(username, password, displayName, false);
    }

    public async loginUser(userId: string, password: string): Promise<Credentials> {
-        const url = `${this.baseUrl}/_matrix/client/v3/login`;
-        const res = await this.request.post(url, {
-            data: {
-                type: "m.login.password",
-                identifier: {
-                    type: "m.id.user",
-                    user: userId,
-                },
-                password: password,
-            },
-        });
-        const json = await res.json();
-
-        return {
-            password,
-            accessToken: json.access_token,
-            userId: json.user_id,
-            deviceId: json.device_id,
-            homeServer: json.home_server || json.user_id.split(":").slice(1).join(":"),
-            username: userId.slice(1).split(":")[0],
-        };
+        return this.csApi.loginUser(userId, password);
    }

    public async setThreepid(userId: string, medium: string, address: string): Promise<void> {
-        if (this.adminToken === undefined) {
-            const result = await this.registerUserInternal("admin", "totalyinsecureadminpassword", undefined, true);
-            this.adminToken = result.accessToken;
-        }
-
-        const url = `${this.baseUrl}/_synapse/admin/v2/users/${userId}`;
-        const res = await this.request.put(url, {
-            data: {
-                threepids: [
-                    {
-                        medium,
-                        address,
-                    },
-                ],
-            },
-            headers: {
-                Authorization: `Bearer ${this.adminToken}`,
-            },
+        await this.adminRequest("PUT", `/v2/users/${userId}`, {
+            threepids: [
+                {
+                    medium,
+                    address,
+                },
+            ],
        });
-
-        if (!res.ok()) {
-            throw await res.json();
-        }
+    }
+}
+
+export class StartedSynapseWithMasContainer extends StartedSynapseContainer {
+    constructor(
+        container: StartedTestContainer,
+        baseUrl: string,
+        registrationSharedSecret: string,
+        private readonly mas: StartedMatrixAuthenticationServiceContainer,
+    ) {
+        super(container, baseUrl, registrationSharedSecret);
+    }
+
+    protected async getAdminToken(): Promise<string> {
+        if (this.adminTokenPromise === undefined) {
+            this.adminTokenPromise = this.mas.getAdminToken();
+        }
+        return this.adminTokenPromise;
+    }
+
+    public registerUser(username: string, password: string, displayName?: string): Promise<Credentials> {
+        return this.mas.registerUser(username, password, displayName);
+    }
+
+    public async setThreepid(userId: string, medium: string, address: string): Promise<void> {
+        return this.mas.setThreepid(userId, medium, address);
    }
 }