Upload profile from Playwright runs

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

.eslintrc.js

@@ -271,6 +271,7 @@ module.exports = {
             },
             rules: {
                 "react-hooks/rules-of-hooks": ["off"],
+                "@typescript-eslint/no-floating-promises": ["error"],
             },
         },
         {

.github/CODEOWNERS

@@ -15,5 +15,5 @@
 /src/i18n/strings
 /src/i18n/strings/en_EN.json  @element-hq/element-web-reviewers
 # Ignore the synapse plugin as this is updated by GHA for docker image updating
-/playwright/plugins/homeserver/synapse/index.ts
+/playwright/testcontainers/synapse.ts
 

.github/labels.yml

@@ -210,6 +210,9 @@
 - name: "X-Upcoming-Release-Blocker"
   description: "This does not affect the current release cycle but will affect the next one"
   color: "e99695"
+- name: "X-Run-All-Tests"
+  description: "When applied to PRs, it'll run the full gamut of end-to-end tests on the PR"
+  color: "ff7979"
 - name: "Z-Actions"
   color: "ededed"
 - name: "Z-Cache-Confusion"
@@ -232,6 +235,15 @@
 - name: "Z-Flaky-Test"
   description: "A test is raising false alarms"
   color: "ededed"
+- name: "Z-Flaky-Test-Chrome"
+  description: "Flaky playwright test in Chrome"
+  color: "ededed"
+- name: "Z-Flaky-Test-Firefox"
+  description: "Flaky playwright test in Firefox"
+  color: "ededed"
+- name: "Z-Flaky-Test-Webkit"
+  description: "Flaky playwright test in Webkit"
+  color: "ededed"
 - name: "Z-Flaky-Jest-Test"
   description: "A Jest test is raising false alarms"
   color: "ededed"

.github/workflows/build.yml

@@ -27,10 +27,17 @@ jobs:
                     - macos-14
                 isDevelop:
                     - ${{ github.event_name == 'push' && github.ref_name == 'develop' }}
+                isPullRequest:
+                    - ${{ github.event_name == 'pull_request' }}
                 # Skip the ubuntu-24.04 build for the develop branch as the dedicated CD build_develop workflow handles that
+                # Skip the non-linux builds for pull requests as Windows is awfully slow, so run in merge queue only
                 exclude:
                     - isDevelop: true
                       image: ubuntu-24.04
+                    - isPullRequest: true
+                      image: windows-2022
+                    - isPullRequest: true
+                      image: macos-14
         runs-on: ${{ matrix.image }}
         defaults:
             run:

.github/workflows/dockerhub.yaml

@@ -24,7 +24,7 @@ jobs:
               uses: sigstore/cosign-installer@dc72c7d5c4d10cd6bcb8cf6e3fd625a9e5e537da # v3
 
             - name: Set up QEMU
-              uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3
+              uses: docker/setup-qemu-action@53851d14592bedcffcf25ea515637cff71ef929a # v3
 
             - name: Set up Docker Buildx
               uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3
@@ -51,7 +51,7 @@ jobs:
 
             - name: Build and push
               id: build-and-push
-              uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6
+              uses: docker/build-push-action@b32b51a8eda65d6793cd0494a773d4f6bcef32dc # v6
               with:
                   context: .
                   push: true

.github/workflows/end-to-end-tests.yaml

@@ -114,13 +114,13 @@ jobs:
                     - Chrome
                     - Firefox
                     - WebKit
-                isCron:
-                    - ${{ github.event_name == 'schedule' }}
-                # Skip the Firefox & Safari runs unless this was a cron trigger
+                runAllTests:
+                    - ${{ github.event_name == 'schedule' || contains(github.event.pull_request.labels.*.name, 'X-Run-All-Tests') }}
+                # Skip the Firefox & Safari runs unless this was a cron trigger or PR has X-Run-All-Tests label
                 exclude:
-                    - isCron: false
+                    - runAllTests: false
                       project: Firefox
-                    - isCron: false
+                    - runAllTests: false
                       project: WebKit
         steps:
             - uses: actions/checkout@v4
@@ -166,11 +166,21 @@ jobs:
 
             # We skip tests tagged with @mergequeue when running on PRs, but run them in MQ and everywhere else
             - name: Run Playwright tests
+              env:
+                  PWTEST_PROFILE_DIR: profile
               run: |
                   yarn playwright test \
                       --shard "${{ matrix.runner }}/${{ needs.build.outputs.num-runners }}" \
                       --project="${{ matrix.project }}" \
-                      ${{ github.event_name == 'pull_request' && '--grep-invert @mergequeue' || '' }}
+                      ${{ (github.event_name == 'pull_request' && matrix.runAllTests == false ) && '--grep-invert @mergequeue' || '' }}
+
+            - name: Upload profile
+              if: always()
+              uses: actions/upload-artifact@v4
+              with:
+                  name: profile-${{ matrix.project }}-${{ matrix.runner }}
+                  path: profile
+                  retention-days: 1
 
             - name: Upload blob report to GitHub Actions Artifacts
               if: always()

.github/workflows/playwright-image-updates.yaml

@@ -17,7 +17,7 @@ jobs:
                   docker pull "$IMAGE"
                   INSPECT=$(docker inspect --format='{{index .RepoDigests 0}}' "$IMAGE")
                   DIGEST=${INSPECT#*@}
-                  sed -i "s/const DOCKER_TAG.*/const DOCKER_TAG = \"develop@$DIGEST\";/" playwright/plugins/homeserver/synapse/index.ts
+                  sed -i "s/const TAG.*/const TAG = \"develop@$DIGEST\";/" playwright/testcontainers/synapse.ts
               env:
                   IMAGE: ghcr.io/element-hq/synapse:develop
 

.github/workflows/static_analysis.yaml

@@ -132,9 +132,3 @@ jobs:
 
             - name: Run linter
               run: "yarn run lint:knip"
-
-            - name: Install Deps
-              run: "scripts/layered.sh"
-
-            - name: Dead Code Analysis
-              run: "yarn run analyse:unused-exports"

CHANGELOG.md

@@ -1,3 +1,21 @@
+Changes in [1.11.90](https://github.com/element-hq/element-web/releases/tag/v1.11.90) (2025-01-14)
+==================================================================================================
+## ✨ Features
+
+* Docker: run as non-root ([#28849](https://github.com/element-hq/element-web/pull/28849)). Contributed by @richvdh.
+* Docker: allow configuration of HTTP listen port via env var ([#28840](https://github.com/element-hq/element-web/pull/28840)). Contributed by @richvdh.
+* Update matrix-wysiwyg to consume WASM asset ([#28838](https://github.com/element-hq/element-web/pull/28838)). Contributed by @t3chguy.
+* OIDC settings tweaks ([#28787](https://github.com/element-hq/element-web/pull/28787)). Contributed by @t3chguy.
+* Delabs native OIDC support ([#28615](https://github.com/element-hq/element-web/pull/28615)). Contributed by @t3chguy.
+* Move room header info button to right-most position ([#28754](https://github.com/element-hq/element-web/pull/28754)). Contributed by @t3chguy.
+* Enable key backup by default ([#28691](https://github.com/element-hq/element-web/pull/28691)). Contributed by @dbkr.
+
+## 🐛 Bug Fixes
+
+* Fix building the automations mermaid diagram ([#28881](https://github.com/element-hq/element-web/pull/28881)). Contributed by @dbkr.
+* Playwright: wait for the network listener on the postgres db ([#28808](https://github.com/element-hq/element-web/pull/28808)). Contributed by @dbkr.
+
+
 Changes in [1.11.89](https://github.com/element-hq/element-web/releases/tag/v1.11.89) (2024-12-18)
 ==================================================================================================
 This is a patch release to fix a bug which could prevent loading stored crypto state from storage, and also to fix URL previews when switching back to a room.

docs/playwright.md

@@ -23,21 +23,19 @@ element-web project is fine: leave it running it a different terminal as you wou
 when developing. Alternatively if you followed the development set up from element-web then
 Playwright will be capable of running the webserver on its own if it isn't already running.
 
-The tests use Docker to launch Homeserver (Synapse or Dendrite) instances to test against, so you'll also
-need to have Docker installed and working in order to run the Playwright tests.
+The tests use [testcontainers](https://node.testcontainers.org/) to launch Homeserver (Synapse or Dendrite)
+instances to test against, so you'll also need to one of the
+[supported container runtimes](#supporter-container-runtimes)
+installed and working in order to run the Playwright tests.
 
 There are a few different ways to run the tests yourself. The simplest is to run:
 
 ```shell
-docker pull ghcr.io/element-hq/synapse:develop
 yarn run test:playwright
 ```
 
 This will run the Playwright tests once, non-interactively.
 
-Note: you don't need to run the `docker pull` command every time, but you should
-do it regularly to ensure you are running against an up-to-date Synapse.
-
 You can also run individual tests this way too, as you'd expect:
 
 ```shell
@@ -61,29 +59,28 @@ Some tests are excluded from running on certain browsers due to incompatibilitie
 
 ## How the Tests Work
 
-Everything Playwright-related lives in the `playwright/` subdirectory of react-sdk
+Everything Playwright-related lives in the `playwright/` subdirectory
 as is typical for Playwright tests. Likewise, tests live in `playwright/e2e`.
 
-`playwright/plugins/homeservers` contains Playwright plugins that starts instances
-of Synapse/Dendrite in Docker containers. These servers are what Element-web runs
-against in the tests.
+`playwright/testcontainers` contains the testcontainers which start instances
+of Synapse/Dendrite. These servers are what Element-web runs against in the tests.
 
 Synapse can be launched with different configurations in order to test element
-in different configurations. `playwright/plugins/homeserver/synapse/templates`
-contains template configuration files for each different configuration.
+in different configurations. You can specify `synapseConfigOptions` as such:
 
-Each test suite can then launch whatever Synapse instances it needs in whatever
-configurations.
+```typescript
+test.use({
+    synapseConfigOptions: {
+        // The config options to pass to the Synapse instance
+    },
+});
+```
 
-Note that although tests should stop the Homeserver instances after running and the
-plugin also stop any remaining instances after all tests have run, it is possible
-to be left with some stray containers if, for example, you terminate a test such
-that the `after()` does not run and also exit Playwright uncleanly. All the containers
-it starts are prefixed, so they are easy to recognise. They can be removed safely.
-
-After each test run, logs from the Synapse instances are saved in `playwright/logs/synapse`
-with each instance in a separate directory named after its ID. These logs are removed
-at the start of each test run.
+The appropriate homeserver will be launched by the Playwright worker and reused for all tests which match the worker configuration.
+Due to homeservers being reused between tests, please use unique names for any rooms put into the room directory as
+they may be visible from other tests, the suggested approach is to use `testInfo.testId` within the name or lodash's uniqueId.
+We remove public rooms from the room directory between tests but deleting users doesn't have a homeserver agnostic solution.
+The logs from testcontainers will be attached to any reports output from Playwright.
 
 ## Writing Tests
 
@@ -113,25 +110,6 @@ Homeserver instances should be reasonably cheap to start (you may see the first
 while as it pulls the Docker image).
 You do not need to explicitly clean up the instance as it will be cleaned up by the fixture.
 
-### Synapse Config Templates
-
-When a Synapse instance is started, it's given a config generated from one of the config
-templates in `playwright/plugins/homeserver/synapse/templates`. There are a couple of special files
-in these templates:
-
-- `homeserver.yaml`:
-  Template substitution happens in this file. Template variables are:
-    - `REGISTRATION_SECRET`: The secret used to register users via the REST API.
-    - `MACAROON_SECRET_KEY`: Generated each time for security
-    - `FORM_SECRET`: Generated each time for security
-    - `PUBLIC_BASEURL`: The localhost url + port combination the synapse is accessible at
-- `localhost.signing.key`: A signing key is auto-generated and saved to this file.
-  Config templates should not contain a signing key and instead assume that one will exist
-  in this file.
-
-All other files in the template are copied recursively to `/data/`, so the file `foo.html`
-in a template can be referenced in the config as `/data/foo.html`.
-
 ### Logging In
 
 We again heavily leverage the magic of [Playwright fixtures](https://playwright.dev/docs/test-fixtures).
@@ -227,6 +205,8 @@ has to be disabled in Playwright on Firefox & Webkit to retain routing functiona
 Anything testing VoIP/microphone will need to have `@no-webkit` as fake microphone functionality is not available
 there at this time.
 
+If you wish to run all tests in a PR, you can give it the label `X-Run-All-Tests`.
+
 ## Supporter container runtimes
 
 We use testcontainers to spin up various instances of Synapse, Matrix Authentication Service, and more.

knip.ts

@@ -10,13 +10,13 @@ export default {
         "playwright/**",
         "test/**",
         "res/decoder-ring/**",
+        "res/jitsi_external_api.min.js",
+        "docs/**",
+        // Used by jest
+        "__mocks__/maplibre-gl.js",
     ],
     project: ["**/*.{js,ts,jsx,tsx}"],
     ignore: [
-        "docs/**",
-        "res/jitsi_external_api.min.js",
-        // Used by jest
-        "__mocks__/maplibre-gl.js",
         // Keep for now
         "src/hooks/useLocalStorageState.ts",
         "src/components/views/elements/InfoTooltip.tsx",
@@ -37,13 +37,8 @@ export default {
         // False positive
         "sw.js",
         // Used by webpack
-        "buffer",
         "process",
         "util",
-        // Used by workflows
-        "ts-prune",
-        // Required due to bug in bloom-filters https://github.com/Callidon/bloom-filters/issues/75
-        "@types/seedrandom",
     ],
     ignoreBinaries: [
         // Used in scripts & workflows

package.json

@@ -1,6 +1,6 @@
 {
     "name": "element-web",
-    "version": "1.11.89",
+    "version": "1.11.90",
     "description": "A feature-rich client for Matrix.org",
     "author": "New Vector Ltd.",
     "repository": {
@@ -66,7 +66,6 @@
         "test:playwright:screenshots:build": "docker build playwright -t element-web-playwright",
         "test:playwright:screenshots:run": "docker run --rm --network host -e BASE_URL -e CI -v $(pwd):/work/ -v $(node -e 'console.log(require(`path`).dirname(require.resolve(`matrix-js-sdk/package.json`)))'):/work/node_modules/matrix-js-sdk -v /var/run/docker.sock:/var/run/docker.sock -v /tmp/:/tmp/ -it element-web-playwright --grep @screenshot --project=Chrome",
         "coverage": "yarn test --coverage",
-        "analyse:unused-exports": "ts-node ./scripts/analyse_unused_exports.ts",
         "analyse:webpack-bundles": "webpack-bundle-analyzer webpack-stats.json webapp",
         "update:jitsi": "curl -s https://meet.element.io/libs/external_api.min.js > ./res/jitsi_external_api.min.js"
     },
@@ -90,7 +89,8 @@
         "@matrix-org/spec": "^1.7.0",
         "@sentry/browser": "^8.0.0",
         "@types/png-chunks-extract": "^1.0.2",
-        "@vector-im/compound-design-tokens": "^2.0.1",
+        "@types/react-virtualized": "^9.21.30",
+        "@vector-im/compound-design-tokens": "^2.1.0",
         "@vector-im/compound-web": "^7.5.0",
         "@vector-im/matrix-wysiwyg": "2.38.0",
         "@zxcvbn-ts/core": "^3.0.4",
@@ -144,6 +144,7 @@
         "react-dom": "^18.3.1",
         "react-focus-lock": "^2.5.1",
         "react-transition-group": "^4.4.1",
+        "react-virtualized": "^9.22.5",
         "rfc4648": "^1.4.0",
         "sanitize-filename": "^1.6.3",
         "sanitize-html": "2.14.0",
@@ -256,12 +257,12 @@
         "lint-staged": "^15.0.2",
         "mailhog": "^4.16.0",
         "matrix-web-i18n": "^3.2.1",
-        "mini-css-extract-plugin": "2.9.0",
+        "mini-css-extract-plugin": "2.9.2",
         "minimist": "^1.2.6",
         "modernizr": "^3.12.0",
         "node-fetch": "^2.6.7",
         "playwright-core": "^1.45.1",
-        "postcss": "8.4.38",
+        "postcss": "8.4.46",
         "postcss-easings": "^4.0.0",
         "postcss-hexrgba": "2.1.0",
         "postcss-import": "16.1.0",
@@ -285,7 +286,6 @@
         "terser-webpack-plugin": "^5.3.9",
         "testcontainers": "^10.16.0",
         "ts-node": "^10.9.1",
-        "ts-prune": "^0.10.3",
         "typescript": "5.7.2",
         "util": "^0.12.5",
         "web-streams-polyfill": "^4.0.0",

playwright/e2e/accessibility/keyboard-navigation.spec.ts

@@ -123,7 +123,7 @@ test.describe("Landmark navigation tests", () => {
         await expect(page.getByText("Bob joined the room")).toBeVisible();
 
         // Close the room
-        page.goto("/#/home");
+        await page.goto("/#/home");
 
         // Pressing Control+F6 will first focus the space button
         await page.keyboard.press("ControlOrMeta+F6");

playwright/e2e/app-loading/guest-registration.spec.ts

@@ -13,7 +13,7 @@ Please see LICENSE files in the repository root for full details.
 import { expect, test } from "../../element-web-test";
 
 test.use({
-    synapseConfigOptions: {
+    synapseConfig: {
         allow_guest_access: true,
     },
 });

playwright/e2e/chat-export/html-export.spec.ts

@@ -95,7 +95,7 @@ test.describe("HTML Export", () => {
         async ({ page, app, room }) => {
             // Set a fixed time rather than masking off the line with the time in it: we don't need to worry
             // about the width changing and we can actually test this line looks correct.
-            page.clock.setSystemTime(new Date("2024-01-01T00:00:00Z"));
+            await page.clock.setSystemTime(new Date("2024-01-01T00:00:00Z"));
 
             // Send a bunch of messages to populate the room
             for (let i = 1; i < 10; i++) {

playwright/e2e/composer/RTE.spec.ts

@@ -165,7 +165,7 @@ test.describe("Composer", () => {
             // Type another
             await page.locator("div[contenteditable=true]").pressSequentially("my message 1");
             // Send message
-            page.locator("div[contenteditable=true]").press("Enter");
+            await page.locator("div[contenteditable=true]").press("Enter");
             // It was sent
             await expect(page.locator(".mx_EventTile_last .mx_EventTile_body").getByText("my message 1")).toBeVisible();
         });

playwright/e2e/crypto/backups-mas.spec.ts

@@ -0,0 +1,95 @@
+/*
+Copyright 2024 New Vector Ltd.
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { test, expect } from "../../element-web-test";
+import { registerAccountMas } from "../oidc";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
+import { TestClientServerAPI } from "../csAPI";
+import { masHomeserver } from "../../plugins/homeserver/synapse/masHomeserver.ts";
+
+// These tests register an account with MAS because then we go through the "normal" registration flow
+// and crypto gets set up. Using the 'user' fixture create a user and synthesizes an existing login,
+// which is faster but leaves us without crypto set up.
+test.use(masHomeserver);
+test.describe("Encryption state after registration", () => {
+    test.skip(isDendrite, "does not yet support MAS");
+
+    test("Key backup is enabled by default", async ({ page, mailhogClient, app }, testInfo) => {
+        await page.goto("/#/login");
+        await page.getByRole("button", { name: "Continue" }).click();
+        await registerAccountMas(page, mailhogClient, `alice_${testInfo.testId}`, "alice@email.com", "Pa$sW0rD!");
+
+        await app.settings.openUserSettings("Security & Privacy");
+        await expect(page.getByText("This session is backing up your keys.")).toBeVisible();
+    });
+
+    test("user is prompted to set up recovery", async ({ page, mailhogClient, app }, testInfo) => {
+        await page.goto("/#/login");
+        await page.getByRole("button", { name: "Continue" }).click();
+        await registerAccountMas(page, mailhogClient, `alice_${testInfo.testId}`, "alice@email.com", "Pa$sW0rD!");
+
+        await page.getByRole("button", { name: "Add room" }).click();
+        await page.getByRole("menuitem", { name: "New room" }).click();
+        await page.getByRole("textbox", { name: "Name" }).fill("test room");
+        await page.getByRole("button", { name: "Create room" }).click();
+
+        await expect(page.getByRole("heading", { name: "Set up recovery" })).toBeVisible();
+    });
+});
+
+test.describe("Key backup reset from elsewhere", () => {
+    test.skip(isDendrite, "does not yet support MAS");
+
+    test("Key backup is disabled when reset from elsewhere", async ({
+        page,
+        mailhogClient,
+        request,
+        homeserver,
+    }, testInfo) => {
+        const testUsername = `alice_${testInfo.testId}`;
+        const testPassword = "Pa$sW0rD!";
+
+        // there's a delay before keys are uploaded so the error doesn't appear immediately: use a fake
+        // clock so we can skip the delay
+        await page.clock.install();
+
+        await page.goto("/#/login");
+        await page.getByRole("button", { name: "Continue" }).click();
+        await registerAccountMas(page, mailhogClient, testUsername, "alice@email.com", testPassword);
+
+        await page.getByRole("button", { name: "Add room" }).click();
+        await page.getByRole("menuitem", { name: "New room" }).click();
+        await page.getByRole("textbox", { name: "Name" }).fill("test room");
+        await page.getByRole("button", { name: "Create room" }).click();
+
+        const accessToken = await page.evaluate(() => window.mxMatrixClientPeg.get().getAccessToken());
+
+        const csAPI = new TestClientServerAPI(request, homeserver, accessToken);
+
+        const backupInfo = await csAPI.getCurrentBackupInfo();
+
+        await csAPI.deleteBackupVersion(backupInfo.version);
+
+        await page.getByRole("textbox", { name: "Send an encrypted message…" }).fill("/discardsession");
+        await page.getByRole("button", { name: "Send message" }).click();
+
+        await page.getByRole("textbox", { name: "Send an encrypted message…" }).fill("Message with broken key backup");
+        await page.getByRole("button", { name: "Send message" }).click();
+
+        // Should be the message we sent plus the room creation event
+        await expect(page.locator(".mx_EventTile")).toHaveCount(2);
+        await expect(
+            page.locator(".mx_RoomView_MessageList > .mx_EventTile_last .mx_EventTile_receiptSent"),
+        ).toBeVisible();
+
+        // Wait for it to try uploading the key
+        await page.clock.fastForward(20000);
+
+        await expect(page.getByRole("heading", { level: 1, name: "New Recovery Method" })).toBeVisible();
+    });
+});

playwright/e2e/crypto/backups.spec.ts

@@ -9,10 +9,6 @@ Please see LICENSE files in the repository root for full details.
 import { type Page } from "@playwright/test";
 
 import { test, expect } from "../../element-web-test";
-import { registerAccountMas } from "../oidc";
-import { isDendrite } from "../../plugins/homeserver/dendrite";
-import { TestClientServerAPI } from "../csAPI";
-import { masHomeserver } from "../../plugins/homeserver/synapse/masHomeserver.ts";
 
 async function expectBackupVersionToBe(page: Page, version: string) {
     await expect(page.locator(".mx_SecureBackupPanel_statusList tr:nth-child(5) td")).toHaveText(
@@ -22,85 +18,6 @@ async function expectBackupVersionToBe(page: Page, version: string) {
     await expect(page.locator(".mx_SecureBackupPanel_statusList tr:nth-child(6) td")).toHaveText(version);
 }
 
-// These tests register an account with MAS because then we go through the "normal" registration flow
-// and crypto gets set up. Using the 'user' fixture create a a user an synthesizes an existing login,
-// which is faster but leaves us without crypto set up.
-test.describe("Encryption state after registration", () => {
-    test.use(masHomeserver);
-    test.skip(isDendrite, "does not yet support MAS");
-
-    test("Key backup is enabled by default", async ({ page, mailhogClient, app }) => {
-        await page.goto("/#/login");
-        await page.getByRole("button", { name: "Continue" }).click();
-        await registerAccountMas(page, mailhogClient, "alice", "alice@email.com", "Pa$sW0rD!");
-
-        await app.settings.openUserSettings("Security & Privacy");
-        await expect(page.getByText("This session is backing up your keys.")).toBeVisible();
-    });
-
-    test("user is prompted to set up recovery", async ({ page, mailhogClient, app }) => {
-        await page.goto("/#/login");
-        await page.getByRole("button", { name: "Continue" }).click();
-        await registerAccountMas(page, mailhogClient, "alice", "alice@email.com", "Pa$sW0rD!");
-
-        await page.getByRole("button", { name: "Add room" }).click();
-        await page.getByRole("menuitem", { name: "New room" }).click();
-        await page.getByRole("textbox", { name: "Name" }).fill("test room");
-        await page.getByRole("button", { name: "Create room" }).click();
-
-        await expect(page.getByRole("heading", { name: "Set up recovery" })).toBeVisible();
-    });
-});
-
-test.describe("Key backup reset from elsewhere", () => {
-    test.use(masHomeserver);
-    test.skip(isDendrite, "does not yet support MAS");
-
-    test("Key backup is disabled when reset from elsewhere", async ({ page, mailhogClient, request, homeserver }) => {
-        const testUsername = "alice";
-        const testPassword = "Pa$sW0rD!";
-
-        // there's a delay before keys are uploaded so the error doesn't appear immediately: use a fake
-        // clock so we can skip the delay
-        await page.clock.install();
-
-        await page.goto("/#/login");
-        await page.getByRole("button", { name: "Continue" }).click();
-        await registerAccountMas(page, mailhogClient, testUsername, "alice@email.com", testPassword);
-
-        await page.getByRole("button", { name: "Add room" }).click();
-        await page.getByRole("menuitem", { name: "New room" }).click();
-        await page.getByRole("textbox", { name: "Name" }).fill("test room");
-        await page.getByRole("button", { name: "Create room" }).click();
-
-        // @ts-ignore - this runs in the browser scope where mxMatrixClientPeg is a thing. Here, it is not.
-        const accessToken = await page.evaluate(() => mxMatrixClientPeg.get().getAccessToken());
-
-        const csAPI = new TestClientServerAPI(request, homeserver, accessToken);
-
-        const backupInfo = await csAPI.getCurrentBackupInfo();
-
-        await csAPI.deleteBackupVersion(backupInfo.version);
-
-        await page.getByRole("textbox", { name: "Send an encrypted message…" }).fill("/discardsession");
-        await page.getByRole("button", { name: "Send message" }).click();
-
-        await page.getByRole("textbox", { name: "Send an encrypted message…" }).fill("Message with broken key backup");
-        await page.getByRole("button", { name: "Send message" }).click();
-
-        // Should be the message we sent plus the room creation event
-        await expect(page.locator(".mx_EventTile")).toHaveCount(2);
-        await expect(
-            page.locator(".mx_RoomView_MessageList > .mx_EventTile_last .mx_EventTile_receiptSent"),
-        ).toBeVisible();
-
-        // Wait for it to try uploading the key
-        await page.clock.fastForward(20000);
-
-        await expect(page.getByRole("heading", { level: 1, name: "New Recovery Method" })).toBeVisible();
-    });
-});
-
 test.describe("Backups", () => {
     test.use({
         displayName: "Hanako",

playwright/e2e/crypto/dehydration.spec.ts

@@ -16,34 +16,34 @@ const ROOM_NAME = "Test room";
 const NAME = "Alice";
 
 function getMemberTileByName(page: Page, name: string): Locator {
-    return page.locator(`.mx_EntityTile, [title="${name}"]`);
+    return page.locator(`.mx_MemberTileView, [title="${name}"]`);
 }
 
+test.use({
+    displayName: NAME,
+    synapseConfig: {
+        experimental_features: {
+            msc2697_enabled: false,
+            msc3814_enabled: true,
+        },
+    },
+    config: async ({ config, context }, use) => {
+        const wellKnown = {
+            ...config.default_server_config,
+            "org.matrix.msc3814": true,
+        };
+
+        await context.route("https://localhost/.well-known/matrix/client", async (route) => {
+            await route.fulfill({ json: wellKnown });
+        });
+
+        await use(config);
+    },
+});
+
 test.describe("Dehydration", () => {
     test.skip(isDendrite, "does not yet support dehydration v2");
 
-    test.use({
-        displayName: NAME,
-        synapseConfigOptions: {
-            experimental_features: {
-                msc2697_enabled: false,
-                msc3814_enabled: true,
-            },
-        },
-        config: async ({ config, context }, use) => {
-            const wellKnown = {
-                ...config.default_server_config,
-                "org.matrix.msc3814": true,
-            };
-
-            await context.route("https://localhost/.well-known/matrix/client", async (route) => {
-                await route.fulfill({ json: wellKnown });
-            });
-
-            await use(config);
-        },
-    });
-
     test("Create dehydrated device", async ({ page, user, app }, workerInfo) => {
         // Create a backup (which will create SSSS, and dehydrated device)
 
@@ -88,7 +88,7 @@ test.describe("Dehydration", () => {
         await viewRoomSummaryByName(page, app, ROOM_NAME);
 
         await page.locator(".mx_RightPanel").getByRole("menuitem", { name: "People" }).click();
-        await expect(page.locator(".mx_MemberList")).toBeVisible();
+        await expect(page.locator(".mx_MemberListView")).toBeVisible();
 
         await getMemberTileByName(page, NAME).click();
         await page.locator(".mx_UserInfo_devices .mx_UserInfo_expand").click();

playwright/e2e/crypto/device-verification.spec.ts

@@ -212,7 +212,7 @@ test.describe("Device verification", { tag: "@no-webkit" }, () => {
         /* on the bot side, wait for the verifier to exist ... */
         const verifier = await awaitVerifier(botVerificationRequest);
         // ... confirm ...
-        botVerificationRequest.evaluate((verificationRequest) => verificationRequest.verifier.verify());
+        void botVerificationRequest.evaluate((verificationRequest) => verificationRequest.verifier.verify());
         // ... and then check the emoji match
         await doTwoWaySasVerification(page, verifier);
 

playwright/e2e/crypto/invisible-crypto.spec.ts

@@ -11,6 +11,7 @@ import { bootstrapCrossSigningForClient } from "../../pages/client.ts";
 
 /** Tests for the "invisible crypto" behaviour -- i.e., when the "exclude insecure devices" setting is enabled */
 test.describe("Invisible cryptography", () => {
+    test.slow();
     test.use({
         displayName: "Alice",
         botCreateOpts: { displayName: "Bob" },

playwright/e2e/crypto/user-verification.spec.ts

@@ -74,7 +74,7 @@ test.describe("User verification", () => {
         /* on the bot side, wait for the verifier to exist ... */
         const botVerifier = await awaitVerifier(bobVerificationRequest);
         // ... confirm ...
-        botVerifier.evaluate((verifier) => verifier.verify());
+        void botVerifier.evaluate((verifier) => verifier.verify());
         // ... and then check the emoji match
         await doTwoWaySasVerification(page, botVerifier);
 

playwright/e2e/crypto/utils.ts

@@ -59,7 +59,7 @@ export function handleSasVerification(verifier: JSHandle<Verifier>): Promise<Emo
         return new Promise<EmojiMapping[]>((resolve) => {
             const onShowSas = (event: ShowSasCallbacks) => {
                 verifier.off("show_sas" as VerifierEvent, onShowSas);
-                event.confirm();
+                void event.confirm();
                 resolve(event.sas.emoji);
             };
 
@@ -313,7 +313,7 @@ export async function autoJoin(client: Client) {
     await client.evaluate((cli) => {
         cli.on(window.matrixcs.RoomMemberEvent.Membership, (event, member) => {
             if (member.membership === "invite" && member.userId === cli.getUserId()) {
-                cli.joinRoom(member.roomId);
+                void cli.joinRoom(member.roomId);
             }
         });
     });

playwright/e2e/csAPI.ts

@@ -9,24 +9,24 @@ import { APIRequestContext } from "playwright-core";
 import { KeyBackupInfo } from "matrix-js-sdk/src/crypto-api";
 
 import { HomeserverInstance } from "../plugins/homeserver";
+import { ClientServerApi } from "../plugins/utils/api.ts";
 
 /**
  * A small subset of the Client-Server API used to manipulate the state of the
  * account on the homeserver independently of the client under test.
  */
-export class TestClientServerAPI {
+export class TestClientServerAPI extends ClientServerApi {
     public constructor(
-        private request: APIRequestContext,
-        private homeserver: HomeserverInstance,
+        request: APIRequestContext,
+        homeserver: HomeserverInstance,
         private accessToken: string,
-    ) {}
+    ) {
+        super(homeserver.baseUrl);
+        this.setRequest(request);
+    }
 
     public async getCurrentBackupInfo(): Promise<KeyBackupInfo | null> {
-        const res = await this.request.get(`${this.homeserver.baseUrl}/_matrix/client/v3/room_keys/version`, {
-            headers: { Authorization: `Bearer ${this.accessToken}` },
-        });
-
-        return await res.json();
+        return this.request("GET", `/v3/room_keys/version`, this.accessToken);
     }
 
     /**
@@ -34,15 +34,6 @@ export class TestClientServerAPI {
      * @param version The version to delete
      */
     public async deleteBackupVersion(version: string): Promise<void> {
-        const res = await this.request.delete(
-            `${this.homeserver.baseUrl}/_matrix/client/v3/room_keys/version/${version}`,
-            {
-                headers: { Authorization: `Bearer ${this.accessToken}` },
-            },
-        );
-
-        if (!res.ok) {
-            throw new Error(`Failed to delete backup version: ${res.status}`);
-        }
+        await this.request("DELETE", `/v3/room_keys/version/${version}`, this.accessToken);
     }
 }

playwright/e2e/forgot-password/forgot-password.spec.ts

@@ -6,30 +6,40 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */
 
-import { expect, test } from "../../element-web-test";
+import { expect, test as base } from "../../element-web-test";
 import { selectHomeserver } from "../utils";
 import { emailHomeserver } from "../../plugins/homeserver/synapse/emailHomeserver.ts";
 import { isDendrite } from "../../plugins/homeserver/dendrite";
+import { Credentials } from "../../plugins/homeserver";
 
-const username = "user1234";
-// this has to be password-like enough to please zxcvbn. Needless to say it's just from pwgen.
-const password = "oETo7MPf0o";
 const email = "user@nowhere.dummy";
 
+const test = base.extend<{ credentials: Pick<Credentials, "username" | "password"> }>({
+    // eslint-disable-next-line no-empty-pattern
+    credentials: async ({}, use, testInfo) => {
+        await use({
+            username: `user_${testInfo.testId}`,
+            // this has to be password-like enough to please zxcvbn. Needless to say it's just from pwgen.
+            password: "oETo7MPf0o",
+        });
+    },
+});
+
+test.use(emailHomeserver);
+test.use({
+    config: {
+        // The only thing that we really *need* (otherwise Element refuses to load) is a default homeserver.
+        // We point that to a guaranteed-invalid domain.
+        default_server_config: {
+            "m.homeserver": {
+                base_url: "https://server.invalid",
+            },
+        },
+    },
+});
+
 test.describe("Forgot Password", () => {
     test.skip(isDendrite, "not yet wired up");
-    test.use(emailHomeserver);
-    test.use({
-        config: {
-            // The only thing that we really *need* (otherwise Element refuses to load) is a default homeserver.
-            // We point that to a guaranteed-invalid domain.
-            default_server_config: {
-                "m.homeserver": {
-                    base_url: "https://server.invalid",
-                },
-            },
-        },
-    });
 
     test("renders properly", { tag: "@screenshot" }, async ({ page, homeserver }) => {
         await page.goto("/");
@@ -44,31 +54,35 @@ test.describe("Forgot Password", () => {
         await expect(page.getByRole("main")).toMatchScreenshot("forgot-password.png");
     });
 
-    test("renders email verification dialog properly", { tag: "@screenshot" }, async ({ page, homeserver }) => {
-        const user = await homeserver.registerUser(username, password);
+    test(
+        "renders email verification dialog properly",
+        { tag: "@screenshot" },
+        async ({ page, homeserver, credentials }) => {
+            const user = await homeserver.registerUser(credentials.username, credentials.password);
 
-        await homeserver.setThreepid(user.userId, "email", email);
+            await homeserver.setThreepid(user.userId, "email", email);
 
-        await page.goto("/");
+            await page.goto("/");
 
-        await page.getByRole("link", { name: "Sign in" }).click();
-        await selectHomeserver(page, homeserver.baseUrl);
+            await page.getByRole("link", { name: "Sign in" }).click();
+            await selectHomeserver(page, homeserver.baseUrl);
 
-        await page.getByRole("button", { name: "Forgot password?" }).click();
+            await page.getByRole("button", { name: "Forgot password?" }).click();
 
-        await page.getByRole("textbox", { name: "Email address" }).fill(email);
+            await page.getByRole("textbox", { name: "Email address" }).fill(email);
 
-        await page.getByRole("button", { name: "Send email" }).click();
+            await page.getByRole("button", { name: "Send email" }).click();
 
-        await page.getByRole("button", { name: "Next" }).click();
+            await page.getByRole("button", { name: "Next" }).click();
 
-        await page.getByRole("textbox", { name: "New Password", exact: true }).fill(password);
-        await page.getByRole("textbox", { name: "Confirm new password", exact: true }).fill(password);
+            await page.getByRole("textbox", { name: "New Password", exact: true }).fill(credentials.password);
+            await page.getByRole("textbox", { name: "Confirm new password", exact: true }).fill(credentials.password);
 
-        await page.getByRole("button", { name: "Reset password" }).click();
+            await page.getByRole("button", { name: "Reset password" }).click();
 
-        await expect(page.getByRole("button", { name: "Resend" })).toBeInViewport();
+            await expect(page.getByRole("button", { name: "Resend" })).toBeInViewport();
 
-        await expect(page.locator(".mx_Dialog")).toMatchScreenshot("forgot-password-verify-email.png");
-    });
+            await expect(page.locator(".mx_Dialog")).toMatchScreenshot("forgot-password-verify-email.png");
+        },
+    );
 });

playwright/e2e/integration-manager/kick.spec.ts

@@ -69,29 +69,13 @@ async function sendActionFromIntegrationManager(
     await iframe.getByRole("button", { name: "Press to send action" }).click();
 }
 
-async function clickUntilGone(page: Page, selector: string, attempt = 0) {
-    if (attempt === 11) {
-        throw new Error("clickUntilGone attempt count exceeded");
-    }
-
-    await page.locator(selector).last().click();
-
-    const count = await page.locator(selector).count();
-    if (count > 0) {
-        return clickUntilGone(page, selector, ++attempt);
-    }
-}
-
 async function expectKickedMessage(page: Page, shouldExist: boolean) {
-    // Expand any event summaries, we can't use a click multiple here because clicking one might de-render others
-    // This is quite horrible but seems the most stable way of clicking 0-N buttons,
-    // one at a time with a full re-evaluation after each click
-    await clickUntilGone(page, ".mx_GenericEventListSummary_toggle[aria-expanded=false]");
-
-    // Check for the event message (or lack thereof)
-    await expect(page.getByText(`${USER_DISPLAY_NAME} removed ${BOT_DISPLAY_NAME}: ${KICK_REASON}`)).toBeVisible({
-        visible: shouldExist,
-    });
+    await expect(async () => {
+        await page.locator(".mx_GenericEventListSummary_toggle[aria-expanded=false]").last().click();
+        await expect(page.getByText(`${USER_DISPLAY_NAME} removed ${BOT_DISPLAY_NAME}: ${KICK_REASON}`)).toBeVisible({
+            visible: shouldExist,
+        });
+    }).toPass();
 }
 
 test.describe("Integration Manager: Kick", () => {

playwright/e2e/knock/manage-knocks.spec.ts

@@ -50,7 +50,7 @@ test.describe("Manage Knocks", () => {
     });
 
     test("should deny knock using bar", async ({ page, app, bot, room }) => {
-        bot.knockRoom(room.roomId);
+        await bot.knockRoom(room.roomId);
 
         const roomKnocksBar = page.locator(".mx_RoomKnocksBar");
         await expect(roomKnocksBar.getByRole("heading", { name: "Asking to join" })).toBeVisible();

playwright/e2e/lazy-loading/lazy-loading.spec.ts

@@ -78,7 +78,7 @@ test.describe("Lazy Loading", () => {
     }
 
     function getMemberInMemberlist(page: Page, name: string): Locator {
-        return page.locator(".mx_MemberList .mx_EntityTile_name").filter({ hasText: name });
+        return page.locator(".mx_MemberListView .mx_MemberTileView_name").filter({ hasText: name });
     }
 
     async function checkMemberList(page: Page, charlies: Bot[]) {

playwright/e2e/login/consent.spec.ts

@@ -9,12 +9,12 @@ Please see LICENSE files in the repository root for full details.
 import { test, expect } from "../../element-web-test";
 import { consentHomeserver } from "../../plugins/homeserver/synapse/consentHomeserver.ts";
 
-test.describe("Consent", () => {
-    test.use(consentHomeserver);
-    test.use({
-        displayName: "Bob",
-    });
+test.use(consentHomeserver);
+test.use({
+    displayName: "Bob",
+});
 
+test.describe("Consent", () => {
     test("should prompt the user to consent to terms when server deems it necessary", async ({
         context,
         page,

playwright/e2e/login/login-consent.spec.ts

@@ -9,13 +9,11 @@ Please see LICENSE files in the repository root for full details.
 import { Page } from "playwright-core";
 
 import { expect, test } from "../../element-web-test";
-import { doTokenRegistration } from "./utils";
-import { isDendrite } from "../../plugins/homeserver/dendrite";
 import { selectHomeserver } from "../utils";
 import { Credentials, HomeserverInstance } from "../../plugins/homeserver";
 import { consentHomeserver } from "../../plugins/homeserver/synapse/consentHomeserver.ts";
-import { legacyOAuthHomeserver } from "../../plugins/homeserver/synapse/legacyOAuthHomeserver.ts";
 
+// This test requires fixed credentials for the device signing keys below to work
 const username = "user1234";
 const password = "p4s5W0rD";
 
@@ -70,38 +68,53 @@ const DEVICE_SIGNING_KEYS_BODY = {
     },
 };
 
-async function login(page: Page, homeserver: HomeserverInstance) {
+async function login(page: Page, homeserver: HomeserverInstance, credentials: Credentials) {
     await page.getByRole("link", { name: "Sign in" }).click();
     await selectHomeserver(page, homeserver.baseUrl);
 
-    await page.getByRole("textbox", { name: "Username" }).fill(username);
-    await page.getByPlaceholder("Password").fill(password);
+    await page.getByRole("textbox", { name: "Username" }).fill(credentials.username);
+    await page.getByPlaceholder("Password").fill(credentials.password);
     await page.getByRole("button", { name: "Sign in" }).click();
 }
 
-test.describe("Login", () => {
-    test.use({
-        config: {
-            // The only thing that we really *need* (otherwise Element refuses to load) is a default homeserver.
-            // We point that to a guaranteed-invalid domain.
-            default_server_config: {
-                "m.homeserver": {
-                    base_url: "https://server.invalid",
-                },
+// This test suite uses the same userId for all tests in the suite
+// due to DEVICE_SIGNING_KEYS_BODY being specific to that userId,
+// so we restart the Synapse container to make it forget everything.
+test.use(consentHomeserver);
+test.use({
+    config: {
+        // The only thing that we really *need* (otherwise Element refuses to load) is a default homeserver.
+        // We point that to a guaranteed-invalid domain.
+        default_server_config: {
+            "m.homeserver": {
+                base_url: "https://server.invalid",
             },
         },
-    });
+    },
+    context: async ({ context, homeserver }, use) => {
+        // Restart the homeserver to wipe its in-memory db so we can reuse the same user ID without cross-signing prompts
+        await homeserver.restart();
+        await use(context);
+    },
+    credentials: async ({ context, homeserver }, use) => {
+        const displayName = "Dave";
+        const credentials = await homeserver.registerUser(username, password, displayName);
+        console.log(`Registered test user @user:localhost with displayname ${displayName}`);
 
-    test.describe("Password login", () => {
-        test.use(consentHomeserver);
-
-        let creds: Credentials;
-
-        test.beforeEach(async ({ homeserver }) => {
-            creds = await homeserver.registerUser(username, password);
+        await use({
+            ...credentials,
+            displayName,
         });
 
+        // Restart the homeserver to wipe its in-memory db so we can reuse the same user ID without cross-signing prompts
+        await homeserver.restart();
+    },
+});
+
+test.describe("Login", () => {
+    test.describe("Password login", () => {
         test("Loads the welcome page by default; then logs in with an existing account and lands on the home screen", async ({
+            credentials,
             page,
             homeserver,
             checkA11y,
@@ -135,16 +148,16 @@ test.describe("Login", () => {
             // cy.percySnapshot("Login");
             await checkA11y();
 
-            await page.getByRole("textbox", { name: "Username" }).fill(username);
-            await page.getByPlaceholder("Password").fill(password);
+            await page.getByRole("textbox", { name: "Username" }).fill(credentials.username);
+            await page.getByPlaceholder("Password").fill(credentials.password);
             await page.getByRole("button", { name: "Sign in" }).click();
 
             await expect(page).toHaveURL(/\/#\/home$/);
         });
 
-        test("Follows the original link after login", async ({ page, homeserver }) => {
+        test("Follows the original link after login", async ({ page, homeserver, credentials }) => {
             await page.goto("/#/room/!room:id"); // should redirect to the welcome page
-            await login(page, homeserver);
+            await login(page, homeserver, credentials);
 
             await expect(page).toHaveURL(/\/#\/room\/!room:id$/);
             await expect(page.getByRole("button", { name: "Join the discussion" })).toBeVisible();
@@ -155,9 +168,10 @@ test.describe("Login", () => {
                 page,
                 homeserver,
                 request,
+                credentials,
             }) => {
                 const res = await request.post(`${homeserver.baseUrl}/_matrix/client/v3/keys/device_signing/upload`, {
-                    headers: { Authorization: `Bearer ${creds.accessToken}` },
+                    headers: { Authorization: `Bearer ${credentials.accessToken}` },
                     data: DEVICE_SIGNING_KEYS_BODY,
                 });
                 if (res.status() / 100 !== 2) {
@@ -166,7 +180,7 @@ test.describe("Login", () => {
                 expect(res.status() / 100).toEqual(2);
 
                 await page.goto("/");
-                await login(page, homeserver);
+                await login(page, homeserver, credentials);
 
                 await expect(page.getByRole("heading", { name: "Verify this device", level: 1 })).toBeVisible();
 
@@ -184,10 +198,14 @@ test.describe("Login", () => {
                     page,
                     homeserver,
                     request,
+                    credentials,
                 }) => {
                     const res = await request.post(
                         `${homeserver.baseUrl}/_matrix/client/v3/keys/device_signing/upload`,
-                        { headers: { Authorization: `Bearer ${creds.accessToken}` }, data: DEVICE_SIGNING_KEYS_BODY },
+                        {
+                            headers: { Authorization: `Bearer ${credentials.accessToken}` },
+                            data: DEVICE_SIGNING_KEYS_BODY,
+                        },
                     );
                     if (res.status() / 100 !== 2) {
                         console.log("Uploading dummy keys failed", await res.json());
@@ -195,7 +213,7 @@ test.describe("Login", () => {
                     expect(res.status() / 100).toEqual(2);
 
                     await page.goto("/");
-                    await login(page, homeserver);
+                    await login(page, homeserver, credentials);
 
                     await expect(page.getByRole("heading", { name: "Verify this device", level: 1 })).toBeVisible();
 
@@ -214,11 +232,15 @@ test.describe("Login", () => {
                     page,
                     homeserver,
                     request,
+                    credentials,
                 }) => {
-                    console.log(`uid ${creds.userId} body`, DEVICE_SIGNING_KEYS_BODY);
+                    console.log(`uid ${credentials.userId} body`, DEVICE_SIGNING_KEYS_BODY);
                     const res = await request.post(
                         `${homeserver.baseUrl}/_matrix/client/v3/keys/device_signing/upload`,
-                        { headers: { Authorization: `Bearer ${creds.accessToken}` }, data: DEVICE_SIGNING_KEYS_BODY },
+                        {
+                            headers: { Authorization: `Bearer ${credentials.accessToken}` },
+                            data: DEVICE_SIGNING_KEYS_BODY,
+                        },
                     );
                     if (res.status() / 100 !== 2) {
                         console.log("Uploading dummy keys failed", await res.json());
@@ -226,9 +248,9 @@ test.describe("Login", () => {
                     expect(res.status() / 100).toEqual(2);
 
                     await page.goto("/");
-                    await login(page, homeserver);
+                    await login(page, homeserver, credentials);
 
-                    const h1 = await page.getByRole("heading", { name: "Verify this device", level: 1 });
+                    const h1 = page.getByRole("heading", { name: "Verify this device", level: 1 });
                     await expect(h1).toBeVisible();
 
                     await expect(h1.locator(".mx_CompleteSecurity_skip")).toHaveCount(0);
@@ -237,25 +259,7 @@ test.describe("Login", () => {
         });
     });
 
-    // tests for old-style SSO login, in which we exchange tokens with Synapse, and Synapse talks to an auth server
-    test.describe("SSO login", () => {
-        test.skip(isDendrite, "does not yet support SSO");
-        test.use(legacyOAuthHomeserver);
-
-        test("logs in with SSO and lands on the home screen", async ({ page, homeserver }) => {
-            // If this test fails with a screen showing "Timeout connecting to remote server", it is most likely due to
-            // your firewall settings: Synapse is unable to reach the OIDC server.
-            //
-            // If you are using ufw, try something like:
-            //    sudo ufw allow in on docker0
-            //
-            await doTokenRegistration(page, homeserver);
-        });
-    });
-
     test.describe("logout", () => {
-        test.use(consentHomeserver);
-
         test("should go to login page on logout", async ({ page, user }) => {
             await page.getByRole("button", { name: "User menu" }).click();
             await expect(page.getByText(user.displayName, { exact: true })).toBeVisible();
@@ -267,29 +271,4 @@ test.describe("Login", () => {
             await expect(page).toHaveURL(/\/#\/login$/);
         });
     });
-
-    test.describe("logout with logout_redirect_url", () => {
-        test.use(consentHomeserver);
-        test.use({
-            config: {
-                // We redirect to decoder-ring because it's a predictable page that isn't Element itself.
-                // We could use example.org, matrix.org, or something else, however this puts dependency of external
-                // infrastructure on our tests. In the same vein, we don't really want to figure out how to ship a
-                // `test-landing.html` page when running with an uncontrolled Element (via `yarn start`).
-                // Using the decoder-ring is just as fine, and we can search for strategic names.
-                logout_redirect_url: "/decoder-ring/",
-            },
-        });
-
-        test("should respect logout_redirect_url", async ({ page, user }) => {
-            await page.getByRole("button", { name: "User menu" }).click();
-            await expect(page.getByText(user.displayName, { exact: true })).toBeVisible();
-
-            // give a change for the outstanding requests queue to settle before logging out
-            await page.waitForTimeout(2000);
-
-            await page.locator(".mx_UserMenu_contextMenu").getByRole("menuitem", { name: "Sign out" }).click();
-            await expect(page).toHaveURL(/\/decoder-ring\/$/);
-        });
-    });
 });

playwright/e2e/login/login-sso.spec.ts

@@ -0,0 +1,29 @@
+/*
+Copyright 2024 New Vector Ltd.
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { test } from "../../element-web-test";
+import { doTokenRegistration } from "./utils";
+import { isDendrite } from "../../plugins/homeserver/dendrite";
+import { legacyOAuthHomeserver } from "../../plugins/homeserver/synapse/legacyOAuthHomeserver.ts";
+
+test.use(legacyOAuthHomeserver);
+
+// tests for old-style SSO login, in which we exchange tokens with Synapse, and Synapse talks to an auth server
+test.describe("SSO login", () => {
+    test.skip(isDendrite, "does not yet support SSO");
+
+    test("logs in with SSO and lands on the home screen", async ({ page, homeserver }, testInfo) => {
+        // If this test fails with a screen showing "Timeout connecting to remote server", it is most likely due to
+        // your firewall settings: Synapse is unable to reach the OIDC server.
+        //
+        // If you are using ufw, try something like:
+        //    sudo ufw allow in on docker0
+        //
+        await doTokenRegistration(page, homeserver, testInfo);
+    });
+});

playwright/e2e/login/logout_redirect_url.spec.ts

@@ -0,0 +1,35 @@
+/*
+Copyright 2024 New Vector Ltd.
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { expect, test } from "../../element-web-test";
+import { consentHomeserver } from "../../plugins/homeserver/synapse/consentHomeserver.ts";
+
+test.use(consentHomeserver);
+test.use({
+    config: {
+        // We redirect to decoder-ring because it's a predictable page that isn't Element itself.
+        // We could use example.org, matrix.org, or something else, however this puts dependency of external
+        // infrastructure on our tests. In the same vein, we don't really want to figure out how to ship a
+        // `test-landing.html` page when running with an uncontrolled Element (via `yarn start`).
+        // Using the decoder-ring is just as fine, and we can search for strategic names.
+        logout_redirect_url: "/decoder-ring/",
+    },
+});
+
+test.describe("logout with logout_redirect_url", () => {
+    test("should respect logout_redirect_url", async ({ page, user }) => {
+        await page.getByRole("button", { name: "User menu" }).click();
+        await expect(page.getByText(user.displayName, { exact: true })).toBeVisible();
+
+        // give a change for the outstanding requests queue to settle before logging out
+        await page.waitForTimeout(2000);
+
+        await page.locator(".mx_UserMenu_contextMenu").getByRole("menuitem", { name: "Sign out" }).click();
+        await expect(page).toHaveURL(/\/decoder-ring\/$/);
+    });
+});

playwright/e2e/login/soft_logout.spec.ts

@@ -6,122 +6,38 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */
 
-import { Page } from "@playwright/test";
-
 import { test, expect } from "../../element-web-test";
-import { doTokenRegistration } from "./utils";
-import { Credentials } from "../../plugins/homeserver";
-import { legacyOAuthHomeserver } from "../../plugins/homeserver/synapse/legacyOAuthHomeserver.ts";
+import { interceptRequestsWithSoftLogout } from "./utils";
 
-test.describe("Soft logout", () => {
-    test.use({
-        displayName: "Alice",
-        config: {
-            // The only thing that we really *need* (otherwise Element refuses to load) is a default homeserver.
-            // We point that to a guaranteed-invalid domain.
-            default_server_config: {
-                "m.homeserver": {
-                    base_url: "https://server.invalid",
-                },
+test.use({
+    displayName: "Alice",
+    config: {
+        // The only thing that we really *need* (otherwise Element refuses to load) is a default homeserver.
+        // We point that to a guaranteed-invalid domain.
+        default_server_config: {
+            "m.homeserver": {
+                base_url: "https://server.invalid",
             },
         },
-    });
-
-    test.describe("with password user", () => {
-        test("shows the soft-logout page when a request fails, and allows a re-login", async ({ page, user }) => {
-            await interceptRequestsWithSoftLogout(page, user);
-            await expect(page.getByText("You're signed out")).toBeVisible();
-            await page.getByPlaceholder("Password").fill(user.password);
-            await page.getByPlaceholder("Password").press("Enter");
-
-            // back to the welcome page
-            await expect(page).toHaveURL(/\/#\/home/);
-            await expect(
-                page.getByRole("heading", { name: "Now, let's help you get started", exact: true }),
-            ).toBeVisible();
-        });
-
-        test("still shows the soft-logout page when the page is reloaded after a soft-logout", async ({
-            page,
-            user,
-        }) => {
-            await interceptRequestsWithSoftLogout(page, user);
-            await expect(page.getByText("You're signed out")).toBeVisible();
-            await page.reload();
-            await expect(page.getByText("You're signed out")).toBeVisible();
-        });
-    });
-
-    test.describe("with SSO user", () => {
-        test.use(legacyOAuthHomeserver);
-        test.use({
-            user: async ({ page, homeserver }, use) => {
-                const user = await doTokenRegistration(page, homeserver);
-
-                // Eventually, we should end up at the home screen.
-                await expect(page).toHaveURL(/\/#\/home$/);
-                await expect(page.getByRole("heading", { name: "Welcome Alice", exact: true })).toBeVisible();
-
-                await use(user);
-            },
-        });
-
-        test("shows the soft-logout page when a request fails, and allows a re-login", async ({ page, user }) => {
-            await expect(page.getByRole("heading", { name: "Welcome Alice", exact: true })).toBeVisible();
-
-            await interceptRequestsWithSoftLogout(page, user);
-
-            await expect(page.getByText("You're signed out")).toBeVisible();
-            await page.getByRole("button", { name: "Continue with OAuth test" }).click();
-
-            // click the submit button
-            await page.getByRole("button", { name: "Submit" }).click();
-
-            // Synapse prompts us to grant permission to Element
-            await expect(page.getByRole("heading", { name: "Continue to your account" })).toBeVisible();
-            await page.getByRole("link", { name: "Continue" }).click();
-
-            // back to the welcome page
-            await expect(page).toHaveURL(/\/#\/home$/);
-            await expect(page.getByRole("heading", { name: "Welcome Alice", exact: true })).toBeVisible();
-        });
-    });
+    },
 });
 
-/**
- * Intercept calls to /sync and have them fail with a soft-logout
- *
- * Any further requests to /sync with the same access token are blocked.
- */
-async function interceptRequestsWithSoftLogout(page: Page, user: Credentials): Promise<void> {
-    await page.route("**/_matrix/client/*/sync*", async (route, req) => {
-        const accessToken = await req.headerValue("Authorization");
+test.describe("Soft logout with password user", () => {
+    test("shows the soft-logout page when a request fails, and allows a re-login", async ({ page, user }) => {
+        await interceptRequestsWithSoftLogout(page, user);
+        await expect(page.getByText("You're signed out")).toBeVisible();
+        await page.getByPlaceholder("Password").fill(user.password);
+        await page.getByPlaceholder("Password").press("Enter");
 
-        // now, if the access token on this request matches the expired one, block it
-        if (accessToken === `Bearer ${user.accessToken}`) {
-            console.log("Intercepting request with soft-logged-out access token");
-            await route.fulfill({
-                status: 401,
-                json: {
-                    errcode: "M_UNKNOWN_TOKEN",
-                    error: "Soft logout",
-                    soft_logout: true,
-                },
-            });
-            return;
-        }
-
-        // otherwise, pass through as normal
-        await route.continue();
+        // back to the welcome page
+        await expect(page).toHaveURL(/\/#\/home/);
+        await expect(page.getByRole("heading", { name: "Now, let's help you get started", exact: true })).toBeVisible();
     });
 
-    const promise = page.waitForResponse((resp) => resp.url().includes("/sync") && resp.status() === 401);
-
-    // do something to make the active /sync return: create a new room
-    await page.evaluate(() => {
-        // don't wait for this to complete: it probably won't, because of the broken sync
-        window.mxMatrixClientPeg.get().createRoom({});
+    test("still shows the soft-logout page when the page is reloaded after a soft-logout", async ({ page, user }) => {
+        await interceptRequestsWithSoftLogout(page, user);
+        await expect(page.getByText("You're signed out")).toBeVisible();
+        await page.reload();
+        await expect(page.getByText("You're signed out")).toBeVisible();
     });
-
-    await promise;
-}
+});

playwright/e2e/login/soft_logout_oauth.spec.ts

@@ -0,0 +1,59 @@
+/*
+Copyright 2024 New Vector Ltd.
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { test, expect } from "../../element-web-test";
+import { doTokenRegistration, interceptRequestsWithSoftLogout } from "./utils";
+import { legacyOAuthHomeserver } from "../../plugins/homeserver/synapse/legacyOAuthHomeserver.ts";
+
+test.use({
+    displayName: "Alice",
+    config: {
+        // The only thing that we really *need* (otherwise Element refuses to load) is a default homeserver.
+        // We point that to a guaranteed-invalid domain.
+        default_server_config: {
+            "m.homeserver": {
+                base_url: "https://server.invalid",
+            },
+        },
+    },
+});
+
+test.use(legacyOAuthHomeserver);
+test.describe("Soft logout with SSO user", () => {
+    test.use({
+        user: async ({ page, homeserver }, use, testInfo) => {
+            const user = await doTokenRegistration(page, homeserver, testInfo);
+
+            // Eventually, we should end up at the home screen.
+            await expect(page).toHaveURL(/\/#\/home$/);
+            await expect(page.getByRole("heading", { name: "Welcome Alice", exact: true })).toBeVisible();
+
+            await use(user);
+        },
+    });
+
+    test("shows the soft-logout page when a request fails, and allows a re-login", async ({ page, user }) => {
+        await expect(page.getByRole("heading", { name: "Welcome Alice", exact: true })).toBeVisible();
+
+        await interceptRequestsWithSoftLogout(page, user);
+
+        await expect(page.getByText("You're signed out")).toBeVisible();
+        await page.getByRole("button", { name: "Continue with OAuth test" }).click();
+
+        // click the submit button
+        await page.getByRole("button", { name: "Submit" }).click();
+
+        // Synapse prompts us to grant permission to Element
+        await expect(page.getByRole("heading", { name: "Continue to your account" })).toBeVisible();
+        await page.getByRole("link", { name: "Continue" }).click();
+
+        // back to the welcome page
+        await expect(page).toHaveURL(/\/#\/home$/);
+        await expect(page.getByRole("heading", { name: "Welcome Alice", exact: true })).toBeVisible();
+    });
+});

playwright/e2e/login/utils.ts

@@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */
 
-import { Page, expect } from "@playwright/test";
+import { Page, expect, TestInfo } from "@playwright/test";
 
 import { Credentials, HomeserverInstance } from "../../plugins/homeserver";
 
@@ -15,12 +15,13 @@ import { Credentials, HomeserverInstance } from "../../plugins/homeserver";
 export async function doTokenRegistration(
     page: Page,
     homeserver: HomeserverInstance,
+    testInfo: TestInfo,
 ): Promise<Credentials & { displayName: string }> {
     await page.goto("/#/login");
 
     await page.getByRole("button", { name: "Edit" }).click();
     await page.getByRole("textbox", { name: "Other homeserver" }).fill(homeserver.baseUrl);
-    await page.getByRole("button", { name: "Continue" }).click();
+    await page.getByRole("button", { name: "Continue", exact: true }).click();
     // wait for the dialog to go away
     await expect(page.locator(".mx_ServerPickerDialog")).toHaveCount(0);
 
@@ -35,7 +36,7 @@ export async function doTokenRegistration(
 
     // Synapse prompts us to pick a user ID
     await expect(page.getByRole("heading", { name: "Create your account" })).toBeVisible();
-    await page.getByRole("textbox", { name: "Username (required)" }).fill("alice");
+    await page.getByRole("textbox", { name: "Username (required)" }).fill(`alice_${testInfo.testId}`);
 
     // wait for username validation to start, and complete
     await expect(page.locator("#field-username-output")).toHaveText("");
@@ -56,5 +57,44 @@ export async function doTokenRegistration(
         homeServer: window.mxMatrixClientPeg.get().getHomeserverUrl(),
         password: null,
         displayName: "Alice",
+        username: window.mxMatrixClientPeg.get().getUserIdLocalpart(),
     }));
 }
+
+/**
+ * Intercept calls to /sync and have them fail with a soft-logout
+ *
+ * Any further requests to /sync with the same access token are blocked.
+ */
+export async function interceptRequestsWithSoftLogout(page: Page, user: Credentials): Promise<void> {
+    await page.route("**/_matrix/client/*/sync*", async (route, req) => {
+        const accessToken = await req.headerValue("Authorization");
+
+        // now, if the access token on this request matches the expired one, block it
+        if (accessToken === `Bearer ${user.accessToken}`) {
+            console.log("Intercepting request with soft-logged-out access token");
+            await route.fulfill({
+                status: 401,
+                json: {
+                    errcode: "M_UNKNOWN_TOKEN",
+                    error: "Soft logout",
+                    soft_logout: true,
+                },
+            });
+            return;
+        }
+
+        // otherwise, pass through as normal
+        await route.continue();
+    });
+
+    const promise = page.waitForResponse((resp) => resp.url().includes("/sync") && resp.status() === 401);
+
+    // do something to make the active /sync return: create a new room
+    await page.evaluate(() => {
+        // don't wait for this to complete: it probably won't, because of the broken sync
+        void window.mxMatrixClientPeg.get().createRoom({});
+    });
+
+    await promise;
+}

playwright/e2e/oidc/index.ts

@@ -33,7 +33,7 @@ export async function registerAccountMas(
         expect(messages.items).toHaveLength(1);
     }).toPass();
     expect(messages.items[0].to).toEqual(`${username} <${email}>`);
-    const [code] = messages.items[0].text.match(/(\d{6})/);
+    const [, code] = messages.items[0].text.match(/Your verification code to confirm this email address is: (\d{6})/);
 
     await page.getByRole("textbox", { name: "6-digit code" }).fill(code);
     await page.getByRole("button", { name: "Continue" }).click();

playwright/e2e/oidc/oidc-native.spec.ts

@@ -12,12 +12,20 @@ import { ElementAppPage } from "../../pages/ElementAppPage.ts";
 import { isDendrite } from "../../plugins/homeserver/dendrite";
 import { masHomeserver } from "../../plugins/homeserver/synapse/masHomeserver.ts";
 
+test.use(masHomeserver);
 test.describe("OIDC Native", { tag: ["@no-firefox", "@no-webkit"] }, () => {
-    test.use(masHomeserver);
     test.skip(isDendrite, "does not yet support MAS");
     test.slow(); // trace recording takes a while here
 
-    test("can register the oauth2 client and an account", async ({ context, page, homeserver, mailhogClient, mas }) => {
+    test("can register the oauth2 client and an account", async ({
+        context,
+        page,
+        homeserver,
+        mailhogClient,
+        mas,
+    }, testInfo) => {
+        await page.clock.install();
+
         const tokenUri = `${mas.baseUrl}/oauth2/token`;
         const tokenApiPromise = page.waitForRequest(
             (request) => request.url() === tokenUri && request.postDataJSON()["grant_type"] === "authorization_code",
@@ -25,11 +33,14 @@ test.describe("OIDC Native", { tag: ["@no-firefox", "@no-webkit"] }, () => {
 
         await page.goto("/#/login");
         await page.getByRole("button", { name: "Continue" }).click();
-        await registerAccountMas(page, mailhogClient, "alice", "alice@email.com", "Pa$sW0rD!");
+
+        const userId = `alice_${testInfo.testId}`;
+        await registerAccountMas(page, mailhogClient, userId, "alice@email.com", "Pa$sW0rD!");
 
         // Eventually, we should end up at the home screen.
         await expect(page).toHaveURL(/\/#\/home$/, { timeout: 10000 });
-        await expect(page.getByRole("heading", { name: "Welcome alice", exact: true })).toBeVisible();
+        await expect(page.getByRole("heading", { name: `Welcome ${userId}`, exact: true })).toBeVisible();
+        await page.clock.runFor(20000); // run the timer so we see the token request
 
         const tokenApiRequest = await tokenApiPromise;
         expect(tokenApiRequest.postDataJSON()["grant_type"]).toBe("authorization_code");

playwright/e2e/one-to-one-chat/one-to-one-chat.spec.ts

@@ -17,8 +17,8 @@ const test = base.extend<{
 test.describe("1:1 chat room", () => {
     test.use({
         displayName: "Jeff",
-        user2: async ({ homeserver }, use) => {
-            const credentials = await homeserver.registerUser("user1234", "p4s5W0rD", "Timmy");
+        user2: async ({ homeserver }, use, testInfo) => {
+            const credentials = await homeserver.registerUser(`user2_${testInfo.testId}`, "p4s5W0rD", "Timmy");
             await use(credentials);
         },
     });

playwright/e2e/polls/pollHistory.spec.ts

@@ -134,7 +134,7 @@ test.describe("Poll history", () => {
 
         await expect(dialog.getByText(pollParams2.title)).toBeAttached();
         await expect(dialog.getByText(pollParams1.title)).toBeAttached();
-        dialog.getByText("Active polls").click();
+        await dialog.getByText("Active polls").click();
 
         // no more active polls
         await expect(page.getByText("There are no active polls in this room")).toBeAttached();

playwright/e2e/read-receipts/reactions-in-threads.spec.ts

@@ -70,11 +70,7 @@ test.describe("Read receipts", { tag: "@mergequeue" }, () => {
                 // Given a thread exists and I have marked it as read
                 await util.goTo(room1);
                 await util.assertRead(room2);
-                await util.receiveMessages(room2, [
-                    "Msg1",
-                    msg.threadedOff("Msg1", "Reply1"),
-                    msg.reactionTo("Reply1", "🪿"),
-                ]);
+                await util.receiveMessages(room2, ["Msg1", msg.threadedOff("Msg1", "Reply1")]);
                 await util.assertUnread(room2, 1);
                 await util.markAsRead(room2);
                 await util.assertRead(room2);

playwright/e2e/read-receipts/read-receipts.spec.ts

@@ -100,12 +100,7 @@ test.describe("Read receipts", { tag: "@mergequeue" }, () => {
         await page.goto(`/#/room/${selectedRoomId}`);
     });
 
-    // Disabled due to flakiness: https://github.com/element-hq/element-web/issues/26895
-    test.skip("With sync accumulator, considers main thread and unthreaded receipts #24629", async ({
-        page,
-        app,
-        bot,
-    }) => {
+    test("With sync accumulator, considers main thread and unthreaded receipts #24629", async ({ page, app, bot }) => {
         // Details are in https://github.com/vector-im/element-web/issues/24629
         // This proves we've fixed one of the "stuck unreads" issues.
 

playwright/e2e/register/email.spec.ts

@@ -10,28 +10,29 @@ import { test, expect } from "../../element-web-test";
 import { emailHomeserver } from "../../plugins/homeserver/synapse/emailHomeserver.ts";
 import { isDendrite } from "../../plugins/homeserver/dendrite";
 
+test.use(emailHomeserver);
+test.use({
+    config: ({ config }, use) =>
+        use({
+            ...config,
+            default_server_config: {
+                ...config.default_server_config,
+                "m.identity_server": {
+                    base_url: "https://server.invalid",
+                },
+            },
+        }),
+});
+
 test.describe("Email Registration", async () => {
     test.skip(isDendrite, "not yet wired up");
-    test.use(emailHomeserver);
-    test.use({
-        config: ({ config }, use) =>
-            use({
-                ...config,
-                default_server_config: {
-                    ...config.default_server_config,
-                    "m.identity_server": {
-                        base_url: "https://server.invalid",
-                    },
-                },
-            }),
-    });
 
     test.beforeEach(async ({ homeserver, page }) => {
         await page.goto("/#/register");
     });
 
     test(
-        "registers an account and lands on the use case selection screen",
+        "registers an account and lands on the home page",
         { tag: "@screenshot" },
         async ({ page, mailhogClient, request, checkA11y }) => {
             await expect(page.getByRole("textbox", { name: "Username" })).toBeVisible();
@@ -56,7 +57,7 @@ test.describe("Email Registration", async () => {
             const [emailLink] = messages.items[0].text.match(/http.+/);
             await request.get(emailLink); // "Click" the link in the email
 
-            await expect(page.locator(".mx_UseCaseSelection_skip")).toBeVisible();
+            await expect(page.getByText("Welcome alice")).toBeVisible();
         },
     );
 });

playwright/e2e/register/register.spec.ts

@@ -9,20 +9,20 @@ Please see LICENSE files in the repository root for full details.
 import { test, expect } from "../../element-web-test";
 import { consentHomeserver } from "../../plugins/homeserver/synapse/consentHomeserver.ts";
 
-test.describe("Registration", () => {
-    test.use(consentHomeserver);
-    test.use({
-        config: {
-            // The only thing that we really *need* (otherwise Element refuses to load) is a default homeserver.
-            // We point that to a guaranteed-invalid domain.
-            default_server_config: {
-                "m.homeserver": {
-                    base_url: "https://server.invalid",
-                },
+test.use(consentHomeserver);
+test.use({
+    config: {
+        // The only thing that we really *need* (otherwise Element refuses to load) is a default homeserver.
+        // We point that to a guaranteed-invalid domain.
+        default_server_config: {
+            "m.homeserver": {
+                base_url: "https://server.invalid",
             },
         },
-    });
+    },
+});
 
+test.describe("Registration", () => {
     test.beforeEach(async ({ page }) => {
         await page.goto("/#/register");
     });
@@ -71,12 +71,6 @@ test.describe("Registration", () => {
             await expect(termsPolicy.getByLabel("Privacy Policy")).toBeVisible();
 
             await page.getByRole("button", { name: "Accept", exact: true }).click();
-
-            await expect(page.locator(".mx_UseCaseSelection_skip")).toBeVisible();
-            await expect(page).toMatchScreenshot("use-case-selection.png", screenshotOptions);
-            await checkA11y();
-            await page.getByRole("button", { name: "Skip", exact: true }).click();
-
             await expect(page).toHaveURL(/\/#\/home$/);
 
             /*

playwright/e2e/right-panel/memberlist.spec.ts

@@ -0,0 +1,48 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { test, expect } from "../../element-web-test";
+import { Bot } from "../../pages/bot";
+
+const ROOM_NAME = "Test room";
+const NAME = "Alice";
+
+test.use({
+    synapseConfig: {
+        presence: {
+            enabled: false,
+            include_offline_users_on_sync: false,
+        },
+    },
+    displayName: NAME,
+    disablePresence: true,
+});
+
+test.describe("Memberlist", () => {
+    test.beforeEach(async ({ app, user, page, homeserver }, testInfo) => {
+        testInfo.setTimeout(testInfo.timeout + 30_000);
+        const id = await app.client.createRoom({ name: ROOM_NAME });
+        const newBots: Bot[] = [];
+        const names = ["Bob", "Bob", "Susan"];
+        for (let i = 0; i < 3; i++) {
+            const displayName = names[i];
+            const autoAcceptInvites = displayName !== "Susan";
+            const bot = new Bot(page, homeserver, { displayName, startClient: true, autoAcceptInvites });
+            await bot.prepareClient();
+            await app.client.inviteUser(id, bot.credentials?.userId);
+            newBots.push(bot);
+        }
+    });
+
+    test("Renders correctly", { tag: "@screenshot" }, async ({ page, app }) => {
+        await app.viewRoomByName(ROOM_NAME);
+        const memberlist = await app.toggleMemberlistPanel();
+        await expect(memberlist.locator(".mx_MemberTileView")).toHaveCount(4);
+        await expect(memberlist.getByText("(Invited)")).toHaveCount(1);
+        await expect(page.locator(".mx_MemberListView")).toMatchScreenshot("with-four-members.png");
+    });
+});

playwright/e2e/right-panel/right-panel.spec.ts

@@ -24,7 +24,7 @@ const ROOM_ADDRESS_LONG =
     "loremIpsumDolorSitAmetConsecteturAdipisicingElitSedDoEiusmodTemporIncididuntUtLaboreEtDoloreMagnaAliqua";
 
 function getMemberTileByName(page: Page, name: string): Locator {
-    return page.locator(`.mx_EntityTile, [title="${name}"]`);
+    return page.locator(`.mx_MemberTileView, [title="${name}"]`);
 }
 
 test.describe("RightPanel", () => {
@@ -47,6 +47,7 @@ test.describe("RightPanel", () => {
             // Set a local room address
             const localAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Local Addresses" });
             await localAddresses.getByRole("textbox").fill(ROOM_ADDRESS_LONG);
+            await expect(page.getByText("This address is available to use")).toBeVisible();
             await localAddresses.getByRole("button", { name: "Add" }).click();
             await expect(localAddresses.getByText(`#${ROOM_ADDRESS_LONG}:localhost`)).toHaveClass(
                 "mx_EditableItem_item",
@@ -107,14 +108,14 @@ test.describe("RightPanel", () => {
             await viewRoomSummaryByName(page, app, ROOM_NAME);
 
             await page.locator(".mx_RightPanel").getByRole("menuitem", { name: "People" }).click();
-            await expect(page.locator(".mx_MemberList")).toBeVisible();
+            await expect(page.locator(".mx_MemberListView")).toBeVisible();
 
             await getMemberTileByName(page, NAME).click();
             await expect(page.locator(".mx_UserInfo")).toBeVisible();
             await expect(page.locator(".mx_UserInfo_profile").getByText(NAME)).toBeVisible();
 
             await page.getByTestId("base-card-back-button").click();
-            await expect(page.locator(".mx_MemberList")).toBeVisible();
+            await expect(page.locator(".mx_MemberListView")).toBeVisible();
 
             await page.getByLabel("Room info").nth(1).click();
             await checkRoomSummaryCard(page, ROOM_NAME);
@@ -130,14 +131,14 @@ test.describe("RightPanel", () => {
                 .locator(".mx_RoomInfoLine_private")
                 .getByRole("button", { name: /\d member/ })
                 .click();
-            await expect(page.locator(".mx_MemberList")).toBeVisible();
+            await expect(page.locator(".mx_MemberListView")).toBeVisible();
 
             await getMemberTileByName(page, NAME).click();
             await expect(page.locator(".mx_UserInfo")).toBeVisible();
             await expect(page.locator(".mx_UserInfo_profile").getByText(NAME)).toBeVisible();
 
             await page.getByTestId("base-card-back-button").click();
-            await expect(page.locator(".mx_MemberList")).toBeVisible();
+            await expect(page.locator(".mx_MemberListView")).toBeVisible();
         });
     });
 });

playwright/e2e/room-directory/room-directory.spec.ts

@@ -30,6 +30,7 @@ test.describe("Room Directory", () => {
             // First add a local address `gaming`
             const localAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Local Addresses" });
             await localAddresses.getByRole("textbox").fill("gaming");
+            await expect(page.getByText("This address is available to use")).toBeVisible();
             await localAddresses.getByRole("button", { name: "Add" }).click();
             await expect(localAddresses.getByText("#gaming:localhost")).toHaveClass("mx_EditableItem_item");
 

playwright/e2e/room_options/marked_unread.spec.ts

@@ -48,6 +48,6 @@ test.describe("Mark as Unread", () => {
         await roomTile.getByRole("button", { name: "Room options" }).click();
         await page.getByRole("menuitem", { name: "Mark as unread" }).click();
 
-        expect(page.getByLabel(TEST_ROOM_NAME + " Unread messages.")).toBeVisible();
+        await expect(page.getByLabel(TEST_ROOM_NAME + " Unread messages.")).toBeVisible();
     });
 });

playwright/e2e/settings/account-user-settings-tab.spec.ts

@@ -34,14 +34,14 @@ test.describe("Account user settings tab", () => {
         await expect(profile.getByRole("textbox", { name: "Display Name" })).toHaveValue(USER_NAME);
 
         // Assert that a userId is rendered
-        expect(uut.getByLabel("Username")).toHaveText(user.userId);
+        await expect(uut.getByLabel("Username")).toHaveText(user.userId);
 
         // Wait until spinners disappear
         await expect(uut.getByTestId("accountSection").locator(".mx_Spinner")).not.toBeVisible();
         await expect(uut.getByTestId("discoverySection").locator(".mx_Spinner")).not.toBeVisible();
 
         const accountSection = uut.getByTestId("accountSection");
-        accountSection.scrollIntoViewIfNeeded();
+        await accountSection.scrollIntoViewIfNeeded();
         // Assert that input areas for changing a password exists
         await expect(accountSection.getByLabel("Current password")).toBeVisible();
         await expect(accountSection.getByLabel("New Password")).toBeVisible();

playwright/e2e/settings/device-management.spec.ts

@@ -7,7 +7,10 @@ Please see LICENSE files in the repository root for full details.
 */
 
 import { test, expect } from "../../element-web-test";
+import { uiaLongSessionTimeoutHomeserver } from "../../plugins/homeserver/synapse/uiaLongSessionTimeoutHomeserver.ts";
 
+// This is needed to not get stopped by UIA when deleting other devices
+test.use(uiaLongSessionTimeoutHomeserver);
 test.describe("Device manager", () => {
     test.use({
         displayName: "Alice",

playwright/e2e/settings/general-room-settings-tab.spec.ts

@@ -41,6 +41,7 @@ test.describe("General room settings tab", () => {
         // 1. Set the room-address to be a really long string
         const longString = "abcasdhjasjhdaj1jh1asdhasjdhajsdhjavhjksd".repeat(4);
         await settings.locator("#roomAliases input[label='Room address']").fill(longString);
+        await expect(page.getByText("This address is available to use")).toBeVisible();
         await settings.locator("#roomAliases").getByText("Add", { exact: true }).click();
 
         // 2. wait for the new setting to apply ...

playwright/e2e/settings/preferences-user-settings-tab.spec.ts

@@ -24,7 +24,7 @@ test.describe("Preferences user settings tab", () => {
     });
 
     test("should be rendered properly", { tag: "@screenshot" }, async ({ app, page, user }) => {
-        page.setViewportSize({ width: 1024, height: 3300 });
+        await page.setViewportSize({ width: 1024, height: 3300 });
         const tab = await app.settings.openUserSettings("Preferences");
         // Assert that the top heading is rendered
         await expect(tab.getByRole("heading", { name: "Preferences" })).toBeVisible();
@@ -61,7 +61,7 @@ test.describe("Preferences user settings tab", () => {
         // Click the button to display the dropdown menu
         await timezoneInput.getByRole("button", { name: "Set timezone" }).click();
         // Select a different value
-        timezoneInput.getByRole("option", { name: /Africa\/Abidjan/ }).click();
+        await timezoneInput.getByRole("option", { name: /Africa\/Abidjan/ }).click();
         // Check the new value
         await expect(timezoneValue.getByText("Africa/Abidjan")).toBeVisible();
     });

playwright/e2e/share-dialog/share-dialog.spec.ts

@@ -23,7 +23,7 @@ test.describe("Share dialog", () => {
 
         const dialog = page.getByRole("dialog", { name: "Share room" });
         await expect(dialog.getByText(`https://matrix.to/#/${room.roomId}`)).toBeVisible();
-        expect(dialog).toMatchScreenshot("share-dialog-room.png", {
+        await expect(dialog).toMatchScreenshot("share-dialog-room.png", {
             // QRCode and url changes at every run
             mask: [page.locator(".mx_QRCode"), page.locator(".mx_ShareDialog_top > span")],
         });
@@ -40,7 +40,7 @@ test.describe("Share dialog", () => {
 
         const dialog = page.getByRole("dialog", { name: "Share User" });
         await expect(dialog.getByText(`https://matrix.to/#/${user.userId}`)).toBeVisible();
-        expect(dialog).toMatchScreenshot("share-dialog-user.png", {
+        await expect(dialog).toMatchScreenshot("share-dialog-user.png", {
             // QRCode changes at every run
             mask: [page.locator(".mx_QRCode")],
         });
@@ -57,7 +57,7 @@ test.describe("Share dialog", () => {
 
         const dialog = page.getByRole("dialog", { name: "Share Room Message" });
         await expect(dialog.getByRole("checkbox", { name: "Link to selected message" })).toBeChecked();
-        expect(dialog).toMatchScreenshot("share-dialog-event.png", {
+        await expect(dialog).toMatchScreenshot("share-dialog-event.png", {
             // QRCode and url changes at every run
             mask: [page.locator(".mx_QRCode"), page.locator(".mx_ShareDialog_top > span")],
         });

playwright/e2e/sliding-sync/sliding-sync.spec.ts

@@ -108,7 +108,6 @@ test.describe("Sliding Sync", () => {
         await page.getByRole("menuitemradio", { name: "A-Z" }).dispatchEvent("click");
         await expect(page.locator(".mx_StyledRadioButton_checked").getByText("A-Z")).toBeVisible();
 
-        await page.pause();
         await checkOrder(["Apple", "Orange", "Pineapple", "Test Room"], page);
     });
 
@@ -276,7 +275,7 @@ test.describe("Sliding Sync", () => {
         // now rescind the invite
         await bot.evaluate(
             async (client, { roomRescind, clientUserId }) => {
-                client.kick(roomRescind, clientUserId);
+                await client.kick(roomRescind, clientUserId);
             },
             { roomRescind, clientUserId },
         );
@@ -295,7 +294,7 @@ test.describe("Sliding Sync", () => {
             is_direct: true,
         });
         await app.client.evaluate(async (client, roomId) => {
-            client.setRoomTag(roomId, "m.favourite", { order: 0.5 });
+            await client.setRoomTag(roomId, "m.favourite", { order: 0.5 });
         }, roomId);
         await expect(page.getByRole("group", { name: "Favourites" }).getByText("Favourite DM")).toBeVisible();
         await expect(page.getByRole("group", { name: "People" }).getByText("Favourite DM")).not.toBeAttached();
@@ -371,37 +370,42 @@ test.describe("Sliding Sync", () => {
             roomIds.push(id);
             await expect(page.getByRole("treeitem", { name: fruit })).toBeVisible();
         }
-        const [roomAId, roomPId] = roomIds;
+        const [roomAId, roomPId, roomOId] = roomIds;
 
-        const assertUnsubExists = (request: Request, subRoomId: string, unsubRoomId: string) => {
+        const matchRoomSubRequest = (subRoomId: string) => (request: Request) => {
+            if (!request.url().includes("/sync")) return false;
             const body = request.postDataJSON();
-            // There may be a request without a txn_id, ignore it, as there won't be any subscription changes
-            if (body.txn_id === undefined) {
-                return;
-            }
-            expect(body.unsubscribe_rooms).toEqual([unsubRoomId]);
-            expect(body.room_subscriptions).not.toHaveProperty(unsubRoomId);
-            expect(body.room_subscriptions).toHaveProperty(subRoomId);
+            return body.txn_id && body.room_subscriptions?.[subRoomId];
+        };
+        const matchRoomUnsubRequest = (unsubRoomId: string) => (request: Request) => {
+            if (!request.url().includes("/sync")) return false;
+            const body = request.postDataJSON();
+            return (
+                body.txn_id && body.unsubscribe_rooms?.includes(unsubRoomId) && !body.room_subscriptions?.[unsubRoomId]
+            );
         };
 
-        let promise = page.waitForRequest(/sync/);
-
-        // Select the Test Room
-        await page.getByRole("treeitem", { name: "Apple", exact: true }).click();
-
-        // and wait for playwright to get the request
-        const roomSubscriptions = (await promise).postDataJSON().room_subscriptions;
+        // Select the Test Room and wait for playwright to get the request
+        const [request] = await Promise.all([
+            page.waitForRequest(matchRoomSubRequest(roomAId)),
+            page.getByRole("treeitem", { name: "Apple", exact: true }).click(),
+        ]);
+        const roomSubscriptions = request.postDataJSON().room_subscriptions;
         expect(roomSubscriptions, "room_subscriptions is object").toBeDefined();
 
-        // Switch to another room
-        promise = page.waitForRequest(/sync/);
-        await page.getByRole("treeitem", { name: "Pineapple", exact: true }).click();
-        assertUnsubExists(await promise, roomPId, roomAId);
+        // Switch to another room and wait for playwright to get the request
+        await Promise.all([
+            page.waitForRequest(matchRoomSubRequest(roomPId)),
+            page.waitForRequest(matchRoomUnsubRequest(roomAId)),
+            page.getByRole("treeitem", { name: "Pineapple", exact: true }).click(),
+        ]);
 
-        // And switch to even another room
-        promise = page.waitForRequest(/sync/);
-        await page.getByRole("treeitem", { name: "Apple", exact: true }).click();
-        assertUnsubExists(await promise, roomPId, roomAId);
+        // And switch to even another room and wait for playwright to get the request
+        await Promise.all([
+            page.waitForRequest(matchRoomSubRequest(roomOId)),
+            page.waitForRequest(matchRoomUnsubRequest(roomPId)),
+            page.getByRole("treeitem", { name: "Orange", exact: true }).click(),
+        ]);
 
         // TODO: Add tests for encrypted rooms
     });

playwright/e2e/spaces/threads-activity-centre/index.ts

@@ -13,6 +13,7 @@ import { test as base, expect } from "../../../element-web-test";
 import { Bot } from "../../../pages/bot";
 import { Client } from "../../../pages/client";
 import { ElementAppPage } from "../../../pages/ElementAppPage";
+import { Credentials } from "../../../plugins/homeserver";
 
 type RoomRef = { name: string; roomId: string };
 
@@ -336,12 +337,14 @@ export class Helpers {
      * @param room1
      * @param room2
      * @param msg - MessageBuilder
+     * @param user - the user to mention in the first message
      * @param hasMention - whether to include a mention in the first message
      */
     async populateThreads(
         room1: { name: string; roomId: string },
         room2: { name: string; roomId: string },
         msg: MessageBuilder,
+        user: Credentials,
         hasMention = true,
     ) {
         if (hasMention) {
@@ -350,9 +353,9 @@ export class Helpers {
                 msg.threadedOff("Msg1", {
                     "body": "User",
                     "format": "org.matrix.custom.html",
-                    "formatted_body": "<a href='https://matrix.to/#/@user:localhost'>User</a>",
+                    "formatted_body": `<a href="https://matrix.to/#/${user.userId}">User</a>`,
                     "m.mentions": {
-                        user_ids: ["@user:localhost"],
+                        user_ids: [user.userId],
                     },
                 }),
             ]);

playwright/e2e/spaces/threads-activity-centre/threadsActivityCentre.spec.ts

@@ -46,16 +46,21 @@ test.describe("Threads Activity Centre", { tag: "@no-firefox" }, () => {
         await util.assertNotificationTac();
     });
 
-    test("should show a highlight indicator when there is a mention in a thread", async ({ room1, util, msg }) => {
+    test("should show a highlight indicator when there is a mention in a thread", async ({
+        room1,
+        util,
+        msg,
+        user,
+    }) => {
         await util.goTo(room1);
         await util.receiveMessages(room1, [
             "Msg1",
             msg.threadedOff("Msg1", {
                 "body": "User",
                 "format": "org.matrix.custom.html",
-                "formatted_body": "<a href='https://matrix.to/#/@user:localhost'>User</a>",
+                "formatted_body": `<a href="https://matrix.to/#/${user.userId}">User</a>`,
                 "m.mentions": {
-                    user_ids: ["@user:localhost"],
+                    user_ids: [user.userId],
                 },
             }),
         ]);
@@ -64,26 +69,30 @@ test.describe("Threads Activity Centre", { tag: "@no-firefox" }, () => {
         await util.assertHighlightIndicator();
     });
 
-    test("should show the rooms with unread threads", { tag: "@screenshot" }, async ({ room1, room2, util, msg }) => {
+    test(
+        "should show the rooms with unread threads",
+        { tag: "@screenshot" },
+        async ({ room1, room2, util, msg, user }) => {
+            await util.goTo(room2);
+            await util.populateThreads(room1, room2, msg, user);
+            // The indicator should be shown
+            await util.assertHighlightIndicator();
+
+            // Verify that we have the expected rooms in the TAC
+            await util.openTac();
+            await util.assertRoomsInTac([
+                { room: room2.name, notificationLevel: "highlight" },
+                { room: room1.name, notificationLevel: "notification" },
+            ]);
+
+            // Verify that we don't have a visual regression
+            await expect(util.getTacPanel()).toMatchScreenshot("tac-panel-mix-unread.png");
+        },
+    );
+
+    test("should update with a thread is read", { tag: "@screenshot" }, async ({ room1, room2, util, msg, user }) => {
         await util.goTo(room2);
-        await util.populateThreads(room1, room2, msg);
-        // The indicator should be shown
-        await util.assertHighlightIndicator();
-
-        // Verify that we have the expected rooms in the TAC
-        await util.openTac();
-        await util.assertRoomsInTac([
-            { room: room2.name, notificationLevel: "highlight" },
-            { room: room1.name, notificationLevel: "notification" },
-        ]);
-
-        // Verify that we don't have a visual regression
-        await expect(util.getTacPanel()).toMatchScreenshot("tac-panel-mix-unread.png");
-    });
-
-    test("should update with a thread is read", { tag: "@screenshot" }, async ({ room1, room2, util, msg }) => {
-        await util.goTo(room2);
-        await util.populateThreads(room1, room2, msg);
+        await util.populateThreads(room1, room2, msg, user);
 
         // Click on the first room in TAC
         await util.openTac();
@@ -104,9 +113,9 @@ test.describe("Threads Activity Centre", { tag: "@no-firefox" }, () => {
         await expect(util.getTacPanel()).toMatchScreenshot("tac-panel-notification-unread.png");
     });
 
-    test("should order by recency after notification level", async ({ room1, room2, util, msg }) => {
+    test("should order by recency after notification level", async ({ room1, room2, util, msg, user }) => {
         await util.goTo(room2);
-        await util.populateThreads(room1, room2, msg, false);
+        await util.populateThreads(room1, room2, msg, user, false);
 
         await util.openTac();
         await util.assertRoomsInTac([

playwright/e2e/spotlight/spotlight.spec.ts

@@ -6,8 +6,8 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */
 
-import type { AccountDataEvents } from "matrix-js-sdk/src/matrix";
-import { test, expect } from "../../element-web-test";
+import type { AccountDataEvents, Visibility } from "matrix-js-sdk/src/matrix";
+import { test as base, expect } from "../../element-web-test";
 import { Filter } from "../../pages/Spotlight";
 import { Bot } from "../../pages/bot";
 import type { Locator, Page } from "@playwright/test";
@@ -38,41 +38,37 @@ async function startDM(app: ElementAppPage, page: Page, name: string): Promise<v
     await expect(page.getByRole("group", { name: "People" }).getByText(name)).toBeAttached();
 }
 
-test.describe("Spotlight", () => {
-    const bot1Name = "BotBob";
-    let bot1: Bot;
-
-    const bot2Name = "ByteBot";
-    let bot2: Bot;
-
-    const room1Name = "247";
-    let room1Id: string;
-
-    const room2Name = "Lounge";
-    let room2Id: string;
-
-    const room3Name = "Public";
-    let room3Id: string;
-
-    test.use({
-        displayName: "Jim",
-    });
-
-    test.beforeEach(async ({ page, homeserver, app, user }) => {
-        bot1 = new Bot(page, homeserver, { displayName: bot1Name, autoAcceptInvites: true });
-        bot2 = new Bot(page, homeserver, { displayName: bot2Name, autoAcceptInvites: true });
-        const Visibility = await page.evaluate(() => (window as any).matrixcs.Visibility);
-
-        room1Id = await app.client.createRoom({ name: room1Name, visibility: Visibility.Public });
-
-        await bot1.joinRoom(room1Id);
-        const bot1UserId = await bot1.evaluate((client) => client.getUserId());
-        room2Id = await bot2.createRoom({ name: room2Name, visibility: Visibility.Public });
-        await bot2.inviteUser(room2Id, bot1UserId);
-
-        room3Id = await bot2.createRoom({
-            name: room3Name,
-            visibility: Visibility.Public,
+type RoomRef = { name: string; roomId: string };
+const test = base.extend<{
+    bot1: Bot;
+    bot2: Bot;
+    room1: RoomRef;
+    room2: RoomRef;
+    room3: RoomRef;
+}>({
+    bot1: async ({ page, homeserver }, use, testInfo) => {
+        const bot = new Bot(page, homeserver, { displayName: `BotBob_${testInfo.testId}`, autoAcceptInvites: true });
+        await use(bot);
+    },
+    bot2: async ({ page, homeserver }, use, testInfo) => {
+        const bot = new Bot(page, homeserver, { displayName: `ByteBot_${testInfo.testId}`, autoAcceptInvites: true });
+        await use(bot);
+    },
+    room1: async ({ app }, use) => {
+        const name = "247";
+        const roomId = await app.client.createRoom({ name, visibility: "public" as Visibility });
+        await use({ name, roomId });
+    },
+    room2: async ({ bot2 }, use) => {
+        const name = "Lounge";
+        const roomId = await bot2.createRoom({ name, visibility: "public" as Visibility });
+        await use({ name, roomId });
+    },
+    room3: async ({ bot2 }, use) => {
+        const name = "Public";
+        const roomId = await bot2.createRoom({
+            name,
+            visibility: "public" as Visibility,
             initial_state: [
                 {
                     type: "m.room.history_visibility",
@@ -83,9 +79,26 @@ test.describe("Spotlight", () => {
                 },
             ],
         });
-        await bot2.inviteUser(room3Id, bot1UserId);
+        await use({ name, roomId });
+    },
+    context: async ({ context, homeserver }, use) => {
+        // Restart the homeserver to wipe its in-memory db so we can reuse the same user ID without cross-signing prompts
+        await homeserver.restart();
+        await use(context);
+    },
+});
 
-        await page.goto("/#/room/" + room1Id);
+test.describe("Spotlight", () => {
+    test.use({
+        displayName: "Jim",
+    });
+
+    test.beforeEach(async ({ page, user, bot1, bot2, room1, room2, room3 }) => {
+        await bot1.joinRoom(room1.roomId);
+        await bot2.inviteUser(room2.roomId, bot1.credentials.userId);
+        await bot2.inviteUser(room3.roomId, bot1.credentials.userId);
+
+        await page.goto(`/#/room/${room1.roomId}`);
         await expect(page.locator(".mx_RoomSublist_skeletonUI")).not.toBeAttached();
     });
 
@@ -117,69 +130,69 @@ test.describe("Spotlight", () => {
         await expect(spotlight.dialog.locator(".mx_SpotlightDialog_filter")).not.toBeAttached();
     });
 
-    test("should find joined rooms", async ({ page, app }) => {
+    test("should find joined rooms", async ({ page, app, room1 }) => {
         const spotlight = await app.openSpotlight();
         await page.waitForTimeout(500); // wait for the dialog to settle
-        await spotlight.search(room1Name);
+        await spotlight.search(room1.name);
         const resultLocator = spotlight.results;
         await expect(resultLocator).toHaveCount(1);
-        await expect(resultLocator.first()).toContainText(room1Name);
+        await expect(resultLocator.first()).toContainText(room1.name);
         await resultLocator.first().click();
-        await expect(page).toHaveURL(new RegExp(`#/room/${room1Id}`));
-        await expect(roomHeaderName(page)).toContainText(room1Name);
+        await expect(page).toHaveURL(new RegExp(`#/room/${room1.roomId}`));
+        await expect(roomHeaderName(page)).toContainText(room1.name);
     });
 
-    test("should find known public rooms", async ({ page, app }) => {
+    test("should find known public rooms", async ({ page, app, room1 }) => {
         const spotlight = await app.openSpotlight();
         await page.waitForTimeout(500); // wait for the dialog to settle
         await spotlight.filter(Filter.PublicRooms);
-        await spotlight.search(room1Name);
+        await spotlight.search(room1.name);
         const resultLocator = spotlight.results;
         await expect(resultLocator).toHaveCount(1);
-        await expect(resultLocator.first()).toContainText(room1Name);
+        await expect(resultLocator.first()).toContainText(room1.name);
         await expect(resultLocator.first()).toContainText("View");
         await resultLocator.first().click();
-        await expect(page).toHaveURL(new RegExp(`#/room/${room1Id}`));
-        await expect(roomHeaderName(page)).toContainText(room1Name);
+        await expect(page).toHaveURL(new RegExp(`#/room/${room1.roomId}`));
+        await expect(roomHeaderName(page)).toContainText(room1.name);
     });
 
-    test("should find unknown public rooms", async ({ page, app }) => {
+    test("should find unknown public rooms", async ({ page, app, room2 }) => {
         const spotlight = await app.openSpotlight();
         await page.waitForTimeout(500); // wait for the dialog to settle
         await spotlight.filter(Filter.PublicRooms);
-        await spotlight.search(room2Name);
+        await spotlight.search(room2.name);
         const resultLocator = spotlight.results;
         await expect(resultLocator).toHaveCount(1);
-        await expect(resultLocator.first()).toContainText(room2Name);
+        await expect(resultLocator.first()).toContainText(room2.name);
         await expect(resultLocator.first()).toContainText("Join");
         await resultLocator.first().click();
-        await expect(page).toHaveURL(new RegExp(`#/room/${room2Id}`));
+        await expect(page).toHaveURL(new RegExp(`#/room/${room2.roomId}`));
         await expect(page.locator(".mx_RoomView_MessageList")).toHaveCount(1);
-        await expect(roomHeaderName(page)).toContainText(room2Name);
+        await expect(roomHeaderName(page)).toContainText(room2.name);
     });
 
-    test("should find unknown public world readable rooms", async ({ page, app }) => {
+    test("should find unknown public world readable rooms", async ({ page, app, room3 }) => {
         const spotlight = await app.openSpotlight();
         await page.waitForTimeout(500); // wait for the dialog to settle
         await spotlight.filter(Filter.PublicRooms);
-        await spotlight.search(room3Name);
+        await spotlight.search(room3.name);
         const resultLocator = spotlight.results;
         await expect(resultLocator).toHaveCount(1);
-        await expect(resultLocator.first()).toContainText(room3Name);
+        await expect(resultLocator.first()).toContainText(room3.name);
         await expect(resultLocator.first()).toContainText("View");
         await resultLocator.first().click();
-        await expect(page).toHaveURL(new RegExp(`#/room/${room3Id}`));
+        await expect(page).toHaveURL(new RegExp(`#/room/${room3.roomId}`));
         await page.getByRole("button", { name: "Join the discussion" }).click();
-        await expect(roomHeaderName(page)).toHaveText(room3Name);
+        await expect(roomHeaderName(page)).toHaveText(room3.name);
     });
 
     // TODO: We currently can’t test finding rooms on other homeservers/other protocols
     // We obviously don’t have federation or bridges in local e2e tests
-    test.skip("should find unknown public rooms on other homeservers", async ({ page, app }) => {
+    test.skip("should find unknown public rooms on other homeservers", async ({ page, app, room3 }) => {
         const spotlight = await app.openSpotlight();
         await page.waitForTimeout(500); // wait for the dialog to settle
         await spotlight.filter(Filter.PublicRooms);
-        await spotlight.search(room3Name);
+        await spotlight.search(room3.name);
         await page.locator("[aria-haspopup=true][role=button]").click();
 
         await page
@@ -194,20 +207,20 @@ test.describe("Spotlight", () => {
 
         const resultLocator = spotlight.results;
         await expect(resultLocator).toHaveCount(1);
-        await expect(resultLocator.first()).toContainText(room3Name);
-        await expect(resultLocator.first()).toContainText(room3Id);
+        await expect(resultLocator.first()).toContainText(room3.name);
+        await expect(resultLocator.first()).toContainText(room3.roomId);
     });
 
-    test("should find known people", async ({ page, app }) => {
+    test("should find known people", async ({ page, app, bot1 }) => {
         const spotlight = await app.openSpotlight();
         await page.waitForTimeout(500); // wait for the dialog to settle
         await spotlight.filter(Filter.People);
-        await spotlight.search(bot1Name);
+        await spotlight.search(bot1.credentials.displayName);
         const resultLocator = spotlight.results;
         await expect(resultLocator).toHaveCount(1);
-        await expect(resultLocator.first()).toContainText(bot1Name);
+        await expect(resultLocator.first()).toContainText(bot1.credentials.displayName);
         await resultLocator.first().click();
-        await expect(roomHeaderName(page)).toHaveText(bot1Name);
+        await expect(roomHeaderName(page)).toHaveText(bot1.credentials.displayName);
     });
 
     /**
@@ -217,42 +230,41 @@ test.describe("Spotlight", () => {
      *
      * https://github.com/matrix-org/synapse/issues/16472
      */
-    test("should find unknown people", async ({ page, app }) => {
+    test("should find unknown people", async ({ page, app, bot2 }) => {
         const spotlight = await app.openSpotlight();
         await page.waitForTimeout(500); // wait for the dialog to settle
         await spotlight.filter(Filter.People);
-        await spotlight.search(bot2Name);
+        await spotlight.search(bot2.credentials.displayName);
         const resultLocator = spotlight.results;
         await expect(resultLocator).toHaveCount(1);
-        await expect(resultLocator.first()).toContainText(bot2Name);
+        await expect(resultLocator.first()).toContainText(bot2.credentials.displayName);
         await resultLocator.first().click();
-        await expect(roomHeaderName(page)).toHaveText(bot2Name);
+        await expect(roomHeaderName(page)).toHaveText(bot2.credentials.displayName);
     });
 
-    test("should find group DMs by usernames or user ids", async ({ page, app }) => {
+    test("should find group DMs by usernames or user ids", async ({ page, app, bot1, bot2, room1 }) => {
         // First we want to share a room with both bots to ensure we’ve got their usernames cached
-        const bot2UserId = await bot2.evaluate((client) => client.getUserId());
-        await app.client.inviteUser(room1Id, bot2UserId);
+        await app.client.inviteUser(room1.roomId, bot2.credentials.userId);
 
         // Starting a DM with ByteBot (will be turned into a group dm later)
         let spotlight = await app.openSpotlight();
         await page.waitForTimeout(500); // wait for the dialog to settle
         await spotlight.filter(Filter.People);
-        await spotlight.search(bot2Name);
+        await spotlight.search(bot2.credentials.displayName);
         let resultLocator = spotlight.results;
         await expect(resultLocator).toHaveCount(1);
-        await expect(resultLocator.first()).toContainText(bot2Name);
+        await expect(resultLocator.first()).toContainText(bot2.credentials.displayName);
         await resultLocator.first().click();
 
         // Send first message to actually start DM
-        await expect(roomHeaderName(page)).toHaveText(bot2Name);
+        await expect(roomHeaderName(page)).toHaveText(bot2.credentials.displayName);
         const locator = page.getByRole("textbox", { name: "Send a message…" });
         await locator.fill("Hey!");
         await locator.press("Enter");
 
         // Assert DM exists by checking for the first message and the room being in the room list
         await expect(page.locator(".mx_EventTile_body").filter({ hasText: "Hey!" })).toBeAttached({ timeout: 3000 });
-        await expect(page.getByRole("group", { name: "People" })).toContainText(bot2Name);
+        await expect(page.getByRole("group", { name: "People" })).toContainText(bot2.credentials.displayName);
 
         // Invite BotBob into existing DM with ByteBot
         const dmRooms = await app.client.evaluate((client, userId) => {
@@ -260,18 +272,17 @@ test.describe("Spotlight", () => {
                 .getAccountData("m.direct" as keyof AccountDataEvents)
                 ?.getContent<Record<string, string[]>>();
             return map[userId] ?? [];
-        }, bot2UserId);
+        }, bot2.credentials.userId);
         expect(dmRooms).toHaveLength(1);
         const groupDmName = await app.client.evaluate((client, id) => client.getRoom(id).name, dmRooms[0]);
-        const bot1UserId = await bot1.evaluate((client) => client.getUserId());
-        await app.client.inviteUser(dmRooms[0], bot1UserId);
+        await app.client.inviteUser(dmRooms[0], bot1.credentials.userId);
         await expect(roomHeaderName(page).first()).toContainText(groupDmName);
         await expect(page.getByRole("group", { name: "People" }).first()).toContainText(groupDmName);
 
         // Search for BotBob by id, should return group DM and user
         spotlight = await app.openSpotlight();
         await spotlight.filter(Filter.People);
-        await spotlight.search(bot1UserId);
+        await spotlight.search(bot1.credentials.userId);
         await page.waitForTimeout(1000); // wait for the dialog to settle
         resultLocator = spotlight.results;
         await expect(resultLocator).toHaveCount(2);
@@ -284,7 +295,7 @@ test.describe("Spotlight", () => {
         // Search for ByteBot by id, should return group DM and user
         spotlight = await app.openSpotlight();
         await spotlight.filter(Filter.People);
-        await spotlight.search(bot2UserId);
+        await spotlight.search(bot2.credentials.userId);
         await page.waitForTimeout(1000); // wait for the dialog to settle
         resultLocator = spotlight.results;
         await expect(resultLocator).toHaveCount(2);
@@ -297,11 +308,10 @@ test.describe("Spotlight", () => {
     });
 
     // Test against https://github.com/vector-im/element-web/issues/22851
-    test("should show each person result only once", async ({ page, app }) => {
+    test("should show each person result only once", async ({ page, app, bot1 }) => {
         const spotlight = await app.openSpotlight();
         await page.waitForTimeout(500); // wait for the dialog to settle
         await spotlight.filter(Filter.People);
-        const bot1UserId = await bot1.evaluate((client) => client.getUserId());
 
         // 2 rounds of search to simulate the bug conditions. Specifically, the first search
         // should have 1 result (not 2) and the second search should also have 1 result (instead
@@ -310,24 +320,24 @@ test.describe("Spotlight", () => {
         // We search for user ID to trigger the profile lookup within the dialog.
         for (let i = 0; i < 2; i++) {
             console.log("Iteration: " + i);
-            await spotlight.search(bot1UserId);
+            await spotlight.search(bot1.credentials.userId);
             await page.waitForTimeout(1000); // wait for the dialog to settle
             const resultLocator = spotlight.results;
             await expect(resultLocator).toHaveCount(1);
-            await expect(resultLocator.first()).toContainText(bot1UserId);
+            await expect(resultLocator.first()).toContainText(bot1.credentials.userId);
         }
     });
 
-    test("should allow opening group chat dialog", async ({ page, app }) => {
+    test("should allow opening group chat dialog", async ({ page, app, bot2 }) => {
         const spotlight = await app.openSpotlight();
         await page.waitForTimeout(500); // wait for the dialog to settle
         await spotlight.filter(Filter.People);
-        await spotlight.search(bot2Name);
+        await spotlight.search(bot2.credentials.displayName);
         await page.waitForTimeout(3000); // wait for the dialog to settle
 
         const resultLocator = spotlight.results;
         await expect(resultLocator).toHaveCount(1);
-        await expect(resultLocator.first()).toContainText(bot2Name);
+        await expect(resultLocator.first()).toContainText(bot2.credentials.displayName);
 
         await expect(spotlight.dialog.locator(".mx_SpotlightDialog_startGroupChat")).toContainText(
             "Start a group chat",
@@ -336,18 +346,18 @@ test.describe("Spotlight", () => {
         await expect(page.getByRole("dialog")).toContainText("Direct Messages");
     });
 
-    test("should close spotlight after starting a DM", async ({ page, app }) => {
-        await startDM(app, page, bot1Name);
+    test("should close spotlight after starting a DM", async ({ page, app, bot1 }) => {
+        await startDM(app, page, bot1.credentials.displayName);
         await expect(page.locator(".mx_SpotlightDialog")).toHaveCount(0);
     });
 
-    test("should show the same user only once", async ({ page, app }) => {
-        await startDM(app, page, bot1Name);
+    test("should show the same user only once", async ({ page, app, bot1 }) => {
+        await startDM(app, page, bot1.credentials.displayName);
         await page.goto("/#/home");
         const spotlight = await app.openSpotlight();
         await page.waitForTimeout(500); // wait for the dialog to settle
         await spotlight.filter(Filter.People);
-        await spotlight.search(bot1Name);
+        await spotlight.search(bot1.credentials.displayName);
         await page.waitForTimeout(3000); // wait for the dialog to settle
         await expect(spotlight.dialog.locator(".mx_Spinner")).not.toBeAttached();
         const resultLocator = spotlight.results;

playwright/e2e/threads/threads.spec.ts

@@ -24,8 +24,7 @@ test.describe("Threads", () => {
         });
     });
 
-    // Flaky: https://github.com/vector-im/element-web/issues/26452
-    test.skip("should be usable for a conversation", { tag: "@screenshot" }, async ({ page, app, bot }) => {
+    test("should be usable for a conversation", { tag: "@screenshot" }, async ({ page, app, bot }) => {
         const roomId = await app.client.createRoom({});
         await app.client.inviteUser(roomId, bot.credentials.userId);
         await bot.joinRoom(roomId);
@@ -76,7 +75,7 @@ test.describe("Threads", () => {
             mask: mask,
         });
         await app.settings.setValue("layout", null, SettingLevel.DEVICE, Layout.Bubble);
-        await expect(page.locator(".mx_ThreadView .mx_EventTile[data-layout='bubble']")).toBeVisible();
+        await expect(page.locator(".mx_ThreadView .mx_EventTile[data-layout='bubble']")).toHaveCount(2);
 
         await expect(page.locator(".mx_ThreadView")).toMatchScreenshot("Initial_ThreadView_on_bubble_layout.png", {
             mask: mask,
@@ -136,8 +135,8 @@ test.describe("Threads", () => {
         await page.getByRole("gridcell", { name: "👋" }).click();
 
         locator = page.locator(".mx_ThreadView");
-        // Make sure the CSS style for spacing is applied to mx_ReactionsRow on group/modern layout
-        await expect(locator.locator(".mx_EventTile[data-layout=group] .mx_ReactionsRow")).toHaveCSS(
+        // Make sure the CSS style for spacing is applied to mx_EventTile_footer on group/modern layout
+        await expect(locator.locator(".mx_EventTile[data-layout=group] .mx_EventTile_footer")).toHaveCSS(
             "margin-inline-start",
             ThreadViewGroupSpacingStart,
         );
@@ -164,7 +163,7 @@ test.describe("Threads", () => {
         locator = page.locator(
             ".mx_ThreadView .mx_GenericEventListSummary[data-layout=bubble] .mx_EventTile_info.mx_EventTile_last",
         );
-        expect(locator.locator(".mx_EventTile_line .mx_EventTile_content"))
+        await expect(locator.locator(".mx_EventTile_line .mx_EventTile_content"))
             // 76px: ThreadViewGroupSpacingStart + 14px + 6px
             // 14px: avatar width
             // See: _EventTile.pcss
@@ -202,12 +201,14 @@ test.describe("Threads", () => {
         await locator.click();
 
         // Wait until the response is redacted
-        await expect(
-            page.locator(".mx_ThreadView").locator(".mx_EventTile_last .mx_EventTile_receiptSent"),
-        ).toBeVisible();
+        // XXX: one would expect this redaction to be shown in the thread the message was in, but due to redactions
+        // stripping the thread_id, it is instead shown in the main timeline
+        await expect(page.locator(".mx_MainSplit_timeline").locator(".mx_EventTile_last")).toContainText(
+            "Message deleted",
+        );
 
         // Take snapshots in group layout and bubble layout (IRC layout is not available on ThreadView)
-        await expect(page.locator(".mx_ThreadView .mx_EventTile[data-layout='group']")).toBeVisible();
+        await expect(page.locator(".mx_ThreadView .mx_EventTile[data-layout='group']")).toHaveCount(2);
         await expect(page.locator(".mx_ThreadView")).toMatchScreenshot(
             "ThreadView_with_redacted_messages_on_group_layout.png",
             {
@@ -215,7 +216,7 @@ test.describe("Threads", () => {
             },
         );
         await app.settings.setValue("layout", null, SettingLevel.DEVICE, Layout.Bubble);
-        await expect(page.locator(".mx_ThreadView .mx_EventTile[data-layout='bubble']")).toBeVisible();
+        await expect(page.locator(".mx_ThreadView .mx_EventTile[data-layout='bubble']")).toHaveCount(2);
         await expect(page.locator(".mx_ThreadView")).toMatchScreenshot(
             "ThreadView_with_redacted_messages_on_bubble_layout.png",
             {
@@ -233,8 +234,8 @@ test.describe("Threads", () => {
 
         // User closes right panel after clicking back to thread list
         locator = page.locator(".mx_ThreadPanel");
-        locator.getByRole("button", { name: "Threads" }).click();
-        locator.getByRole("button", { name: "Close" }).click();
+        await locator.getByRole("button", { name: "Threads" }).click();
+        await locator.getByRole("button", { name: "Close" }).click();
 
         // Bot responds to thread
         await bot.sendMessage(roomId, "How are things?", threadId);
@@ -243,9 +244,8 @@ test.describe("Threads", () => {
         await expect(locator.locator(".mx_ThreadSummary_sender").getByText("BotBob")).toBeAttached();
         await expect(locator.locator(".mx_ThreadSummary_content").getByText("How are things?")).toBeAttached();
 
-        locator = page.getByRole("button", { name: "Threads" });
-        await expect(locator).toHaveAttribute("data-indicator", "default"); // User asserts thread list unread indicator
-        // await expect(locator).toHaveClass(/mx_LegacyRoomHeader_button--unread/);
+        locator = page.getByRole("banner").getByRole("button", { name: "Threads" });
+        await expect(locator).toHaveAttribute("data-indicator", "success"); // User asserts thread list unread indicator
         await locator.click(); // User opens thread list
 
         // User asserts thread with correct root & latest events & unread dot
@@ -273,20 +273,18 @@ test.describe("Threads", () => {
         await expect(locator.getByText("Great!")).toBeAttached();
         await locator.locator(".mx_EventTile_line").hover();
         await locator.locator(".mx_EventTile_line").getByRole("button", { name: "Edit" }).click();
-        await locator.getByRole("textbox").fill(" How about yourself?{enter}");
+        await locator.getByRole("textbox").pressSequentially(" How about yourself?"); // fill would overwrite the original text
         await locator.getByRole("textbox").press("Enter");
 
         locator = page.locator(".mx_RoomView_body .mx_ThreadSummary");
         await expect(locator.locator(".mx_ThreadSummary_sender").getByText("Tom")).toBeAttached();
-        await expect(
-            locator.locator(".mx_ThreadSummary_content").getByText("Great! How about yourself?"),
-        ).toBeAttached();
+        await expect(locator.locator(".mx_ThreadSummary_content")).toHaveText("Great! How about yourself?");
 
         // User closes right panel
         await page.locator(".mx_ThreadPanel").getByRole("button", { name: "Close" }).click();
 
         // Bot responds to thread and saves the id of their message to @eventId
-        const { event_id: eventId } = await bot.sendMessage(roomId, threadId, "I'm very good thanks");
+        const { event_id: eventId } = await bot.sendMessage(roomId, "I'm very good thanks", threadId);
 
         // User asserts
         locator = page.locator(".mx_RoomView_body .mx_ThreadSummary");
@@ -344,7 +342,7 @@ test.describe("Threads", () => {
 
             await expect(page.locator(".mx_ThreadView_timelinePanelWrapper")).toHaveCount(1);
 
-            (await app.openMessageComposerOptions(true)).getByRole("menuitem", { name: "Voice Message" }).click();
+            await (await app.openMessageComposerOptions(true)).getByRole("menuitem", { name: "Voice Message" }).click();
             await page.waitForTimeout(3000);
             await app.getComposer(true).getByRole("button", { name: "Send voice message" }).click();
             await expect(page.locator(".mx_ThreadView .mx_MVoiceMessageBody")).toHaveCount(1);

playwright/e2e/timeline/timeline.spec.ts

@@ -590,10 +590,6 @@ test.describe("Timeline", () => {
             "should set inline start padding to a hidden event line",
             { tag: "@screenshot" },
             async ({ page, app, room }) => {
-                test.skip(
-                    true,
-                    "Disabled due to screenshot test being flaky - https://github.com/element-hq/element-web/issues/26890",
-                );
                 await sendEvent(app.client, room.roomId);
                 await page.goto(`/#/room/${room.roomId}`);
                 await app.settings.setValue("showHiddenEventsInTimeline", null, SettingLevel.DEVICE, true);
@@ -607,7 +603,12 @@ test.describe("Timeline", () => {
                 await messageEdit(page);
 
                 // Click timestamp to highlight hidden event line
-                await page.locator(".mx_RoomView_body .mx_EventTile_info .mx_MessageTimestamp").click();
+                const timestamp = page.locator(".mx_RoomView_body .mx_EventTile_info a", {
+                    has: page.locator(".mx_MessageTimestamp"),
+                });
+                // wait for the remote echo otherwise we get an error modal due to a 404 on the /event/ API
+                await expect(timestamp).not.toHaveAttribute("href", /~!/);
+                await timestamp.locator(".mx_MessageTimestamp").click();
 
                 // should not add inline start padding to a hidden event line on IRC layout
                 await app.settings.setValue("layout", null, SettingLevel.DEVICE, Layout.IRC);

playwright/e2e/user-onboarding/user-onboarding-new.spec.ts

@@ -1,79 +0,0 @@
-/*
-Copyright 2024 New Vector Ltd.
-Copyright 2022 The Matrix.org Foundation C.I.C.
-
-SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE files in the repository root for full details.
-*/
-
-import { test, expect } from "../../element-web-test";
-
-test.describe("User Onboarding (new user)", () => {
-    test.use({
-        displayName: "Jane Doe",
-    });
-
-    // This first beforeEach happens before the `user` fixture runs
-    test.beforeEach(async ({ page }) => {
-        await page.addInitScript(() => {
-            window.localStorage.setItem("mx_registration_time", "1656633601");
-        });
-    });
-
-    test.beforeEach(async ({ page, user }) => {
-        await expect(page.locator(".mx_UserOnboardingPage")).toBeVisible();
-        await expect(page.getByRole("button", { name: "Welcome" })).toBeVisible();
-        await expect(page.locator(".mx_UserOnboardingList")).toBeVisible();
-    });
-
-    test("page is shown and preference exists", { tag: "@screenshot" }, async ({ page, app }) => {
-        await expect(page.locator(".mx_UserOnboardingPage")).toMatchScreenshot(
-            "User-Onboarding-new-user-page-is-shown-and-preference-exists-1.png",
-        );
-        await app.settings.openUserSettings("Preferences");
-        await expect(page.getByText("Show shortcut to welcome checklist above the room list")).toBeVisible();
-    });
-
-    test("app download dialog", { tag: "@screenshot" }, async ({ page }) => {
-        await page.getByRole("button", { name: "Download apps" }).click();
-        await expect(
-            page.getByRole("dialog").getByRole("heading", { level: 1, name: "Download Element" }),
-        ).toBeVisible();
-        await expect(page.locator(".mx_Dialog")).toMatchScreenshot(
-            "User-Onboarding-new-user-app-download-dialog-1.png",
-            {
-                // Set a constant bg behind the modal to ensure screenshot stability
-                css: `
-                    .mx_AppDownloadDialog_wrapper {
-                        background: black;
-                    }
-                `,
-            },
-        );
-    });
-
-    test("using find friends action should increase progress", async ({ page, homeserver }) => {
-        const bot = await homeserver.registerUser("botbob", "password", "BotBob");
-
-        const oldProgress = parseFloat(await page.getByRole("progressbar").getAttribute("value"));
-        await page.getByRole("button", { name: "Find friends" }).click();
-        await page.locator(".mx_InviteDialog_editor").getByRole("textbox").fill(bot.userId);
-        await page.getByRole("button", { name: "Go" }).click();
-        await expect(page.locator(".mx_InviteDialog_buttonAndSpinner")).not.toBeVisible();
-
-        const message = "Hi!";
-        const composer = page.getByRole("textbox", { name: "Send a message…" });
-        await composer.fill(`${message}`);
-        await composer.press("Enter");
-        await expect(page.locator(".mx_MTextBody.mx_EventTile_content", { hasText: message })).toBeVisible();
-
-        await page.goto("/#/home");
-        await expect(page.locator(".mx_UserOnboardingPage")).toBeVisible();
-        await expect(page.getByRole("button", { name: "Welcome" })).toBeVisible();
-        await expect(page.locator(".mx_UserOnboardingList")).toBeVisible();
-
-        await page.waitForTimeout(500); // await progress bar animation
-        const progress = parseFloat(await page.getByRole("progressbar").getAttribute("value"));
-        expect(progress).toBeGreaterThan(oldProgress);
-    });
-});

playwright/e2e/user-onboarding/user-onboarding-old.spec.ts

@@ -1,28 +0,0 @@
-/*
-Copyright 2024 New Vector Ltd.
-Copyright 2022 The Matrix.org Foundation C.I.C.
-
-SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
-Please see LICENSE files in the repository root for full details.
-*/
-
-import { test, expect } from "../../element-web-test";
-
-test.describe("User Onboarding (old user)", () => {
-    test.use({
-        displayName: "Jane Doe",
-    });
-
-    test.beforeEach(async ({ page }) => {
-        await page.addInitScript(() => {
-            window.localStorage.setItem("mx_registration_time", "2");
-        });
-    });
-
-    test("page and preference are hidden", async ({ page, user, app }) => {
-        await expect(page.locator(".mx_UserOnboardingPage")).not.toBeVisible();
-        await expect(page.locator(".mx_UserOnboardingButton")).not.toBeVisible();
-        await app.settings.openUserSettings("Preferences");
-        await expect(page.getByText("Show shortcut to welcome checklist above the room list")).not.toBeVisible();
-    });
-});

playwright/e2e/widgets/stickers.spec.ts

@@ -150,7 +150,7 @@ test.describe("Stickers", { tag: ["@no-firefox", "@no-webkit"] }, () => {
         const { content_uri: contentUri } = await app.client.uploadContent(STICKER_IMAGE, { type: "image/png" });
         const widgetHtml = getWidgetHtml(contentUri, "image/png");
         stickerPickerUrl = webserver.start(widgetHtml);
-        setWidgetAccountData(app, user, stickerPickerUrl);
+        await setWidgetAccountData(app, user, stickerPickerUrl);
 
         await app.viewRoomByName(ROOM_NAME_1);
         await expect(page).toHaveURL(`/#/room/${room.roomId}`);
@@ -177,7 +177,7 @@ test.describe("Stickers", { tag: ["@no-firefox", "@no-webkit"] }, () => {
         const { content_uri: contentUri } = await app.client.uploadContent(STICKER_IMAGE, { type: "image/png" });
         const widgetHtml = getWidgetHtml(contentUri, "image/png");
         stickerPickerUrl = webserver.start(widgetHtml);
-        setWidgetAccountData(app, user, stickerPickerUrl, false);
+        await setWidgetAccountData(app, user, stickerPickerUrl, false);
 
         await app.viewRoomByName(ROOM_NAME_1);
         await expect(page).toHaveURL(`/#/room/${room.roomId}`);
@@ -192,7 +192,7 @@ test.describe("Stickers", { tag: ["@no-firefox", "@no-webkit"] }, () => {
         });
         const widgetHtml = getWidgetHtml(contentUri, "application/octet-stream");
         stickerPickerUrl = webserver.start(widgetHtml);
-        setWidgetAccountData(app, user, stickerPickerUrl);
+        await setWidgetAccountData(app, user, stickerPickerUrl);
 
         await app.viewRoomByName(ROOM_NAME_1);
         await expect(page).toHaveURL(`/#/room/${room.roomId}`);

playwright/element-web-test.ts

@@ -6,7 +6,15 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */
 
-import { expect as baseExpect, Locator, Page, ExpectMatcherState, ElementHandle } from "@playwright/test";
+import {
+    expect as baseExpect,
+    Locator,
+    Page,
+    ExpectMatcherState,
+    ElementHandle,
+    PlaywrightTestArgs,
+    Fixtures as _Fixtures,
+} from "@playwright/test";
 import { sanitizeForFilePath } from "playwright-core/lib/utils";
 import AxeBuilder from "@axe-core/playwright";
 import _ from "lodash";
@@ -19,7 +27,7 @@ import { Crypto } from "./pages/crypto";
 import { Toasts } from "./pages/toasts";
 import { Bot, CreateBotOpts } from "./pages/bot";
 import { Webserver } from "./plugins/webserver";
-import { test as base } from "./services.ts";
+import { Options, Services, test as base } from "./services.ts";
 
 // Enable experimental service worker support
 // See https://playwright.dev/docs/service-workers-experimental#how-to-enable
@@ -41,11 +49,11 @@ const CONFIG_JSON: Partial<IConfigOptions> = {
     },
 };
 
-interface CredentialsWithDisplayName extends Credentials {
+export interface CredentialsWithDisplayName extends Credentials {
     displayName: string;
 }
 
-export interface Fixtures {
+export interface TestFixtures {
     axe: AxeBuilder;
     checkA11y: () => Promise<void>;
 
@@ -99,9 +107,12 @@ export interface Fixtures {
     bot: Bot;
     labsFlags: string[];
     webserver: Webserver;
+    disablePresence: boolean;
 }
 
-export const test = base.extend<Fixtures>({
+type CombinedTestFixtures = PlaywrightTestArgs & TestFixtures;
+export type Fixtures = _Fixtures<CombinedTestFixtures, Services & Options, CombinedTestFixtures>;
+export const test = base.extend<TestFixtures>({
     context: async ({ context }, use, testInfo) => {
         // We skip tests instead of using grep-invert to still surface the counts in the html report
         test.skip(
@@ -110,8 +121,9 @@ export const test = base.extend<Fixtures>({
         );
         await use(context);
     },
+    disablePresence: false,
     config: {}, // We merge this atop the default CONFIG_JSON in the page fixture to make extending it easier
-    page: async ({ homeserver, context, page, config, labsFlags }, use) => {
+    page: async ({ homeserver, context, page, config, labsFlags, disablePresence }, use) => {
         await context.route(`http://localhost:8080/config.json*`, async (route) => {
             const json = {
                 ...CONFIG_JSON,
@@ -131,19 +143,24 @@ export const test = base.extend<Fixtures>({
                     return obj;
                 }, {}),
             };
+            if (disablePresence) {
+                json["enable_presence_by_hs_url"] = {
+                    [homeserver.baseUrl]: false,
+                };
+            }
             await route.fulfill({ json });
         });
         await use(page);
     },
 
     displayName: undefined,
-    credentials: async ({ homeserver, displayName: testDisplayName }, use) => {
+    credentials: async ({ context, homeserver, displayName: testDisplayName }, use, testInfo) => {
         const names = ["Alice", "Bob", "Charlie", "Daniel", "Eve", "Frank", "Grace", "Hannah", "Isaac", "Judy"];
         const password = _.uniqueId("password_");
         const displayName = testDisplayName ?? _.sample(names)!;
 
-        const credentials = await homeserver.registerUser("user", password, displayName);
-        console.log(`Registered test user @user:localhost with displayname ${displayName}`);
+        const credentials = await homeserver.registerUser(`user_${testInfo.testId}`, password, displayName);
+        console.log(`Registered test user ${credentials.userId} with displayname ${displayName}`);
 
         await use({
             ...credentials,
@@ -167,6 +184,7 @@ export const test = base.extend<Fixtures>({
                 window.localStorage.setItem(
                     "mx_local_settings",
                     JSON.stringify({
+                        // Retain any other settings which may have already been set
                         ...JSON.parse(window.localStorage.getItem("mx_local_settings") || "{}"),
                         // Ensure the language is set to a consistent value
                         language: "en",

playwright/flaky-reporter.ts

@@ -25,12 +25,15 @@ type PaginationLinks = {
 };
 
 class FlakyReporter implements Reporter {
-    private flakes = new Set<string>();
+    private flakes = new Map<string, TestCase[]>();
 
     public onTestEnd(test: TestCase): void {
         const title = `${test.location.file.split("playwright/e2e/")[1]}: ${test.title}`;
         if (test.outcome() === "flaky") {
-            this.flakes.add(title);
+            if (!this.flakes.has(title)) {
+                this.flakes.set(title, []);
+            }
+            this.flakes.get(title).push(test);
         }
     }
 
@@ -97,12 +100,14 @@ class FlakyReporter implements Reporter {
         if (!GITHUB_TOKEN) return;
 
         const issues = await this.getAllIssues();
-        for (const flake of this.flakes) {
+        for (const [flake, results] of this.flakes) {
             const title = ISSUE_TITLE_PREFIX + "`" + flake + "`";
             const existingIssue = issues.find((issue) => issue.title === title);
             const headers = { Authorization: `Bearer ${GITHUB_TOKEN}` };
             const body = `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`;
 
+            const labels = [LABEL, ...results.map((test) => `${LABEL}-${test.parent.project()?.name}`)];
+
             if (existingIssue) {
                 console.log(`Found issue ${existingIssue.number} for ${flake}, adding comment...`);
                 // Ensure that the test is open
@@ -111,6 +116,11 @@ class FlakyReporter implements Reporter {
                     headers,
                     body: JSON.stringify({ state: "open" }),
                 });
+                await fetch(`${existingIssue.url}/labels`, {
+                    method: "POST",
+                    headers,
+                    body: JSON.stringify({ labels }),
+                });
                 await fetch(`${existingIssue.url}/comments`, {
                     method: "POST",
                     headers,
@@ -124,7 +134,7 @@ class FlakyReporter implements Reporter {
                     body: JSON.stringify({
                         title,
                         body,
-                        labels: [LABEL],
+                        labels: [...labels],
                     }),
                 });
             }

playwright/logger.ts

@@ -0,0 +1,63 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { BrowserContext, Page, TestInfo } from "@playwright/test";
+import { Readable } from "stream";
+import stripAnsi from "strip-ansi";
+
+export class Logger {
+    private pages: Page[] = [];
+    private logs: Record<string, string> = {};
+
+    public getConsumer(container: string) {
+        this.logs[container] = "";
+        return (stream: Readable) => {
+            stream.on("data", (chunk) => {
+                this.logs[container] += chunk.toString();
+            });
+            stream.on("err", (chunk) => {
+                this.logs[container] += "ERR " + chunk.toString();
+            });
+        };
+    }
+
+    public async onTestStarted(context: BrowserContext) {
+        this.pages = [];
+        for (const id in this.logs) {
+            if (id.startsWith("page-")) {
+                delete this.logs[id];
+            } else {
+                this.logs[id] = "";
+            }
+        }
+
+        context.on("console", (msg) => {
+            const page = msg.page();
+            let pageIdx = this.pages.indexOf(page);
+            if (pageIdx === -1) {
+                this.pages.push(page);
+                pageIdx = this.pages.length - 1;
+                this.logs[`page-${pageIdx}`] = `Console logs for page with URL: ${page.url()}\n\n`;
+            }
+            const type = msg.type();
+            const text = msg.text();
+            this.logs[`page-${pageIdx}`] += `${type}: ${text}\n`;
+        });
+    }
+
+    public async onTestFinished(testInfo: TestInfo) {
+        if (testInfo.status !== "passed") {
+            for (const id in this.logs) {
+                if (!this.logs[id]) continue;
+                await testInfo.attach(id, {
+                    body: stripAnsi(this.logs[id]),
+                    contentType: "text/plain",
+                });
+            }
+        }
+    }
+}

playwright/pages/ElementAppPage.ts

@@ -177,6 +177,18 @@ export class ElementAppPage {
         return this.page.locator(".mx_RightPanel");
     }
 
+    /**
+     * Opens/closes the memberlist panel
+     * @returns locator to the memberlist panel
+     */
+    public async toggleMemberlistPanel(): Promise<Locator> {
+        const locator = this.page.locator(".mx_FacePile");
+        await locator.click();
+        const memberlist = this.page.locator(".mx_MemberListView");
+        await memberlist.waitFor();
+        return memberlist;
+    }
+
     /**
      * Get a locator for the tooltip associated with an element
      * @param e The element with the tooltip

playwright/pages/bot.ts

@@ -121,7 +121,7 @@ export class Bot extends Client {
                     return logger as unknown as Logger;
                 }
 
-                const logger = getLogger(`cypress bot ${credentials.userId}`);
+                const logger = getLogger(`bot ${credentials.userId}`);
 
                 const keys = {};
 
@@ -171,7 +171,7 @@ export class Bot extends Client {
                 if (opts.autoAcceptInvites) {
                     cli.on(window.matrixcs.RoomMemberEvent.Membership, (event, member) => {
                         if (member.membership === "invite" && member.userId === cli.getUserId()) {
-                            cli.joinRoom(member.roomId);
+                            void cli.joinRoom(member.roomId);
                         }
                     });
                 }

playwright/pages/client.ts

@@ -179,18 +179,17 @@ export class Client {
     public async createRoom(options: ICreateRoomOpts): Promise<string> {
         const client = await this.prepareClient();
         return await client.evaluate(async (cli, options) => {
-            const roomPromise = new Promise<void>((resolve) => {
-                const onRoom = (room: Room) => {
-                    if (room.roomId === roomId) {
-                        cli.off(window.matrixcs.ClientEvent.Room, onRoom);
-                        resolve();
-                    }
-                };
-                cli.on(window.matrixcs.ClientEvent.Room, onRoom);
-            });
             const { room_id: roomId } = await cli.createRoom(options);
             if (!cli.getRoom(roomId)) {
-                await roomPromise;
+                await new Promise<void>((resolve) => {
+                    const onRoom = (room: Room) => {
+                        if (room.roomId === roomId) {
+                            cli.off(window.matrixcs.ClientEvent.Room, onRoom);
+                            resolve();
+                        }
+                    };
+                    cli.on(window.matrixcs.ClientEvent.Room, onRoom);
+                });
             }
             return roomId;
         }, options);

playwright/plugins/homeserver/dendrite/index.ts

@@ -6,31 +6,32 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */
 
-import { Fixtures, PlaywrightTestArgs } from "@playwright/test";
-
-import { Fixtures as BaseFixtures } from "../../../element-web-test.ts";
 import { DendriteContainer, PineconeContainer } from "../../../testcontainers/dendrite.ts";
-import { Services } from "../../../services.ts";
+import { Fixtures } from "../../../element-web-test.ts";
 
-type Fixture = PlaywrightTestArgs & Services & BaseFixtures;
-export const dendriteHomeserver: Fixtures<Fixture, {}, Fixture> = {
-    _homeserver: async ({ request }, use) => {
-        const container =
-            process.env["PLAYWRIGHT_HOMESERVER"] === "dendrite"
-                ? new DendriteContainer(request)
-                : new PineconeContainer(request);
-        await use(container);
-    },
-    homeserver: async ({ logger, network, _homeserver: homeserver }, use) => {
-        const container = await homeserver
-            .withNetwork(network)
-            .withNetworkAliases("homeserver")
-            .withLogConsumer(logger.getConsumer("dendrite"))
-            .start();
+export const dendriteHomeserver: Fixtures = {
+    _homeserver: [
+        // eslint-disable-next-line no-empty-pattern
+        async ({}, use) => {
+            const container =
+                process.env["PLAYWRIGHT_HOMESERVER"] === "dendrite" ? new DendriteContainer() : new PineconeContainer();
+            await use(container);
+        },
+        { scope: "worker" },
+    ],
+    homeserver: [
+        async ({ logger, network, _homeserver: homeserver }, use) => {
+            const container = await homeserver
+                .withNetwork(network)
+                .withNetworkAliases("homeserver")
+                .withLogConsumer(logger.getConsumer("dendrite"))
+                .start();
 
-        await use(container);
-        await container.stop();
-    },
+            await use(container);
+            await container.stop();
+        },
+        { scope: "worker" },
+    ],
 };
 
 export function isDendrite(): boolean {

playwright/plugins/homeserver/index.ts

@@ -6,8 +6,11 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */
 
+import { ClientServerApi } from "../utils/api.ts";
+
 export interface HomeserverInstance {
     readonly baseUrl: string;
+    readonly csApi: ClientServerApi;
 
     /**
      * Register a user on the given Homeserver using the shared registration secret.
@@ -41,4 +44,5 @@ export interface Credentials {
     homeServer: string;
     password: string | null; // null for password-less users
     displayName?: string;
+    username: string; // the localpart of the userId
 }

playwright/plugins/homeserver/synapse/consentHomeserver.ts

@@ -6,51 +6,52 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */
 
-import { Fixtures } from "@playwright/test";
+import { Fixtures } from "../../../element-web-test.ts";
 
-import { Services } from "../../../services.ts";
-
-export const consentHomeserver: Fixtures<Services, {}, Services> = {
-    _homeserver: async ({ _homeserver: container, mailhog }, use) => {
-        container
-            .withCopyDirectoriesToContainer([
-                { source: "playwright/plugins/homeserver/synapse/res", target: "/data/res" },
-            ])
-            .withConfig({
-                email: {
-                    enable_notifs: false,
-                    smtp_host: "mailhog",
-                    smtp_port: 1025,
-                    smtp_user: "username",
-                    smtp_pass: "password",
-                    require_transport_security: false,
-                    notif_from: "Your Friendly %(app)s homeserver <noreply@example.com>",
-                    app_name: "Matrix",
-                    notif_template_html: "notif_mail.html",
-                    notif_template_text: "notif_mail.txt",
-                    notif_for_new_users: true,
-                    client_base_url: "http://localhost/element",
-                },
-                user_consent: {
-                    template_dir: "/data/res/templates/privacy",
-                    version: "1.0",
-                    server_notice_content: {
-                        msgtype: "m.text",
-                        body: "To continue using this homeserver you must review and agree to the terms and conditions at %(consent_uri)s",
+export const consentHomeserver: Fixtures = {
+    _homeserver: [
+        async ({ _homeserver: container, mailhog }, use) => {
+            container
+                .withCopyDirectoriesToContainer([
+                    { source: "playwright/plugins/homeserver/synapse/res", target: "/data/res" },
+                ])
+                .withConfig({
+                    email: {
+                        enable_notifs: false,
+                        smtp_host: "mailhog",
+                        smtp_port: 1025,
+                        smtp_user: "username",
+                        smtp_pass: "password",
+                        require_transport_security: false,
+                        notif_from: "Your Friendly %(app)s homeserver <noreply@example.com>",
+                        app_name: "Matrix",
+                        notif_template_html: "notif_mail.html",
+                        notif_template_text: "notif_mail.txt",
+                        notif_for_new_users: true,
+                        client_base_url: "http://localhost/element",
                     },
-                    send_server_notice_to_guests: true,
-                    block_events_error:
-                        "To continue using this homeserver you must review and agree to the terms and conditions at %(consent_uri)s",
-                    require_at_registration: true,
-                },
-                server_notices: {
-                    system_mxid_localpart: "notices",
-                    system_mxid_display_name: "Server Notices",
-                    system_mxid_avatar_url: "mxc://localhost/oumMVlgDnLYFaPVkExemNVVZ",
-                    room_name: "Server Notices",
-                },
-            })
-            .withConfigField("listeners[0].resources[0].names", ["client", "consent"]);
-        await use(container);
-    },
+                    user_consent: {
+                        template_dir: "/data/res/templates/privacy",
+                        version: "1.0",
+                        server_notice_content: {
+                            msgtype: "m.text",
+                            body: "To continue using this homeserver you must review and agree to the terms and conditions at %(consent_uri)s",
+                        },
+                        send_server_notice_to_guests: true,
+                        block_events_error:
+                            "To continue using this homeserver you must review and agree to the terms and conditions at %(consent_uri)s",
+                        require_at_registration: true,
+                    },
+                    server_notices: {
+                        system_mxid_localpart: "notices",
+                        system_mxid_display_name: "Server Notices",
+                        system_mxid_avatar_url: "mxc://localhost/oumMVlgDnLYFaPVkExemNVVZ",
+                        room_name: "Server Notices",
+                    },
+                })
+                .withConfigField("listeners[0].resources[0].names", ["client", "consent"]);
+            await use(container);
+        },
+        { scope: "worker" },
+    ],
 };

playwright/plugins/homeserver/synapse/emailHomeserver.ts

@@ -6,23 +6,24 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */
 
-import { Fixtures } from "@playwright/test";
+import { Fixtures } from "../../../element-web-test.ts";
 
-import { Services } from "../../../services.ts";
-
-export const emailHomeserver: Fixtures<Services, {}, Services> = {
-    _homeserver: async ({ _homeserver: container, mailhog }, use) => {
-        container.withConfig({
-            enable_registration_without_verification: undefined,
-            disable_msisdn_registration: undefined,
-            registrations_require_3pid: ["email"],
-            email: {
-                smtp_host: "mailhog",
-                smtp_port: 1025,
-                notif_from: "Your Friendly %(app)s homeserver <noreply@example.com>",
-                app_name: "my_branded_matrix_server",
-            },
-        });
-        await use(container);
-    },
+export const emailHomeserver: Fixtures = {
+    _homeserver: [
+        async ({ _homeserver: container, mailhog }, use) => {
+            container.withConfig({
+                enable_registration_without_verification: undefined,
+                disable_msisdn_registration: undefined,
+                registrations_require_3pid: ["email"],
+                email: {
+                    smtp_host: "mailhog",
+                    smtp_port: 1025,
+                    notif_from: "Your Friendly %(app)s homeserver <noreply@example.com>",
+                    app_name: "my_branded_matrix_server",
+                },
+            });
+            await use(container);
+        },
+        { scope: "worker" },
+    ],
 };

playwright/plugins/homeserver/synapse/legacyOAuthHomeserver.ts

@@ -6,43 +6,56 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */
 
-import { Fixtures } from "@playwright/test";
 import { TestContainers } from "testcontainers";
 
-import { Services } from "../../../services.ts";
 import { OAuthServer } from "../../oauth_server";
+import { Fixtures } from "../../../element-web-test.ts";
 
-export const legacyOAuthHomeserver: Fixtures<Services, {}, Services> = {
-    _homeserver: async ({ _homeserver: container }, use) => {
-        const server = new OAuthServer();
-        const port = server.start();
-
-        await TestContainers.exposeHostPorts(port);
-        container.withConfig({
-            oidc_providers: [
-                {
-                    idp_id: "test",
-                    idp_name: "OAuth test",
-                    issuer: `http://localhost:${port}/oauth`,
-                    authorization_endpoint: `http://localhost:${port}/oauth/auth.html`,
-                    // the token endpoint receives requests from synapse,
-                    // rather than the webapp, so needs to escape the docker container.
-                    token_endpoint: `http://host.testcontainers.internal:${port}/oauth/token`,
-                    userinfo_endpoint: `http://host.testcontainers.internal:${port}/oauth/userinfo`,
-                    client_id: "synapse",
-                    discover: false,
-                    scopes: ["profile"],
-                    skip_verification: true,
-                    client_auth_method: "none",
-                    user_mapping_provider: {
-                        config: {
-                            display_name_template: "{{ user.name }}",
+export const legacyOAuthHomeserver: Fixtures = {
+    oAuthServer: [
+        // eslint-disable-next-line no-empty-pattern
+        async ({}, use) => {
+            const server = new OAuthServer();
+            await use(server);
+            server.stop();
+        },
+        { scope: "worker" },
+    ],
+    context: async ({ context, oAuthServer }, use, testInfo) => {
+        oAuthServer.onTestStarted(testInfo);
+        await use(context);
+    },
+    _homeserver: [
+        async ({ oAuthServer, _homeserver: homeserver }, use) => {
+            const port = oAuthServer.start();
+            await TestContainers.exposeHostPorts(port);
+            homeserver.withConfig({
+                oidc_providers: [
+                    {
+                        idp_id: "test",
+                        idp_name: "OAuth test",
+                        issuer: `http://localhost:${port}/oauth`,
+                        authorization_endpoint: `http://localhost:${port}/oauth/auth.html`,
+                        // the token endpoint receives requests from synapse,
+                        // rather than the webapp, so needs to escape the docker container.
+                        token_endpoint: `http://host.testcontainers.internal:${port}/oauth/token`,
+                        userinfo_endpoint: `http://host.testcontainers.internal:${port}/oauth/userinfo`,
+                        client_id: "synapse",
+                        discover: false,
+                        scopes: ["profile"],
+                        skip_verification: true,
+                        client_auth_method: "none",
+                        user_mapping_provider: {
+                            config: {
+                                display_name_template: "{{ user.name }}",
+                            },
                         },
                     },
-                },
-            ],
-        });
-        await use(container);
-        server.stop();
-    },
+                ],
+            });
+
+            await use(homeserver);
+        },
+        { scope: "worker" },
+    ],
 };

playwright/plugins/homeserver/synapse/masHomeserver.ts

@@ -6,58 +6,57 @@ SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Com
 Please see LICENSE files in the repository root for full details.
 */
 
-import { Fixtures, PlaywrightTestArgs } from "@playwright/test";
-
-import { Services } from "../../../services.ts";
-import { Fixtures as BaseFixtures } from "../../../element-web-test.ts";
 import { MatrixAuthenticationServiceContainer } from "../../../testcontainers/mas.ts";
+import { Fixtures } from "../../../element-web-test.ts";
 
-type Fixture = PlaywrightTestArgs & Services & BaseFixtures;
-export const masHomeserver: Fixtures<Fixture, {}, Fixture> = {
-    mas: async ({ _homeserver: homeserver, logger, network, postgres, mailhog }, use) => {
-        const config = {
-            clients: [
-                {
-                    client_id: "0000000000000000000SYNAPSE",
-                    client_auth_method: "client_secret_basic",
-                    client_secret: "SomeRandomSecret",
+export const masHomeserver: Fixtures = {
+    mas: [
+        async ({ _homeserver: homeserver, logger, network, postgres, mailhog }, use) => {
+            const config = {
+                clients: [
+                    {
+                        client_id: "0000000000000000000SYNAPSE",
+                        client_auth_method: "client_secret_basic",
+                        client_secret: "SomeRandomSecret",
+                    },
+                ],
+                matrix: {
+                    homeserver: "localhost",
+                    secret: "AnotherRandomSecret",
+                    endpoint: "http://homeserver:8008",
                 },
-            ],
-            matrix: {
-                homeserver: "localhost",
-                secret: "AnotherRandomSecret",
-                endpoint: "http://homeserver:8008",
-            },
-        };
+            };
 
-        const container = await new MatrixAuthenticationServiceContainer(postgres)
-            .withNetwork(network)
-            .withNetworkAliases("mas")
-            .withLogConsumer(logger.getConsumer("mas"))
-            .withConfig(config)
-            .start();
+            const container = await new MatrixAuthenticationServiceContainer(postgres)
+                .withNetwork(network)
+                .withNetworkAliases("mas")
+                .withLogConsumer(logger.getConsumer("mas"))
+                .withConfig(config)
+                .start();
 
-        homeserver.withConfig({
-            enable_registration: undefined,
-            enable_registration_without_verification: undefined,
-            disable_msisdn_registration: undefined,
-            password_config: undefined,
-            experimental_features: {
-                msc3861: {
-                    enabled: true,
-                    issuer: `http://mas:8080/`,
-                    introspection_endpoint: "http://mas:8080/oauth2/introspect",
-                    client_id: config.clients[0].client_id,
-                    client_auth_method: config.clients[0].client_auth_method,
-                    client_secret: config.clients[0].client_secret,
-                    admin_token: config.matrix.secret,
+            homeserver.withConfig({
+                enable_registration: undefined,
+                enable_registration_without_verification: undefined,
+                disable_msisdn_registration: undefined,
+                password_config: undefined,
+                experimental_features: {
+                    msc3861: {
+                        enabled: true,
+                        issuer: `http://mas:8080/`,
+                        introspection_endpoint: "http://mas:8080/oauth2/introspect",
+                        client_id: config.clients[0].client_id,
+                        client_auth_method: config.clients[0].client_auth_method,
+                        client_secret: config.clients[0].client_secret,
+                        admin_token: config.matrix.secret,
+                    },
                 },
-            },
-        });
+            });
 
-        await use(container);
-        await container.stop();
-    },
+            await use(container);
+            await container.stop();
+        },
+        { scope: "worker" },
+    ],
 
     config: async ({ homeserver, context, mas }, use) => {
         const issuer = `${mas.baseUrl}/`;

playwright/plugins/homeserver/synapse/uiaLongSessionTimeoutHomeserver.ts

@@ -0,0 +1,22 @@
+/*
+Copyright 2024 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { Fixtures } from "../../../element-web-test.ts";
+
+export const uiaLongSessionTimeoutHomeserver: Fixtures = {
+    synapseConfig: [
+        async ({ synapseConfig }, use) => {
+            await use({
+                ...synapseConfig,
+                ui_auth: {
+                    session_timeout: "300s",
+                },
+            });
+        },
+        { scope: "worker" },
+    ],
+};

playwright/plugins/oauth_server/index.ts

@@ -9,12 +9,21 @@ Please see LICENSE files in the repository root for full details.
 import http from "http";
 import express from "express";
 import { AddressInfo } from "net";
+import { TestInfo } from "@playwright/test";
+
+import { randB64Bytes } from "../utils/rand.ts";
 
 export class OAuthServer {
     private server?: http.Server;
+    private sub?: string;
+
+    public onTestStarted(testInfo: TestInfo): void {
+        this.sub = testInfo.testId;
+    }
 
     public start(): number {
         if (this.server) this.stop();
+        const token = randB64Bytes(16);
 
         const app = express();
 
@@ -28,7 +37,7 @@ export class OAuthServer {
             const code = req.body.code;
             if (code === "valid_auth_code") {
                 res.send({
-                    access_token: "oauth_access_token",
+                    access_token: token,
                     token_type: "Bearer",
                     expires_in: "3600",
                 });
@@ -43,7 +52,7 @@ export class OAuthServer {
 
             // return an OAuth2 user info object
             res.send({
-                sub: "alice",
+                sub: this.sub,
                 name: "Alice",
             });
         });

playwright/plugins/utils/api.ts

@@ -0,0 +1,76 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { APIRequestContext } from "@playwright/test";
+
+import { Credentials } from "../homeserver";
+
+export type Verb = "GET" | "POST" | "PUT" | "DELETE";
+
+export class Api {
+    private _request?: APIRequestContext;
+
+    constructor(private readonly baseUrl: string) {}
+
+    public setRequest(request: APIRequestContext): void {
+        this._request = request;
+    }
+
+    public async request<R extends {}>(verb: "GET", path: string, token?: string, data?: never): Promise<R>;
+    public async request<R extends {}>(verb: Verb, path: string, token?: string, data?: object): Promise<R>;
+    public async request<R extends {}>(verb: Verb, path: string, token?: string, data?: object): Promise<R> {
+        const url = `${this.baseUrl}${path}`;
+        const res = await this._request.fetch(url, {
+            data,
+            method: verb,
+            headers: token
+                ? {
+                      Authorization: `Bearer ${token}`,
+                  }
+                : undefined,
+        });
+
+        if (!res.ok()) {
+            throw new Error(
+                `Request to ${url} failed with status ${res.status()}: ${JSON.stringify(await res.json())}`,
+            );
+        }
+
+        return res.json();
+    }
+}
+
+export class ClientServerApi extends Api {
+    constructor(baseUrl: string) {
+        super(`${baseUrl}/_matrix/client`);
+    }
+
+    public async loginUser(userId: string, password: string): Promise<Credentials> {
+        const json = await this.request<{
+            access_token: string;
+            user_id: string;
+            device_id: string;
+            home_server: string;
+        }>("POST", "/v3/login", undefined, {
+            type: "m.login.password",
+            identifier: {
+                type: "m.id.user",
+                user: userId,
+            },
+            password: password,
+        });
+
+        return {
+            password,
+            accessToken: json.access_token,
+            userId: json.user_id,
+            deviceId: json.device_id,
+            homeServer: json.home_server || json.user_id.split(":").slice(1).join(":"),
+            username: userId.slice(1).split(":")[0],
+        };
+    }
+}

playwright/services.ts

@@ -7,103 +7,141 @@ Please see LICENSE files in the repository root for full details.
 
 import { test as base } from "@playwright/test";
 import mailhog from "mailhog";
-import { GenericContainer, Network, StartedNetwork, StartedTestContainer, Wait } from "testcontainers";
+import { Network, StartedNetwork } from "testcontainers";
 import { PostgreSqlContainer, StartedPostgreSqlContainer } from "@testcontainers/postgresql";
 
-import { SynapseConfigOptions, SynapseContainer } from "./testcontainers/synapse.ts";
-import { ContainerLogger } from "./testcontainers/utils.ts";
+import { SynapseConfig, SynapseContainer } from "./testcontainers/synapse.ts";
+import { Logger } from "./logger.ts";
 import { StartedMatrixAuthenticationServiceContainer } from "./testcontainers/mas.ts";
 import { HomeserverContainer, StartedHomeserverContainer } from "./testcontainers/HomeserverContainer.ts";
+import { MailhogContainer, StartedMailhogContainer } from "./testcontainers/mailhog.ts";
+import { OAuthServer } from "./plugins/oauth_server";
+
+export interface TestFixtures {
+    mailhogClient: mailhog.API;
+}
 
 export interface Services {
-    logger: ContainerLogger;
+    logger: Logger;
 
     network: StartedNetwork;
     postgres: StartedPostgreSqlContainer;
+    mailhog: StartedMailhogContainer;
 
-    mailhog: StartedTestContainer;
-    mailhogClient: mailhog.API;
-
-    synapseConfigOptions: SynapseConfigOptions;
+    synapseConfig: SynapseConfig;
     _homeserver: HomeserverContainer<any>;
     homeserver: StartedHomeserverContainer;
+    // Set in masHomeserver only
     mas?: StartedMatrixAuthenticationServiceContainer;
+    // Set in legacyOAuthHomeserver only
+    oAuthServer?: OAuthServer;
 }
 
-export const test = base.extend<Services>({
-    // eslint-disable-next-line no-empty-pattern
-    logger: async ({}, use, testInfo) => {
-        const logger = new ContainerLogger();
-        await use(logger);
-        await logger.testFinished(testInfo);
-    },
-    // eslint-disable-next-line no-empty-pattern
-    network: async ({}, use) => {
-        const network = await new Network().start();
-        await use(network);
-        await network.stop();
-    },
-    postgres: async ({ logger, network }, use) => {
-        const container = await new PostgreSqlContainer()
-            .withNetwork(network)
-            .withNetworkAliases("postgres")
-            .withLogConsumer(logger.getConsumer("postgres"))
-            .withTmpFs({
-                "/dev/shm/pgdata/data": "",
-            })
-            .withEnvironment({
-                PG_DATA: "/dev/shm/pgdata/data",
-            })
-            .withCommand([
-                "-c",
-                "shared_buffers=128MB",
-                "-c",
-                `fsync=off`,
-                "-c",
-                `synchronous_commit=off`,
-                "-c",
-                "full_page_writes=off",
-            ])
-            .start();
-        await use(container);
-        await container.stop();
-    },
+export interface Options {}
 
-    mailhog: async ({ logger, network }, use) => {
-        const container = await new GenericContainer("mailhog/mailhog:latest")
-            .withNetwork(network)
-            .withNetworkAliases("mailhog")
-            .withExposedPorts(8025)
-            .withLogConsumer(logger.getConsumer("mailhog"))
-            .withWaitStrategy(Wait.forListeningPorts())
-            .start();
-        await use(container);
-        await container.stop();
-    },
+export const test = base.extend<TestFixtures, Services & Options>({
+    logger: [
+        // eslint-disable-next-line no-empty-pattern
+        async ({}, use) => {
+            const logger = new Logger();
+            await use(logger);
+        },
+        { scope: "worker" },
+    ],
+    network: [
+        // eslint-disable-next-line no-empty-pattern
+        async ({}, use) => {
+            const network = await new Network().start();
+            await use(network);
+            await network.stop();
+        },
+        { scope: "worker" },
+    ],
+    postgres: [
+        async ({ logger, network }, use) => {
+            const container = await new PostgreSqlContainer()
+                .withNetwork(network)
+                .withNetworkAliases("postgres")
+                .withLogConsumer(logger.getConsumer("postgres"))
+                .withTmpFs({
+                    "/dev/shm/pgdata/data": "",
+                })
+                .withEnvironment({
+                    PG_DATA: "/dev/shm/pgdata/data",
+                })
+                .withCommand([
+                    "-c",
+                    "shared_buffers=128MB",
+                    "-c",
+                    `fsync=off`,
+                    "-c",
+                    `synchronous_commit=off`,
+                    "-c",
+                    "full_page_writes=off",
+                ])
+                .start();
+            await use(container);
+            await container.stop();
+        },
+        { scope: "worker" },
+    ],
+
+    mailhog: [
+        async ({ logger, network }, use) => {
+            const container = await new MailhogContainer()
+                .withNetwork(network)
+                .withNetworkAliases("mailhog")
+                .withLogConsumer(logger.getConsumer("mailhog"))
+                .start();
+            await use(container);
+            await container.stop();
+        },
+        { scope: "worker" },
+    ],
     mailhogClient: async ({ mailhog: container }, use) => {
-        await use(mailhog({ host: container.getHost(), port: container.getMappedPort(8025) }));
+        await container.client.deleteAll();
+        await use(container.client);
     },
 
-    synapseConfigOptions: [{}, { option: true }],
-    _homeserver: async ({ request }, use) => {
-        const container = new SynapseContainer(request);
-        await use(container);
-    },
-    homeserver: async ({ logger, network, _homeserver: homeserver, synapseConfigOptions, mas }, use) => {
-        const container = await homeserver
-            .withNetwork(network)
-            .withNetworkAliases("homeserver")
-            .withLogConsumer(logger.getConsumer("synapse"))
-            .withConfig(synapseConfigOptions)
-            .start();
+    synapseConfig: [{}, { scope: "worker" }],
+    _homeserver: [
+        // eslint-disable-next-line no-empty-pattern
+        async ({}, use) => {
+            const container = new SynapseContainer();
+            await use(container);
+        },
+        { scope: "worker" },
+    ],
+    homeserver: [
+        async ({ logger, network, _homeserver: homeserver, synapseConfig, mas }, use) => {
+            const container = await homeserver
+                .withNetwork(network)
+                .withNetworkAliases("homeserver")
+                .withLogConsumer(logger.getConsumer("synapse"))
+                .withConfig(synapseConfig)
+                .withMatrixAuthenticationService(mas)
+                .start();
 
-        await use(container);
-        await container.stop();
-    },
-    // eslint-disable-next-line no-empty-pattern
-    mas: async ({}, use) => {
-        // we stub the mas fixture to allow `homeserver` to depend on it to ensure
-        // when it is specified by `masHomeserver` it is started before the homeserver
-        await use(undefined);
+            await use(container);
+            await container.stop();
+        },
+        { scope: "worker" },
+    ],
+    mas: [
+        // eslint-disable-next-line no-empty-pattern
+        async ({}, use) => {
+            // we stub the mas fixture to allow `homeserver` to depend on it to ensure
+            // when it is specified by `masHomeserver` it is started before the homeserver
+            await use(undefined);
+        },
+        { scope: "worker" },
+    ],
+
+    context: async ({ logger, context, request, homeserver, mailhogClient }, use, testInfo) => {
+        homeserver.setRequest(request);
+        await logger.onTestStarted(context);
+        await use(context);
+        await logger.onTestFinished(testInfo);
+        await homeserver.onTestFinished(testInfo);
     },
 });

playwright/stale-screenshot-reporter.ts

@@ -20,9 +20,13 @@ const snapshotRoot = path.join(__dirname, "snapshots");
 
 class StaleScreenshotReporter implements Reporter {
     private screenshots = new Set<string>();
+    private failing = false;
     private success = true;
 
     public onTestEnd(test: TestCase): void {
+        if (!test.ok()) {
+            this.failing = true;
+        }
         for (const annotation of test.annotations) {
             if (annotation.type === "_screenshot") {
                 this.screenshots.add(annotation.description);
@@ -39,6 +43,7 @@ class StaleScreenshotReporter implements Reporter {
     }
 
     public async onExit(): Promise<void> {
+        if (this.failing) return;
         const screenshotFiles = new Set(await glob(`**/*.png`, { cwd: snapshotRoot }));
         for (const screenshot of screenshotFiles) {
             if (screenshot.split("-").at(-1) !== "linux.png") {