Run Playwright tests on Firefox & "Safari" nightly (#28757)

* Run Playwright tests on Firefox

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Update playwright.config.ts

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Update end-to-end-tests.yaml

* Iterate

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Finalise

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* Documentation

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

* typo

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>

---------

Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
This commit is contained in:
Michael Telatynski
2024-12-19 23:42:09 +00:00
committed by GitHub
parent be181d2c79
commit 5448de5dd6
23 changed files with 266 additions and 161 deletions

View File

@@ -13,7 +13,7 @@ import { SettingLevel } from "../../../src/settings/SettingLevel";
import { Layout } from "../../../src/settings/enums/Layout";
import { ElementAppPage } from "../../pages/ElementAppPage";
test.describe("Audio player", () => {
test.describe("Audio player", { tag: ["@no-firefox", "@no-webkit"] }, () => {
test.use({
displayName: "Hanako",
});

View File

@@ -51,90 +51,94 @@ test.describe("Backups", () => {
displayName: "Hanako",
});
test("Create, delete and recreate a keys backup", async ({ page, user, app }, workerInfo) => {
// Create a backup
const securityTab = await app.settings.openUserSettings("Security & Privacy");
test(
"Create, delete and recreate a keys backup",
{ tag: "@no-webkit" },
async ({ page, user, app }, workerInfo) => {
// Create a backup
const securityTab = await app.settings.openUserSettings("Security & Privacy");
await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible();
await securityTab.getByRole("button", { name: "Set up", exact: true }).click();
await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible();
await securityTab.getByRole("button", { name: "Set up", exact: true }).click();
const currentDialogLocator = page.locator(".mx_Dialog");
const currentDialogLocator = page.locator(".mx_Dialog");
// It's the first time and secure storage is not set up, so it will create one
await expect(currentDialogLocator.getByRole("heading", { name: "Set up Secure Backup" })).toBeVisible();
await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click();
await expect(currentDialogLocator.getByRole("heading", { name: "Save your Security Key" })).toBeVisible();
await currentDialogLocator.getByRole("button", { name: "Copy", exact: true }).click();
// copy the recovery key to use it later
const securityKey = await app.getClipboard();
await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click();
// It's the first time and secure storage is not set up, so it will create one
await expect(currentDialogLocator.getByRole("heading", { name: "Set up Secure Backup" })).toBeVisible();
await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click();
await expect(currentDialogLocator.getByRole("heading", { name: "Save your Security Key" })).toBeVisible();
await currentDialogLocator.getByRole("button", { name: "Copy", exact: true }).click();
// copy the recovery key to use it later
const securityKey = await app.getClipboard();
await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click();
await expect(currentDialogLocator.getByRole("heading", { name: "Secure Backup successful" })).toBeVisible();
await currentDialogLocator.getByRole("button", { name: "Done", exact: true }).click();
await expect(currentDialogLocator.getByRole("heading", { name: "Secure Backup successful" })).toBeVisible();
await currentDialogLocator.getByRole("button", { name: "Done", exact: true }).click();
// Open the settings again
await app.settings.openUserSettings("Security & Privacy");
await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible();
// Open the settings again
await app.settings.openUserSettings("Security & Privacy");
await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible();
// expand the advanced section to see the active version in the reports
await page
.locator(".mx_Dialog .mx_SettingsSubsection_content details .mx_SecureBackupPanel_advanced")
.locator("..")
.click();
// expand the advanced section to see the active version in the reports
await page
.locator(".mx_Dialog .mx_SettingsSubsection_content details .mx_SecureBackupPanel_advanced")
.locator("..")
.click();
await expectBackupVersionToBe(page, "1");
await expectBackupVersionToBe(page, "1");
await securityTab.getByRole("button", { name: "Delete Backup", exact: true }).click();
await expect(currentDialogLocator.getByRole("heading", { name: "Delete Backup" })).toBeVisible();
// Delete it
await currentDialogLocator.getByTestId("dialog-primary-button").click(); // Click "Delete Backup"
await securityTab.getByRole("button", { name: "Delete Backup", exact: true }).click();
await expect(currentDialogLocator.getByRole("heading", { name: "Delete Backup" })).toBeVisible();
// Delete it
await currentDialogLocator.getByTestId("dialog-primary-button").click(); // Click "Delete Backup"
// Create another
await securityTab.getByRole("button", { name: "Set up", exact: true }).click();
await expect(currentDialogLocator.getByRole("heading", { name: "Security Key" })).toBeVisible();
await currentDialogLocator.getByLabel("Security Key").fill(securityKey);
await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click();
// Create another
await securityTab.getByRole("button", { name: "Set up", exact: true }).click();
await expect(currentDialogLocator.getByRole("heading", { name: "Security Key" })).toBeVisible();
await currentDialogLocator.getByLabel("Security Key").fill(securityKey);
await currentDialogLocator.getByRole("button", { name: "Continue", exact: true }).click();
// Should be successful
await expect(currentDialogLocator.getByRole("heading", { name: "Success!" })).toBeVisible();
await currentDialogLocator.getByRole("button", { name: "OK", exact: true }).click();
// Should be successful
await expect(currentDialogLocator.getByRole("heading", { name: "Success!" })).toBeVisible();
await currentDialogLocator.getByRole("button", { name: "OK", exact: true }).click();
// Open the settings again
await app.settings.openUserSettings("Security & Privacy");
await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible();
// Open the settings again
await app.settings.openUserSettings("Security & Privacy");
await expect(securityTab.getByRole("heading", { name: "Secure Backup" })).toBeVisible();
// expand the advanced section to see the active version in the reports
await page
.locator(".mx_Dialog .mx_SettingsSubsection_content details .mx_SecureBackupPanel_advanced")
.locator("..")
.click();
// expand the advanced section to see the active version in the reports
await page
.locator(".mx_Dialog .mx_SettingsSubsection_content details .mx_SecureBackupPanel_advanced")
.locator("..")
.click();
await expectBackupVersionToBe(page, "2");
await expectBackupVersionToBe(page, "2");
// ==
// Ensure that if you don't have the secret storage passphrase the backup won't be created
// ==
// ==
// Ensure that if you don't have the secret storage passphrase the backup won't be created
// ==
// First delete version 2
await securityTab.getByRole("button", { name: "Delete Backup", exact: true }).click();
await expect(currentDialogLocator.getByRole("heading", { name: "Delete Backup" })).toBeVisible();
// Click "Delete Backup"
await currentDialogLocator.getByTestId("dialog-primary-button").click();
// First delete version 2
await securityTab.getByRole("button", { name: "Delete Backup", exact: true }).click();
await expect(currentDialogLocator.getByRole("heading", { name: "Delete Backup" })).toBeVisible();
// Click "Delete Backup"
await currentDialogLocator.getByTestId("dialog-primary-button").click();
// Try to create another
await securityTab.getByRole("button", { name: "Set up", exact: true }).click();
await expect(currentDialogLocator.getByRole("heading", { name: "Security Key" })).toBeVisible();
// But cancel the security key dialog, to simulate not having the secret storage passphrase
await currentDialogLocator.getByTestId("dialog-cancel-button").click();
// Try to create another
await securityTab.getByRole("button", { name: "Set up", exact: true }).click();
await expect(currentDialogLocator.getByRole("heading", { name: "Security Key" })).toBeVisible();
// But cancel the security key dialog, to simulate not having the secret storage passphrase
await currentDialogLocator.getByTestId("dialog-cancel-button").click();
await expect(currentDialogLocator.getByRole("heading", { name: "Starting backup…" })).toBeVisible();
// check that it failed
await expect(currentDialogLocator.getByText("Unable to create key backup")).toBeVisible();
// cancel
await currentDialogLocator.getByTestId("dialog-cancel-button").click();
await expect(currentDialogLocator.getByRole("heading", { name: "Starting backup…" })).toBeVisible();
// check that it failed
await expect(currentDialogLocator.getByText("Unable to create key backup")).toBeVisible();
// cancel
await currentDialogLocator.getByTestId("dialog-cancel-button").click();
// go back to the settings to check that no backup was created (the setup button should still be there)
await app.settings.openUserSettings("Security & Privacy");
await expect(securityTab.getByRole("button", { name: "Set up", exact: true })).toBeVisible();
});
// go back to the settings to check that no backup was created (the setup button should still be there)
await app.settings.openUserSettings("Security & Privacy");
await expect(securityTab.getByRole("button", { name: "Set up", exact: true })).toBeVisible();
},
);
});

View File

@@ -21,7 +21,7 @@ import {
} from "./utils";
import { Bot } from "../../pages/bot";
test.describe("Device verification", () => {
test.describe("Device verification", { tag: "@no-webkit" }, () => {
let aliceBotClient: Bot;
/** The backup version that was set up by the bot client. */

View File

@@ -25,7 +25,7 @@ const test = base.extend<Fixtures>({
},
});
test.describe("migration", function () {
test.describe("migration", { tag: "@no-webkit" }, function () {
test.use({ displayName: "Alice" });
test("Should support migration from legacy crypto", async ({ context, user, page }, workerInfo) => {

View File

@@ -10,7 +10,8 @@ import { Locator, Page } from "@playwright/test";
import { test, expect } from "../../element-web-test";
test.describe("Location sharing", () => {
// Firefox headless lacks WebGL support https://bugzilla.mozilla.org/show_bug.cgi?id=1375585
test.describe("Location sharing", { tag: "@no-firefox" }, () => {
const selectLocationShareTypeOption = (page: Page, shareType: string): Locator => {
return page.getByTestId(`share-location-option-${shareType}`);
};

View File

@@ -9,7 +9,7 @@ Please see LICENSE files in the repository root for full details.
import { test, expect, registerAccountMas } from ".";
import { isDendrite } from "../../plugins/homeserver/dendrite";
test.describe("OIDC Aware", () => {
test.describe("OIDC Aware", { tag: ["@no-firefox", "@no-webkit"] }, () => {
test.skip(isDendrite, "does not yet support MAS");
test.slow(); // trace recording takes a while here

View File

@@ -10,7 +10,7 @@ import { test, expect, registerAccountMas } from ".";
import { isDendrite } from "../../plugins/homeserver/dendrite";
import { ElementAppPage } from "../../pages/ElementAppPage.ts";
test.describe("OIDC Native", () => {
test.describe("OIDC Native", { tag: ["@no-firefox", "@no-webkit"] }, () => {
test.skip(isDendrite, "does not yet support MAS");
test.slow(); // trace recording takes a while here

View File

@@ -9,7 +9,9 @@ Please see LICENSE files in the repository root for full details.
import { test, expect } from "../../element-web-test";
test.describe("Registration", () => {
test.use({ startHomeserverOpts: "consent" });
test.use({
startHomeserverOpts: "consent",
});
test.beforeEach(async ({ page }) => {
await page.goto("/#/register");

View File

@@ -39,7 +39,7 @@ test.describe("FilePanel", () => {
await expect(page.locator(".mx_FilePanel")).toBeVisible();
});
test.describe("render", () => {
test.describe("render", { tag: ["@no-firefox", "@no-webkit"] }, () => {
test("should render empty state", { tag: "@screenshot" }, async ({ page }) => {
// Wait until the information about the empty state is rendered
await expect(page.locator(".mx_EmptyState")).toBeVisible();

View File

@@ -15,37 +15,43 @@ test.describe("Room Directory", () => {
botCreateOpts: { displayName: "Paul" },
});
test("should allow admin to add alias & publish room to directory", async ({ page, app, user, bot }) => {
const roomId = await app.client.createRoom({
name: "Gaming",
preset: "public_chat" as Preset,
});
test(
"should allow admin to add alias & publish room to directory",
{ tag: "@no-webkit" },
async ({ page, app, user, bot }) => {
const roomId = await app.client.createRoom({
name: "Gaming",
preset: "public_chat" as Preset,
});
await app.viewRoomByName("Gaming");
await app.settings.openRoomSettings();
await app.viewRoomByName("Gaming");
await app.settings.openRoomSettings();
// First add a local address `gaming`
const localAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Local Addresses" });
await localAddresses.getByRole("textbox").fill("gaming");
await localAddresses.getByRole("button", { name: "Add" }).click();
await expect(localAddresses.getByText("#gaming:localhost")).toHaveClass("mx_EditableItem_item");
// First add a local address `gaming`
const localAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Local Addresses" });
await localAddresses.getByRole("textbox").fill("gaming");
await localAddresses.getByRole("button", { name: "Add" }).click();
await expect(localAddresses.getByText("#gaming:localhost")).toHaveClass("mx_EditableItem_item");
// Publish into the public rooms directory
const publishedAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Published Addresses" });
await expect(publishedAddresses.locator("#canonicalAlias")).toHaveValue("#gaming:localhost");
const checkbox = publishedAddresses
.locator(".mx_SettingsFlag", { hasText: "Publish this room to the public in localhost's room directory?" })
.getByRole("switch");
await checkbox.check();
await expect(checkbox).toBeChecked();
// Publish into the public rooms directory
const publishedAddresses = page.locator(".mx_SettingsFieldset", { hasText: "Published Addresses" });
await expect(publishedAddresses.locator("#canonicalAlias")).toHaveValue("#gaming:localhost");
const checkbox = publishedAddresses
.locator(".mx_SettingsFlag", {
hasText: "Publish this room to the public in localhost's room directory?",
})
.getByRole("switch");
await checkbox.check();
await expect(checkbox).toBeChecked();
await app.closeDialog();
await app.closeDialog();
const resp = await bot.publicRooms({});
expect(resp.total_room_count_estimate).toEqual(1);
expect(resp.chunk).toHaveLength(1);
expect(resp.chunk[0].room_id).toEqual(roomId);
});
const resp = await bot.publicRooms({});
expect(resp.total_room_count_estimate).toEqual(1);
expect(resp.chunk).toHaveLength(1);
expect(resp.chunk[0].room_id).toEqual(roomId);
},
);
test(
"should allow finding published rooms in directory",

View File

@@ -36,7 +36,7 @@ test.describe("General room settings tab", () => {
await expect(settings.getByText("Show more")).toBeVisible();
});
test("long address should not cause dialog to overflow", async ({ page, app }) => {
test("long address should not cause dialog to overflow", { tag: "@no-webkit" }, async ({ page, app }) => {
const settings = await app.settings.openRoomSettings("General");
// 1. Set the room-address to be a really long string
const longString = "abcasdhjasjhdaj1jh1asdhasjdhajsdhjavhjksd".repeat(4);

View File

@@ -31,7 +31,7 @@ test.describe("Preferences user settings tab", () => {
await expect(tab).toMatchScreenshot("Preferences-user-settings-tab-should-be-rendered-properly-1.png");
});
test("should be able to change the app language", async ({ uut, user }) => {
test("should be able to change the app language", { tag: ["@no-firefox", "@no-webkit"] }, async ({ uut, user }) => {
// Check language and region setting dropdown
const languageInput = uut.getByRole("button", { name: "Language Dropdown" });
await languageInput.scrollIntoViewIfNeeded();

View File

@@ -55,38 +55,44 @@ test.describe("Spaces", () => {
botCreateOpts: { displayName: "BotBob" },
});
test("should allow user to create public space", { tag: "@screenshot" }, async ({ page, app, user }) => {
const contextMenu = await openSpaceCreateMenu(page);
await expect(contextMenu).toMatchScreenshot("space-create-menu.png");
test(
"should allow user to create public space",
{ tag: ["@screenshot", "@no-webkit"] },
async ({ page, app, user }) => {
const contextMenu = await openSpaceCreateMenu(page);
await expect(contextMenu).toMatchScreenshot("space-create-menu.png");
await contextMenu.getByRole("button", { name: /Public/ }).click();
await contextMenu.getByRole("button", { name: /Public/ }).click();
await contextMenu
.locator('.mx_SpaceBasicSettings_avatarContainer input[type="file"]')
.setInputFiles("playwright/sample-files/riot.png");
await contextMenu.getByRole("textbox", { name: "Name" }).fill("Let's have a Riot");
await expect(contextMenu.getByRole("textbox", { name: "Address" })).toHaveValue("lets-have-a-riot");
await contextMenu.getByRole("textbox", { name: "Description" }).fill("This is a space to reminisce Riot.im!");
await contextMenu.getByRole("button", { name: "Create" }).click();
await contextMenu
.locator('.mx_SpaceBasicSettings_avatarContainer input[type="file"]')
.setInputFiles("playwright/sample-files/riot.png");
await contextMenu.getByRole("textbox", { name: "Name" }).fill("Let's have a Riot");
await expect(contextMenu.getByRole("textbox", { name: "Address" })).toHaveValue("lets-have-a-riot");
await contextMenu
.getByRole("textbox", { name: "Description" })
.fill("This is a space to reminisce Riot.im!");
await contextMenu.getByRole("button", { name: "Create" }).click();
// Create the default General & Random rooms, as well as a custom "Jokes" room
await expect(page.getByPlaceholder("General")).toBeVisible();
await expect(page.getByPlaceholder("Random")).toBeVisible();
await page.getByPlaceholder("Support").fill("Jokes");
await page.getByRole("button", { name: "Continue" }).click();
// Create the default General & Random rooms, as well as a custom "Jokes" room
await expect(page.getByPlaceholder("General")).toBeVisible();
await expect(page.getByPlaceholder("Random")).toBeVisible();
await page.getByPlaceholder("Support").fill("Jokes");
await page.getByRole("button", { name: "Continue" }).click();
// Copy matrix.to link
await page.getByRole("button", { name: "Share invite link" }).click();
expect(await app.getClipboardText()).toEqual("https://matrix.to/#/#lets-have-a-riot:localhost");
// Copy matrix.to link
await page.getByRole("button", { name: "Share invite link" }).click();
expect(await app.getClipboardText()).toEqual("https://matrix.to/#/#lets-have-a-riot:localhost");
// Go to space home
await page.getByRole("button", { name: "Go to my first room" }).click();
// Go to space home
await page.getByRole("button", { name: "Go to my first room" }).click();
// Assert rooms exist in the room list
await expect(page.getByRole("treeitem", { name: "General" })).toBeVisible();
await expect(page.getByRole("treeitem", { name: "Random" })).toBeVisible();
await expect(page.getByRole("treeitem", { name: "Jokes" })).toBeVisible();
});
// Assert rooms exist in the room list
await expect(page.getByRole("treeitem", { name: "General" })).toBeVisible();
await expect(page.getByRole("treeitem", { name: "Random" })).toBeVisible();
await expect(page.getByRole("treeitem", { name: "Jokes" })).toBeVisible();
},
);
test("should allow user to create private space", { tag: "@screenshot" }, async ({ page, app, user }) => {
const menu = await openSpaceCreateMenu(page);
@@ -157,7 +163,7 @@ test.describe("Spaces", () => {
).toBeVisible();
});
test("should allow user to invite another to a space", async ({ page, app, user, bot }) => {
test("should allow user to invite another to a space", { tag: "@no-webkit" }, async ({ page, app, user, bot }) => {
await app.client.createSpace({
visibility: "public" as any,
room_alias_name: "space",

View File

@@ -9,7 +9,7 @@
import { expect, test } from ".";
import { CommandOrControl } from "../../utils";
test.describe("Threads Activity Centre", () => {
test.describe("Threads Activity Centre", { tag: "@no-firefox" }, () => {
test.use({
displayName: "Alice",
botCreateOpts: { displayName: "Other User" },

View File

@@ -324,7 +324,7 @@ test.describe("Threads", () => {
});
});
test("can send voice messages", async ({ page, app, user }) => {
test("can send voice messages", { tag: ["@no-firefox", "@no-webkit"] }, async ({ page, app, user }) => {
// Increase right-panel size, so that voice messages fit
await page.evaluate(() => {
window.localStorage.setItem("mx_rhs_size", "600");
@@ -353,7 +353,7 @@ test.describe("Threads", () => {
test(
"should send location and reply to the location on ThreadView",
{ tag: "@screenshot" },
{ tag: ["@screenshot", "@no-firefox"] },
async ({ page, app, bot }) => {
const roomId = await app.client.createRoom({});
await app.client.inviteUser(roomId, bot.credentials.userId);

View File

@@ -90,7 +90,7 @@ test.describe("Timeline", () => {
let oldAvatarUrl: string;
let newAvatarUrl: string;
test.describe("useOnlyCurrentProfiles", () => {
test.describe("useOnlyCurrentProfiles", { tag: ["@no-firefox", "@no-webkit"] }, () => {
test.beforeEach(async ({ app, user }) => {
({ content_uri: oldAvatarUrl } = await app.client.uploadContent(OLD_AVATAR, { type: "image/png" }));
await app.client.setAvatarUrl(oldAvatarUrl);
@@ -876,7 +876,7 @@ test.describe("Timeline", () => {
});
});
test.describe("message sending", () => {
test.describe("message sending", { tag: ["@no-firefox", "@no-webkit"] }, () => {
const MESSAGE = "Hello world";
const reply = "Reply";
const viewRoomSendMessageAndSetupReply = async (page: Page, app: ElementAppPage, roomId: string) => {
@@ -914,7 +914,6 @@ test.describe("Timeline", () => {
});
test("can reply with a voice message", async ({ page, app, room, context }) => {
await context.grantPermissions(["microphone"]);
await viewRoomSendMessageAndSetupReply(page, app, room.roomId);
const composerOptions = await app.openMessageComposerOptions();

View File

@@ -128,7 +128,7 @@ async function setWidgetAccountData(
});
}
test.describe("Stickers", () => {
test.describe("Stickers", { tag: ["@no-firefox", "@no-webkit"] }, () => {
test.use({
displayName: "Sally",
room: async ({ app }, use) => {