nikrozman/element-web
1
0
Fork 0
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Implement dendrite & pinecone support in Playwright (#11943)

Browse Source 
Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
This commit is contained in:
Michael Telatynski
2023-11-27 14:40:45 +00:00
committed by GitHub
parent a3cf11ac74
commit f7aab0e042
25 changed files with 589 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

198
playwright/plugins/homeserver/synapse/index.ts Normal file
View File

@@ -0,0 +1,198 @@
/*
Copyright 2023 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import * as path from "node:path";
import * as os from "node:os";
import * as crypto from "node:crypto";
import * as fse from "fs-extra";
import { APIRequestContext } from "@playwright/test";
import { getFreePort } from "../../utils/port";
import { Docker } from "../../docker";
import { HomeserverConfig, HomeserverInstance, Homeserver, StartHomeserverOpts, Credentials } from "..";
import { randB64Bytes } from "../../utils/rand";
async function cfgDirFromTemplate(opts: StartHomeserverOpts): Promise<HomeserverConfig> {
const templateDir = path.join(__dirname, "templates", opts.template);
const stats = await fse.stat(templateDir);
if (!stats?.isDirectory) {
throw new Error(`No such template: ${opts.template}`);
}
const tempDir = await fse.mkdtemp(path.join(os.tmpdir(), "react-sdk-synapsedocker-"));
// copy the contents of the template dir, omitting homeserver.yaml as we'll template that
console.log(`Copy ${templateDir} -> ${tempDir}`);
await fse.copy(templateDir, tempDir, { filter: (f) => path.basename(f) !== "homeserver.yaml" });
const registrationSecret = randB64Bytes(16);
const macaroonSecret = randB64Bytes(16);
const formSecret = randB64Bytes(16);
const port = await getFreePort();
const baseUrl = `http://localhost:${port}`;
// now copy homeserver.yaml, applying substitutions
const templateHomeserver = path.join(templateDir, "homeserver.yaml");
const outputHomeserver = path.join(tempDir, "homeserver.yaml");
console.log(`Gen ${templateHomeserver} -> ${outputHomeserver}`);
let hsYaml = await fse.readFile(templateHomeserver, "utf8");
hsYaml = hsYaml.replace(/{{REGISTRATION_SECRET}}/g, registrationSecret);
hsYaml = hsYaml.replace(/{{MACAROON_SECRET_KEY}}/g, macaroonSecret);
hsYaml = hsYaml.replace(/{{FORM_SECRET}}/g, formSecret);
hsYaml = hsYaml.replace(/{{PUBLIC_BASEURL}}/g, baseUrl);
if (opts.oAuthServerPort) {
hsYaml = hsYaml.replace(/{{OAUTH_SERVER_PORT}}/g, opts.oAuthServerPort.toString());
}
hsYaml = hsYaml.replace(/{{HOST_DOCKER_INTERNAL}}/g, await Docker.hostnameOfHost());
if (opts.variables) {
let fetchedHostContainer: Awaited<ReturnType<typeof Docker.hostnameOfHost>> | null = null;
for (const key in opts.variables) {
let value = String(opts.variables[key]);
if (value === "{{HOST_DOCKER_INTERNAL}}") {
if (!fetchedHostContainer) {
fetchedHostContainer = await Docker.hostnameOfHost();
}
value = fetchedHostContainer;
}
hsYaml = hsYaml.replace(new RegExp("%" + key + "%", "g"), value);
}
}
await fse.writeFile(outputHomeserver, hsYaml);
// now generate a signing key (we could use synapse's config generation for
// this, or we could just do this...)
// NB. This assumes the homeserver.yaml specifies the key in this location
const signingKey = randB64Bytes(32);
const outputSigningKey = path.join(tempDir, "localhost.signing.key");
console.log(`Gen -> ${outputSigningKey}`);
await fse.writeFile(outputSigningKey, `ed25519 x ${signingKey}`);
return {
port,
baseUrl,
configDir: tempDir,
registrationSecret,
};
}
export class Synapse implements Homeserver, HomeserverInstance {
protected docker: Docker = new Docker();
public config: HomeserverConfig & { serverId: string };
public constructor(private readonly request: APIRequestContext) {}
/**
* Start a synapse instance: the template must be the name of
* one of the templates in the playwright/plugins/synapsedocker/templates
* directory.
*
* Any value in `opts.variables` that is set to `{{HOST_DOCKER_INTERNAL}}'
* will be replaced with 'host.docker.internal' (if we are on Docker) or
* 'host.containers.internal' if we are on Podman.
*/
public async start(opts: StartHomeserverOpts): Promise<HomeserverInstance> {
if (this.config) await this.stop();
const synCfg = await cfgDirFromTemplate(opts);
console.log(`Starting synapse with config dir ${synCfg.configDir}...`);
const dockerSynapseParams = ["--rm", "-v", `${synCfg.configDir}:/data`, "-p", `${synCfg.port}:8008/tcp`];
if (await Docker.isPodman()) {
// Make host.containers.internal work to allow Synapse to talk to the test OIDC server.
dockerSynapseParams.push("--network");
dockerSynapseParams.push("slirp4netns:allow_host_loopback=true");
} else {
// Make host.docker.internal work to allow Synapse to talk to the test OIDC server.
dockerSynapseParams.push("--add-host");
dockerSynapseParams.push("host.docker.internal:host-gateway");
}
const synapseId = await this.docker.run({
image: "matrixdotorg/synapse:develop",
containerName: `react-sdk-playwright-synapse`,
params: dockerSynapseParams,
cmd: ["run"],
});
console.log(`Started synapse with id ${synapseId} on port ${synCfg.port}.`);
// Await Synapse healthcheck
await this.docker.exec([
"curl",
"--connect-timeout",
"30",
"--retry",
"30",
"--retry-delay",
"1",
"--retry-all-errors",
"--silent",
"http://localhost:8008/health",
]);
this.config = {
...synCfg,
serverId: synapseId,
};
return this;
}
public async stop(): Promise<void> {
if (!this.config) throw new Error("Missing existing synapse instance, did you call stop() before start()?");
const id = this.config.serverId;
const synapseLogsPath = path.join("playwright", "synapselogs", id);
await fse.ensureDir(synapseLogsPath);
await this.docker.persistLogsToFile({
stdoutFile: path.join(synapseLogsPath, "stdout.log"),
stderrFile: path.join(synapseLogsPath, "stderr.log"),
});
await this.docker.stop();
await fse.remove(this.config.configDir);
console.log(`Stopped synapse id ${id}.`);
}
public async registerUser(username: string, password: string, displayName?: string): Promise<Credentials> {
const url = `${this.config.baseUrl}/_synapse/admin/v1/register`;
const { nonce } = await this.request.get(url).then((r) => r.json());
const mac = crypto
.createHmac("sha1", this.config.registrationSecret)
.update(`${nonce}\0${username}\0${password}\0notadmin`)
.digest("hex");
const res = await this.request.post(url, {
data: {
nonce,
username,
password,
mac,
admin: false,
displayname: displayName,
},
});
if (!res.ok()) {
throw await res.json();
}
const data = await res.json();
return {
homeServer: data.home_server,
accessToken: data.access_token,
userId: data.user_id,
deviceId: data.device_id,
password,
};
}
}

3
playwright/plugins/homeserver/synapse/templates/COPYME/README.md Normal file
View File

@@ -0,0 +1,3 @@
# Meta-template for synapse templates
To make another template, you can copy this directory

72
playwright/plugins/homeserver/synapse/templates/COPYME/homeserver.yaml Normal file
View File

@@ -0,0 +1,72 @@
server_name: "localhost"
pid_file: /data/homeserver.pid
# XXX: This won't actually be right: it lets docker allocate an ephemeral port,
# so we have a chicken-and-egg problem
public_baseurl: http://localhost:8008/
# Listener is always port 8008 (configured in the container)
listeners:
- port: 8008
tls: false
bind_addresses: ["::"]
type: http
x_forwarded: true
resources:
- names: [client, federation, consent]
compress: false
# An sqlite in-memory database is fast & automatically wipes each time
database:
name: "sqlite3"
args:
database: ":memory:"
# Needs to be configured to log to the console like a good docker process
log_config: "/data/log.config"
rc_messages_per_second: 10000
rc_message_burst_count: 10000
rc_registration:
per_second: 10000
burst_count: 10000
rc_login:
address:
per_second: 10000
burst_count: 10000
account:
per_second: 10000
burst_count: 10000
failed_attempts:
per_second: 10000
burst_count: 10000
media_store_path: "/data/media_store"
uploads_path: "/data/uploads"
enable_registration: true
enable_registration_without_verification: true
disable_msisdn_registration: false
# These placeholders will be be replaced with values generated at start
registration_shared_secret: "{{REGISTRATION_SECRET}}"
report_stats: false
macaroon_secret_key: "{{MACAROON_SECRET_KEY}}"
form_secret: "{{FORM_SECRET}}"
# Signing key must be here: it will be generated to this file
signing_key_path: "/data/localhost.signing.key"
email:
enable_notifs: false
smtp_host: "localhost"
smtp_port: 25
smtp_user: "exampleusername"
smtp_pass: "examplepassword"
require_transport_security: False
notif_from: "Your Friendly %(app)s homeserver <noreply@example.com>"
app_name: Matrix
notif_template_html: notif_mail.html
notif_template_text: notif_mail.txt
notif_for_new_users: True
client_base_url: "http://localhost/element"
trusted_key_servers:
- server_name: "matrix.org"
suppress_key_server_warning: true

50
playwright/plugins/homeserver/synapse/templates/COPYME/log.config Normal file
View File

@@ -0,0 +1,50 @@
# Log configuration for Synapse.
#
# This is a YAML file containing a standard Python logging configuration
# dictionary. See [1] for details on the valid settings.
#
# Synapse also supports structured logging for machine readable logs which can
# be ingested by ELK stacks. See [2] for details.
#
# [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
# [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html
version: 1
formatters:
precise:
format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
handlers:
# A handler that writes logs to stderr. Unused by default, but can be used
# instead of "buffer" and "file" in the logger handlers.
console:
class: logging.StreamHandler
formatter: precise
loggers:
synapse.storage.SQL:
# beware: increasing this to DEBUG will make synapse log sensitive
# information such as access tokens.
level: INFO
twisted:
# We send the twisted logging directly to the file handler,
# to work around https://github.com/matrix-org/synapse/issues/3471
# when using "buffer" logger. Use "console" to log to stderr instead.
handlers: [console]
propagate: false
root:
level: INFO
# Write logs to the `buffer` handler, which will buffer them together in memory,
# then write them to a file.
#
# Replace "buffer" with "console" to log to stderr instead. (Note that you'll
# also need to update the configuration for the `twisted` logger above, in
# this case.)
#
handlers: [console]
disable_existing_loggers: false

1
playwright/plugins/homeserver/synapse/templates/consent/README.md Normal file
View File

@@ -0,0 +1 @@
A synapse configured with user privacy consent enabled

84
playwright/plugins/homeserver/synapse/templates/consent/homeserver.yaml Normal file
View File

@@ -0,0 +1,84 @@
server_name: "localhost"
pid_file: /data/homeserver.pid
public_baseurl: "{{PUBLIC_BASEURL}}"
listeners:
- port: 8008
tls: false
bind_addresses: ["::"]
type: http
x_forwarded: true
resources:
- names: [client, federation, consent]
compress: false
database:
name: "sqlite3"
args:
database: ":memory:"
log_config: "/data/log.config"
rc_messages_per_second: 10000
rc_message_burst_count: 10000
rc_registration:
per_second: 10000
burst_count: 10000
rc_login:
address:
per_second: 10000
burst_count: 10000
account:
per_second: 10000
burst_count: 10000
failed_attempts:
per_second: 10000
burst_count: 10000
media_store_path: "/data/media_store"
uploads_path: "/data/uploads"
enable_registration: true
enable_registration_without_verification: true
disable_msisdn_registration: false
registration_shared_secret: "{{REGISTRATION_SECRET}}"
report_stats: false
macaroon_secret_key: "{{MACAROON_SECRET_KEY}}"
form_secret: "{{FORM_SECRET}}"
signing_key_path: "/data/localhost.signing.key"
email:
enable_notifs: false
smtp_host: "localhost"
smtp_port: 25
smtp_user: "exampleusername"
smtp_pass: "examplepassword"
require_transport_security: False
notif_from: "Your Friendly %(app)s homeserver <noreply@example.com>"
app_name: Matrix
notif_template_html: notif_mail.html
notif_template_text: notif_mail.txt
notif_for_new_users: True
client_base_url: "http://localhost/element"
user_consent:
template_dir: /data/res/templates/privacy
version: 1.0
server_notice_content:
msgtype: m.text
body: >-
To continue using this homeserver you must review and agree to the
terms and conditions at %(consent_uri)s
send_server_notice_to_guests: True
block_events_error: >-
To continue using this homeserver you must review and agree to the
terms and conditions at %(consent_uri)s
require_at_registration: true
server_notices:
system_mxid_localpart: notices
system_mxid_display_name: "Server Notices"
system_mxid_avatar_url: "mxc://localhost:5005/oumMVlgDnLYFaPVkExemNVVZ"
room_name: "Server Notices"
trusted_key_servers:
- server_name: "matrix.org"
suppress_key_server_warning: true

50
playwright/plugins/homeserver/synapse/templates/consent/log.config Normal file
View File

@@ -0,0 +1,50 @@
# Log configuration for Synapse.
#
# This is a YAML file containing a standard Python logging configuration
# dictionary. See [1] for details on the valid settings.
#
# Synapse also supports structured logging for machine readable logs which can
# be ingested by ELK stacks. See [2] for details.
#
# [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
# [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html
version: 1
formatters:
precise:
format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
handlers:
# A handler that writes logs to stderr. Unused by default, but can be used
# instead of "buffer" and "file" in the logger handlers.
console:
class: logging.StreamHandler
formatter: precise
loggers:
synapse.storage.SQL:
# beware: increasing this to DEBUG will make synapse log sensitive
# information such as access tokens.
level: DEBUG
twisted:
# We send the twisted logging directly to the file handler,
# to work around https://github.com/matrix-org/synapse/issues/3471
# when using "buffer" logger. Use "console" to log to stderr instead.
handlers: [console]
propagate: false
root:
level: DEBUG
# Write logs to the `buffer` handler, which will buffer them together in memory,
# then write them to a file.
#
# Replace "buffer" with "console" to log to stderr instead. (Note that you'll
# also need to update the configuration for the `twisted` logger above, in
# this case.)
#
handlers: [console]
disable_existing_loggers: false

19
playwright/plugins/homeserver/synapse/templates/consent/res/templates/privacy/en/1.0.html Normal file
View File

@@ -0,0 +1,19 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>Test Privacy policy</title>
</head>
<body>
{% if has_consented %}
<p>Thank you, you've already accepted the license.</p>
{% else %}
<p>Please accept the license!</p>
<form method="post" action="consent">
<input type="hidden" name="v" value="{{version}}" />
<input type="hidden" name="u" value="{{user}}" />
<input type="hidden" name="h" value="{{userhmac}}" />
<input type="submit" value="Sure thing!" />
</form>
{% endif %}
</body>
</html>

9
playwright/plugins/homeserver/synapse/templates/consent/res/templates/privacy/en/success.html Normal file
View File

@@ -0,0 +1,9 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>Test Privacy policy</title>
</head>
<body>
<p>Danke schoen</p>
</body>
</html>

1
playwright/plugins/homeserver/synapse/templates/default/README.md Normal file
View File

@@ -0,0 +1 @@
A synapse configured with user privacy consent disabled

94
playwright/plugins/homeserver/synapse/templates/default/homeserver.yaml Normal file
View File

@@ -0,0 +1,94 @@
server_name: "localhost"
pid_file: /data/homeserver.pid
public_baseurl: "{{PUBLIC_BASEURL}}"
listeners:
- port: 8008
tls: false
bind_addresses: ["::"]
type: http
x_forwarded: true
resources:
- names: [client]
compress: false
database:
name: "sqlite3"
args:
database: ":memory:"
log_config: "/data/log.config"
rc_messages_per_second: 10000
rc_message_burst_count: 10000
rc_registration:
per_second: 10000
burst_count: 10000
rc_joins:
local:
per_second: 9999
burst_count: 9999
remote:
per_second: 9999
burst_count: 9999
rc_joins_per_room:
per_second: 9999
burst_count: 9999
rc_3pid_validation:
per_second: 1000
burst_count: 1000
rc_invites:
per_room:
per_second: 1000
burst_count: 1000
per_user:
per_second: 1000
burst_count: 1000
rc_login:
address:
per_second: 10000
burst_count: 10000
account:
per_second: 10000
burst_count: 10000
failed_attempts:
per_second: 10000
burst_count: 10000
media_store_path: "/data/media_store"
uploads_path: "/data/uploads"
enable_registration: true
enable_registration_without_verification: true
disable_msisdn_registration: false
registration_shared_secret: "{{REGISTRATION_SECRET}}"
report_stats: false
macaroon_secret_key: "{{MACAROON_SECRET_KEY}}"
form_secret: "{{FORM_SECRET}}"
signing_key_path: "/data/localhost.signing.key"
trusted_key_servers:
- server_name: "matrix.org"
suppress_key_server_warning: true
ui_auth:
session_timeout: "300s"
oidc_providers:
- idp_id: test
idp_name: "OAuth test"
issuer: "http://localhost:{{OAUTH_SERVER_PORT}}/oauth"
authorization_endpoint: "http://localhost:{{OAUTH_SERVER_PORT}}/oauth/auth.html"
# the token endpoint receives requests from synapse, rather than the webapp, so needs to escape the docker container.
# Hence, HOST_DOCKER_INTERNAL rather than localhost. This is set to
# host.docker.internal on Docker and host.containers.internal on Podman.
token_endpoint: "http://{{HOST_DOCKER_INTERNAL}}:{{OAUTH_SERVER_PORT}}/oauth/token"
userinfo_endpoint: "http://{{HOST_DOCKER_INTERNAL}}:{{OAUTH_SERVER_PORT}}/oauth/userinfo"
client_id: "synapse"
discover: false
scopes: ["profile"]
skip_verification: true
user_mapping_provider:
config:
display_name_template: "{{ user.name }}"

50
playwright/plugins/homeserver/synapse/templates/default/log.config Normal file
View File

@@ -0,0 +1,50 @@
# Log configuration for Synapse.
#
# This is a YAML file containing a standard Python logging configuration
# dictionary. See [1] for details on the valid settings.
#
# Synapse also supports structured logging for machine readable logs which can
# be ingested by ELK stacks. See [2] for details.
#
# [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
# [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html
version: 1
formatters:
precise:
format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
handlers:
# A handler that writes logs to stderr. Unused by default, but can be used
# instead of "buffer" and "file" in the logger handlers.
console:
class: logging.StreamHandler
formatter: precise
loggers:
synapse.storage.SQL:
# beware: increasing this to DEBUG will make synapse log sensitive
# information such as access tokens.
level: DEBUG
twisted:
# We send the twisted logging directly to the file handler,
# to work around https://github.com/matrix-org/synapse/issues/3471
# when using "buffer" logger. Use "console" to log to stderr instead.
handlers: [console]
propagate: false
root:
level: DEBUG
# Write logs to the `buffer` handler, which will buffer them together in memory,
# then write them to a file.
#
# Replace "buffer" with "console" to log to stderr instead. (Note that you'll
# also need to update the configuration for the `twisted` logger above, in
# this case.)
#
handlers: [console]
disable_existing_loggers: false

1
playwright/plugins/homeserver/synapse/templates/email/README.md Normal file
View File

@@ -0,0 +1 @@
A synapse configured to require an email for registration

44
playwright/plugins/homeserver/synapse/templates/email/homeserver.yaml Normal file
View File

@@ -0,0 +1,44 @@
server_name: "localhost"
pid_file: /data/homeserver.pid
public_baseurl: "{{PUBLIC_BASEURL}}"
listeners:
- port: 8008
tls: false
bind_addresses: ["::"]
type: http
x_forwarded: true
resources:
- names: [client]
compress: false
database:
name: "sqlite3"
args:
database: ":memory:"
log_config: "/data/log.config"
media_store_path: "/data/media_store"
uploads_path: "/data/uploads"
enable_registration: true
registrations_require_3pid:
- email
registration_shared_secret: "{{REGISTRATION_SECRET}}"
report_stats: false
macaroon_secret_key: "{{MACAROON_SECRET_KEY}}"
form_secret: "{{FORM_SECRET}}"
signing_key_path: "/data/localhost.signing.key"
trusted_key_servers:
- server_name: "matrix.org"
suppress_key_server_warning: true
ui_auth:
session_timeout: "300s"
email:
smtp_host: "%SMTP_HOST%"
smtp_port: %SMTP_PORT%
notif_from: "Your Friendly %(app)s homeserver <noreply@example.com>"
app_name: my_branded_matrix_server

50
playwright/plugins/homeserver/synapse/templates/email/log.config Normal file
View File

@@ -0,0 +1,50 @@
# Log configuration for Synapse.
#
# This is a YAML file containing a standard Python logging configuration
# dictionary. See [1] for details on the valid settings.
#
# Synapse also supports structured logging for machine readable logs which can
# be ingested by ELK stacks. See [2] for details.
#
# [1]: https://docs.python.org/3.7/library/logging.config.html#configuration-dictionary-schema
# [2]: https://matrix-org.github.io/synapse/latest/structured_logging.html
version: 1
formatters:
precise:
format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s - %(message)s'
handlers:
# A handler that writes logs to stderr. Unused by default, but can be used
# instead of "buffer" and "file" in the logger handlers.
console:
class: logging.StreamHandler
formatter: precise
loggers:
synapse.storage.SQL:
# beware: increasing this to DEBUG will make synapse log sensitive
# information such as access tokens.
level: INFO
twisted:
# We send the twisted logging directly to the file handler,
# to work around https://github.com/matrix-org/synapse/issues/3471
# when using "buffer" logger. Use "console" to log to stderr instead.
handlers: [console]
propagate: false
root:
level: INFO
# Write logs to the `buffer` handler, which will buffer them together in memory,
# then write them to a file.
#
# Replace "buffer" with "console" to log to stderr instead. (Note that you'll
# also need to update the configuration for the `twisted` logger above, in
# this case.)
#
handlers: [console]
disable_existing_loggers: false