/* Copyright 2024 New Vector Ltd. Copyright 2021 The Matrix.org Foundation C.I.C. SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE files in the repository root for full details. */ import EventEmitter from "events"; import { SimpleObservable } from "matrix-widget-api"; import { logger } from "matrix-js-sdk/src/logger"; import { UPDATE_EVENT } from "../stores/AsyncStore"; import { arrayFastResample } from "../utils/arrays"; import { type IDestroyable } from "../utils/IDestroyable"; import { PlaybackClock } from "./PlaybackClock"; import { createAudioContext, decodeOgg } from "./compat"; import { clamp } from "../utils/numbers"; import { DEFAULT_WAVEFORM, PLAYBACK_WAVEFORM_SAMPLES } from "./consts"; import { PlaybackEncoder } from "../PlaybackEncoder"; export enum PlaybackState { Decoding = "decoding", Stopped = "stopped", // no progress on timeline Paused = "paused", // some progress on timeline Playing = "playing", // active progress through timeline } const THUMBNAIL_WAVEFORM_SAMPLES = 100; // arbitrary: [30,120] export interface PlaybackInterface { readonly currentState: PlaybackState; readonly liveData: SimpleObservable; readonly timeSeconds: number; readonly durationSeconds: number; skipTo(timeSeconds: number): Promise; } export class Playback extends EventEmitter implements IDestroyable, PlaybackInterface { /** * Stable waveform for representing a thumbnail of the media. Values are * guaranteed to be between zero and one, inclusive. */ public readonly thumbnailWaveform: number[]; private readonly context: AudioContext; private source?: AudioBufferSourceNode | MediaElementAudioSourceNode; private state = PlaybackState.Decoding; private audioBuf?: AudioBuffer; private element?: HTMLAudioElement; private resampledWaveform: number[]; private waveformObservable = new SimpleObservable(); private readonly clock: PlaybackClock; private readonly fileSize: number; /** * Creates a new playback instance from a buffer. * @param {ArrayBuffer} buf The buffer containing the sound sample. * @param {number[]} seedWaveform Optional seed waveform to present until the proper waveform * can be calculated. Contains values between zero and one, inclusive. */ public constructor( private buf: ArrayBuffer, seedWaveform = DEFAULT_WAVEFORM, ) { super(); // Capture the file size early as reading the buffer will result in a 0-length buffer left behind this.fileSize = this.buf.byteLength; this.context = createAudioContext(); this.resampledWaveform = arrayFastResample(seedWaveform ?? DEFAULT_WAVEFORM, PLAYBACK_WAVEFORM_SAMPLES); this.thumbnailWaveform = arrayFastResample(seedWaveform ?? DEFAULT_WAVEFORM, THUMBNAIL_WAVEFORM_SAMPLES); this.waveformObservable.update(this.resampledWaveform); this.clock = new PlaybackClock(this.context); } /** * Size of the audio clip in bytes. May be zero if unknown. This is updated * when the playback goes through phase changes. */ public get sizeBytes(): number { return this.fileSize; } /** * Stable waveform for the playback. Values are guaranteed to be between * zero and one, inclusive. */ public get waveform(): number[] { return this.resampledWaveform; } public get waveformData(): SimpleObservable { return this.waveformObservable; } public get clockInfo(): PlaybackClock { return this.clock; } public get liveData(): SimpleObservable { return this.clock.liveData; } public get timeSeconds(): number { return this.clock.timeSeconds; } public get durationSeconds(): number { return this.clock.durationSeconds; } public get currentState(): PlaybackState { return this.state; } public get isPlaying(): boolean { return this.currentState === PlaybackState.Playing; } public emit(event: PlaybackState, ...args: any[]): boolean { this.state = event; super.emit(event, ...args); super.emit(UPDATE_EVENT, event, ...args); return true; // we don't ever care if the event had listeners, so just return "yes" } public destroy(): void { // Dev note: It's critical that we call stop() during cleanup to ensure that downstream callers // are aware of the final clock position before the user triggered an unload. // noinspection JSIgnoredPromiseFromCall - not concerned about being called async here this.stop(); this.removeAllListeners(); this.clock.destroy(); this.waveformObservable.close(); if (this.element) { URL.revokeObjectURL(this.element.src); this.element.remove(); } } public async prepare(): Promise { // don't attempt to decode the media again // AudioContext.decodeAudioData detaches the array buffer `this.buf` // meaning it cannot be re-read if (this.state !== PlaybackState.Decoding) { return; } // The point where we use an audio element is fairly arbitrary, though we don't want // it to be too low. As of writing, voice messages want to show a waveform but audio // messages do not. Using an audio element means we can't show a waveform preview, so // we try to target the difference between a voice message file and large audio file. // Overall, the point of this is to avoid memory-related issues due to storing a massive // audio buffer in memory, as that can balloon to far greater than the input buffer's // byte length. if (this.buf.byteLength > 5 * 1024 * 1024) { // 5mb logger.log("Audio file too large: processing through