element-web/src/voice-broadcast/models/VoiceBroadcastPlayback.ts

/*
Copyright 2022 The Matrix.org Foundation C.I.C.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import {
    EventType,
    MatrixClient,
    MatrixEvent,
    MsgType,
    RelationType,
} from "matrix-js-sdk/src/matrix";
import { TypedEventEmitter } from "matrix-js-sdk/src/models/typed-event-emitter";

import { Playback, PlaybackState } from "../../audio/Playback";
import { PlaybackManager } from "../../audio/PlaybackManager";
import { UPDATE_EVENT } from "../../stores/AsyncStore";
import { MediaEventHelper } from "../../utils/MediaEventHelper";
import { IDestroyable } from "../../utils/IDestroyable";
import { VoiceBroadcastChunkEventType, VoiceBroadcastInfoEventType, VoiceBroadcastInfoState } from "..";
import { RelationsHelper, RelationsHelperEvent } from "../../events/RelationsHelper";
import { getReferenceRelationsForEvent } from "../../events";
import { VoiceBroadcastChunkEvents } from "../utils/VoiceBroadcastChunkEvents";

export enum VoiceBroadcastPlaybackState {
    Paused,
    Playing,
    Stopped,
    Buffering,
}

export enum VoiceBroadcastPlaybackEvent {
    LengthChanged = "length_changed",
    StateChanged = "state_changed",
    InfoStateChanged = "info_state_changed",
}

interface EventMap {
    [VoiceBroadcastPlaybackEvent.LengthChanged]: (length: number) => void;
    [VoiceBroadcastPlaybackEvent.StateChanged]: (
        state: VoiceBroadcastPlaybackState,
        playback: VoiceBroadcastPlayback
    ) => void;
    [VoiceBroadcastPlaybackEvent.InfoStateChanged]: (state: VoiceBroadcastInfoState) => void;
}

export class VoiceBroadcastPlayback
    extends TypedEventEmitter<VoiceBroadcastPlaybackEvent, EventMap>
    implements IDestroyable {
    private state = VoiceBroadcastPlaybackState.Stopped;
    private infoState: VoiceBroadcastInfoState;
    private chunkEvents = new VoiceBroadcastChunkEvents();
    private playbacks = new Map<string, Playback>();
    private currentlyPlaying: MatrixEvent;
    private lastInfoEvent: MatrixEvent;
    private chunkRelationHelper: RelationsHelper;
    private infoRelationHelper: RelationsHelper;

    public constructor(
        public readonly infoEvent: MatrixEvent,
        private client: MatrixClient,
    ) {
        super();
        this.addInfoEvent(this.infoEvent);
        this.setUpRelationsHelper();
    }

    private setUpRelationsHelper(): void {
        this.infoRelationHelper = new RelationsHelper(
            this.infoEvent,
            RelationType.Reference,
            VoiceBroadcastInfoEventType,
            this.client,
        );
        this.infoRelationHelper.on(RelationsHelperEvent.Add, this.addInfoEvent);
        this.infoRelationHelper.emitCurrent();

        this.chunkRelationHelper = new RelationsHelper(
            this.infoEvent,
            RelationType.Reference,
            EventType.RoomMessage,
            this.client,
        );
        this.chunkRelationHelper.on(RelationsHelperEvent.Add, this.addChunkEvent);
        this.chunkRelationHelper.emitCurrent();
    }

    private addChunkEvent = async (event: MatrixEvent): Promise<boolean> => {
        const eventId = event.getId();

        if (!eventId
            || eventId.startsWith("~!") // don't add local events
            || event.getContent()?.msgtype !== MsgType.Audio // don't add non-audio event
        ) {
            return false;
        }

        this.chunkEvents.addEvent(event);
        this.emit(VoiceBroadcastPlaybackEvent.LengthChanged, this.chunkEvents.getLength());

        if (this.getState() !== VoiceBroadcastPlaybackState.Stopped) {
            await this.enqueueChunk(event);
        }

        if (this.getState() === VoiceBroadcastPlaybackState.Buffering) {
            await this.start();
        }

        return true;
    };

    private addInfoEvent = (event: MatrixEvent): void => {
        if (this.lastInfoEvent && this.lastInfoEvent.getTs() >= event.getTs()) {
            // Only handle newer events
            return;
        }

        const state = event.getContent()?.state;

        if (!Object.values(VoiceBroadcastInfoState).includes(state)) {
            // Do not handle unknown voice broadcast states
            return;
        }

        this.lastInfoEvent = event;
        this.setInfoState(state);
    };

    private async loadChunks(): Promise<void> {
        const relations = getReferenceRelationsForEvent(this.infoEvent, EventType.RoomMessage, this.client);
        const chunkEvents = relations?.getRelations();

        if (!chunkEvents) {
            return;
        }

        this.chunkEvents.addEvents(chunkEvents);

        for (const chunkEvent of chunkEvents) {
            await this.enqueueChunk(chunkEvent);
        }
    }

    private async enqueueChunk(chunkEvent: MatrixEvent) {
        const sequenceNumber = parseInt(chunkEvent.getContent()?.[VoiceBroadcastChunkEventType]?.sequence, 10);
        if (isNaN(sequenceNumber) || sequenceNumber < 1) return;

        const helper = new MediaEventHelper(chunkEvent);
        const blob = await helper.sourceBlob.value;
        const buffer = await blob.arrayBuffer();
        const playback = PlaybackManager.instance.createPlaybackInstance(buffer);
        await playback.prepare();
        playback.clockInfo.populatePlaceholdersFrom(chunkEvent);
        this.playbacks.set(chunkEvent.getId(), playback);
        playback.on(UPDATE_EVENT, (state) => this.onPlaybackStateChange(playback, state));
    }

    private async onPlaybackStateChange(playback: Playback, newState: PlaybackState) {
        if (newState !== PlaybackState.Stopped) {
            return;
        }

        await this.playNext();
    }

    private async playNext(): Promise<void> {
        if (!this.currentlyPlaying) return;

        const next = this.chunkEvents.getNext(this.currentlyPlaying);

        if (next) {
            this.setState(VoiceBroadcastPlaybackState.Playing);
            this.currentlyPlaying = next;
            await this.playbacks.get(next.getId())?.play();
            return;
        }

        if (this.getInfoState() === VoiceBroadcastInfoState.Stopped) {
            this.setState(VoiceBroadcastPlaybackState.Stopped);
        } else {
            // No more chunks available, although the broadcast is not finished → enter buffering state.
            this.setState(VoiceBroadcastPlaybackState.Buffering);
        }
    }

    public getLength(): number {
        return this.chunkEvents.getLength();
    }

    public async start(): Promise<void> {
        if (this.playbacks.size === 0) {
            await this.loadChunks();
        }

        const chunkEvents = this.chunkEvents.getEvents();

        const toPlay = this.getInfoState() === VoiceBroadcastInfoState.Stopped
            ? chunkEvents[0] // start at the beginning for an ended voice broadcast
            : chunkEvents[chunkEvents.length - 1]; // start at the current chunk for an ongoing voice broadcast

        if (this.playbacks.has(toPlay?.getId())) {
            this.setState(VoiceBroadcastPlaybackState.Playing);
            this.currentlyPlaying = toPlay;
            await this.playbacks.get(toPlay.getId()).play();
            return;
        }

        this.setState(VoiceBroadcastPlaybackState.Buffering);
    }

    public get length(): number {
        return this.chunkEvents.getLength();
    }

    public stop(): void {
        this.setState(VoiceBroadcastPlaybackState.Stopped);

        if (this.currentlyPlaying) {
            this.playbacks.get(this.currentlyPlaying.getId()).stop();
        }
    }

    public pause(): void {
        // stopped voice broadcasts cannot be paused
        if (this.getState() === VoiceBroadcastPlaybackState.Stopped) return;

        this.setState(VoiceBroadcastPlaybackState.Paused);
        if (!this.currentlyPlaying) return;
        this.playbacks.get(this.currentlyPlaying.getId()).pause();
    }

    public resume(): void {
        if (!this.currentlyPlaying) {
            // no playback to resume, start from the beginning
            this.start();
            return;
        }

        this.setState(VoiceBroadcastPlaybackState.Playing);
        this.playbacks.get(this.currentlyPlaying.getId()).play();
    }

    /**
     * Toggles the playback:
     * stopped → playing
     * playing → paused
     * paused → playing
     */
    public async toggle() {
        if (this.state === VoiceBroadcastPlaybackState.Stopped) {
            await this.start();
            return;
        }

        if (this.state === VoiceBroadcastPlaybackState.Paused) {
            this.resume();
            return;
        }

        this.pause();
    }

    public getState(): VoiceBroadcastPlaybackState {
        return this.state;
    }

    private setState(state: VoiceBroadcastPlaybackState): void {
        if (this.state === state) {
            return;
        }

        this.state = state;
        this.emit(VoiceBroadcastPlaybackEvent.StateChanged, state, this);
    }

    public getInfoState(): VoiceBroadcastInfoState {
        return this.infoState;
    }

    private setInfoState(state: VoiceBroadcastInfoState): void {
        if (this.infoState === state) {
            return;
        }

        this.infoState = state;
        this.emit(VoiceBroadcastPlaybackEvent.InfoStateChanged, state);
    }

    public destroy(): void {
        this.chunkRelationHelper.destroy();
        this.infoRelationHelper.destroy();
        this.removeAllListeners();

        this.chunkEvents = new VoiceBroadcastChunkEvents();
        this.playbacks.forEach(p => p.destroy());
        this.playbacks = new Map<string, Playback>();
    }
}