Improve handling of animated images, add support for AVIF animations (#30932)

* Only set MSC4230 is_animated flag if we are able to tell if the media is animated Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Set blob type correctly to not need to weave the mimetype around Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Use ImageDecoder to determine whether media is animated or not, adding support for AVIF and other formats Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Fix test Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Iterate Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Add test Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> * Fix test Signed-off-by: Michael Telatynski <7t3chguy@gmail.com> --------- Signed-off-by: Michael Telatynski <7t3chguy@gmail.com>
2025-10-03 14:29:10 +01:00
parent 5f084c28c3
commit e83ddbc98a
6 changed files with 115 additions and 33 deletions
--- a/src/ContentMessages.ts
+++ b/src/ContentMessages.ts
@@ -158,14 +158,17 @@ async function infoForImageFile(matrixClient: MatrixClient, roomId: string, imag
    }

    // We don't await this immediately so it can happen in the background
-    const isAnimatedPromise = blobIsAnimated(imageFile.type, imageFile);
+    const isAnimatedPromise = blobIsAnimated(imageFile);

    const imageElement = await loadImageElement(imageFile);

    const result = await createThumbnail(imageElement.img, imageElement.width, imageElement.height, thumbnailType);
    const imageInfo = result.info;

-    imageInfo["org.matrix.msc4230.is_animated"] = await isAnimatedPromise;
+    const isAnimated = await isAnimatedPromise;
+    if (isAnimated !== undefined) {
+        imageInfo["org.matrix.msc4230.is_animated"] = await isAnimatedPromise;
+    }

    // For lesser supported image types, always include the thumbnail even if it is larger
    if (!ALWAYS_INCLUDE_THUMBNAIL.includes(imageFile.type)) {
--- a/src/components/views/messages/MImageBody.tsx
+++ b/src/components/views/messages/MImageBody.tsx
@@ -311,10 +311,7 @@ export class MImageBodyInner extends React.Component<IProps, IState> {
                    // then we need to check if the image is animated by downloading it.
                    if (
                        content.info?.["org.matrix.msc4230.is_animated"] === false ||
-                        !(await blobIsAnimated(
-                            content.info?.mimetype,
-                            await this.props.mediaEventHelper!.sourceBlob.value,
-                        ))
+                        (await blobIsAnimated(await this.props.mediaEventHelper!.sourceBlob.value)) === false
                    ) {
                        isAnimated = false;
                    }
--- a/src/utils/Image.ts
+++ b/src/utils/Image.ts
@@ -9,7 +9,6 @@
 import { arrayHasDiff } from "./arrays";

 export function mayBeAnimated(mimeType?: string): boolean {
-    // AVIF animation support at the time of writing is only available in Chrome hence not having `blobIsAnimated` check
    return ["image/gif", "image/webp", "image/png", "image/apng", "image/avif"].includes(mimeType!);
 }

@@ -26,8 +25,28 @@ function arrayBufferReadStr(arr: ArrayBuffer, start: number, len: number): strin
    return String.fromCharCode.apply(null, Array.from(arrayBufferRead(arr, start, len)));
 }

-export async function blobIsAnimated(mimeType: string | undefined, blob: Blob): Promise<boolean> {
-    switch (mimeType) {
+/**
+ * Check if a Blob contains an animated image.
+ * @param blob The Blob to check.
+ * @returns True if the image is animated, false if not, or undefined if it could not be determined.
+ */
+export async function blobIsAnimated(blob: Blob): Promise<boolean | undefined> {
+    try {
+        // Try parse the image using ImageDecoder as this is the most coherent way of asserting whether a piece of media
+        // is or is not animated. Limited availability at time of writing, notably Safari lacks support.
+        // https://developer.mozilla.org/en-US/docs/Web/API/ImageDecoder
+        const data = await blob.arrayBuffer();
+        const decoder = new ImageDecoder({ data, type: blob.type });
+        await decoder.tracks.ready;
+        if ([...decoder.tracks].some((track) => track.animated)) {
+            return true;
+        }
+    } catch (e) {
+        console.warn("ImageDecoder not supported or failed to decode image", e);
+        // Not supported by this browser, fall through to manual checks
+    }
+
+    switch (blob.type) {
        case "image/webp": {
            // Only extended file format WEBP images support animation, so grab the expected data range and verify header.
            // Based on https://developers.google.com/speed/webp/docs/riff_container#extended_file_format
@@ -42,7 +61,7 @@ export async function blobIsAnimated(mimeType: string | undefined, blob: Blob):
                const animationFlagMask = 1 << 1;
                return (flags & animationFlagMask) != 0;
            }
-            break;
+            return false;
        }

        case "image/gif": {
@@ -100,9 +119,7 @@ export async function blobIsAnimated(mimeType: string | undefined, blob: Blob):
                }
                i += length + 4;
            }
-            break;
+            return false;
        }
    }
-
-    return false;
 }
--- a/src/utils/MediaEventHelper.ts
+++ b/src/utils/MediaEventHelper.ts
@@ -72,18 +72,25 @@ export class MediaEventHelper implements IDestroyable {
    };

    private fetchSource = (): Promise<Blob> => {
+        const content = this.event.getContent<MediaEventContent>();
        if (this.media.isEncrypted) {
-            const content = this.event.getContent<MediaEventContent>();
            return decryptFile(content.file!, content.info);
        }
-        return this.media.downloadSource().then((r) => r.blob());
+
+        return (
+            this.media
+                .downloadSource()
+                .then((r) => r.blob())
+                // Set the mime type from the event info on the blob
+                .then((blob) => blob.slice(0, blob.size, content.info?.mimetype ?? blob.type))
+        );
    };

    private fetchThumbnail = (): Promise<Blob | null> => {
        if (!this.media.hasThumbnail) return Promise.resolve(null);

+        const content = this.event.getContent<ImageContent>();
        if (this.media.isEncrypted) {
-            const content = this.event.getContent<ImageContent>();
            if (content.info?.thumbnail_file) {
                return decryptFile(content.info.thumbnail_file, content.info.thumbnail_info);
            } else {
@@ -96,7 +103,12 @@ export class MediaEventHelper implements IDestroyable {
        const thumbnailHttp = this.media.thumbnailHttp;
        if (!thumbnailHttp) return Promise.resolve(null);

-        return fetch(thumbnailHttp).then((r) => r.blob());
+        return (
+            fetch(thumbnailHttp)
+                .then((r) => r.blob())
+                // Set the mime type from the event info on the blob
+                .then((blob) => blob.slice(0, blob.size, content.info?.thumbnail_info?.mimetype ?? blob.type))
+        );
    };

    public static isEligible(event: MatrixEvent): boolean {
--- a/test/unit-tests/Image-test.ts
+++ b/test/unit-tests/Image-test.ts
@@ -32,42 +32,55 @@ describe("Image", () => {

    describe("blobIsAnimated", () => {
        it("Animated GIF", async () => {
-            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.gif"))]);
-            expect(await blobIsAnimated("image/gif", img)).toBeTruthy();
+            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.gif"))], {
+                type: "image/gif",
+            });
+            expect(await blobIsAnimated(img)).toBeTruthy();
        });

        it("Static GIF", async () => {
-            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.gif"))]);
-            expect(await blobIsAnimated("image/gif", img)).toBeFalsy();
+            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.gif"))], {
+                type: "image/gif",
+            });
+            expect(await blobIsAnimated(img)).toBeFalsy();
        });

        it("Animated WEBP", async () => {
-            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.webp"))]);
-            expect(await blobIsAnimated("image/webp", img)).toBeTruthy();
+            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.webp"))], {
+                type: "image/webp",
+            });
+            expect(await blobIsAnimated(img)).toBeTruthy();
        });

        it("Static WEBP", async () => {
-            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.webp"))]);
-            expect(await blobIsAnimated("image/webp", img)).toBeFalsy();
+            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.webp"))], {
+                type: "image/webp",
+            });
+            expect(await blobIsAnimated(img)).toBeFalsy();
        });

        it("Static WEBP in extended file format", async () => {
-            const img = new Blob([
-                fs.readFileSync(path.resolve(__dirname, "images", "static-logo-extended-file-format.webp")),
-            ]);
-            expect(await blobIsAnimated("image/webp", img)).toBeFalsy();
+            const img = new Blob(
+                [fs.readFileSync(path.resolve(__dirname, "images", "static-logo-extended-file-format.webp"))],
+                { type: "image/webp" },
+            );
+            expect(await blobIsAnimated(img)).toBeFalsy();
        });

        it("Animated PNG", async () => {
            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "animated-logo.apng"))]);
-            expect(await blobIsAnimated("image/png", img)).toBeTruthy();
-            expect(await blobIsAnimated("image/apng", img)).toBeTruthy();
+            const pngBlob = img.slice(0, img.size, "image/png");
+            const apngBlob = img.slice(0, img.size, "image/apng");
+            expect(await blobIsAnimated(pngBlob)).toBeTruthy();
+            expect(await blobIsAnimated(apngBlob)).toBeTruthy();
        });

        it("Static PNG", async () => {
            const img = new Blob([fs.readFileSync(path.resolve(__dirname, "images", "static-logo.png"))]);
-            expect(await blobIsAnimated("image/png", img)).toBeFalsy();
-            expect(await blobIsAnimated("image/apng", img)).toBeFalsy();
+            const pngBlob = img.slice(0, img.size, "image/png");
+            const apngBlob = img.slice(0, img.size, "image/apng");
+            expect(await blobIsAnimated(pngBlob)).toBeFalsy();
+            expect(await blobIsAnimated(apngBlob)).toBeFalsy();
        });
    });
 });
--- a/test/unit-tests/utils/MediaEventHelper-test.ts
+++ b/test/unit-tests/utils/MediaEventHelper-test.ts
@@ -0,0 +1,40 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE files in the repository root for full details.
+*/
+
+import { MatrixEvent } from "matrix-js-sdk/src/matrix";
+
+import { MediaEventHelper } from "../../../src/utils/MediaEventHelper.ts";
+import { stubClient } from "../../test-utils";
+
+describe("MediaEventHelper", () => {
+    it("should set the mime type on the blob based on the event metadata", async () => {
+        stubClient();
+
+        const event = new MatrixEvent({
+            type: "m.room.message",
+            content: {
+                msgtype: "m.image",
+                body: "image.png",
+                info: {
+                    mimetype: "image/png",
+                    size: 1234,
+                    w: 100,
+                    h: 100,
+                    thumbnail_info: {
+                        mimetype: "image/png",
+                    },
+                    thumbnail_url: "mxc://matrix.org/thumbnail",
+                },
+                url: "mxc://matrix.org/abcdef",
+            },
+        });
+        const helper = new MediaEventHelper(event);
+
+        const blob = await helper.thumbnailBlob.value;
+        expect(blob?.type).toBe(event.getContent().info.thumbnail_info?.mimetype);
+    });
+});