From a5683cde031b1ade21d111382e237f59b2b1e734 Mon Sep 17 00:00:00 2001 From: bxdxnn <267911624+bxdxnn@users.noreply.github.com> Date: Sat, 11 Apr 2026 19:04:33 +0000 Subject: [PATCH] Support gallery messages --- apps/web/res/css/_components.pcss | 1 + .../res/css/views/messages/_MGalleryBody.pcss | 92 +++++++ apps/web/src/@types/matrix-js-sdk.d.ts | 17 ++ apps/web/src/ContentMessages.ts | 232 ++++++++++++++++++ .../views/messages/MGalleryBody.tsx | 151 ++++++++++++ .../views/messages/MessageEvent.tsx | 11 +- apps/web/src/i18n/strings/en_EN.json | 6 +- 7 files changed, 506 insertions(+), 4 deletions(-) create mode 100644 apps/web/res/css/views/messages/_MGalleryBody.pcss create mode 100644 apps/web/src/components/views/messages/MGalleryBody.tsx diff --git a/apps/web/res/css/_components.pcss b/apps/web/res/css/_components.pcss index bdca70276db..9cc88836a3a 100644 --- a/apps/web/res/css/_components.pcss +++ b/apps/web/res/css/_components.pcss @@ -225,6 +225,7 @@ @import "./views/messages/_LegacyCallEvent.pcss"; @import "./views/messages/_MEmoteBody.pcss"; @import "./views/messages/_MFileBody.pcss"; +@import "./views/messages/_MGalleryBody.pcss"; @import "./views/messages/_MImageBody.pcss"; @import "./views/messages/_MImageReplyBody.pcss"; @import "./views/messages/_MJitsiWidgetEvent.pcss"; diff --git a/apps/web/res/css/views/messages/_MGalleryBody.pcss b/apps/web/res/css/views/messages/_MGalleryBody.pcss new file mode 100644 index 00000000000..439fe2ec022 --- /dev/null +++ b/apps/web/res/css/views/messages/_MGalleryBody.pcss @@ -0,0 +1,92 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE files in the repository root for full details. +*/ + +.mx_MGalleryBody { + display: flex; + flex-direction: column; + gap: $spacing-4; +} + +/// Grid layout for gallery items +.mx_MGalleryBody_grid { + display: grid; + grid-template-columns: repeat(2, 1fr); + gap: 2px; + border-radius: var(--MBody-border-radius); + overflow: hidden; + max-width: 600px; +} + +.mx_MGalleryBody_item { + position: relative; + aspect-ratio: 1; + background-color: $background; + overflow: hidden; + cursor: pointer; + + &--square { + aspect-ratio: 1; + } + + &--wide { + grid-column: span 2; + aspect-ratio: 2; + } + + &--tall { + grid-row: span 2; + aspect-ratio: 0.5; + } + + &--large { + grid-column: span 2; + grid-row: span 2; + aspect-ratio: 1; + } + + &--more { + display: flex; + align-items: center; + justify-content: center; + background-color: $background; + color: $primary-content; + font-size: $font-18px; + font-weight: bold; + aspect-ratio: 1; + } +} + +.mx_MGalleryBody_item_thumbnail { + width: 100%; + height: 100%; + object-fit: cover; + display: block; + transition: opacity 0.2s ease-in-out; +} + +.mx_MGalleryBody_item_placeholder { + position: absolute; + inset: 0; + background-color: $background; + animation: mx--anim-pulse 1.75s infinite cubic-bezier(0.4, 0, 0.6, 1); +} + +.mx_MGalleryBody_item_error { + position: absolute; + inset: 0; + display: flex; + align-items: center; + justify-content: center; + background-color: $background; + color: $secondary-content; +} + +.mx_MGalleryBody_empty { + padding: $spacing-8; + text-align: center; + color: $secondary-content; +} \ No newline at end of file diff --git a/apps/web/src/@types/matrix-js-sdk.d.ts b/apps/web/src/@types/matrix-js-sdk.d.ts index 3f36190e4e2..25e2f9974de 100644 --- a/apps/web/src/@types/matrix-js-sdk.d.ts +++ b/apps/web/src/@types/matrix-js-sdk.d.ts @@ -16,9 +16,26 @@ import type { DeviceClientInformation } from "../utils/device/types.ts"; import type { UserWidget } from "../utils/WidgetUtils-types.ts"; import { type MediaPreviewConfig } from "./media_preview.ts"; import { type INVITE_RULES_ACCOUNT_DATA_TYPE, type InviteConfigAccountData } from "./invite-rules.ts"; +import type { ImageInfo, VideoInfo, AudioInfo, FileInfo } from "matrix-js-sdk/src/types"; // Extend Matrix JS SDK types via Typescript declaration merging to support unspecced event fields and types declare module "matrix-js-sdk/src/types" { + export interface GalleryItemContent { + itemtype: string; + body: string; + url?: string; + file?: EncryptedFile; + info?: ImageInfo | VideoInfo | AudioInfo | FileInfo; + } + + export interface GalleryContent { + msgtype: "dm.filament.gallery"; + body: string; + format?: "org.matrix.custom.html"; + formatted_body?: string; + itemtypes: GalleryItemContent[]; + } + export interface FileInfo { /** * @see https://github.com/matrix-org/matrix-spec-proposals/pull/2448 diff --git a/apps/web/src/ContentMessages.ts b/apps/web/src/ContentMessages.ts index ec68f854fe3..8166be0c63a 100644 --- a/apps/web/src/ContentMessages.ts +++ b/apps/web/src/ContentMessages.ts @@ -24,9 +24,27 @@ import { type AudioInfo, type VideoInfo, type EncryptedFile, + type FileInfo, type MediaEventContent, type MediaEventInfo, } from "matrix-js-sdk/src/types"; + +export interface GalleryItemContent { + itemtype: string; + body: string; + url?: string; + file?: EncryptedFile; + info?: ImageInfo | VideoInfo | AudioInfo | FileInfo; +} + +export interface GalleryContent { + msgtype: "dm.filament.gallery"; + body: string; + format?: "org.matrix.custom.html"; + formatted_body?: string; + itemtypes: GalleryItemContent[]; +} + import encrypt from "matrix-encrypt-attachment"; import extractPngChunks from "png-chunks-extract"; import { logger } from "matrix-js-sdk/src/logger"; @@ -61,6 +79,10 @@ import { blobIsAnimated } from "./utils/Image.ts"; // 5669 px (x-axis) , 5669 px (y-axis) , per metre const PHYS_HIDPI = [0x00, 0x00, 0x16, 0x25, 0x00, 0x00, 0x16, 0x25, 0x01]; +const MSG_TYPE_GALLERY = "dm.filament.gallery"; +const GALLERY_MIN_ITEMS = 2; +const GALLERY_MAX_ITEMS = 10; + export class UploadCanceledError extends Error {} export class UploadFailedError extends Error { public constructor(cause: any) { @@ -450,6 +472,10 @@ export default class ContentMessages { return; } + if (files.length >= GALLERY_MIN_ITEMS && files.length <= GALLERY_MAX_ITEMS) { + return this.sendContentListAsGalleryToRoom(files, roomId, relation, replyToEvent, matrixClient, context); + } + if (!this.mediaConfig) { // hot-path optimization to not flash a spinner if we don't need to const modal = Modal.createDialog(Spinner, undefined, "mx_Dialog_spinner"); @@ -535,6 +561,212 @@ export default class ContentMessages { }); } + public async sendContentListAsGalleryToRoom( + files: File[], + roomId: string, + relation: IEventRelation | undefined, + replyToEvent: MatrixEvent | undefined, + matrixClient: MatrixClient, + context = TimelineRenderingType.Room, + ): Promise { + if (matrixClient.isGuest()) { + dis.dispatch({ action: "require_registration" }); + return; + } + + if (files.length < GALLERY_MIN_ITEMS || files.length > GALLERY_MAX_ITEMS) { + logger.warn( + `[Gallery] Invalid number of files: ${files.length}. Must be between ${GALLERY_MIN_ITEMS} and ${GALLERY_MAX_ITEMS}`, + ); + return; + } + + if (!this.mediaConfig) { + const modal = Modal.createDialog(Spinner, undefined, "mx_Dialog_spinner"); + await Promise.race([this.ensureMediaConfigFetched(matrixClient), modal.finished]); + if (!this.mediaConfig) { + return; + } else { + modal.close(); + } + } + + const tooBigFiles: File[] = []; + const okFiles: File[] = []; + + for (const file of files) { + if (this.isFileSizeAcceptable(file)) { + okFiles.push(file); + } else { + tooBigFiles.push(file); + } + } + + if (tooBigFiles.length > 0) { + const { finished } = Modal.createDialog(UploadFailureDialog, { + badFiles: tooBigFiles, + totalFiles: files.length, + contentMessages: this, + }); + const [shouldContinue] = await finished; + if (!shouldContinue) return; + } + + let promBefore: Promise = Promise.resolve(); + const uploadedItems: GalleryItemContent[] = []; + + for (let i = 0; i < okFiles.length; ++i) { + const file = okFiles[i]; + const loopPromiseBefore = promBefore; + + const galleryItem = await doMaybeLocalRoomAction( + roomId, + (actualRoomId) => + this.uploadContentAsGalleryItem( + file, + actualRoomId, + relation, + matrixClient, + replyToEvent ?? undefined, + loopPromiseBefore, + ), + matrixClient, + ); + + if (galleryItem) { + uploadedItems.push(galleryItem); + } + } + + if (uploadedItems.length < GALLERY_MIN_ITEMS) { + logger.warn(`[Gallery] Not enough items uploaded: ${uploadedItems.length}`); + return; + } + + const galleryContent: GalleryContent = { + msgtype: MSG_TYPE_GALLERY, + body: _t("timeline|gallery|caption_default"), + itemtypes: uploadedItems, + }; + + attachMentions(matrixClient.getSafeUserId(), galleryContent, null, replyToEvent); + attachRelation(galleryContent, relation); + if (replyToEvent) { + addReplyToMessageContent(galleryContent, replyToEvent); + } + + if (SettingsStore.getValue("Performance.addSendMessageTimingMetadata")) { + decorateStartSendingTime(galleryContent); + } + + const threadId = relation?.rel_type === THREAD_RELATION_TYPE.name ? relation.event_id : undefined; + + try { + const response = await matrixClient.sendMessage(roomId, threadId ?? null, galleryContent as any); + + if (SettingsStore.getValue("Performance.addSendMessageTimingMetadata")) { + sendRoundTripMetric(matrixClient, roomId, response.event_id); + } + + dis.dispatch({ action: "message_sent" }); + } catch (error) { + const unwrappedError = error instanceof UploadFailedError && error.cause ? error.cause : error; + if (unwrappedError instanceof HTTPError && unwrappedError.httpStatus === 413) { + this.mediaConfig = null; + } + Modal.createDialog(ErrorDialog, { + title: _t("upload_failed_title"), + description: _t("timeline|gallery|upload_failed"), + }); + } + + if (replyToEvent) { + dis.dispatch({ + action: "reply_to_event", + event: null, + context, + }); + } + + dis.dispatch({ + action: Action.FocusSendMessageComposer, + context, + }); + } + + private async uploadContentAsGalleryItem( + file: File, + roomId: string, + relation: IEventRelation | undefined, + matrixClient: MatrixClient, + replyToEvent: MatrixEvent | undefined, + promBefore?: Promise, + ): Promise { + const fileName = file.name || _t("common|attachment"); + const content: GalleryItemContent = { + itemtype: MsgType.File, + body: fileName, + }; + + if (file.type) { + content.info = { mimetype: file.type, size: file.size } as ImageInfo; + } + + const upload = new RoomUpload(roomId, fileName, relation, file.size); + this.inprogress.push(upload); + dis.dispatch({ action: Action.UploadStarted, upload }); + + function onProgress(progress: UploadProgress): void { + upload.onProgress(progress); + dis.dispatch({ action: Action.UploadProgress, upload }); + } + + try { + if (file.type.startsWith("image/")) { + content.itemtype = MsgType.Image; + try { + const imageInfo = await infoForImageFile(matrixClient, roomId, file); + content.info = imageInfo as ImageInfo; + } catch (e) { + logger.error(e); + content.itemtype = MsgType.File; + } + } else if (file.type.startsWith("audio/")) { + content.itemtype = MsgType.Audio; + try { + const audioInfo = await infoForAudioFile(file); + content.info = audioInfo as AudioInfo; + } catch (e) { + logger.error(e); + content.itemtype = MsgType.File; + } + } else if (file.type.startsWith("video/")) { + content.itemtype = MsgType.Video; + try { + const videoInfo = await infoForVideoFile(matrixClient, roomId, file); + content.info = videoInfo as VideoInfo; + } catch (e) { + logger.error(e); + content.itemtype = MsgType.File; + } + } + + if (upload.cancelled) throw new UploadCanceledError(); + const result = await uploadFile(matrixClient, roomId, file, onProgress, upload.abortController); + content.file = result.file; + content.url = result.url; + + if (promBefore) await promBefore; + if (upload.cancelled) throw new UploadCanceledError(); + + removeElement(this.inprogress, (e) => e.promise === upload.promise); + return content; + } catch (error) { + removeElement(this.inprogress, (e) => e.promise === upload.promise); + return null; + } + } + public getCurrentUploads(relation?: IEventRelation): RoomUpload[] { return this.inprogress.filter((roomUpload) => { const noRelation = !relation && !roomUpload.relation; diff --git a/apps/web/src/components/views/messages/MGalleryBody.tsx b/apps/web/src/components/views/messages/MGalleryBody.tsx new file mode 100644 index 00000000000..c88d35b783f --- /dev/null +++ b/apps/web/src/components/views/messages/MGalleryBody.tsx @@ -0,0 +1,151 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR GPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE files in the repository root for full details. +*/ + +import React, { useState, type JSX } from "react"; +import { _t } from "../../../languageHandler"; +import TextualBody from "./TextualBody"; +import { type IBodyProps } from "./IBodyProps"; +import type { FileInfo, ImageInfo, VideoInfo, AudioInfo, EncryptedFile } from "matrix-js-sdk/src/types"; + +export interface GalleryItemContent { + itemtype: string; + body: string; + url?: string; + file?: EncryptedFile; + info?: FileInfo | ImageInfo | VideoInfo | AudioInfo; +} + +export interface GalleryContent { + msgtype: "dm.filament.gallery"; + body: string; + format?: "org.matrix.custom.html"; + formatted_body?: string; + itemtypes: GalleryItemContent[]; +} + +interface GalleryItemProps { + item: GalleryItemContent; + index: number; + onClick: (index: number) => void; + forExport?: boolean; +} + +const GalleryItem: React.FC = ({ item, index, onClick, forExport }) => { + const [loaded, setLoaded] = useState(false); + const [error, setError] = useState(false); + + const imageUrl = item.url ?? item.file?.url; + const imageInfo = item.info; + const width = (imageInfo as ImageInfo | VideoInfo)?.w ?? 200; + const height = (imageInfo as ImageInfo | VideoInfo)?.h ?? 200; + + const handleClick = (): void => { + onClick(index); + }; + + const handleLoad = (): void => { + setLoaded(true); + }; + + const handleError = (): void => { + setError(true); + }; + + const aspectRatio = width / height || 1; + const isWide = aspectRatio > 1.5; + const isTall = aspectRatio < 0.67; + + let thumbnail: JSX.Element | null = null; + if (!forExport && imageUrl && !error) { + thumbnail = ( + {item.body} + ); + } else if (forExport && imageUrl) { + thumbnail = ( + {item.body} + ); + } + + let placeholder: JSX.Element | null = null; + if (!loaded && !error) { + placeholder =
; + } + + const gridClass = + index === 0 + ? "mx_MGalleryBody_item--large" + : isWide + ? "mx_MGalleryBody_item--wide" + : isTall + ? "mx_MGalleryBody_item--tall" + : "mx_MGalleryBody_item--square"; + + return ( +
+ {placeholder} + {thumbnail} + {error &&
{_t("common|attachment")}
} +
+ ); +}; + +const MGalleryBody: React.FC = (props) => { + const content = props.mxEvent.getContent(); + const items = content.itemtypes ?? []; + const hasCaption = content.body && content.body.length > 0; + + const handleItemClick = (index: number): void => { + const item = items[index]; + if (item.url) { + window.open(item.url, "_blank"); + } + }; + + const renderImageGrid = (): React.ReactNode => { + if (items.length === 0) { + return
{_t("common|attachment")}
; + } + + const displayItems = items.slice(0, 4); + const remainingCount = items.length - 4; + + return ( +
+ {displayItems.map((item, index) => ( + + ))} + {remainingCount > 0 && ( +
+{remainingCount}
+ )} +
+ ); + }; + + return ( +
+ {renderImageGrid()} + {hasCaption && } +
+ ); +}; + +export default MGalleryBody; diff --git a/apps/web/src/components/views/messages/MessageEvent.tsx b/apps/web/src/components/views/messages/MessageEvent.tsx index 549f38226c6..c10c7194815 100644 --- a/apps/web/src/components/views/messages/MessageEvent.tsx +++ b/apps/web/src/components/views/messages/MessageEvent.tsx @@ -33,6 +33,7 @@ import MPollBody from "./MPollBody"; import MLocationBody from "./MLocationBody"; import MjolnirBody from "./MjolnirBody"; import MBeaconBody from "./MBeaconBody"; +import MGalleryBody from "./MGalleryBody"; import { type GetRelationsForEvent, type IEventTileOps } from "../rooms/EventTile"; import { DecryptionFailureBodyFactory, @@ -63,6 +64,8 @@ export interface IOperableEventTile { getEventTileOps(): IEventTileOps | null; } +const MSG_TYPE_GALLERY = "dm.filament.gallery"; + const baseBodyTypes = new Map>([ [MsgType.Text, TextualBody], [MsgType.Notice, TextualBody], @@ -71,6 +74,7 @@ const baseBodyTypes = new Map>([ [MsgType.File, (props: IBodyProps) => renderMBody(props, FileBodyFactory)!], [MsgType.Audio, MVoiceOrAudioBody], [MsgType.Video, VideoBodyFactory], + [MSG_TYPE_GALLERY, MGalleryBody], ]); const baseEvTypes = new Map>([ [EventType.Sticker, MStickerBody], @@ -294,9 +298,10 @@ export default class MessageEvent extends React.Component implements IMe } const hasCaption = - [MsgType.Image, MsgType.File, MsgType.Audio, MsgType.Video].includes(msgtype as MsgType) && - content.filename && - content.filename !== content.body; + ([MsgType.Image, MsgType.File, MsgType.Audio, MsgType.Video].includes(msgtype as MsgType) && + content.filename && + content.filename !== content.body) || + msgtype === MSG_TYPE_GALLERY; const bodyProps: IBodyProps = { ref: this.body, mxEvent: this.props.mxEvent, diff --git a/apps/web/src/i18n/strings/en_EN.json b/apps/web/src/i18n/strings/en_EN.json index 481a2ad478b..3dceb05ed7d 100644 --- a/apps/web/src/i18n/strings/en_EN.json +++ b/apps/web/src/i18n/strings/en_EN.json @@ -3743,7 +3743,11 @@ "one_user": "%(displayName)s is typing …", "two_users": "%(names)s and %(lastPerson)s are typing …" }, - "undecryptable_tooltip": "This message could not be decrypted" + "undecryptable_tooltip": "This message could not be decrypted", + "gallery": { + "caption_default": "Gallery", + "upload_failed": "Failed to upload gallery" + } }, "truncated_list_n_more": { "other": "And %(count)s more..."