From 299fcfd4d1a98997f821148b660131dc8512f2b5 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Tue, 9 Mar 2021 19:33:54 +0100 Subject: [PATCH 01/11] extract base class from image tile to inherit from for video tile --- .../room/timeline/tiles/BaseMediaTile.js | 149 ++++++++++++++++++ .../session/room/timeline/tiles/ImageTile.js | 114 +------------- 2 files changed, 151 insertions(+), 112 deletions(-) create mode 100644 src/domain/session/room/timeline/tiles/BaseMediaTile.js diff --git a/src/domain/session/room/timeline/tiles/BaseMediaTile.js b/src/domain/session/room/timeline/tiles/BaseMediaTile.js new file mode 100644 index 00000000..9cd3222d --- /dev/null +++ b/src/domain/session/room/timeline/tiles/BaseMediaTile.js @@ -0,0 +1,149 @@ +/* +Copyright 2020 Bruno Windels +Copyright 2020 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {MessageTile} from "./MessageTile.js"; +import {SendStatus} from "../../../../../matrix/room/sending/PendingEvent.js"; +const MAX_HEIGHT = 300; +const MAX_WIDTH = 400; + +export class BaseMediaTile extends MessageTile { + constructor(options) { + super(options); + this._decryptedThumbnail = null; + this._decryptedFile = null; + this._error = null; + if (!this.isPending) { + this._tryLoadEncryptedThumbnail(); + } + } + + get isUploading() { + return this.isPending && this._entry.pendingEvent.status === SendStatus.UploadingAttachments; + } + + get uploadPercentage() { + const {pendingEvent} = this._entry; + return pendingEvent && Math.round((pendingEvent.attachmentsSentBytes / pendingEvent.attachmentsTotalBytes) * 100); + } + + get sendStatus() { + const {pendingEvent} = this._entry; + switch (pendingEvent?.status) { + case SendStatus.Waiting: + return this.i18n`Waiting…`; + case SendStatus.EncryptingAttachments: + case SendStatus.Encrypting: + return this.i18n`Encrypting…`; + case SendStatus.UploadingAttachments: + return this.i18n`Uploading…`; + case SendStatus.Sending: + return this.i18n`Sending…`; + case SendStatus.Error: + return this.i18n`Error: ${pendingEvent.error.message}`; + default: + return ""; + } + } + + get thumbnailUrl() { + if (this._decryptedThumbnail) { + return this._decryptedThumbnail.url; + } else { + const thumbnailMxc = this._getContent().info?.thumbnail_url; + if (thumbnailMxc) { + return this._mediaRepository.mxcUrlThumbnail(thumbnailMxc, this.width, this.height, "scale"); + } + } + if (this._entry.isPending) { + const attachment = this._entry.pendingEvent.getAttachment("info.thumbnail_url"); + return attachment && attachment.localPreview.url; + } + if (this.mimeType?.startsWith("image/")) { + if (this._decryptedFile) { + return this._decryptedFile.url; + } else { + const mxcUrl = this._getContent()?.url; + if (typeof mxcUrl === "string") { + return this._mediaRepository.mxcUrlThumbnail(mxcUrl, this.width, this.height, "scale"); + } + } + } + return ""; + } + + get width() { + const info = this._getContent()?.info; + return Math.round(info?.w * this._scaleFactor()); + } + + get height() { + const info = this._getContent()?.info; + return Math.round(info?.h * this._scaleFactor()); + } + + get mimeType() { + const info = this._getContent()?.info; + return info?.mimetype; + } + + get label() { + return this._getContent().body; + } + + get error() { + if (this._error) { + return `Could not decrypt media: ${this._error.message}`; + } + return null; + } + + + async _loadEncryptedFile(file) { + const blob = await this._mediaRepository.downloadEncryptedFile(file, true); + if (this.isDisposed) { + blob.dispose(); + return; + } + return this.track(blob); + } + + async _tryLoadEncryptedThumbnail() { + try { + const thumbnailFile = this._getContent().info?.thumbnail_file; + const file = this._getContent().file; + if (thumbnailFile) { + this._decryptedThumbnail = await this._loadEncryptedFile(thumbnailFile); + this.emitChange("thumbnailUrl"); + } else if (file && this.mimeType?.startsWith("image/")) { // is the main resource an image? then try that for a thumbnail + this._decryptedFile = await this._loadEncryptedFile(file); + this.emitChange("thumbnailUrl"); + } + } catch (err) { + this._error = err; + this.emitChange("error"); + } + } + + _scaleFactor() { + const info = this._getContent()?.info; + const scaleHeightFactor = MAX_HEIGHT / info?.h; + const scaleWidthFactor = MAX_WIDTH / info?.w; + // take the smallest scale factor, to respect all constraints + // we should not upscale images, so limit scale factor to 1 upwards + return Math.min(scaleWidthFactor, scaleHeightFactor, 1); + } +} diff --git a/src/domain/session/room/timeline/tiles/ImageTile.js b/src/domain/session/room/timeline/tiles/ImageTile.js index fc70551d..eae2b926 100644 --- a/src/domain/session/room/timeline/tiles/ImageTile.js +++ b/src/domain/session/room/timeline/tiles/ImageTile.js @@ -15,20 +15,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {MessageTile} from "./MessageTile.js"; -import {SendStatus} from "../../../../../matrix/room/sending/PendingEvent.js"; -const MAX_HEIGHT = 300; -const MAX_WIDTH = 400; +import {BaseMediaTile} from "./BaseMediaTile.js"; -export class ImageTile extends MessageTile { +export class ImageTile extends BaseMediaTile { constructor(options) { super(options); - this._decryptedThumbail = null; - this._decryptedImage = null; - this._error = null; - if (!this.isPending) { - this.tryLoadEncryptedThumbnail(); - } this._lightboxUrl = this.urlCreator.urlForSegments([ // ensure the right room is active if in grid view this.navigation.segment("room", this._room.id), @@ -36,32 +27,6 @@ export class ImageTile extends MessageTile { ]); } - async _loadEncryptedFile(file) { - const blob = await this._mediaRepository.downloadEncryptedFile(file, true); - if (this.isDisposed) { - blob.dispose(); - return; - } - return this.track(blob); - } - - async tryLoadEncryptedThumbnail() { - try { - const thumbnailFile = this._getContent().info?.thumbnail_file; - const file = this._getContent().file; - if (thumbnailFile) { - this._decryptedThumbail = await this._loadEncryptedFile(thumbnailFile); - this.emitChange("thumbnailUrl"); - } else if (file) { - this._decryptedImage = await this._loadEncryptedFile(file); - this.emitChange("thumbnailUrl"); - } - } catch (err) { - this._error = err; - this.emitChange("error"); - } - } - get lightboxUrl() { if (!this.isPending) { return this._lightboxUrl; @@ -69,81 +34,6 @@ export class ImageTile extends MessageTile { return ""; } - get isUploading() { - return this.isPending && this._entry.pendingEvent.status === SendStatus.UploadingAttachments; - } - - get uploadPercentage() { - const {pendingEvent} = this._entry; - return pendingEvent && Math.round((pendingEvent.attachmentsSentBytes / pendingEvent.attachmentsTotalBytes) * 100); - } - - get sendStatus() { - const {pendingEvent} = this._entry; - switch (pendingEvent?.status) { - case SendStatus.Waiting: - return this.i18n`Waiting…`; - case SendStatus.EncryptingAttachments: - case SendStatus.Encrypting: - return this.i18n`Encrypting…`; - case SendStatus.UploadingAttachments: - return this.i18n`Uploading…`; - case SendStatus.Sending: - return this.i18n`Sending…`; - case SendStatus.Error: - return this.i18n`Error: ${pendingEvent.error.message}`; - default: - return ""; - } - } - - get thumbnailUrl() { - if (this._decryptedThumbail) { - return this._decryptedThumbail.url; - } else if (this._decryptedImage) { - return this._decryptedImage.url; - } - if (this._entry.isPending) { - const attachment = this._entry.pendingEvent.getAttachment("url"); - return attachment && attachment.localPreview.url; - } - const mxcUrl = this._getContent()?.url; - if (typeof mxcUrl === "string") { - return this._mediaRepository.mxcUrlThumbnail(mxcUrl, this.thumbnailWidth, this.thumbnailHeight, "scale"); - } - return ""; - } - - _scaleFactor() { - const info = this._getContent()?.info; - const scaleHeightFactor = MAX_HEIGHT / info?.h; - const scaleWidthFactor = MAX_WIDTH / info?.w; - // take the smallest scale factor, to respect all constraints - // we should not upscale images, so limit scale factor to 1 upwards - return Math.min(scaleWidthFactor, scaleHeightFactor, 1); - } - - get thumbnailWidth() { - const info = this._getContent()?.info; - return Math.round(info?.w * this._scaleFactor()); - } - - get thumbnailHeight() { - const info = this._getContent()?.info; - return Math.round(info?.h * this._scaleFactor()); - } - - get label() { - return this._getContent().body; - } - - get error() { - if (this._error) { - return `Could not decrypt image: ${this._error.message}`; - } - return null; - } - get shape() { return "image"; } From b955cac7ce5df52dd118d50f329c596d95a5eac3 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Tue, 9 Mar 2021 19:34:22 +0100 Subject: [PATCH 02/11] extract base class from image view to inherit video view from --- .../ui/session/room/timeline/BaseMediaView.js | 60 +++++++++++++++++++ .../web/ui/session/room/timeline/ImageView.js | 47 ++------------- 2 files changed, 65 insertions(+), 42 deletions(-) create mode 100644 src/platform/web/ui/session/room/timeline/BaseMediaView.js diff --git a/src/platform/web/ui/session/room/timeline/BaseMediaView.js b/src/platform/web/ui/session/room/timeline/BaseMediaView.js new file mode 100644 index 00000000..7dc538e6 --- /dev/null +++ b/src/platform/web/ui/session/room/timeline/BaseMediaView.js @@ -0,0 +1,60 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {TemplateView} from "../../../general/TemplateView.js"; +import {renderMessage} from "./common.js"; + +export class BaseMediaView extends TemplateView { + render(t, vm) { + const heightRatioPercent = (vm.height / vm.width) * 100; + let spacerStyle = `padding-top: ${heightRatioPercent}%;`; + if (vm.platform.isIE11) { + // preserving aspect-ratio in a grid with padding percentages + // does not work in IE11, so we assume people won't use it + // with viewports narrower than 400px where thumbnails will get + // scaled. If they do, the thumbnail will still scale, but + // there will be whitespace underneath the picture + // An alternative would be to use position: absolute but that + // can slow down rendering, and was bleeding through the lightbox. + spacerStyle = `height: ${vm.height}px`; + } + const children = [ + t.div({className: "spacer", style: spacerStyle}), + this.renderMedia(t, vm), + t.time(vm.date + " " + vm.time), + ]; + if (vm.isPending) { + const cancel = t.button({onClick: () => vm.abortSending(), className: "link"}, vm.i18n`Cancel`); + const sendStatus = t.div({ + className: { + sendStatus: true, + hidden: vm => !vm.sendStatus + }, + }, [vm => vm.sendStatus, " ", cancel]); + const progress = t.progress({ + min: 0, + max: 100, + value: vm => vm.uploadPercentage, + className: {hidden: vm => !vm.isUploading} + }); + children.push(sendStatus, progress); + } + return renderMessage(t, vm, [ + t.div({className: "media", style: `max-width: ${vm.width}px`}, children), + t.if(vm => vm.error, t.createTemplate((t, vm) => t.p({className: "error"}, vm.error))) + ]); + } +} diff --git a/src/platform/web/ui/session/room/timeline/ImageView.js b/src/platform/web/ui/session/room/timeline/ImageView.js index 855ab23f..f4c1ecf7 100644 --- a/src/platform/web/ui/session/room/timeline/ImageView.js +++ b/src/platform/web/ui/session/room/timeline/ImageView.js @@ -14,54 +14,17 @@ See the License for the specific language governing permissions and limitations under the License. */ -import {TemplateView} from "../../../general/TemplateView.js"; -import {renderMessage} from "./common.js"; +import {BaseMediaView} from "./BaseMediaView.js"; -export class ImageView extends TemplateView { - render(t, vm) { - const heightRatioPercent = (vm.thumbnailHeight / vm.thumbnailWidth) * 100; - let spacerStyle = `padding-top: ${heightRatioPercent}%;`; - if (vm.platform.isIE11) { - // preserving aspect-ratio in a grid with padding percentages - // does not work in IE11, so we assume people won't use it - // with viewports narrower than 400px where thumbnails will get - // scaled. If they do, the thumbnail will still scale, but - // there will be whitespace underneath the picture - // An alternative would be to use position: absolute but that - // can slow down rendering, and was bleeding through the lightbox. - spacerStyle = `height: ${vm.thumbnailHeight}px`; - } +export class ImageView extends BaseMediaView { + renderMedia(t, vm) { const img = t.img({ loading: "lazy", src: vm => vm.thumbnailUrl, alt: vm => vm.label, title: vm => vm.label, - style: `max-width: ${vm.thumbnailWidth}px; max-height: ${vm.thumbnailHeight}px;` + style: `max-width: ${vm.width}px; max-height: ${vm.height}px;` }); - const children = [ - t.div({className: "spacer", style: spacerStyle}), - vm.isPending ? img : t.a({href: vm.lightboxUrl}, img), - t.time(vm.date + " " + vm.time), - ]; - if (vm.isPending) { - const cancel = t.button({onClick: () => vm.abortSending(), className: "link"}, vm.i18n`Cancel`); - const sendStatus = t.div({ - className: { - sendStatus: true, - hidden: vm => !vm.sendStatus - }, - }, [vm => vm.sendStatus, " ", cancel]); - const progress = t.progress({ - min: 0, - max: 100, - value: vm => vm.uploadPercentage, - className: {hidden: vm => !vm.isUploading} - }); - children.push(sendStatus, progress); - } - return renderMessage(t, vm, [ - t.div({className: "picture", style: `max-width: ${vm.thumbnailWidth}px`}, children), - t.if(vm => vm.error, t.createTemplate((t, vm) => t.p({className: "error"}, vm.error))) - ]); + return vm.isPending ? img : t.a({href: vm.lightboxUrl}, img); } } From ee6f3e5457a1018c7f0b62502374afdf4a295874 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Tue, 9 Mar 2021 19:35:10 +0100 Subject: [PATCH 03/11] render video messages --- .../session/room/timeline/tiles/VideoTile.js | 43 +++++++++++++++++++ .../session/room/timeline/tilesCreator.js | 3 ++ .../web/ui/css/themes/element/theme.css | 25 +++++------ src/platform/web/ui/css/timeline.css | 5 ++- src/platform/web/ui/general/html.js | 2 +- .../web/ui/session/room/TimelineList.js | 2 + .../web/ui/session/room/timeline/VideoView.js | 38 ++++++++++++++++ 7 files changed, 103 insertions(+), 15 deletions(-) create mode 100644 src/domain/session/room/timeline/tiles/VideoTile.js create mode 100644 src/platform/web/ui/session/room/timeline/VideoView.js diff --git a/src/domain/session/room/timeline/tiles/VideoTile.js b/src/domain/session/room/timeline/tiles/VideoTile.js new file mode 100644 index 00000000..13ed68f9 --- /dev/null +++ b/src/domain/session/room/timeline/tiles/VideoTile.js @@ -0,0 +1,43 @@ +/* +Copyright 2020 Bruno Windels +Copyright 2020 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {BaseMediaTile} from "./BaseMediaTile.js"; + +export class VideoTile extends BaseMediaTile { + async loadVideo() { + const file = this._getContent().file; + if (file && !this._decryptedFile) { + this._decryptedFile = await this._loadEncryptedFile(file); + this.emitChange("videoUrl"); + } + } + + get videoUrl() { + if (this._decryptedFile) { + return this._decryptedFile.url; + } + const mxcUrl = this._getContent()?.url; + if (typeof mxcUrl === "string") { + return this._mediaRepository.mxcUrl(mxcUrl); + } + return ""; + } + + get shape() { + return "video"; + } +} diff --git a/src/domain/session/room/timeline/tilesCreator.js b/src/domain/session/room/timeline/tilesCreator.js index 1901efe2..ed43cd3a 100644 --- a/src/domain/session/room/timeline/tilesCreator.js +++ b/src/domain/session/room/timeline/tilesCreator.js @@ -17,6 +17,7 @@ limitations under the License. import {GapTile} from "./tiles/GapTile.js"; import {TextTile} from "./tiles/TextTile.js"; import {ImageTile} from "./tiles/ImageTile.js"; +import {VideoTile} from "./tiles/VideoTile.js"; import {FileTile} from "./tiles/FileTile.js"; import {LocationTile} from "./tiles/LocationTile.js"; import {RoomNameTile} from "./tiles/RoomNameTile.js"; @@ -44,6 +45,8 @@ export function tilesCreator(baseOptions) { return new TextTile(options); case "m.image": return new ImageTile(options); + case "m.video": + return new VideoTile(options); case "m.file": return new FileTile(options); case "m.location": diff --git a/src/platform/web/ui/css/themes/element/theme.css b/src/platform/web/ui/css/themes/element/theme.css index 4c6e1dbf..4c41ad97 100644 --- a/src/platform/web/ui/css/themes/element/theme.css +++ b/src/platform/web/ui/css/themes/element/theme.css @@ -509,7 +509,7 @@ ul.Timeline > li.messageStatus .message-container > p { .message-container { padding: 1px 10px 0px 10px; margin: 5px 10px 0 10px; - /* so the .picture can grow horizontally and its spacer can grow vertically */ + /* so the .media can grow horizontally and its spacer can grow vertically */ width: 100%; } @@ -555,14 +555,14 @@ ul.Timeline > li.messageStatus .message-container > p { } -.message-container .picture { +.message-container .media { display: grid; margin-top: 4px; width: 100%; } -.message-container .picture > a { +.message-container .media > a { text-decoration: none; width: 100%; display: block; @@ -570,12 +570,12 @@ ul.Timeline > li.messageStatus .message-container > p { /* .spacer grows with an inline padding-top to the size of the image, so the timeline doesn't jump when the image loads */ -.message-container .picture > * { +.message-container .media > * { grid-row: 1; grid-column: 1; } -.message-container .picture img { +.message-container .media img, .message-container .media video { width: 100%; height: auto; /* for IE11 to still scale even though the spacer is too tall */ @@ -587,29 +587,30 @@ so the timeline doesn't jump when the image loads */ where we can trust the spacer to always have the correct height, otherwise the image starts with height 0 and with loading=lazy only loads when the top comes into view*/ -.hydrogen:not(.legacy) .message-container .picture img { +.hydrogen:not(.legacy) .message-container .media img, +.hydrogen:not(.legacy) .message-container .media video { align-self: stretch; } -.message-container .picture > .sendStatus { +.message-container .media > .sendStatus { align-self: end; justify-self: start; font-size: 0.8em; } -.message-container .picture > progress { +.message-container .media > progress { align-self: center; justify-self: center; width: 75%; } -.message-container .picture > time { +.message-container .media > time { align-self: end; justify-self: end; } -.message-container .picture > time, -.message-container .picture > .sendStatus { +.message-container .media > time, +.message-container .media > .sendStatus { color: #2e2f32; display: block; padding: 2px; @@ -617,7 +618,7 @@ only loads when the top comes into view*/ background-color: rgba(255, 255, 255, 0.75); border-radius: 4px; } -.message-container .picture > .spacer { +.message-container .media > .spacer { /* TODO: can we implement this with a pseudo element? or perhaps they are not grid items? */ width: 100%; /* don't stretch height as it is a spacer, just in case it doesn't match with image height */ diff --git a/src/platform/web/ui/css/timeline.css b/src/platform/web/ui/css/timeline.css index ee9ffdbb..8a766a54 100644 --- a/src/platform/web/ui/css/timeline.css +++ b/src/platform/web/ui/css/timeline.css @@ -37,11 +37,12 @@ limitations under the License. margin: 5px 0; } -.message-container .picture { +.message-container .media { display: block; } -.message-container .picture > img { +.message-container .media > img, +.message-container .media > video { display: block; } diff --git a/src/platform/web/ui/general/html.js b/src/platform/web/ui/general/html.js index a965a6ee..19b670d7 100644 --- a/src/platform/web/ui/general/html.js +++ b/src/platform/web/ui/general/html.js @@ -94,7 +94,7 @@ export const TAG_NAMES = { [HTML_NS]: [ "br", "a", "ol", "ul", "li", "div", "h1", "h2", "h3", "h4", "h5", "h6", "p", "strong", "em", "span", "img", "section", "main", "article", "aside", - "pre", "button", "time", "input", "textarea", "label", "form", "progress", "output"], + "pre", "button", "time", "input", "textarea", "label", "form", "progress", "output", "video"], [SVG_NS]: ["svg", "circle"] }; diff --git a/src/platform/web/ui/session/room/TimelineList.js b/src/platform/web/ui/session/room/TimelineList.js index 9d946fce..414649be 100644 --- a/src/platform/web/ui/session/room/TimelineList.js +++ b/src/platform/web/ui/session/room/TimelineList.js @@ -18,6 +18,7 @@ import {ListView} from "../../general/ListView.js"; import {GapView} from "./timeline/GapView.js"; import {TextMessageView} from "./timeline/TextMessageView.js"; import {ImageView} from "./timeline/ImageView.js"; +import {VideoView} from "./timeline/VideoView.js"; import {FileView} from "./timeline/FileView.js"; import {MissingAttachmentView} from "./timeline/MissingAttachmentView.js"; import {AnnouncementView} from "./timeline/AnnouncementView.js"; @@ -30,6 +31,7 @@ function viewClassForEntry(entry) { case "message-status": return TextMessageView; case "image": return ImageView; + case "video": return VideoView; case "file": return FileView; case "missing-attachment": return MissingAttachmentView; } diff --git a/src/platform/web/ui/session/room/timeline/VideoView.js b/src/platform/web/ui/session/room/timeline/VideoView.js new file mode 100644 index 00000000..37a6af1d --- /dev/null +++ b/src/platform/web/ui/session/room/timeline/VideoView.js @@ -0,0 +1,38 @@ +/* +Copyright 2020 Bruno Windels + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import {BaseMediaView} from "./BaseMediaView.js"; + +export class VideoView extends BaseMediaView { + renderMedia(t, vm) { + return t.video({ + // provide empty data url if video is not decrypted yet. + // Chrome/Electron need this to enable the play button. + src: vm => vm.videoUrl || `data:${vm.mimeType},`, + title: vm => vm.label, + controls: true, + preload: "none", + poster: vm => vm.thumbnailUrl, + onPlay: async evt => { + if (!vm.videoUrl) { + await vm.loadVideo(); + evt.target.play(); + } + }, + style: `max-width: ${vm.width}px; max-height: ${vm.height}px;` + }); + } +} From c6ff56a9424e40868ecccd0d88e082d87b6f9333 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Tue, 9 Mar 2021 19:35:25 +0100 Subject: [PATCH 04/11] send video messages --- src/domain/session/room/RoomViewModel.js | 51 +++++++++++++- src/platform/web/Platform.js | 6 +- src/platform/web/dom/ImageHandle.js | 67 ++++++++++++++++--- .../web/ui/session/room/MessageComposer.js | 1 + 4 files changed, 114 insertions(+), 11 deletions(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 83910675..6a83446a 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -187,6 +187,43 @@ export class RoomViewModel extends ViewModel { }); } + async _pickAndSendVideo() { + try { + if (!this.platform.hasReadPixelPermission()) { + alert("Please allow canvas image data access, so we can scale your images down."); + return; + } + const file = await this.platform.openFile("video/*"); + if (!file) { + return; + } + if (!file.blob.mimeType.startsWith("video/")) { + return this._sendFile(file); + } + let video = await this.platform.loadVideo(file.blob); + const content = { + body: file.name, + msgtype: "m.video", + info: videoToInfo(video) + }; + const attachments = { + "url": this._room.createAttachment(video.blob, file.name), + }; + + const limit = await this.platform.settingsStorage.getInt("sentImageSizeLimit"); + const maxDimension = limit || Math.min(video.maxDimension, 800); + const thumbnail = await video.scale(maxDimension); + content.info.thumbnail_info = imageToInfo(thumbnail); + attachments["info.thumbnail_url"] = + this._room.createAttachment(thumbnail.blob, file.name); + await this._room.sendEvent("m.room.message", content, attachments); + } catch (err) { + this._sendError = err; + this.emitChange("error"); + console.error(err.stack); + } + } + async _pickAndSendPicture() { try { if (!this.platform.hasReadPixelPermission()) { @@ -221,7 +258,9 @@ export class RoomViewModel extends ViewModel { } await this._room.sendEvent("m.room.message", content, attachments); } catch (err) { - console.error(err); + this._sendError = err; + this.emitChange("error"); + console.error(err.stack); } } @@ -259,6 +298,10 @@ class ComposerViewModel extends ViewModel { this._roomVM._pickAndSendFile(); } + sendVideo() { + this._roomVM._pickAndSendVideo(); + } + get canSend() { return !this._isEmpty; } @@ -283,3 +326,9 @@ function imageToInfo(image) { size: image.blob.size }; } + +function videoToInfo(video) { + const info = imageToInfo(video); + info.duration = video.duration; + return info; +} diff --git a/src/platform/web/Platform.js b/src/platform/web/Platform.js index 565aefe3..e4ad3a88 100644 --- a/src/platform/web/Platform.js +++ b/src/platform/web/Platform.js @@ -32,7 +32,7 @@ import {Crypto} from "./dom/Crypto.js"; import {estimateStorageUsage} from "./dom/StorageEstimate.js"; import {WorkerPool} from "./dom/WorkerPool.js"; import {BlobHandle} from "./dom/BlobHandle.js"; -import {hasReadPixelPermission, ImageHandle} from "./dom/ImageHandle.js"; +import {hasReadPixelPermission, ImageHandle, VideoHandle} from "./dom/ImageHandle.js"; import {downloadInIframe} from "./dom/download.js"; function addScript(src) { @@ -184,6 +184,10 @@ export class Platform { return ImageHandle.fromBlob(blob); } + async loadVideo(blob) { + return VideoHandle.fromBlob(blob); + } + hasReadPixelPermission() { return hasReadPixelPermission(); } diff --git a/src/platform/web/dom/ImageHandle.js b/src/platform/web/dom/ImageHandle.js index 8676364c..ae05e868 100644 --- a/src/platform/web/dom/ImageHandle.js +++ b/src/platform/web/dom/ImageHandle.js @@ -27,18 +27,18 @@ export class ImageHandle { this.blob = blob; this.width = width; this.height = height; - this._imgElement = imgElement; + this._domElement = imgElement; } get maxDimension() { return Math.max(this.width, this.height); } - async _getImgElement() { - if (!this._imgElement) { - this._imgElement = await loadImgFromBlob(this.blob); + async _getDomElement() { + if (!this._domElement) { + this._domElement = await loadImgFromBlob(this.blob); } - return this._imgElement; + return this._domElement; } async scale(maxDimension) { @@ -46,18 +46,18 @@ export class ImageHandle { const scaleFactor = Math.min(1, maxDimension / (aspectRatio >= 1 ? this.width : this.height)); const scaledWidth = Math.round(this.width * scaleFactor); const scaledHeight = Math.round(this.height * scaleFactor); - const canvas = document.createElement("canvas"); canvas.width = scaledWidth; canvas.height = scaledHeight; const ctx = canvas.getContext("2d"); - const img = await this._getImgElement(); - ctx.drawImage(img, 0, 0, scaledWidth, scaledHeight); + const drawableElement = await this._getDomElement(); + ctx.drawImage(drawableElement, 0, 0, scaledWidth, scaledHeight); let mimeType = this.blob.mimeType === "image/jpeg" ? "image/jpeg" : "image/png"; let nativeBlob; if (canvas.toBlob) { nativeBlob = await new Promise(resolve => canvas.toBlob(resolve, mimeType)); } else if (canvas.msToBlob) { + // TODO: provide a mimetype override in blob handle for this case mimeType = "image/png"; nativeBlob = canvas.msToBlob(); } else { @@ -72,6 +72,21 @@ export class ImageHandle { } } +export class VideoHandle extends ImageHandle { + get duration() { + if (typeof this._domElement.duration === "number") { + return Math.round(this._domElement.duration * 1000); + } + return undefined; + } + + static async fromBlob(blob) { + const video = await loadVideoFromBlob(blob); + const {videoWidth, videoHeight} = video; + return new VideoHandle(blob, videoWidth, videoHeight, video); + } +} + export function hasReadPixelPermission() { const canvas = document.createElement("canvas"); canvas.width = 1; @@ -91,7 +106,8 @@ export function hasReadPixelPermission() { async function loadImgFromBlob(blob) { const img = document.createElement("img"); let detach; - const loadPromise = new Promise((resolve, reject) => { + const loadPromise = new Promise((resolve, _reject) => { + const reject = evt => _reject(evt.target.error); detach = () => { img.removeEventListener("load", resolve); img.removeEventListener("error", reject); @@ -104,3 +120,36 @@ async function loadImgFromBlob(blob) { detach(); return img; } + +async function loadVideoFromBlob(blob) { + const video = document.createElement("video"); + video.muted = true; + let detach; + const loadPromise = new Promise((resolve, _reject) => { + const reject = evt => _reject(evt.target.error); + detach = () => { + video.removeEventListener("loadedmetadata", resolve); + video.removeEventListener("error", reject); + }; + video.addEventListener("loadedmetadata", resolve); + video.addEventListener("error", reject); + }); + video.src = blob.url; + video.load(); + await loadPromise; + // seek to the first 1/10s to make sure that drawing the video + // on a canvas won't give a blank image + const seekPromise = new Promise((resolve, _reject) => { + const reject = evt => _reject(evt.target.error); + detach = () => { + video.removeEventListener("seeked", resolve); + video.removeEventListener("error", reject); + }; + video.addEventListener("seeked", resolve); + video.addEventListener("error", reject); + }); + video.currentTime = 0.1; + await seekPromise; + detach(); + return video; +} diff --git a/src/platform/web/ui/session/room/MessageComposer.js b/src/platform/web/ui/session/room/MessageComposer.js index 32f3fc05..8da69478 100644 --- a/src/platform/web/ui/session/room/MessageComposer.js +++ b/src/platform/web/ui/session/room/MessageComposer.js @@ -66,6 +66,7 @@ export class MessageComposer extends TemplateView { } else { const vm = this.value; this._attachmentPopup = new Popup(new Menu([ + Menu.option(vm.i18n`Send video`, () => vm.sendVideo()).setIcon("video"), Menu.option(vm.i18n`Send picture`, () => vm.sendPicture()).setIcon("picture"), Menu.option(vm.i18n`Send file`, () => vm.sendFile()).setIcon("file"), ])); From 35e6dffd0b486bf87f155b1e3879c9a39e870a71 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 10 Mar 2021 10:01:07 +0100 Subject: [PATCH 05/11] don't rely on mime-types to decide to use main url for thumbnail so it does not break when clients forget to set the mimetype --- src/domain/session/room/timeline/tiles/BaseMediaTile.js | 9 ++++++--- src/domain/session/room/timeline/tiles/VideoTile.js | 4 ++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/BaseMediaTile.js b/src/domain/session/room/timeline/tiles/BaseMediaTile.js index 9cd3222d..ea3812d7 100644 --- a/src/domain/session/room/timeline/tiles/BaseMediaTile.js +++ b/src/domain/session/room/timeline/tiles/BaseMediaTile.js @@ -72,7 +72,7 @@ export class BaseMediaTile extends MessageTile { const attachment = this._entry.pendingEvent.getAttachment("info.thumbnail_url"); return attachment && attachment.localPreview.url; } - if (this.mimeType?.startsWith("image/")) { + if (this._isMainResourceImage()) { if (this._decryptedFile) { return this._decryptedFile.url; } else { @@ -111,7 +111,6 @@ export class BaseMediaTile extends MessageTile { return null; } - async _loadEncryptedFile(file) { const blob = await this._mediaRepository.downloadEncryptedFile(file, true); if (this.isDisposed) { @@ -128,7 +127,7 @@ export class BaseMediaTile extends MessageTile { if (thumbnailFile) { this._decryptedThumbnail = await this._loadEncryptedFile(thumbnailFile); this.emitChange("thumbnailUrl"); - } else if (file && this.mimeType?.startsWith("image/")) { // is the main resource an image? then try that for a thumbnail + } else if (file && this._isMainResourceImage()) { // is the main resource an image? then try that for a thumbnail this._decryptedFile = await this._loadEncryptedFile(file); this.emitChange("thumbnailUrl"); } @@ -146,4 +145,8 @@ export class BaseMediaTile extends MessageTile { // we should not upscale images, so limit scale factor to 1 upwards return Math.min(scaleWidthFactor, scaleHeightFactor, 1); } + + _isMainResourceImage() { + return true; // overwritten in VideoTile + } } diff --git a/src/domain/session/room/timeline/tiles/VideoTile.js b/src/domain/session/room/timeline/tiles/VideoTile.js index 13ed68f9..fbcf26c6 100644 --- a/src/domain/session/room/timeline/tiles/VideoTile.js +++ b/src/domain/session/room/timeline/tiles/VideoTile.js @@ -40,4 +40,8 @@ export class VideoTile extends BaseMediaTile { get shape() { return "video"; } + + _isMainResourceImage() { + return false; + } } From 9bc1d38d7370dd5938a94298ac368620d0d6bbba Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 10 Mar 2021 13:38:28 +0100 Subject: [PATCH 06/11] extract dom event => promise code --- src/platform/web/dom/ImageHandle.js | 33 ++++----------------------- src/platform/web/dom/utils.js | 35 +++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 29 deletions(-) create mode 100644 src/platform/web/dom/utils.js diff --git a/src/platform/web/dom/ImageHandle.js b/src/platform/web/dom/ImageHandle.js index ae05e868..fae3799e 100644 --- a/src/platform/web/dom/ImageHandle.js +++ b/src/platform/web/dom/ImageHandle.js @@ -15,6 +15,7 @@ limitations under the License. */ import {BlobHandle} from "./BlobHandle.js"; +import {domEventAsPromise} from "./utils.js"; export class ImageHandle { static async fromBlob(blob) { @@ -106,15 +107,7 @@ export function hasReadPixelPermission() { async function loadImgFromBlob(blob) { const img = document.createElement("img"); let detach; - const loadPromise = new Promise((resolve, _reject) => { - const reject = evt => _reject(evt.target.error); - detach = () => { - img.removeEventListener("load", resolve); - img.removeEventListener("error", reject); - }; - img.addEventListener("load", resolve); - img.addEventListener("error", reject); - }); + const loadPromise = domEventAsPromise(img, "load"); img.src = blob.url; await loadPromise; detach(); @@ -124,32 +117,14 @@ async function loadImgFromBlob(blob) { async function loadVideoFromBlob(blob) { const video = document.createElement("video"); video.muted = true; - let detach; - const loadPromise = new Promise((resolve, _reject) => { - const reject = evt => _reject(evt.target.error); - detach = () => { - video.removeEventListener("loadedmetadata", resolve); - video.removeEventListener("error", reject); - }; - video.addEventListener("loadedmetadata", resolve); - video.addEventListener("error", reject); - }); + const loadPromise = domEventAsPromise(video, "loadedmetadata"); video.src = blob.url; video.load(); await loadPromise; // seek to the first 1/10s to make sure that drawing the video // on a canvas won't give a blank image - const seekPromise = new Promise((resolve, _reject) => { - const reject = evt => _reject(evt.target.error); - detach = () => { - video.removeEventListener("seeked", resolve); - video.removeEventListener("error", reject); - }; - video.addEventListener("seeked", resolve); - video.addEventListener("error", reject); - }); + const seekPromise = domEventAsPromise(video, "seeked"); video.currentTime = 0.1; await seekPromise; - detach(); return video; } diff --git a/src/platform/web/dom/utils.js b/src/platform/web/dom/utils.js new file mode 100644 index 00000000..43a26640 --- /dev/null +++ b/src/platform/web/dom/utils.js @@ -0,0 +1,35 @@ +/* +Copyright 2020 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export function domEventAsPromise(element, successEvent) { + return new Promise((resolve, reject) => { + let detach; + const handleError = evt => { + detach(); + reject(evt.target.error); + }; + const handleSuccess = () => { + detach(); + resolve(); + }; + detach = () => { + element.removeEventListener(successEvent, handleSuccess); + element.removeEventListener("error", handleError); + }; + element.addEventListener(successEvent, handleSuccess); + element.addEventListener("error", handleError); + }); +} From c8265b235803e119f3a32d96422d845bda1f362e Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 10 Mar 2021 13:39:04 +0100 Subject: [PATCH 07/11] also allow other video mime types, otherwise playback fails in e2ee room --- src/platform/web/dom/BlobHandle.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/platform/web/dom/BlobHandle.js b/src/platform/web/dom/BlobHandle.js index b029555b..32dd94c0 100644 --- a/src/platform/web/dom/BlobHandle.js +++ b/src/platform/web/dom/BlobHandle.js @@ -55,6 +55,8 @@ const ALLOWED_BLOB_MIMETYPES = { 'video/mp4': true, 'video/webm': true, 'video/ogg': true, + 'video/quicktime': true, + 'video/VP8': true, 'audio/mp4': true, 'audio/webm': true, From 948249bb3de9838d11a8a9fceefe85ea10826b28 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 10 Mar 2021 13:39:34 +0100 Subject: [PATCH 08/11] better error reporting for unsupported codecs when uploading --- src/domain/session/room/RoomViewModel.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 6a83446a..a6d5454a 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -188,12 +188,13 @@ export class RoomViewModel extends ViewModel { } async _pickAndSendVideo() { + let file; try { if (!this.platform.hasReadPixelPermission()) { alert("Please allow canvas image data access, so we can scale your images down."); return; } - const file = await this.platform.openFile("video/*"); + file = await this.platform.openFile("video/*"); if (!file) { return; } @@ -218,7 +219,11 @@ export class RoomViewModel extends ViewModel { this._room.createAttachment(thumbnail.blob, file.name); await this._room.sendEvent("m.room.message", content, attachments); } catch (err) { - this._sendError = err; + if (err instanceof window.MediaError && err.code === 4) { + this._sendError = new Error(`this browser does not support videos of type ${file?.blob.mimeType}.`); + } else { + this._sendError = err; + } this.emitChange("error"); console.error(err.stack); } From e8c8455f43e7537727a1672d92268b16bad5ce90 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 10 Mar 2021 13:39:58 +0100 Subject: [PATCH 09/11] safari hack/fix --- src/platform/web/dom/ImageHandle.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/platform/web/dom/ImageHandle.js b/src/platform/web/dom/ImageHandle.js index fae3799e..5ca1a586 100644 --- a/src/platform/web/dom/ImageHandle.js +++ b/src/platform/web/dom/ImageHandle.js @@ -124,6 +124,9 @@ async function loadVideoFromBlob(blob) { // seek to the first 1/10s to make sure that drawing the video // on a canvas won't give a blank image const seekPromise = domEventAsPromise(video, "seeked"); + // needed for safari to reliably fire the seeked event, + // somewhat hacky but using raf for example didn't do the trick + await new Promise(r => setTimeout(r, 200)); video.currentTime = 0.1; await seekPromise; return video; From a672b0c78a9c699b58a1a58b838b795a468f1538 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 10 Mar 2021 13:40:11 +0100 Subject: [PATCH 10/11] better error handling in video decoding --- .../room/timeline/tiles/BaseMediaTile.js | 7 ++- .../web/ui/session/room/timeline/VideoView.js | 53 +++++++++++++++---- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/src/domain/session/room/timeline/tiles/BaseMediaTile.js b/src/domain/session/room/timeline/tiles/BaseMediaTile.js index ea3812d7..26862902 100644 --- a/src/domain/session/room/timeline/tiles/BaseMediaTile.js +++ b/src/domain/session/room/timeline/tiles/BaseMediaTile.js @@ -106,11 +106,16 @@ export class BaseMediaTile extends MessageTile { get error() { if (this._error) { - return `Could not decrypt media: ${this._error.message}`; + return `Could not load media: ${this._error.message}`; } return null; } + setViewError(err) { + this._error = err; + this.emitChange("error"); + } + async _loadEncryptedFile(file) { const blob = await this._mediaRepository.downloadEncryptedFile(file, true); if (this.isDisposed) { diff --git a/src/platform/web/ui/session/room/timeline/VideoView.js b/src/platform/web/ui/session/room/timeline/VideoView.js index 37a6af1d..340cae6d 100644 --- a/src/platform/web/ui/session/room/timeline/VideoView.js +++ b/src/platform/web/ui/session/room/timeline/VideoView.js @@ -15,10 +15,11 @@ limitations under the License. */ import {BaseMediaView} from "./BaseMediaView.js"; +import {domEventAsPromise} from "../../../../dom/utils.js"; export class VideoView extends BaseMediaView { - renderMedia(t, vm) { - return t.video({ + renderMedia(t) { + const video = t.video({ // provide empty data url if video is not decrypted yet. // Chrome/Electron need this to enable the play button. src: vm => vm.videoUrl || `data:${vm.mimeType},`, @@ -26,13 +27,47 @@ export class VideoView extends BaseMediaView { controls: true, preload: "none", poster: vm => vm.thumbnailUrl, - onPlay: async evt => { - if (!vm.videoUrl) { - await vm.loadVideo(); - evt.target.play(); - } - }, - style: `max-width: ${vm.width}px; max-height: ${vm.height}px;` + onPlay: this._onPlay.bind(this), + style: vm => `max-width: ${vm.width}px; max-height: ${vm.height}px;${vm.isPending ? "z-index: -1": ""}` }); + + video.addEventListener("error", this._onError.bind(this)); + + return video; + } + + async _onPlay(evt) { + const vm = this.value; + // download and decrypt the video if needed, + if (!vm.videoUrl) { + try { + const video = evt.target; + // this will trigger the src to update + await vm.loadVideo(); + // important to only listen for this after src has changed, + // or we get the error for the placeholder data url + const loadPromise = domEventAsPromise(video, "loadeddata"); + // now, reload the video and play + video.load(); + await loadPromise; + video.play(); + } catch (err) {/* errors are already caught in error event handler */} + } + } + + _onError(evt) { + const vm = this.value; + const video = evt.target; + const err = video.error; + if (err instanceof window.MediaError && err.code === 4) { + if (!video.src.startsWith("data:")) { + vm.setViewError(new Error(`this browser does not support videos of type ${vm.mimeType}.`)); + } else { + // ignore placeholder url failing to load + return; + } + } else { + vm.setViewError(err); + } } } From 9506bf1b811fd6e1717c254bbd948a41d47382db Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Wed, 10 Mar 2021 13:43:55 +0100 Subject: [PATCH 11/11] clean up video upload error handling --- src/domain/session/room/RoomViewModel.js | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index a6d5454a..43eeb75c 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -188,20 +188,29 @@ export class RoomViewModel extends ViewModel { } async _pickAndSendVideo() { - let file; try { if (!this.platform.hasReadPixelPermission()) { alert("Please allow canvas image data access, so we can scale your images down."); return; } - file = await this.platform.openFile("video/*"); + const file = await this.platform.openFile("video/*"); if (!file) { return; } if (!file.blob.mimeType.startsWith("video/")) { return this._sendFile(file); } - let video = await this.platform.loadVideo(file.blob); + let video; + try { + video = await this.platform.loadVideo(file.blob); + } catch (err) { + // TODO: extract platform dependent code from view model + if (err instanceof window.MediaError && err.code === 4) { + throw new Error(`this browser does not support videos of type ${file?.blob.mimeType}.`); + } else { + throw err; + } + } const content = { body: file.name, msgtype: "m.video", @@ -219,11 +228,7 @@ export class RoomViewModel extends ViewModel { this._room.createAttachment(thumbnail.blob, file.name); await this._room.sendEvent("m.room.message", content, attachments); } catch (err) { - if (err instanceof window.MediaError && err.code === 4) { - this._sendError = new Error(`this browser does not support videos of type ${file?.blob.mimeType}.`); - } else { - this._sendError = err; - } + this._sendError = err; this.emitChange("error"); console.error(err.stack); }