Merge pull request #268 from vector-im/bwindels/video-messages

video messages

src/domain/session/room/RoomViewModel.js

@@ -187,6 +187,53 @@ export class RoomViewModel extends ViewModel {
         });
     }
 
+    async _pickAndSendVideo() {
+        try {
+            if (!this.platform.hasReadPixelPermission()) {
+                alert("Please allow canvas image data access, so we can scale your images down.");
+                return;
+            }
+            const file = await this.platform.openFile("video/*");
+            if (!file) {
+                return;
+            }
+            if (!file.blob.mimeType.startsWith("video/")) {
+                return this._sendFile(file);
+            }
+            let video;
+            try {
+                video = await this.platform.loadVideo(file.blob);
+            } catch (err) {
+                // TODO: extract platform dependent code from view model
+                if (err instanceof window.MediaError && err.code === 4) {
+                    throw new Error(`this browser does not support videos of type ${file?.blob.mimeType}.`);
+                } else {
+                    throw err;
+                }
+            }
+            const content = {
+                body: file.name,
+                msgtype: "m.video",
+                info: videoToInfo(video)
+            };
+            const attachments = {
+                "url": this._room.createAttachment(video.blob, file.name),
+            };
+
+            const limit = await this.platform.settingsStorage.getInt("sentImageSizeLimit");
+            const maxDimension = limit || Math.min(video.maxDimension, 800);
+            const thumbnail = await video.scale(maxDimension);
+            content.info.thumbnail_info = imageToInfo(thumbnail);
+            attachments["info.thumbnail_url"] = 
+                this._room.createAttachment(thumbnail.blob, file.name);
+            await this._room.sendEvent("m.room.message", content, attachments);
+        } catch (err) {
+            this._sendError = err;
+            this.emitChange("error");
+            console.error(err.stack);
+        }
+    }
+
     async _pickAndSendPicture() {
         try {
             if (!this.platform.hasReadPixelPermission()) {
@@ -221,7 +268,9 @@ export class RoomViewModel extends ViewModel {
             }
             await this._room.sendEvent("m.room.message", content, attachments);
         } catch (err) {
-            console.error(err);
+            this._sendError = err;
+            this.emitChange("error");
+            console.error(err.stack);
         }
     }
     
@@ -259,6 +308,10 @@ class ComposerViewModel extends ViewModel {
         this._roomVM._pickAndSendFile();
     }
 
+    sendVideo() {
+        this._roomVM._pickAndSendVideo();
+    }
+
     get canSend() {
         return !this._isEmpty;
     }
@@ -283,3 +336,9 @@ function imageToInfo(image) {
         size: image.blob.size
     };
 }
+
+function videoToInfo(video) {
+    const info = imageToInfo(video);
+    info.duration = video.duration;
+    return info;
+}

src/domain/session/room/timeline/tiles/BaseMediaTile.js

@@ -0,0 +1,157 @@
+/*
+Copyright 2020 Bruno Windels <bruno@windels.cloud>
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {MessageTile} from "./MessageTile.js";
+import {SendStatus} from "../../../../../matrix/room/sending/PendingEvent.js";
+const MAX_HEIGHT = 300;
+const MAX_WIDTH = 400;
+
+export class BaseMediaTile extends MessageTile {
+    constructor(options) {
+        super(options);
+        this._decryptedThumbnail = null;
+        this._decryptedFile = null;
+        this._error = null;
+        if (!this.isPending) {
+            this._tryLoadEncryptedThumbnail();
+        }
+    }
+
+    get isUploading() {
+        return this.isPending && this._entry.pendingEvent.status === SendStatus.UploadingAttachments;
+    }
+
+    get uploadPercentage() {
+        const {pendingEvent} = this._entry;
+        return pendingEvent && Math.round((pendingEvent.attachmentsSentBytes / pendingEvent.attachmentsTotalBytes) * 100);
+    }
+
+    get sendStatus() {
+        const {pendingEvent} = this._entry;
+        switch (pendingEvent?.status) {
+            case SendStatus.Waiting:
+                return this.i18n`Waiting…`;
+            case SendStatus.EncryptingAttachments:
+            case SendStatus.Encrypting:
+                return this.i18n`Encrypting…`;
+            case SendStatus.UploadingAttachments:
+                return this.i18n`Uploading…`;
+            case SendStatus.Sending:
+                return this.i18n`Sending…`;
+            case SendStatus.Error:
+                return this.i18n`Error: ${pendingEvent.error.message}`;
+            default:
+                return "";
+        }
+    }
+
+    get thumbnailUrl() {
+        if (this._decryptedThumbnail) {
+            return this._decryptedThumbnail.url;
+        } else {
+            const thumbnailMxc = this._getContent().info?.thumbnail_url;
+            if (thumbnailMxc) {
+                return this._mediaRepository.mxcUrlThumbnail(thumbnailMxc, this.width, this.height, "scale");
+            }
+        }
+        if (this._entry.isPending) {
+            const attachment = this._entry.pendingEvent.getAttachment("info.thumbnail_url");
+            return attachment && attachment.localPreview.url;
+        }
+        if (this._isMainResourceImage()) {
+            if (this._decryptedFile) {
+                return this._decryptedFile.url;
+            } else {
+                const mxcUrl = this._getContent()?.url;
+                if (typeof mxcUrl === "string") {
+                    return this._mediaRepository.mxcUrlThumbnail(mxcUrl, this.width, this.height, "scale");
+                }
+            }
+        }
+        return "";
+    }
+
+    get width() {
+        const info = this._getContent()?.info;
+        return Math.round(info?.w * this._scaleFactor());
+    }
+
+    get height() {
+        const info = this._getContent()?.info;
+        return Math.round(info?.h * this._scaleFactor());
+    }
+
+    get mimeType() {
+        const info = this._getContent()?.info;
+        return info?.mimetype;
+    }
+
+    get label() {
+        return this._getContent().body;
+    }
+
+    get error() {
+        if (this._error) {
+            return `Could not load media: ${this._error.message}`;
+        }
+        return null;
+    }
+
+    setViewError(err) {
+        this._error = err;
+        this.emitChange("error");
+    }
+
+    async _loadEncryptedFile(file) {
+        const blob = await this._mediaRepository.downloadEncryptedFile(file, true);
+        if (this.isDisposed) {
+            blob.dispose();
+            return;
+        }
+        return this.track(blob);
+    }
+
+    async _tryLoadEncryptedThumbnail() {
+        try {
+            const thumbnailFile = this._getContent().info?.thumbnail_file;
+            const file = this._getContent().file;
+            if (thumbnailFile) {
+                this._decryptedThumbnail = await this._loadEncryptedFile(thumbnailFile);
+                this.emitChange("thumbnailUrl");
+            } else if (file && this._isMainResourceImage()) { // is the main resource an image? then try that for a thumbnail
+                this._decryptedFile = await this._loadEncryptedFile(file);
+                this.emitChange("thumbnailUrl");
+            }
+        } catch (err) {
+            this._error = err;
+            this.emitChange("error");
+        }
+    }
+
+    _scaleFactor() {
+        const info = this._getContent()?.info;
+        const scaleHeightFactor = MAX_HEIGHT / info?.h;
+        const scaleWidthFactor = MAX_WIDTH / info?.w;
+        // take the smallest scale factor, to respect all constraints
+        // we should not upscale images, so limit scale factor to 1 upwards
+        return Math.min(scaleWidthFactor, scaleHeightFactor, 1);
+    }
+
+    _isMainResourceImage() {
+        return true; // overwritten in VideoTile
+    }
+}

src/domain/session/room/timeline/tiles/ImageTile.js

@@ -15,20 +15,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {MessageTile} from "./MessageTile.js";
+import {BaseMediaTile} from "./BaseMediaTile.js";
-import {SendStatus} from "../../../../../matrix/room/sending/PendingEvent.js";
-const MAX_HEIGHT = 300;
-const MAX_WIDTH = 400;
 
-export class ImageTile extends MessageTile {
+export class ImageTile extends BaseMediaTile {
     constructor(options) {
         super(options);
-        this._decryptedThumbail = null;
-        this._decryptedImage = null;
-        this._error = null;
-        if (!this.isPending) {
-            this.tryLoadEncryptedThumbnail();
-        }
         this._lightboxUrl = this.urlCreator.urlForSegments([
             // ensure the right room is active if in grid view
             this.navigation.segment("room", this._room.id),
@@ -36,32 +27,6 @@ export class ImageTile extends MessageTile {
         ]);
     }
 
-    async _loadEncryptedFile(file) {
-        const blob = await this._mediaRepository.downloadEncryptedFile(file, true);
-        if (this.isDisposed) {
-            blob.dispose();
-            return;
-        }
-        return this.track(blob);
-    }
-
-    async tryLoadEncryptedThumbnail() {
-        try {
-            const thumbnailFile = this._getContent().info?.thumbnail_file;
-            const file = this._getContent().file;
-            if (thumbnailFile) {
-                this._decryptedThumbail = await this._loadEncryptedFile(thumbnailFile);
-                this.emitChange("thumbnailUrl");
-            } else if (file) {
-                this._decryptedImage = await this._loadEncryptedFile(file);
-                this.emitChange("thumbnailUrl");
-            }
-        } catch (err) {
-            this._error = err;
-            this.emitChange("error");
-        }
-    }
-
     get lightboxUrl() {
         if (!this.isPending) {
             return this._lightboxUrl;
@@ -69,81 +34,6 @@ export class ImageTile extends MessageTile {
         return "";
     }
 
-    get isUploading() {
-        return this.isPending && this._entry.pendingEvent.status === SendStatus.UploadingAttachments;
-    }
-
-    get uploadPercentage() {
-        const {pendingEvent} = this._entry;
-        return pendingEvent && Math.round((pendingEvent.attachmentsSentBytes / pendingEvent.attachmentsTotalBytes) * 100);
-    }
-
-    get sendStatus() {
-        const {pendingEvent} = this._entry;
-        switch (pendingEvent?.status) {
-            case SendStatus.Waiting:
-                return this.i18n`Waiting…`;
-            case SendStatus.EncryptingAttachments:
-            case SendStatus.Encrypting:
-                return this.i18n`Encrypting…`;
-            case SendStatus.UploadingAttachments:
-                return this.i18n`Uploading…`;
-            case SendStatus.Sending:
-                return this.i18n`Sending…`;
-            case SendStatus.Error:
-                return this.i18n`Error: ${pendingEvent.error.message}`;
-            default:
-                return "";
-        }
-    }
-
-    get thumbnailUrl() {
-        if (this._decryptedThumbail) {
-            return this._decryptedThumbail.url;
-        } else if (this._decryptedImage) {
-            return this._decryptedImage.url;
-        }
-        if (this._entry.isPending) {
-            const attachment = this._entry.pendingEvent.getAttachment("url");
-            return attachment && attachment.localPreview.url;
-        }
-        const mxcUrl = this._getContent()?.url;
-        if (typeof mxcUrl === "string") {
-            return this._mediaRepository.mxcUrlThumbnail(mxcUrl, this.thumbnailWidth, this.thumbnailHeight, "scale");
-        }
-        return "";
-    }
-
-    _scaleFactor() {
-        const info = this._getContent()?.info;
-        const scaleHeightFactor = MAX_HEIGHT / info?.h;
-        const scaleWidthFactor = MAX_WIDTH / info?.w;
-        // take the smallest scale factor, to respect all constraints
-        // we should not upscale images, so limit scale factor to 1 upwards
-        return Math.min(scaleWidthFactor, scaleHeightFactor, 1);
-    }
-
-    get thumbnailWidth() {
-        const info = this._getContent()?.info;
-        return Math.round(info?.w * this._scaleFactor());
-    }
-
-    get thumbnailHeight() {
-        const info = this._getContent()?.info;
-        return Math.round(info?.h * this._scaleFactor());
-    }
-
-    get label() {
-        return this._getContent().body;
-    }
-
-    get error() {
-        if (this._error) {
-            return `Could not decrypt image: ${this._error.message}`;
-        }
-        return null;
-    }
-
     get shape() {
         return "image";
     }

src/domain/session/room/timeline/tiles/VideoTile.js

@@ -0,0 +1,47 @@
+/*
+Copyright 2020 Bruno Windels <bruno@windels.cloud>
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {BaseMediaTile} from "./BaseMediaTile.js";
+
+export class VideoTile extends BaseMediaTile {
+    async loadVideo() {
+        const file = this._getContent().file;
+        if (file && !this._decryptedFile) {
+            this._decryptedFile = await this._loadEncryptedFile(file);
+            this.emitChange("videoUrl");
+        }
+    }
+
+    get videoUrl() {
+        if (this._decryptedFile) {
+            return this._decryptedFile.url;
+        }
+        const mxcUrl = this._getContent()?.url;
+        if (typeof mxcUrl === "string") {
+            return this._mediaRepository.mxcUrl(mxcUrl);
+        }
+        return "";
+    }
+
+    get shape() {
+        return "video";
+    }
+
+    _isMainResourceImage() {
+        return false;
+    }
+}

src/domain/session/room/timeline/tilesCreator.js

@@ -17,6 +17,7 @@ limitations under the License.
 import {GapTile} from "./tiles/GapTile.js";
 import {TextTile} from "./tiles/TextTile.js";
 import {ImageTile} from "./tiles/ImageTile.js";
+import {VideoTile} from "./tiles/VideoTile.js";
 import {FileTile} from "./tiles/FileTile.js";
 import {LocationTile} from "./tiles/LocationTile.js";
 import {RoomNameTile} from "./tiles/RoomNameTile.js";
@@ -44,6 +45,8 @@ export function tilesCreator(baseOptions) {
                             return new TextTile(options);
                         case "m.image":
                             return new ImageTile(options);
+                        case "m.video":
+                            return new VideoTile(options);
                         case "m.file":
                             return new FileTile(options);
                         case "m.location":

src/platform/web/Platform.js

@@ -32,7 +32,7 @@ import {Crypto} from "./dom/Crypto.js";
 import {estimateStorageUsage} from "./dom/StorageEstimate.js";
 import {WorkerPool} from "./dom/WorkerPool.js";
 import {BlobHandle} from "./dom/BlobHandle.js";
-import {hasReadPixelPermission, ImageHandle} from "./dom/ImageHandle.js";
+import {hasReadPixelPermission, ImageHandle, VideoHandle} from "./dom/ImageHandle.js";
 import {downloadInIframe} from "./dom/download.js";
 
 function addScript(src) {
@@ -184,6 +184,10 @@ export class Platform {
         return ImageHandle.fromBlob(blob);
     }
 
+    async loadVideo(blob) {
+        return VideoHandle.fromBlob(blob);
+    }
+
     hasReadPixelPermission() {
         return hasReadPixelPermission();
     }

src/platform/web/dom/BlobHandle.js

@@ -55,6 +55,8 @@ const ALLOWED_BLOB_MIMETYPES = {
     'video/mp4': true,
     'video/webm': true,
     'video/ogg': true,
+    'video/quicktime': true,
+    'video/VP8': true,
 
     'audio/mp4': true,
     'audio/webm': true,

src/platform/web/dom/ImageHandle.js

@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 import {BlobHandle} from "./BlobHandle.js";
+import {domEventAsPromise} from "./utils.js";
 
 export class ImageHandle {
     static async fromBlob(blob) {
@@ -27,18 +28,18 @@ export class ImageHandle {
         this.blob = blob;
         this.width = width;
         this.height = height;
-        this._imgElement = imgElement;
+        this._domElement = imgElement;
     }
 
     get maxDimension() {
         return Math.max(this.width, this.height);
     }
 
-    async _getImgElement() {
+    async _getDomElement() {
-        if (!this._imgElement) {
+        if (!this._domElement) {
-            this._imgElement = await loadImgFromBlob(this.blob);
+            this._domElement = await loadImgFromBlob(this.blob);
         }
-        return this._imgElement;
+        return this._domElement;
     }
 
     async scale(maxDimension) {
@@ -46,18 +47,18 @@ export class ImageHandle {
         const scaleFactor = Math.min(1, maxDimension / (aspectRatio >= 1 ? this.width : this.height));
         const scaledWidth = Math.round(this.width * scaleFactor);
         const scaledHeight = Math.round(this.height * scaleFactor);
-
         const canvas = document.createElement("canvas");
         canvas.width = scaledWidth;
         canvas.height = scaledHeight;
         const ctx = canvas.getContext("2d");
-        const img = await this._getImgElement();
+        const drawableElement = await this._getDomElement();
-        ctx.drawImage(img, 0, 0, scaledWidth, scaledHeight);
+        ctx.drawImage(drawableElement, 0, 0, scaledWidth, scaledHeight);
         let mimeType = this.blob.mimeType === "image/jpeg" ? "image/jpeg" : "image/png";
         let nativeBlob;
         if (canvas.toBlob) {
             nativeBlob = await new Promise(resolve => canvas.toBlob(resolve, mimeType));
         } else if (canvas.msToBlob) {
+            // TODO: provide a mimetype override in blob handle for this case
             mimeType = "image/png";
             nativeBlob = canvas.msToBlob();
         } else {
@@ -72,6 +73,21 @@ export class ImageHandle {
     }
 }
 
+export class VideoHandle extends ImageHandle {
+    get duration() {
+        if (typeof this._domElement.duration === "number") {
+            return Math.round(this._domElement.duration * 1000);
+        }
+        return undefined;
+    }
+
+    static async fromBlob(blob) {
+        const video = await loadVideoFromBlob(blob);
+        const {videoWidth, videoHeight} = video;
+        return new VideoHandle(blob, videoWidth, videoHeight, video);
+    }
+}
+
 export function hasReadPixelPermission() {
     const canvas = document.createElement("canvas");
     canvas.width = 1;
@@ -91,16 +107,27 @@ export function hasReadPixelPermission() {
 async function loadImgFromBlob(blob) {
     const img = document.createElement("img");
     let detach;
-    const loadPromise = new Promise((resolve, reject) => {
+    const loadPromise = domEventAsPromise(img, "load");
-        detach = () => {
-            img.removeEventListener("load", resolve);
-            img.removeEventListener("error", reject);
-        };
-        img.addEventListener("load", resolve);
-        img.addEventListener("error", reject);
-    });
     img.src = blob.url;
     await loadPromise;
     detach();
     return img;
 }
+
+async function loadVideoFromBlob(blob) {
+    const video = document.createElement("video");
+    video.muted = true;
+    const loadPromise = domEventAsPromise(video, "loadedmetadata");
+    video.src = blob.url;
+    video.load();
+    await loadPromise;
+    // seek to the first 1/10s to make sure that drawing the video
+    // on a canvas won't give a blank image
+    const seekPromise = domEventAsPromise(video, "seeked");
+    // needed for safari to reliably fire the seeked event,
+    // somewhat hacky but using raf for example didn't do the trick
+    await new Promise(r => setTimeout(r, 200));
+    video.currentTime = 0.1;
+    await seekPromise;
+    return video;
+}

src/platform/web/dom/utils.js

@@ -0,0 +1,35 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export function domEventAsPromise(element, successEvent) {
+    return new Promise((resolve, reject) => {
+        let detach;
+        const handleError = evt => {
+            detach();
+            reject(evt.target.error);
+        };
+        const handleSuccess = () => {
+            detach();
+            resolve();
+        };
+        detach = () => {
+            element.removeEventListener(successEvent, handleSuccess);
+            element.removeEventListener("error", handleError);
+        };
+        element.addEventListener(successEvent, handleSuccess);
+        element.addEventListener("error", handleError);
+    });
+}

src/platform/web/ui/css/themes/element/theme.css

@@ -509,7 +509,7 @@ ul.Timeline > li.messageStatus .message-container > p {
 .message-container {
     padding: 1px 10px 0px 10px;
     margin: 5px 10px 0 10px;
-    /* so the .picture can grow horizontally and its spacer can grow vertically */
+    /* so the .media can grow horizontally and its spacer can grow vertically */
     width: 100%;
 }
 
@@ -555,14 +555,14 @@ ul.Timeline > li.messageStatus .message-container > p {
 }
 
 
-.message-container .picture {
+.message-container .media {
     display: grid;
     margin-top: 4px;
     width: 100%;
 }
 
 
-.message-container .picture > a {
+.message-container .media > a {
     text-decoration: none;
     width: 100%;
     display: block;
@@ -570,12 +570,12 @@ ul.Timeline > li.messageStatus .message-container > p {
 
 /* .spacer grows with an inline padding-top to the size of the image,
 so the timeline doesn't jump when the image loads */
-.message-container .picture > * {
+.message-container .media > * {
     grid-row: 1;
     grid-column: 1;
 }
 
-.message-container .picture img {
+.message-container .media img, .message-container .media video {
     width: 100%;
     height: auto;
     /* for IE11 to still scale even though the spacer is too tall */
@@ -587,29 +587,30 @@ so the timeline doesn't jump when the image loads */
 where we can trust the spacer to always have the correct height,
 otherwise the image starts with  height 0 and with loading=lazy
 only loads when the top comes into view*/
-.hydrogen:not(.legacy) .message-container .picture img {
+.hydrogen:not(.legacy) .message-container .media img,
+.hydrogen:not(.legacy) .message-container .media video {
     align-self: stretch;
 }
 
-.message-container .picture > .sendStatus {
+.message-container .media > .sendStatus {
     align-self: end;
     justify-self: start;
     font-size: 0.8em;
 }
 
-.message-container .picture > progress {
+.message-container .media > progress {
     align-self: center;
     justify-self: center;
     width: 75%;
 }
 
-.message-container .picture > time {
+.message-container .media > time {
     align-self: end;
     justify-self: end;
 }
 
-.message-container .picture > time,
+.message-container .media > time,
-.message-container .picture > .sendStatus {
+.message-container .media > .sendStatus {
     color: #2e2f32;
     display: block;
     padding: 2px;
@@ -617,7 +618,7 @@ only loads when the top comes into view*/
     background-color: rgba(255, 255, 255, 0.75);
     border-radius: 4px;
 }
-.message-container .picture > .spacer {
+.message-container .media > .spacer {
     /* TODO: can we implement this with a pseudo element? or perhaps they are not grid items? */
     width: 100%;
     /* don't stretch height as it is a spacer, just in case it doesn't match with image height */

src/platform/web/ui/css/timeline.css

@@ -37,11 +37,12 @@ limitations under the License.
     margin: 5px 0;
 }
 
-.message-container .picture {
+.message-container .media {
     display: block;
 }
 
-.message-container .picture > img {
+.message-container .media > img,
+.message-container .media > video {
     display: block;
 }
 

src/platform/web/ui/general/html.js

@@ -94,7 +94,7 @@ export const TAG_NAMES = {
     [HTML_NS]: [
         "br", "a", "ol", "ul", "li", "div", "h1", "h2", "h3", "h4", "h5", "h6",
         "p", "strong", "em", "span", "img", "section", "main", "article", "aside",
-        "pre", "button", "time", "input", "textarea", "label", "form", "progress", "output"],
+        "pre", "button", "time", "input", "textarea", "label", "form", "progress", "output", "video"],
     [SVG_NS]: ["svg", "circle"]
 };
 

src/platform/web/ui/session/room/MessageComposer.js

@@ -66,6 +66,7 @@ export class MessageComposer extends TemplateView {
         } else {
             const vm = this.value;
             this._attachmentPopup = new Popup(new Menu([
+                Menu.option(vm.i18n`Send video`, () => vm.sendVideo()).setIcon("video"),
                 Menu.option(vm.i18n`Send picture`, () => vm.sendPicture()).setIcon("picture"),
                 Menu.option(vm.i18n`Send file`, () => vm.sendFile()).setIcon("file"),
             ]));

src/platform/web/ui/session/room/TimelineList.js

@@ -18,6 +18,7 @@ import {ListView} from "../../general/ListView.js";
 import {GapView} from "./timeline/GapView.js";
 import {TextMessageView} from "./timeline/TextMessageView.js";
 import {ImageView} from "./timeline/ImageView.js";
+import {VideoView} from "./timeline/VideoView.js";
 import {FileView} from "./timeline/FileView.js";
 import {MissingAttachmentView} from "./timeline/MissingAttachmentView.js";
 import {AnnouncementView} from "./timeline/AnnouncementView.js";
@@ -30,6 +31,7 @@ function viewClassForEntry(entry) {
         case "message-status":
             return TextMessageView;
         case "image": return ImageView;
+        case "video": return VideoView;
         case "file": return FileView;
         case "missing-attachment": return MissingAttachmentView;
     }

src/platform/web/ui/session/room/timeline/BaseMediaView.js

@@ -0,0 +1,60 @@
+/*
+Copyright 2020 Bruno Windels <bruno@windels.cloud>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {TemplateView} from "../../../general/TemplateView.js";
+import {renderMessage} from "./common.js";
+
+export class BaseMediaView extends TemplateView {
+    render(t, vm) {
+        const heightRatioPercent = (vm.height / vm.width) * 100; 
+        let spacerStyle = `padding-top: ${heightRatioPercent}%;`;
+        if (vm.platform.isIE11) {
+            // preserving aspect-ratio in a grid with padding percentages
+            // does not work in IE11, so we assume people won't use it
+            // with viewports narrower than 400px where thumbnails will get
+            // scaled. If they do, the thumbnail will still scale, but
+            // there will be whitespace underneath the picture
+            // An alternative would be to use position: absolute but that
+            // can slow down rendering, and was bleeding through the lightbox.
+            spacerStyle = `height: ${vm.height}px`;
+        }
+        const children = [
+            t.div({className: "spacer", style: spacerStyle}),
+            this.renderMedia(t, vm),
+            t.time(vm.date + " " + vm.time),
+        ];
+        if (vm.isPending) {
+            const cancel = t.button({onClick: () => vm.abortSending(), className: "link"}, vm.i18n`Cancel`);
+            const sendStatus = t.div({
+                className: {
+                    sendStatus: true,
+                    hidden: vm => !vm.sendStatus
+                },
+            }, [vm => vm.sendStatus, " ", cancel]);
+            const progress = t.progress({
+                min: 0,
+                max: 100,
+                value: vm => vm.uploadPercentage,
+                className: {hidden: vm => !vm.isUploading}
+            });
+            children.push(sendStatus, progress);
+        }
+        return renderMessage(t, vm, [
+            t.div({className: "media", style: `max-width: ${vm.width}px`}, children),
+            t.if(vm => vm.error, t.createTemplate((t, vm) => t.p({className: "error"}, vm.error)))
+        ]);
+    }
+}

src/platform/web/ui/session/room/timeline/ImageView.js

@@ -14,54 +14,17 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import {TemplateView} from "../../../general/TemplateView.js";
+import {BaseMediaView} from "./BaseMediaView.js";
-import {renderMessage} from "./common.js";
 
-export class ImageView extends TemplateView {
+export class ImageView extends BaseMediaView {
-    render(t, vm) {
+    renderMedia(t, vm) {
-        const heightRatioPercent = (vm.thumbnailHeight / vm.thumbnailWidth) * 100; 
-        let spacerStyle = `padding-top: ${heightRatioPercent}%;`;
-        if (vm.platform.isIE11) {
-            // preserving aspect-ratio in a grid with padding percentages
-            // does not work in IE11, so we assume people won't use it
-            // with viewports narrower than 400px where thumbnails will get
-            // scaled. If they do, the thumbnail will still scale, but
-            // there will be whitespace underneath the picture
-            // An alternative would be to use position: absolute but that
-            // can slow down rendering, and was bleeding through the lightbox.
-            spacerStyle = `height: ${vm.thumbnailHeight}px`;
-        }
         const img = t.img({
             loading: "lazy",
             src: vm => vm.thumbnailUrl,
             alt: vm => vm.label,
             title: vm => vm.label,
-            style: `max-width: ${vm.thumbnailWidth}px; max-height: ${vm.thumbnailHeight}px;`
+            style: `max-width: ${vm.width}px; max-height: ${vm.height}px;`
         });
-        const children = [
+        return vm.isPending ? img : t.a({href: vm.lightboxUrl}, img);
-            t.div({className: "spacer", style: spacerStyle}),
-            vm.isPending ? img : t.a({href: vm.lightboxUrl}, img),
-            t.time(vm.date + " " + vm.time),
-        ];
-        if (vm.isPending) {
-            const cancel = t.button({onClick: () => vm.abortSending(), className: "link"}, vm.i18n`Cancel`);
-            const sendStatus = t.div({
-                className: {
-                    sendStatus: true,
-                    hidden: vm => !vm.sendStatus
-                },
-            }, [vm => vm.sendStatus, " ", cancel]);
-            const progress = t.progress({
-                min: 0,
-                max: 100,
-                value: vm => vm.uploadPercentage,
-                className: {hidden: vm => !vm.isUploading}
-            });
-            children.push(sendStatus, progress);
-        }
-        return renderMessage(t, vm, [
-            t.div({className: "picture", style: `max-width: ${vm.thumbnailWidth}px`}, children),
-            t.if(vm => vm.error, t.createTemplate((t, vm) => t.p({className: "error"}, vm.error)))
-        ]);
     }
 }

src/platform/web/ui/session/room/timeline/VideoView.js

@@ -0,0 +1,73 @@
+/*
+Copyright 2020 Bruno Windels <bruno@windels.cloud>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {BaseMediaView} from "./BaseMediaView.js";
+import {domEventAsPromise} from "../../../../dom/utils.js";
+
+export class VideoView extends BaseMediaView {
+    renderMedia(t) {
+        const video = t.video({
+            // provide empty data url if video is not decrypted yet.
+            // Chrome/Electron need this to enable the play button.
+            src: vm => vm.videoUrl || `data:${vm.mimeType},`,
+            title: vm => vm.label,
+            controls: true,
+            preload: "none",
+            poster: vm => vm.thumbnailUrl,
+            onPlay: this._onPlay.bind(this),
+            style: vm => `max-width: ${vm.width}px; max-height: ${vm.height}px;${vm.isPending ? "z-index: -1": ""}`
+        });
+
+        video.addEventListener("error", this._onError.bind(this));
+
+        return video;
+    }
+
+    async _onPlay(evt) {
+        const vm = this.value;
+        // download and decrypt the video if needed,
+        if (!vm.videoUrl) {
+            try {
+                const video = evt.target;
+                // this will trigger the src to update
+                await vm.loadVideo();
+                // important to only listen for this after src has changed,
+                // or we get the error for the placeholder data url
+                const loadPromise = domEventAsPromise(video, "loadeddata");
+                // now, reload the video and play
+                video.load();
+                await loadPromise;
+                video.play();
+            } catch (err) {/* errors are already caught in error event handler */}
+        }  
+    }
+
+    _onError(evt) {
+        const vm = this.value;
+        const video = evt.target;
+        const err = video.error;
+        if (err instanceof window.MediaError && err.code === 4) {
+            if (!video.src.startsWith("data:")) {
+                vm.setViewError(new Error(`this browser does not support videos of type ${vm.mimeType}.`));
+            } else {
+                // ignore placeholder url failing to load
+                return;
+            }
+        } else {
+            vm.setViewError(err);
+        }
+    }
+}