From c6ff56a9424e40868ecccd0d88e082d87b6f9333 Mon Sep 17 00:00:00 2001 From: Bruno Windels Date: Tue, 9 Mar 2021 19:35:25 +0100 Subject: [PATCH] send video messages --- src/domain/session/room/RoomViewModel.js | 51 +++++++++++++- src/platform/web/Platform.js | 6 +- src/platform/web/dom/ImageHandle.js | 67 ++++++++++++++++--- .../web/ui/session/room/MessageComposer.js | 1 + 4 files changed, 114 insertions(+), 11 deletions(-) diff --git a/src/domain/session/room/RoomViewModel.js b/src/domain/session/room/RoomViewModel.js index 83910675..6a83446a 100644 --- a/src/domain/session/room/RoomViewModel.js +++ b/src/domain/session/room/RoomViewModel.js @@ -187,6 +187,43 @@ export class RoomViewModel extends ViewModel { }); } + async _pickAndSendVideo() { + try { + if (!this.platform.hasReadPixelPermission()) { + alert("Please allow canvas image data access, so we can scale your images down."); + return; + } + const file = await this.platform.openFile("video/*"); + if (!file) { + return; + } + if (!file.blob.mimeType.startsWith("video/")) { + return this._sendFile(file); + } + let video = await this.platform.loadVideo(file.blob); + const content = { + body: file.name, + msgtype: "m.video", + info: videoToInfo(video) + }; + const attachments = { + "url": this._room.createAttachment(video.blob, file.name), + }; + + const limit = await this.platform.settingsStorage.getInt("sentImageSizeLimit"); + const maxDimension = limit || Math.min(video.maxDimension, 800); + const thumbnail = await video.scale(maxDimension); + content.info.thumbnail_info = imageToInfo(thumbnail); + attachments["info.thumbnail_url"] = + this._room.createAttachment(thumbnail.blob, file.name); + await this._room.sendEvent("m.room.message", content, attachments); + } catch (err) { + this._sendError = err; + this.emitChange("error"); + console.error(err.stack); + } + } + async _pickAndSendPicture() { try { if (!this.platform.hasReadPixelPermission()) { @@ -221,7 +258,9 @@ export class RoomViewModel extends ViewModel { } await this._room.sendEvent("m.room.message", content, attachments); } catch (err) { - console.error(err); + this._sendError = err; + this.emitChange("error"); + console.error(err.stack); } } @@ -259,6 +298,10 @@ class ComposerViewModel extends ViewModel { this._roomVM._pickAndSendFile(); } + sendVideo() { + this._roomVM._pickAndSendVideo(); + } + get canSend() { return !this._isEmpty; } @@ -283,3 +326,9 @@ function imageToInfo(image) { size: image.blob.size }; } + +function videoToInfo(video) { + const info = imageToInfo(video); + info.duration = video.duration; + return info; +} diff --git a/src/platform/web/Platform.js b/src/platform/web/Platform.js index 565aefe3..e4ad3a88 100644 --- a/src/platform/web/Platform.js +++ b/src/platform/web/Platform.js @@ -32,7 +32,7 @@ import {Crypto} from "./dom/Crypto.js"; import {estimateStorageUsage} from "./dom/StorageEstimate.js"; import {WorkerPool} from "./dom/WorkerPool.js"; import {BlobHandle} from "./dom/BlobHandle.js"; -import {hasReadPixelPermission, ImageHandle} from "./dom/ImageHandle.js"; +import {hasReadPixelPermission, ImageHandle, VideoHandle} from "./dom/ImageHandle.js"; import {downloadInIframe} from "./dom/download.js"; function addScript(src) { @@ -184,6 +184,10 @@ export class Platform { return ImageHandle.fromBlob(blob); } + async loadVideo(blob) { + return VideoHandle.fromBlob(blob); + } + hasReadPixelPermission() { return hasReadPixelPermission(); } diff --git a/src/platform/web/dom/ImageHandle.js b/src/platform/web/dom/ImageHandle.js index 8676364c..ae05e868 100644 --- a/src/platform/web/dom/ImageHandle.js +++ b/src/platform/web/dom/ImageHandle.js @@ -27,18 +27,18 @@ export class ImageHandle { this.blob = blob; this.width = width; this.height = height; - this._imgElement = imgElement; + this._domElement = imgElement; } get maxDimension() { return Math.max(this.width, this.height); } - async _getImgElement() { - if (!this._imgElement) { - this._imgElement = await loadImgFromBlob(this.blob); + async _getDomElement() { + if (!this._domElement) { + this._domElement = await loadImgFromBlob(this.blob); } - return this._imgElement; + return this._domElement; } async scale(maxDimension) { @@ -46,18 +46,18 @@ export class ImageHandle { const scaleFactor = Math.min(1, maxDimension / (aspectRatio >= 1 ? this.width : this.height)); const scaledWidth = Math.round(this.width * scaleFactor); const scaledHeight = Math.round(this.height * scaleFactor); - const canvas = document.createElement("canvas"); canvas.width = scaledWidth; canvas.height = scaledHeight; const ctx = canvas.getContext("2d"); - const img = await this._getImgElement(); - ctx.drawImage(img, 0, 0, scaledWidth, scaledHeight); + const drawableElement = await this._getDomElement(); + ctx.drawImage(drawableElement, 0, 0, scaledWidth, scaledHeight); let mimeType = this.blob.mimeType === "image/jpeg" ? "image/jpeg" : "image/png"; let nativeBlob; if (canvas.toBlob) { nativeBlob = await new Promise(resolve => canvas.toBlob(resolve, mimeType)); } else if (canvas.msToBlob) { + // TODO: provide a mimetype override in blob handle for this case mimeType = "image/png"; nativeBlob = canvas.msToBlob(); } else { @@ -72,6 +72,21 @@ export class ImageHandle { } } +export class VideoHandle extends ImageHandle { + get duration() { + if (typeof this._domElement.duration === "number") { + return Math.round(this._domElement.duration * 1000); + } + return undefined; + } + + static async fromBlob(blob) { + const video = await loadVideoFromBlob(blob); + const {videoWidth, videoHeight} = video; + return new VideoHandle(blob, videoWidth, videoHeight, video); + } +} + export function hasReadPixelPermission() { const canvas = document.createElement("canvas"); canvas.width = 1; @@ -91,7 +106,8 @@ export function hasReadPixelPermission() { async function loadImgFromBlob(blob) { const img = document.createElement("img"); let detach; - const loadPromise = new Promise((resolve, reject) => { + const loadPromise = new Promise((resolve, _reject) => { + const reject = evt => _reject(evt.target.error); detach = () => { img.removeEventListener("load", resolve); img.removeEventListener("error", reject); @@ -104,3 +120,36 @@ async function loadImgFromBlob(blob) { detach(); return img; } + +async function loadVideoFromBlob(blob) { + const video = document.createElement("video"); + video.muted = true; + let detach; + const loadPromise = new Promise((resolve, _reject) => { + const reject = evt => _reject(evt.target.error); + detach = () => { + video.removeEventListener("loadedmetadata", resolve); + video.removeEventListener("error", reject); + }; + video.addEventListener("loadedmetadata", resolve); + video.addEventListener("error", reject); + }); + video.src = blob.url; + video.load(); + await loadPromise; + // seek to the first 1/10s to make sure that drawing the video + // on a canvas won't give a blank image + const seekPromise = new Promise((resolve, _reject) => { + const reject = evt => _reject(evt.target.error); + detach = () => { + video.removeEventListener("seeked", resolve); + video.removeEventListener("error", reject); + }; + video.addEventListener("seeked", resolve); + video.addEventListener("error", reject); + }); + video.currentTime = 0.1; + await seekPromise; + detach(); + return video; +} diff --git a/src/platform/web/ui/session/room/MessageComposer.js b/src/platform/web/ui/session/room/MessageComposer.js index 32f3fc05..8da69478 100644 --- a/src/platform/web/ui/session/room/MessageComposer.js +++ b/src/platform/web/ui/session/room/MessageComposer.js @@ -66,6 +66,7 @@ export class MessageComposer extends TemplateView { } else { const vm = this.value; this._attachmentPopup = new Popup(new Menu([ + Menu.option(vm.i18n`Send video`, () => vm.sendVideo()).setIcon("video"), Menu.option(vm.i18n`Send picture`, () => vm.sendPicture()).setIcon("picture"), Menu.option(vm.i18n`Send file`, () => vm.sendFile()).setIcon("file"), ]));