feat(media): Blurhash

package.json

@@ -48,6 +48,7 @@
     "@webcomponents/custom-elements": "^1.2.4",
     "babel-loader": "^8.0.6",
     "babel-plugin-transform-react-remove-prop-types": "^0.4.24",
+    "blurhash": "^1.1.3",
     "cheerio": "^1.0.0-rc.2",
     "child-process-promise": "^2.2.1",
     "chokidar": "^3.0.1",
@@ -109,7 +110,8 @@
     "mocha": "^6.1.4",
     "now": "^15.7.0",
     "standard": "^13.1.0",
-    "testcafe": "^1.2.1"
+    "testcafe": "^1.2.1",
+    "worker-loader": "^2.0.0"
   },
   "engines": {
     "node": ">= 8"
@@ -150,7 +152,10 @@
       "customElements",
       "AbortController",
       "matchMedia",
-      "MessageChannel"
+      "MessageChannel",
+      "ImageData",
+      "OffscreenCanvas",
+      "postMessage"
     ],
     "ignore": [
       "dist",

src/routes/_components/LazyImage.html

@@ -53,7 +53,8 @@
       height: void 0,
       ariaHidden: false,
       alt: '',
-      title: ''
+      title: '',
+      blurhash: void 0
     }),
     computed: {
       computedStyle: ({ background }) => {
@@ -71,7 +72,7 @@
         return `object-position: ${coordsToPercent(focus.x)}% ${100 - coordsToPercent(focus.y)}%;`
       },
       fillFixSize: ({ forceSize, $largeInlineMedia }) => !$largeInlineMedia && !forceSize,
-      displaySrc: ({ error, src, fallback }) => ((error && fallback) || src)
+      displaySrc: ({ blurhash, error, src, fallback }) => (blurhash || (error && fallback) || src)
     }
   }
 </script>

src/routes/_components/NonAutoplayGifv.html

@@ -17,14 +17,17 @@
       alt={label || ''}
       title={label || ''}
       src={staticSrc}
+      blurhash={blurhash}
       fallback={oneTransparentPixel}
       {width}
       {height}
       background="var(--loading-bg)"
       {focus}
     />
   {/if}
-  <PlayVideoIcon className={playing ? 'hidden' : ''}/>
+  {#if !blurhash}
+    <PlayVideoIcon className={playing ? 'hidden' : ''}/>
+  {/if}
 </div>
 <style>
   .non-autoplay-gifv {

src/routes/_components/status/Media.html

@@ -1,23 +1,40 @@
 {#if type === 'video' || type === 'audio'}
-  <button id={elementId}
-          type="button"
-          class="play-video-button focus-after {$largeInlineMedia ? '' : 'fixed-size'} {type === 'audio' ? 'play-audio-button' : ''}"
-          aria-label="Play video: {description}"
-          style="width: {inlineWidth}px; height: {inlineHeight}px;">
-    <PlayVideoIcon />
+  {#if blurhash}
     {#if type === 'video'}
       <LazyImage
         alt={description}
         title={description}
         src={previewUrl}
         fallback={oneTransparentPixel}
+        blurhash={blurhash}
         width={inlineWidth}
         height={inlineHeight}
         background="var(--loading-bg)"
         {focus}
       />
     {/if}
-  </button>
+  {:else}
+    <button id={elementId}
+            type="button"
+            class="play-video-button focus-after {$largeInlineMedia ? '' : 'fixed-size'} {type === 'audio' ? 'play-audio-button' : ''}"
+            aria-label="Play video: {description}"
+            style="width: {inlineWidth}px; height: {inlineHeight}px;">
+      <PlayVideoIcon />
+      {#if type === 'video'}
+        <LazyImage
+          alt={description}
+          title={description}
+          src={previewUrl}
+          fallback={oneTransparentPixel}
+          blurhash={blurhash}
+          width={inlineWidth}
+          height={inlineHeight}
+          background="var(--loading-bg)"
+          {focus}
+        />
+      {/if}
+    </button>
+  {/if}
 {:else}
   <button id={elementId}
           type="button"
@@ -40,6 +57,7 @@
       class={noNativeWidthHeight ? 'no-native-width-height' : ''}
       label="Animated GIF: {description}"
       poster={previewUrl}
+      blurhash={blurhash}
       src={url}
       staticSrc={previewUrl}
       width={inlineWidth}
@@ -53,6 +71,7 @@
       title={description}
       src={previewUrl}
       fallback={oneTransparentPixel}
+      blurhash={blurhash}
       width={inlineWidth}
       height={inlineHeight}
       background="var(--loading-bg)"
@@ -91,11 +110,16 @@
   import LazyImage from '../LazyImage.html'
   import AutoplayVideo from '../AutoplayVideo.html'
   import { registerClickDelegate } from '../../_utils/delegate'
+  import { decode } from '../../_utils/blurhash'
 
-  export default {
-    oncreate () {
-      const { elementId } = this.get()
+export default {
+    async oncreate () {
+      const { elementId, media } = this.get()
       registerClickDelegate(this, elementId, () => this.onClick())
+
+      if (media.blurhash) {
+        this.set({ decodedBlurhash: await decode(media.blurhash) })
+      }
     },
     computed: {
       focus: ({ meta }) => meta && meta.focus,
@@ -126,6 +150,7 @@
       elementId: ({ media, uuid }) => `media-${uuid}-${media.id}`,
       description: ({ media }) => media.description || '',
       previewUrl: ({ media }) => media.preview_url,
+      blurhash: ({ showBlurhash, decodedBlurhash }) => showBlurhash && decodedBlurhash,
       url: ({ media }) => media.url,
       type: ({ media }) => media.type
     },
@@ -141,6 +166,7 @@
     },
     data: () => ({
       oneTransparentPixel: ONE_TRANSPARENT_PIXEL,
+      decodedBlurhash: ONE_TRANSPARENT_PIXEL,
       mouseover: void 0
     }),
     store: () => store,

src/routes/_components/status/MediaAttachments.html

@@ -1,7 +1,7 @@
 <div class={computedClass}
      style="grid-template-columns: repeat({nCols}, 1fr);" >
   {#each mediaAttachments as media, index}
-    <Media {media} {uuid} {mediaAttachments} {index} />
+    <Media {media} {uuid} {mediaAttachments} {index} {showBlurhash} />
   {/each}
 </div>
 <style>
@@ -39,6 +39,7 @@
   }
 
   .status-media.status-media-is-sensitive {
+    height: inherit;
     margin: 0;
   }
 </style>
@@ -55,6 +56,10 @@
         twoCols && 'two-cols',
         !$largeInlineMedia && 'grouped-images'
       ),
+      showBlurhash:
+        ({ sensitive, sensitiveShown, mediaAttachments }) => {
+          return sensitive && mediaAttachments.every(attachment => !!attachment.blurhash) ? !sensitiveShown : false
+        },
       nCols:
         ({ mediaAttachments, $largeInlineMedia }) => {
           return (!$largeInlineMedia && mediaAttachments.length > 1) ? 2 : 1

src/routes/_components/status/StatusMediaAttachments.html

@@ -10,28 +10,32 @@
       <SvgIcon className="status-sensitive-media-svg" href="#fa-eye-slash" />
     </div>
   </button>
-  <MediaAttachments {mediaAttachments} {sensitive} {uuid} />
   {:else}
     <button id={elementId}
             type="button"
             class="status-sensitive-media-button"
             aria-label="Show sensitive media" >
 
-      <div class="status-sensitive-media-warning">
-        Sensitive content. Click to show.
+      <div class="{customWarningClass}">
+        <div class="status-sensitive-media-warning-text">
+          Sensitive content. Click to show.
+        </div>
       </div>
       <div class="svg-wrapper">
         <SvgIcon className="status-sensitive-media-svg" href="#fa-eye" />
       </div>
     </button>
   {/if}
+  {#if sensitiveShown || canUseBlurhash}
+    <MediaAttachments {mediaAttachments} {sensitive} {sensitiveShown} {uuid} />
+  {/if}
   </div>
 </div>
 {#if enableShortcuts}
   <Shortcut scope={shortcutScope} key="y" on:pressed="toggleSensitiveMedia()"/>
 {/if}
 {:else}
-  <MediaAttachments {mediaAttachments} {sensitive} {uuid} />
+  <MediaAttachments {mediaAttachments} {sensitive} {sensitiveShown} {uuid} />
 {/if}
 <style>
   .status-sensitive-media-container {
@@ -81,6 +85,7 @@
   }
 
   .status-sensitive-media-hidden .status-sensitive-media-button {
+    position: absolute;
     right: 0;
     bottom: 0;
     width: 100%;
@@ -96,7 +101,6 @@
   }
 
   .status-sensitive-media-container .status-sensitive-media-warning {
-    position: absolute;
     top: 0;
     left: 0;
     right: 0;
@@ -109,6 +113,21 @@
     padding: 0 10px;
   }
 
+  .status-sensitive-media-container .status-sensitive-media-warning-transparent {
+    position: absolute;
+  }
+
+  .status-sensitive-media-container .status-sensitive-media-warning-opaque {
+    background: var(--mask-bg);
+    height: 100%;
+  }
+
+  .status-sensitive-media-container .status-sensitive-media-warning-transparent .status-sensitive-media-warning-text {
+    background: var(--mask-bg);
+    padding: 10px;
+    border-radius: 6px;
+  }
+
   .status-sensitive-media-container .svg-wrapper {
     display: flex;
     align-items: flex-start;
@@ -119,6 +138,7 @@
   }
   .status-sensitive-media-hidden .svg-wrapper {
     position: absolute;
+    background: none;
     top: 0;
     left: 0;
     right: 0;
@@ -171,6 +191,7 @@
         $largeInlineMedia ? 'not-grouped-images' : 'grouped-images'
       ),
       mediaAttachments: ({ originalStatus }) => originalStatus.media_attachments,
+      canUseBlurhash: ({ $ignoreBlurhash, mediaAttachments }) => !$ignoreBlurhash && mediaAttachments && mediaAttachments.every(media => !!media.blurhash),
       sensitiveShown: ({ $sensitivesShown, uuid }) => !!$sensitivesShown[uuid],
       sensitive: ({ originalStatus, $markMediaAsSensitive, $neverMarkMediaAsSensitive }) => (
         !$neverMarkMediaAsSensitive && ($markMediaAsSensitive || originalStatus.sensitive)
@@ -181,7 +202,11 @@
           return ''
         }
         return `padding-bottom: ${Math.ceil(mediaAttachments.length / 2) * 29}%;`
-      }
+      },
+      customWarningClass: ({ canUseBlurhash }) => classname(
+        'status-sensitive-media-warning',
+        canUseBlurhash ? 'status-sensitive-media-warning-transparent' : 'status-sensitive-media-warning-opaque'
+      )
     },
     methods: {
       toggleSensitiveMedia () {

src/routes/_pages/settings/general.html

@@ -8,6 +8,11 @@ <h2>Media</h2>
              bind:checked="$neverMarkMediaAsSensitive" on:change="onChange(event)">
       <label for="choice-never-mark-media-sensitive">Show sensitive media by default</label>
     </div>
+    <div class="setting-group">
+        <input type="checkbox" id="choice-use-blurhash"
+               bind:checked="$ignoreBlurhash" on:change="onChange(event)">
+        <label for="choice-use-blurhash">Show a plain gray color for sensitive media</label>
+      </div>
     <div class="setting-group">
       <input type="checkbox" id="choice-mark-media-sensitive"
              bind:checked="$markMediaAsSensitive" on:change="onChange(event)">

src/routes/_store/store.js

@@ -30,6 +30,7 @@ const persistedState = {
   loggedInInstancesInOrder: [],
   markMediaAsSensitive: false,
   neverMarkMediaAsSensitive: false,
+  ignoreBlurhash: false,
   omitEmojiInDisplayNames: undefined,
   pinnedPages: {},
   pushSubscriptions: {},

src/routes/_utils/blurhash.js

@@ -0,0 +1,49 @@
+import BlurhashWorker from 'worker-loader!../_workers/blurhash' // eslint-disable-line
+
+const RESOLUTION = 32
+let worker
+let canvas
+
+export function init () {
+  worker = worker || new BlurhashWorker()
+}
+
+export async function decode (blurhash) {
+  return new Promise((resolve, reject) => {
+    try {
+      init()
+
+      const onMessage = ({ data: { encoded, decoded, imageData, error } }) => {
+        if (encoded !== blurhash) {
+          return
+        }
+
+        worker.removeEventListener('message', onMessage)
+
+        if (error) {
+          return reject(error)
+        }
+
+        if (decoded) {
+          resolve(decoded)
+        } else {
+          if (!canvas) {
+            canvas = document.createElement('canvas')
+            canvas.height = RESOLUTION
+            canvas.width = RESOLUTION
+          }
+
+          canvas.getContext('2d').putImageData(imageData, 0, 0)
+          canvas.toBlob(blob => {
+            resolve(URL.createObjectURL(blob))
+          })
+        }
+      }
+
+      worker.addEventListener('message', onMessage)
+      worker.postMessage({ encoded: blurhash })
+    } catch (e) {
+      reject(e)
+    }
+  })
+}

src/routes/_workers/blurhash.js

@@ -0,0 +1,26 @@
+import { decode as decodeBlurHash } from 'blurhash'
+
+const RESOLUTION = 32
+const OFFSCREEN_CANVAS = typeof OffscreenCanvas === 'function'
+  ? new OffscreenCanvas(RESOLUTION, RESOLUTION) : null
+
+self.addEventListener('message', ({ data: { encoded } }) => {
+  try {
+    const pixels = decodeBlurHash(encoded, RESOLUTION, RESOLUTION)
+
+    if (pixels) {
+      const imageData = new ImageData(pixels, RESOLUTION, RESOLUTION)
+
+      if (OFFSCREEN_CANVAS) {
+        OFFSCREEN_CANVAS.getContext('2d').putImageData(imageData, 0, 0)
+        OFFSCREEN_CANVAS.convertToBlob().then(blob => {
+          postMessage({ encoded, decoded: URL.createObjectURL(blob), imageData: null, error: null })
+        })
+      } else {
+        postMessage({ encoded, imageData, decoded: null, error: null })
+      }
+    }
+  } catch (error) {
+    postMessage({ encoded, decoded: null, imageData: null, error })
+  }
+})