Add blurhash (mastodon#10630)

* Add blurhash * Use fallback color for spoiler when blurhash missing * Federate the blurhash and accept it as long as it's at most 5x5 * Display unknown media attachments as blurhash placeholders * Improve style of embed actions and spoiler button * Change blurhash resolution from 3x3 to 4x4 * Improve dependency definitions * Fix code style issues
best-friends · May 4, 2019 · d9a0f27 · d9a0f27
1 parent 563e74f
commit d9a0f27
Show file tree

Hide file tree

Showing 22 changed files with 233 additions and 59 deletions.
diff --git a/Gemfile b/Gemfile
@@ -21,6 +21,7 @@ gem 'fog-openstack', '~> 0.3', require: false
 gem 'paperclip', '~> 6.0'
 gem 'paperclip-av-transcoder', '~> 0.6'
 gem 'streamio-ffmpeg', '~> 3.0'
+gem 'blurhash', '~> 0.1'
 
 gem 'active_model_serializers', '~> 0.10'
 gem 'addressable', '~> 2.6'

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -99,6 +99,8 @@ GEM
       rack (>= 0.9.0)
     binding_of_caller (0.8.0)
       debug_inspector (>= 0.0.1)
+    blurhash (0.1.2)
+      ffi (~> 1.10.0)
     bootsnap (1.4.4)
       msgpack (~> 1.0)
     brakeman (4.5.0)
@@ -663,6 +665,7 @@ DEPENDENCIES
   aws-sdk-s3 (~> 1.36)
   better_errors (~> 2.5)
   binding_of_caller (~> 0.7)
+  blurhash (~> 0.1)
   bootsnap (~> 1.4)
   brakeman (~> 4.5)
   browser

diff --git a/app/javascript/mastodon/components/media_gallery.js b/app/javascript/mastodon/components/media_gallery.js
@@ -7,6 +7,7 @@ import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
 import { isIOS } from '../is_mobile';
 import classNames from 'classnames';
 import { autoPlayGif, displayMedia } from '../initial_state';
+import { decode } from 'blurhash';
 
 const messages = defineMessages({
   toggle_visible: { id: 'media_gallery.toggle_visible', defaultMessage: 'Toggle visibility' },
@@ -21,6 +22,7 @@ class Item extends React.PureComponent {
     size: PropTypes.number.isRequired,
     onClick: PropTypes.func.isRequired,
     displayWidth: PropTypes.number,
+    visible: PropTypes.bool.isRequired,
   };
 
   static defaultProps = {
@@ -29,6 +31,10 @@ class Item extends React.PureComponent {
     size: 1,
   };
 
+  state = {
+    loaded: false,
+  };
+
   handleMouseEnter = (e) => {
     if (this.hoverToPlay()) {
       e.target.play();
@@ -62,8 +68,40 @@ class Item extends React.PureComponent {
     e.stopPropagation();
   }
 
+  componentDidMount () {
+    if (this.props.attachment.get('blurhash')) {
+      this._decode();
+    }
+  }
+
+  componentDidUpdate (prevProps) {
+    if (prevProps.attachment.get('blurhash') !== this.props.attachment.get('blurhash') && this.props.attachment.get('blurhash')) {
+      this._decode();
+    }
+  }
+
+  _decode () {
+    const hash   = this.props.attachment.get('blurhash');
+    const pixels = decode(hash, 32, 32);
+
+    if (pixels) {
+      const ctx       = this.canvas.getContext('2d');
+      const imageData = new ImageData(pixels, 32, 32);
+
+      ctx.putImageData(imageData, 0, 0);
+    }
+  }
+
+  setCanvasRef = c => {
+    this.canvas = c;
+  }
+
+  handleImageLoad = () => {
+    this.setState({ loaded: true });
+  }
+
   render () {
-    const { attachment, index, size, standalone, displayWidth } = this.props;
+    const { attachment, index, size, standalone, displayWidth, visible } = this.props;
 
     let width  = 50;
     let height = 100;
@@ -116,12 +154,20 @@ class Item extends React.PureComponent {
 
     let thumbnail = '';
 
-    if (attachment.get('type') === 'image') {
+    if (attachment.get('type') === 'unknown') {
+      return (
+        <div className={classNames('media-gallery__item', { standalone })} key={attachment.get('id')} style={{ left: left, top: top, right: right, bottom: bottom, width: `${width}%`, height: `${height}%` }}>
+          <a className='media-gallery__item-thumbnail' href={attachment.get('remote_url')} target='_blank' style={{ cursor: 'pointer' }} >
+            <canvas width={32} height={32} ref={this.setCanvasRef} className='media-gallery__preview' />
+          </a>
+        </div>
+      );
+    } else if (attachment.get('type') === 'image') {
       const previewUrl   = attachment.get('preview_url');
       const previewWidth = attachment.getIn(['meta', 'small', 'width']);
 
-      const originalUrl    = attachment.get('url');
-      const originalWidth  = attachment.getIn(['meta', 'original', 'width']);
+      const originalUrl   = attachment.get('url');
+      const originalWidth = attachment.getIn(['meta', 'original', 'width']);
 
       const hasSize = typeof originalWidth === 'number' && typeof previewWidth === 'number';
 
@@ -147,6 +193,7 @@ class Item extends React.PureComponent {
             alt={attachment.get('description')}
             title={attachment.get('description')}
             style={{ objectPosition: `${x}% ${y}%` }}
+            onLoad={this.handleImageLoad}
           />
         </a>
       );
@@ -176,7 +223,8 @@ class Item extends React.PureComponent {
 
     return (
       <div className={classNames('media-gallery__item', { standalone })} key={attachment.get('id')} style={{ left: left, top: top, right: right, bottom: bottom, width: `${width}%`, height: `${height}%` }}>
-        {thumbnail}
+        <canvas width={32} height={32} ref={this.setCanvasRef} className={classNames('media-gallery__preview', { 'media-gallery__preview--hidden': visible && this.state.loaded })} />
+        {visible && thumbnail}
       </div>
     );
   }
@@ -225,6 +273,7 @@ class MediaGallery extends React.PureComponent {
     if (node /*&& this.isStandaloneEligible()*/) {
       // offsetWidth triggers a layout, so only calculate when we need to
       if (this.props.cacheWidth) this.props.cacheWidth(node.offsetWidth);
+
       this.setState({
         width: node.offsetWidth,
       });
@@ -242,7 +291,7 @@ class MediaGallery extends React.PureComponent {
 
     const width = this.state.width || defaultWidth;
 
-    let children;
+    let children, spoilerButton;
 
     const style = {};
 
@@ -256,35 +305,28 @@ class MediaGallery extends React.PureComponent {
       style.height = height;
     }
 
-    if (!visible) {
-      let warning;
+    const size = media.take(4).size;
 
-      if (sensitive) {
-        warning = <FormattedMessage id='status.sensitive_warning' defaultMessage='Sensitive content' />;
-      } else {
-        warning = <FormattedMessage id='status.media_hidden' defaultMessage='Media hidden' />;
-      }
+    if (this.isStandaloneEligible()) {
+      children = <Item standalone onClick={this.handleClick} attachment={media.get(0)} displayWidth={width} visible={visible} />;
+    } else {
+      children = media.take(4).map((attachment, i) => <Item key={attachment.get('id')} onClick={this.handleClick} attachment={attachment} index={i} size={size} displayWidth={width} visible={visible} />);
+    }
 
-      children = (
-        <button type='button' className='media-spoiler' onClick={this.handleOpen} style={style} ref={this.handleRef}>
-          <span className='media-spoiler__warning'>{warning}</span>
-          <span className='media-spoiler__trigger'><FormattedMessage id='status.sensitive_toggle' defaultMessage='Click to view' /></span>
+    if (visible) {
+      spoilerButton = <IconButton title={intl.formatMessage(messages.toggle_visible)} icon={visible ? 'eye' : 'eye-slash'} overlay onClick={this.handleOpen} />;
+    } else {
+      spoilerButton = (
+        <button type='button' onClick={this.handleOpen} className='spoiler-button__overlay'>
+          <span className='spoiler-button__overlay__label'>{sensitive ? <FormattedMessage id='status.sensitive_warning' defaultMessage='Sensitive content' /> : <FormattedMessage id='status.media_hidden' defaultMessage='Media hidden' />}</span>
         </button>
       );
-    } else {
-      const size = media.take(4).size;
-
-      if (this.isStandaloneEligible()) {
-        children = <Item standalone onClick={this.handleClick} attachment={media.get(0)} displayWidth={width} />;
-      } else {
-        children = media.take(4).map((attachment, i) => <Item key={attachment.get('id')} onClick={this.handleClick} attachment={attachment} index={i} size={size} displayWidth={width} />);
-      }
     }
 
     return (
       <div className='media-gallery' style={style} ref={this.handleRef}>
-        <div className={classNames('spoiler-button', { 'spoiler-button--visible': visible })}>
-          <IconButton title={intl.formatMessage(messages.toggle_visible)} icon={visible ? 'eye' : 'eye-slash'} overlay onClick={this.handleOpen} />
+        <div className={classNames('spoiler-button', { 'spoiler-button--minified': visible })}>
+          {spoilerButton}
         </div>
 
         {children}

diff --git a/app/javascript/mastodon/components/status.js b/app/javascript/mastodon/components/status.js
@@ -274,7 +274,7 @@ class Status extends ImmutablePureComponent {
     if (status.get('poll')) {
       media = <PollContainer pollId={status.get('poll')} />;
     } else if (status.get('media_attachments').size > 0) {
-      if (this.props.muted || status.get('media_attachments').some(item => item.get('type') === 'unknown')) {
+      if (this.props.muted) {
         media = (
           <AttachmentList
             compact
@@ -289,6 +289,7 @@ class Status extends ImmutablePureComponent {
             {Component => (
               <Component
                 preview={video.get('preview_url')}
+                blurhash={video.get('blurhash')}
                 src={video.get('url')}
                 alt={video.get('description')}
                 width={this.props.cachedMediaWidth}

diff --git a/app/javascript/mastodon/features/report/components/status_check_box.js b/app/javascript/mastodon/features/report/components/status_check_box.js
@@ -35,6 +35,7 @@ export default class StatusCheckBox extends React.PureComponent {
             {Component => (
               <Component
                 preview={video.get('preview_url')}
+                blurhash={video.get('blurhash')}
                 src={video.get('url')}
                 alt={video.get('description')}
                 width={239}

diff --git a/app/javascript/mastodon/features/status/components/detailed_status.js b/app/javascript/mastodon/features/status/components/detailed_status.js
@@ -5,7 +5,6 @@ import Avatar from '../../../components/avatar';
 import DisplayName from '../../../components/display_name';
 import StatusContent from '../../../components/status_content';
 import MediaGallery from '../../../components/media_gallery';
-import AttachmentList from '../../../components/attachment_list';
 import { Link } from 'react-router-dom';
 import { FormattedDate, FormattedNumber } from 'react-intl';
 import Card from './card';
@@ -109,14 +108,13 @@ export default class DetailedStatus extends ImmutablePureComponent {
     if (status.get('poll')) {
       media = <PollContainer pollId={status.get('poll')} />;
     } else if (status.get('media_attachments').size > 0) {
-      if (status.get('media_attachments').some(item => item.get('type') === 'unknown')) {
-        media = <AttachmentList media={status.get('media_attachments')} />;
-      } else if (status.getIn(['media_attachments', 0, 'type']) === 'video') {
+      if (status.getIn(['media_attachments', 0, 'type']) === 'video') {
         const video = status.getIn(['media_attachments', 0]);
 
         media = (
           <Video
             preview={video.get('preview_url')}
+            blurhash={video.get('blurhash')}
             src={video.get('url')}
             alt={video.get('description')}
             width={300}

diff --git a/app/javascript/mastodon/features/ui/components/media_modal.js b/app/javascript/mastodon/features/ui/components/media_modal.js
@@ -144,6 +144,7 @@ class MediaModal extends ImmutablePureComponent {
         return (
           <Video
             preview={image.get('preview_url')}
+            blurhash={image.get('blurhash')}
             src={image.get('url')}
             width={image.get('width')}
             height={image.get('height')}

diff --git a/app/javascript/mastodon/features/ui/components/video_modal.js b/app/javascript/mastodon/features/ui/components/video_modal.js
@@ -20,6 +20,7 @@ export default class VideoModal extends ImmutablePureComponent {
         <div>
           <Video
             preview={media.get('preview_url')}
+            blurhash={media.get('blurhash')}
             src={media.get('url')}
             startTime={time}
             onCloseVideo={onClose}