Add OCR tool to media editing modal (#11566)

mastodon · Aug 15, 2019 · 28636f4 · 28636f4
1 parent f178a01
commit 28636f4
Show file tree

Hide file tree

Showing 10 changed files with 275 additions and 34 deletions.
diff --git a/app/javascript/mastodon/features/compose/components/upload_form.js b/app/javascript/mastodon/features/compose/components/upload_form.js
@@ -4,6 +4,7 @@ import UploadProgressContainer from '../containers/upload_progress_container';
 import ImmutablePureComponent from 'react-immutable-pure-component';
 import UploadContainer from '../containers/upload_container';
 import SensitiveButtonContainer from '../containers/sensitive_button_container';
+import { FormattedMessage } from 'react-intl';
 
 export default class UploadForm extends ImmutablePureComponent {
 
@@ -16,7 +17,7 @@ export default class UploadForm extends ImmutablePureComponent {
 
     return (
       <div className='compose-form__upload-wrapper'>
-        <UploadProgressContainer />
+        <UploadProgressContainer icon='upload' message={<FormattedMessage id='upload_progress.label' defaultMessage='Uploading…' />} />
 
         <div className='compose-form__uploads-wrapper'>
           {mediaIds.map(id => (

diff --git a/app/javascript/mastodon/features/compose/components/upload_progress.js b/app/javascript/mastodon/features/compose/components/upload_progress.js
@@ -2,18 +2,19 @@ import React from 'react';
 import PropTypes from 'prop-types';
 import Motion from '../../ui/util/optional_motion';
 import spring from 'react-motion/lib/spring';
-import { FormattedMessage } from 'react-intl';
 import Icon from 'mastodon/components/icon';
 
 export default class UploadProgress extends React.PureComponent {
 
   static propTypes = {
     active: PropTypes.bool,
     progress: PropTypes.number,
+    icon: PropTypes.string.isRequired,
+    message: PropTypes.node.isRequired,
   };
 
   render () {
-    const { active, progress } = this.props;
+    const { active, progress, icon, message } = this.props;
 
     if (!active) {
       return null;
@@ -22,11 +23,11 @@ export default class UploadProgress extends React.PureComponent {
     return (
       <div className='upload-progress'>
         <div className='upload-progress__icon'>
-          <Icon id='upload' />
+          <Icon id={icon} />
         </div>
 
         <div className='upload-progress__message'>
-          <FormattedMessage id='upload_progress.label' defaultMessage='Uploading...' />
+          {message}
 
           <div className='upload-progress__backdrop'>
             <Motion defaultStyle={{ width: 0 }} style={{ width: spring(progress) }}>

diff --git a/app/javascript/mastodon/features/ui/components/focal_point_modal.js b/app/javascript/mastodon/features/ui/components/focal_point_modal.js
@@ -10,6 +10,11 @@ import { FormattedMessage, defineMessages, injectIntl } from 'react-intl';
 import IconButton from 'mastodon/components/icon_button';
 import Button from 'mastodon/components/button';
 import Video from 'mastodon/features/video';
+import { TesseractWorker } from 'tesseract.js';
+import Textarea from 'react-textarea-autosize';
+import UploadProgress from 'mastodon/features/compose/components/upload_progress';
+import CharacterCounter from 'mastodon/features/compose/components/character_counter';
+import { length } from 'stringz';
 
 const messages = defineMessages({
   close: { id: 'lightbox.close', defaultMessage: 'Close' },
@@ -29,6 +34,12 @@ const mapDispatchToProps = (dispatch, { id }) => ({
 
 });
 
+const removeExtraLineBreaks = str => str.replace(/\n\n/g, '******')
+  .replace(/\n/g, ' ')
+  .replace(/\*\*\*\*\*\*/g, '\n\n');
+
+const assetHost = process.env.CDN_HOST || '';
+
 export default @connect(mapStateToProps, mapDispatchToProps)
 @injectIntl
 class FocalPointModal extends ImmutablePureComponent {
@@ -47,6 +58,7 @@ class FocalPointModal extends ImmutablePureComponent {
     dragging: false,
     description: '',
     dirty: false,
+    progress: 0,
   };
 
   componentWillMount () {
@@ -133,9 +145,27 @@ class FocalPointModal extends ImmutablePureComponent {
     this.node = c;
   }
 
+  handleTextDetection = () => {
+    const { media } = this.props;
+
+    const worker = new TesseractWorker({
+      workerPath: `${assetHost}/packs/ocr/worker.min.js`,
+      corePath: `${assetHost}/packs/ocr/tesseract-core.wasm.js`,
+      langPath: `${assetHost}/ocr/lang-data`,
+    });
+
+    this.setState({ detecting: true });
+
+    worker.recognize(media.get('url'))
+      .progress(({ progress }) => this.setState({ progress }))
+      .finally(() => worker.terminate())
+      .then(({ text }) => this.setState({ description: removeExtraLineBreaks(text), dirty: true, detecting: false }))
+      .catch(() => this.setState({ detecting: false }));
+  }
+
   render () {
     const { media, intl, onClose } = this.props;
-    const { x, y, dragging, description, dirty } = this.state;
+    const { x, y, dragging, description, dirty, detecting, progress } = this.state;
 
     const width  = media.getIn(['meta', 'original', 'width']) || null;
     const height = media.getIn(['meta', 'original', 'height']) || null;
@@ -158,15 +188,27 @@ class FocalPointModal extends ImmutablePureComponent {
 
             <label className='setting-text-label' htmlFor='upload-modal__description'><FormattedMessage id='upload_form.description' defaultMessage='Describe for the visually impaired' /></label>
 
-            <textarea
-              id='upload-modal__description'
-              className='setting-text light'
-              value={description}
-              onChange={this.handleChange}
-              autoFocus
-            />
+            <div className='setting-text__wrapper'>
+              <Textarea
+                id='upload-modal__description'
+                className='setting-text light'
+                value={detecting ? '…' : description}
+                onChange={this.handleChange}
+                disabled={detecting}
+                autoFocus
+              />
+
+              <div className='setting-text__modifiers'>
+                <UploadProgress progress={progress * 100} active={detecting} icon='file-text-o' message={<FormattedMessage id='upload_modal.analyzing_picture' defaultMessage='Analyzing picture…' />} />
+              </div>
+            </div>
+
+            <div className='setting-text__toolbar'>
+              <button disabled={detecting || media.get('type') !== 'image'} className='link-button' onClick={this.handleTextDetection}><FormattedMessage id='upload_modal.detect_text' defaultMessage='Detect text from picture' /></button>
+              <CharacterCounter max={420} text={detecting ? '' : description} />
+            </div>
 
-            <Button disabled={!dirty} text={intl.formatMessage(messages.apply)} onClick={this.handleSubmit} />
+            <Button disabled={!dirty || detecting || length(description) > 420} text={intl.formatMessage(messages.apply)} onClick={this.handleSubmit} />
           </div>
 
           <div className='report-modal__statuses'>

diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
@@ -3,6 +3,27 @@
   -ms-overflow-style: -ms-autohiding-scrollbar;
 }
 
+.link-button {
+  display: block;
+  font-size: 15px;
+  line-height: 20px;
+  color: $ui-highlight-color;
+  border: 0;
+  background: transparent;
+  padding: 0;
+  cursor: pointer;
+
+  &:hover,
+  &:active {
+    text-decoration: underline;
+  }
+
+  &:disabled {
+    color: $ui-primary-color;
+    cursor: default;
+  }
+}
+
 .button {
   background-color: $ui-highlight-color;
   border: 10px none;
@@ -637,18 +658,6 @@
     .character-counter__wrapper {
       align-self: center;
       margin-right: 4px;
-
-      .character-counter {
-        cursor: default;
-        font-family: $font-sans-serif, sans-serif;
-        font-size: 14px;
-        font-weight: 600;
-        color: $lighter-text-color;
-
-        &.character-counter--over {
-          color: $warning-red;
-        }
-      }
     }
   }
 
@@ -665,6 +674,18 @@
   }
 }
 
+.character-counter {
+  cursor: default;
+  font-family: $font-sans-serif, sans-serif;
+  font-size: 14px;
+  font-weight: 600;
+  color: $lighter-text-color;
+
+  &.character-counter--over {
+    color: $warning-red;
+  }
+}
+
 .no-reduce-motion .spoiler-input {
   transition: height 0.4s ease, opacity 0.4s ease;
 }
@@ -4555,16 +4576,48 @@ a.status-card.compact:hover {
     padding: 10px;
     font-family: inherit;
     font-size: 14px;
-    resize: vertical;
+    resize: none;
     border: 0;
     outline: 0;
     border-radius: 4px;
     border: 1px solid $ui-secondary-color;
-    margin-bottom: 20px;
+    min-height: 100px;
+    max-height: 50vh;
+    margin-bottom: 10px;
 
     &:focus {
       border: 1px solid darken($ui-secondary-color, 8%);
     }
+
+    &__wrapper {
+      background: $white;
+      border: 1px solid $ui-secondary-color;
+      margin-bottom: 10px;
+      border-radius: 4px;
+
+      .setting-text {
+        border: 0;
+        margin-bottom: 0;
+        border-radius: 0;
+
+        &:focus {
+          border: 0;
+        }
+      }
+
+      &__modifiers {
+        color: $inverted-text-color;
+        font-family: inherit;
+        font-size: 14px;
+        background: $white;
+      }
+    }
+
+    &__toolbar {
+      display: flex;
+      justify-content: space-between;
+      margin-bottom: 20px;
+    }
   }
 
   .setting-text-label {

diff --git a/config/initializers/content_security_policy.rb b/config/initializers/content_security_policy.rb
@@ -20,11 +20,11 @@
   if Rails.env.development?
     webpacker_urls = %w(ws http).map { |protocol| "#{protocol}#{Webpacker.dev_server.https? ? 's' : ''}://#{Webpacker.dev_server.host_with_port}" }
 
-    p.connect_src :self, :blob, assets_host, Rails.configuration.x.streaming_api_base_url, *webpacker_urls
-    p.script_src  :self, :unsafe_inline, :unsafe_eval, assets_host
+    p.connect_src :self, :data, :blob, assets_host, Rails.configuration.x.streaming_api_base_url, *webpacker_urls
+    p.script_src  :self, :blob, :unsafe_inline, :unsafe_eval, assets_host
   else
-    p.connect_src :self, :blob, assets_host, Rails.configuration.x.streaming_api_base_url
-    p.script_src  :self, assets_host
+    p.connect_src :self, :data, :blob, assets_host, Rails.configuration.x.streaming_api_base_url
+    p.script_src  :self, :blob, assets_host
   end
 end
 

diff --git a/config/webpack/development.js b/config/webpack/development.js
@@ -56,5 +56,6 @@ module.exports = merge(sharedConfig, {
       settings.dev_server.watch_options,
       watchOptions
     ),
+    writeToDisk: filePath => /ocr/.test(filePath),
   },
 });
diff --git a/config/webpack/shared.js b/config/webpack/shared.js
@@ -5,6 +5,7 @@ const { basename, dirname, join, relative, resolve } = require('path');
 const { sync } = require('glob');
 const MiniCssExtractPlugin = require('mini-css-extract-plugin');
 const AssetsManifestPlugin = require('webpack-assets-manifest');
+const CopyPlugin = require('copy-webpack-plugin');
 const extname = require('path-complete-extname');
 const { env, settings, themes, output } = require('./configuration');
 const rules = require('./rules');
@@ -84,6 +85,10 @@ module.exports = {
       writeToDisk: true,
       publicPath: true,
     }),
+    new CopyPlugin([
+      { from: 'node_modules/tesseract.js/dist/worker.min.js', to: 'ocr' },
+      { from: 'node_modules/tesseract.js-core/tesseract-core.wasm.js', to: 'ocr' },
+    ]),
   ],
 
   resolve: {

diff --git a/package.json b/package.json
@@ -84,6 +84,7 @@
     "blurhash": "^1.0.0",
     "classnames": "^2.2.5",
     "compression-webpack-plugin": "^3.0.0",
+    "copy-webpack-plugin": "^5.0.4",
     "cross-env": "^5.1.4",
     "css-loader": "^3.2.0",
     "cssnano": "^4.1.10",
@@ -155,6 +156,7 @@
     "stringz": "^2.0.0",
     "substring-trie": "^1.0.2",
     "terser-webpack-plugin": "^1.4.1",
+    "tesseract.js": "^2.0.0-alpha.13",
     "throng": "^4.0.0",
     "tiny-queue": "^0.2.1",
     "uuid": "^3.1.0",

diff --git a/public/ocr/lang-data/eng.traineddata.gz b/public/ocr/lang-data/eng.traineddata.gz