ph-audio-image-video-recorder.html

<!--
`ph-audio-image-video-recorder` can be used to record audio and video from the devices microphones and cameras. It uses only the
standard APIs **Web Audio API** and **UserMedia API**. It also overlays the live video output with a audio visualization.

Please note that this component is a fancy wrapper around those APIs and provides you with the essential functionality
for recording video and audio. There are also some filters added, which the user can apply on the video. These filters
are merged into the video stream. They're only applied via the CSS3 selector `filter`.
The APIs are at this time (January 2015) not well supported by the majority of the browsers available. Please use with
caution and don't use it in production :)

Please note that the audio and video stream can not be recorded as one blob record containing audio and video in one
blob. This is caused by the API limitation. As a result of that a video exists of a recorded video stream and a
separate recorded audio stream. The recorded audio and video will be started at the same time, when the user starts to
play a recorded video. So it's possible that the video and audio will run out of sync.

Use the attribute `mode` to switch between the types of recording (video (audio+video), audio (only audio), image).

== **Please Note** ==

This component only works on those browsers:

- iOS 8+ on Safari only supports the Audio WebAPI, so there's no Video/Image access available
- Android 4.4.4+ on Chrome 40+ supports Audio and Video, 5+ integrates the evergreen Chrome, so it works there too
- Windows Internet Explorer does NOT support any of the used APIs

== **Please Note** ==

You can use the `ph-audio-image-video-recorder` element like the following.

Example 1:

    <ph-audio-image-video-recorder mode="image"></ph-audio-image-video-recorder>

Example 2:

    <ph-audio-image-video-recorder mode="video" visualizeAudio="true"></ph-audio-image-video-recorder>

Example 3:

    <ph-audio-image-video-recorder mode="audio" visualizeAudio="true"></ph-audio-image-video-recorder>

@group Inovex Survey Elements
@element ph-audio-image-video-recorder
@homepage inovex.de
-->

<!-- 3rd party dependecies -->
<link rel="import" href="../polymer/polymer.html">
<link rel="import" href="../core-icons/av-icons.html">
<link rel="import" href="../core-icons/image-icons.html">
<link rel="import" href="../paper-radio-button/paper-radio-button.html">
<link rel="import" href="../paper-radio-group/paper-radio-group.html">
<link rel="import" href="../paper-button/paper-button.html">
<link rel="import" href="../paper-slider/paper-slider.html">

<!-- self-defined elements -->
<link rel="import" href="../ph-core-audio/ph-core-audio.html">
<link rel="import" href="../ph-core-image/ph-core-image.html">
<link rel="import" href="../ph-core-video/ph-core-video.html">
<link rel="import" href="../ph-core-usermedia/ph-core-usermedia.html">
<link rel="import" href="../ph-core-mediastreamtrack/ph-core-mediastreamtrack.html">

<!-- mixins for shared functionality -->
<link rel="import" href="../ph-polymer-mixins/mixins-helper.html">

<polymer-element name="ph-audio-image-video-recorder" attributes="mode visualizeAudio">
  <template>
    <!-- Stylesheet -->
    <link rel="stylesheet" href="ph-audio-image-video-recorder.css">

    <div vertical layout center>

      <!-- START video / device container -->
      <div horizontal layout center>

        <!-- START video container -->
        <div id="videoContainer" class="[[ mode ]]" flex>

          <!-- START UI State: ACCESS GRANTED -->
          <template if="{{ _currentUIState == 'deviceAccessGranted' }}">
            <template if="[[ mode == 'video' || mode == 'image' ]]">
              <!-- live camera -->
              <video flex id="liveVideo" class="{{ _filterSelected }}" muted></video>
            </template>

            <!-- audio visualizer overlay -->
            <template if="[[ mode == 'audio' || mode == 'video' ]]">
              <ph-core-audio
                id="audioComponent"
                visualizeAudio="{{ visualizeAudio }}"
                gain="{{ _gainValue }}"
                >
              </ph-core-audio>
            </template>
          </template>
          <!-- END UI State: ACCESS GRANTED -->

          <!-- START UI State: MEDIA RECORDED -->
          <template if="{{ _currentUIState == 'mediaRecorded' }}">
            <!-- playback from existing video -->
            <template if="[[ mode == 'video' ]]">
              <video id="playbackVideo"
                     class="{{ _filterSelected }}" on-ended="{{ onPlaybackEnded }}"
                     on-timeupdate="{{ onPlaybackTimeupdate }}"></video>
              <audio id="playbackAudio" src="{{ _audioObjectURL }}" volume="1.0" type="audio/wave"></audio>
              <div id="progressLine"></div>
            </template>

            <template if="[[ mode == 'audio' ]]">
              <audio id="playbackAudio" src="{{ _audioObjectURL }}" volume="1.0" type="audio/wave"
                     on-ended="{{ onPlaybackEnded }}" on-timeupdate="{{ onPlaybackTimeupdate }}"></audio>
              <div id="progressLine"></div>
            </template>

            <template if="[[ mode == 'image' ]]">
              <!-- picture from camera -->
              <img src="{{ _imageDataURL }}" class="{{ _filterSelected }}">
            </template>
          </template>
          <!-- END UI State: MEDIA RECORDED -->
        </div>
        <!-- END video container -->

        <!-- START device list -->
        <div self-end> <!-- positioning on bottom -->
          <div id="deviceList" layout vertical>
            <template if="{{ (mode == 'video' || mode == 'image') && _cameras.length > 0 }}">
              <paper-radio-group selected="{{ _selectedCameraId }}" class="spaceToTop" layout vertical>
                <template repeat="{{ device, index in _cameras }}">
                  <paper-radio-button
                    name="{{ device.id }}"
                    label="Camera {{ index + 1 }} {{ device.facing != '' ? '(' + device.facing + ')' : '' }}"
                    on-tap="{{ onCameraTapped }}">
                  </paper-radio-button>
                </template>
              </paper-radio-group>
            </template>

            <template if="[[ mode == 'video' || mode == 'audio' ]]">
              <paper-radio-group selected="{{ _selectedMicrophoneId }}" class="spaceToTop" layout vertical>
                <template repeat="{{ device, index in _microphones }}">
                  <paper-radio-button
                    name="{{ device.id }}"
                    label="Microphone {{ index + 1 }} {{ device.facing != '' ? '(' + device.facing + ')' : '' }}"
                    on-tap="{{ onMicrophoneTapped }}">
                  </paper-radio-button>
                </template>
              </paper-radio-group>

              <paper-slider min="0" max="100" value="[[ _gainValueInit * 100 ]]"
                            on-immediate-value-change="{{ onMicrophoneVolumeChange }}"></paper-slider>
            </template>
          </div>
        </div>
        <!-- END device list -->
      </div>
      <!-- END video / device container -->

      <!-- START button container -->
      <div id="buttonContainer" class="spaceToTop">
        <!-- First enable camera/mic to record anything -->
        <template if="{{ _currentUIState == 'waitingForDeviceAccess' }}">
          <!-- activate button -->
          <paper-button raised on-tap="{{ onEnableTapped }}" class="spaceToTop">
            <core-icon icon="av:videocam"></core-icon>
            <div>Grant access</div>
          </paper-button>
        </template>

        <template if="{{ _currentUIState == 'deviceAccessGranted' }}">
          <div center layout horizontal>
            <div center layout vertical>
              <template bind ref="audioVideoImageButtons"></template>
              <!-- filters -->
              <template if="[[ mode == 'video' || mode == 'image' ]]">
                <paper-radio-group selected="{{ _filterSelected }}" class="spaceToTop">
                  <template repeat="{{ filter, index in _filters }}">
                    <paper-radio-button
                      name="{{ filter }}"
                      label="{{ filter }}"
                      on-tap="{{ onFilterTapped }}">
                    </paper-radio-button>
                  </template>
                </paper-radio-group>
              </template>
            </div>
          </div>
        </template>

        <template if="{{ _currentUIState == 'mediaRecorded' }}">
          <div center layout horizontal>

            <template if="[[ mode == 'audio' || mode == 'video' ]]">
              <!-- play button -->
              <template if="{{ _currentPlayerState == 'stopped' || _currentPlayerState == 'paused' }}">
                <paper-button id="playButton" raised on-tap="{{ onPlayTapped }}">
                  <core-icon icon="av:play-arrow"></core-icon>
                </paper-button>
              </template>

              <!-- pause button -->
              <template if="{{ _currentPlayerState == 'playing' }}">
                <paper-button id="pauseButton" raised on-tap="{{ onPauseTapped }}">
                  <core-icon icon="av:pause"></core-icon>
                </paper-button>
              </template>

              <!-- stop button -->
              <paper-button id="stopButton" raised on-tap="{{ onStopTapped }}">
                <core-icon icon="av:stop"></core-icon>
              </paper-button>
            </template>

            <template if="[[ mode == 'audio' || mode == 'video' || mode == 'image' ]]">
              <!-- redo button -->
              <paper-button raised on-tap="{{ onRedoTapped }}">
                <core-icon icon="av:replay"></core-icon>
                <div>Aufnahme wiederholen</div>
              </paper-button>
            </template>
          </div>
        </template>
      </div>
      <!-- END button container -->

      <!-- START console container -->
      <div id="consoleContainer" class="spaceToTop">
        <template if="{{ (mode == 'audio' || mode == 'video') && _recordingNow }}">
          Recording ... {{ _recordingTime }}s
        </template>

        <template if="{{ mode == 'audio' && !_recordingNow && _currentUIState == 'mediaRecorded' }}">
          Recorded {{ _totalDuration }}s
        </template>

        <template if="{{ mode == 'video' && !_recordingNow && _currentUIState == 'mediaRecorded' }}">
          Frames captured: {{ _videoFramesCaptured }}, {{ _totalDuration }}s Video (~{{ _videoFps }} fps)
        </template>
      </div>
      <!-- END console container -->

    </div>

    <!-- START bound templates -->
    <template id="audioVideoImageButtons">
      <template if="[[ mode == 'audio' || mode == 'video' ]]">
        <paper-button id="recordButton" raised on-tap="{{ onRecordingTapped }}">

          <!-- glow/pulsate the record icon in recording mode -->
          <template if="{{ !_recordingNow }}">
            <core-icon icon="radio-button-off"></core-icon>
          </template>

          <template if="{{ _recordingNow }}">
            <core-icon icon="radio-button-on" class="pulsate"></core-icon>
          </template>

          <div>
            <template if="{{ !_recordingNow }}">Start record</template>
            <template if="{{ _recordingNow }}">Stop record</template>
          </div>
        </paper-button>
      </template>

      <template if="[[ mode == 'image' ]]">
        <paper-button id="snapButton" raised on-tap="{{ onSnapTapped }}">
          <core-icon icon="image:photo-camera"></core-icon>
          <div>Capture image</div>
        </paper-button>
      </template>
    </template>
    <!-- END bound templates -->

  </template>

  <script>
    (function () {
      Polymer(Polymer.mixin({
        publish: {

          /**
           * The usermedia mode to record from. Valid modes are `video`, `audio` and `image`.
           *
           * @attribute mode
           * @type String
           * @default 'video'
           */
          mode: 'video',

          /**
           * Enables or disables the audio visualizer.
           *
           * @attribute visualizeAudio
           * @type Boolean
           * @default true
           */
          visualizeAudio: true
        },

        _filters: [                         // CSS 3 filters
          'No filter',
          'grayscale',
          'sepia',
          'invert',
          'brightness',
          'blur'
        ],
        _filterSelected: 'No filter',     // Selected filter

        // the UI states
        _uiStateEnum: {
          WAITING_FOR_DEVICE_ACCESS:  'waitingForDeviceAccess',
          DEVICE_ACCESS_GRANTED:      'deviceAccessGranted',
          MEDIA_RECORDED:             'mediaRecorded'
        },
        _currentUIState: '',

        // substates (for the player)
        _playerStateEnum: {
          STOPPED: 'stopped',
          PLAYING: 'playing',
          PAUSED: 'paused'
        },
        _currentPlayerState: 'mediaStopped',

        _stream: null,                      // GetUserMedia stream object
        _microphones: [],                   // list of provided microphones
        _cameras: [],                       // list of provided cameras
        _selectedMicrophoneId: '',          // id of selected microphone
        _selectedCameraId: '',              // id of selected camera

        // image specific
        _imageDataURL: null,                // image data URL (stores the image)

        // video specific
        _videoBlob: null,                   // video data URL (stores the video)
        _videoFramesCaptured: 0,            // captured video frames in blob
        _videoFps: 0,                       // fps of the video
        _videoDim: {                        // video resolution
          width: 320,
          height: 240
        },

        // audio specific
        _audioBlob: null,                   // audio data URL (stores the audio)
        _gainValueInit: 0.8,                // intial gain value
        _gainValue: 0.8,                    // current gain value

        _recordingNow: false,               // flag that indicates if the recording is running (video/audio only)
        _totalDuration: null,               // audio/video duration in seconds
        _recordingTime: 0,                  // elapsed time of the recording

        // components
        _audioComponent: null,              // reference to the audio component
        _imageComponent: null,              // reference to the image component
        _videoComponent: null,              // reference to the video component

        // entry point
        ready: function () {
          if (!this._validParameters()) {
            console.log('One of your parameter is not valid, please check.');
            return;
          }

          if (!this._browserSupported()) {
            console.log('Your browser does not support the current mode, sorry :(');
            return;
          }

          this._currentUIState = this._uiStateEnum.WAITING_FOR_DEVICE_ACCESS;
        },

        // ===== UI triggered methods =====

        // all: event handler to initialize the UserMedia API
        onEnableTapped: function (event, detail, sender) {
          this._initGetUserMedia();
        },

        // all: event handler to dismiss the current recording to start a new one
        onRedoTapped: function (event, detail, sender) {
          this._initVideoAudio();
        },

        // audio: event handler to reinit the UserMedia stream, when the microphone source changes
        onMicrophoneTapped: function (event, detail, sender) {
          if (this._currentUIState == this._uiStateEnum.DEVICE_ACCESS_GRANTED && !this._recordingNow) {
            this._initGetUserMedia();
          }
        },

        // audio
        onMicrophoneVolumeChange: function (event, detail, sender) {
          if (this._currentUIState == this._uiStateEnum.DEVICE_ACCESS_GRANTED) {
            this._gainValue = sender.immediateValue / 100;
          }
        },

        // image: event handler to grap a picture of the current stream frame
        onSnapTapped: function (event, detail, sender) {
          this._takePicture();
          this._currentUIState = this._uiStateEnum.MEDIA_RECORDED;
        },

        // image, video: event handler to reinit the UserMedia stream, when the camera source changes
        onCameraTapped: function (event, detail, sender) {
          if (this._currentUIState == this._uiStateEnum.DEVICE_ACCESS_GRANTED && !this._recordingNow) {
            this._initGetUserMedia();
          }
        },

        // audio, video: event handler to start and stop recording
        onRecordingTapped: function (event, detail, sender) {
          this._recordingNow = !this._recordingNow;

          // recording
          if (this._recordingNow) {
            this._startRecord();
          } else {
            this._stopRecord();
          }
        },

        // audio, video: event handler to sync the current playback time with the visual progress bar
        onPlaybackTimeupdate: function (event, detail, sender) {
          var percentPlayed = (sender.currentTime / this._totalDuration) * 100;
          this._id('#progressLine').style.width = percentPlayed + '%';
        },

        // audio, video: event handler to start playing the recorded stream(s) (based on the current mode)
        onPlayTapped: function (event, detail, sender) {
          this._currentPlayerState = this._playerStateEnum.PLAYING;

          if (this.mode == 'video') {
            this._id('#playbackVideo').play();
          }
          this._id('#playbackAudio').play();
        },

        // audio, video: event handler to pause the playing audio/video (based on the current mode)
        onPauseTapped: function (event, detail, sender) {
          this._currentPlayerState = this._playerStateEnum.PAUSED;

          if (this.mode == 'video') {
            this._id('#playbackVideo').pause();
          }
          this._id('#playbackAudio').pause();
        },

        // audio, video: event handler to stop the playing audio/video (based on the current mode)
        onStopTapped: function (event, detail, sender) {
          this._currentPlayerState = this._playerStateEnum.STOPPED;

          if (this.mode == 'video') {
            this._id('#playbackVideo').load();
          }
          this._id('#playbackAudio').load();
        },

        // image, video: event handler to apply the selected filter
        onFilterTapped: function (event, detail, sender) {
          this._filterSelected = sender.getAttribute('name');
        },

        // video
        onPlaybackEnded: function (event, detail, sender) {
          this._currentPlayerState = this._playerStateEnum.STOPPED;
          this._id('#progressLine').style.width = '100%';
        },

        // ===== private methods =====

        // @todo write mixin helper to check parameter array against regexes + data types
        _validParameters: function() {
          var param = [];
          param['mode'] = this.mode == 'image' || this.mode == 'video' || this.mode == 'audio';
          param['visualizeAudio'] = this.visualizeAudio === true || this.visualizeAudio === false;
          return param['mode'] && param['visualizeAudio'];
        },

        // checks the browser support of each component
        _browserSupported: function() {
          switch (this.mode) {
            case 'image':
              var c = new PhCoreImage();
              return c.isSupported();
              break;
            case 'video':
              var c = new PhCoreVideo();
              return c.isSupported();
              break;
            case 'audio':
              var c = new PhCoreAudio();
              return c.isSupported();
              break;
            default:
              break;
          }
        },

        // called when the ui state changes
        _currentUIStateChanged: function () {
          switch (this._currentUIState) {
            case this._uiStateEnum.WAITING_FOR_DEVICE_ACCESS:
              this._onWaitingForDeviceAccess();
              break;
            case this._uiStateEnum.DEVICE_ACCESS_GRANTED:
              this._onDeviceAccessGranted();
              break;
            default:
              break;
          }
        },

        // called when the user gives access to the requested sources
        _onDeviceAccessGranted: function () {
          // we must short delay to catch the DOM changes. This is a common Polymer pattern.
          this.async(function () {
            // init live videostream
            if (this.mode == 'image' || this.mode == 'video') {
              var liveVideo = this._id('#liveVideo');
              liveVideo.src = window.URL.createObjectURL(this._stream);
              liveVideo.play();
            }

            // init image component
            if (this.mode == 'image') {
              this._imageComponent = new PhCoreImage();
              this._imageComponent.init(this._id('#liveVideo'), this._videoDim.width, this._videoDim.height);
            }

            // init audio component
            if (this.mode == 'audio' || this.mode == 'video') {
              this._audioComponent = this._id('#audioComponent');
              this._audioComponent.init(this._stream);
            }

            // init video component
            if (this.mode == 'video') {
              this._videoComponent = new PhCoreVideo();
              this._videoComponent.init(this._id('#liveVideo'), this._videoDim.width, this._videoDim.height);
            }
          });
        },

        // called when the component is initialized
        _onWaitingForDeviceAccess: function () {
          // we must short delay to catch the DOM changes. This is a common Polymer pattern.
          this.async(function () {
            // get device sources
            try {
              var mst = new PhCoreMediaStreamTrack();
              mst.getDevices(this._onDevicesReady.bind(this));
            } catch (e) {
              console.error(e);
            }
          });
        },

        // called when the devices are received from the MediaStreamTrack API
        _onDevicesReady: function(devices) {
          this._microphones = devices.microphones;
          this._cameras = devices.cameras;

          // default is first? *assumption*
          this._selectedCameraId = this._cameras[0].id;
          this._selectedMicrophoneId = this._microphones[0].id;
        },

        // Resets the UI State to the second state available (after the user granted access to the mic + webcam).
        // This method is used to do the first initializing (BEFORE the very first recording) and to reset the
        // state AFTER a recording was made.
        _initVideoAudio: function () {
          // if we only change the source, we must manually trigger the method, because the changed watcher only
          // watches on state changes and will never trigger the method in such a case.
          if (this._currentUIState == this._uiStateEnum.DEVICE_ACCESS_GRANTED) {
            this._onDeviceAccessGranted();
          }

          this._currentUIState = this._uiStateEnum.DEVICE_ACCESS_GRANTED;
        },

        // Initializes the MediaCapture API
        _initGetUserMedia: function () {
          var usermediaConfig = {
            enabled: {
              audio: this.mode == 'image' ? false : true,
              video: this.mode == 'audio' ? false : true
            },
            audio: {
              mandatory: {
                googEchoCancellation: false,
                googAutoGainControl: false,
                googNoiseSuppression: false,
                googHighpassFilter: false
              },
              optional: [{sourceId: this._selectedMicrophoneId}]
            },
            video: {
              mandatory: {
                maxWidth: this._videoDim.width,
                maxHeight: this._videoDim.height
              },
              optional: [{sourceId: this._selectedCameraId}]
            }
          };

          var umc = new PhCoreUserMedia();
          umc.init(usermediaConfig, this._onUserMediaSuccess.bind(this), this._onUserMediaError.bind(this));
        },

        // success callback of UserMedia API
        _onUserMediaSuccess: function (stream) {
          // save the stream in this object to access it everywhere
          this._stream = stream;
          this._initVideoAudio();
        },

        // error callback of UserMedia API
        _onUserMediaError: function (error) {
          switch (error.name) {
            case 'PermissionDeniedError':
              alert('It seems that you have not allowed your Browser to access any media device. Please remove that ' +
              'restriction and try again.');
              break;
          }
        },

        // retrieves the data url from the image component
        _takePicture: function () {
          this._imageDataURL = this._imageComponent.getPictureDataURL('image/jpeg', 0.92);
        },

        // updates the length of the current recording every second
        _updateRecordingDuration: function () {
          var audioComponent = this._audioComponent;
          var update = function () {
            // @todo as long as this component is just a prototype we ignore the fact, that audio and video is not
            // correctly synced. We just play them both at the same time. The audio component ist the component wich
            // is used in video and audio mode, so we just use it to get the duration of the recording.
            this._recordingTime = Math.round(audioComponent.getDuration() / 1000);

            if (this._recordingNow == true) {
              this._updateRecordingDuration();
            }
          };

          update();
          this.async(update, null, 1000);
        },

        // starts the record for the current mode
        _startRecord: function () {
          if (this.mode == 'video') {
            this._videoComponent.startRecord();
          }
          this._audioComponent.startRecord();
          this._updateRecordingDuration();
        },

        // stops the record for the current mode
        _stopRecord: function () {
          if (this.mode == 'video' || this.mode == 'audio') {
            this._totalDuration = (this._audioComponent.getDuration() / 1000);
          }

          // switch from live to recorded video
          this._currentUIState = this._uiStateEnum.MEDIA_RECORDED;
          this._currentPlayerState = this._playerStateEnum.STOPPED;

          // video
          if (this.mode == 'video') {
            this._videoComponent.stopRecord();
            this._videoComponent.getRecordedBlob((function(videoInfo) {
              this._videoBlob = videoInfo.blob;
              this._videoFramesCaptured = videoInfo.frames;
              this._videoFps = videoInfo.fps;

              var playbackVideo = this._id('#playbackVideo');
              this._id('#playbackVideo').src = window.URL.createObjectURL(this._videoBlob);  // crossbrowser safe!
              playbackVideo.style.width = this._videoDim.width * 2 + 'px';
              playbackVideo.style.height = this._videoDim.height * 2 + 'px';
            }).bind(this));
          }

          // audio
          this._audioComponent.stopRecord();
          this._audioComponent.getRecordedBlob((function (audioBlob) {
            this._audioBlob = audioBlob;
            this._audioObjectURL = window.URL.createObjectURL(this._audioBlob);
          }).bind(this));
        }
      }, mixinsHelper));
    })();
  </script>
</polymer-element>