Skip to content

Commit

Permalink
Optimise for tablet.
Browse files Browse the repository at this point in the history
  • Loading branch information
@gmarty
gmarty committed Aug 23, 2016
1 parent 48f381d commit 5579f9f
Show file tree
Hide file tree
Showing 8 changed files with 82 additions and 57 deletions.
1 change: 1 addition & 0 deletions app/css/app.css
View file
Expand Up @@ -84,6 +84,7 @@ button {

/* App specific styles */
.app-view-container {
overflow: auto;
height: 100%;
padding: 5rem 0 0;
background: #fff;
Expand Down
1 change: 0 additions & 1 deletion app/index.html
View file
Expand Up @@ -34,7 +34,6 @@
</head>
<body>
<section class="app-view-container"></section>
<section class="microphone"></section>
<section class="full-screen"></section>
</body>
</html>
8 changes: 0 additions & 8 deletions app/js/controllers/reminders.js
View file
Expand Up @@ -4,7 +4,6 @@ import ReactDOM from 'components/react-dom';
import BaseController from './base';

import Reminders from '../views/reminders';
import Microphone from '../views/microphone';
import FullScreen from '../views/full-screen';

export default class RemindersController extends BaseController {
Expand All @@ -16,13 +15,6 @@ export default class RemindersController extends BaseController {
}), this.mountNode
);

ReactDOM.render(
React.createElement(Microphone, {
speechController: this.speechController,
server: this.server,
}), document.querySelector('.microphone')
);

ReactDOM.render(
React.createElement(FullScreen), document.querySelector('.full-screen')
);
Expand Down
18 changes: 13 additions & 5 deletions app/js/lib/intent-parser/reminder-confirmation.js
View file
Expand Up @@ -160,16 +160,24 @@ export default class Confirmation {
*/
[p.formatHoursAndMinutes](date) {
date = moment(date);
let format;

if (date.minute() === 0) {
return date.format('h A'); // 7 PM
format = date.format('h A'); // 7 PM
} else if (date.minute() === 15) {
return date.format('[quarter past] h A');
format = date.format('[quarter past] h A');
} else if (date.minute() === 30) {
return date.format('[half past] h A');
format = date.format('[half past] h A');
} else if (date.minute() === 45) {
const nextHour = date.add(1, 'hour');
return nextHour.format('[quarter to] h A');
format = nextHour.format('[quarter to] h A');
} else {
format = date.format('h m A'); // 6 24 AM
}
return date.format('h m A'); // 6 24 AM

// Some speech synthesisers pronounce "AM" as "ham" (not "A. M.").
return format
.replace(/ AM$/gi, ' A.M.')
.replace(/ PM$/gi, ' P.M.');
}
}
9 changes: 3 additions & 6 deletions app/js/lib/speech-controller.js
View file
Expand Up @@ -12,7 +12,6 @@ const p = Object.freeze({
idle: Symbol('idle'),

// Methods
startListeningForWakeword: Symbol('startListeningForWakeword'),
stopListeningForWakeword: Symbol('stopListeningForWakeword'),
listenForUtterance: Symbol('listenForUtterance'),
handleSpeechRecognitionEnd: Symbol('handleSpeechRecognitionEnd'),
Expand Down Expand Up @@ -68,7 +67,7 @@ export default class SpeechController extends EventDispatcher {
}

start() {
return this[p.startListeningForWakeword]();
return this.startListeningForWakeword();
}

startSpeechRecognition() {
Expand All @@ -77,17 +76,15 @@ export default class SpeechController extends EventDispatcher {
return this[p.stopListeningForWakeword]()
.then(this[p.listenForUtterance].bind(this))
.then(this[p.handleSpeechRecognitionEnd].bind(this))
.then(this[p.startListeningForWakeword].bind(this))
.catch((err) => {
console.log('startSpeechRecognition err', err);
this.emit(EVENT_INTERFACE[4], { type: EVENT_INTERFACE[4] });
this[p.startListeningForWakeword]();
});
}

stopSpeechRecognition() {
return this[p.speechRecogniser].abort()
.then(this[p.startListeningForWakeword].bind(this));
.then(this.startListeningForWakeword.bind(this));
}

/**
Expand All @@ -100,7 +97,7 @@ export default class SpeechController extends EventDispatcher {
return this[p.speechSynthesis].speak(text);
}

[p.startListeningForWakeword]() {
startListeningForWakeword() {
this.emit(EVENT_INTERFACE[0], { type: EVENT_INTERFACE[0] });
this[p.idle] = true;

Expand Down
39 changes: 19 additions & 20 deletions app/js/lib/speech/synthesis.js
View file
Expand Up @@ -38,29 +38,28 @@ export default class SpeechSynthesis {
* @return {Promise} A promise that resolves when the utterance is finished.
*/
speak(text = '') {
if (!text) {
return Promise.resolve();
}

const utterance = new SpeechSynthesisUtterance(text);

if (this[p.preferredVoice]) {
// Use a preferred voice if available.
utterance.voice = this[p.preferredVoice];
}
utterance.lang = 'en-GB';
utterance.pitch = VOICE_PITCH;
utterance.rate = VOICE_RATE;

this[p.synthesis].speak(utterance);

return new Promise((resolve, reject) => {
utterance.addEventListener('end', () => {
if (!text) {
return resolve();
}

const utterance = new SpeechSynthesisUtterance(text);

if (this[p.preferredVoice]) {
// Use a preferred voice if available.
utterance.voice = this[p.preferredVoice];
}
utterance.lang = 'en-GB';
utterance.pitch = VOICE_PITCH;
utterance.rate = VOICE_RATE;
utterance.onend = () => {
resolve();
});
utterance.addEventListener('error', () => {
};
utterance.onerror = () => {
reject();
});
};

this[p.synthesis].speak(utterance);
});
}

Expand Down
30 changes: 15 additions & 15 deletions app/js/views/microphone.jsx
View file
Expand Up @@ -5,7 +5,7 @@ export default class Microphone extends React.Component {
super(props);

this.state = {
isListening: false,
isListeningToSpeech: false,
};

this.speechController = props.speechController;
Expand All @@ -16,23 +16,23 @@ export default class Microphone extends React.Component {
this.bufferSource = null;
this.timeout = null;

this.onWakeWord = this.onWakeWord.bind(this);
this.onSpeechRecognitionEnd = this.onSpeechRecognitionEnd.bind(this);
this.startListeningToSpeech = this.startListeningToSpeech.bind(this);
this.stopListeningToSpeech = this.stopListeningToSpeech.bind(this);
this.onClickMic = this.onClickMic.bind(this);
}

componentDidMount() {
this.loadAudio();

this.speechController.on('wakeheard', this.onWakeWord);
//this.speechController.on('wakeheard', this.startListeningToSpeech);
this.speechController.on('speechrecognitionstop',
this.onSpeechRecognitionEnd);
this.stopListeningToSpeech);
}

componentWillUnmount() {
this.speechController.off('wakeheard', this.onWakeWord);
//this.speechController.off('wakeheard', this.startListeningToSpeech);
this.speechController.off('speechrecognitionstop',
this.onSpeechRecognitionEnd);
this.stopListeningToSpeech);
}

loadAudio() {
Expand Down Expand Up @@ -73,20 +73,20 @@ export default class Microphone extends React.Component {
this.bufferSource = null;
}

onWakeWord() {
startListeningToSpeech() {
this.playBleep();
this.setState({ isListening: true });
this.setState({ isListeningToSpeech: true });
}

onSpeechRecognitionEnd() {
stopListeningToSpeech() {
this.stopBleep();
this.setState({ isListening: false });
this.setState({ isListeningToSpeech: false });
}

onClickMic() {
if (!this.state.isListening) {
if (!this.state.isListeningToSpeech) {
this.playBleep();
this.setState({ isListening: true });
this.setState({ isListeningToSpeech: true });
this.timeout = setTimeout(() => {
// When the sound finished playing
this.stopBleep();
Expand All @@ -97,7 +97,7 @@ export default class Microphone extends React.Component {

clearTimeout(this.timeout);
this.stopBleep();
this.setState({ isListening: false });
this.setState({ isListeningToSpeech: false });
this.speechController.stopSpeechRecognition();
}

Expand All @@ -106,7 +106,7 @@ export default class Microphone extends React.Component {
return null;
}

const className = this.state.isListening ? 'listening' : '';
const className = this.state.isListeningToSpeech ? 'listening' : '';

return (
<div className={className} onClick={this.onClickMic}>
Expand Down
33 changes: 31 additions & 2 deletions app/js/views/reminders.jsx
View file
Expand Up @@ -2,6 +2,7 @@ import React from 'components/react';
import moment from 'components/moment';

import Toaster from './toaster';
import Microphone from '../views/microphone';
import RemindersList from './reminders/reminders-list';

export default class Reminders extends React.Component {
Expand All @@ -17,7 +18,9 @@ export default class Reminders extends React.Component {

this.refreshInterval = null;
this.toaster = null;
this.microphone = null;
this.debugEvent = this.debugEvent.bind(this);
this.onWakeWord = this.onWakeWord.bind(this);
this.onReminder = this.onReminder.bind(this);
this.onParsingFailure = this.onParsingFailure.bind(this);
this.onWebPushMessage = this.onWebPushMessage.bind(this);
Expand All @@ -28,7 +31,10 @@ export default class Reminders extends React.Component {
}

componentDidMount() {
this.refreshReminders();
this.refreshReminders()
.then(() => {
console.log('Reminders loaded');
});

// Refresh the page every 5 minutes if idle.
this.refreshInterval = setInterval(() => {
Expand All @@ -43,6 +49,8 @@ export default class Reminders extends React.Component {
this.speechController.on('speechrecognitionstart', this.debugEvent);
this.speechController.on('speechrecognitionstop', this.debugEvent);
this.speechController.on('reminder', this.debugEvent);

this.speechController.on('wakeheard', this.onWakeWord);
this.speechController.on('reminder', this.onReminder);
this.speechController.on('parsing-failed', this.onParsingFailure);
this.server.on('push-message', this.onWebPushMessage);
Expand All @@ -57,6 +65,8 @@ export default class Reminders extends React.Component {
this.speechController.off('speechrecognitionstart', this.debugEvent);
this.speechController.off('speechrecognitionstop', this.debugEvent);
this.speechController.off('reminder', this.debugEvent);

this.speechController.off('wakeheard', this.onWakeWord);
this.speechController.off('reminder', this.onReminder);
this.speechController.off('parsing-failed', this.onParsingFailure);
this.server.off('push-message', this.onWebPushMessage);
Expand All @@ -72,7 +82,8 @@ export default class Reminders extends React.Component {
}

refreshReminders() {
this.server.reminders.getAll()
// @todo Add a loader.
return this.server.reminders.getAll()
.then((reminders) => {
this.setState({ reminders });
});
Expand All @@ -85,9 +96,15 @@ export default class Reminders extends React.Component {
this.setState({ reminders });
}

onWakeWord() {
this.microphone.startListeningToSpeech();
}

onReminder(evt) {
const { recipients, action, due, confirmation } = evt.result;

this.microphone.stopListeningToSpeech();

// @todo Nice to have: optimistic update.
// https://github.com/fxbox/calendar/issues/32
this.server.reminders
Expand All @@ -102,7 +119,9 @@ export default class Reminders extends React.Component {
this.toaster.success(confirmation);
this.speechController.speak(confirmation)
.then(() => {
console.log('Utterance terminated.');
this.toaster.hide();
this.speechController.startListeningForWakeword();
});
})
.catch((res) => {
Expand All @@ -113,16 +132,21 @@ export default class Reminders extends React.Component {
this.speechController.speak(message)
.then(() => {
this.toaster.hide();
this.speechController.startListeningForWakeword();
});
});
}

onParsingFailure() {
this.microphone.stopListeningToSpeech();

const message = 'I did not understand that. Can you repeat?';
this.toaster.warning(message);
this.speechController.speak(message)
.then(() => {
console.log('Utterance terminated.');
this.toaster.hide();
this.speechController.startListeningForWakeword();
});
}

Expand All @@ -143,6 +167,11 @@ export default class Reminders extends React.Component {
return (
<section className="reminders">
<Toaster ref={(t) => this.toaster = t}/>
<div className="microphone">
<Microphone ref={(t) => this.microphone = t}
speechController={this.speechController}
server={this.server}/>
</div>
<RemindersList reminders={this.state.reminders}
server={this.server}
refreshReminders={this.refreshReminders}/>
Expand Down

0 comments on commit 5579f9f

Please sign in to comment.