From ce45224dffce8d9a48c5ce7d0d1e50be6cbbf570 Mon Sep 17 00:00:00 2001 From: Brendan Reville Date: Fri, 25 Oct 2019 22:37:23 -0700 Subject: [PATCH 01/15] Use React for UI Beginnings of a model in which we keep the app running with its own state machine, rendering to a couple canvases at full frame rate, using its own model classes... but with React used to draw the HTML-based UI on top of it, for headers, footers, and the body of the app. This first attempt just tells React to re-render 10 times per second. In this first commit, the first scene is rendered with both its original UI and a React-render duplicate UI on top. The React UI doesn't yet shrink text below certain widths, though. --- public/index.html | 25 +++++++--- src/demo/index.jsx | 17 ++++++- src/demo/ui.jsx | 117 +++++++++++++++++++++++++++++++++++++++++++++ src/index.js | 1 - webpack.config.js | 1 - 5 files changed, 151 insertions(+), 10 deletions(-) create mode 100644 src/demo/ui.jsx delete mode 100644 src/index.js diff --git a/public/index.html b/public/index.html index e5706cde..559eef6a 100644 --- a/public/index.html +++ b/public/index.html @@ -4,12 +4,6 @@ ML Activities Playground -
- -
-
- -
- -
-
From c66c33b0938dac9cd93526cdf1e3948ee97f8712 Mon Sep 17 00:00:00 2001 From: Brendan Reville Date: Sat, 26 Oct 2019 13:18:58 -0700 Subject: [PATCH 05/15] Continue -> Skip on predict scene --- src/demo/ui.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/demo/ui.jsx b/src/demo/ui.jsx index 315c6ea6..178814f8 100644 --- a/src/demo/ui.jsx +++ b/src/demo/ui.jsx @@ -326,7 +326,7 @@ class Predict extends React.Component { style={styles.continueButton} onClick={() => toMode(Modes.Pond)} > - Continue + Skip From d38f8742cb08cd7a30f63118a022169e4f3bc2d9 Mon Sep 17 00:00:00 2001 From: Brendan Reville Date: Sat, 26 Oct 2019 13:22:50 -0700 Subject: [PATCH 06/15] Remove more unused code --- src/MLActivities.jsx | 74 ---- src/activities/imageRecognition/Draggable.jsx | 29 -- src/activities/imageRecognition/Droppable.jsx | 38 --- .../imageRecognition/ImageRecognition.jsx | 320 ------------------ .../imageRecognition/PredictionUpload.jsx | 32 -- .../imageRecognition/TrainingImageUpload.jsx | 48 --- src/activities/rps/IntroScreen.jsx | 52 --- src/activities/rps/PlayRound.jsx | 65 ---- src/activities/rps/PlayRoundInstructions.jsx | 57 ---- src/activities/rps/PlayRoundResult.jsx | 89 ----- src/activities/rps/RPS.jsx | 246 -------------- src/activities/rps/TrainingScreen.jsx | 105 ------ 12 files changed, 1155 deletions(-) delete mode 100644 src/MLActivities.jsx delete mode 100644 src/activities/imageRecognition/Draggable.jsx delete mode 100644 src/activities/imageRecognition/Droppable.jsx delete mode 100644 src/activities/imageRecognition/ImageRecognition.jsx delete mode 100644 src/activities/imageRecognition/PredictionUpload.jsx delete mode 100644 src/activities/imageRecognition/TrainingImageUpload.jsx delete mode 100644 src/activities/rps/IntroScreen.jsx delete mode 100644 src/activities/rps/PlayRound.jsx delete mode 100644 src/activities/rps/PlayRoundInstructions.jsx delete mode 100644 src/activities/rps/PlayRoundResult.jsx delete mode 100644 src/activities/rps/RPS.jsx delete mode 100644 src/activities/rps/TrainingScreen.jsx diff --git a/src/MLActivities.jsx b/src/MLActivities.jsx deleted file mode 100644 index 5a93e00f..00000000 --- a/src/MLActivities.jsx +++ /dev/null @@ -1,74 +0,0 @@ -import React from 'react'; -import RPS from './activities/rps/RPS'; -import ImageRecognition from './activities/imageRecognition/ImageRecognition'; -import Button from 'react-bootstrap/lib/Button'; -import Row from 'react-bootstrap/lib/Row'; -import Col from 'react-bootstrap/lib/Col'; -import Grid from 'react-bootstrap/lib/Grid'; -import Panel from 'react-bootstrap/lib/Panel'; - -const Activity = Object.freeze({ - None: 0, - RPS: 1, - ImageRecognition: 2 -}); - -module.exports = class MLActivities extends React.Component { - state = { - currentActivity: Activity.None - }; - - render() { - return ( - - - - -

ML Activities Playground

- {this.state.currentActivity !== Activity.None && ( - - )} - {this.state.currentActivity === Activity.None && ( -
- - -
- )} - {this.state.currentActivity === Activity.RPS && ( - - - - )} - {this.state.currentActivity === Activity.ImageRecognition && ( - - - - )} - - - - - ); - } -}; diff --git a/src/activities/imageRecognition/Draggable.jsx b/src/activities/imageRecognition/Draggable.jsx deleted file mode 100644 index 8bbcd8a6..00000000 --- a/src/activities/imageRecognition/Draggable.jsx +++ /dev/null @@ -1,29 +0,0 @@ -import React from 'react'; -import $ from 'jquery'; -import 'jquery-ui/ui/widgets/draggable'; -import * as PropTypes from 'react/lib/ReactPropTypes'; -window.jQuery = $; -require('jquery-ui-touch-punch'); - -module.exports = class ImageRecognition extends React.Component { - componentDidMount() { - $(this.draggableDiv).draggable({revert: true}); - } - - render() { - return ( -
(this.draggableDiv = element)} - > - {this.props.children} -
- ); - } -}; - -module.exports.propTypes = { - children: PropTypes.node, - guid: PropTypes.string -}; diff --git a/src/activities/imageRecognition/Droppable.jsx b/src/activities/imageRecognition/Droppable.jsx deleted file mode 100644 index b70676ef..00000000 --- a/src/activities/imageRecognition/Droppable.jsx +++ /dev/null @@ -1,38 +0,0 @@ -import React from 'react'; -import $ from 'jquery'; -import 'jquery-ui/ui/effects/effect-drop'; -import 'jquery-ui/ui/widgets/droppable'; -import * as PropTypes from 'react/lib/ReactPropTypes'; -window.jQuery = $; -require('jquery-ui-touch-punch'); - -module.exports = class ImageRecognition extends React.Component { - componentDidMount() { - $(this.droppableDiv).droppable({ - classes: { - 'ui-droppable-active': 'ui-state-active', - 'ui-droppable-hover': 'ui-state-hover' - }, - tolerance: 'touch', - drop: (event, ui) => { - this.props.onDrop(ui.draggable.data('guid')); - } - }); - } - - render() { - return ( -
(this.droppableDiv = element)} - > - {this.props.children} -
- ); - } -}; - -module.exports.propTypes = { - children: PropTypes.node, - onDrop: PropTypes.func -}; diff --git a/src/activities/imageRecognition/ImageRecognition.jsx b/src/activities/imageRecognition/ImageRecognition.jsx deleted file mode 100644 index c3bfe012..00000000 --- a/src/activities/imageRecognition/ImageRecognition.jsx +++ /dev/null @@ -1,320 +0,0 @@ -import React from 'react'; -import SimpleTrainer from '../../utils/SimpleTrainer'; -import Draggable from './Draggable'; -import Droppable from './Droppable'; -import TrainingImageUpload from './TrainingImageUpload'; -import PredictionUpload from './PredictionUpload'; -import Row from 'react-bootstrap/lib/Row'; -import Col from 'react-bootstrap/lib/Col'; -import Button from 'react-bootstrap/lib/Button'; - -const ActivityState = Object.freeze({ - Loading: 0, - Training: 1, - Playing: 2 -}); - -const NO_PREDICTION = -1; -const defaultState = { - classes: [ - { - name: 'Dogs', - examples: 0 - }, - { - name: 'Cats', - examples: 0 - } - ], - activityState: ActivityState.Loading, - predictedClass: NO_PREDICTION -}; - -const activityImages = [ - {guid: 'a', url: 'images/dog1.png'}, - {guid: 'b', url: 'images/dog2.png'}, - {guid: 'c', url: 'images/dog3.png'}, - {guid: 'd', url: 'images/cat1.jpg'}, - {guid: 'e', url: 'images/cat2.jpg'}, - {guid: 'f', url: 'images/cat3.jpg'} -]; - -const IMAGE_SIZE = 227; - -function loadImage(url, size) { - return new Promise(resolve => { - const image = new Image(); - image.width = size; - image.height = size; - image.addEventListener('load', () => { - resolve(image); - }); - image.src = url; - }); -} - -module.exports = class ImageRecognition extends React.Component { - constructor(props) { - super(props); - this.simpleTrainer = new SimpleTrainer(); - } - - state = defaultState; - - componentDidMount() { - this.simpleTrainer.initializeClassifiers().then(() => { - this.setState({activityState: ActivityState.Training}); - }); - } - - render() { - if (this.state.activityState === ActivityState.Loading) { - return
Loading machine learning model data...
; - } - - return ( -
- {this.state.activityState === ActivityState.Training && ( -
- - - {this.state.classes.map((classData, i) => { - return ( - { - this.simpleTrainer.addExampleImage(image, i); - this.updateExampleCounts(i); - }} - /> - ); - })} - - - - -

Drag images to train your machine learning algorithm

- - - - - - - - - - {activityImages.map((image, i) => { - return ( - - { - // loadImage(image.url, IMAGE_SIZE).then((img) => { - // this.simpleTrainer.predictFromImage(img).then((result) => { - // console.log(result); - // }); - // }); - // }} - src={image.url} - className="thumbnail" - style={{ - display: 'inline-block' - }} - width={100} - height={100} - /> - - ); - })} - - - - {this.state.classes.map((classData, i) => { - return ( - - { - const image = activityImages.find(e => { - return e.guid === guid; - }); - loadImage(image.url, IMAGE_SIZE).then(image => { - this.simpleTrainer.addExampleImage(image, i); - this.updateExampleCounts(i); - }); - }} - > - {classData.name} - -
- {classData.examples.toString()} -
- - ); - })} - -
- )} - {this.state.activityState === ActivityState.Training && ( - - - - - - - )} - {this.state.activityState === ActivityState.Playing && ( -
- - - { - this.simpleTrainer.predictFromImage(img).then(trainingResult => { - this.setState({trainingResult}); - }); - }} - /> - - - - -

Tap an image to classify it

- {activityImages.map((image, i) => { - return ( - { - loadImage(image.url, IMAGE_SIZE).then(img => { - this.simpleTrainer.predictFromImage(img).then(result => { - this.setState({ - trainingResult: result - }); - }); - }); - }} - width={100} - height={100} - /> - ); - })} - - -
- )} - {this.state.activityState === ActivityState.Playing && - !!this.state.trainingResult && ( - - -

Predicted Category:

-

- { - this.state.classes[ - this.state.trainingResult.predictedClassId - ].name - } -

- - {JSON.stringify(this.state.trainingResult)} - - - )} - {this.state.activityState === ActivityState.Playing && ( - - - - - - )} -
- ); - } - - resetAllExampleCounts() { - const classes = this.state.classes; - this.state.classes.forEach((c, i) => { - classes[i].examples = 0; - }); - this.setState({classes: classes}); - } - - async updateExampleCounts(i) { - const classes = this.state.classes; - classes[i].examples = this.simpleTrainer.getExampleCount(i); - return this.setState({classes: classes}); - } - - async playRound() { - if (this.video.isPlaying()) { - if (this.simpleTrainer.getNumClasses() > 0) { - let frameDataURI = this.video.getFrameDataURI(400); - - let predictionResult = await this.simpleTrainer.predictFromImage( - this.video.getVideoElement() - ); - this.setState( - { - lastPrediction: { - predictedClass: predictionResult.predictedClassId, - confidence: - predictionResult.confidencesByClassId[ - predictionResult.predictedClassId - ], - playerPlayedImage: frameDataURI, - confidencesByClassId: predictionResult.confidencesByClassId - } - }, - null - ); - } - } - } -}; diff --git a/src/activities/imageRecognition/PredictionUpload.jsx b/src/activities/imageRecognition/PredictionUpload.jsx deleted file mode 100644 index d9a7e7af..00000000 --- a/src/activities/imageRecognition/PredictionUpload.jsx +++ /dev/null @@ -1,32 +0,0 @@ -import React, {PropTypes} from 'react'; - -module.exports = class SingleUpload extends React.Component { - static propTypes = { - predictClass: PropTypes.func - }; - - onUpload() { - var file = document.getElementById('predictfile').files[0]; - if (!file) { - return; - } - var url = URL.createObjectURL(file), // create an Object URL - img = new Image(); // create a temp. image object - - var _this = this; - img.onload = function() { - _this.props.predictClass(img); - }; - - img.src = url; // start convertion file - } - - render() { - return ( -
- Upload your own: - this.onUpload()} /> -
- ); - } -}; diff --git a/src/activities/imageRecognition/TrainingImageUpload.jsx b/src/activities/imageRecognition/TrainingImageUpload.jsx deleted file mode 100644 index 207d33ee..00000000 --- a/src/activities/imageRecognition/TrainingImageUpload.jsx +++ /dev/null @@ -1,48 +0,0 @@ -import React, {PropTypes} from 'react'; - -module.exports = class MultiUpload extends React.Component { - static propTypes = { - className: PropTypes.string, - addTrainingExample: PropTypes.func - }; - - onUpload() { - var files = document.getElementById(this.props.className + 'file').files; - for (var i = 0; i < files.length; ++i) { - var file = files[i]; - if (!file) { - return; - } - var url = URL.createObjectURL(file), // create an Object URL - img = new Image(); // create a temp. image object - var _this = this; - img.onload = function() { - // The height and width doesn't always load so set them if they're 0 - if (!img.width) { - img.width = 500; - } - if (!img.height) { - img.height = 500; - } - _this.props.addTrainingExample(img); - URL.revokeObjectURL(url); - }; - - img.src = url; // start convertion file - } - } - - render() { - return ( -
- {'Upload your own ' + this.props.className + ':'} - this.onUpload()} - multiple - /> -
- ); - } -}; diff --git a/src/activities/rps/IntroScreen.jsx b/src/activities/rps/IntroScreen.jsx deleted file mode 100644 index 0575ebaf..00000000 --- a/src/activities/rps/IntroScreen.jsx +++ /dev/null @@ -1,52 +0,0 @@ -import React from 'react'; -import Col from 'react-bootstrap/lib/Col'; -import Row from 'react-bootstrap/lib/Row'; -import Button from 'react-bootstrap/lib/Button'; -import * as PropTypes from 'react/lib/ReactPropTypes'; - -module.exports = class IntroScreen extends React.Component { - constructor(props) { - super(props); - } - - render() { - return ( - - -

Train a computer to “see” Rock, Paper, Scissors

-

- On the next screen, you’ll be prompted to let your browser to access - the camera. Please click “Allow” -

- -

- If your device doesn’t have a camera: -

-

- Visit code.org/ai from using a modern smartphone that has a camera. -

-

- What is the camera access for?: -

-

You’ll be “training” a computer vision algorithm.

-

- None of the images seen by the camera will ever leave your computer - or be shared or stored by anybody. They’ll be used only for this - tutorial and deleted after you leave this web page. -

- - - ); - } -}; - -module.exports.propTypes = { - onClickContinue: PropTypes.func -}; diff --git a/src/activities/rps/PlayRound.jsx b/src/activities/rps/PlayRound.jsx deleted file mode 100644 index 1ffde6b0..00000000 --- a/src/activities/rps/PlayRound.jsx +++ /dev/null @@ -1,65 +0,0 @@ -import React from 'react'; -import Col from 'react-bootstrap/lib/Col'; -import Row from 'react-bootstrap/lib/Row'; -import * as PropTypes from 'react/lib/ReactPropTypes'; - -module.exports = class PlayRound extends React.Component { - constructor(props) { - super(props); - } - - state = { - countdownNumber: 3 - }; - - componentDidMount() { - setTimeout(() => this.decrementCountdown(), 1000); - setTimeout(() => this.decrementCountdown(), 2000); - setTimeout(() => this.decrementCountdown(), 3000); - setTimeout(() => this.props.onPlayRound(), 3500); - } - - componentWillUnmount() { - console.log('unounting'); - this.hasMounted = false; - } - - decrementCountdown() { - this.setState({countdownNumber: this.state.countdownNumber - 1}); - } - - render() { - return ( - - - - ); - } -}; - -module.exports.propTypes = { - onPlayRound: PropTypes.func, - onMountVideo: PropTypes.func, - imageSize: PropTypes.number -}; diff --git a/src/activities/rps/PlayRoundInstructions.jsx b/src/activities/rps/PlayRoundInstructions.jsx deleted file mode 100644 index 4a3e86ce..00000000 --- a/src/activities/rps/PlayRoundInstructions.jsx +++ /dev/null @@ -1,57 +0,0 @@ -import React from 'react'; -import Col from 'react-bootstrap/lib/Col'; -import Row from 'react-bootstrap/lib/Row'; -import Button from 'react-bootstrap/lib/Button'; -import * as PropTypes from 'react/lib/ReactPropTypes'; - -module.exports = class PlayRoundInstructions extends React.Component { - constructor(props) { - super(props); - } - - render() { - return ( -
- - -

- Have you provided enough training data? -

-

- Did you take enough photos for the machine learning algorithm to - distinguish ROCK from PAPER from SCISSORS? Let’s find out! -

- - - - - - - - - -

- If the computer vision doesn’t seem to work very well, try going - back to Train more. -

-

- If it works well, see if it can recognize somebody else’s hand, - especially somebody with different skin color. -

- - -
- ); - } -}; - -module.exports.propTypes = { - onClickContinue: PropTypes.func -}; diff --git a/src/activities/rps/PlayRoundResult.jsx b/src/activities/rps/PlayRoundResult.jsx deleted file mode 100644 index 857b4d36..00000000 --- a/src/activities/rps/PlayRoundResult.jsx +++ /dev/null @@ -1,89 +0,0 @@ -import React from 'react'; -import Col from 'react-bootstrap/lib/Col'; -import Button from 'react-bootstrap/lib/Button'; -import Row from 'react-bootstrap/lib/Row'; -import * as PropTypes from 'react/lib/ReactPropTypes'; - -module.exports = class PlayRound extends React.Component { - constructor(props) { - super(props); - } - - componentDidMount() {} - - render() { - return ( -
- - -

- {this.props.winner > 0 - ? 'YOU WIN! 🙌' - : this.props.winner === 0 - ? 'DRAW 🤷‍♀️' - : 'YOU LOSE 😭'} -

- - - - -

- You: -

- -

- {this.props.playerPlayed} ( - {Math.floor(this.props.confidence * 100)}% sure) -

- - -

- Computer: -

-

- {this.props.computerPlayedEmoji} -

-

{this.props.computerPlayed}

- - - - - - - - - - - - - - - -
- ); - } -}; - -module.exports.propTypes = { - onPlayAgain: PropTypes.func, - onTrainMore: PropTypes.func, - onContinue: PropTypes.func, - roundResult: PropTypes.object, - - confidence: PropTypes.number, - - playerPlayed: PropTypes.string, - playerPlayedImage: PropTypes.string, - computerPlayed: PropTypes.string, - computerPlayedEmoji: PropTypes.string, - - winner: PropTypes.number -}; diff --git a/src/activities/rps/RPS.jsx b/src/activities/rps/RPS.jsx deleted file mode 100644 index 32707777..00000000 --- a/src/activities/rps/RPS.jsx +++ /dev/null @@ -1,246 +0,0 @@ -/** - * RPS Activity. Based off of: - * - https://github.com/ryansloan/rps-ml - * - https://github.com/googlecreativelab/teachable-machine-boilerplate - */ - -import React from 'react'; - -import SimpleTrainer from '../../utils/SimpleTrainer'; -import Video from '../../utils/Video.js'; - -import IntroScreen from './IntroScreen'; -import TrainingScreen from './TrainingScreen'; -import PlayRound from './PlayRound'; -import PlayRoundInstructions from './PlayRoundInstructions'; -import PlayRoundResult from './PlayRoundResult'; - -const IMAGE_SIZE = 227; -const CLASS_NAMES = ['rock', 'paper', 'scissors']; - -const NO_CLASS = -1; - -const ActivityScreen = Object.freeze({ - IntroInstructions: 0, - TrainClass1: 1, - TrainClass2: 2, - TrainClass3: 3, - PlayRoundInstructions: 4, - PlayRound: 5, - PlayRoundResult: 6 -}); - -const defaultState = { - currentScreen: ActivityScreen.IntroInstructions, - predictedClass: NO_CLASS, - confidencesByClassId: [], - trainingImages0: [], - trainingImages1: [], - trainingImages2: [], - roundResult: null, - roundPrediction: null -}; -module.exports = class Main extends React.Component { - constructor(props) { - super(props); - this.video = new Video(IMAGE_SIZE); - this.simpleTrainer = new SimpleTrainer(); - } - - state = defaultState; - - componentDidMount() { - this.simpleTrainer.initializeClassifiers(); - } - - rpsToEmoji(rps) { - switch (rps) { - case 'rock': - return '✊'; - case 'scissors': - return '✌'; - default: - return '✋'; - } - } - - render() { - return ( -
- {this.state.currentScreen === ActivityScreen.IntroInstructions && ( - { - this.setState( - { - currentScreen: ActivityScreen.TrainClass1 - }, - null - ); - }} - /> - )} - {this.trainingScreen(0)} - {this.trainingScreen(1)} - {this.trainingScreen(2)} - {this.state.currentScreen === ActivityScreen.PlayRoundInstructions && ( - { - this.setState( - { - currentScreen: ActivityScreen.PlayRound - }, - null - ); - }} - /> - )} - {this.state.currentScreen === ActivityScreen.PlayRound && ( - { - this.video.loadVideo(videoElement); - }} - onPlayRound={() => { - this.playRound().then(() => { - return this.setState( - { - currentScreen: ActivityScreen.PlayRoundResult - }, - null - ); - }); - }} - /> - )} - {this.state.currentScreen === ActivityScreen.PlayRoundResult && ( - { - this.setState( - { - currentScreen: ActivityScreen.PlayRound - }, - null - ); - }} - onTrainMore={() => { - this.setState( - { - currentScreen: ActivityScreen.TrainClass1 - }, - null - ); - }} - onContinue={() => { - this.setState(defaultState, null); - }} - /> - )} -
- ); - } - - trainingScreen(index) { - const thisScreen = ActivityScreen[`TrainClass${index + 1}`]; - const nextScreen = - index + 1 >= CLASS_NAMES.length - ? ActivityScreen.PlayRoundInstructions - : ActivityScreen[`TrainClass${index + 2}`]; - return ( - this.state.currentScreen === thisScreen && ( - { - this.trainExample(index); - }} - onContinueClicked={() => { - this.setState( - { - currentScreen: nextScreen - }, - null - ); - }} - onMountVideo={videoElement => { - this.video.loadVideo(videoElement); - }} - imageSize={IMAGE_SIZE} - trainingClass={CLASS_NAMES[index]} - exampleCount={this.simpleTrainer.getExampleCount(index)} - trainingImages={this.state[`trainingImages${index}`]} - /> - ) - ); - } - - /** - * @param {number} index - * @returns {Promise} - */ - async trainExample(index) { - if (this.video.isPlaying()) { - this.simpleTrainer.addExampleImage(this.video.getVideoElement(), index); - this.setState({ - ['trainingImages' + index]: this.state['trainingImages' + index].concat( - this.video.getFrameDataURI() - ) - }); - } - } - - async playRound() { - if (this.video.isPlaying()) { - if (this.simpleTrainer.getNumClasses() > 0) { - let frameDataURI = this.video.getFrameDataURI(400); - - let predictionResult = await this.simpleTrainer.predictFromImage( - this.video.getVideoElement() - ); - let computerChoice = CLASS_NAMES[Math.floor(Math.random() * 3)]; - let playerChoice = CLASS_NAMES[predictionResult.predictedClassId]; - const winner = this.pickWinner(playerChoice, computerChoice); - - this.setState( - { - roundPrediction: { - predictedClass: predictionResult.predictedClassId, - confidence: - predictionResult.confidencesByClassId[ - predictionResult.predictedClassId - ], - playerPlayedImage: frameDataURI, - - playerPlayed: playerChoice, - computerPlayed: computerChoice, - computerPlayedEmoji: this.rpsToEmoji(computerChoice), - - confidencesByClassId: predictionResult.confidencesByClassId - }, - roundResult: { - winner: winner - } - }, - null - ); - } - } - } - - pickWinner(playerChoice, computerChoice) { - return this.beats(playerChoice, computerChoice); - } - - beats(x, y) { - const keyBeatsValue = { - rock: 'scissors', - paper: 'rock', - scissors: 'paper' - }; - - return keyBeatsValue[x] === y ? 1 : keyBeatsValue[y] === x ? -1 : 0; - } -}; diff --git a/src/activities/rps/TrainingScreen.jsx b/src/activities/rps/TrainingScreen.jsx deleted file mode 100644 index cf0cf9c7..00000000 --- a/src/activities/rps/TrainingScreen.jsx +++ /dev/null @@ -1,105 +0,0 @@ -import React from 'react'; -import Col from 'react-bootstrap/lib/Col'; -import Button from 'react-bootstrap/lib/Button'; -import Row from 'react-bootstrap/lib/Row'; -import * as PropTypes from 'react/lib/ReactPropTypes'; - -module.exports = class TrainingScreen extends React.Component { - constructor(props) { - super(props); - } - - componentWillUnmount() { - this.hasMounted = false; - } - - render() { - return ( -
- - -

- - “Train” the computer to see{' '} - {this.props.trainingClass.endsWith('s') ? '' : 'a'}{' '} - {this.props.trainingClass} - -

-

- Show a rock, and click Train to take photos, so the machine - learning algorithm can “learn” how to recognize a rock. -

- - - - - - - - - - - - - {this.props.trainingImages.map((image, i) => { - return ( - - ); - })} - - - {this.props.trainingImages && this.props.trainingImages.length > 0 && ( - - - - - - )} -
- ); - } -}; - -module.exports.propTypes = { - onMountVideo: PropTypes.func, - onTrainClicked: PropTypes.func, - onContinueClicked: PropTypes.func, - imageSize: PropTypes.number, - trainingClass: PropTypes.string, - - exampleCount: PropTypes.number, - trainingImages: PropTypes.arrayOf(PropTypes.string) -}; From 6b1c7c640b9b31e6b2318f9543fb39eb40e916e6 Mon Sep 17 00:00:00 2001 From: Brendan Reville Date: Sat, 26 Oct 2019 13:43:40 -0700 Subject: [PATCH 07/15] More UI tweaks --- src/demo/helpers.js | 3 +++ src/demo/renderer.js | 2 +- src/demo/ui.jsx | 46 +++++++++++++++++++++++--------------------- 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/src/demo/helpers.js b/src/demo/helpers.js index bd6a77cd..c9b406b4 100644 --- a/src/demo/helpers.js +++ b/src/demo/helpers.js @@ -9,6 +9,9 @@ export const backgroundPathForMode = mode => { imgName = 'underwater'; } + // Temporarily show background for every mode. + imgName = 'underwater'; + return imgName ? backgroundPath(imgName) : null; }; diff --git a/src/demo/renderer.js b/src/demo/renderer.js index c157e5c7..456cf44a 100644 --- a/src/demo/renderer.js +++ b/src/demo/renderer.js @@ -289,7 +289,7 @@ const drawFrame = state => { size, size, '#F0F0F0', - '#000000' + '#F0F0F0' ); }; diff --git a/src/demo/ui.jsx b/src/demo/ui.jsx index 178814f8..513dd941 100644 --- a/src/demo/ui.jsx +++ b/src/demo/ui.jsx @@ -8,19 +8,17 @@ import {onClassifyFish} from './models/train'; const styles = { Header: { position: 'absolute', - top: 0, + top: 10, width: '100%', - //height: 50, display: 'flex', alignItems: 'center', justifyContent: 'center', - fontSize: 32 + fontSize: 48 }, Footer: { position: 'absolute', bottom: 0, width: '100%', - height: 50, display: 'flex', justifyContent: 'space-between' }, @@ -38,18 +36,13 @@ const styles = { button: { cursor: 'pointer' }, - heading: { - border: '2px solid black', - borderRadius: 20, - padding: '10px 45px', - fontSize: 24 - }, activityIntroText: { position: 'absolute', - fontSize: 24, + fontSize: 22, top: '20%', - width: '98%', - left: '1%', + left: '50%', + width: '80%', + transform: 'translateX(-50%)', textAlign: 'center' }, trainingIntroBot: { @@ -67,12 +60,14 @@ const styles = { left: '50%' }, continueButton: { - marginLeft: 'auto' + marginLeft: 'auto', + marginRight: 10, + marginBottom: 10 }, wordsText: { textAlign: 'center', marginTop: 20, - fontSize: 24 + fontSize: 22 }, button1col: { width: '20%', @@ -83,9 +78,10 @@ const styles = { }, trainQuestionText: { position: 'absolute', - top: '20%', + top: '18%', left: '50%', - transform: 'translateX(-50%)' + transform: 'translateX(-50%)', + fontSize: 22 }, trainButtonYes: { position: 'absolute', @@ -99,9 +95,15 @@ const styles = { }, pondText: { position: 'absolute', - top: '90%', - left: '50%', - transform: 'translateX(-50%)' + bottom: '10%', + left: '60%', + transform: 'translateX(-50%)', + fontSize: 22, + width: '60%', + backgroundColor: 'rgba(0,0,0,0.7)', + padding: '3%', + borderRadius: 10, + color: 'white' }, trainBot: { position: 'absolute', @@ -119,8 +121,8 @@ const styles = { pondBot: { position: 'absolute', height: '50%', - left: '10%', - bottom: '5%' + left: 0, + bottom: 0 } }; From ed4342a025dcf56e2806e52d7f68bfc9e680a3ab Mon Sep 17 00:00:00 2001 From: Brendan Reville Date: Sat, 26 Oct 2019 13:51:22 -0700 Subject: [PATCH 08/15] Update React each time state changes. This adds the ability to set a callback handler with the state manager which will be called when state is set. --- src/demo/index.jsx | 18 ++++++++++++------ src/demo/state.js | 11 +++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/demo/index.jsx b/src/demo/index.jsx index 835d2185..1aaf7133 100644 --- a/src/demo/index.jsx +++ b/src/demo/index.jsx @@ -1,6 +1,6 @@ import $ from 'jquery'; import constants, {Modes} from './constants'; -import {setState} from './state'; +import {setState, setSetStateCallback} from './state'; import {init as initScene} from './init'; import {render} from './renderer'; @@ -33,9 +33,15 @@ $(document).ready(() => { // requestAnimationFrame on itself. render(); - // Start the React renderer. - const renderElement = document.getElementById('container-react'); - setInterval(() => { - ReactDOM.render(, renderElement); - }, 1000); + // Render the UI. + renderUI(); + + // And have the render UI handler be called every time state is set. + setSetStateCallback(renderUI); }); + +// Tell React to explicitly render the UI. +export const renderUI = () => { + const renderElement = document.getElementById('container-react'); + ReactDOM.render(, renderElement); +}; diff --git a/src/demo/state.js b/src/demo/state.js index 708edf9f..9c862d3d 100644 --- a/src/demo/state.js +++ b/src/demo/state.js @@ -1,3 +1,5 @@ +let setStateCallback = null; + const initialState = { currentMode: null, fishData: [], @@ -26,5 +28,14 @@ export const getState = function() { export const setState = function(newState) { state = {...state, ...newState}; + + if (setStateCallback) { + setStateCallback(); + } + return state; }; + +export const setSetStateCallback = (callback) => { + setStateCallback = callback; +} \ No newline at end of file From 87c44312ce7f6e5903dc2a3d1f62370dfde02905 Mon Sep 17 00:00:00 2001 From: Brendan Reville Date: Sat, 26 Oct 2019 14:47:57 -0700 Subject: [PATCH 09/15] URL parameter for which word set & sway fish on predict screen Use ?words=small or ?words=large to specify which set of words to show. --- public/index.html | 14 -------- src/demo/index.jsx | 6 +++- src/demo/renderer.js | 17 ++++++---- src/demo/state.js | 2 +- src/demo/ui.jsx | 81 ++++++++++++++++++++++++-------------------- 5 files changed, 61 insertions(+), 59 deletions(-) diff --git a/public/index.html b/public/index.html index cb121c54..ca2ed964 100644 --- a/public/index.html +++ b/public/index.html @@ -45,20 +45,6 @@ button:focus { outline: white auto 5px; } - .ui-button-1-col { - width: 20%; - display: block; - margin: 0 auto; - margin-top: 2%; - margin-bottom: 2%; - } - .ui-button-3-col { - width: 20%; - margin-left: 6%; - margin-right: 6%; - margin-top: 2%; - margin-bottom: 2%; - } @media screen and (max-width: 800px) { button { font-size: 90%; diff --git a/src/demo/index.jsx b/src/demo/index.jsx index 1aaf7133..3901928e 100644 --- a/src/demo/index.jsx +++ b/src/demo/index.jsx @@ -16,6 +16,9 @@ $(document).ready(() => { canvas.width = backgroundCanvas.width = constants.canvasWidth; canvas.height = backgroundCanvas.height = constants.canvasHeight; + // Temporarily use URL parameter to set some state. + const smallWordSet = window.location.href.indexOf("words=small") !== -1; + // Set initial state for UI elements. setState({ currentMode: Modes.Loading, @@ -23,7 +26,8 @@ $(document).ready(() => { backgroundCanvas, uiContainer: document.getElementById('ui-container'), headerContainer: document.getElementById('header-container'), - footerContainer: document.getElementById('footer-container') + footerContainer: document.getElementById('footer-container'), + smallWordSet: smallWordSet }); // Initialize our first model. diff --git a/src/demo/renderer.js b/src/demo/renderer.js index 456cf44a..7f687ee8 100644 --- a/src/demo/renderer.js +++ b/src/demo/renderer.js @@ -195,7 +195,7 @@ const getYForFish = (numFish, fishIdx, state, offsetX, predictedClassId) => { // Move fish down a little on predict screen. if (state.currentMode === Modes.Predicting) { - y += 100; + y += 130; // And drop the fish down even more if they are not liked. const doesLike = predictedClassId === ClassType.Like; @@ -207,6 +207,11 @@ const getYForFish = (numFish, fishIdx, state, offsetX, predictedClassId) => { y += screenX - midScreenX; } } + + // And sway fish vertically on the predicting screen. + const swayValue = (($time() * 360) / (20 * 1000) + (fishIdx + 1) * 10) % 360; + const swayOffsetY = Math.sin(((swayValue * Math.PI) / 180) * 6) * 8; + y += swayOffsetY; } return y; @@ -339,9 +344,9 @@ const drawPondFishImages = () => { const canvas = getState().canvas; const ctx = canvas.getContext('2d'); getState().pondFish.forEach(fish => { - var swayValue = (($time() * 360) / (20 * 1000) + (fish.id + 1) * 10) % 360; - var swayOffsetX = Math.sin(((swayValue * Math.PI) / 180) * 2) * 120; - var swayOffsetY = Math.sin(((swayValue * Math.PI) / 180) * 6) * 8; + const swayValue = (($time() * 360) / (20 * 1000) + (fish.id + 1) * 10) % 360; + const swayOffsetX = Math.sin(((swayValue * Math.PI) / 180) * 2) * 120; + const swayOffsetY = Math.sin(((swayValue * Math.PI) / 180) * 6) * 8; drawSingleFish(fish, fish.x + swayOffsetX, fish.y + swayOffsetY, ctx); }); @@ -445,8 +450,8 @@ export const clearCanvas = canvas => { // Draw an overlay over the whole scene. Used for fades. function drawOverlays() { - var duration = $time() - currentModeStartTime; - var amount = 1 - duration / 800; + const duration = $time() - currentModeStartTime; + let amount = 1 - duration / 800; if (amount < 0) { amount = 0; } diff --git a/src/demo/state.js b/src/demo/state.js index 9c862d3d..20617f00 100644 --- a/src/demo/state.js +++ b/src/demo/state.js @@ -38,4 +38,4 @@ export const setState = function(newState) { export const setSetStateCallback = (callback) => { setStateCallback = callback; -} \ No newline at end of file +} diff --git a/src/demo/ui.jsx b/src/demo/ui.jsx index 513dd941..06991693 100644 --- a/src/demo/ui.jsx +++ b/src/demo/ui.jsx @@ -6,7 +6,7 @@ import {init as initScene} from './init'; import {onClassifyFish} from './models/train'; const styles = { - Header: { + header: { position: 'absolute', top: 10, width: '100%', @@ -15,19 +15,19 @@ const styles = { justifyContent: 'center', fontSize: 48 }, - Footer: { + footer: { position: 'absolute', bottom: 0, width: '100%', display: 'flex', justifyContent: 'space-between' }, - Body: { + body: { position: 'relative', width: '100%', paddingTop: '56.25%' // for 16:9 }, - Content: { + content: { position: 'absolute', top: '10%', left: 0, @@ -36,6 +36,25 @@ const styles = { button: { cursor: 'pointer' }, + continueButton: { + marginLeft: 'auto', + marginRight: 10, + marginBottom: 10 + }, + button1col: { + width: '20%', + display: 'block', + margin: '0 auto', + marginTop: '2%', + marginBottom: '2%' + }, + button3col: { + width: '20%', + marginLeft: '6%', + marginRight: '6%', + marginTop: '2%', + marginBottom: '2%' + }, activityIntroText: { position: 'absolute', fontSize: 22, @@ -47,35 +66,21 @@ const styles = { }, trainingIntroBot: { position: 'absolute', - //height: '50%', transform: 'translateX(-50%)', top: '30%', left: '50%' }, activityIntroBot: { position: 'absolute', - //height: '50%', transform: 'translateX(-50%)', top: '50%', left: '50%' }, - continueButton: { - marginLeft: 'auto', - marginRight: 10, - marginBottom: 10 - }, wordsText: { textAlign: 'center', marginTop: 20, fontSize: 22 }, - button1col: { - width: '20%', - display: 'block', - margin: '0 auto', - marginTop: '2%', - marginBottom: '2%' - }, trainQuestionText: { position: 'absolute', top: '18%', @@ -93,18 +98,6 @@ const styles = { top: '80%', left: '60%' }, - pondText: { - position: 'absolute', - bottom: '10%', - left: '60%', - transform: 'translateX(-50%)', - fontSize: 22, - width: '60%', - backgroundColor: 'rgba(0,0,0,0.7)', - padding: '3%', - borderRadius: 10, - color: 'white' - }, trainBot: { position: 'absolute', height: '50%', @@ -118,6 +111,18 @@ const styles = { left: '50%', transform: 'translateX(-50%)' }, + pondText: { + position: 'absolute', + bottom: '10%', + left: '60%', + transform: 'translateX(-50%)', + fontSize: 22, + width: '60%', + backgroundColor: 'rgba(0,0,0,0.7)', + padding: '3%', + borderRadius: 10, + color: 'white' + }, pondBot: { position: 'absolute', height: '50%', @@ -128,25 +133,25 @@ const styles = { class Body extends React.Component { render() { - return
{this.props.children}
; + return
{this.props.children}
; } } class Header extends React.Component { render() { - return
{this.props.children}
; + return
{this.props.children}
; } } class Content extends React.Component { render() { - return
{this.props.children}
; + return
{this.props.children}
; } } class Footer extends React.Component { render() { - return
{this.props.children}
; + return
{this.props.children}
; } } @@ -217,8 +222,7 @@ class Words extends React.Component { currentItems() { const state = getState(); - const iterationCount = state.iterationCount; - const itemSet = iterationCount === 0 ? 0 : 1; + const itemSet = state.smallWordSet ? 0 : 1; return this.items[itemSet]; } @@ -232,8 +236,11 @@ class Words extends React.Component { } render() { + const state = getState(); const currentItems = this.currentItems(); - const buttonStyle = styles.button1col; + const buttonStyle = state.smallWordSet + ? styles.button1col + : styles.button3col; return ( From c209485d93f93579eb3437c60cea16b5662adf91 Mon Sep 17 00:00:00 2001 From: Brendan Reville Date: Sat, 26 Oct 2019 14:53:08 -0700 Subject: [PATCH 10/15] Show prediction label at midpoint of screen --- src/demo/renderer.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/demo/renderer.js b/src/demo/renderer.js index 7f687ee8..1bef3ffc 100644 --- a/src/demo/renderer.js +++ b/src/demo/renderer.js @@ -166,7 +166,7 @@ const getOffsetForTime = (t, totalFish) => { let amount = t / moveTime; // Apply an S-curve to that amount. - amount = amount - Math.sin(amount*2*Math.PI) / (2*Math.PI); + amount = amount - Math.sin(amount * 2 * Math.PI) / (2 * Math.PI); return ( constants.fishCanvasWidth * totalFish - @@ -209,7 +209,8 @@ const getYForFish = (numFish, fishIdx, state, offsetX, predictedClassId) => { } // And sway fish vertically on the predicting screen. - const swayValue = (($time() * 360) / (20 * 1000) + (fishIdx + 1) * 10) % 360; + const swayValue = + (($time() * 360) / (20 * 1000) + (fishIdx + 1) * 10) % 360; const swayOffsetY = Math.sin(((swayValue * Math.PI) / 180) * 6) * 8; y += swayOffsetY; } @@ -253,7 +254,11 @@ const drawMovingFish = state => { if (state.currentMode === Modes.Predicting) { if (fish.result) { - drawPrediction(fish.result.predictedClassId, state.word, x, y, ctx); + const midScreenX = + constants.canvasWidth / 2 - constants.fishCanvasWidth / 2; + if (x > midScreenX) { + drawPrediction(fish.result.predictedClassId, state.word, x, y, ctx); + } } else { predictFish(state, i).then(prediction => { fish.result = prediction; @@ -344,7 +349,8 @@ const drawPondFishImages = () => { const canvas = getState().canvas; const ctx = canvas.getContext('2d'); getState().pondFish.forEach(fish => { - const swayValue = (($time() * 360) / (20 * 1000) + (fish.id + 1) * 10) % 360; + const swayValue = + (($time() * 360) / (20 * 1000) + (fish.id + 1) * 10) % 360; const swayOffsetX = Math.sin(((swayValue * Math.PI) / 180) * 2) * 120; const swayOffsetY = Math.sin(((swayValue * Math.PI) / 180) * 6) * 8; From 8ff2f0d943e9ba9030811c93fafa6181b6231720 Mon Sep 17 00:00:00 2001 From: Brendan Reville Date: Sat, 26 Oct 2019 15:00:29 -0700 Subject: [PATCH 11/15] Fix JS --- src/demo/ui.jsx | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/demo/ui.jsx b/src/demo/ui.jsx index 06991693..0c4d80d1 100644 --- a/src/demo/ui.jsx +++ b/src/demo/ui.jsx @@ -1,4 +1,5 @@ import React from 'react'; +import PropTypes from 'prop-types'; import {getState, setState} from './state'; import {Modes} from './constants'; import {toMode} from './helpers'; @@ -132,33 +133,56 @@ const styles = { }; class Body extends React.Component { + static propTypes = { + children: PropTypes.node + }; + render() { return
{this.props.children}
; } } class Header extends React.Component { + static propTypes = { + children: PropTypes.node + }; + render() { return
{this.props.children}
; } } class Content extends React.Component { + static propTypes = { + children: PropTypes.node + }; + render() { return
{this.props.children}
; } } class Footer extends React.Component { + static propTypes = { + children: PropTypes.node + }; + render() { return
{this.props.children}
; } } class Button extends React.Component { + static propTypes = { + style: PropTypes.object, + children: PropTypes.node, + onClick: PropTypes.func + }; + render() { return (