Add firebase functions Vision API demo

.gitignore

@@ -1,2 +1,3 @@
 node_modules/
-*.log
+*.log
+*.firebaserc

README.md

@@ -2,7 +2,25 @@
 
 # ML API next talk demos
 
-This repo includes 3 demos from my [Google Next talk](https://youtu.be/w1xNTLH1zlA) on the ML APIs. To run the demos, follow the instructions below.
+This repo includes 4 demos from my [Google Next talk](https://youtu.be/w1xNTLH1zlA) and [Google I/O talk](https://www.youtube.com/watch?v=ETeeSYMGZn0) on the Cloud ML APIs. To run the demos, follow the instructions below.
+
+## Vision API
+
+1. `cd` into `vision-api-firebase`
+2. Create a project in the [Firebase console](http://firebase.google.com/console) and install the [Firebase CLI](https://firebase.google.com/docs/cli/)
+3. Run `firebase login` via the CLI and then `firebase init functions` to initialize the Firebase SDK for Cloud Functions. When prompted, don't overwrite `functions/package.json` or `functions/index.js`.
+4. In your Cloud console for the same project, enable the Vision API
+5. Generate a service account for your project by navigating to the "Project settings" tab in your Firebase console and then selecting "Service Accouts". Click "Generate New Private Key" and save the file to your `functions/` directory in a file called `keyfile.json`:
+
+![Project settings](project-settings.png)
+
+![Service accounts](service-accounts.png)
+
+6. In `functions/index.js` replace both instances of `your-firebase-project-id` with the ID of your Firebase project
+7. Deploy your Cloud Function by running `firebase deploy --only functions`
+8. From the Authentication tab in your Firebase console, enable *Twitter authentication* (you can use whichever auth provider you'd like, I chose Twitter).
+9. Run the frontend locally by running `firebase serve` from the `vision-api-firebase/` directory of this project. Navigate to `localhost:5000` to try uploading a photo. After uploading a photo check your Functions logs and then your Firebase Database to confirm the function executed correctly.
+10. Deploy the frontend by running `firebase deploy --only hosting`. For future deploys you can run `firebase deploy` to deploy Functions and Hosting simultaneously.
 
 ## Speech API
 

vision-api-firebase/cors.json

@@ -0,0 +1,7 @@
+[
+  {
+    "origin": ["*"],
+    "method": ["GET"],
+    "maxAgeSeconds": 3600
+  }
+]

vision-api-firebase/database.rules.json

@@ -0,0 +1,9 @@
+{
+  "rules": {
+    ".read": true,
+    ".write": true,
+    "entities": {
+    	".indexOn": ".value"
+    }
+  }
+}

vision-api-firebase/firebase.json

@@ -0,0 +1,14 @@
+{
+  "database": {
+    "rules": "database.rules.json"
+  },
+  "hosting": {
+    "public": ".",
+    "rewrites": [
+      {
+        "source": "**",
+        "destination": "/index.html"
+      }
+    ]
+  }
+}

vision-api-firebase/functions/index.js

@@ -0,0 +1,127 @@
+'use strict';
+
+const functions = require('firebase-functions');
+const config = require('./local.json');
+
+const fbConfig = {
+  projectId: "your-firebase-project-id",
+  keyfileName: 'keyfile.json'
+};
+
+const vision = require('@google-cloud/vision')(fbConfig);
+
+var admin = require("firebase-admin");
+var serviceAccount = require("./keyfile.json");
+admin.initializeApp({
+  credential: admin.credential.cert(serviceAccount),
+  databaseURL: "https://your-firebase-project-id.firebaseio.com"
+});
+
+const db = admin.database();
+const imageRef = db.ref('images');
+const entitiesRef = db.ref('entities');
+const labelsRef = db.ref('labels');
+const faceRef = db.ref('faces');
+const emojiRef = db.ref('emojis');
+const latestImgDataRef = db.ref('latestImgData');
+const emotions = ['anger','joy','sorrow','surprise'];
+let userRef;
+
+function detectFacesAndLabels(faces, entities) {
+    if (faces) {
+      for (let i in faces) {
+        let face = faces[i];
+        for (let j in emotions) {
+          let emotion = emotions[j];
+          if ((face[emotion + 'Likelihood'] === "VERY_LIKELY") || (face[emotion + 'Likelihood'] === "LIKELY") || (face[emotion + 'Likelihood'] === "POSSIBLE")) {
+            faceRef.child(emotion).transaction(function(data) {
+              if (data != null) {
+                data++;
+              } else {
+                data = 1;
+              }
+              return data;
+            });
+          }
+        }
+      }
+    }
+
+    if (entities) {
+      for (let i in entities) {
+        let entity = entities[i].description.toLowerCase();
+        entity.replace(/\.|#|\$|\[|\]|\//g,''); // Remove ".", "#", "$", "[", or "]" (illegal Firebase path name)
+        entitiesRef.child(entity).transaction(function(data) {
+          if (data != null) {
+            data++
+          } else {
+            data= 1;
+          }
+          return data;
+        });
+      }
+    }
+}
+
+exports.callVision = functions.storage.object().onChange(event => {
+  const obj = event.data;
+
+  const gcsUrl = "gs://" + obj.bucket + "/" + obj.name;
+  const userId = obj.name.substring(0, obj.name.indexOf('/'));
+  userRef = db.ref('users').child(userId);
+
+  return Promise.resolve()
+    .then(() => {
+      if (obj.resourceState === 'not_exists') {
+        // This was a deletion event, we don't want to process this
+        return;
+      }
+      if (!obj.bucket) {
+        throw new Error('Bucket not provided. Make sure you have a "bucket" property in your request');
+      }
+      if (!obj.name) {
+        throw new Error('Filename not provided. Make sure you have a "name" property in your request');
+      }
+
+      let visionReq = {
+        "image": {
+          "source": {
+            "imageUri": gcsUrl
+          }
+        },
+        "features": [
+          {
+            "type": "FACE_DETECTION"
+          },
+          {
+            "type": "LABEL_DETECTION"
+          },
+          {
+            "type": "LANDMARK_DETECTION"
+          },
+          {
+            "type": "WEB_DETECTION"
+          },
+          {
+            "type": "IMAGE_PROPERTIES"
+          },
+          {
+            "type": "SAFE_SEARCH_DETECTION"
+          }
+        ]
+      }
+
+      return vision.annotate(visionReq);
+    })
+    .then(([visionData]) => {
+      let imgMetadata = visionData[0];
+      console.log('got vision data: ',imgMetadata);
+      imageRef.push(imgMetadata);
+      userRef.child('visionData').set(imgMetadata);
+      latestImgDataRef.set(imgMetadata);
+      return detectFacesAndLabels(imgMetadata['faceAnnotations'], imgMetadata['webDetection']['webEntities']);
+    })
+    .then(() => {
+      console.log(`Parsed vision annotation and wrote to Firebase`);
+    });
+});

vision-api-firebase/functions/keyfile.json

@@ -0,0 +1,12 @@
+{
+  "type": "service_account",
+  "project_id": "realtime-selfies",
+  "private_key_id": "52a62e4d7b852c32685e45d4febd64ec5b5eedc7",
+  "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCbyRDGaISYkPCv\nfygVv7swgC3CQoag1p6N5emha1H7CoCsd2IN07hu2M6qFW8RZjhgFlViS3Uv+VwG\nKqqE1m6J1NnwcnpCPTS1x2efrfysigIQZQ0P3p4LyhR8twkaiNIm5suwqKlDXaWz\nDrl1MZAwzBDlgkNKCMgFOVaMSN7HP3yyQnIKYEJWr33W2NTD53aJ8jz/Ie+yeUzt\n1H3cLi1OatSnOvDXEFNM19Kbvj4Tr4S8xGxeUmrCkHSxvnslbDuaw2HFvB5+yTDb\nIeqlM0JOcF+JVRhb4jYr1C1hrP5xklDzKzpFzfUECisfWmkBxxieOKCA2S2SknDJ\nXfKJA6/HAgMBAAECggEAUhnuQFKXBvzkC+mcy5GHarEy7H05DIzvdOMldM/lncNU\nOxCT2auqOKkEibjQF0BjF6jNiQcxlH37lLhps6Z7DSLjVQhQDJbLL74Oi6YbUydM\nnS1Ua/esHZR7pZqlLpnK/3uE/+5zfjBdgneRIAVl84Goqfwz0FG+cJpeVYKV/jVR\nayMNgpQM8+CAykkerH2hvL7fJUIWxPhsSL0U858HmQKBlICe0i+fJm0qnBvmT9m0\neBlu6XElH9+7cqRff/O8tvO9osXNJPtTIiImq6DYGt6Hbp83eXsDQ65b7e0KWAoO\nJphRyEP/+CjUqirf4YHg6iNOA+rqOsfSDj0+XMau2QKBgQDU10/TnldlV3dPEN3u\nNBfxBUm1Xrodcua689aYqDoTAw1nzYLBLVRrQD8zpwhYsRYKCS5VoKucfGGEDGMU\nKvn9O5QhppKzh2p2/dn1rs7g33iBEHULcTH7G+fAMdXnfiAzbZEH4VvRqItAoiu0\nFPs5Ma+qXw9t9vutqoB1Pkew9QKBgQC7X/c/GaPA4lT1TY82mTToSz+hO9iup+iZ\nL+c/L40NpFgbQpTq/FYjbt5MzsH9OUVFLJQ8W3F+voRM9WCat+wv1uBLYGm7SnBy\ncnz35E2gxTdIeeh2Vq8KDpHoYM2yXuh5h023y4Zg2G6zNByg9FfuuNbJDvonkco9\nN273SNB4SwKBgGEMqGaK7bjU8B8KRtfFwyDwU1KzFdQ1v0WBx9kl5A0lOCibycJB\n5BtfWTI5OJIQdUcwNoNu1rFs+Z4Xc9oPWpwAXaQWaxAXcBE/4PGousZIv47CLUyB\nWdPxPnQhhTKgDRjGHfpk5NQtsQlQqPLdGkxS/pGF9OgkVVAzzY0oT9I9AoGBAKp1\n6nNwOuYNwWaMUZ5FELnHQzLGRCDYRiWeS8zS0Iq5mSHrl5iTSTXg9cGAU0CwKlF8\n9bpXIlBAuBFfJax7aBY5cEGCi43EcbncZ84I14pMADgiF5YY1BLdIGX8MwVzjCCL\nM+vh7vUJ0OOco/LVd22IoHW292KqIgdA9+VlS6ozAoGAImr6XNstunRUARI19Q+w\nUR0Qpwiz/SCW1P1ZhRx+ppUkc1YWNpaQsIAticDYgX9nQCMW3TgsHKHNJB8bJXMt\npfNxDl40xPR1Xb78MOWHLo+WC589TEG+x/s7ELkvt6a90IvRcC1rkU1MrqLE2xYo\n6QPFCELi90B6fDwRwh3//pc=\n-----END PRIVATE KEY-----\n",
+  "client_email": "realtime-selfies@appspot.gserviceaccount.com",
+  "client_id": "111247916172156281662",
+  "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+  "token_uri": "https://accounts.google.com/o/oauth2/token",
+  "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+  "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/realtime-selfies%40appspot.gserviceaccount.com"
+}

vision-api-firebase/functions/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "functions",
+  "description": "Cloud Functions for Firebase",
+  "dependencies": {
+    "@google-cloud/vision": "^0.11.2",
+    "firebase-admin": "^4.1.2",
+    "firebase-functions": "^0.5",
+    "google-cloud": "^0.53.0"
+  },
+  "private": true
+}

vision-api-firebase/index.html

@@ -0,0 +1,84 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>Realtime Photo Fun</title>
+    <link rel="stylesheet" href="main.css">
+    <!-- update the version number as needed -->
+
+
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.5.0/Chart.bundle.js"></script>
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.0/jquery.min.js"></script>
+    <!-- <script src="https://www.gstatic.com/firebasejs/3.9.0/firebase.js"></script> -->
+    <link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
+<link rel="stylesheet" href="https://code.getmdl.io/1.3.0/material.indigo-pink.min.css">
+<script defer src="https://code.getmdl.io/1.3.0/material.min.js"></script>
+    <script src="/__/firebase/3.9.0/firebase-app.js"></script>
+    <!-- include only the Firebase features as you need -->
+    <script src="/__/firebase/3.9.0/firebase-auth.js"></script>
+    <script src="/__/firebase/3.9.0/firebase-database.js"></script>
+    <script src="/__/firebase/3.9.0/firebase-messaging.js"></script>
+    <script src="/__/firebase/3.9.0/firebase-storage.js"></script>
+    <!-- initialize the SDK after all desired features are loaded -->
+    <script src="/__/firebase/init.js"></script>
+    <script src="https://wurfl.io/wurfl.js"></script>
+    <script src="main.js"></script>
+
+  </head>
+  <body>
+    <div class="mdl-layout mdl-js-layout mdl-layout--fixed-header">
+      <header class="mdl-layout__header">
+        <div class="mdl-layout__header-row">
+          <img src="icon.png" class="icon"/>
+          <input type="button" id="loadFileXml" value="Add a photo!" onclick="document.getElementById('img-select').click();" class="mdl-button mdl-js-button mdl-button--raised btn"/>
+          <input type="file" style="display:none;" id="img-select" name="file" accept="image/*" capture="camera"/>
+          <div class="mdl-layout-spacer"></div>
+          <div class="data-spin-container">
+            <div class="mdl-spinner mdl-js-spinner img-load-spinner"></div>
+            <div class="permission-denied">Permission Denied</div>
+          </div>
+
+        </div>
+      </header>
+
+      <main class="mdl-layout__content">
+        <div class="page-content">
+        <canvas id="c"></canvas>
+          <div class="mdl-grid desktop latest grid-3-section">
+              <div class="mdl-cell mdl-cell--4-col selfie-grid desktop">
+                <span class="cell-header"><h4>Latest photo</h4></span>
+
+                <div class="selfie-container"><img id="latest-selfie"/></div>
+              </div>
+
+            <div class="mdl-cell mdl-cell--4-col selfie-grid only-mobile">
+              <span class="cell-header"><h4>Latest image data</h4></span>
+              <div id="user-img-data"></div>
+                <div class="data-spin-container">
+                  <div class="mdl-spinner mdl-js-spinner data-load-spinner"></div>
+                </div>
+
+
+            </div>
+            <div class="mdl-cell mdl-cell--4-col selfie-grid">
+            <div id="total-selfies"><h4>Total photos: <span id="num-selfies"></span></h4></div>
+            </div>
+          </div>
+
+
+          <div class="mdl-grid">
+            <div class="mdl-cell mdl-cell--5-col">
+              <div class="chart-container"><canvas id="faceChart" width="500" height="500"></canvas></div>
+            </div>
+            <div class="mdl-cell mdl-cell--1-col"></div>
+            <div class="mdl-cell mdl-cell--6-col">
+              <div class="chart-container"><canvas id="labelsChart" width="500" height="500"></canvas></div>
+            </div>
+          </div>
+          <br/>
+        </div>
+      </main>
+    </div>
+  </body>
+</html>