Add dataset size tutorial. Make modifications for testing directories. (

#34)

LICENSE.md

@@ -362,7 +362,7 @@ file or class name and description of purpose be included on the
 same "printed page" as the copyright notice for easier
 identification within third-party archives.
 
-Copyright 2016, [Google Cloud Platform Community Authors][authors]
+Copyright 2017, [Google Cloud Platform Community Authors][authors]
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

README.md

@@ -19,7 +19,7 @@ Found out how you can contribute at
 
 ## License
 
-Copyright 2016, [Google Cloud Platform Community Authors][authors]
+Copyright 2017, [Google Cloud Platform Community Authors][authors]
 
 _Except as otherwise noted, the content in this repository is licensed under the
 [Creative Commons Attribution 4.0 International (CC BY 4.0) License][cca], and

bin/build

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+
+/**
+ * Copyright 2017, Google, Inc.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const utils = require('../test/utils');
+
+const { TUTORIALS_PATH, EMBEDMD_REGEXP } = utils;
+const files = fs.readdirSync(TUTORIALS_PATH);
+
+files.forEach((entry, i) => {
+  let content, filename;
+  const dir = path.join(TUTORIALS_PATH, '/', entry);
+  fs.stat(dir, (err, stats) => {
+    if (err) {
+      throw err;
+    }
+
+    if (stats.isDirectory()) {
+      filename = path.join(dir, 'index.md');
+      fs.readFile(filename, { encoding: 'utf8' }, (err, content) => {
+        if (err) {
+          throw err;
+          return;
+        }
+
+        if (EMBEDMD_REGEXP.test(content)) {
+          const cmd = `embedmd -w ${filename}`;
+          console.log(cmd);
+          utils.runAsync(cmd, dir)
+            .then(console.log).catch(console.error);
+        }
+      });
+    }
+  });
+});

circle.yml

@@ -1,4 +1,4 @@
-# Copyright 2016, Google, Inc.
+# Copyright 2017, Google, Inc.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at

docs/styleguide.md

@@ -7,6 +7,7 @@ This document provides guidance for contributors to the Google Cloud Platform (G
 * [General content guidelines](#general-content-guidelines)
 * [General style guidelines](#general-style-guidelines)
 * [Voice and tone](#voice-and-tone)
+* [Inlcuding source code](#includeing-source-code)
 * [Writing resources](#writing-resources)
 
 ## Caveats
@@ -314,6 +315,35 @@ If a sentence is long, even with  straightforward word choices, break it up into
 
 Re-read what you wrote and then eliminate all the unnecessary words.
 
+## Inluding source code
+
+If you would like to include source code within your tutorial, you have two
+options:
+
+### Option 1
+
+Just embed the source code directly in the tutorial. Wrap the code in three
+backticks or indent by four spaces to achieve proper formatting.
+
+This option is the simplest, but offer no way to test the code, and does not
+allow the user to view actual source code files as they might exist in a real
+project.
+
+For an example, see [Run Koa.js on Google App Engine Flexible Environment](https://github.com/GoogleCloudPlatform/community/blob/master/tutorials/run-koajs-on-google-app-engine.md).
+
+### Option 2
+
+Instead of a markdown file in the `tutorials/` directory, create a folder for
+your files. The markdown for the tutorial should be in an `index.md` file within
+the new folder, and the rest of the source code files must be in the new folder
+as well. Snippets from the source code files can be included in the markdown
+file using [EmbedMd](https://github.com/campoy/embedmd).
+
+This option is more complicated, but allows us to test the code, and allows the
+user to view real source code files.
+
+For an example, see [Using Node.js to Calculate the Size of a BigQuery Dataset](https://github.com/GoogleCloudPlatform/community/blob/master/tutorials/using-nodejs-to-calculate-the-size-of-a-bigquery-dataset).
+
 ## Writing resources
 
 Learn more about strong writing.

package.json

@@ -2,7 +2,8 @@
   "private": true,
   "scripts": {
     "lint": "semistandard \"**/*.js\"",
-    "test": "npm run lint && mocha --recursive -t 30000"
+    "build": "node ./bin/build",
+    "test": "npm run lint && mocha -t 120000 ./test/*"
   },
   "semistandard": {
     "globals": [
@@ -18,7 +19,7 @@
     ]
   },
   "devDependencies": {
-    "mocha": "^2.5.3",
-    "semistandard": "^8.0.0"
+    "mocha": "3.2.0",
+    "semistandard": "9.2.1"
   }
 }

test/tutorials.test.js

@@ -1,15 +1,17 @@
-// Copyright 2015-2016, Google, Inc.
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/**
+ * Copyright 2017, Google, Inc.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 'use strict';
 
@@ -19,6 +21,7 @@ const path = require('path');
 const utils = require('./utils');
 const {
   FILENAME_REGEXP,
+  DIR_REGEXP,
   TUTORIAL_YAML_REGEXP,
   TITLE_REGEXP,
   DESCRIPTION_REGEXP,
@@ -30,24 +33,61 @@ const {
 
 const files = fs.readdirSync(TUTORIALS_PATH);
 
-describe('tutorials/', function () {
-  files.forEach(function (entry, i) {
-    describe(entry, function () {
-      before(function (done) {
-        fs.readFile(path.join(TUTORIALS_PATH, '/', entry), { encoding: 'utf8' }, (err, content) => {
+describe('tutorials/', () => {
+  files.forEach((entry, i) => {
+    describe(entry, () => {
+      let content, filename;
+      const dir = path.join(TUTORIALS_PATH, '/', entry);
+      const stats = fs.statSync(dir);
+
+      if (stats.isDirectory()) {
+        filename = path.join(dir, 'index.md');
+      } else if (stats.isFile()) {
+        filename = dir;
+      } else {
+        throw new Error(`Unrecognized file: ${entry}`);
+      }
+
+      before((done) => {
+        fs.readFile(filename, { encoding: 'utf8' }, (err, _content) => {
+          if (err) {
+            done(err);
+            return;
+          }
+          content = _content;
+          done();
+        });
+      });
+
+      it('tests pass, if any', (done) => {
+        // TODO: Handle tests for other languages
+        fs.stat(path.join(dir, 'package.json'), (err, stats) => {
           if (err) {
-            return done(err);
+            // Ignore error
+            done();
+            return;
           }
-          this.content = content;
-          return done();
+
+          utils.runAsync('npm install', dir)
+            .then(() => utils.runAsync('npm test', dir))
+            .then((output) => {
+              console.log(output);
+              done();
+            }).catch(done);
         });
       });
-      it('filename', function () {
-        assert(FILENAME_REGEXP.test(entry), `filename should be of the form ${FILENAME_REGEXP}. Actual: ${entry}.`);
+
+      it('filename', () => {
+        if (stats.isDirectory()) {
+          assert(DIR_REGEXP.test(entry), `filename should be of the form ${DIR_REGEXP}. Actual: ${entry}.`);
+        } else {
+          assert(FILENAME_REGEXP.test(entry), `filename should be of the form ${FILENAME_REGEXP}. Actual: ${entry}.`);
+        }
       });
-      it('frontmatter', function () {
-        const matches = TUTORIAL_YAML_REGEXP.exec(this.content);
-        assert(TUTORIAL_YAML_REGEXP.test(this.content), `frontmatter should be of the form ${TUTORIAL_YAML_REGEXP}. Actual: ${this.content}`);
+
+      it('frontmatter', () => {
+        const matches = TUTORIAL_YAML_REGEXP.exec(content);
+        assert(TUTORIAL_YAML_REGEXP.test(content), `frontmatter should be of the form ${TUTORIAL_YAML_REGEXP}. Actual: ${content}`);
         const [
           ,
           title,

test/utils.js

@@ -1,30 +1,35 @@
-// Copyright 2015-2016, Google, Inc.
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+/**
+ * Copyright 2017, Google, Inc.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 'use strict';
 
-var path = require('path');
+const childProcess = require(`child_process`);
+const path = require('path');
 
-exports.TUTORIALS_PATH = path.join(__dirname, '/../tutorials');
+exports.TUTORIALS_PATH = path.join(__dirname, '../tutorials');
 
-exports.FILENAME_REGEXP = /^[a-z0-9\-]+\.md$/; // e.g. setting-up-postgres.md
-exports.TITLE_REGEXP = /^[a-zA-Z0-9\s\.\-\(\)&!]+$/; // e.g. How to Set Up PostgreSQL on Compute Engine
-exports.DESCRIPTION_REGEXP = /^[a-zA-Z0-9\s\.,\-\(\)&!]+$/; // e.g. Learn how to get PostgreSQL running on Compute Engine
-exports.GITHUB_REGEXP = /^[a-zA-Z0-9\-]+$/; // e.g. jimtravisgoog
-exports.TAGS_REGEXP = /^[a-zA-Z0-9\.,\s\-]+$/; // e.g. Compute Engine, PostgreSQL
+exports.FILENAME_REGEXP = /^[a-z0-9-]+\.md$/; // e.g. setting-up-postgres.md
+exports.DIR_REGEXP = /^[a-z0-9-]+$/; // e.g. using-nodejs-to-calculate-the-size-of-a-bigquery-dataset
+exports.TITLE_REGEXP = /^[a-zA-Z0-9\s.\-()&!]+$/; // e.g. How to Set Up PostgreSQL on Compute Engine
+exports.DESCRIPTION_REGEXP = /^[a-zA-Z0-9\s.,\-()&!]+$/; // e.g. Learn how to get PostgreSQL running on Compute Engine
+exports.GITHUB_REGEXP = /^[a-zA-Z0-9-]+$/; // e.g. jimtravisgoog
+exports.TAGS_REGEXP = /^[a-zA-Z0-9.,\s-]+$/; // e.g. Compute Engine, PostgreSQL
 exports.DATE_REGEXP = /^[0-1]\d\/[0-3]\d\/\d\d\d\d$/; // e.g. 03/31/2016
 exports.MEDIUM_REGEXP = /^[a-zA-Z0-9\s]+$/; // e.g. BigQuery
-exports.SIZE_REGEXP = /^[0-9\.]+\s[A-Z]+$/; // e.g. 15 TB
+exports.SIZE_REGEXP = /^[0-9.]+\s[A-Z]+$/; // e.g. 15 TB
+exports.EMBEDMD_REGEXP = /\[embedmd]:#/g; // e.g. [embedmd]:#
 
 // e.g.
 //
@@ -36,3 +41,19 @@ exports.SIZE_REGEXP = /^[0-9\.]+\s[A-Z]+$/; // e.g. 15 TB
 // date_published: 6/3/2016
 // ---
 exports.TUTORIAL_YAML_REGEXP = /^---\ntitle: (.+)\ndescription: (.+)\nauthor: (.+)\ntags: (.+)\ndate_published: (.+)\n---\n/;
+
+exports.runAsync = (cmd, cwd, cb) => {
+  return new Promise((resolve, reject) => {
+    childProcess.exec(cmd, { cwd: cwd }, (err, stdout, stderr) => {
+      if (err) {
+        reject(err);
+        return;
+      }
+      if (stdout) {
+        resolve(stdout.toString().trim());
+      } else {
+        resolve(stdout);
+      }
+    });
+  });
+};

tutorials/using-nodejs-to-calculate-the-size-of-a-bigquery-dataset/index.js

@@ -0,0 +1,33 @@
+'use strict';
+
+// Read and validate the input arguments
+const [projectId, datasetId] = process.argv.slice(2);
+
+if (!projectId || !datasetId) {
+  console.log('Usage: node index.js PROJECT_ID DATASET_ID');
+  console.log('Example: node index.js bigquery-public-data hacker_news');
+  process.exit();
+}
+
+// Instantiate a BigQuery client
+const bigquery = require('@google-cloud/bigquery')({
+  projectId: projectId
+});
+
+// References an existing dataset, e.g. "my_dataset"
+const dataset = bigquery.dataset(datasetId);
+
+// Lists all tables in the dataset
+dataset.getTables()
+  .then((results) => results[0])
+  // Retrieve the metadata for each table
+  .then((tables) => Promise.all(tables.map((table) => table.get())))
+  .then((results) => results.map((result) => result[0]))
+  // Select the size of each table
+  .then((tables) => tables.map((table) => (parseInt(table.metadata.numBytes, 10) / 1000) / 1000))
+  // Sum up the sizes
+  .then((sizes) => sizes.reduce((cur, prev) => cur + prev, 0))
+  // Print and return the size
+  .then((sum) => {
+    console.log(`Size of ${dataset.id}: ${sum} MB`);
+  });