Merge pull request #1879 from BryceLTaylor/Verbose-Testing

Verbose testing

.gitignore

@@ -28,3 +28,4 @@ ENV
 /test/integration-cypress/cypress/screenshots
 /test/integration-cypress/cypress/videos
 /test/integration-cypress/package-lock.json
+/test/integration/node_modules/*

Makefile

@@ -70,6 +70,9 @@ integration:
 
 smoke:
 	$(TAP) ./test/integration/smoke-testing/*.js --timeout=3600
+
+smoke-verbose:
+	$(TAP) ./test/integration/smoke-testing/*.js --timeout=3600 -R spec
 
 localization:
 	$(TAP) ./test/localization/*.js

package.json

@@ -7,6 +7,7 @@
     "stop": "make stop",
     "test": "make test",
     "smoke": "make smoke",
+    "smoke-verbose": "make smoke-verbose",
     "watch": "make watch",
     "build": "make build",
     "dev": "make watch && make start &"

test/integration/package.json

@@ -1,6 +1,6 @@
 {
     "devDependencies": {
-        "selenium-webdriver": "2.45.0",
-        "chromedriver": "2.33.0"
+        "selenium-webdriver": "3.6.0",
+        "chromedriver": "2.37.0"
     }
 }

test/helpers/selenium-helpers.js → test/integration/selenium-helpers.js

@@ -1,4 +1,4 @@
-const webdriver = require('selenium-webdriver');
+var webdriver = require('selenium-webdriver');
 
 const driver = new webdriver.Builder()
     .forBrowser('chrome')

test/integration/smoke-testing/test-login-failures.js

@@ -0,0 +1,130 @@
+const {
+    clickText,
+    findByXpath,
+    clickButton,
+    driver,
+    until,
+    By
+} = require('../selenium-helpers.js');
+
+var username = process.env.SMOKE_USERNAME;
+var password = process.env.SMOKE_PASSWORD;
+
+
+var tap = require('tap');
+const test = tap.test;
+
+var rootUrl = process.env.ROOT_URL || 'https://scratch.ly';
+var url = rootUrl + '/users/' + username;
+
+tap.plan(5);
+
+tap.tearDown(function () {
+    driver.quit();
+});
+
+tap.beforeEach(function () {
+    return driver.get(url);
+});
+
+/*
+ * This test fails sometimes because blank username eventually
+ * triggers the captcha page, which is a bug:
+ * https://github.com/LLK/scratchr2/issues/4762
+ */
+test('Trying to sign in with no username and no password using scratchr2 navbar', t => {
+    clickText('Sign in')
+        .then(() => clickButton('Sign in'))
+        .then(() => driver.wait(until
+            .elementLocated(By.xpath('//form[@id="login"]/button[@type="submit"]')))
+        )
+        .then(() => driver.wait(until
+            .elementLocated(By.xpath('//form[@id="login"]/div[@class="error"]')))
+        )
+        .then(() => findByXpath('//form/div[@class="error"]'))
+        .then(element => element.getText())
+        .then(text => t.match(text, 'This field is required.',
+            '"This field is required" error should be displayed'))
+        .then(() => t.end());
+});
+
+/*
+ * This test fails sometimes because blank username eventually
+ * triggers the captcha page, which is a bug:
+ * https://github.com/LLK/scratchr2/issues/4762
+ */
+test('Trying to sign in with no username using scratchr2 navbar', t => {
+    clickText('Sign in')
+        .then(() => findByXpath('//input[@name="password"]'))
+        .then((element) => element.sendKeys(password))
+        .then(() => clickButton('Sign in'))
+        .then(() => driver.wait(until
+            .elementLocated(By.xpath('//form[@id="login"]/button[@type="submit"]')))
+        )
+        .then(() => driver.wait(until
+            .elementLocated(By.xpath('//form[@id="login"]/div[@class="error"]')))
+        )
+        .then(() => findByXpath('//form/div[@class="error"]'))
+        .then((element) => element.getText())
+        .then((text) => t.match(text, 'This field is required.',
+            '"This field is required" error should be displayed'))
+        .then(() => t.end());
+});
+
+test('Trying to sign in with no password using scratchr2 navbar', t => {
+    var nonsenseusername = Math.random().toString(36)
+        .replace(/[^a-z]+/g, '')
+        .substr(0, 5);
+    clickText('Sign in')
+        .then(() => findByXpath('//input[@id="login_dropdown_username"]'))
+        .then((element) => element.sendKeys(nonsenseusername))
+        .then(() => clickButton('Sign in'))
+        .then(() => driver.wait(until
+            .elementLocated(By.xpath('//form[@id="login"]/button[@type="submit"]'))))
+        .then(() => driver.wait(until
+            .elementLocated(By.xpath('//form[@id="login"]/div[@class="error"]'))))
+        .then(() => findByXpath('//form/div[@class="error"]'))
+        .then((element) => element.getText())
+        .then((text) => t.match(text, 'This field is required.',
+            '"This field is required" error should be displayed'))
+        .then(() => t.end());
+});
+
+test('Trying to sign in with the wrong username using scratchr2 navbar', t => {
+    var nonsenseusername = Math.random().toString(36)
+        .replace(/[^a-z]+/g, '')
+        .substr(0, 5);
+    clickText('Sign in')
+        .then(() => findByXpath('//input[@id="login_dropdown_username"]'))
+        .then((element) => element.sendKeys(nonsenseusername))
+        .then(() => findByXpath('//input[@name="password"]'))
+        .then((element) => element.sendKeys(password))
+        .then(() => clickButton('Sign in'))
+        .then(() => driver.wait(until
+            .elementLocated(By.xpath('//form[@id="login"]/button[@type="submit"]'))))
+        .then(() => driver.wait(until
+            .elementLocated(By.xpath('//form[@id="login"]/div[@class="error"]'))))
+        .then(() => findByXpath('//form/div[@class="error"]'))
+        .then((element) => element.getText())
+        .then((text) => t.match(text, 'Incorrect username or password.',
+            '"Incorrect username or password" error should be displayed'))
+        .then(() => t.end());
+});
+
+test('Trying to sign in with the wrong password using scratchr2 navbar', t => {
+    clickText('Sign in')
+        .then(() => findByXpath('//input[@id="login_dropdown_username"]'))
+        .then((element) => element.sendKeys(username))
+        .then(() => findByXpath('//input[@name="password"]'))
+        .then((element) => element.sendKeys('nonsensepassword'))
+        .then(() => clickButton('Sign in'))
+        .then(() => driver.wait(until
+            .elementLocated(By.xpath('//form[@id="login"]/button[@type="submit"]'))))
+        .then(() => driver.wait(until
+            .elementLocated(By.xpath('//form[@id="login"]/div[@class="error"]'))))
+        .then(() => findByXpath('//form/div[@class="error"]'))
+        .then((element) => element.getText())
+        .then((text) => t.match(text, 'Incorrect username or password.',
+            '"Incorrect username or password" error should be displayed'))
+        .then(() => t.end());
+});

test/integration/smoke-testing/test_signing_in_and_my_stuff.js → test/integration/smoke-testing/test-my-stuff.js

@@ -10,10 +10,8 @@ const {
     findByXpath,
     clickXpath,
     clickButton,
-    driver,
-    until,
-    By
-} = require('../../helpers/selenium-helpers.js');
+    driver
+} = require('../selenium-helpers.js');
 
 var username = process.env.SMOKE_USERNAME;
 var password = process.env.SMOKE_PASSWORD;
@@ -23,9 +21,9 @@ var tap = require('tap');
 const test = tap.test;
 
 var rootUrl = process.env.ROOT_URL || 'https://scratch.ly';
-var url = rootUrl + '/users/anyuser';
+var url = rootUrl + '/users/' + username;
 
-tap.plan(12);
+tap.plan(7);
 
 tap.tearDown(function () {
     driver.quit();
@@ -35,108 +33,6 @@ tap.beforeEach(function () {
     return driver.get(url);
 });
 
-/*
- * This test fails sometimes because blank username eventually
- * triggers the captcha page, which is a bug:
- * https://github.com/LLK/scratchr2/issues/4762
- */
-test('Trying to sign in with no username and no password using scratchr2 navbar', t => {
-    clickText('Sign in')
-        .then(() => clickButton('Sign in'))
-        .then(() => driver.wait(until
-            .elementLocated(By.xpath('//form[@id="login"]/button[@type="submit"]')))
-        )
-        .then(() => driver.wait(until
-            .elementLocated(By.xpath('//form[@id="login"]/div[@class="error"]')))
-        )
-        .then(() => findByXpath('//form/div[@class="error"]'))
-        .then(element => element.getText())
-        .then(text => t.match(text, 'This field is required.',
-            '"This field is required" error should be displayed'))
-        .then(() => t.end());
-});
-
-/*
- * This test fails sometimes because blank username eventually
- * triggers the captcha page, which is a bug:
- * https://github.com/LLK/scratchr2/issues/4762
- */
-test('Trying to sign in with no username using scratchr2 navbar', t => {
-    clickText('Sign in')
-        .then(() => findByXpath('//input[@name="password"]'))
-        .then((element) => element.sendKeys(password))
-        .then(() => clickButton('Sign in'))
-        .then(() => driver.wait(until
-            .elementLocated(By.xpath('//form[@id="login"]/button[@type="submit"]')))
-        )
-        .then(() => driver.wait(until
-            .elementLocated(By.xpath('//form[@id="login"]/div[@class="error"]')))
-        )
-        .then(() => findByXpath('//form/div[@class="error"]'))
-        .then((element) => element.getText())
-        .then((text) => t.match(text, 'This field is required.',
-            '"This field is required" error should be displayed'))
-        .then(() => t.end());
-});
-
-test('Trying to sign in with no password using scratchr2 navbar', t => {
-    var nonsenseusername = Math.random().toString(36)
-        .replace(/[^a-z]+/g, '')
-        .substr(0, 5);
-    clickText('Sign in')
-        .then(() => findByXpath('//input[@id="login_dropdown_username"]'))
-        .then((element) => element.sendKeys(nonsenseusername))
-        .then(() => clickButton('Sign in'))
-        .then(() => driver.wait(until
-            .elementLocated(By.xpath('//form[@id="login"]/button[@type="submit"]'))))
-        .then(() => driver.wait(until
-            .elementLocated(By.xpath('//form[@id="login"]/div[@class="error"]'))))
-        .then(() => findByXpath('//form/div[@class="error"]'))
-        .then((element) => element.getText())
-        .then((text) => t.match(text, 'This field is required.',
-            '"This field is required" error should be displayed'))
-        .then(() => t.end());
-});
-
-test('Trying to sign in with the wrong username using scratchr2 navbar', t => {
-    var nonsenseusername = Math.random().toString(36)
-        .replace(/[^a-z]+/g, '')
-        .substr(0, 5);
-    clickText('Sign in')
-        .then(() => findByXpath('//input[@id="login_dropdown_username"]'))
-        .then((element) => element.sendKeys(nonsenseusername))
-        .then(() => findByXpath('//input[@name="password"]'))
-        .then((element) => element.sendKeys(password))
-        .then(() => clickButton('Sign in'))
-        .then(() => driver.wait(until
-            .elementLocated(By.xpath('//form[@id="login"]/button[@type="submit"]'))))
-        .then(() => driver.wait(until
-            .elementLocated(By.xpath('//form[@id="login"]/div[@class="error"]'))))
-        .then(() => findByXpath('//form/div[@class="error"]'))
-        .then((element) => element.getText())
-        .then((text) => t.match(text, 'Incorrect username or password.',
-            '"Incorrect username or password" error should be displayed'))
-        .then(() => t.end());
-});
-
-test('Trying to sign in with the wrong password using scratchr2 navbar', t => {
-    clickText('Sign in')
-        .then(() => findByXpath('//input[@id="login_dropdown_username"]'))
-        .then((element) => element.sendKeys(username))
-        .then(() => findByXpath('//input[@name="password"]'))
-        .then((element) => element.sendKeys('nonsensepassword'))
-        .then(() => clickButton('Sign in'))
-        .then(() => driver.wait(until
-            .elementLocated(By.xpath('//form[@id="login"]/button[@type="submit"]'))))
-        .then(() => driver.wait(until
-            .elementLocated(By.xpath('//form[@id="login"]/div[@class="error"]'))))
-        .then(() => findByXpath('//form/div[@class="error"]'))
-        .then((element) => element.getText())
-        .then((text) => t.match(text, 'Incorrect username or password.',
-            '"Incorrect username or password" error should be displayed'))
-        .then(() => t.end());
-});
-
 test('Sign in to Scratch using scratchr2 navbar', t => {
     clickText('Sign in')
         .then(() => findByXpath('//input[@id="login_dropdown_username"]'))

test/integration/smoke-testing/test_footer_links.js

@@ -3,13 +3,13 @@
  *
  * Test cases: https://github.com/LLK/scratch-www/wiki/Most-Important-Workflows
  */
- 
+
 const tap = require('tap');
 
 const {
     driver,
     webdriver
-} = require('../../helpers/selenium-helpers.js');
+} = require('../selenium-helpers.js');
 
 // Selenium's promise driver will be deprecated, so we should not rely on it
 webdriver.SELENIUM_PROMISE_MANAGER = 0;