Adds background workers and crafts and cleans up a lot of code

* adds composer scripts for `lint`, `test`, `start`
* adds background worker to Heroku Proc file
* set default tests to skip functional tests
* adds new table - job_queue for holding jobs for the worker
* adds support for sqlite (mostly for fast unit testing)
* adds mockery and vfsstream for testing
* config options for workers added
* reorganized configs and setting files
* added config for tests
* changed config/tests.php to only be loaded as needed
* added UdoitDB database abstraction class that handles reconnecting for long running jobs
* added UdoitJob class that manages running and creating queued jobs
* expanded UdoitUtils to centeralize more functions of the application
* added worker that processes jobs in the background
* added a ton of unit tests
* updated default.js to accomidate changes to background workers
* updated a lot of rel require/include references to use absolute refs
* heavily simplified index.php to act purely as an LTI launch target
* simplified oauth2responce.php
* updated parseResults.php to use GET rather then POST
* changed process.php to simply store the jobs to do
* changed progress.php to simply report on the status of jobs
* added missing data from the sample report.json
* moved the code used to display the scanner from index.php to a new file scanner.php
* all .sh scripts moved to composer scripts
* Adopted composer autoloading for all classes

.travis.yml

@@ -9,6 +9,6 @@ php:
   - '5.4'
   - '5.5'
   - '5.6'
-script: 
-  - 'find . -type f -iname "*.php" -not -path "./lib/quail/tests/*" -not -path "./vendor/*" | xargs -n1 php -l'
-  - './vendor/phpunit/phpunit/phpunit --testsuite "UDOIT"'
+script:
+  - 'composer lint'
+  - 'composer test'

HEROKU.md

@@ -119,7 +119,7 @@ If you need to check that the tables exist, you can connect to Postgres using so
 * `Select * from users;` or `Select * from reports;` will show you their contents
 * `\q` quits the psql terminal
 
-If needed, you can manually run the table creation script: `Heroku run php lib/db_create_tables.php`
+If needed, you can manually run the table creation script: `heroku run composer dbsetup`
 
 ## Table Schema
-The table schema can be found in [lib/db_create_tables.php](lib/db_create_tables.php)
+The table schema can be found in [bin/db_create_tables.php](bin/db_create_tables.php)

Procfile

@@ -1 +1,2 @@
 web: vendor/bin/heroku-php-apache2 public/
+worker: php lib/worker.php

README.md

@@ -96,10 +96,10 @@ Edit `config/localConfig.php`:
 To create the required tables, run the creation script below.  You'll need to complete the db steps above first.
 
 ```
-$ php lib/db_create_tables.php
+$ php composer.phar dbsetup
 ```
 
-The table schema can be found in [lib/db_create_tables.php](lib/db_create_tables.php)
+The table schema can be found in [bin/db_create_tables.php](bin/db_create_tables.php)
 
 ## Configuration and Setup
 If you didn't already make `config/localConfig.php` when you set up the database, do it now.
@@ -210,7 +210,7 @@ For quick local development, set `$UDOIT_ENV = ENV_DEV;` in `config/localConfig.
 From the public directory, run:
 
 ```
-$ php -S localhost:8000
+$ php composer.phar start
 ```
 
 Then open [http://localhost:8000 in a browser](http://localhost:8000).
@@ -219,13 +219,19 @@ Then open [http://localhost:8000 in a browser](http://localhost:8000).
 We use phpunit to run unit tests on UDOIT.  To run the tests, type the following command:
 
 ```
-$ ./vendor/phpunit/phpunit/phpunit
+$ php composer.phar test
+```
+
+We included a Dockerfile, docker-compose.yml, and tests script to run your tests in a predictable environment.  To run tests using docker run this command:
+
+```
+$ php composer.phar test:docker
 ```
 
-By default, phpunit will run all tests, including the functional tests that require access to outside APIs.  If you would like to exclude those tests, run this command:
+By default, we exclude functional tests that include external APIs.  If you would like to run those tests, run this command:
 
 ```
-$ ./vendor/phpunit/phpunit/phpunit --exclude-group functional
+$ ./vendor/phpunit/phpunit/phpunit
 ```
 
 ## Contributors

app.json

@@ -54,6 +54,6 @@
     }
   ],
   "scripts": {
-    "postdeploy": "php lib/db_create_tables.php"
+    "postdeploy": "composer dbsetup"
   }
 }

bin/db_create_tables.php

@@ -0,0 +1,136 @@
+<?php
+require_once(__DIR__.'/../config/settings.php');
+global $db_type;
+if ($db_type == 'sqlite' || $db_type == 'test')
+{
+    // SQLITE (mostly for testing)
+    $tables = [
+        '
+            CREATE TABLE IF NOT EXISTS reports (
+                id integer PRIMARY KEY AUTOINCREMENT,
+                user_id integer,
+                course_id integer,
+                report_json text,
+                date_run timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
+                errors integer,
+                suggestions integer
+            );
+        ',
+
+        '
+            CREATE TABLE IF NOT EXISTS users (
+                id integer CONSTRAINT users_pk PRIMARY KEY AUTOINCREMENT,
+                api_key varchar(255),
+                refresh_token varchar(255),
+                canvas_url varchar(255),
+                date_created timestamp with time zone DEFAULT CURRENT_TIMESTAMP
+            );
+        ',
+
+        '
+            CREATE TABLE IF NOT EXISTS job_queue (
+                id integer PRIMARY KEY AUTOINCREMENT,
+                job_group varchar(255),
+                user_id integer,
+                job_type varchar(255),
+                data text,
+                results text,
+                status varchar(255),
+                date_created timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
+                date_completed timestamp with time zone
+            );
+        '
+    ];
+}
+if ($db_type == 'pgsql')
+{
+    // POSTGRESQL
+    $tables = [
+        '
+            CREATE TABLE IF NOT EXISTS reports (
+                id SERIAL PRIMARY KEY,
+                user_id integer,
+                course_id integer,
+                report_json text,
+                date_run timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
+                errors integer,
+                suggestions integer
+            );
+        ',
+
+        '
+            CREATE TABLE IF NOT EXISTS users (
+                id integer CONSTRAINT users_pk PRIMARY KEY,
+                api_key varchar(255),
+                refresh_token varchar(255),
+                canvas_url varchar(255),
+                date_created timestamp with time zone DEFAULT CURRENT_TIMESTAMP
+            );
+        ',
+
+        '
+            CREATE TABLE IF NOT EXISTS job_queue (
+                id SERIAL PRIMARY KEY,
+                job_group varchar(255),
+                user_id integer,
+                job_type varchar(255),
+                data text,
+                results text,
+                status varchar(255),
+                date_created timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
+                date_completed timestamp with time zone
+            );
+        '
+    ];
+}
+if ($db_type == 'mysql')
+{
+    // MYSQL
+    $tables = [
+        '
+            CREATE TABLE IF NOT EXISTS `reports` (
+                `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+                `user_id` int(10) unsigned NOT NULL,
+                `course_id` int(10) unsigned NOT NULL,
+                `report_json` MEDIUMTEXT NOT NULL,
+                `date_run` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+                `errors` int(10) unsigned NOT NULL,
+                `suggestions` int(10) unsigned NOT NULL,
+                PRIMARY KEY (`id`)
+            ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+        ',
+
+        '
+            CREATE TABLE IF NOT EXISTS `users` (
+                `id` int(10) unsigned NOT NULL,
+                `api_key` varchar(255) NOT NULL,
+                `refresh_token` varchar(255) NOT NULL,
+                `canvas_url` varchar(255) NOT NULL,
+                `date_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                PRIMARY KEY (`id`),
+                UNIQUE KEY `id` (`id`)
+            ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+        ',
+
+        '
+            CREATE TABLE IF NOT EXISTS `job_queue` (
+                `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+                job_group varchar(255) NOT NULL,
+                `user_id` int(10) unsigned NOT NULL,
+                `job_type` varchar(255) NOT NULL,
+                `data` text,
+                `results` text,
+                `status` varchar(255) NOT NULL DEFAULT "new",
+                `date_created` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+                `date_completed` timestamp,
+                PRIMARY KEY (`id`),
+            ) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+        '
+    ];
+}
+
+//  run every query
+foreach ($tables as $sql)
+{
+    UdoitDB::query($sql);
+}

bin/move_reports_to_db.php

@@ -0,0 +1,68 @@
+<?php
+/**
+*   Copyright (C) 2014 University of Central Florida, created by Jacob Bates, Eric Colon, Fenel Joseph, and Emily Sachs.
+*
+*   This program is free software: you can redistribute it and/or modify
+*   it under the terms of the GNU General Public License as published by
+*   the Free Software Foundation, either version 3 of the License, or
+*   (at your option) any later version.
+*
+*   This program is distributed in the hope that it will be useful,
+*   but WITHOUT ANY WARRANTY; without even the implied warranty of
+*   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+*   GNU General Public License for more details.
+*
+*   You should have received a copy of the GNU General Public License
+*   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*
+*   Primary Author Contact:  Jacob Bates <jacob.bates@ucf.edu>
+*/
+
+require_once(__DIR__.'/../config/settings.php');
+$dbh = include(__DIR__.'/../lib/db.php');
+
+$rows = $dbh->query("SELECT * FROM {$db_reports_table}")->fetchAll();
+
+if( ! isset($rows[0]['file_path']))
+{
+	exit("It looks like this script doesnt need to be run");
+}
+
+if( ! isset($rows[0]['report_json']))
+{
+	// Quick hack to add report column since we dont have migrations yet
+	$column_type = $db_type == 'mysql' ? 'MEDIUMTEXT' : 'TEXT';
+	$dbh->query("ALTER TABLE {$db_reports_table} ADD report_json {$column_type}");
+}
+
+$sth = $dbh->prepare("UPDATE {$db_reports_table} set report_json = :report_json WHERE id = :id");
+$count_moved = 0;
+
+foreach ($rows as $row)
+{
+	if(empty($row['file_path'])) continue;
+
+	$file = __DIR__. '/'. $row['file_path'];
+	if( ! file_exists($file)){
+		echo("Json file not found {$file} for report id: {$row['id']}\n");
+		continue;
+	}
+
+	$json = file_get_contents($file);
+
+	if(empty($json)) continue;
+
+	$sth->bindValue(':report_json', $json, PDO::PARAM_STR);
+	$sth->bindValue(':id', $row['id'], PDO::PARAM_INT);
+	$res = $sth->execute();
+
+	if(!$res){
+		echo("Failed inserting report for {$row['id']}");
+		continue;
+	}
+
+	$count_moved++;
+}
+
+$dbh->query("ALTER TABLE {$db_reports_table} DROP COLUMN file_path");
+echo("Moved {$count_moved} reports from disk to the database. Feel free to delete the reports directory");