Merge pull request #340 from cfpb/milestone9

Milestone9

.travis.yml

@@ -6,7 +6,7 @@ before_install:
 - npm install -g grunt-cli
 install: npm install
 before_deploy:
-- grunt build
+- scripts/grunt_build.bash
 - grunt zip
 - grunt codedeploy
 deploy:

ABOUT.md

@@ -0,0 +1,25 @@
+# About
+
+The HMDA Pilot is a work in progress by the [Consumer Financial Protection Bureau](http://consumerfinance.gov/) to provide a new way to validate HMDA data. The application makes it easier to review and verify validation edits.
+
+## Validate a HMDA Dataset
+
+1. **Select a .DAT file**
+
+    On the File and Validate page (Step 1), select a properly formatted .DAT file and filing year, then click the Start Validation button. When the validation starts, the file will remain on your computer and will not be uploaded.
+
+2. **Validate Data**
+
+    The validation edits must be performed in the order shown below. For each step, the file must pass all edits and be verified before proceeding with the next step.
+
+    * Syntactical & Validity Edits
+    * Quality & Macro Edits
+    * MSA and IRS Edits
+
+3. **Review Edit Reports**
+
+    Validation errors will be displayed in the Edit Reports and Edit Details pages. If needed, make corrections in your system of record and revalidate the entire file (starting at Step 1).
+
+4. **Review Validation Summary**
+
+    After all edits are complete the Validation Summary will display the Respondent and File information. You do not need to submit the validated file or send an additional information.

COMMON_QUESTIONS.md

@@ -0,0 +1,5 @@
+# Commonly Asked Questions
+
+1. Why can't I skip ahead from Syntactical/validity to Quality before you fixing my Syntactical/Validity edit errors?
+
+1. When you recheck the file that you have fixed, you need to recheck the full file and not just the changes

Gruntfile.js

@@ -64,6 +64,10 @@ module.exports = function (grunt) {
       gruntfile: {
         files: ['Gruntfile.js']
       },
+      markdown: {
+        files: ['ABOUT.md', 'COMMON_QUESTIONS.md'],
+        tasks: ['markdown:help']
+      },
       livereload: {
         options: {
           livereload: '<%= connect.options.livereload %>'
@@ -499,6 +503,17 @@ module.exports = function (grunt) {
     },
     ngdocs: {
       all: ['<%= yeoman.app %>/scripts/{,*/}*.js']
+    },
+    markdown: {
+        help: {
+            files: [{
+                'app/partials/about.html': 'ABOUT.md',
+                'app/views/common_questions.html': 'COMMON_QUESTIONS.md',
+            }],
+            options: {
+                template: 'config/md-to-html.jst',
+            }
+        }
     }
   });
 
@@ -520,6 +535,7 @@ module.exports = function (grunt) {
       'concurrent:server',
       'autoprefixer',
       'replace:local',
+      'markdown:help',
       'connect:livereload',
       'watch'
     ]);
@@ -567,7 +583,8 @@ module.exports = function (grunt) {
       'uglify',
       'filerev',
       'usemin',
-      'htmlmin'
+      'htmlmin',
+      'markdown:help'
     ]);
   });
 

app/partials/about.html

@@ -0,0 +1,23 @@
+<h1 id="about">About</h1>
+<p>The HMDA Pilot is a work in progress by the <a href="http://consumerfinance.gov/">Consumer Financial Protection Bureau</a> to provide a new way to validate HMDA data. The application makes it easier to review and verify validation edits.</p>
+<h2 id="validate-a-hmda-dataset">Validate a HMDA Dataset</h2>
+<ol>
+<li><p><strong>Select a .DAT file</strong></p>
+<p> On the File and Validate page (Step 1), select a properly formatted .DAT file and filing year, then click the Start Validation button. When the validation starts, the file will remain on your computer and will not be uploaded.</p>
+</li>
+<li><p><strong>Validate Data</strong></p>
+<p> The validation edits must be performed in the order shown below. For each step, the file must pass all edits and be verified before proceeding with the next step.</p>
+<ul>
+<li>Syntactical &amp; Validity Edits</li>
+<li>Quality &amp; Macro Edits</li>
+<li>MSA and IRS Edits</li>
+</ul>
+</li>
+<li><p><strong>Review Edit Reports</strong></p>
+<p> Validation errors will be displayed in the Edit Reports and Edit Details pages. If needed, make corrections in your system of record and revalidate the entire file (starting at Step 1).</p>
+</li>
+<li><p><strong>Review Validation Summary</strong></p>
+<p> After all edits are complete the Validation Summary will display the Respondent and File information. You do not need to submit the validated file or send an additional information.</p>
+</li>
+</ol>
+

app/partials/errorDetail-macro.html

@@ -2,7 +2,7 @@
     <tbody ng-repeat="item in error.errors">
         <tr ng-repeat="(key, value) in item.properties">
             <th class="property" scope="row">{{key}}</th>
-            <td class="value">{{value}}</td>
+            <td class="value">{{value | hmdaMacroValue:key}}</td>
         </tr>
     </tbody>
 </table>

app/partials/errorDetail.html

@@ -1,5 +1,13 @@
 <pagination>
-<pagination-size></pagination-size>
+
+<div class="detail-header content-l">
+    <div class="content-l_col content-l_col-2-3">
+        <pagination-size></pagination-size>
+    </div>
+    <div class="toolbar content-l_col content-l_col-1-3">
+        <button class="btn btn__link" hmda-export export="individual" type="{{editType}}" edit-id="{{editId}}" ng-if="error"><span class="cf-icon cf-icon-pdf"></span> Export</button>
+    </div>
+</div>
 
 <table id="{{editType}}" class="error-detail grouped-rows" ng-if="error">
     <thead>
@@ -15,7 +23,7 @@
         <tr ng-repeat="(key, value) in item.properties">
             <th class="{{error.scope === 'lar' ? 'loan-number':'line-number'}}" scope="row" rowspan="{{item.properties | keyLength}}" ng-if="$first">{{item.loanNumber || item.lineNumber}}</th>
             <td class="description">{{ {'property':key, 'lineNumber':item.lineNumber} | hmdaLabel:error.scope}}</td>
-            <td class="value">{{value}}</td>
+            <td class="value">{{value | hmdaValue:error.scope:key}}</td>
             <td class="action" rowspan="{{item.properties | keyLength}}" ng-if="$first && editType !== 'quality'">Correct and revalidate</td>
         </tr>
     </tbody>

app/partials/header.html

@@ -8,6 +8,11 @@
             <h1><a class="title-link" href="#/">HMDA Pilot</a></h1>
         </div>
 
+        <ul class="site-nav">
+            <li><a href="#/about" target="help">About</a></li>
+            <li><a href="#/common-questions" target="help">Common Questions</a></li>
+        </ul>
+
         <div class="site-logo">
             <a href="http://www.consumerfinance.gov"><img class="logo" src="images/logo_210.png" alt="Consumer Financial Protection Bureau"></a>
         </div>

app/partials/progressBar.html

@@ -0,0 +1,6 @@
+<div class="progress-bar">
+    <div role="progressbar" aria-valuemin="0" aria-valuemax="100" aria-valuenow="{{percentageComplete}}" aria-valuetext="Complete" class="meter">
+        <span style="width:{{percentageComplete}}%"></span>
+    </div>
+    <p class="description">{{processStep}}</p>
+</div>

app/partials/wizardNav.html

@@ -1,4 +1,4 @@
-<nav class="nav-wizard">
+<nav class="nav-wizard" ng-if="showWizardNav()">
     <ol>
         <li ng-repeat="step in steps" class="step {{step.stepClass}}">
             <span class="{{step.badgeClass}}">{{step.badgeText}}</span>

app/scripts/app.js

@@ -71,6 +71,12 @@ angular
         templateUrl: 'views/reportIRS.html',
         controller: 'IRSReportCtrl'
       })
+      .when('/about', {
+        templateUrl: 'views/about.html'
+      })
+      .when('/common-questions', {
+        templateUrl: 'views/common_questions.html'
+      })
       .otherwise({
         redirectTo: '/'
       });
@@ -84,7 +90,7 @@ angular
     $rootScope.$watch(function() {
         return HMDAEngine.getHmdaJson();
     }, function(newVal) {
-        if (angular.equals({}, newVal)) {
+        if (angular.equals({}, newVal) && ['/about', '/common-questions'].indexOf($location.path()) === -1) {
             $location.path('/');
         }
     });

app/scripts/controllers/index.js

@@ -13,3 +13,4 @@ app.controller('ErrorDetailCtrl', require('./errorDetail'));
 app.controller('SpecialErrorDetailCtrl', require('./specialErrorDetail'));
 app.controller('IRSReportCtrl', require('./irsReport'));
 app.controller('PaginationCtrl', require('./pagination'));
+app.controller('ProgressBarCtrl', require('./progressBar'));

app/scripts/controllers/progressBar.js

@@ -0,0 +1,19 @@
+'use strict';
+
+/**
+ * @ngdoc function
+ * @name hmdaPilotApp.controller:ProgressBarCtrl
+ * @description
+ * # ProgressBarCtrl
+ * Controller of the hmdaPilotApp
+ */
+module.exports = /*@ngInject*/ function ($scope, HMDAEngine) {
+
+    $scope.percentageComplete = 0;
+
+    HMDAEngine.getProgress().events.on('progressStep', function(percent) {
+        $scope.$apply(function(){
+            $scope.percentageComplete = percent;
+        });
+    });
+};

app/scripts/controllers/selectFile.js

@@ -7,19 +7,20 @@
  * # Select File
  * Controller for selecting a HMDA file and Reporting Year for verification.
  */
-module.exports = /*@ngInject*/ function ($scope, $location, $q, $timeout, FileReader, FileMetadata, HMDAEngine, Wizard, Session) {
-    var fiscalYears = HMDAEngine.getValidYears();
+module.exports = /*@ngInject*/ function ($scope, $location, $q, $timeout, FileReader, FileMetadata, HMDAEngine, Wizard, Session, ngDialog, Configuration) {
+    var progressDialog,
+        fiscalYears = HMDAEngine.getValidYears();
 
     // Set/Reset the state of different objects on load
     Session.reset();
     HMDAEngine.clearHmdaJson();
     HMDAEngine.clearErrors();
+    HMDAEngine.clearProgress();
     $scope.metadata = FileMetadata.clear();
     $scope.wizardSteps = Wizard.initSteps();
 
     // Populate the $scope
     $scope.reportingYears = fiscalYears;
-    $scope.isProcessing = false;
 
     // Initialize the errors for the form fields
     $scope.errors = {};
@@ -45,14 +46,18 @@ module.exports = /*@ngInject*/ function ($scope, $location, $q, $timeout, FileRe
     $scope.submit = function(hmdaData) {
         // Clear out any existing errors
         $scope.errors.global = null;
-        // Toggle processing flag on so that we can notify the user
-        $scope.isProcessing = true;
 
-        $timeout(function() { $scope.process(hmdaData); }, 100); // Pause before starting the conversion so that the DOM can update
+        // Give a name to the current step in the process (shown in the progressDialog)
+        $scope.processStep = 'Processing HMDA file...';
+
+        progressDialog = ngDialog.open(angular.extend(Configuration.progressDialog, {scope: $scope}));
 
+        $timeout(function() { $scope.process(hmdaData); }, 100); // Pause before starting the conversion so that the DOM can update
     };
 
     $scope.process = function(hmdaData) {
+
+        // Enable LocalDB support?
         HMDAEngine.setUseLocalDB(hmdaData.local);
 
         /* istanbul ignore if debug */
@@ -63,8 +68,8 @@ module.exports = /*@ngInject*/ function ($scope, $location, $q, $timeout, FileRe
         // Convert the file to JSON
         HMDAEngine.fileToJson(hmdaData.file, hmdaData.year, function(fileErr) {
             if (fileErr) {
-                // Toggle processing flag off
-                $scope.isProcessing = false;
+                // Close the progress dialog
+                progressDialog.close();
 
                 $scope.errors.global = fileErr;
                 $scope.$apply();
@@ -77,6 +82,9 @@ module.exports = /*@ngInject*/ function ($scope, $location, $q, $timeout, FileRe
                 console.time('total time for syntactical and validity edits');
             }
 
+            // Give a name to the current step in the process (shown in the progressDialog)
+            $scope.processStep = 'Validating Syntactical and Validity edits...';
+
             $q.all([HMDAEngine.runSyntactical(hmdaData.year), HMDAEngine.runValidity(hmdaData.year)])
             .then(function() {
 
@@ -94,12 +102,12 @@ module.exports = /*@ngInject*/ function ($scope, $location, $q, $timeout, FileRe
                 // And go the summary page
                 $location.path('/summarySyntacticalValidity');
 
-                // Toggle processing flag off
-                $scope.isProcessing = false;
+                // Close the progress dialog
+                progressDialog.close();
             })
             .catch(function(err) {
-                // Toggle processing flag off
-                $scope.isProcessing = false;
+                // Close the progress dialog
+                progressDialog.close();
 
                 $scope.errors.global = err.message;
                 return;

app/scripts/controllers/summaryMSA-IRS.js

@@ -24,10 +24,8 @@ module.exports = /*@ngInject*/ function ($scope, $location, Wizard, HMDAEngine,
     }
 
     // Get the list of errors from the HMDAEngine
-    var editErrors = HMDAEngine.getErrors();
-
     $scope.data = {
-        specialErrors: editErrors.special
+        specialErrors: HMDAEngine.getErrors().special
     };
 
     $scope.showIRSReport = function() {

app/scripts/controllers/summaryQualityMacro.js

@@ -7,7 +7,10 @@
  * # SummaryQualityMacroCtrl
  * Controller for the Syntactical and Validity Summary view
  */
-module.exports = /*@ngInject*/ function ($scope, $location, $q, $timeout, HMDAEngine, Wizard, Session) { /*jshint ignore:line*/
+module.exports = /*@ngInject*/ function ($scope, $location, $q, $timeout, HMDAEngine, Wizard, Session, ngDialog, Configuration) { /*jshint ignore:line*/
+
+    // Set/Reset the state of different objects on load
+    HMDAEngine.clearProgress();
 
     Array.prototype.diff = function(a) {
         return this.filter(function(i) { return a.indexOf(i) < 0; });
@@ -39,14 +42,13 @@ module.exports = /*@ngInject*/ function ($scope, $location, $q, $timeout, HMDAEn
 
     // Populate the $scope
     $scope.errors = {};
-    $scope.isProcessing = false;
 
     // Get the list of errors from the HMDAEngine
-    var editErrors = HMDAEngine.getErrors();
+    var progressDialog;
 
     $scope.data = {
-        qualityErrors: editErrors.quality,
-        macroErrors: editErrors.macro
+        qualityErrors: HMDAEngine.getErrors().quality,
+        macroErrors: HMDAEngine.getErrors().macro
     };
 
     $scope.previous = function () {
@@ -58,11 +60,13 @@ module.exports = /*@ngInject*/ function ($scope, $location, $q, $timeout, HMDAEn
     };
 
     $scope.next = function() {
-        if (hasErrors(editErrors.special)) {
+        if (hasErrors(HMDAEngine.getErrors().special)) {
             $location.path('/summaryMSA-IRS');
         } else {
-            // Toggle processing flag on so that we can notify the user
-            $scope.isProcessing = true;
+            // Give a name to the current step in the process (shown in the progressDialog)
+            $scope.processStep = 'Processing MSA/MD Data...';
+
+            progressDialog = ngDialog.open(angular.extend(Configuration.progressDialog, {scope: $scope}));
 
             // Pause before starting the validation so that the DOM can update
             $timeout(function() { $scope.process(); }, 100);
@@ -91,11 +95,11 @@ module.exports = /*@ngInject*/ function ($scope, $location, $q, $timeout, HMDAEn
             // And go the next summary page
             $location.path('/summaryMSA-IRS');
 
-            // Toggle processing flag off
-            $scope.isProcessing = false;
+            // Close the progress dialog
+            progressDialog.close();
         }).catch(function(err) {
-            // Toggle processing flag off
-            $scope.isProcessing = false;
+            // Close the progress dialog
+            progressDialog.close();
 
             $scope.errors.global = err.message;
             return;