Limit for lucene index size for /filter and /group API requests (#25)

* Lucene index optimization: application property "lucene.index.max.segment.size" added to allow control over lucene index segment size.

* Lucene index optimization: do not return total pages count and variant histograms for large files, application property for max lucene index size is "lucene.index.max.size.grouping"

* Lucene index optimization: removed lucene index segment size optimization

* Lucene index optimization: throw exception if group/ method is called for a index exceeding max allowed size

* 'VCF File too large, unable to show data' message will be displayed if the corresponding error occurs

* Infinite scroll support for variations table of large VCF index files (if no 'totalPages' field returned)

* Variants table loading indicator

* Pagination control disappeared - fixed

* Lucene index optimization: return only top 100 results for filter/searchGenes request

* Lucene index optimization: use term API for gene lucene search

* Do not display diagrams for large vcf files

client/client/app/components/ngbProjectInfoPanel/ngbProjectInfo/ngbVariantDensityDiagram/ngbVariantDensityDiagram.controller.js

@@ -59,7 +59,8 @@ export default class ngbVariantDensityDiagramController {
                         return v;
                     }
                 },
-                y: (d) => d.value
+                y: (d) => d.value,
+                noData: 'No Data Available'
             },
             title: {
                 enable: true,
@@ -95,7 +96,8 @@ export default class ngbVariantDensityDiagramController {
             this.projectContext.variantsDataByChromosomes.length === 0;
         if (this.projectContext.reference && this.projectContext.variantsDataByChromosomes) {
             await this.updateDiagram(this.projectContext.variantsDataByChromosomes,
-                this.projectContext.isVariantsGroupByChromosomesLoading);
+                this.projectContext.isVariantsGroupByChromosomesLoading,
+                this.projectContext.variantsGroupByChromosomesError);
             this.isProgressShown = this.projectContext.isVariantsGroupByChromosomesLoading;
             this._scope.$apply();
         }
@@ -153,12 +155,16 @@ export default class ngbVariantDensityDiagramController {
     }
 
 
-    async updateDiagram(variants, isLoading) {
+    async updateDiagram(variants, isLoading, error) {
         if (isLoading) {
             return;
         }
-        (!variants || variants.length === 0) ?
-            this._scope.data = [] : this._scope.data = this.makeNvD3ChartObjectFromData(variants);
+        if (!variants || variants.length === 0) {
+            this._scope.options.chart.noData = error || 'No Data Available';
+            this._scope.data = [];
+        } else {
+            this._scope.data = this.makeNvD3ChartObjectFromData(variants);
+        }
         this._scope.api && angular.isFunction(this._scope.api.update) ? this._scope.api.update() : '';
     }
 }

client/client/app/components/ngbProjectInfoPanel/ngbProjectInfo/ngbVariantQualityDiagram/ngbVariantQualityDiagram.controller.js

@@ -44,7 +44,7 @@ export default class ngbVariantQualityDiagramController {
                 yAxis : {
                     tickFormat : (t) => t
                 },
-
+                noData: 'No Data Available'
             },
             title: {
                 enable: true,
@@ -84,7 +84,8 @@ export default class ngbVariantQualityDiagramController {
             this.projectContext.variantsDataByQuality.length === 0;
         if (this.projectContext.reference && this.projectContext.variantsDataByQuality) {
             await this.updateDiagram(this.projectContext.variantsDataByQuality,
-                this.projectContext.isVariantsGroupByQualityLoading);
+                this.projectContext.isVariantsGroupByQualityLoading,
+                this.projectContext.variantsGroupByQualityError);
             this.isProgressShown = this.projectContext.isVariantsGroupByQualityLoading;
             this._scope.$apply();
         }
@@ -136,13 +137,16 @@ export default class ngbVariantQualityDiagramController {
 
     }
 
-    async updateDiagram(variantQualities, isLoading) {
+    async updateDiagram(variantQualities, isLoading, error) {
         if (isLoading) {
             return;
         }
-        (!variantQualities || variantQualities.length === 0) ?
-            this._scope.data = [] : this._scope.data = this.makeNvD3ChartObjectFromData(variantQualities);
-
+        if (!variantQualities || variantQualities.length === 0) {
+            this._scope.options.chart.noData = error || 'No Data Available';
+            this._scope.data = [];
+        } else {
+            this._scope.data = this.makeNvD3ChartObjectFromData(variantQualities);
+        }
         this._scope.api && angular.isFunction(this._scope.api.update) ? this._scope.api.update() : '';
     }
 

client/client/app/components/ngbProjectInfoPanel/ngbProjectInfo/ngbVariantTypeDiagram/ngbVariantTypeDiagram.controller.js

@@ -54,7 +54,8 @@ export default class ngbVariantTypeDiagramController {
                         return d;
                     }
                 },
-                y: (d)=> d.value
+                y: (d)=> d.value,
+                noData: 'No Data Available'
             },
             title: {
                 enable: true,
@@ -152,17 +153,23 @@ export default class ngbVariantTypeDiagramController {
             this.projectContext.variantsDataByType.length === 0;
         if (this.projectContext.reference && this.projectContext.variantsDataByType) {
             await this.updateDiagram(this.projectContext.variantsDataByType,
-                this.projectContext.isVariantsGroupByTypeLoading);
+                this.projectContext.isVariantsGroupByTypeLoading,
+                this.projectContext.variantsGroupByTypeError);
             this.isProgressShown = this.projectContext.isVariantsGroupByTypeLoading;
             this._scope.$apply();
         }
     }
 
-    async updateDiagram(variants, isLoading) {
+    async updateDiagram(variants, isLoading, error) {
         if (isLoading) {
             return;
         }
-        (!variants || variants.length === 0) ? this._scope.data = [] : this._scope.data = this.makeNvD3ChartObjectFromData(variants);
-        this._scope.api && angular.isFunction(this._scope.api.update) ? this._scope.api.update() : '';		
+        if (!variants || variants.length === 0) {
+            this._scope.options.chart.noData = error || 'No Data Available';
+            this._scope.data = [];
+        } else {
+            this._scope.data = this.makeNvD3ChartObjectFromData(variants);
+        }
+        this._scope.api && angular.isFunction(this._scope.api.update) ? this._scope.api.update() : '';
     }
 }

client/client/app/components/ngbProjectInfoPanel/ngbProjectInfoPanel.controller.js

@@ -17,6 +17,6 @@ export default class ngbProjectInfoPanelController {
     }
 
     get containsVcfFiles() {
-        return this.projectContext.containsVcfFiles;
+        return this.projectContext.containsVcfFiles && !this.projectContext.variantsGroupError;
     }
 }

client/client/app/components/ngbVariantsTablePanel/index.js

@@ -8,13 +8,18 @@ import ngbVariantsTablePanel from './ngbVariantsTablePanel.component.js';
 import controller from './ngbVariantsTablePanel.controller';
 
 // Import components
+import  ngbVariantsLoadingIndicator from './ngbVariantsLoadingIndicator';
 import  ngbVariantsTable from './ngbVariantsTable';
 import  ngbVariantsTableColumn from './ngbVariantsTableColumn';
 import  ngbVariantsTablePaginate from './ngbVariantsTablePaginate';
 
 
 // Import external modules
-export default angular.module('ngbVariantsTablePanel', [ngbVariantsTable,ngbVariantsTableColumn,ngbVariantsTablePaginate])
+export default angular.module('ngbVariantsTablePanel', [
+    ngbVariantsTable,
+    ngbVariantsTableColumn,
+    ngbVariantsTablePaginate,
+    ngbVariantsLoadingIndicator])
     .component('ngbVariantsTablePanel', ngbVariantsTablePanel)
     .controller(controller.UID, controller)
     .name;

client/client/app/components/ngbVariantsTablePanel/ngbVariantsLoadingIndicator/index.js

@@ -0,0 +1,12 @@
+import angular from 'angular';
+import './ngbVariantsLoadingIndicator.scss';
+
+// Import internal modules
+import component from './ngbVariantsLoadingIndicator.component';
+import controller from './ngbVariantsLoadingIndicator.controller';
+
+// Import external modules
+export default angular.module('ngbVariantsLoadingIndicator', [])
+    .controller(controller.UID, controller)
+    .component('ngbVariantsLoadingIndicator', component)
+    .name;

client/client/app/components/ngbVariantsTablePanel/ngbVariantsLoadingIndicator/ngbVariantsLoadingIndicator.component.js

@@ -0,0 +1,6 @@
+import controller from './ngbVariantsLoadingIndicator.controller';
+
+export default  {
+    controller: controller.UID,
+    template: require('./ngbVariantsLoadingIndicator.tpl.html')
+};

client/client/app/components/ngbVariantsTablePanel/ngbVariantsLoadingIndicator/ngbVariantsLoadingIndicator.controller.js

@@ -0,0 +1,34 @@
+import  baseController from '../../../shared/baseController';
+
+export default class ngbVariantsLoadingIndicatorController extends baseController {
+
+    static get UID() {
+        return 'ngbVariantsLoadingIndicatorController';
+    }
+
+    projectContext;
+    isProgressShown = false;
+    _scope;
+
+    constructor(dispatcher, projectContext, $scope) {
+        super(dispatcher);
+        this._scope = $scope;
+        Object.assign(this, {
+            $scope,
+            dispatcher,
+            projectContext
+        });
+
+        this.initEvents();
+        this.refresh(false);
+    }
+
+    events = {
+        'variants:page:loading:finished': ::this.refresh,
+        'variants:page:loading:started': ::this.refresh
+    };
+
+    refresh () {
+        this.isProgressShown = this.projectContext.variantsPageLoading;
+    }
+}

client/client/app/components/ngbVariantsTablePanel/ngbVariantsLoadingIndicator/ngbVariantsLoadingIndicator.scss

@@ -0,0 +1,11 @@
+ngb-variants-loading-indicator {
+  align-self: center;
+  height: 22px;
+  .indicator {
+    margin-right: 10px;
+    margin-top: 4px;
+    md-progress-circular {
+      svg path { stroke: white; }
+    }
+  }
+}

client/client/app/components/ngbVariantsTablePanel/ngbVariantsLoadingIndicator/ngbVariantsLoadingIndicator.tpl.html

@@ -0,0 +1,7 @@
+<div class="indicator">
+    <md-progress-circular
+            md-mode="indeterminate"
+            ng-show="$ctrl.isProgressShown"
+            md-diameter="16">
+    </md-progress-circular>
+</div>

client/client/app/components/ngbVariantsTablePanel/ngbVariantsTable/ngbVariantsTable.controller.js

@@ -251,7 +251,10 @@ export default class ngbVariantsTableController extends baseController {
         this.projectContext.loadVariations(this.projectContext.lastPageVariations).then((data) => {
             this.gridApi.infiniteScroll.saveScrollPercentage();
             this.gridOptions.data = this.gridOptions.data.concat(data);
-            this.gridApi.infiniteScroll.dataLoaded(this.projectContext.firstPageVariations > 1, this.projectContext.lastPageVariations < this.projectContext.totalPagesCountVariations);
+            this.gridApi.infiniteScroll.dataLoaded(
+                this.projectContext.firstPageVariations > 1,
+                (this.projectContext.totalPagesCountVariations === undefined && this.projectContext.hasMoreVariations)
+                || this.projectContext.lastPageVariations < this.projectContext.totalPagesCountVariations);
         });
     }
 
@@ -267,7 +270,10 @@ export default class ngbVariantsTableController extends baseController {
 
             const self = this;
             this.$timeout(function () {
-                self.gridApi.infiniteScroll.dataLoaded(self.projectContext.firstPageVariations > 1, self.projectContext.lastPageVariations < self.projectContext.totalPagesCountVariations);
+                self.gridApi.infiniteScroll.dataLoaded(
+                    self.projectContext.firstPageVariations > 1,
+                    (self.projectContext.totalPagesCountVariations === undefined && self.projectContext.hasMoreVariations)
+                    || self.projectContext.lastPageVariations < self.projectContext.totalPagesCountVariations);
             });
         });
     }
@@ -283,7 +289,10 @@ export default class ngbVariantsTableController extends baseController {
             const self = this;
             this.gridOptions.data = data;
             this.$timeout(function () {
-                self.gridApi.infiniteScroll.resetScroll(self.projectContext.firstPageVariations > 1, self.projectContext.lastPageVariations < self.projectContext.totalPagesCountVariations);
+                self.gridApi.infiniteScroll.resetScroll(
+                    self.projectContext.firstPageVariations > 1,
+                    (self.projectContext.totalPagesCountVariations === undefined && self.projectContext.hasMoreVariations)
+                    || self.projectContext.lastPageVariations < self.projectContext.totalPagesCountVariations);
             });
         });
     }
@@ -311,7 +320,10 @@ export default class ngbVariantsTableController extends baseController {
             const self = this;
             this.gridOptions.data = data;
             this.$timeout(function () {
-                self.gridApi.infiniteScroll.resetScroll(self.projectContext.firstPageVariations > 1, self.projectContext.lastPageVariations < self.projectContext.totalPagesCountVariations);
+                self.gridApi.infiniteScroll.resetScroll(
+                    self.projectContext.firstPageVariations > 1,
+                    (self.projectContext.totalPagesCountVariations === undefined && self.projectContext.hasMoreVariations)
+                    || self.projectContext.lastPageVariations < self.projectContext.totalPagesCountVariations);
             });
         });
     }
@@ -337,7 +349,10 @@ export default class ngbVariantsTableController extends baseController {
 
             const self = this;
             this.$timeout(function () {
-                self.gridApi.infiniteScroll.dataLoaded(self.projectContext.firstPageVariations > 1, self.projectContext.lastPageVariations < self.projectContext.totalPagesCountVariations);
+                self.gridApi.infiniteScroll.dataLoaded(
+                    self.projectContext.firstPageVariations > 1,
+                    (self.projectContext.totalPagesCountVariations === undefined && self.projectContext.hasMoreVariations)
+                    || self.projectContext.lastPageVariations < self.projectContext.totalPagesCountVariations);
             });
         }
     }

client/client/app/components/ngbVariantsTablePanel/ngbVariantsTablePaginate/ngbVariantsTablePaginate.controller.js

@@ -43,7 +43,7 @@ export default class ngbVariantsTablePaginateController extends baseController {
 
     setPage(page, loadData)
     {
-        if (page < 1 || page > this.totalPages) {
+        if (this.totalPages === undefined || page < 1 || page > this.totalPages) {
             return;
         }
 
@@ -57,6 +57,9 @@ export default class ngbVariantsTablePaginateController extends baseController {
     getPages() {
         const totalPages = this.totalPages;
         const currentPage = this.currentPage;
+        if (totalPages === undefined || currentPage === undefined) {
+            return [];
+        }
 
         let minimumPage = Math.max(1, currentPage - 3);
         let maximumPage = Math.min(totalPages, currentPage + 3);

client/client/app/shared/components/ngbGoldenLayout/ngbViewActions/ngbViewActions.constant.js

@@ -9,6 +9,13 @@ const variantsTablePaginationAction = {
     },
     isDefault: false
 };
+const variantsLoadingIndicatorAction = {
+    name: 'variantsLoadingIndicator',
+    liStyle: {
+        width: 'auto'
+    },
+    isDefault: false
+};
 const closeAllTracksAction = {
     name: 'closeAllTracks',
     isDefault: false
@@ -58,15 +65,16 @@ const genomeAnnotationsAction = {
 export default {
     actions: {
         closeAllTracks: closeAllTracksAction,
-        variantsTableColumn: variantsTableColumnAction,
-        variantsTablePagination: variantsTablePaginationAction,
-        variantsResetFilter: variantsResetFilterActions,
         fitAllTracks: fitAllTracksAction,
+        genomeAnnotations: genomeAnnotationsAction,
         organizeTracks: organizeTracksAction,
-        genomeAnnotations: genomeAnnotationsAction
+        variantsLoadingIndicator: variantsLoadingIndicatorAction,
+        variantsResetFilter: variantsResetFilterActions,
+        variantsTableColumn: variantsTableColumnAction,
+        variantsTablePagination: variantsTablePaginationAction
     },
     viewActions: {
         ngbBrowser: [genomeAnnotationsAction, fitAllTracksAction, organizeTracksAction, closeAllTracksAction],
-        ngbVariantsTablePanel: [variantsTablePaginationAction, variantsResetFilterActions, variantsTableColumnAction]
+        ngbVariantsTablePanel: [variantsTablePaginationAction, variantsLoadingIndicatorAction, variantsResetFilterActions, variantsTableColumnAction]
     }
 };

client/client/app/shared/components/ngbGoldenLayout/ngbViewActions/ngbViewActions.tpl.html

@@ -1,6 +1,7 @@
 <li ng-repeat="action in viewActions[viewName] track by $index" ng-style="action.liStyle" ng-if="action.isVisible === undefined || action.isVisible(projectContext)">
     <ngb-variants-table-column ng-if="action === actions.variantsTableColumn"></ngb-variants-table-column>
     <ngb-variants-table-paginate ng-if="action === actions.variantsTablePagination"></ngb-variants-table-paginate>
+    <ngb-variants-loading-indicator ng-if="action === actions.variantsLoadingIndicator"></ngb-variants-loading-indicator>
     <ngb-close-all-tracks ng-if="action === actions.closeAllTracks && tracksAreSelected()"></ngb-close-all-tracks>
     <ngb-genome-annotations ng-if="action === actions.genomeAnnotations && tracksAreSelected() && (action.isVisible === undefined || action.isVisible(projectContext))"></ngb-genome-annotations>
     <ngb-view-action ng-if="action.isDefault && (action.isVisible === undefined || action.isVisible(projectContext))" event="action.event" label="action.label" icon="action.icon"></ngb-view-action>