Merge b39e3f3 into 1bfeff5

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Superdesk Analytics Changelog
 
+## [1.7.3] 2020-01-31
+### Fixes
+- [SDESK-4847] Implement CSV download for tables (#113)
+- [SDESK-4695] Convert relative dates to absolute for date_histogram bounds (#112)
+- [SDESK-4695] (fix): Histogram aggregations failing for relative dates (#114)
+- fix(install): Fix installing mkdir for highcharts-export-server (#115)
+- fix(packages): Update versions based on GitHub recommendations (#116)
+
 ## [1.7.2] 2019-11-05
 ### Improvements
 - [SDESK-4779] Add subject to publishing based reports (#111)

client/charts/SDChart/Axis.js

@@ -168,6 +168,14 @@ export class Axis {
          */
         this.excludeEmpty = false;
 
+        /**
+         * @ngdoc property
+         * @name SDChart.Axis#includeTotal
+         * @type {Boolean}
+         * @description If true, then adds the 'Total' column in table outputs
+         */
+        this.includeTotal = true;
+
         this._sortedCategories = undefined;
     }
 
@@ -190,6 +198,7 @@ export class Axis {
      * @param {String} [options.sortOrder] - If undefined, do not sort, otherwise sort categories in
      * ascending or descending order based on series values (if series data is provided as an object)
      * @param {Boolean} [options.excludeEmpty=false] If true, remove values with 0 from the categories and series
+     * @param {Boolean} [options.includeTotal] - If true, then adds the 'Total' column in table outputs
      * @return {SDChart.Axis}
      * @description Sets the options for the axis
      */
@@ -283,7 +292,9 @@ export class Axis {
      * @description Generate the x-axis config
      */
     genXAxisConfig(config) {
-        const axisConfig = {type: this.type};
+        const axisConfig = {
+            type: this.type === 'table' ? 'category' : this.type,
+        };
 
         if (this.categories !== undefined) {
             axisConfig.categories = this.getCategories();
@@ -386,4 +397,4 @@ export class Axis {
 
         return config;
     }
-}
+}

client/charts/SDChart/Chart.js

@@ -1,6 +1,7 @@
 import {forEach, get, sum, drop, cloneDeep} from 'lodash';
 
 import {Axis} from './Axis';
+import {convertHtmlStringToText} from '../../utils';
 
 /**
  * @ngdoc class
@@ -656,9 +657,30 @@ export class Chart {
             rows: rows,
             title: this.getTitle(),
             subtitle: this.getSubtitle(),
+            genCSV: this.genCSVFromTable.bind(this),
         };
     }
 
+    genCSVFromTable() {
+        let csv = '';
+
+        csv += `"${this.config.headers.join('","')}"\n`;
+        this.config.rows.forEach((row) => {
+            let text = row.map((cell) => {
+                let cellText = convertHtmlStringToText(cell);
+
+                return typeof cellText === 'number' ?
+                    cellText :
+                    `"${cellText.replace(/"/g, '""')}"`;
+            })
+                .join(',');
+
+            csv += text + '\n';
+        });
+
+        return csv.replace(/\n/g, '\r\n');
+    }
+
     /**
      * @ngdoc method
      * @name SDChart.Chart#genMultiTableConfig
@@ -669,8 +691,7 @@ export class Chart {
         const axis = this.axis[0];
 
         const headers = [axis.xTitle].concat(
-            axis.series.map((series) => series.getName()),
-            'Total'
+            axis.series.map((series) => series.getName())
         );
 
         const rows = axis.getTranslatedCategories().map((category) => ([category]));
@@ -681,13 +702,17 @@ export class Chart {
             });
         });
 
-        rows.forEach((row) => {
-            row.push(
-                sum(
-                    drop(row)
-                )
-            );
-        });
+        if (axis.includeTotal) {
+            headers.push('Total');
+
+            rows.forEach((row) => {
+                row.push(
+                    sum(
+                        drop(row)
+                    )
+                );
+            });
+        }
 
         return {
             id: this.id,
@@ -699,6 +724,7 @@ export class Chart {
             rows: rows,
             title: this.getTitle(),
             subtitle: this.getSubtitle(),
+            genCSV: this.genCSVFromTable.bind(this),
         };
     }
 

client/charts/SDChart/Series.js

@@ -325,6 +325,10 @@ export class Series {
             type: this.type !== undefined ? this.type : this.axis.defaultChartType || 'bar',
         };
 
+        if (series.type === 'table') {
+            series.type = 'bar';
+        }
+
         this.setDataConfig(series);
         this.setPointConfig(series);
         this.setStyleConfig(series);

client/charts/SDChart/tests/SDChart.Chart.spec.js

@@ -145,4 +145,66 @@ describe('SDChart.Chart', () => {
             fullHeight: true,
         }));
     });
+
+    it('can generate CSV from table', () => {
+        const chart = new SDChart.Chart({
+            id: 'test_table',
+            title: 'Testing Table to CSV',
+            chartType: 'table',
+        });
+
+        const axis = chart.addAxis()
+            .setOptions({
+                dfaultChartType: 'table',
+                xTitle: 'Sent',
+                includeTotal: false,
+                categories: [
+                    '11/10/2019 15:43',
+                    '12/10/2019 15:43',
+                    '13/10/2019 15:43',
+                ],
+            });
+
+        axis.addSeries()
+            .setOptions({
+                name: 'Slugline',
+                data: [
+                    'Cri Aust',
+                    'Test',
+                    'Three',
+                ],
+            });
+
+        axis.addSeries()
+            .setOptions({
+                name: 'Version',
+                data: [
+                    1,
+                    3,
+                    7,
+                ],
+            });
+
+        axis.addSeries()
+            .setOptions({
+                name: 'Reason',
+                data: [
+                    '<div><p>Single line text</p></div>',
+                    '<div><p>Multi line text</p><p>second line<br />third "line" test</p></div>',
+                    '',
+                ],
+            });
+
+        const config = chart.genConfig();
+        const csv = config.genCSV();
+
+        expect(csv).toBe('"Sent","Slugline","Version","Reason"\r\n' +
+            '"11/10/2019 15:43","Cri Aust",1,"Single line text"\r\n' +
+            '"12/10/2019 15:43","Test",3,"Multi line text\r\n' +
+            '\r\n' +
+            'second line\r\n' +
+            'third ""line"" test"\r\n' +
+            '"13/10/2019 15:43","Three",7,""\r\n'
+        );
+    });
 });

client/charts/directives/Chart.js

@@ -72,7 +72,7 @@ export function Chart(chartManager, $timeout, $sce, _) {
              * @description Using the chartManager, download the chart data as a CSV file
              */
             scope.downloadAsCSV = function() {
-                chartManager.downloadCSV(scope.config.id, 'chart.csv');
+                chartManager.downloadCSV(scope.config);
             };
 
             /**

client/charts/services/ChartManager.js

@@ -103,13 +103,18 @@ export function ChartManager(_, Highcharts, notify) {
     /**
      * @ngdoc method
      * @name ChartManager#downloadCSV
-     * @param {String} id - The ID assocaited with the chart instance
-     * @param {String} filename - The name of the downloaded file
+     * @param {Object} config - The chart config object, containing the ID of the ID associated with the chart instance
      * @description Converts the chart data to a CSV string, then downloads as a CSV file
      */
-    this.downloadCSV = function(id, filename) {
+    this.downloadCSV = function(config) {
+        const id = config.id;
+        const filename = `${id}.csv`;
+
         if (_.get(this.charts, `${id}.getCSV`)) {
-            const csv = this.charts[id].getCSV();
+            // Either use a custom defined genCSV or the one from Highcharts
+            const csv = config.genCSV ?
+                config.genCSV() :
+                this.charts[id].getCSV();
             const link = document.createElement('a');
 
             link.setAttribute('href', 'data:text/text;charset=utf-8,' + encodeURIComponent(csv));

client/charts/tests/ChartConfig.spec.js

@@ -313,6 +313,7 @@ describe('chartConfig', () => {
             chart: {type: 'column'},
             title: 'Tables',
             subtitle: 'For Today',
+            genCSV: jasmine.any(Function),
             xAxis: [{
                 title: {text: 'Category'},
                 categories: ['Basketball', 'Advisories', 'Cricket'],
@@ -397,6 +398,7 @@ describe('chartConfig', () => {
             chart: {type: 'column'},
             title: 'Tables',
             subtitle: 'For Today',
+            genCSV: jasmine.any(Function),
             xAxis: [{
                 title: {text: 'Category'},
                 categories: ['Cricket', 'Basketball', 'Advisories'],

client/saved_reports/directives/SaveGenerateReport.js

@@ -43,17 +43,17 @@ export function SaveGenerateReport(_, privileges, savedReports) {
                 // and translations from the current report controller
                 if (showSaveForm) {
                     scope.getReportParams()
-                    .then((params) => {
-                        scope.currentParams = Object.assign(
-                            {},
-                            params,
-                            {
-                                name: scope.currentTemplate.name,
-                                description: scope.currentTemplate.description,
-                            }
-                        );
-                        scope.showSaveForm = showSaveForm;
-                    });
+                        .then((params) => {
+                            scope.currentParams = Object.assign(
+                                {},
+                                params,
+                                {
+                                    name: scope.currentTemplate.name,
+                                    description: scope.currentTemplate.description,
+                                }
+                            );
+                            scope.showSaveForm = showSaveForm;
+                        });
                 } else {
                     scope.showSaveForm = showSaveForm;
                     scope.currentParams = {};

client/utils.js

@@ -276,3 +276,55 @@ export const compileAndGetHTML = ($compile, scope, template, data = {}) => {
 
     return html;
 };
+
+/**
+ * Utility to select and copy the text within a parent node
+ * @param {Node} node - The parent node used to select all text from
+ * @returns {string}
+ */
+export const getTextFromDOMNode = (node) => {
+    let innerText;
+
+    if (typeof document.selection !== 'undefined' && typeof document.body.createTextRange !== 'undefined') {
+        const range = document.body.createTextRange();
+
+        range.moveToElementText(node);
+        innerText = range.text;
+    } else if (typeof window.getSelection !== 'undefined' && typeof document.createRange !== 'undefined') {
+        const selection = window.getSelection();
+
+        selection.selectAllChildren(node);
+        innerText = '' + selection;
+        selection.removeAllRanges();
+    }
+    return innerText;
+};
+
+/**
+ * Utility to convert text into HTML DOM Nodes, then select and return the text shown
+ * This replicates the user highlighting the text within those nodes, using the Browser
+ * to do the grunt work for us.
+ * @param {String|Number} data - The html in string format, i.e. '<div><p>testing</p></div>'
+ * @returns {string|Number}
+ */
+export const convertHtmlStringToText = (data) => {
+    if (typeof data !== 'string' || !data.startsWith('<')) {
+        return data;
+    }
+
+    const node = document.createElement('div');
+    let text;
+
+    node.innerHTML = data;
+
+    // Attach the node to the document before selecting
+    // This is required otherwise the browser won't be able to
+    // select the text
+    document.body.appendChild(node);
+    text = getTextFromDOMNode(node);
+
+    // Make sure we clean up after adding the node to the document
+    document.body.removeChild(node);
+
+    return text;
+};

package.json

@@ -1,6 +1,6 @@
 {
   "name": "superdesk-analytics",
-  "version": "1.7.2",
+  "version": "1.7.3",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -25,7 +25,7 @@
     "ts-loader": "3.5.0"
   },
   "devDependencies": {
-    "eslint": "^3.7.1",
+    "eslint": "^4.19.1",
     "eslint-plugin-jasmine": "^1.8.1",
     "eslint-plugin-angular": "^1.6.1",
     "jasmine-core": "^2.99.1",
@@ -42,7 +42,7 @@
     "superdesk-core": "superdesk/superdesk-client-core#1.30",
     "angular-embed": "superdesk/angular-embed#d75968e",
     "angular-embedly": "Urigo/angular-embedly#0.0.8",
-    "jquery": "3.3.1",
+    "jquery": "^3.4.1",
     "jquery.gridster": "dustmoo/gridster.js#c306335397816beceb74e4a176067baef34a0359",
     "rangy": "1.3.0",
     "typescript-eslint-parser": "^18.0.0"

server/analytics/__init__.py

@@ -30,7 +30,7 @@
 from superdesk.celery_app import celery
 from superdesk.default_settings import celery_queue, crontab
 
-__version__ = '1.7.2'
+__version__ = '1.7.3'
 
 
 def init_schedule_task(app):