Merge pull request #2577 from GordonSmith/DDL_JAVA

fix:  Add extra references to assist the DDL2 schema generation

packages/ddl-shim/src/ddl2.ts

@@ -5,12 +5,13 @@ export type IServiceType = "wuresult" | "hipie" | "roxie";
 export type IDatasourceType = IServiceType | "logicalfile" | "form" | "databomb";
 export type DatasourceType = ILogicalFile | IForm | IDatabomb | IWUResult | IHipieService | IRoxieService;
 
-export type IPrimativeFieldType = "boolean" | "number" | "number64" | "string";
-export type IFieldType = IPrimativeFieldType | "range" | "dataset" | "object";
+export type IPrimative = boolean | number | string;
+export type IRange = [undefined | IPrimative, undefined | IPrimative];
+export type IFieldType = "boolean" | "number" | "number64" | "string" | "range" | "dataset" | "object";
 export interface IField {
     id: string;
     type: IFieldType;
-    default?: boolean | number | string | [undefined | IPrimativeFieldType, undefined | IPrimativeFieldType] | IField[];
+    default?: IPrimative | IRange | IField[];
     children?: IField[];
 }
 
@@ -262,7 +263,47 @@ export interface IView {
 
 //  DDL  ======================================================================
 export interface Schema {
-    version: "0.0.21";
+    version: "0.0.22";
     datasources: DatasourceType[];
     dataviews: IView[];
+    //  The following defs are only provided to assist the Java code generation (from the the generated schema)  ---
+    defs?: {
+        fieldTypes: {
+            field: IField;
+            primative: IPrimative;
+            range: IRange;
+        };
+        datasourceTypes: {
+            datasource: IDatasource;
+            logicalFile: ILogicalFile;
+            form: IForm;
+            databomb: IDatabomb;
+            wuresult: IWUResult;
+            hipieService: IHipieService;
+            roxieService: IRoxieService;
+        };
+        datasourceRefTypes: {
+            wuResultRef: IWUResultRef;
+            roxieServiceRef: IRoxieServiceRef;
+        };
+        activityTypes: {
+            filter: IFilter;
+            project: IProject;
+            groupby: IGroupBy;
+            sort: ISort;
+            limit: ILimit;
+            mappings: IMappings;
+        };
+        aggregateTypes: {
+            aggregate: IAggregate;
+            count: ICount;
+        };
+        transformationTypes: {
+            equals: IEquals;
+            calculated: ICalculated;
+            scale: IScale;
+            template: ITemplate;
+            map: IMap;
+        };
+    };
 }

packages/ddl-shim/src/upgrade.ts

@@ -544,7 +544,7 @@ class DDLUpgrade {
                     return {
                         type: "range" as DDL2.IFieldType,
                         id: field.id,
-                        default: field.properties.default ? field.properties.default as [DDL2.IPrimativeFieldType, DDL2.IPrimativeFieldType] : [undefined, undefined]
+                        default: field.properties.default ? field.properties.default as [DDL2.IPrimative, DDL2.IPrimative] : [undefined, undefined]
                     };
                 case "dataset":
                     return {
@@ -581,7 +581,7 @@ class DDLUpgrade {
 
     output2output(output: DDL1.IOutput, target: DDL2.OutputDict) {
         target[output.id] = {
-            fields: []// this.filters2fields(output.filter)
+            fields: this.filters2fields(output.filter)
         };
     }
 
@@ -592,11 +592,11 @@ class DDLUpgrade {
             return idParts.length === 1 || idParts[1] === "range";
         }).map(filter => {
             const idParts = filter.fieldid.split("-");
-            return {
-                type: "string" as DDL2.IFieldType,
+            const retVal: DDL2.IField = {
                 id: idParts[0],
-                default: ""
+                type: "string"
             };
+            return retVal;
         });
     }
 
@@ -626,7 +626,7 @@ class DDLUpgrade {
 
     write(): DDL2.Schema {
         return {
-            version: "0.0.21",
+            version: "0.0.22",
             datasources: this.writeDatasources(),
             dataviews: this.writeDataviews()
         };

packages/marshaller/src/dashy.ts

@@ -1,16 +1,19 @@
 import { JSEditor, JSONEditor } from "@hpcc-js/codemirror";
 import { PropertyExt, Utility, Widget } from "@hpcc-js/common";
-import { DDL1, ddl2Schema, upgrade as ddlUpgrade } from "@hpcc-js/ddl-shim";
+import { DDL1, ddl2Schema, upgrade } from "@hpcc-js/ddl-shim";
 import { DatasourceTable } from "@hpcc-js/dgrid";
 import { Graph } from "@hpcc-js/graph";
 import { PropertyEditor } from "@hpcc-js/other";
 import { CommandPalette, CommandRegistry, ContextMenu, SplitPanel, TabPanel } from "@hpcc-js/phosphor";
+import { scopedLogger } from "@hpcc-js/util";
 import { Activity, DatasourceAdapt } from "./ddl2/activities/activity";
 import { Dashboard, IDashboardPersist } from "./ddl2/dashboard";
 import { DDLEditor } from "./ddl2/ddleditor";
 import { GraphAdapter } from "./ddl2/graphadapter";
 import { Element, ElementContainer } from "./ddl2/model";
 
+const logger = scopedLogger("marshaller/dashy");
+
 import "../src/dashy.css";
 
 class Palette extends PropertyExt {
@@ -112,16 +115,22 @@ export class Dashy extends SplitPanel {
         this._elementContainer.refresh();
     }
 
-    restore(json: IDashboardPersist) {
+    restore(json: IDashboardPersist): Promise<void> {
         this._elementContainer.clear();
         this._dashboard.restore(json);
-        this._elementContainer.refresh();
-        this._dashboard.render();
+        return Promise.all([
+            this._elementContainer.refresh(),
+            this._dashboard.renderPromise()
+        ]).then(promises => {
+            for (const error of this._elementContainer.validate()) {
+                logger.warning(error.elementID + " (" + error.source + "):  " + error.msg);
+            }
+        });
     }
 
     importV1DDL(ddl: DDL1.DDLSchema, baseUrl: string, wuid?: string) {
         this._ddlv1.json(ddl);
-        const ddl2 = ddlUpgrade(ddl, baseUrl, wuid);
+        const ddl2 = upgrade(ddl, baseUrl, wuid);
         this._ddlv2.json(ddl2);
         this._tabDDL
             .addWidget(this._ddlv1, "v1")
@@ -423,8 +432,6 @@ export class Dashy extends SplitPanel {
                     const reader = new FileReader();
                     reader.onload = (function (theFile) {
                         return function (e) {
-                            console.log("e readAsText = ", e);
-                            console.log("e readAsText target = ", e.target);
                             try {
                                 const json = JSON.parse(e.target.result);
                                 if (json.visualizationversion) {

packages/marshaller/src/ddl2/activities/databomb.ts

@@ -50,10 +50,9 @@ export class Databomb extends Activity {
         for (row0 of this._jasonData) {
             const retVal: DDL2.IField[] = [];
             for (const key in row0) {
-                const rowType: DDL2.IPrimativeFieldType = typeof row0[key] as DDL2.IPrimativeFieldType;
                 retVal.push({
                     id: key,
-                    type: rowType
+                    type: typeof row0[key] as DDL2.IFieldType
                 });
             }
             return retVal;
@@ -134,7 +133,7 @@ export class Form extends Activity {
             retVal.push(
                 {
                     id: key,
-                    type: typeof row0[key] as DDL2.IPrimativeFieldType,
+                    type: typeof row0[key] as DDL2.IFieldType,
                     default: row0[key]
                 });
         }

packages/marshaller/src/ddl2/activities/sort.ts

@@ -136,7 +136,6 @@ export class Sort extends Activity {
         }
 
         if (sortByArr.length) {
-            console.log("Sort Len:  " + data.length);
             return [...data].sort((l: any, r: any) => {
                 for (const item of sortByArr) {
                     const retVal2 = item.compare(l[item.id], r[item.id]);

packages/marshaller/src/ddl2/ddl.ts

@@ -1,5 +1,6 @@
 import { MultiChartPanel } from "@hpcc-js/composite";
 import { DDL2 } from "@hpcc-js/ddl-shim";
+import { scopedLogger } from "@hpcc-js/util";
 import { Activity, ActivityPipeline, ReferencedFields } from "./activities/activity";
 import { Databomb, Form } from "./activities/databomb";
 import { DSPicker } from "./activities/dspicker";
@@ -14,6 +15,8 @@ import { WUResult } from "./activities/wuresult";
 import { Dashboard } from "./dashboard";
 import { Element, ElementContainer } from "./model";
 
+const logger = scopedLogger("marshaller/ddl2/ddl");
+
 function mergeFieldArray(targetArr: DDL2.IField[], sourceArr: DDL2.IField[]): DDL2.IField[] {
     const existing: string[] = targetArr.map(f => f.id);
     return targetArr.concat(sourceArr.filter(f => existing.indexOf(f.id) < 0));
@@ -301,16 +304,22 @@ export class DDLAdapter {
         this.readDatasource(ddlDS, ds);
         const dsDetails = ds instanceof DSPicker ? ds.details() : ds;
         if (dsDetails instanceof WUResult && DDL2.isWUResultRef(ddlDSRef)) {
-            dsDetails.resultName(ddlDSRef.output);
+            dsDetails
+                .resultName(ddlDSRef.output)
+                .responseFields((ddlDS as DDL2.IWUResult).outputs[ddlDSRef.output].fields)
+                ;
         } else if (dsDetails instanceof RoxieRequest && DDL2.isRoxieServiceRef(ddlDSRef)) {
-            dsDetails.resultName(ddlDSRef.output);
-            dsDetails.request(ddlDSRef.request.map(rf => {
-                return new Param(this._elementContainer)
-                    .source(rf.source)
-                    .remoteField(rf.remoteFieldID)
-                    .localField(rf.localFieldID)
-                    ;
-            }));
+            dsDetails
+                .resultName(ddlDSRef.output)
+                .request(ddlDSRef.request.map(rf => {
+                    return new Param(this._elementContainer)
+                        .source(rf.source)
+                        .remoteField(rf.remoteFieldID)
+                        .localField(rf.localFieldID)
+                        ;
+                }))
+                .responseFields((ddlDS as DDL2.IRoxieService).outputs[ddlDSRef.output].fields)
+                ;
         }
         return this;
     }
@@ -330,7 +339,7 @@ export class DDLAdapter {
             } else if (activity instanceof Mappings) {
                 return this.writeMappings(activity);
             } else {
-                console.log(`Unknown activity type:  ${activity.classID()}`);
+                logger.warning(`Unknown activity type:  ${activity.classID()}`);
             }
         }).filter(activity => !!activity);
     }
@@ -341,7 +350,7 @@ export class DDLAdapter {
         const chart = element.chart();
         for (const prop of chart.publishedProperties()) {
             if (prop.id === "fields") continue;
-            console.log(`${prop.id}=>${(chart as any)[`${prop.id}`]()}`);
+            logger.debug(`${prop.id}=>${(chart as any)[`${prop.id}`]()}`);
             if ((chart as any)[`${prop.id}_modified`]()) {
                 const val = (chart as any)[`${prop.id}`]();
                 if (!(val instanceof Object)) {
@@ -435,7 +444,7 @@ export class DDLAdapter {
     write(): DDL2.Schema {
         this._dsDedup.clear();
         const retVal: DDL2.Schema = {
-            version: "0.0.21",
+            version: "0.0.22",
             datasources: this.writeDatasources(),
             dataviews: this.writeDDLViews()
         };