Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Generate date fact for DATEs without schema reference.

  • Loading branch information...
commit 8b132c17e2ad64b25a86b7f9eb0ac3aa34f5af9b 1 parent d41b7ff
ZD authored
View
76 backend/src/main/java/com/gooddata/connector/DateColumnsExtender.java
@@ -54,19 +54,15 @@ public DateColumnsExtender(SourceSchema schema) {
dates = schema.getDates();
for(int i= 0; i < dates.size(); i++) {
SourceColumn c = dates.get(i);
- String sr = c.getSchemaReference();
- // only process the dates with a schema reference
- if(sr != null && sr.length() > 0) {
- dateColumnIndexes.add(schema.getColumnIndex(c));
- String fmt = c.getFormat();
- if(fmt == null || fmt.length() <= 0) {
- if(c.isDatetime())
- fmt = Constants.DEFAULT_DATETIME_FMT_STRING;
- else
- fmt = Constants.DEFAULT_DATE_FMT_STRING;
- }
- dateColumnFormats.add(DateTimeFormat.forPattern(fmt));
+ dateColumnIndexes.add(schema.getColumnIndex(c));
+ String fmt = c.getFormat();
+ if(fmt == null || fmt.length() <= 0) {
+ if(c.isDatetime())
+ fmt = Constants.DEFAULT_DATETIME_FMT_STRING;
+ else
+ fmt = Constants.DEFAULT_DATE_FMT_STRING;
}
+ dateColumnFormats.add(DateTimeFormat.forPattern(fmt));
}
}
@@ -81,13 +77,10 @@ public DateColumnsExtender(SourceSchema schema) {
List<String> rowExt = new ArrayList<String>();
for(int i= 0; i < dates.size(); i++) {
SourceColumn c = dates.get(i);
- String sr = c.getSchemaReference();
- if(sr != null && sr.length() > 0) {
- rowExt.add(StringUtil.toIdentifier(c.getName()) + N.DT_SLI_SFX);
- if(c.isDatetime()) {
- rowExt.add(StringUtil.toIdentifier(c.getName()) + N.TM_SLI_SFX);
- rowExt.add(N.TM_PFX+StringUtil.toIdentifier(c.getName())+"_"+N.ID);
- }
+ rowExt.add(StringUtil.toIdentifier(c.getName()) + N.DT_SLI_SFX);
+ if(c.isDatetime()) {
+ rowExt.add(StringUtil.toIdentifier(c.getName()) + N.TM_SLI_SFX);
+ rowExt.add(N.TM_PFX+StringUtil.toIdentifier(c.getName())+"_"+N.ID);
}
}
if(rowExt.size() > 0)
@@ -109,32 +102,22 @@ public DateColumnsExtender(SourceSchema schema) {
List<String> rowExt = new ArrayList<String>();
for(int i = 0; i < dateColumnIndexes.size(); i++) {
SourceColumn c = dates.get(i);
- String sr = c.getSchemaReference();
- if(sr != null && sr.length() > 0) {
- String dateValue = row[dateColumnIndexes.get(i)];
- if(dateValue != null && dateValue.trim().length()>0) {
- try {
- DateTimeFormatter formatter = dateColumnFormats.get(i);
- DateTime dt = formatter.parseDateTime(dateValue);
- Days ds = Days.daysBetween(base, dt);
- rowExt.add(Integer.toString(ds.getDays() + 1));
- if(c.isDatetime()) {
- int ts = dt.getSecondOfDay();
- rowExt.add(Integer.toString(ts));
- String scs = Integer.toString(ts);
- rowExt.add((scs.length()>1)?(scs):("0"+scs));
- }
- }
- catch (IllegalArgumentException e) {
- l.debug("Can't parse date "+dateValue);
- rowExt.add("");
- if(c.isDatetime()) {
- rowExt.add("");
- rowExt.add("00");
- }
+ String dateValue = row[dateColumnIndexes.get(i)];
+ if(dateValue != null && dateValue.trim().length()>0) {
+ try {
+ DateTimeFormatter formatter = dateColumnFormats.get(i);
+ DateTime dt = formatter.parseDateTime(dateValue);
+ Days ds = Days.daysBetween(base, dt);
+ rowExt.add(Integer.toString(ds.getDays() + 1));
+ if(c.isDatetime()) {
+ int ts = dt.getSecondOfDay();
+ rowExt.add(Integer.toString(ts));
+ String scs = Integer.toString(ts);
+ rowExt.add((scs.length()>1)?(scs):("0"+scs));
}
}
- else {
+ catch (IllegalArgumentException e) {
+ l.debug("Can't parse date "+dateValue);
rowExt.add("");
if(c.isDatetime()) {
rowExt.add("");
@@ -142,6 +125,13 @@ public DateColumnsExtender(SourceSchema schema) {
}
}
}
+ else {
+ rowExt.add("");
+ if(c.isDatetime()) {
+ rowExt.add("");
+ rowExt.add("00");
+ }
+ }
}
if(rowExt.size() > 0)
return mergeArrays(row, rowExt.toArray(new String[]{}));
View
22 backend/src/main/java/com/gooddata/modeling/generator/MaqlGenerator.java
@@ -549,19 +549,21 @@ public String generateMaqlDdlAdd() {
String sfn = StringUtil.toIdentifier(folder);
folderStatement = ", FOLDER {ffld." + sfn + "}";
}
- String stat = "";
+ String stat = "CREATE FACT {" + identifier + "} VISUAL(TITLE \"" + lcn
+ + " (Date)\"" + folderStatement + ") AS {" + getFactTableName() + "."+N.DT_PFX + scn +"};\n"
+ + "ALTER DATATYPE {" + getFactTableName() + "."+N.DT_PFX + scn +"} INT;\n"
+ + "ALTER DATASET {" + schema.getDatasetName() + "} ADD {"+ identifier + "};\n\n";
+ if(includeTime) {
+ stat += "CREATE FACT {" + N.TM + "." + identifier + "} VISUAL(TITLE \"" + lcn
+ + " (Time)\"" + folderStatement + ") AS {" + getFactTableName() + "."+N.TM_PFX + scn +"};\n"
+ + "ALTER DATASET {" + schema.getDatasetName() + "} ADD {"+ N.TM + "." + identifier + "};\n\n";
+ }
if(reference != null && reference.length() > 0) {
reference = StringUtil.toIdentifier(reference);
- stat += "CREATE FACT {" + identifier + "} VISUAL(TITLE \"" + lcn
- + " (Date)\"" + folderStatement + ") AS {" + getFactTableName() + "."+N.DT_PFX + scn +"};\n"
- + "ALTER DATASET {" + schema.getDatasetName() + "} ADD {"+ identifier + "};\n\n";
stat += "# CONNECT THE DATE TO THE DATE DIMENSION\n";
stat += "ALTER ATTRIBUTE {"+reference+"."+N.DT_ATTR_NAME+"} ADD KEYS {"+getFactTableName() +
"."+N.DT_PFX + scn + "_"+N.ID+"};\n\n";
if(includeTime) {
- stat += "CREATE FACT {" + N.TM + "." + identifier + "} VISUAL(TITLE \"" + lcn
- + " (Time)\"" + folderStatement + ") AS {" + getFactTableName() + "."+N.TM_PFX + scn +"};\n"
- + "ALTER DATASET {" + schema.getDatasetName() + "} ADD {"+ N.TM + "." + identifier + "};\n\n";
stat += "# CONNECT THE TIME TO THE TIME DIMENSION\n";
stat += "ALTER ATTRIBUTE {"+N.TM_ATTR_NAME+reference+"} ADD KEYS {"+getFactTableName() +
"."+N.TM_PFX + scn + "_"+N.ID+"};\n\n";
@@ -571,13 +573,15 @@ public String generateMaqlDdlAdd() {
}
public String generateMaqlDdlDrop() {
- String script = "";
String reference = column.getSchemaReference();
boolean includeTime = column.isDatetime();
+ String script = "DROP {" + identifier + "};\n";
+ if(includeTime) {
+ script += "DROP {" + N.TM_PFX + identifier + "};\n";
+ }
if(reference != null && reference.length() > 0) {
reference = StringUtil.toIdentifier(reference);
script += "# DISCONNECT THE DATE DIMENSION\n";
- script += "DROP {" + identifier + "};\n";
script += "ALTER ATTRIBUTE {"+reference+"."+N.DT_ATTR_NAME+"} DROP KEYS {"+getFactTableName() +
"."+N.DT_PFX + scn + "_"+N.ID+"};\n\n";
if(includeTime) {
View
9 connector/src/main/java/com/gooddata/connector/CsvConnector.java
@@ -107,8 +107,15 @@ public void extract(String dir) throws IOException {
while (row != null) {
rowCnt++;
if(row.length != colCnt) {
- throw new InvalidParameterException("The delimited file "+this.getDataFile()+" has different number of columns than " +
+ if(!(row.length == 1 && row[0].length() == 0)) {
+ // this is not empty line
+ throw new InvalidParameterException("The delimited file "+this.getDataFile()+" has different number of columns than " +
"it's configuration file. Row="+rowCnt);
+ }
+ else {
+ row = cr.readNext();
+ continue;
+ }
}
if(identityColumn>=0) {
String key = "";
Please sign in to comment.
Something went wrong with that request. Please try again.