Skip to content
This repository has been archived by the owner on Apr 10, 2024. It is now read-only.

Commit

Permalink
Generate date fact for DATEs without schema reference.
Browse files Browse the repository at this point in the history
  • Loading branch information
ZD committed Dec 16, 2010
1 parent d41b7ff commit 8b132c1
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 53 deletions.
76 changes: 33 additions & 43 deletions backend/src/main/java/com/gooddata/connector/DateColumnsExtender.java
View file
Expand Up @@ -54,19 +54,15 @@ public DateColumnsExtender(SourceSchema schema) {
dates = schema.getDates();
for(int i= 0; i < dates.size(); i++) {
SourceColumn c = dates.get(i);
String sr = c.getSchemaReference();
// only process the dates with a schema reference
if(sr != null && sr.length() > 0) {
dateColumnIndexes.add(schema.getColumnIndex(c));
String fmt = c.getFormat();
if(fmt == null || fmt.length() <= 0) {
if(c.isDatetime())
fmt = Constants.DEFAULT_DATETIME_FMT_STRING;
else
fmt = Constants.DEFAULT_DATE_FMT_STRING;
}
dateColumnFormats.add(DateTimeFormat.forPattern(fmt));
dateColumnIndexes.add(schema.getColumnIndex(c));
String fmt = c.getFormat();
if(fmt == null || fmt.length() <= 0) {
if(c.isDatetime())
fmt = Constants.DEFAULT_DATETIME_FMT_STRING;
else
fmt = Constants.DEFAULT_DATE_FMT_STRING;
}
dateColumnFormats.add(DateTimeFormat.forPattern(fmt));
}
}

Expand All @@ -81,13 +77,10 @@ public String[] extendHeader(String[] header) {
List<String> rowExt = new ArrayList<String>();
for(int i= 0; i < dates.size(); i++) {
SourceColumn c = dates.get(i);
String sr = c.getSchemaReference();
if(sr != null && sr.length() > 0) {
rowExt.add(StringUtil.toIdentifier(c.getName()) + N.DT_SLI_SFX);
if(c.isDatetime()) {
rowExt.add(StringUtil.toIdentifier(c.getName()) + N.TM_SLI_SFX);
rowExt.add(N.TM_PFX+StringUtil.toIdentifier(c.getName())+"_"+N.ID);
}
rowExt.add(StringUtil.toIdentifier(c.getName()) + N.DT_SLI_SFX);
if(c.isDatetime()) {
rowExt.add(StringUtil.toIdentifier(c.getName()) + N.TM_SLI_SFX);
rowExt.add(N.TM_PFX+StringUtil.toIdentifier(c.getName())+"_"+N.ID);
}
}
if(rowExt.size() > 0)
Expand All @@ -109,39 +102,36 @@ public String[] extendRow(String[] row) {
List<String> rowExt = new ArrayList<String>();
for(int i = 0; i < dateColumnIndexes.size(); i++) {
SourceColumn c = dates.get(i);
String sr = c.getSchemaReference();
if(sr != null && sr.length() > 0) {
String dateValue = row[dateColumnIndexes.get(i)];
if(dateValue != null && dateValue.trim().length()>0) {
try {
DateTimeFormatter formatter = dateColumnFormats.get(i);
DateTime dt = formatter.parseDateTime(dateValue);
Days ds = Days.daysBetween(base, dt);
rowExt.add(Integer.toString(ds.getDays() + 1));
if(c.isDatetime()) {
int ts = dt.getSecondOfDay();
rowExt.add(Integer.toString(ts));
String scs = Integer.toString(ts);
rowExt.add((scs.length()>1)?(scs):("0"+scs));
}
}
catch (IllegalArgumentException e) {
l.debug("Can't parse date "+dateValue);
rowExt.add("");
if(c.isDatetime()) {
rowExt.add("");
rowExt.add("00");
}
String dateValue = row[dateColumnIndexes.get(i)];
if(dateValue != null && dateValue.trim().length()>0) {
try {
DateTimeFormatter formatter = dateColumnFormats.get(i);
DateTime dt = formatter.parseDateTime(dateValue);
Days ds = Days.daysBetween(base, dt);
rowExt.add(Integer.toString(ds.getDays() + 1));
if(c.isDatetime()) {
int ts = dt.getSecondOfDay();
rowExt.add(Integer.toString(ts));
String scs = Integer.toString(ts);
rowExt.add((scs.length()>1)?(scs):("0"+scs));
}
}
else {
catch (IllegalArgumentException e) {
l.debug("Can't parse date "+dateValue);
rowExt.add("");
if(c.isDatetime()) {
rowExt.add("");
rowExt.add("00");
}
}
}
else {
rowExt.add("");
if(c.isDatetime()) {
rowExt.add("");
rowExt.add("00");
}
}
}
if(rowExt.size() > 0)
return mergeArrays(row, rowExt.toArray(new String[]{}));
Expand Down
22 changes: 13 additions & 9 deletions backend/src/main/java/com/gooddata/modeling/generator/MaqlGenerator.java
View file
Expand Up @@ -549,19 +549,21 @@ public String generateMaqlDdlAdd() {
String sfn = StringUtil.toIdentifier(folder);
folderStatement = ", FOLDER {ffld." + sfn + "}";
}
String stat = "";
String stat = "CREATE FACT {" + identifier + "} VISUAL(TITLE \"" + lcn
+ " (Date)\"" + folderStatement + ") AS {" + getFactTableName() + "."+N.DT_PFX + scn +"};\n"
+ "ALTER DATATYPE {" + getFactTableName() + "."+N.DT_PFX + scn +"} INT;\n"
+ "ALTER DATASET {" + schema.getDatasetName() + "} ADD {"+ identifier + "};\n\n";
if(includeTime) {
stat += "CREATE FACT {" + N.TM + "." + identifier + "} VISUAL(TITLE \"" + lcn
+ " (Time)\"" + folderStatement + ") AS {" + getFactTableName() + "."+N.TM_PFX + scn +"};\n"
+ "ALTER DATASET {" + schema.getDatasetName() + "} ADD {"+ N.TM + "." + identifier + "};\n\n";
}
if(reference != null && reference.length() > 0) {
reference = StringUtil.toIdentifier(reference);
stat += "CREATE FACT {" + identifier + "} VISUAL(TITLE \"" + lcn
+ " (Date)\"" + folderStatement + ") AS {" + getFactTableName() + "."+N.DT_PFX + scn +"};\n"
+ "ALTER DATASET {" + schema.getDatasetName() + "} ADD {"+ identifier + "};\n\n";
stat += "# CONNECT THE DATE TO THE DATE DIMENSION\n";
stat += "ALTER ATTRIBUTE {"+reference+"."+N.DT_ATTR_NAME+"} ADD KEYS {"+getFactTableName() +
"."+N.DT_PFX + scn + "_"+N.ID+"};\n\n";
if(includeTime) {
stat += "CREATE FACT {" + N.TM + "." + identifier + "} VISUAL(TITLE \"" + lcn
+ " (Time)\"" + folderStatement + ") AS {" + getFactTableName() + "."+N.TM_PFX + scn +"};\n"
+ "ALTER DATASET {" + schema.getDatasetName() + "} ADD {"+ N.TM + "." + identifier + "};\n\n";
stat += "# CONNECT THE TIME TO THE TIME DIMENSION\n";
stat += "ALTER ATTRIBUTE {"+N.TM_ATTR_NAME+reference+"} ADD KEYS {"+getFactTableName() +
"."+N.TM_PFX + scn + "_"+N.ID+"};\n\n";
Expand All @@ -571,13 +573,15 @@ public String generateMaqlDdlAdd() {
}

public String generateMaqlDdlDrop() {
String script = "";
String reference = column.getSchemaReference();
boolean includeTime = column.isDatetime();
String script = "DROP {" + identifier + "};\n";
if(includeTime) {
script += "DROP {" + N.TM_PFX + identifier + "};\n";
}
if(reference != null && reference.length() > 0) {
reference = StringUtil.toIdentifier(reference);
script += "# DISCONNECT THE DATE DIMENSION\n";
script += "DROP {" + identifier + "};\n";
script += "ALTER ATTRIBUTE {"+reference+"."+N.DT_ATTR_NAME+"} DROP KEYS {"+getFactTableName() +
"."+N.DT_PFX + scn + "_"+N.ID+"};\n\n";
if(includeTime) {
Expand Down
9 changes: 8 additions & 1 deletion connector/src/main/java/com/gooddata/connector/CsvConnector.java
View file
Expand Up @@ -107,8 +107,15 @@ public void extract(String dir) throws IOException {
while (row != null) {
rowCnt++;
if(row.length != colCnt) {
throw new InvalidParameterException("The delimited file "+this.getDataFile()+" has different number of columns than " +
if(!(row.length == 1 && row[0].length() == 0)) {
// this is not empty line
throw new InvalidParameterException("The delimited file "+this.getDataFile()+" has different number of columns than " +
"it's configuration file. Row="+rowCnt);
}
else {
row = cr.readNext();
continue;
}
}
if(identityColumn>=0) {
String key = "";
Expand Down

0 comments on commit 8b132c1

Please sign in to comment.