Skip to content
This repository has been archived by the owner on Apr 10, 2024. It is now read-only.

Commit

Permalink
Fixed CSV testing issues
Browse files Browse the repository at this point in the history
  • Loading branch information
ZD committed Oct 14, 2010
1 parent 589df5e commit dea6051
Show file tree
Hide file tree
Showing 9 changed files with 58 additions and 9 deletions.
6 changes: 3 additions & 3 deletions backend/pom.xml
View file
Expand Up @@ -103,8 +103,8 @@
<version>1.4</version>
</dependency>
<dependency>
<groupId>net.sf.opencsv</groupId>
<artifactId>opencsv</artifactId>
</dependency>
<groupId>net.sf.opencsv</groupId>
<artifactId>opencsv</artifactId>
</dependency>
</dependencies>
</project>
1 change: 1 addition & 0 deletions backend/src/main/java/com/gooddata/connector/Constants.java
View file
Expand Up @@ -34,6 +34,7 @@ public class Constants {
public final static String[] DISCARD_CHARS = {"$", "%", ",", "(", ")", "?", "?", "?"};

public final static int MAX_TABLE_NAME_LENGTH = 44;
public final static int MAX_SCHEMA_NAME_LENGTH = 24;

public final static String DEFAULT_DATE_FMT_STRING = "yyyy-MM-dd";
public final static String DEFAULT_DATE_LABEL = "date.mdyy";
Expand Down
1 change: 1 addition & 0 deletions backend/src/main/java/com/gooddata/modeling/model/SourceSchema.java
View file
Expand Up @@ -35,6 +35,7 @@
import java.util.ArrayList;
import java.util.List;

import com.gooddata.connector.Constants;
import com.gooddata.exception.ModelException;
import com.gooddata.util.StringUtil;
import com.thoughtworks.xstream.XStream;
Expand Down
4 changes: 2 additions & 2 deletions cli-distro/examples/jdbc/fundamentals.txt
View file
Expand Up @@ -9,10 +9,10 @@ StoreProject(fileName="examples/jdbc/pid");

# Generate config file. This command is commented out as we have done that already.
# If you change the JDBC input data you need to re-generate the config file
#GenerateJdbcConfig(name="FUNDAMENTALS",configFile="examples/jdbc/fundamentals.config.xml",driver="org.apache.derby.jdbc.EmbeddedDriver",url="jdbc:derby:../examples/jdbc/fundamentals",query="SELECT * FROM FUNDAMENTALS");
#GenerateJdbcConfig(name="FUNDAMENTALS",configFile="examples/jdbc/fundamentals.config.xml",driver="org.apache.derby.jdbc.EmbeddedDriver",url="jdbc:derby:examples/jdbc/fundamentals",query="SELECT * FROM FUNDAMENTALS");

# load JDBC data
LoadJdbc(configFile="examples/jdbc/fundamentals.config.xml",driver="org.apache.derby.jdbc.EmbeddedDriver",url="jdbc:derby:../examples/jdbc/fundamentals",query="SELECT * FROM FUNDAMENTALS");
LoadJdbc(configFile="examples/jdbc/fundamentals.config.xml",driver="org.apache.derby.jdbc.EmbeddedDriver",url="jdbc:derby:examples/jdbc/fundamentals",query="SELECT * FROM FUNDAMENTALS");

# Generate the data model MAQL script
GenerateMaql(maqlFile="examples/jdbc/fundamentals.maql");
Expand Down
10 changes: 8 additions & 2 deletions cli-distro/pom.xml
View file
Expand Up @@ -55,13 +55,19 @@
<dependency>
<groupId>postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>8.4-701.jdbc4</version>
</dependency>
<dependency>
<groupId>net.sourceforge.jtds</groupId>
<artifactId>jtds</artifactId>
<version>1.2.4</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
</dependency>
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
</dependency>
</dependencies>
<build>
<plugins>
Expand Down
2 changes: 0 additions & 2 deletions cli-distro/tests/drop_snapshots/cmd.txt
View file
Expand Up @@ -54,8 +54,6 @@ ExecuteMaql(maqlFile="tests/drop_snapshots/salary.maql");
# Transfer the salary data
TransferLastSnapshot(incremental="false");

DropSnapshots();

# Load the salary data file, using the XML file describing the data
LoadCsv(csvDataFile="tests/drop_snapshots/salary.csv",header="true",configFile="tests/drop_snapshots/salary.xml");

Expand Down
12 changes: 12 additions & 0 deletions connector/src/main/java/com/gooddata/connector/CsvConnector.java
View file
Expand Up @@ -138,6 +138,9 @@ static SourceSchema guessSourceSchema (InputStream configStream, URL dataUrl, St
if (configStream != null) {
srcSchm = SourceSchema.createSchema(configStream);
} else {
int idmax = Constants.MAX_SCHEMA_NAME_LENGTH - 3;
if (name.length() > idmax)
name = name.substring(0, idmax);
srcSchm = SourceSchema.createSchema(name);
}
final int knownColumns = srcSchm.getColumns().size();
Expand All @@ -148,6 +151,9 @@ public String transform(String str) {
int idmax = Constants.MAX_TABLE_NAME_LENGTH - srcSchm.getName().length() - 3; // good enough for 999 long names
if (idorig.length() <= idmax)
return idorig;
if(idmax < 8)
throw new InvalidParameterException("The schema name '"+srcSchm+"' is too long. Please use a name " +
"up to 32 characters.");
return idorig.substring(0, idmax);

}
Expand All @@ -169,6 +175,12 @@ public String transform(String str) {
final SourceColumn sc;
final String identifier = idGen.transform(header);
final String title = titleGen.transform(header);
if(identifier == null || identifier.length() <= 0) {
throw new InvalidParameterException("The CSV header can't contain empty names or names with all non-latin characters.");
}
if(title == null || title.length() <= 0) {
throw new InvalidParameterException("The CSV header can't contain empty names or names with all non-latin characters.");
}
if (defaultLdmType != null) {
sc = new SourceColumn(identifier, defaultLdmType, title, folder);
} else {
Expand Down
7 changes: 7 additions & 0 deletions connector/src/main/java/com/gooddata/csv/DataTypeGuess.java
View file
Expand Up @@ -35,6 +35,7 @@
import java.util.Set;

import com.gooddata.connector.Constants;
import com.gooddata.exception.InvalidParameterException;
import com.gooddata.util.CSVReader;

import com.gooddata.modeling.model.SourceColumn;
Expand Down Expand Up @@ -163,6 +164,9 @@ public SourceColumn[] guessCsvSchema(CSVReader cr) throws IOException {
int countdown = 1000;
while(row != null && countdown-- >0) {
for(int i=0; i< row.length; i++) {
if(i >= excludedColumnTypes.size())
throw new InvalidParameterException("The CSV file contains rows with different number of columns." +
" Quitting.");
Set<String> types = excludedColumnTypes.get(i);
String value = row[i];
String dateFormat = getDateFormat(value);
Expand All @@ -186,6 +190,9 @@ public SourceColumn[] guessCsvSchema(CSVReader cr) throws IOException {
if (defaultLdmType != null)
ldmType = defaultLdmType;
else {
if(i >= excludedColumnTypes.size())
throw new InvalidParameterException("The CSV file contains rows with different number of columns." +
" Quitting.");
final Set<String> excludedColumnType = excludedColumnTypes.get(i);
if(!excludedColumnType.contains(SourceColumn.LDM_TYPE_DATE))
ldmType = SourceColumn.LDM_TYPE_DATE;
Expand Down
24 changes: 24 additions & 0 deletions pom.xml
View file
Expand Up @@ -211,6 +211,30 @@
<groupId>com.ibm.icu</groupId>
<artifactId>icu4j</artifactId>
<version>4.0.1</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.12</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>org.apache.derby</groupId>
<artifactId>derby</artifactId>
<version>10.6.1.0</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>8.4-701.jdbc4</version>
<scope>runtime</scope>
</dependency>
<dependency>
<groupId>net.sourceforge.jtds</groupId>
<artifactId>jtds</artifactId>
<version>1.2.4</version>
<scope>runtime</scope>
</dependency>
</dependencies>
</dependencyManagement>
Expand Down

0 comments on commit dea6051

Please sign in to comment.