Skip to content

Commit

Permalink
TEIID-3599: Adding support to read data from multiple excel files
Browse files Browse the repository at this point in the history
  • Loading branch information
@rareddy
rareddy committed Feb 19, 2016
1 parent d31fe1b commit 6447d81
Show file tree
Hide file tree
Showing 3 changed files with 81 additions and 13 deletions.
35 changes: 25 additions & 10 deletions ...ors/translator-excel/src/main/java/org/teiid/translator/excel/ExcelMetadataProcessor.java
View file
Expand Up @@ -26,7 +26,6 @@
import java.io.IOException;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;

import javax.resource.ResourceException;

Expand All @@ -38,10 +37,17 @@
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.teiid.logging.LogConstants;
import org.teiid.logging.LogManager;
import org.teiid.metadata.*;
import org.teiid.metadata.Column;
import org.teiid.metadata.Column.SearchType;
import org.teiid.translator.*;
import org.teiid.metadata.ExtensionMetadataProperty;
import org.teiid.metadata.MetadataFactory;
import org.teiid.metadata.Table;
import org.teiid.translator.FileConnection;
import org.teiid.translator.MetadataProcessor;
import org.teiid.translator.TranslatorException;
import org.teiid.translator.TranslatorProperty;
import org.teiid.translator.TranslatorProperty.PropertyType;
import org.teiid.translator.TypeFacility;

public class ExcelMetadataProcessor implements MetadataProcessor<FileConnection> {

Expand All @@ -68,6 +74,12 @@ public void process(MetadataFactory mf, FileConnection conn) throws TranslatorEx
}
try {
File xlsFile = conn.getFile(this.excelFileName);
if (xlsFile.isDirectory()) {
File[] files = xlsFile.listFiles();
if (files.length > 0) {
xlsFile = files[0];
}
}
if (xlsFile.isDirectory() || !xlsFile.exists()) {
throw new TranslatorException(ExcelPlugin.Event.TEIID23005, ExcelPlugin.Util.gs(ExcelPlugin.Event.TEIID23005, xlsFile.getName()));
}
Expand All @@ -85,7 +97,7 @@ else if (extension.equalsIgnoreCase("xlsx")) { //$NON-NLS-1$
int sheetCount = workbook.getNumberOfSheets();
for (int i = 0; i < sheetCount; i++) {
Sheet sheet = workbook.getSheetAt(i);
addTable(mf, sheet, xlsFile.getName());
addTable(mf, sheet, xlsFile.getName(), this.excelFileName);
}
} finally {
xlsFileStream.close();
Expand All @@ -97,7 +109,7 @@ else if (extension.equalsIgnoreCase("xlsx")) { //$NON-NLS-1$
}
}

private void addTable(MetadataFactory mf, Sheet sheet, String xlsName) {
private void addTable(MetadataFactory mf, Sheet sheet, String xlsName, String originalName) {
int firstRowNumber = sheet.getFirstRowNum();
Row headerRow = null;
int firstCellNumber = -1;
Expand Down Expand Up @@ -131,7 +143,7 @@ private void addTable(MetadataFactory mf, Sheet sheet, String xlsName) {
AtomicInteger columnCount = new AtomicInteger();
Table table = mf.addTable(sheet.getSheetName());
table.setNameInSource(sheet.getSheetName());
table.setProperty(ExcelMetadataProcessor.FILE, xlsName);
table.setProperty(ExcelMetadataProcessor.FILE, originalName);

// add implicit row_id column based on row number from excel sheet
Column column = mf.addColumn(ROW_ID, TypeFacility.RUNTIME_NAMES.INTEGER, table);
Expand Down Expand Up @@ -222,7 +234,8 @@ private String getCellType(Cell cell) {
}
}

@TranslatorProperty(display="Header Row Number", category=PropertyType.IMPORT, description="Row number that contains the header information")
@TranslatorProperty(display="Header Row Number", category=PropertyType.IMPORT,
description="Row number that contains the header information")
public int getHeaderRowNumber() {
return headerRowNumber;
}
Expand All @@ -236,7 +249,8 @@ public void setHeaderRowNumber(int headerRowNumber) {
}
}

@TranslatorProperty(display="Data Row Number", category=PropertyType.IMPORT, description="Row number from which data rows start from")
@TranslatorProperty(display = "Data Row Number", category = PropertyType.IMPORT,
description = "Row number from which data rows start from")
public int getDataRowNumber() {
return dataRowNumber;
}
Expand All @@ -250,8 +264,9 @@ public void setDataRowNumber(int dataRowNumber) {
}
}

@TranslatorProperty(display="Excel File", category=PropertyType.IMPORT, description="Name of the Excel file to read metadata from", required=true)
@TranslatorProperty(display="Excel File", category=PropertyType.IMPORT,
description="Name of the Excel file to read metadata from", required=true)
public String getExcelFileName() {
return excelFileName;
}
}
}
20 changes: 20 additions & 0 deletions connectors/translator-excel/src/test/java/org/teiid/translator/excel/TestExcelExecution.java
View file
Expand Up @@ -88,6 +88,26 @@ public void testExecutionNoDataNumberXLS() throws Exception {
assertEquals("[[13, FirstName, LastName, Age], [14, John, Doe, 44.0], [15, Jane, Smith, 40.0], [16, Matt, Liek, 13.0], [17, Sarah, Byne, 10.0], [18, Rocky, Dog, 3.0], [19, Total, null, 110.0]]", results.toString());
}

@Test
public void testFileGlob() throws Exception {
String ddl = "CREATE FOREIGN TABLE Sheet1 (\n" +
" ROW_ID integer OPTIONS (SEARCHABLE 'All_Except_Like', \"teiid_excel:CELL_NUMBER\" 'ROW_ID'),\n" +
" column1 string OPTIONS (SEARCHABLE 'Unsearchable', \"teiid_excel:CELL_NUMBER\" '7'),\n" +
" column2 string OPTIONS (SEARCHABLE 'Unsearchable', \"teiid_excel:CELL_NUMBER\" '8'),\n" +
" column3 string OPTIONS (SEARCHABLE 'Unsearchable', \"teiid_excel:CELL_NUMBER\" '9'),\n" +
" CONSTRAINT PK0 PRIMARY KEY(ROW_ID)\n" +
") OPTIONS (\"teiid_excel:FILE\" '*.xls');";

FileConnection connection = Mockito.mock(FileConnection.class);
File f = Mockito.mock(File.class);
Mockito.stub(f.isDirectory()).toReturn(true);
Mockito.stub(f.listFiles()).toReturn(new File[] {UnitTestUtil.getTestDataFile("names.xls")});
Mockito.stub(connection.getFile("*.xls")).toReturn(f);

ArrayList results = helpExecute(ddl, connection, "select * from Sheet1");
assertEquals("[[13, FirstName, LastName, Age], [14, John, Doe, 44.0], [15, Jane, Smith, 40.0], [16, Matt, Liek, 13.0], [17, Sarah, Byne, 10.0], [18, Rocky, Dog, 3.0], [19, Total, null, 110.0]]", results.toString());
}

@Test
public void testExecutionNoDataNumberXLSX() throws Exception {
String ddl = "CREATE FOREIGN TABLE Sheet1 (\n" +
Expand Down
39 changes: 36 additions & 3 deletions ...translator-excel/src/test/java/org/teiid/translator/excel/TestExcelMetadataProcessor.java
View file
Expand Up @@ -23,6 +23,7 @@

import static org.junit.Assert.*;

import java.io.File;
import java.util.Properties;

import javax.resource.ResourceException;
Expand All @@ -45,14 +46,25 @@
@SuppressWarnings("nls")
public class TestExcelMetadataProcessor {

static String getDDL(Properties props) throws TranslatorException, ResourceException {
static String getDDL(Properties props) throws TranslatorException, ResourceException {
return getDDL(props, null);
}
static String getDDL(Properties props, String filename) throws TranslatorException, ResourceException {
ExcelExecutionFactory translator = new ExcelExecutionFactory();
translator.start();

String xlsName = props.getProperty("importer.excelFileName");
MetadataFactory mf = new MetadataFactory("vdb", 1, "people", SystemMetadata.getInstance().getRuntimeTypeMap(), props, null);
FileConnection connection = Mockito.mock(FileConnection.class);
Mockito.stub(connection.getFile(xlsName)).toReturn(UnitTestUtil.getTestDataFile(xlsName));
if (xlsName.contains("*.")) {
Mockito.stub(connection.getFile(xlsName)).toReturn(UnitTestUtil.getTestDataFile(xlsName));
File f = Mockito.mock(File.class);
Mockito.stub(f.isDirectory()).toReturn(true);
Mockito.stub(f.listFiles()).toReturn(new File[] {UnitTestUtil.getTestDataFile(filename)});
Mockito.stub(connection.getFile(xlsName)).toReturn(f);
} else {
Mockito.stub(connection.getFile(xlsName)).toReturn(UnitTestUtil.getTestDataFile(xlsName));
}
translator.getMetadata(mf, connection);

TransformationMetadata metadata = RealMetadataFactory.createTransformationMetadata(mf.asMetadataStore(), "vdb", new FunctionTree("foo", new UDFSource(translator.getPushDownFunctions())));
Expand Down Expand Up @@ -265,5 +277,26 @@ public void testDataTypeFromNullCell() throws Exception {
") OPTIONS (NAMEINSOURCE 'Sheet1', \"teiid_excel:FILE\" 'names.xlsx', \"teiid_excel:FIRST_DATA_ROW_NUMBER\" '2');";

assertEquals(expectedDDL, ddl);
}
}

@Test
public void testFileGlob() throws Exception {
Properties props = new Properties();
props.setProperty("importer.excelFileName", "*.xlsx");
props.setProperty("importer.headerRowNumber", "1");
props.setProperty("importer.dataRowNumber", "2");
String ddl = getDDL(props, "names.xlsx");
String expectedDDL = "SET NAMESPACE 'http://www.teiid.org/translator/excel/2014' AS teiid_excel;\n" +
"\n" +
"CREATE FOREIGN TABLE Sheet1 (\n" +
"\tROW_ID integer OPTIONS (SEARCHABLE 'All_Except_Like', \"teiid_excel:CELL_NUMBER\" 'ROW_ID'),\n" +
"\tFirstName string OPTIONS (SEARCHABLE 'Unsearchable', \"teiid_excel:CELL_NUMBER\" '1'),\n" +
"\tLastName string OPTIONS (SEARCHABLE 'Unsearchable', \"teiid_excel:CELL_NUMBER\" '2'),\n" +
"\tAge double OPTIONS (SEARCHABLE 'Unsearchable', \"teiid_excel:CELL_NUMBER\" '3'),\n" +
"\t\"time\" double OPTIONS (SEARCHABLE 'Unsearchable', \"teiid_excel:CELL_NUMBER\" '4'),\n"+
"\tCONSTRAINT PK0 PRIMARY KEY(ROW_ID)\n" +
") OPTIONS (NAMEINSOURCE 'Sheet1', \"teiid_excel:FILE\" '*.xlsx', \"teiid_excel:FIRST_DATA_ROW_NUMBER\" '2');";

assertEquals(expectedDDL, ddl);
}
}

0 comments on commit 6447d81

Please sign in to comment.