Add system.dataset.fromExcel

build.gradle.kts

@@ -15,8 +15,8 @@ ignitionModule {
     fileName.set("Ignition-Extensions.modl")
     id.set("org.imdc.extensions.IgnitionExtensions")
     moduleVersion.set("${project.version}")
-
     moduleDescription.set("Useful but niche extensions to Ignition for power users")
+    license.set("LICENSE.md")
     requiredIgnitionVersion.set(libs.versions.ignition.get())
 
     projectScopes.putAll(

common/src/main/kotlin/org/imdc/extensions/common/DatasetExtensions.kt

@@ -6,9 +6,19 @@ import com.inductiveautomation.ignition.common.script.PyArgParser
 import com.inductiveautomation.ignition.common.script.builtin.KeywordArgs
 import com.inductiveautomation.ignition.common.script.hints.ScriptFunction
 import com.inductiveautomation.ignition.common.util.DatasetBuilder
+import org.apache.poi.ss.usermodel.CellType.BOOLEAN
+import org.apache.poi.ss.usermodel.CellType.FORMULA
+import org.apache.poi.ss.usermodel.CellType.NUMERIC
+import org.apache.poi.ss.usermodel.CellType.STRING
+import org.apache.poi.ss.usermodel.DateUtil
+import org.apache.poi.ss.usermodel.WorkbookFactory
 import org.python.core.Py
 import org.python.core.PyFunction
 import org.python.core.PyObject
+import java.io.File
+import java.math.BigDecimal
+import java.util.Date
+import kotlin.math.max
 
 object DatasetExtensions {
     @Suppress("unused")
@@ -35,9 +45,7 @@ object DatasetExtensions {
             List(dataset.columnCount) { Any::class.java }
         }
 
-        val builder = DatasetBuilder.newBuilder()
-            .colNames(dataset.columnNames)
-            .colTypes(columnTypes)
+        val builder = DatasetBuilder.newBuilder().colNames(dataset.columnNames).colTypes(columnTypes)
 
         for (row in dataset.rowIndices) {
             val columnValues = Array<PyObject>(dataset.columnCount) { col ->
@@ -103,41 +111,59 @@ object DatasetExtensions {
     @Suppress("unused")
     @ScriptFunction(docBundlePrefix = "DatasetExtensions")
     @KeywordArgs(
-        names = ["dataset", "output"],
-        types = [Dataset::class, Appendable::class],
+        names = ["dataset", "output", "includeTypes"],
+        types = [Dataset::class, Appendable::class, Boolean::class],
     )
     fun print(args: Array<PyObject>, keywords: Array<String>) {
         val parsedArgs = PyArgParser.parseArgs(
             args,
             keywords,
-            arrayOf("dataset", "output"),
-            arrayOf(Dataset::class.java, PyObject::class.java),
+            arrayOf("dataset", "output", "includeTypes"),
+            Array(3) { PyObject::class.java },
             "print",
         )
         val dataset = parsedArgs.requirePyObject("dataset").toJava<Dataset>()
         val appendable = parsedArgs.getPyObject("output")
             .orElse(Py.getSystemState().stdout)
             .let(::PyObjectAppendable)
+        val includeTypes = parsedArgs.getBoolean("includeTypes").orElse(false)
 
-        return appendable.printDataset(dataset)
+        return printDataset(appendable, dataset, includeTypes)
     }
 
-    internal fun Appendable.printDataset(dataset: Dataset, separator: String = "|") {
+    internal fun printDataset(appendable: Appendable, dataset: Dataset, includeTypes: Boolean = false) {
+        val typeNames = List<String>(dataset.columnCount) { column ->
+            if (includeTypes) {
+                dataset.getColumnType(column).simpleName
+            } else {
+                ""
+            }
+        }
+
         val columnWidths = IntArray(dataset.columnCount) { column ->
             maxOf(
+                // longest value in a row
                 if (dataset.rowCount > 0) {
                     dataset.rowIndices.maxOf { row -> dataset[row, column].toString().length }
                 } else {
                     0
                 },
-                dataset.getColumnName(column).length,
+                // longest value in a header
+                if (includeTypes) {
+                    dataset.getColumnName(column).length + typeNames[column].length + 3 // 3 = two parens and a space
+                } else {
+                    dataset.getColumnName(column).length
+                },
+                // absolute minimum width for markdown table (and human eyeballs)
                 3,
             )
         }
 
+        val separator = "|"
+
         fun Sequence<String>.joinToBuffer() {
             joinTo(
-                buffer = this@printDataset,
+                buffer = appendable,
                 separator = " $separator ",
                 prefix = "$separator ",
                 postfix = " $separator\n",
@@ -148,7 +174,16 @@ object DatasetExtensions {
         sequence {
             yield("Row")
             for (column in dataset.columnIndices) {
-                yield(dataset.getColumnName(column).padStart(columnWidths[column]))
+                val headerValue = buildString {
+                    append(dataset.getColumnName(column))
+                    if (includeTypes) {
+                        append(" (").append(typeNames[column]).append(")")
+                    }
+                    while (length < columnWidths[column]) {
+                        insert(0, ' ')
+                    }
+                }
+                yield(headerValue)
             }
         }.joinToBuffer()
 
@@ -171,4 +206,140 @@ object DatasetExtensions {
             }.joinToBuffer()
         }
     }
+
+    @Suppress("unused")
+    @ScriptFunction(docBundlePrefix = "DatasetExtensions")
+    @KeywordArgs(
+        names = ["input", "headerRow", "sheetNumber", "firstRow", "lastRow", "firstColumn", "lastColumn"],
+        types = [ByteArray::class, Integer::class, Integer::class, Integer::class, Integer::class, Integer::class, Integer::class],
+    )
+    fun fromExcel(args: Array<PyObject>, keywords: Array<String>): Dataset {
+        val parsedArgs = PyArgParser.parseArgs(
+            args,
+            keywords,
+            arrayOf(
+                "input",
+                "headerRow",
+                "sheetNumber",
+                "firstRow",
+                "lastRow",
+                "firstColumn",
+                "lastColumn",
+            ),
+            Array(7) { Any::class.java },
+            "fromExcel",
+        )
+
+        when (val input = parsedArgs.requirePyObject("input").toJava<Any>()) {
+            is String -> WorkbookFactory.create(File(input))
+            is ByteArray -> WorkbookFactory.create(input.inputStream().buffered())
+            else -> throw Py.TypeError("Unable to create Workbook from input; should be string or binary data. Got ${input::class.simpleName} instead.")
+        }.use { workbook ->
+            val sheetNumber = parsedArgs.getInteger("sheetNumber").orElse(0)
+            val sheet = workbook.getSheetAt(sheetNumber)
+
+            val headerRow = parsedArgs.getInteger("headerRow").orElse(-1)
+            val firstRow = parsedArgs.getInteger("firstRow").orElseGet { max(sheet.firstRowNum, headerRow + 1) }
+            val lastRow = parsedArgs.getInteger("lastRow").orElseGet { sheet.lastRowNum }
+
+            val dataRange = firstRow..lastRow
+
+            if (firstRow >= lastRow) {
+                throw Py.ValueError("firstRow ($firstRow) must be less than lastRow ($lastRow)")
+            }
+            if (headerRow >= 0 && headerRow in dataRange) {
+                throw Py.ValueError("headerRow must not be in firstRow..lastRow ($dataRange)")
+            }
+
+            val columnRow = sheet.getRow(if (headerRow >= 0) headerRow else firstRow)
+            val firstColumn = parsedArgs.getInteger("firstColumn").orElseGet { columnRow.firstCellNum.toInt() }
+            val lastColumn =
+                parsedArgs.getInteger("lastColumn").map { it + 1 }.orElseGet { columnRow.lastCellNum.toInt() }
+            if (firstColumn >= lastColumn) {
+                throw Py.ValueError("firstColumn ($firstColumn) must be less than lastColumn ($lastColumn)")
+            }
+
+            val columnCount = lastColumn - firstColumn
+
+            val dataset = DatasetBuilder()
+            dataset.colNames(
+                List(columnCount) {
+                    if (headerRow >= 0) {
+                        columnRow.getCell(it + firstColumn).toString()
+                    } else {
+                        "Col $it"
+                    }
+                },
+            )
+
+            var typesSet = false
+            val columnTypes = mutableListOf<Class<*>>()
+
+            for (i in dataRange) {
+                if (i == headerRow) {
+                    continue
+                }
+
+                val row = sheet.getRow(i)
+
+                val rowValues = Array<Any?>(columnCount) { j ->
+                    val cell = row.getCell(j + firstColumn)
+
+                    when (cell?.cellType.takeUnless { it == FORMULA } ?: cell.cachedFormulaResultType) {
+                        NUMERIC -> {
+                            if (DateUtil.isCellDateFormatted(cell)) {
+                                if (!typesSet) {
+                                    columnTypes.add(Date::class.java)
+                                }
+                                cell.dateCellValue
+                            } else {
+                                val numericCellValue = cell.numericCellValue
+                                if (BigDecimal(numericCellValue).scale() == 0) {
+                                    if (!typesSet) {
+                                        columnTypes.add(Int::class.javaObjectType)
+                                    }
+                                    numericCellValue.toInt()
+                                } else {
+                                    if (!typesSet) {
+                                        columnTypes.add(Double::class.javaObjectType)
+                                    }
+                                    numericCellValue
+                                }
+                            }
+                        }
+
+                        STRING -> {
+                            if (!typesSet) {
+                                columnTypes.add(String::class.java)
+                            }
+                            cell.stringCellValue
+                        }
+
+                        BOOLEAN -> {
+                            if (!typesSet) {
+                                columnTypes.add(Boolean::class.javaObjectType)
+                            }
+                            cell.booleanCellValue
+                        }
+
+                        else -> {
+                            if (!typesSet) {
+                                columnTypes.add(Any::class.java)
+                            }
+                            null
+                        }
+                    }
+                }
+
+                if (!typesSet) {
+                    typesSet = true
+                    dataset.colTypes(columnTypes)
+                }
+
+                dataset.addRow(*rowValues)
+            }
+
+            return dataset.build()
+        }
+    }
 }

common/src/main/resources/org/imdc/extensions/common/DatasetExtensions.properties

@@ -12,4 +12,15 @@ filter.returns=A modified dataset.
 print.desc=Prints a dataset to standard output, or the provided buffer.
 print.param.dataset=The dataset to print. Must not be null.
 print.param.output=The output destination. Defaults to sys.stdout.
+print.param.includeTypes=If True, includes the type of the column in the first row. Defaults to False.
 print.returns=None.
+
+fromExcel.desc=Creates a dataset by reading select cells from an Excel spreadsheet.
+fromExcel.param.input=The Excel document to read - either the path to a file on disk, or a byte array with the contents of a file.
+fromExcel.param.headerRow=The row number to use for the column names in the output dataset.
+fromExcel.param.sheetNumber=The sheet number (zero-indexed) in the Excel document to extract data from.
+fromExcel.param.firstRow=The first row (zero-indexed) in the Excel document to retrieve data from. If not supplied, the first non-empty row will be used.
+fromExcel.param.lastRow=The last row (zero-indexed) in the Excel document to retrieve data from. If not supplied, the last non-empty row will be used.
+fromExcel.param.firstColumn=The first column (zero-indexed) in the Excel document to retrieve data from. If not supplied, the first non-empty column will be used.
+fromExcel.param.lastColumn=The last column (zero-indexed) in the Excel document to retrieve data from. If not supplied, the last non-empty column will be used.
+fromExcel.returns=A Dataset created from the Excel document. Types are assumed based on the first row of input data.

common/src/main/resources/org/imdc/extensions/common/UtilitiesExtensions.properties

@@ -1,8 +1,10 @@
 getContext.desc=Returns the current scope's context object directly.
 getContext.returns=The current scope's context.
+
 deepCopy.desc=Deep copies the inner object structure into plain Python lists, dictionaries, and primitives.
 deepCopy.param.object=The object to convert.
 deepCopy.returns=A plain Python primitive object.
+
 evalExpression.desc=Evaluates the supplied expression. Provide keyword arguments to populate values to curly braces.
 evalExpression.param.expression=The expression to evaluate.
 evalExpression.returns=A QualifiedValue with the result of the provided expression.