Tests: refactor and improve tests and remove unnecessary/untested code

lingo-db · Jul 21, 2023 · 0315019 · 0315019
1 parent fa9c394
commit 0315019
Show file tree

Hide file tree

Showing 64 changed files with 913 additions and 1,184 deletions.
diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
@@ -98,6 +98,7 @@ jobs:
       - name: Test
         run: |
             chmod +x /build/lingodb/*
+            find ./test/sqlite-small/ -maxdepth 1 -type f -name '*.test' | xargs -L 1 -P $(nproc) /build/lingodb/sqlite-tester
             find ./test/sqlite/ -maxdepth 1 -type f -name '*.test' | xargs -L 1 -P $(nproc) /build/lingodb/sqlite-tester
   build_lingodb_python:
     runs-on: ubuntu-latest
@@ -145,10 +146,9 @@ jobs:
             /build/llvm/bin/llvm-lit -v /build/lingodb/test/lit
             mkdir /coverage-data
             lcov --capture --directory /build/lingodb  --output-file /coverage-data/coverage.info
-            lcov --remove /coverage-data/coverage.info -o /coverage-data/filtered-coverage.info '/build/llvm/*' '**/llvm-project/*' '*.inc' '**/arrow/*' '**/pybind11/*' '**/vendored/*' '/usr/*'
+            lcov --remove /coverage-data/coverage.info -o /coverage-data/filtered-coverage.info '/build/llvm/*' '**/llvm-project/*' '*.inc' '**/arrow/*' '**/pybind11/*' '**/vendored/*' '/usr/*' '/build/lingodb/*'
       - uses: actions/checkout@master
       - uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          files: /coverage-data/filtered-coverage.info
-          fail_ci_if_error: true
+          files: /coverage-data/filtered-coverage.info
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -125,6 +125,7 @@ test_pylingodb:
 .sqlite_tests:
    script:
       #- export LINGODB_EXECUTION_MODE=CHEAP
+      - time find ./test/sqlite-small/ -maxdepth 1 -type f -name '*.test' | xargs -L 1 -P $(nproc) /build/lingodb/sqlite-tester
       - time find ./test/sqlite/ -maxdepth 1 -type f -name '*.test' | xargs -L 1 -P $(nproc) /build/lingodb/sqlite-tester
 
 .sqlite_dataset_tests:

diff --git a/Makefile b/Makefile
@@ -65,8 +65,20 @@ build/lingodb-debug-coverage/.stamp: $(LDB_DEPS)
 
 .PHONY: run-test
 run-test: build/lingodb-debug/.stamp
-	cmake --build $(dir $<) --target mlir-db-opt run-mlir run-sql mlir-doc -- -j${NPROCS}
-	env LD_LIBRARY_PATH=${ROOT_DIR}/build/arrow/install/lib ./build/llvm-build/bin/llvm-lit -v $(dir $<)/test/lit -j 1
+	cmake --build $(dir $<) --target mlir-db-opt run-mlir run-sql mlir-doc sqlite-tester sql-to-mlir  -- -j${NPROCS}
+	./build/llvm-build/bin/llvm-lit -v $(dir $<)/test/lit -j 1
+	find ./test/sqlite-small/ -maxdepth 1 -type f -name '*.test' | xargs -L 1 -P ${NPROCS} $(dir $<)/sqlite-tester
+
+.PHONY: test-coverage
+test-coverage: build/lingodb-debug-coverage/.stamp
+	cmake --build $(dir $<) --target mlir-db-opt run-mlir run-sql sql-to-mlir -- -j${NPROCS}
+	./build/llvm-build/bin/llvm-lit -v $(dir $<)/test/lit
+	lcov --capture --directory $(dir $<)  --output-file $(dir $<)/coverage.info
+	lcov --remove $(dir $<)/coverage.info -o $(dir $<)/filtered-coverage.info \
+			'**/build/llvm-build/*' '**/llvm-project/*' '*.inc' '**/arrow/*' '**/pybind11/*' '**/vendored/*' '/usr/*' '**/build/lingodb-debug-coverage/*'
+	genhtml  --ignore-errors source $(dir $<)/filtered-coverage.info --legend --title "lcov-test" --output-directory=$(dir $<)/coverage-report
+
+
 .PHONY: run-benchmark
 run-benchmark: build/lingodb-release/.stamp build/lingodb-debug/.stamp resources/data/tpch-1/.stamp
 	cmake --build $(dir $<) --target run-sql -- -j${NPROCS}

diff --git a/Readme.md b/Readme.md
@@ -2,7 +2,14 @@
   <img src="https://www.lingo-db.com/images/lingodb-black-title.png" height="50">
 </div>
 <p>&nbsp;</p>
-
+<p align="center">
+  <a href="https://github.com/lingo-db/lingo-db/actions">
+    <img src="https://github.com/lingo-db/lingo-db/actions/workflows/workflow.yml/badge.svg?branch=main" alt="Github Actions Badge">
+  </a>
+  <a href="https://codecov.io/gh/lingo-db/lingo-db" >
+    <img src="https://codecov.io/gh/lingo-db/lingo-db/branch/main/graph/badge.svg?token=7RC3UD5YEA"/>
+  </a>
+</p>
 
 # LingoDB
 LingoDB is a cutting-edge data processing system that leverages compiler technology to achieve unprecedented flexibility and extensibility without sacrificing performance. It supports a wide range of data-processing workflows beyond relational SQL queries, thanks to declarative sub-operators. Furthermore, LingoDB can perform cross-domain optimization by interleaving optimization passes of different domains and its flexibility enables sustainable support for heterogeneous hardware.

diff --git a/include/execution/Frontend.h b/include/execution/Frontend.h
@@ -32,7 +32,6 @@ class Frontend {
 };
 std::unique_ptr<Frontend> createMLIRFrontend();
 std::unique_ptr<Frontend> createSQLFrontend();
-std::unique_ptr<Frontend> createBatchedSQLFrontend();
 
 } //namespace execution
 

diff --git a/include/frontend/SQL/Parser.h b/include/frontend/SQL/Parser.h
@@ -458,9 +458,6 @@ struct Parser {
    //translate a complete from clause into a single value of type tuple stream (connect single items with cross-products)
    mlir::Value translateFromClause(mlir::OpBuilder& builder, SelectStmt* stmt, TranslationContext& context, TranslationContext::ResolverScope& scope);
 
-   //translate sort specification into array attribute for relalg::SortOp
-   mlir::ArrayAttr translateSortSpec(List* sortClause, mlir::OpBuilder& builder, TranslationContext& context, TargetInfo);
-
    //translate list of constant values into relalg::ConstRelationOp
    std::pair<mlir::Value, TargetInfo> translateConstRelation(List* valuesLists, mlir::OpBuilder& builder);
 

diff --git a/include/mlir/Dialect/util/FunctionHelper.h b/include/mlir/Dialect/util/FunctionHelper.h
@@ -40,8 +40,6 @@ class FunctionSpec {
 
 class FunctionHelper {
    ModuleOp parentModule;
-   static void* resolveFunction(std::string mangledName);
-
    public:
    static void visitAllFunctions(const std::function<void(std::string, void*)>& fn);
    void setParentModule(const ModuleOp& parentModule);

diff --git a/lib/execution/Frontend.cpp b/lib/execution/Frontend.cpp
@@ -128,78 +128,10 @@ class SQLFrontend : public execution::Frontend {
       return parallismAllowed;
    }
 };
-class BatchedSQLFrontend : public execution::Frontend {
-   mlir::MLIRContext context;
-   mlir::OwningOpRef<mlir::ModuleOp> module;
-   void loadFromString(std::string sql) override {
-      initializeContext(context);
-
-      mlir::OpBuilder builder(&context);
-
-      mlir::ModuleOp moduleOp = builder.create<mlir::ModuleOp>(builder.getUnknownLoc());
-      std::stringstream batched(sql);
-      builder.setInsertionPointToStart(moduleOp.getBody());
-      auto* queryBlock = new mlir::Block;
-      std::vector<mlir::Type> returnTypes;
-      size_t i = 0;
-      {
-         mlir::OpBuilder::InsertionGuard guard(builder);
-         builder.setInsertionPointToStart(queryBlock);
-         while (true) {
-            std::stringstream query;
-            std::string line;
-            std::getline(batched, line);
-            if (batched.eof()) {
-               //exit from repl loop
-               break;
-            }
-            while (true) {
-               if (!batched.good()) {
-                  if (batched.eof()) {
-                     query << line << std::endl;
-                  }
-                  break;
-               }
-               query << line << std::endl;
-               if (!line.empty() && line.find(';') == line.size() - 1) {
-                  break;
-               }
-               std::getline(batched, line);
-            }
-            frontend::sql::Parser translator(query.str(), *catalog, moduleOp);
-            auto val = translator.translate(builder);
-            if (val.has_value()) {
-               builder.create<mlir::subop::SetResultOp>(builder.getUnknownLoc(), i++, val.value());
-            }
-         }
-         builder.create<mlir::func::ReturnOp>(builder.getUnknownLoc());
-      }
-      mlir::func::FuncOp funcOp = builder.create<mlir::func::FuncOp>(builder.getUnknownLoc(), "main", builder.getFunctionType({}, {}));
-      funcOp.getBody().push_back(queryBlock);
-      module = moduleOp;
-   }
-   void loadFromFile(std::string fileName) override {
-      std::ifstream istream{fileName};
-      if (!istream) {
-         error.emit() << "Error can't load file " << fileName;
-      }
-      std::stringstream buffer;
-      buffer << istream.rdbuf();
-      std::string sqlQuery = buffer.str();
-      loadFromString(sqlQuery);
-   }
-   mlir::ModuleOp* getModule() override {
-      assert(module);
-      return module.operator->();
-   }
-};
 } // namespace
 std::unique_ptr<execution::Frontend> execution::createMLIRFrontend() {
    return std::make_unique<MLIRFrontend>();
 }
 std::unique_ptr<execution::Frontend> execution::createSQLFrontend() {
    return std::make_unique<SQLFrontend>();
-}
-std::unique_ptr<execution::Frontend> execution::createBatchedSQLFrontend() {
-   return std::make_unique<BatchedSQLFrontend>();
 }
diff --git a/lib/frontend/SQL/Parser.cpp b/lib/frontend/SQL/Parser.cpp
@@ -174,66 +174,6 @@ mlir::Value frontend::sql::Parser::translateWhenCaseExpression(mlir::OpBuilder&
 
    return ifOp.getResult(0);
 }
-mlir::ArrayAttr frontend::sql::Parser::translateSortSpec(List* sortClause, mlir::OpBuilder& builder, TranslationContext& context, TargetInfo targetInfo) {
-   std::vector<mlir::Attribute> mapping;
-   for (auto* cell = sortClause->head; cell != nullptr; cell = cell->next) {
-      auto* temp = reinterpret_cast<Node*>(cell->data.ptr_value);
-      mlir::relalg::SortSpec spec = mlir::relalg::SortSpec::asc;
-      switch (temp->type) {
-         case T_SortBy: {
-            auto* sort = reinterpret_cast<SortBy*>(temp);
-
-            switch (sort->sortby_dir_) {
-               case SORTBY_DESC: {
-                  spec = mlir::relalg::SortSpec::desc;
-                  break;
-               }
-               case SORTBY_ASC: // fall through
-               case SORTBY_DEFAULT: {
-                  spec = mlir::relalg::SortSpec::asc;
-                  break;
-               }
-               default: {
-                  error("unknown sort type");
-               }
-            }
-
-            auto* target = sort->node_;
-            const mlir::tuples::Column* attr;
-            switch (target->type) {
-               case T_ColumnRef: {
-                  attr = resolveColRef(target, context);
-                  break;
-               }
-               case T_FakeNode: { //
-                  attr = context.getAttribute(reinterpret_cast<FakeNode*>(target)->colId);
-                  break;
-               }
-               case T_A_Const: {
-                  auto* constExpr = reinterpret_cast<A_Const*>(target);
-                  auto constVal = constExpr->val_;
-                  switch (constVal.type_) {
-                     case T_Integer: {
-                        attr = targetInfo.namedResults.at(constVal.val_.ival_ - 1).second;
-                        break;
-                     }
-                     default: error("unsupported sort specification");
-                  }
-                  break;
-               }
-               default: error("can only sort with column refs");
-            }
-
-            mapping.push_back(mlir::relalg::SortSpecificationAttr::get(builder.getContext(), attrManager.createRef(attr), spec));
-            break;
-         }
-         default: {
-            error("unknown orderby type");
-         }
-      }
-   }
-   return builder.getArrayAttr(mapping);
-}
 mlir::Value frontend::sql::Parser::translateFuncCallExpression(Node* node, mlir::OpBuilder& builder, mlir::Location loc, TranslationContext& context) {
    auto* funcCall = reinterpret_cast<FuncCall*>(node);
    //expr = FuncCallTransform(parse_result,,context);

diff --git a/lib/util/FunctionHelper.cpp b/lib/util/FunctionHelper.cpp
@@ -68,9 +68,6 @@ mlir::util::FunctionSpec::FunctionSpec(const std::string& name, const std::strin
    getFunctions().insert({mangledName, *this});
 }
 
-void* mlir::util::FunctionHelper::resolveFunction(std::string mangledName) {
-   return getFunctions().at(mangledName).getPointer();
-}
 
 void mlir::util::FunctionHelper::visitAllFunctions(const std::function<void(std::string, void*)>& fn) {
    for(auto f:getFunctions()){

diff --git a/resources/data/test/test.arrow b/resources/data/test/test.arrow
diff --git a/resources/data/test/test.arrow.sample b/resources/data/test/test.arrow.sample
diff --git a/resources/data/test/test.metadata.json b/resources/data/test/test.metadata.json
@@ -1,115 +1 @@
-{
-  "columns": [
-    {
-      "distinct_values": 1,
-      "name": "str",
-      "type": {
-        "base": "string",
-        "nullable": true,
-        "props": [
-          20
-        ]
-      }
-    },
-    {
-      "distinct_values": 1,
-      "name": "float32",
-      "type": {
-        "base": "float",
-        "nullable": true,
-        "props": [
-          32
-        ]
-      }
-    },
-    {
-      "distinct_values": 1,
-      "name": "float64",
-      "type": {
-        "base": "float",
-        "nullable": true,
-        "props": [
-          64
-        ]
-      }
-    },
-    {
-      "distinct_values": 1,
-      "name": "decimal",
-      "type": {
-        "base": "decimal",
-        "nullable": true,
-        "props": [
-          5,
-          2
-        ]
-      }
-    },
-    {
-      "distinct_values": 1,
-      "name": "int32",
-      "type": {
-        "base": "int",
-        "nullable": true,
-        "props": [
-          32
-        ]
-      }
-    },
-    {
-      "distinct_values": 1,
-      "name": "int64",
-      "type": {
-        "base": "int",
-        "nullable": true,
-        "props": [
-          64
-        ]
-      }
-    },
-    {
-      "distinct_values": 1,
-      "name": "bool",
-      "type": {
-        "base": "bool",
-        "nullable": true,
-        "props": []
-      }
-    },
-    {
-      "distinct_values": 1,
-      "name": "date32",
-      "type": {
-        "base": "date",
-        "nullable": true,
-        "props": [
-          "day"
-        ]
-      }
-    },
-    {
-      "distinct_values": 1,
-      "name": "date64",
-      "type": {
-        "base": "date",
-        "nullable": true,
-        "props": [
-          "millisecond"
-        ]
-      }
-    },
-    {
-      "distinct_values": 2,
-      "name": "primaryKeyHashValue",
-      "type": {
-        "base": "index",
-        "nullable": false,
-        "props": []
-      }
-    }
-  ],
-  "num_rows": 2,
-  "pkey": [
-    "float64"
-  ]
-}
+{"columns":[{"distinct_values":1,"name":"str","type":{"base":"string","nullable":true,"props":[20]}},{"distinct_values":1,"name":"float32","type":{"base":"float","nullable":true,"props":[32]}},{"distinct_values":1,"name":"float64","type":{"base":"float","nullable":true,"props":[64]}},{"distinct_values":1,"name":"decimal","type":{"base":"decimal","nullable":true,"props":[5,2]}},{"distinct_values":1,"name":"int32","type":{"base":"int","nullable":true,"props":[32]}},{"distinct_values":1,"name":"int64","type":{"base":"int","nullable":true,"props":[64]}},{"distinct_values":1,"name":"bool","type":{"base":"bool","nullable":true,"props":[]}},{"distinct_values":1,"name":"date32","type":{"base":"date","nullable":true,"props":["day"]}},{"distinct_values":1,"name":"date64","type":{"base":"date","nullable":true,"props":["millisecond"]}}],"indices":[{"columns":["float64"],"name":"pk_hash","type":0}],"num_rows":2,"pkey":["float64"]}
diff --git a/resources/data/test/test.pk_hash.arrow b/resources/data/test/test.pk_hash.arrow
diff --git a/resources/sql/test/initialize.sql b/resources/sql/test/initialize.sql
@@ -11,4 +11,4 @@ create table test(
     date64 timestamp,
     primary key(float64)
 );
-INSERT into test(str, float32, float64, decimal, int32, int64, bool, date32, date64) values ('str', 1.1, 1.1, 1.10, 1, 1, 1, '1966-01-02', '1996-01-02'), (null, null, null, null, null, null, null, null, null);
+INSERT into test(str, float32, float64, decimal, int32, int64, bool, date32, date64) values ('str', 1.1, 1.1, 1.10, 1, 1, 1, '1996-01-02', '1996-01-02'), (null, null, null, null, null, null, null, null, null);
diff --git a/test/lit/DSA/ops.mlir b/test/lit/DSA/ops.mlir
@@ -0,0 +1,9 @@
+// RUN: mlir-db-opt -allow-unregistered-dialect %s -split-input-file -mlir-print-debuginfo -mlir-print-local-scope  | FileCheck %s
+// CHECK:   %{{.*}} = dsa.for %{{.*}} in %{{.*}} : !util.buffer<i8>  iter_args(%{{.*}} = %c0_i8) -> (i8){
+func.func @test(%arg0: !util.buffer<i8>) -> i8 {
+  %c0_i8 = arith.constant 0 : i8
+  %0 = dsa.for %arg1 in %arg0 : !util.buffer<i8>  iter_args(%arg2 = %c0_i8) -> (i8){
+    dsa.yield %arg2 : i8
+  }
+  return %0 : i8
+}