test(it): Use the parser from SQLcl to filter valid statements from O…

…racle docs
felipebz · Jun 20, 2024 · 9bbeb5f · 9bbeb5f
1 parent f1ae98d
commit 9bbeb5f
Show file tree

Hide file tree

Showing 624 changed files with 601 additions and 2,886 deletions.
diff --git a/.gitignore b/.gitignore
@@ -61,3 +61,4 @@ gradle-app.setting
 # Cache of project
 .gradletasknamecache
 .kotlin/
+zpa-checks/tools/
diff --git a/zpa-checks/build.gradle.kts b/zpa-checks/build.gradle.kts
@@ -1,3 +1,9 @@
+import de.undercouch.gradle.tasks.download.Download
+
+plugins {
+    id("de.undercouch.download") version "5.6.0"
+}
+
 dependencies {
     implementation(Libs.flr_core)
     implementation(Libs.flr_xpath)
@@ -10,7 +16,25 @@ testing {
         val integrationTest by registering(JvmTestSuite::class) {
             testType.set(TestSuiteType.INTEGRATION_TEST)
 
+            val downloadZipFile by tasks.creating(Download::class) {
+                val sqlclVersion = "24.1.0.087.0929"
+                src("https://download.oracle.com/otn_software/java/sqldeveloper/sqlcl-$sqlclVersion.zip")
+                overwrite(false)
+                dest(layout.projectDirectory.dir("tools").file("sqlcl-$sqlclVersion.zip"))
+            }
+
+            val downloadAndUnzipFile = tasks.register<Copy>("downloadAndUnzipFile") {
+                dependsOn(downloadZipFile)
+                from(zipTree(downloadZipFile.dest))
+                into(layout.projectDirectory.dir("tools"))
+            }
+
+            tasks.build {
+                dependsOn(downloadAndUnzipFile)
+            }
+
             dependencies {
+                implementation(layout.projectDirectory.dir("tools/sqlcl/lib").asFileTree)
                 implementation(project())
                 implementation(Libs.jackson)
                 implementation(project(":zpa-core"))

diff --git a/zpa-checks/src/integrationTest/kotlin/org/sonar/plsqlopen/it/OracleDocsExtractor.kt b/zpa-checks/src/integrationTest/kotlin/org/sonar/plsqlopen/it/OracleDocsExtractor.kt
@@ -19,6 +19,10 @@
  */
 package org.sonar.plsqlopen.it
 
+import oracle.dbtools.parser.Lexer
+import oracle.dbtools.parser.plsql.SyntaxError
+import oracle.dbtools.raptor.newscriptrunner.ISQLCommand
+import oracle.dbtools.raptor.newscriptrunner.ScriptParser
 import org.jsoup.Jsoup
 import java.io.File
 import java.nio.file.Paths
@@ -53,24 +57,18 @@ class OracleDocsExtractor {
                         Jsoup.parse(stream, Charsets.UTF_8.name(), "").run {
                             select("pre.oac_no_warn").forEachIndexed { index, element ->
                                 var text = element.text()
+                                    .replace('’', '\'')
 
-                                val lines = text.lines()
-                                val line = lines.indexOfFirst { it.contains("---") }
-                                try {
-                                    if (line != -1) {
-                                        text = text.lines().take(line - 2).joinToString(separator = "\n")
-                                    }
+                                val fileContent = extractValidStatementsFrom(text)
 
-                                    if (text.isNotEmpty()) {
-                                        val name = "${File(entry.name).nameWithoutExtension}-$index.sql"
-                                        val path = entry.name.substring(entry.name.indexOf(parent))
-                                        text = "-- https://docs.oracle.com/en/database/oracle/oracle-database/23/$path\n$text"
+                                if (fileContent.isNotEmpty()) {
+                                    val name = "${File(entry.name).nameWithoutExtension}-$index.sql"
+                                    val path = entry.name.substring(entry.name.indexOf(parent))
+                                    text = "-- https://docs.oracle.com/en/database/oracle/oracle-database/23/$path\n$fileContent"
 
-                                        val pathOutput = Paths.get(outputDir.absolutePath, parent, name).toFile()
-                                        pathOutput.parentFile.mkdirs()
-                                        pathOutput.writeText(text, Charsets.UTF_8)
-                                    }
-                                } catch (e: Exception) {
+                                    val pathOutput = Paths.get(outputDir.absolutePath, parent, name).toFile()
+                                    pathOutput.parentFile.mkdirs()
+                                    pathOutput.writeText(text, Charsets.UTF_8)
                                 }
                             }
                         }
@@ -79,4 +77,29 @@ class OracleDocsExtractor {
             }
     }
 
+    private fun extractValidStatementsFrom(text: String): String {
+        val parser = ScriptParser(text)
+        var cmd: ISQLCommand
+        var validText = ""
+        while ((parser.next().also { cmd = it }) != null) {
+            val sql = cmd.sqlOrigWithTerminator
+            val syntaxError = SyntaxError.checkSyntax(
+                sql,
+                arrayOf("select", "sql_statement", "sql_statements")
+            )
+            if (syntaxError == null) {
+                val tokens = Lexer.parse(sql)
+
+                // ignore the command if it doesn't have any token (e.g. comment line)
+                if (tokens.isNotEmpty()) {
+                    if (validText.isNotEmpty()) {
+                        validText += "\n"
+                    }
+                    validText += sql
+                }
+            }
+        }
+        return validText
+    }
+
 }
diff --git a/...ntegrationTest/resources/expected/oracle-database_23/ColumnsShouldHaveTableNameCheck.json b/...ntegrationTest/resources/expected/oracle-database_23/ColumnsShouldHaveTableNameCheck.json
@@ -1,4 +1,9 @@
 {
+  "sqlrf/Comments-0.sql" : [
+    2,
+    2,
+    3
+  ],
   "sqlrf/Comments-48.sql" : [
     2
   ],

diff --git a/...s/src/integrationTest/resources/expected/oracle-database_23/ExplicitInParameterCheck.json b/...s/src/integrationTest/resources/expected/oracle-database_23/ExplicitInParameterCheck.json
@@ -35,37 +35,37 @@
     27,
     28,
     30,
-    42,
-    42,
-    42,
-    56,
-    56,
-    66,
-    66,
+    41,
+    41,
+    41,
+    55,
+    55,
+    65,
+    65,
+    74,
+    74,
     75,
     75,
-    76,
-    76,
+    86,
+    86,
+    86,
     87,
-    87,
-    87,
-    88,
-    99,
+    98,
+    98,
+    98,
     99,
     99,
-    100,
-    100,
+    109,
+    110,
     110,
     111,
     111,
+    111,
     112,
     112,
-    112,
-    113,
-    113,
+    174,
     175,
-    176,
-    203
+    202
   ],
   "sqlrf/Using-Extensible-Indexing-2.sql" : [
     3,

diff --git a/.../integrationTest/resources/expected/oracle-database_23/FunctionWithOutParameterCheck.json b/.../integrationTest/resources/expected/oracle-database_23/FunctionWithOutParameterCheck.json
@@ -4,10 +4,10 @@
     23,
     27,
     28,
-    36,
-    110,
-    175,
-    176
+    35,
+    109,
+    174,
+    175
   ],
   "sqlrf/Using-Extensible-Indexing-2.sql" : [
     5

diff --git a/...hecks/src/integrationTest/resources/expected/oracle-database_23/InequalityUsageCheck.json b/...hecks/src/integrationTest/resources/expected/oracle-database_23/InequalityUsageCheck.json
@@ -12,11 +12,11 @@
     5
   ],
   "sqlrf/Using-Extensible-Indexing-1.sql" : [
+    130,
+    130,
     131,
     131,
-    132,
-    132,
-    151
+    150
   ],
   "sqlrf/Using-Extensible-Indexing-2.sql" : [
     39

diff --git a/.../src/integrationTest/resources/expected/oracle-database_23/InsertWithoutColumnsCheck.json b/.../src/integrationTest/resources/expected/oracle-database_23/InsertWithoutColumnsCheck.json
@@ -1,32 +1,35 @@
 {
   "sqlrf/ALTER-SESSION-13.sql" : [
-    5
+    4
   ],
   "sqlrf/ALTER-TABLE-114.sql" : [
     2,
-    5
+    4
   ],
   "sqlrf/ALTER-TABLE-31.sql" : [
+    3,
     4,
     5,
     6,
-    7,
-    8
+    7
   ],
   "sqlrf/ALTER-TABLE-91.sql" : [
+    4,
     5,
-    6,
-    7
+    6
   ],
   "sqlrf/COLLATION-0.sql" : [
-    6
+    5
   ],
   "sqlrf/COMMIT-0.sql" : [
     2
   ],
   "sqlrf/CREATE-TABLE-50.sql" : [
     2
   ],
+  "sqlrf/CREATE-VIEW-7.sql" : [
+    2
+  ],
   "sqlrf/CURRENT_TIMESTAMP-2.sql" : [
     2
   ],
@@ -38,7 +41,7 @@
   ],
   "sqlrf/DEREF-0.sql" : [
     2,
-    5
+    4
   ],
   "sqlrf/Data-Types-18.sql" : [
     2
@@ -57,13 +60,13 @@
     9
   ],
   "sqlrf/Data-Types-3.sql" : [
+    3,
     4,
     5,
-    6,
-    7
+    6
   ],
   "sqlrf/HEXTORAW-0.sql" : [
-    4
+    3
   ],
   "sqlrf/INSERT-0.sql" : [
     2
@@ -139,6 +142,7 @@
     3
   ],
   "sqlrf/SELECT-102.sql" : [
+    3,
     4,
     5,
     6,
@@ -157,17 +161,16 @@
     19,
     20,
     21,
-    22,
-    23
+    22
   ],
   "sqlrf/SELECT-60.sql" : [
     2
   ],
   "sqlrf/SELECT-77.sql" : [
+    5,
     6,
     7,
-    8,
-    9
+    8
   ],
   "sqlrf/SELECT-86.sql" : [
     2
@@ -196,8 +199,8 @@
   ],
   "sqlrf/SQL-JSON-Conditions-37.sql" : [
     2,
-    5,
-    8
+    4,
+    6
   ],
   "sqlrf/Sequence-Pseudocolumns-4.sql" : [
     2
@@ -206,27 +209,39 @@
     2
   ],
   "sqlrf/TO_BINARY_DOUBLE-0.sql" : [
-    5
+    4
   ],
   "sqlrf/TO_CHAR-datetime-2.sql" : [
     3,
     7
   ],
-  "sqlrf/TRANSLATE-USING-0.sql" : [
-    4
-  ],
   "sqlrf/Type-Constructor-Expressions-0.sql" : [
     8
   ],
   "sqlrf/Using-XML-in-SQL-Statements-1.sql" : [
     2
   ],
+  "sqlrf/Using-XML-in-SQL-Statements-5.sql" : [
+    2
+  ],
+  "sqlrf/XMLTRANSFORM-0.sql" : [
+    3
+  ],
+  "sqlrf/create-domain-20.sql" : [
+    2
+  ],
+  "sqlrf/create-domain-21.sql" : [
+    2
+  ],
   "sqlrf/domain_name-2.sql" : [
     2
   ],
   "sqlrf/domain_name-6.sql" : [
     2
   ],
+  "sqlrf/sys_row_etag-0.sql" : [
+    3
+  ],
   "sqlrf/to_vector-1.sql" : [
     2
   ],