29 validity check email format

.cljfmt.edn

@@ -0,0 +1 @@
+{:paths ["src" "tests/unit" "tests/integration"]}

.python-version

@@ -0,0 +1 @@
+3.10.0/envs/dqt

README.org

@@ -166,11 +166,11 @@ Run =docker compose up= to have postgress running
   bb test:watch
 #+end_src
 
-#+begin_src
+#+begin_src shell
   $ bin/koacha
 #+end_src
 
-#+begin_src
+#+begin_src shell
   $ bin/koacha --watch
 #+end_src
 

bin/dqt

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -ex
+
+# export CLASSPATH=//.m2/repository/org/postgresql/postgresql/42.3.5/postgresql-42.3.5.jar
+# export CLASSPATH=/Users/WarFox/.m2/repository/org/postgresql/postgresql/42.3.5/postgresql-42.3.5.jar
+
+# java -cp ~/.m2/repository/org/postgresql/postgresql/42.3.5/postgresql-42.3.5.jar:./target/dqt-0.1.0-SNAPSHOT.jar dqt.core run -d examples/postgres.edn -t examples/tables/employees.edn
+# java -cp ./target/dqt-0.1.0-SNAPSHOT.jar dqt.core run -d examples/postgres.edn -t examples/tables/employees.edn
+# java -jar ./target/dqt-0.1.0-SNAPSHOT.jar run -d examples/postgres.edn -t examples/tables/employees.edn
+
+# java -cp ":./target/dqt-0.1.0-SNAPSHOT.jar" dqt.core run -d examples/postgres.edn -t examples/tables/employees.edn
+
+java -cp "/Users/WarFox/.m2/repository/org/postgresql/postgresql/42.3.5/postgresql-42.3.5.jar:./target/dqt-0.1.0-SNAPSHOT.jar" dqt.core run -d examples/postgres.edn -t examples/tables/employees.edn

dev/user.clj

@@ -0,0 +1,142 @@
+(ns user
+  (:require
+    [dqt.core :as core]
+    [honey.sql :as honey]
+    [honey.sql.helpers :refer [select from where]]
+    [next.jdbc :as jdbc]
+    [next.jdbc.sql :as sql]))
+
+
+(def db
+  {:dbtype     "postgresql"
+   :dbname     "postgres"
+   :host       "localhost"
+   :port       5432
+   :user       "postgres"
+   :password   "postgres"
+   :ssl        false
+   :sslfactory "org.postgresql.ssl.NonValidatingFactory"})
+
+
+(def sqlmap
+  {:select [:first-name :last-name :hired-date]
+   :from    [:employees]
+   :where   [:= :employee-id 100]})
+
+
+;; information-schema column
+(->> {:select [:column-name :data-type :is-nullable]
+      :from   :information_schema.columns
+      :where  [:= :table-name "employees"]}
+     honey/format
+     (jdbc/execute! db))
+
+
+(-> (select [[:count :*]])
+    (from :foo)
+    honey/format)
+
+
+(-> (select :*)
+    (select :employee-id)
+    (select [[:count :*] :my-count])
+    (select [[:avg :salary] :average-salary])
+    (from :foo)
+    (where [:= :a 1] [:< :b 100])
+    honey/format)
+
+
+(-> {:select [:id
+              [[:over
+                [[:avg :salary]
+                 {:partition-by [:department]
+                  :order-by     [:designation]}
+                 :Average]
+                [[:max :salary]
+                 :w
+                 :MaxSalary]]]]
+     :from   [:employee]
+     :window [:w {:partition-by [:department]}]}
+    (honey/format {:pretty true}))
+
+
+(-> (select :%count.*) (from :foo) honey/format)
+
+(-> (select [:%count.*]) (from :foo) honey/format)
+
+(-> (select [[:count :*]]) (from :foo) honey/format)
+
+(-> (select :%max.id) (from :foo) honey/format)
+
+
+;; not a list, but vectors or symbols
+(-> (select [[:count :*] :my-count]
+            :employee-id
+            [[:avg :salary] :average-salary])
+    honey/format)
+
+
+;; need a list
+(-> {:select [[[:count :*] :my-count]
+              :employee-id
+              [[:avg :salary] :average-salary]]}
+    honey/format)
+
+
+;; function call with :% notation
+(-> {:select [[:%count.* :my-count]
+              :employee-id
+              [:%avg.salary :average-salary]]}
+    honey/format)
+
+
+(jdbc/execute! ds
+               (honey/format (count* :employees)))
+
+
+(sql/query ds
+           (honey/format (count* :employees)))
+
+
+(comment
+
+  (defn create-files
+    []
+    (migratus/create {:migration-dir "migrations"} "regions")
+    (migratus/create {:migration-dir "migrations"} "countries")
+    (migratus/create {:migration-dir "migrations"} "locations")
+    (migratus/create {:migration-dir "migrations"} "departments")
+    (migratus/create {:migration-dir "migrations"} "jobs")
+    (migratus/create {:migration-dir "migrations"} "employees")
+    (migratus/create {:migration-dir "migrations"} "dependents")))
+
+
+(defn this-jar
+  "utility function to get the name of jar in which this function is invoked"
+  [& [ns]]
+  (-> (or ns (class *ns*))
+      .getProtectionDomain .getCodeSource .getLocation .toURI .getPath))
+
+
+(defn version-3
+  []
+  (this-jar "dqt.cli"))
+
+
+(defn version-2
+  []
+  (let [location (..
+                   (Class/forName "dqt.core.$_main")
+                   getProtectionDomain
+                   getCodeSource
+                   getLocation)]
+    (println location)
+    (println "Version "
+             (.getValue
+               (..
+                 (java.util.jar.Manifest.
+                   (.openStream
+                     (java.net.URL.
+                       (str "jar:" location "!/META-INF/MANIFEST.MF"))))
+                 getMainAttributes)
+               "Build-number"))))

examples/postgres.edn

@@ -0,0 +1,8 @@
+{:dbtype     "postgresql"
+ :dbname     "postgres"
+ :host       #or [#env DATABASE_HOSTNAME "localhost"]
+ :user       "postgres"
+ :password   "postgres"
+ :ssl        false
+ :classname  "org.postgres.Driver"
+ :sslfactory "org.postgresql.ssl.NonValidatingFactory"}

examples/tables/employees.edn

@@ -0,0 +1,38 @@
+{:table-name :employees
+ :metrics    [:row-count
+              :missing-count
+              :missing-percentage
+              :values-count
+              :values-percentage
+              :invalid-count
+              :invalid-percentage
+              :valid-count
+              :valid-percentage
+              :avg-length
+              :max-length
+              :min-length
+              :avg
+              :sum
+              :max
+              :min
+              :stddev
+              :variance]
+
+ :tests      [[:row-count = 40]
+              [:avg-length-phone-number = 12]
+              [:stddev-salary > 4500]
+              [:sum-salary > 20000]
+              [:count-manager-id = 39]
+              [:missing-count-manager-id = 0]
+              [:missing-percentage-manager-id < 20]
+              [:values-percentage-manager-id < 30]
+              [:max-length-email <= 35]]
+
+ :columns    [:dob
+              {:valid-format :date
+               :tests
+               [[:invalid-percentage == 0]]}
+              :first-name
+              {:valid-format :number-percentage
+               :tests
+               [[:invalid-percentage == 0]]}]}

resources/sql/information-schema.sql

@@ -0,0 +1,3 @@
+SELECT column_name, data_type, is_nullable
+  FROM information_schema.COLUMNS
+ WHERE TABLE_NAME = 'employees';

resources/sql/row-count.sql

@@ -0,0 +1,3 @@
+-- :name count
+SELECT COUNT(*)
+FROM employees;

src/dqt/config.clj

@@ -0,0 +1 @@
+(ns dqt.config)

src/dqt/driver.clj

@@ -0,0 +1,27 @@
+(ns dqt.driver
+  (:import
+   (java.sql
+    Driver
+    DriverManager)))
+
+;; (defn register
+;;   [driver]
+
+;;   )
+;; Class drvClass = driverLoader.loadClass(driverClassName);
+;; Driver driver = drvClass.newInstance();
+;; (DriverManager/registerDriver)
+
+(comment
+  ;; for (Enumeration<Driver> e = DriverManager.getDrivers(); e.hasMoreElements();) {
+  ;;                          Driver driver = e.nextElement();
+  ;;                          drivers.add(driver.getClass().getName());
+                         ;; }
+  )
+(defn- classes
+  [driver]
+  (-> driver .getClass .getName))
+
+(defn drivers
+  []
+  (map classes (enumeration-seq (DriverManager/getDrivers))))

src/dqt/query_runner.clj

@@ -4,12 +4,16 @@
    [next.jdbc :as jdbc]
    [next.jdbc.result-set :as rs]))
 
+(defn ->sql
+  [q]
+  (honey/format q))
+
 (defn execute!
   "Execute query and builds result set with keys in kebab case"
   ([db query]
    (execute! db query {}))
   ([db query opts]
-   (let [formatted-query (honey/format query)]
+   (let [formatted-query (->sql query)]
      (println formatted-query)
      (jdbc/execute! db formatted-query
                     (assoc opts
@@ -20,7 +24,7 @@
   ([db query]
    (execute-one! db query {}))
   ([db query opts]
-   (let [formatted-query (honey/format query)]
+   (let [formatted-query (->sql query)]
      (println formatted-query)
      (jdbc/execute-one! db formatted-query
                         (assoc opts

src/dqt/system.clj

@@ -18,8 +18,10 @@
                                       :columns-metadata (ig/ref ::columns-metadata-enriched))
    ::calculated-metrics        {:columns     (ig/ref ::columns-metadata-enriched)
                                 :sql-metrics (ig/ref ::sql-metrics)}
+   ::validity-checks           {:columns (ig/ref ::columns-metadata-enriched)}
    ::test-results              {:sql-metrics        (ig/ref ::sql-metrics)
                                 :calculated-metrics (ig/ref ::calculated-metrics)
+                                :validity-checks    (ig/ref ::validity-checks)
                                 :tests              (:tests options)}
    ::report                    (ig/ref ::test-results)})
 
@@ -39,6 +41,11 @@
   [_ {:keys [db table-name columns-metadata]}]
   (m/get-metrics db table-name columns-metadata))
 
+(defmethod ig/init-key ::validity-checks
+  [_ {:keys [columns]}]
+  (println "validity checks")
+  (println columns))
+
 (defmethod ig/init-key ::calculated-metrics
   [_ {:keys [columns sql-metrics]}]
   (into {} (map #(m/calculated-metrics % sql-metrics) columns)))

src/dqt/validity_checks.clj

@@ -0,0 +1,14 @@
+(ns dqt.validity-checks
+  (:require [dqt.query-runner :as q]))
+
+;; select EmailAddress from FindInvalidEmailAddressDemo
+;; -> where EmailAddress NOT LIKE '%_@_%._%';
+
+(def email-regex #"%_@_%._%")
+
+(defn email
+  "Returns list of invalid emails from the column"
+  [column table]
+  {:select [column]
+   :from table
+   :where [column :not :like email-regex]})

tables/department.yml

@@ -0,0 +1,22 @@
+table_name: department
+metrics:
+  - row_count
+  - missing_count
+  - missing_percentage
+  - values_count
+  - values_percentage
+  - invalid_count
+  - invalid_percentage
+  - valid_count
+  - valid_percentage
+  - avg_length
+  - max_length
+  - min_length
+  - avg
+  - sum
+  - max
+  - min
+  - stddev
+  - variance
+tests:
+  - row_count > 0

tables/employee_department.yml

@@ -0,0 +1,22 @@
+table_name: employee_department
+metrics:
+  - row_count
+  - missing_count
+  - missing_percentage
+  - values_count
+  - values_percentage
+  - invalid_count
+  - invalid_percentage
+  - valid_count
+  - valid_percentage
+  - avg_length
+  - max_length
+  - min_length
+  - avg
+  - sum
+  - max
+  - min
+  - stddev
+  - variance
+tests:
+  - row_count > 0