Support migration to datomic, support for saving datoms in nippy, refactoring of core

.gitignore

@@ -2,8 +2,6 @@
 /classes
 /checkouts
 profiles.clj
-pom.xml
-pom.xml.asc
 *.jar
 *.class
 /.lein-*

README.md

@@ -7,7 +7,7 @@ Migration tool for Datahike from and to other databases.
 Make sure your source and target databases exist. You can run the migration on the commandline:
 
 ```bash
-lein run -s datomic-cloud.edn -t datahike-pg.edn -d datomic-cloud:datahike
+lein run -s datomic-cloud.edn -t datahike-pg.edn
 ```
 
 Use `lein run -- -h` for further instructions. See the `*-example.edn` files for `dataomic-cloud` and `datahike-pg` example configurations.
@@ -17,23 +17,32 @@ Alternatively open your Clojure project, add `io.lambdaforge/wanderung` to your
 ```clojure
 (require '[wanderung.core :as w])
 
-(def datomic-cfg {:name "your-database"
+(def datomic-cfg {:wanderung/type :datomic
+                  :name "your-database"
                   :server-type :ion
                   :region      "eu-west-1"
                   :system      "your-system"
                   :endpoint    "http://entry.your-system.eu-west-1.datomic.net:8182/"
                   :proxy-port  8182})
 
-(def datahike-cfg {:store {:backend :file
+(def datahike-cfg {:wanderung/type :datahike
+                   :store {:backend :file
                            :path "/your-data-path"}
                    :name "from-datomic"
                    :schema-flexibility :write
-                   :keep-history? true}) 
+                   :keep-history? true})
 ;; if the database doesn't exist, wanderung will create a Datahike database
-                           
-(w/migrate [:datomic-cloud :datahike] datomic-cfg datahike-cfg)
+
+(w/migrate datomic-cfg datahike-cfg)
 ```
 
+## Tests
+
+Before using Wanderung for performing a migration, you may wish to run tests that to check that Wanderung works correctly. In order to do so, you need to perform the following steps:
+
+1. Install [Datomic dev-local](https://docs.datomic.com/cloud/dev-local.html).
+2. Run the tests by calling `lein test`. In case they fail or in case there are errors, do `lein clean` and attempt to run the tests again.
+
 ## License
 
 Copyright © 2020 lambdaforge UG (haftungsbeschränkt)

datahike-pg-example.edn

@@ -1,4 +1,5 @@
-{:store {:backend :jdbc
+{:wanderung/type :datahike
+ :store {:backend :jdbc
          :dbtype "postgresql"
          :user "datahike"
          :password "is-awesome"

datomic-cloud-example.edn

@@ -1,4 +1,5 @@
-{:name "your-database"
+{:wanderung/type :datomic
+ :name "your-database"
  :server-type :ion
  :region      "eu-west-1"
  :system      "your-system"

nippy-example.edn

@@ -0,0 +1,3 @@
+;; Store the raw datoms as nippy-encoded data in a file.
+{:wanderung/type :nippy
+ :filename "backups/mybackup.nippy"}

pom.xml

@@ -0,0 +1,137 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>io.lambdaforge</groupId>
+  <artifactId>wanderung</artifactId>
+  <packaging>jar</packaging>
+  <version>0.1.1-SNAPSHOT</version>
+  <name>wanderung</name>
+  <description>Data migration tool for Datahike</description>
+  <url>http://example.com/FIXME</url>
+  <licenses>
+    <license>
+      <name>EPL-2.0 OR GPL-2.0-or-later WITH Classpath-exception-2.0</name>
+      <url>https://www.eclipse.org/legal/epl-2.0/</url>
+    </license>
+  </licenses>
+  <scm>
+    <url>https://github.com/lambdaforge/wanderung</url>
+    <connection>scm:git:git://github.com/lambdaforge/wanderung.git</connection>
+    <developerConnection>scm:git:ssh://git@github.com/lambdaforge/wanderung.git</developerConnection>
+    <tag>a5d6d5b6e2f3018a996dd2aad04d477938900aeb</tag>
+  </scm>
+  <build>
+    <sourceDirectory>src</sourceDirectory>
+    <testSourceDirectory>test</testSourceDirectory>
+    <resources>
+      <resource>
+        <directory>resources</directory>
+      </resource>
+    </resources>
+    <testResources>
+      <testResource>
+        <directory>resources</directory>
+      </testResource>
+    </testResources>
+    <directory>target</directory>
+    <outputDirectory>target/classes</outputDirectory>
+    <plugins/>
+  </build>
+  <repositories>
+    <repository>
+      <id>central</id>
+      <url>https://repo1.maven.org/maven2/</url>
+      <snapshots>
+        <enabled>false</enabled>
+      </snapshots>
+      <releases>
+        <enabled>true</enabled>
+      </releases>
+    </repository>
+    <repository>
+      <id>clojars</id>
+      <url>https://repo.clojars.org/</url>
+      <snapshots>
+        <enabled>true</enabled>
+      </snapshots>
+      <releases>
+        <enabled>true</enabled>
+      </releases>
+    </repository>
+  </repositories>
+  <dependencyManagement>
+    <dependencies/>
+  </dependencyManagement>
+  <dependencies>
+    <dependency>
+      <groupId>org.clojure</groupId>
+      <artifactId>clojure</artifactId>
+      <version>1.10.1</version>
+    </dependency>
+    <dependency>
+      <groupId>io.replikativ</groupId>
+      <artifactId>datahike-jdbc</artifactId>
+      <version>0.1.2-SNAPSHOT</version>
+    </dependency>
+    <dependency>
+      <groupId>io.replikativ</groupId>
+      <artifactId>datahike</artifactId>
+      <version>0.3.3-SNAPSHOT</version>
+      <exclusions>
+        <exclusion>
+          <artifactId>konserve</artifactId>
+          <groupId>io.replikativ</groupId>
+        </exclusion>
+      </exclusions>
+    </dependency>
+    <dependency>
+      <groupId>com.datomic</groupId>
+      <artifactId>client-cloud</artifactId>
+      <version>0.8.78</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.datomic</groupId>
+      <artifactId>client-pro</artifactId>
+      <version>0.9.63</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.datomic</groupId>
+      <artifactId>dev-local</artifactId>
+      <version>0.9.232</version>
+    </dependency>
+    <dependency>
+      <groupId>com.cognitect</groupId>
+      <artifactId>transit-clj</artifactId>
+      <version>0.8.313</version>
+    </dependency>
+    <dependency>
+      <groupId>com.taoensso</groupId>
+      <artifactId>nippy</artifactId>
+      <version>3.1.1</version>
+    </dependency>
+    <dependency>
+      <groupId>org.clojure</groupId>
+      <artifactId>tools.cli</artifactId>
+      <version>1.0.206</version>
+    </dependency>
+    <dependency>
+      <groupId>org.clojure</groupId>
+      <artifactId>test.check</artifactId>
+      <version>0.9.0</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>lambdaisland</groupId>
+      <artifactId>kaocha</artifactId>
+      <version>1.0.632</version>
+      <scope>test</scope>
+    </dependency>
+  </dependencies>
+</project>
+
+<!-- This file was autogenerated by Leiningen.
+  Please do not edit it directly; instead edit project.clj and regenerate it.
+  It should not be considered canonical data. For more information see
+  https://github.com/technomancy/leiningen -->

project.clj

@@ -7,8 +7,13 @@
                  [io.replikativ/datahike-jdbc "0.1.2-SNAPSHOT"]
                  [io.replikativ/datahike "0.3.3-SNAPSHOT" :exclusions [io.replikativ/konserve]]
                  [com.datomic/client-cloud "0.8.78" :scope "provided"]
+                 [com.datomic/client-pro "0.9.63" :scope "provided"]
+                 [com.datomic/dev-local "0.9.232"]
                  [com.cognitect/transit-clj "0.8.313"]
-                 [org.clojure/tools.cli "1.0.194"]]
-  :profiles {:dev {:dependencies [[org.clojure/test.check "0.9.0"]]}}
+                 [com.taoensso/nippy "3.1.1"]
+                 [org.clojure/tools.cli "1.0.206"]]
+  :resource-paths ["resources"]
+  :profiles {:dev {:dependencies [[org.clojure/test.check "0.9.0"]]}
+             :test {:dependencies [[lambdaisland/kaocha "1.0.632"]]}}
   :main wanderung.core
   :repl-options {:init-ns wanderung.core})

resources/testdatoms.edn

@@ -0,0 +1 @@
+[[13194139534312 :db/txInstant #inst "2021-06-02T09:45:14.334-00:00" 13194139534312 true] [17592186045417 :db/ident :small 13194139534312 true] [17592186045418 :db/ident :medium 13194139534312 true] [17592186045419 :db/ident :large 13194139534312 true] [17592186045420 :db/ident :xlarge 13194139534312 true] [17592186045421 :db/ident :shirt 13194139534312 true] [17592186045422 :db/ident :pants 13194139534312 true] [17592186045423 :db/ident :dress 13194139534312 true] [17592186045424 :db/ident :hat 13194139534312 true] [17592186045425 :db/ident :red 13194139534312 true] [17592186045426 :db/ident :green 13194139534312 true] [17592186045427 :db/ident :blue 13194139534312 true] [17592186045428 :db/ident :yellow 13194139534312 true] [13194139534325 :db/txInstant #inst "2021-06-02T09:45:14.354-00:00" 13194139534325 true] [0 :db.install/attribute 72 13194139534325 true] [0 :db.install/attribute 73 13194139534325 true] [0 :db.install/attribute 74 13194139534325 true] [0 :db.install/attribute 75 13194139534325 true] [72 :db/unique :db.unique/identity 13194139534325 true] [72 :db/ident :inv/sku 13194139534325 true] [72 :db/cardinality :db.cardinality/one 13194139534325 true] [72 :db/valueType :db.type/string 13194139534325 true] [73 :db/cardinality :db.cardinality/one 13194139534325 true] [73 :db/valueType :db.type/ref 13194139534325 true] [73 :db/ident :inv/color 13194139534325 true] [74 :db/valueType :db.type/ref 13194139534325 true] [74 :db/cardinality :db.cardinality/one 13194139534325 true] [74 :db/ident :inv/size 13194139534325 true] [75 :db/cardinality :db.cardinality/one 13194139534325 true] [75 :db/valueType :db.type/ref 13194139534325 true] [75 :db/ident :inv/type 13194139534325 true] [13194139534326 :db/txInstant #inst "2021-06-02T09:45:14.382-00:00" 13194139534326 true] [17592186045431 :inv/color 17592186045425 13194139534326 true] [17592186045431 :inv/type 17592186045421 13194139534326 true] [17592186045431 :inv/sku "SKU-0" 13194139534326 true] [17592186045431 :inv/size 17592186045417 13194139534326 true] [17592186045432 :inv/size 17592186045417 13194139534326 true] [17592186045432 :inv/sku "SKU-1" 13194139534326 true] [17592186045432 :inv/color 17592186045425 13194139534326 true] [17592186045432 :inv/type 17592186045422 13194139534326 true] [17592186045433 :inv/type 17592186045423 13194139534326 true] [17592186045433 :inv/size 17592186045417 13194139534326 true] [17592186045433 :inv/color 17592186045425 13194139534326 true] [17592186045433 :inv/sku "SKU-2" 13194139534326 true] [17592186045434 :inv/sku "SKU-3" 13194139534326 true] [17592186045434 :inv/color 17592186045425 13194139534326 true] [17592186045434 :inv/size 17592186045417 13194139534326 true] [17592186045434 :inv/type 17592186045424 13194139534326 true] [17592186045435 :inv/sku "SKU-4" 13194139534326 true] [17592186045435 :inv/type 17592186045421 13194139534326 true] [17592186045435 :inv/color 17592186045425 13194139534326 true] [17592186045435 :inv/size 17592186045418 13194139534326 true]]

src/wanderung/core.clj

@@ -2,34 +2,80 @@
   (:require [datahike.api :as d]
             [datomic.client.api :as dt]
             [wanderung.datomic-cloud :as wdc]
+            [wanderung.datahike :as wd]
+            [wanderung.datom :as datom]
             [clojure.tools.cli :refer [parse-opts]]
             [clojure.string :refer [split]]
             [clojure.java.io :as io]
+            [taoensso.nippy :as nippy]
             [datahike-jdbc.core])
   (:gen-class))
 
-(defmulti migrate (fn [direction source-configuration target-configuration] direction))
-
-(defmethod migrate [:datomic-cloud :datahike] [_ datomic-config datahike-config]
-  (let [datomic-conn (dt/connect (dt/client (dissoc datomic-config :name)) {:db-name (:name datomic-config)})
-        datomic-data (wdc/extract-datomic-cloud-data datomic-conn)
-        datahike-conn (if (d/database-exists? datahike-config)
-                        (do
-                          (println "➜ Connecting to Datahike...")
-                          (d/connect datahike-config))
-                        (do
-                          (println "➜ Datahike database does not exist.")
-                          (println "➜ Creating database...")
-                          (d/create-database datahike-config)
-                          (println "  ✓ Done")
-                          (println "➜ Connecting to Datahike...")
-                          (d/connect datahike-config)))]
+
+;;;------- Basic datoms interface -------
+
+;; Two elementary methods for moving datoms to and from a database.
+(defmulti datoms-from-storage (fn [config] (:wanderung/type config)))
+(defmulti datoms-to-storage (fn [config datoms] (:wanderung/type config)))
+
+;;; Datomic
+(defn datomic-connect [datomic-config]
+  (dt/connect (dt/client (dissoc datomic-config :name))
+              {:db-name (:name datomic-config)}))
+
+(defmethod datoms-from-storage :datomic [storage]
+  (-> storage
+      datomic-connect
+      wdc/extract-datomic-cloud-data))
+
+(defmethod datoms-to-storage :datomic [storage datoms]
+  (wdc/transact-datoms-to-datomic (datomic-connect storage) datoms))
+
+;;; Nippy
+(defmethod datoms-from-storage :nippy [storage]
+  (nippy/thaw-from-file (:filename storage)))
+
+(defmethod datoms-to-storage :nippy [storage datoms]
+  (let [filename (:filename storage)]
+    (io/make-parents filename)
+    (nippy/freeze-to-file filename datoms)))
+
+;;; Datahike
+(defn datahike-maybe-create-and-connect [config]
+  (when (not (d/database-exists? config))
+       (println "➜ Datahike database does not exist.")
+       (println "➜ Creating database...")
+       (d/create-database config)
+       (println "  ✓ Done"))
+  (println "➜ Connecting to Datahike...")
+  (let [result (d/connect config)]
     (println "  ✓ Done")
-    @(d/load-entities datahike-conn datomic-data)
-    true))
+    result))
+
+(defmethod datoms-from-storage :datahike [storage]
+  (-> storage
+      d/connect
+      wd/extract-datahike-data))
+
+(defmethod datoms-to-storage :datahike [storage datoms]
+  @(d/load-entities (datahike-maybe-create-and-connect storage) datoms))
+
+;;;------- Migrations -------
 
-(defmethod migrate :default [direction _ _]
-  (throw (IllegalArgumentException. (str "Direction " direction " not supported."))))
+(defmulti migrate (fn [source-configuration target-configuration]
+                    (mapv :wanderung/type [source-configuration
+                                           target-configuration])))
+
+#_(defmethod migrate [:datahike :datahike] [src-cfg tgt-cfg]
+    ... optimized implementation for specific migration goes here ...)
+
+(defmethod migrate :default [src tgt]
+  (->> (datoms-from-storage src) (datoms-to-storage tgt)))
+
+;;;------- CLI -------
+
+(defn multimethod-for-dispatch-value? [method x]
+  (-> (methods method) keys set (contains? x)))
 
 (def cli-options
   [["-s" "--source SOURCE" "Source EDN configuration file"
@@ -38,24 +84,57 @@
    ["-t" "--target TARGET" "Target EDN configuration file"
     :parse-fn identity
     :validate [#(.exists (io/file %)) "Target configuration does not exist."]]
-   ["-d" "--direction SOURCE_SYSTEM:TARGET_SYSTEM" "Migration directions separated by colon: e.g. datomic-cloud:datahike"
-    :parse-fn #(->> (split % #":") (mapv keyword))
-    :validate [#(-> (methods migrate) keys set (contains? %)) "Direction not allowed."]]
-   ["-h" "--help"]])
+   ["-h" "--help"]
+   ["-c" "--check"]])
+
+(defn load-config [filename]
+  (-> filename
+      slurp
+      read-string))
+
+(defn execute-migration [options]
+  (let [{:keys [source target help check]} options
+        src-cfg (load-config source)
+        tgt-cfg (load-config target)
+        src-type (:wanderung/type src-cfg)
+        tgt-type (:wanderung/type tgt-cfg)]
+    (cond
+      (not (multimethod-for-dispatch-value? datoms-from-storage src-type))
+      (println "Cannot use" src-type "as source database.")
+
+      (not (multimethod-for-dispatch-value? datoms-to-storage tgt-type))
+      (println "Cannot use" tgt-type "as target database.")
+
+      (:wanderung/read-only? tgt-cfg)
+      (println "Cannot migrate to read-only database.")
+
+      :default (do
+                 (println "➜ Start migrating from" src-type "to" tgt-type "...")
+                 (migrate src-cfg tgt-cfg)
+                 (println "  ✓ Done")
+                 (when check
+                   (if (multimethod-for-dispatch-value? datoms-from-storage tgt-type)
+                     (do
+                       (println "➜ Comparing datoms between source and target...")
+                       (if (datom/similar-datoms? (datoms-from-storage src-cfg)
+                                                  (datoms-from-storage tgt-cfg))
+                         (println "  ✓ Success: Datoms look the same.")
+                         (println "ERROR: The datoms differ between source and target.")))
+                     (println "ERROR: The target does not support reading datoms")))))))
+
 
 (defn -main [& args]
-  (let [{{:keys [source target direction help]} :options summary :summary errors :errors} (parse-opts args cli-options)]
+  (let [{options :options
+         summary :summary
+         errors :errors} (parse-opts args cli-options)]
     (if errors
       (->> errors (map println) doall)
-      (if help
+      (if (:help options)
         (do
           (println "Run migrations to datahike from various sources")
           (println "USAGE:")
           (println summary))
-        (do
-          (println "➜ Start migrating from " (first direction) "to" (second direction) "...")
-          (migrate direction (-> source slurp read-string) (-> target slurp read-string))
-          (println "  ✓ Done"))))))
+        (execute-migration options)))))
 
 (comment