Enum Support (#6)

* Support enums and nested types in translate * Tests passing * Handle enums in codec. * Versioning/README stuff * Formatting issues / unused imports * Better README for lein plugin * Formatting issues / unused imports
datopia · Oct 17, 2021 · 1f257ae · 1f257ae
1 parent baa0b5b
commit 1f257ae
Show file tree

Hide file tree

Showing 14 changed files with 194 additions and 50 deletions.
diff --git a/README.md b/README.md
@@ -4,19 +4,19 @@ Clojure-idiomatic protobuf3 schema generation & codec functionality.
 
 ## Subprojects
 
-### [io.datopia/stickler-translate](translate)
+### [org.datopia/stickler-translate](translate)
 
 protobuf->edn schema generation library.
 
 [![Clojars
-Project](http://clojars.org/io.datopia/stickler-translate/latest-version.svg)](http://clojars.org/io.datopia/stickler-translate)
+Project](http://clojars.org/org.datopia/stickler-translate/latest-version.svg)](http://clojars.org/org.datopia/stickler-translate)
 
-### [io.datopia/stickler-codec](codec)
+### [org.datopia/stickler-codec](codec)
 
 Runtime codec library.
 
 [![Clojars
-Project](http://clojars.org/io.datopia/stickler-codec/latest-version.svg)](http://clojars.org/io.datopia/stickler-codec)
+Project](http://clojars.org/org.datopia/stickler-codec/latest-version.svg)](http://clojars.org/org.datopia/stickler-codec)
 
 ## Contributors
 

diff --git a/codec/README.md b/codec/README.md
@@ -1,10 +1,10 @@
-# io.datopia/stickler-codec
+# org.datopia/stickler-codec
 
  [![Clojars
-Project](http://clojars.org/io.datopia/stickler-codec/latest-version.svg)](http://clojars.org/io.datopia/stickler-codec)
+Project](http://clojars.org/org.datopia/stickler-codec/latest-version.svg)](http://clojars.org/org.datopia/stickler-codec)
 
 Small runtime library for encoding/decoding protobuf messages via schemas
-generated by [io.datopia/stickler-translate](../translate).
+generated by [org.datopia/stickler-translate](../translate).
 
 ## Documentation
  - [API Docs](https://datopia.github.io/stickler/stickler-codec/)

diff --git a/codec/project.clj b/codec/project.clj
@@ -1,4 +1,4 @@
-(defproject io.datopia/stickler-codec "0.1.1-SNAPSHOT"
+(defproject org.datopia/stickler-codec "0.1.1-SNAPSHOT"
   :description        "Idiomatic Clojure codec functionality for protobuf3."
   :url                "https://github.com/datopia/stickler"
   :license            {:name "MIT License"
@@ -9,6 +9,7 @@
   :java-source-paths  ["src/java"]
   :profiles
   {:dev {:java-source-paths ["test/gen-java"]
+         :resource-paths    ["test/resources"]
          :global-vars       {*warn-on-reflection* true}
          :aliases
          {"test-prep" ["run" "-m" "stickler.test-prep"]}
@@ -17,7 +18,7 @@
                         :metadata   {:doc/format :markdown}
                         :themes     [:default [:datopia {:datopia/github "https://github.com/datopia/stickler"}]]}
          :dependencies
-         [[io.datopia/stickler-translate         "0.1.0"]
+         [[org.datopia/stickler-translate        "0.1.1-SNAPSHOT"]
           [io.datopia/codox-theme                "0.1.0"]
           [com.squareup.wire/wire-java-generator "2.3.0-RC1"]
           [org.clojure/test.check                "0.10.0-alpha3"]]}})
diff --git a/codec/src/stickler/codec.clj b/codec/src/stickler/codec.clj
@@ -1,6 +1,4 @@
 (ns stickler.codec
-  (:require [clojure.java.io :as io]
-            [clojure.edn     :as edn])
   (:import [java.io ByteArrayOutputStream ByteArrayInputStream]
            [java.nio.charset Charset]
            [io.datopia.stickler CodecUtil]))
@@ -47,6 +45,12 @@
 
 (declare encode-stream)
 
+(defn- encode-enum-field-value [schema ^ByteArrayInputStream stream field v]
+  {:pre [(keyword? v)]}
+  (let [t (:type field)
+        m (-> schema t :fields)]
+    (CodecUtil/writeVarint32 stream (unchecked-int (m v)))))
+
 (defn- encode-message-field-value [schema stream v]
   (let [sub-stream (ByteArrayOutputStream.)]
     (encode-stream schema sub-stream v)
@@ -72,7 +76,9 @@
     :double   (CodecUtil/writeDouble     stream (unchecked-double v))
     :bytes    (encode-byte-array stream v)
     :string   (encode-byte-array stream (.getBytes ^String v ^Charset utf8))
-    (encode-message-field-value schema stream v)))
+    (if (:enum? ((:type field) schema))
+      (encode-enum-field-value schema stream field v)
+      (encode-message-field-value schema stream v))))
 
 (defn- encode-packed-field [schema stream field v]
   (when-not (empty? v)
@@ -103,6 +109,12 @@
     (.read stream body 0 len)
     (decode-bytes schema (:type field) body)))
 
+(defn- decode-enum-field-value [schema ^ByteArrayInputStream stream field]
+  (let [x (CodecUtil/readVarint32 stream)
+        t (:type field)
+        m (-> schema t :tag->kw)]
+    (m x)))
+
 (defn- decode-field-value [schema ^ByteArrayInputStream stream field & [len]]
   (case (:type field)
     :uint32   (CodecUtil/readUnsigned32 stream)
@@ -122,7 +134,10 @@
     :double   (CodecUtil/readDouble     stream)
     :bytes    (decode-byte-array stream len)
     :string   (String. ^bytes (decode-byte-array stream len) ^Charset utf8)
-    (decode-message-field-value schema stream field)))
+
+    (if (:enum? ((:type field) schema))
+      (decode-enum-field-value   schema stream field)
+      (decode-message-field-value schema stream field))))
 
 (defn- decode-packed-field [schema ^ByteArrayInputStream stream field]
   (let [size (CodecUtil/readVarint32 stream)
@@ -198,12 +213,29 @@
          (compare t-a t-b))))
     fields))
 
+(defn- prepare-enums [schema]
+  (reduce
+   (fn [[msgs enums] [k msg]]
+     (if (:enum? msg)
+       [msgs (assoc enums k
+               (assoc msg :tag->kw (into {}
+                                     (for [[k v] (:fields msg)]
+                                       [v k]))))]
+       [(assoc msgs k msg) enums]))
+   [{} {}]
+   schema))
+
+(defn- prepare-messages [schema]
+  (for [[msg-k msg-schema] schema
+        :let [msg-schema (update msg-schema :fields sorted-map-by-tag)
+              tag->f
+              (into {}
+                (for [[field-k {tag :tag :as field}] (:fields msg-schema)]
+                  [tag (assoc field :name field-k)]))]]
+    [msg-k (assoc msg-schema :tag->field tag->f)]))
+
 (defn prepare-schema [schema]
-  (into {}
-    (for [[msg-k msg-schema] schema
-          :let [msg-schema (update msg-schema :fields sorted-map-by-tag)
-                tag->f
-                (into {}
-                  (for [[field-k {tag :tag :as field}] (:fields msg-schema)]
-                    [tag (assoc field :name field-k)]))]]
-      [msg-k (assoc msg-schema :tag->field tag->f)])))
+  (let [[schema enums] (prepare-enums schema)]
+    (reduce into {}
+            [enums
+             (prepare-messages schema)])))
diff --git a/codec/test/stickler/codec_test.clj b/codec/test/stickler/codec_test.clj
@@ -43,7 +43,8 @@
    :bool         gen/boolean
    :double       gen/double
    :float        gen-float
-   :stickler/msg (gen/return :stickler.test/Scalars)})
+   :stickler/msg (gen/return :stickler.test/Scalars)
+   :size         (gen/elements #{:SMALL :MEDIUM :LARGE})})
 
 (defn- map->gen [m]
   (let [m-gen (apply gen/tuple
@@ -193,6 +194,18 @@
     (Arrays/equals ^bytes (:bytes m)
                    ^bytes (:bytes (roundtrip m)))))
 
+(defspec enum-wire-symmetry trials
+  (wire-symmetry-prop
+   Scalars
+   {:stickler/msg :stickler.test/Scalars
+    :size         (gen/elements #{:SMALL :MEDIUM :LARGE})}
+   map->Scalars))
+
+(defspec enum-roundtrip trials
+  (prop/for-all [m (map->gen {:stickler/msg :stickler.test/Scalars
+                              :size         (gen/elements #{:SMALL :MEDIUM :LARGE})})]
+    (= (:size m) (:size (roundtrip m)))))
+
 (defspec message-wire-symmetry trials
   (wire-symmetry-prop Scalars gen-message map->Scalars))
 

diff --git a/codec/test/stickler/codec_test/util.clj b/codec/test/stickler/codec_test/util.clj
@@ -1,5 +1,7 @@
 (ns stickler.codec-test.util
-  (:import [stickler.test Scalars$Builder]
+  (:import [stickler.test
+            Scalars$Builder
+            Scalars$Size]
            [java.util Arrays]))
 
 ;; taken from encore
@@ -31,6 +33,8 @@
     (:float    m) (.float_   (:float  m))
     (:double   m) (.double_  (:double m))
 
+    (:size     m) (.size     (Scalars$Size/valueOf (name (:size m))))
+
     (contains? m :bool) (.bool (:bool m))))
 
 (defn map->Scalars [m]

diff --git a/codec/test/stickler/test_prep.clj b/codec/test/stickler/test_prep.clj
@@ -35,12 +35,11 @@
   (with-open [writer (io/writer (io/file out-dir "schema.edn"))]
     (pprint/write (stickler/Schema->edn schema) :stream writer)))
 
-(defn -main [& [out-dir schema-out]]
-  {:pre [out-dir]}
+(defn -main []
   (let [tmp-in  (.toFile (Files/createTempDirectory "proto" (make-array FileAttribute 0)))
         proto-f (io/file tmp-in "test.proto")
         in-res  (io/resource "test.proto")]
     (spit proto-f (slurp in-res))
     (let [schema (stickler/dirs->Schema tmp-in)]
-      (generate-java schema (io/file out-dir))
-      (generate-edn  schema (io/file schema-out)))))
+      (generate-java schema (io/file "test/gen-java"))
+      (generate-edn  schema (io/file "test/resources/")))))
diff --git a/lein-stickler/LICENSE b/lein-stickler/LICENSE
@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2018 <All contributors listed in the README.md file>.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/lein-stickler/README.md b/lein-stickler/README.md
@@ -0,0 +1,8 @@
+# lein-stickler
+
+With `org.datopia/lein-stickler` in your Leiningen profile, run `lein stickler
+:dirs dir [dirs ...]` to output an EDN schema to stdout.
+
+## License
+
+[MIT](LICENSE)
diff --git a/lein-stickler/project.clj b/lein-stickler/project.clj
@@ -0,0 +1,7 @@
+(defproject org.datopia/lein-stickler "0.1.0-SNAPSHOT"
+  :description       "Converts protobuf3 schemas into EDN maps."
+  :url               "https://github.com/datopia/stickler"
+  :license           {:name "MIT License"
+                      :url  "http://opensource.org/licenses/MIT"}
+  :dependencies      [[org.datopia/stickler-translate "0.1.1-SNAPSHOT"]]
+  :eval-in-leiningen true)
diff --git a/lein-stickler/src/leiningen/stickler.clj b/lein-stickler/src/leiningen/stickler.clj
@@ -0,0 +1,24 @@
+(ns leiningen.stickler
+  (:require [clojure.string     :as str]
+            [clojure.pprint     :as pprint]
+            [stickler.translate :as translate]))
+
+(defn- parse-args [args]
+  (let [args (map #(cond-> % (str/starts-with? % ":") read-string) args)
+        args (partition-by keyword? args)]
+    (reduce
+     (fn [acc [[arg] vs]]
+       (update acc arg (fnil into #{}) vs))
+     {} (partition 2 args))))
+
+(defn ^{:no-project-needed true} stickler
+  "protobuf3 -> EDN schema, for use with stickler.codec.
+
+   :dirs /proto ...
+      A sequence of absolute directories to process into a protobuf schema.
+   The EDN is pretty-printed to stdout"
+  [_ & args]
+  (let [args (parse-args args)]
+    (-> (apply translate/dirs->Schema (:dirs args))
+        translate/Schema->edn
+        pprint/pprint)))
diff --git a/translate/README.md b/translate/README.md
@@ -1,9 +1,9 @@
 # io.datopia/stickler-translate
 
  [![Clojars
-Project](http://clojars.org/io.datopia/stickler-translate/latest-version.svg)](http://clojars.org/io.datopia/stickler-translate)
+Project](http://clojars.org/org.datopia/stickler-translate/latest-version.svg)](http://clojars.org/org.datopia/stickler-translate)
 
-Conversion of on-disk protobuf3 files into EDN schemas, suitable as inputs for [io.datopia/stickler-codec](../codec).
+Conversion of on-disk protobuf3 files into EDN schemas, suitable as inputs for [org.datopia/stickler-codec](../codec).
 
 ## Documentation
  - [API Docs](https://datopia.github.io/stickler/stickler-translate/)

diff --git a/translate/project.clj b/translate/project.clj
@@ -1,4 +1,4 @@
-(defproject io.datopia/stickler-translate "0.1.1-SNAPSHOT"
+(defproject org.datopia/stickler-translate "0.1.1-SNAPSHOT"
   :description  "protobuf3 -> EDN schema generator."
   :url          "https://github.com/datopia/stickler"
   :license      {:name "MIT License"