diff --git a/README.md b/README.md index 96f7e47..d7eb9e6 100644 --- a/README.md +++ b/README.md @@ -4,19 +4,19 @@ Clojure-idiomatic protobuf3 schema generation & codec functionality. ## Subprojects -### [io.datopia/stickler-translate](translate) +### [org.datopia/stickler-translate](translate) protobuf->edn schema generation library. [![Clojars -Project](http://clojars.org/io.datopia/stickler-translate/latest-version.svg)](http://clojars.org/io.datopia/stickler-translate) +Project](http://clojars.org/org.datopia/stickler-translate/latest-version.svg)](http://clojars.org/org.datopia/stickler-translate) -### [io.datopia/stickler-codec](codec) +### [org.datopia/stickler-codec](codec) Runtime codec library. [![Clojars -Project](http://clojars.org/io.datopia/stickler-codec/latest-version.svg)](http://clojars.org/io.datopia/stickler-codec) +Project](http://clojars.org/org.datopia/stickler-codec/latest-version.svg)](http://clojars.org/org.datopia/stickler-codec) ## Contributors diff --git a/codec/README.md b/codec/README.md index 174b4a8..2450334 100644 --- a/codec/README.md +++ b/codec/README.md @@ -1,10 +1,10 @@ -# io.datopia/stickler-codec +# org.datopia/stickler-codec [![Clojars -Project](http://clojars.org/io.datopia/stickler-codec/latest-version.svg)](http://clojars.org/io.datopia/stickler-codec) +Project](http://clojars.org/org.datopia/stickler-codec/latest-version.svg)](http://clojars.org/org.datopia/stickler-codec) Small runtime library for encoding/decoding protobuf messages via schemas -generated by [io.datopia/stickler-translate](../translate). +generated by [org.datopia/stickler-translate](../translate). ## Documentation - [API Docs](https://datopia.github.io/stickler/stickler-codec/) diff --git a/codec/project.clj b/codec/project.clj index 2d9623f..fbb6084 100644 --- a/codec/project.clj +++ b/codec/project.clj @@ -1,4 +1,4 @@ -(defproject io.datopia/stickler-codec "0.1.1-SNAPSHOT" +(defproject org.datopia/stickler-codec "0.1.1-SNAPSHOT" :description "Idiomatic Clojure codec functionality for protobuf3." :url "https://github.com/datopia/stickler" :license {:name "MIT License" @@ -9,6 +9,7 @@ :java-source-paths ["src/java"] :profiles {:dev {:java-source-paths ["test/gen-java"] + :resource-paths ["test/resources"] :global-vars {*warn-on-reflection* true} :aliases {"test-prep" ["run" "-m" "stickler.test-prep"]} @@ -17,7 +18,7 @@ :metadata {:doc/format :markdown} :themes [:default [:datopia {:datopia/github "https://github.com/datopia/stickler"}]]} :dependencies - [[io.datopia/stickler-translate "0.1.0"] + [[org.datopia/stickler-translate "0.1.1-SNAPSHOT"] [io.datopia/codox-theme "0.1.0"] [com.squareup.wire/wire-java-generator "2.3.0-RC1"] [org.clojure/test.check "0.10.0-alpha3"]]}}) diff --git a/codec/src/stickler/codec.clj b/codec/src/stickler/codec.clj index 9b4fd46..d9f90cc 100644 --- a/codec/src/stickler/codec.clj +++ b/codec/src/stickler/codec.clj @@ -1,6 +1,4 @@ (ns stickler.codec - (:require [clojure.java.io :as io] - [clojure.edn :as edn]) (:import [java.io ByteArrayOutputStream ByteArrayInputStream] [java.nio.charset Charset] [io.datopia.stickler CodecUtil])) @@ -47,6 +45,12 @@ (declare encode-stream) +(defn- encode-enum-field-value [schema ^ByteArrayInputStream stream field v] + {:pre [(keyword? v)]} + (let [t (:type field) + m (-> schema t :fields)] + (CodecUtil/writeVarint32 stream (unchecked-int (m v))))) + (defn- encode-message-field-value [schema stream v] (let [sub-stream (ByteArrayOutputStream.)] (encode-stream schema sub-stream v) @@ -72,7 +76,9 @@ :double (CodecUtil/writeDouble stream (unchecked-double v)) :bytes (encode-byte-array stream v) :string (encode-byte-array stream (.getBytes ^String v ^Charset utf8)) - (encode-message-field-value schema stream v))) + (if (:enum? ((:type field) schema)) + (encode-enum-field-value schema stream field v) + (encode-message-field-value schema stream v)))) (defn- encode-packed-field [schema stream field v] (when-not (empty? v) @@ -103,6 +109,12 @@ (.read stream body 0 len) (decode-bytes schema (:type field) body))) +(defn- decode-enum-field-value [schema ^ByteArrayInputStream stream field] + (let [x (CodecUtil/readVarint32 stream) + t (:type field) + m (-> schema t :tag->kw)] + (m x))) + (defn- decode-field-value [schema ^ByteArrayInputStream stream field & [len]] (case (:type field) :uint32 (CodecUtil/readUnsigned32 stream) @@ -122,7 +134,10 @@ :double (CodecUtil/readDouble stream) :bytes (decode-byte-array stream len) :string (String. ^bytes (decode-byte-array stream len) ^Charset utf8) - (decode-message-field-value schema stream field))) + + (if (:enum? ((:type field) schema)) + (decode-enum-field-value schema stream field) + (decode-message-field-value schema stream field)))) (defn- decode-packed-field [schema ^ByteArrayInputStream stream field] (let [size (CodecUtil/readVarint32 stream) @@ -198,12 +213,29 @@ (compare t-a t-b)))) fields)) +(defn- prepare-enums [schema] + (reduce + (fn [[msgs enums] [k msg]] + (if (:enum? msg) + [msgs (assoc enums k + (assoc msg :tag->kw (into {} + (for [[k v] (:fields msg)] + [v k]))))] + [(assoc msgs k msg) enums])) + [{} {}] + schema)) + +(defn- prepare-messages [schema] + (for [[msg-k msg-schema] schema + :let [msg-schema (update msg-schema :fields sorted-map-by-tag) + tag->f + (into {} + (for [[field-k {tag :tag :as field}] (:fields msg-schema)] + [tag (assoc field :name field-k)]))]] + [msg-k (assoc msg-schema :tag->field tag->f)])) + (defn prepare-schema [schema] - (into {} - (for [[msg-k msg-schema] schema - :let [msg-schema (update msg-schema :fields sorted-map-by-tag) - tag->f - (into {} - (for [[field-k {tag :tag :as field}] (:fields msg-schema)] - [tag (assoc field :name field-k)]))]] - [msg-k (assoc msg-schema :tag->field tag->f)]))) + (let [[schema enums] (prepare-enums schema)] + (reduce into {} + [enums + (prepare-messages schema)]))) diff --git a/codec/test/stickler/codec_test.clj b/codec/test/stickler/codec_test.clj index b753836..cb9c359 100644 --- a/codec/test/stickler/codec_test.clj +++ b/codec/test/stickler/codec_test.clj @@ -43,7 +43,8 @@ :bool gen/boolean :double gen/double :float gen-float - :stickler/msg (gen/return :stickler.test/Scalars)}) + :stickler/msg (gen/return :stickler.test/Scalars) + :size (gen/elements #{:SMALL :MEDIUM :LARGE})}) (defn- map->gen [m] (let [m-gen (apply gen/tuple @@ -193,6 +194,18 @@ (Arrays/equals ^bytes (:bytes m) ^bytes (:bytes (roundtrip m))))) +(defspec enum-wire-symmetry trials + (wire-symmetry-prop + Scalars + {:stickler/msg :stickler.test/Scalars + :size (gen/elements #{:SMALL :MEDIUM :LARGE})} + map->Scalars)) + +(defspec enum-roundtrip trials + (prop/for-all [m (map->gen {:stickler/msg :stickler.test/Scalars + :size (gen/elements #{:SMALL :MEDIUM :LARGE})})] + (= (:size m) (:size (roundtrip m))))) + (defspec message-wire-symmetry trials (wire-symmetry-prop Scalars gen-message map->Scalars)) diff --git a/codec/test/stickler/codec_test/util.clj b/codec/test/stickler/codec_test/util.clj index 7ede59f..9dec15b 100644 --- a/codec/test/stickler/codec_test/util.clj +++ b/codec/test/stickler/codec_test/util.clj @@ -1,5 +1,7 @@ (ns stickler.codec-test.util - (:import [stickler.test Scalars$Builder] + (:import [stickler.test + Scalars$Builder + Scalars$Size] [java.util Arrays])) ;; taken from encore @@ -31,6 +33,8 @@ (:float m) (.float_ (:float m)) (:double m) (.double_ (:double m)) + (:size m) (.size (Scalars$Size/valueOf (name (:size m)))) + (contains? m :bool) (.bool (:bool m)))) (defn map->Scalars [m] diff --git a/codec/test/stickler/test_prep.clj b/codec/test/stickler/test_prep.clj index 54a3efb..fb57bd7 100644 --- a/codec/test/stickler/test_prep.clj +++ b/codec/test/stickler/test_prep.clj @@ -35,12 +35,11 @@ (with-open [writer (io/writer (io/file out-dir "schema.edn"))] (pprint/write (stickler/Schema->edn schema) :stream writer))) -(defn -main [& [out-dir schema-out]] - {:pre [out-dir]} +(defn -main [] (let [tmp-in (.toFile (Files/createTempDirectory "proto" (make-array FileAttribute 0))) proto-f (io/file tmp-in "test.proto") in-res (io/resource "test.proto")] (spit proto-f (slurp in-res)) (let [schema (stickler/dirs->Schema tmp-in)] - (generate-java schema (io/file out-dir)) - (generate-edn schema (io/file schema-out))))) + (generate-java schema (io/file "test/gen-java")) + (generate-edn schema (io/file "test/resources/"))))) diff --git a/lein-stickler/LICENSE b/lein-stickler/LICENSE new file mode 100644 index 0000000..31d3916 --- /dev/null +++ b/lein-stickler/LICENSE @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) 2018 . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/lein-stickler/README.md b/lein-stickler/README.md new file mode 100644 index 0000000..7ce130f --- /dev/null +++ b/lein-stickler/README.md @@ -0,0 +1,8 @@ +# lein-stickler + +With `org.datopia/lein-stickler` in your Leiningen profile, run `lein stickler +:dirs dir [dirs ...]` to output an EDN schema to stdout. + +## License + +[MIT](LICENSE) diff --git a/lein-stickler/project.clj b/lein-stickler/project.clj new file mode 100644 index 0000000..d70fd03 --- /dev/null +++ b/lein-stickler/project.clj @@ -0,0 +1,7 @@ +(defproject org.datopia/lein-stickler "0.1.0-SNAPSHOT" + :description "Converts protobuf3 schemas into EDN maps." + :url "https://github.com/datopia/stickler" + :license {:name "MIT License" + :url "http://opensource.org/licenses/MIT"} + :dependencies [[org.datopia/stickler-translate "0.1.1-SNAPSHOT"]] + :eval-in-leiningen true) diff --git a/lein-stickler/src/leiningen/stickler.clj b/lein-stickler/src/leiningen/stickler.clj new file mode 100644 index 0000000..964df93 --- /dev/null +++ b/lein-stickler/src/leiningen/stickler.clj @@ -0,0 +1,24 @@ +(ns leiningen.stickler + (:require [clojure.string :as str] + [clojure.pprint :as pprint] + [stickler.translate :as translate])) + +(defn- parse-args [args] + (let [args (map #(cond-> % (str/starts-with? % ":") read-string) args) + args (partition-by keyword? args)] + (reduce + (fn [acc [[arg] vs]] + (update acc arg (fnil into #{}) vs)) + {} (partition 2 args)))) + +(defn ^{:no-project-needed true} stickler + "protobuf3 -> EDN schema, for use with stickler.codec. + + :dirs /proto ... + A sequence of absolute directories to process into a protobuf schema. + The EDN is pretty-printed to stdout" + [_ & args] + (let [args (parse-args args)] + (-> (apply translate/dirs->Schema (:dirs args)) + translate/Schema->edn + pprint/pprint))) diff --git a/translate/README.md b/translate/README.md index 3e54490..cad83e4 100644 --- a/translate/README.md +++ b/translate/README.md @@ -1,9 +1,9 @@ # io.datopia/stickler-translate [![Clojars -Project](http://clojars.org/io.datopia/stickler-translate/latest-version.svg)](http://clojars.org/io.datopia/stickler-translate) +Project](http://clojars.org/org.datopia/stickler-translate/latest-version.svg)](http://clojars.org/org.datopia/stickler-translate) -Conversion of on-disk protobuf3 files into EDN schemas, suitable as inputs for [io.datopia/stickler-codec](../codec). +Conversion of on-disk protobuf3 files into EDN schemas, suitable as inputs for [org.datopia/stickler-codec](../codec). ## Documentation - [API Docs](https://datopia.github.io/stickler/stickler-translate/) diff --git a/translate/project.clj b/translate/project.clj index e529587..a59a088 100644 --- a/translate/project.clj +++ b/translate/project.clj @@ -1,4 +1,4 @@ -(defproject io.datopia/stickler-translate "0.1.1-SNAPSHOT" +(defproject org.datopia/stickler-translate "0.1.1-SNAPSHOT" :description "protobuf3 -> EDN schema generator." :url "https://github.com/datopia/stickler" :license {:name "MIT License" diff --git a/translate/src/stickler/translate.clj b/translate/src/stickler/translate.clj index 604a9a7..e0130e6 100644 --- a/translate/src/stickler/translate.clj +++ b/translate/src/stickler/translate.clj @@ -1,13 +1,13 @@ (ns stickler.translate "Convert directories of-disk protobuf3 files into EDN schemas suitable for use - by `io.datopia/sticker-codec`." + by `org.datopia/sticker-codec`." (:require [clojure.java.io :as io] [stickler.translate.util :refer [assoc-when]] [clojure.walk :as walk] [clojure.string :as str] [clojure.pprint :refer [pprint]]) (:import [com.squareup.wire.schema - Schema SchemaLoader ProtoType OneOf IdentifierSet$Builder ProtoMember]) + Schema SchemaLoader ProtoType OneOf IdentifierSet$Builder ProtoMember Type]) (:gen-class)) (defn- un-underscore [s] @@ -41,7 +41,7 @@ (defn- proto->package-name [^com.squareup.wire.schema.ProtoFile proto] (.packageName proto)) -(defn- type->simple-name [ t] +(defn- type->simple-name [t] (-> t .type .simpleName)) (defn- proto+type->key [proto t] @@ -69,8 +69,6 @@ [(->field-name (keyword (.name f))) (cond (.isScalar t) (let [type-k (scalar-proto-type->key t)] - (when (= type-k :enum) - (throw (RuntimeException. "Enums not supported."))) (assoc m :scalar? true :type type-k @@ -80,25 +78,46 @@ :type (proto+type->key proto f) :wire-type msg-wire-type))])) -(defn- convert-one-of [proto ^OneOf one-of] - (let [fields (mapv (partial convert-field proto) (.fields one-of))] - {(->field-name (keyword (.name one-of))) {:one-of (into {} fields)}})) - -(defn- convert-fields [proto fields one-ofs] +(defn- convert-msg [proto fields one-ofs] (let [fields (into {} (map (partial convert-field proto) fields)) one-ofs (for [^OneOf one-of one-ofs - f (.fields one-of) + f (.fields one-of) :let [one-of-k (keyword (.name one-of))]] (-> (convert-field proto f) (assoc-in [1 :one-of] one-of-k)))] (assoc-when {} - :fields (not-empty (into fields one-ofs))))) + :fields (not-empty (into fields one-ofs))))) + +(defn- ->constant-name [s] + (-> s clojure.string/upper-case (clojure.string/replace \_ \-))) + +(defprotocol ConvertType + (-convert-type [t proto])) + +(extend-protocol ConvertType + com.squareup.wire.schema.MessageType + (-convert-type [msg proto] + (convert-msg proto (.fields msg) (.oneOfs msg))) + com.squareup.wire.schema.EnumType + (-convert-type [enum _] + (reduce + (fn conv-enum [m ^com.squareup.wire.schema.EnumConstant c] + (let [k (-> c .name ->constant-name keyword)] + (assoc-in m [:fields k] (.tag c)))) + {:enum? true} + (.constants enum)))) (defn- convert-proto-file [^com.squareup.wire.schema.ProtoFile f] (reduce - (fn [acc ^com.squareup.wire.schema.MessageType t] - (let [t-name (proto+type->key f t)] - (assoc acc t-name (convert-fields f (.fields t) (.oneOfs t))))) + (fn reduce-top-levels [acc ^Type t] + (reduce + (partial apply assoc) + acc + (map + (fn reduce-top-level-and-nested [t] + [(proto+type->key f t) + (-convert-type t f)]) + (into [t] (.nestedTypes t))))) {} (.types f))) @@ -109,7 +128,7 @@ (.build b))) (defn prune-Schema - "Prune the given `schema` such that the sequences of keyword/string + "Prune the given `schema` such that the sequences of string identifiers in `prune-spec`'s `:include` and `:exclude` keys are included/excluded, respectively." [^Schema schema prune-spec] @@ -128,10 +147,23 @@ form)) edn-schema)) +(defn- unfuck-enums [schema] + (let [enums (into #{} (for [[k v] schema :when (:enum? v)] k))] + (walk/postwalk + (fn [form] + (if (and (map? form) (enums (:type form))) + (assoc form :wire-type (type->wire-type :enum)) + form)) + schema))) + (defn Schema->edn "Convert the given `schema` to a map suitable for use by `stickler-codec`." [^Schema schema] - (apply merge (map convert-proto-file (.protoFiles schema)))) + (->> schema + .protoFiles + (map convert-proto-file) + (apply merge) + unfuck-enums)) (defn- dirs->loader [& dirs] (reduce @@ -143,9 +175,12 @@ (defn dirs->Schema "Turn a sequence of `dirs` into a `Schema`. - See [[prune-Schema]], [[Schema->edn]]." + See [[prune-Schema]], [[Schema->edn]]." [& dirs] (.load ^SchemaLoader (apply dirs->loader dirs))) -(defn -main [& argv] - (pprint (Schema->edn (apply dirs->Schema argv)))) +(defn translate [& argv] + (Schema->edn (apply dirs->Schema argv))) + +(defn -main [& args] + (pprint (apply translate args)))