Support bytes as a data type

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "jsonschema-transpiler"
-version = "1.2.0"
+version = "1.3.0"
 authors = ["Anthony Miyaguchi <amiyaguchi@mozilla.com>"]
 description = "A tool to transpile JSON Schema into schemas for data processing"
 license = "MPL-2.0"

src/ast.rs

@@ -14,6 +14,7 @@ pub enum Atom {
     String,
     Datetime,
     JSON,
+    Bytes,
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone)]

src/avro.rs

@@ -142,6 +142,7 @@ impl TranslateFrom<ast::Tag> for Type {
                 ast::Atom::Number => Primitive::Double,
                 ast::Atom::String => Primitive::String,
                 ast::Atom::Datetime => Primitive::String,
+                ast::Atom::Bytes => Primitive::Bytes,
                 ast::Atom::JSON => match handle_error("json atom") {
                     Ok(_) => Primitive::String,
                     Err(reason) => return Err(reason),
@@ -623,4 +624,11 @@ mod tests {
         let avro = json!({"type": "string"});
         assert_from_ast_eq(ast, avro);
     }
+
+    #[test]
+    fn from_ast_bytes() {
+        let ast = json!({"type": {"atom": "bytes"}});
+        let avro = json!({"type": "bytes"});
+        assert_from_ast_eq(ast, avro);
+    }
 }

src/bigquery.rs

@@ -85,6 +85,7 @@ impl TranslateFrom<ast::Tag> for Tag {
                 ast::Atom::Number => Atom::Float64,
                 ast::Atom::String => Atom::String,
                 ast::Atom::Datetime => Atom::Timestamp,
+                ast::Atom::Bytes => Atom::Bytes,
                 ast::Atom::JSON => match handle_error("json atom") {
                     Ok(_) => Atom::String,
                     Err(reason) => return Err(reason),
@@ -567,6 +568,19 @@ mod tests {
         assert_eq!(expect, transform_tag(data));
     }
 
+    #[test]
+    fn test_from_ast_bytes() {
+        let data = json!({
+            "type": {"atom": "bytes"},
+            "nullable": true
+        });
+        let expect = json!({
+           "type": "BYTES",
+           "mode": "NULLABLE",
+        });
+        assert_eq!(expect, transform_tag(data));
+    }
+
     #[test]
     fn test_schema_from_ast_atom() {
         // Nameless tags are top-level fields that should be rooted by default

src/jsonschema.rs

@@ -19,6 +19,7 @@ enum Atom {
     Object,
     Array,
     DateTime,
+    Bytes,
 }
 
 enum Type {
@@ -69,6 +70,8 @@ struct Array {
 #[serde(rename_all = "kebab-case")]
 enum Format {
     DateTime,
+    // Custom format value for casting strings into byte-strings
+    Bytes,
     #[serde(other)]
     Other,
 }
@@ -99,6 +102,9 @@ impl Tag {
             (Value::String(string), Some(Format::DateTime)) if string == "string" => {
                 Type::Atom(Atom::DateTime)
             }
+            (Value::String(string), Some(Format::Bytes)) if string == "string" => {
+                Type::Atom(Atom::Bytes)
+            }
             (Value::String(string), _) => {
                 let atom: Atom = serde_json::from_value(json!(string)).unwrap();
                 Type::Atom(atom)
@@ -139,6 +145,7 @@ impl Tag {
             Atom::Integer => ast::Tag::new(ast::Type::Atom(ast::Atom::Integer), None, false),
             Atom::String => ast::Tag::new(ast::Type::Atom(ast::Atom::String), None, false),
             Atom::DateTime => ast::Tag::new(ast::Type::Atom(ast::Atom::Datetime), None, false),
+            Atom::Bytes => ast::Tag::new(ast::Type::Atom(ast::Atom::Bytes), None, false),
             Atom::Object => match &self.object.properties {
                 Some(properties) => {
                     let mut fields: HashMap<String, ast::Tag> = HashMap::new();
@@ -631,6 +638,16 @@ mod tests {
         assert_eq!(schema.format.unwrap(), Format::DateTime);
     }
 
+    #[test]
+    fn test_deserialize_type_bytes() {
+        let data = json!({
+            "type": "string",
+            "format": "bytes"
+        });
+        let schema: Tag = serde_json::from_value(data).unwrap();
+        assert_eq!(schema.format.unwrap(), Format::Bytes);
+    }
+
     #[test]
     fn test_deserialize_type_alternate_format() {
         let data = json!({
@@ -653,4 +670,17 @@ mod tests {
         });
         assert_eq!(expect, translate(data))
     }
+
+    #[test]
+    fn test_into_ast_atom_bytes() {
+        let data = json!({
+            "type": "string",
+            "format": "bytes"
+        });
+        let expect = json!({
+            "type": {"atom": "bytes"},
+            "nullable": false,
+        });
+        assert_eq!(expect, translate(data))
+    }
 }

tests/resources/translate/atomic.json

@@ -123,6 +123,29 @@
           "type": "string"
         }
       }
+    },
+    {
+      "description": [
+        "Test that strings can be cast into byte strings"
+      ],
+      "name": "test_bytes_format",
+      "compatible": true,
+      "test": {
+        "avro": {
+          "type": "bytes"
+        },
+        "bigquery": [
+          {
+            "mode": "REQUIRED",
+            "name": "root",
+            "type": "BYTES"
+          }
+        ],
+        "json": {
+          "format": "bytes",
+          "type": "string"
+        }
+      }
     }
   ]
 }

tests/transpile_avro.rs

@@ -235,6 +235,30 @@ fn avro_test_datetime() {
     convert_avro(&input, context);
 }
 
+#[test]
+fn avro_test_bytes_format() {
+    let input_data = r#"
+    {
+      "format": "bytes",
+      "type": "string"
+    }
+    "#;
+    let expected_data = r#"
+    {
+      "type": "bytes"
+    }
+    "#;
+    let mut context = Context {
+        ..Default::default()
+    };
+    let input: Value = serde_json::from_str(input_data).unwrap();
+    let expected: Value = serde_json::from_str(expected_data).unwrap();
+    assert_eq!(expected, convert_avro(&input, context));
+
+    context.resolve_method = ResolveMethod::Panic;
+    convert_avro(&input, context);
+}
+
 #[test]
 fn avro_test_map_with_atomics() {
     let input_data = r#"

tests/transpile_bigquery.rs

@@ -231,6 +231,34 @@ fn bigquery_test_datetime() {
     convert_bigquery(&input, context);
 }
 
+#[test]
+fn bigquery_test_bytes_format() {
+    let input_data = r#"
+    {
+      "format": "bytes",
+      "type": "string"
+    }
+    "#;
+    let expected_data = r#"
+    [
+      {
+        "mode": "REQUIRED",
+        "name": "root",
+        "type": "BYTES"
+      }
+    ]
+    "#;
+    let mut context = Context {
+        ..Default::default()
+    };
+    let input: Value = serde_json::from_str(input_data).unwrap();
+    let expected: Value = serde_json::from_str(expected_data).unwrap();
+    assert_eq!(expected, convert_bigquery(&input, context));
+
+    context.resolve_method = ResolveMethod::Panic;
+    convert_bigquery(&input, context);
+}
+
 #[test]
 fn bigquery_test_map_with_atomics() {
     let input_data = r#"