Merge pull request #79 from acmiyaguchi/snakecase

Add `--normalize-case` option for snake_casing column names

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "jsonschema-transpiler"
-version = "1.1.0"
+version = "1.2.0"
 authors = ["Anthony Miyaguchi <amiyaguchi@mozilla.com>"]
 description = "A tool to transpile JSON Schema into schemas for data processing"
 license = "MPL-2.0"
@@ -14,9 +14,10 @@ name = "jst"
 clap = "~2.32"
 env_logger = "0.6.1"
 log = "0.4"
-regex = "1"
+onig = "4.3"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+lazy_static = "1.3.0"
 
 [build-dependencies]
 serde = { version = "1.0", features = ["derive"] }

build.rs

@@ -82,7 +82,7 @@ fn avro_{name}() {{
     {expected}
     "#;
     let mut context = Context {{
-        resolve_method: ResolveMethod::Cast,
+        ..Default::default()
     }};
     let input: Value = serde_json::from_str(input_data).unwrap();
     let expected: Value = serde_json::from_str(expected_data).unwrap();
@@ -109,23 +109,31 @@ fn write_bigquery_tests(mut outfile: &File, suite: &TestSuite) {
     for case in &suite.tests {
         let formatted = format!(
             r##"
-#[test]
+#[test]{should_panic}
 fn bigquery_{name}() {{
     let input_data = r#"
     {input_data}
     "#;
     let expected_data = r#"
     {expected}
     "#;
-    let context = Context {{
-        resolve_method: ResolveMethod::Cast,
+    let mut context = Context {{
+        ..Default::default()
     }};
     let input: Value = serde_json::from_str(input_data).unwrap();
     let expected: Value = serde_json::from_str(expected_data).unwrap();
     assert_eq!(expected, convert_bigquery(&input, context));
+
+    context.resolve_method = ResolveMethod::Panic;
+    convert_bigquery(&input, context);
 }}
 "##,
             name = case.name,
+            should_panic = if case.compatible {
+                ""
+            } else {
+                "\n#[should_panic]"
+            },
             input_data = format_json(case.test.json.clone()),
             expected = format_json(case.test.bigquery.clone()),
         );
@@ -134,7 +142,7 @@ fn bigquery_{name}() {{
 }
 
 fn main() {
-    let test_cases = "tests/resources";
+    let test_cases = "tests/resources/translate";
     let mut avro_fp = File::create("tests/transpile_avro.rs").unwrap();
     let mut bq_fp = File::create("tests/transpile_bigquery.rs").unwrap();
     let format_tests = get_env_var_as_bool("FORMAT_TESTS", true);

scripts/README.md

@@ -49,7 +49,14 @@ $ ./scripts/mps-download-sampled-schemas.py
 $ ./scripts/mps-download-sampled-data.py
 
 # Generates a folder avro/
-$ ./scripts/mps-generate-avro-schemas.sh
+$ ./scripts/mps-generate-schemas.sh
+
+# Alternatively, specify a folder and pass flags
+$ ./scripts/mps-generate-schemas.sh \
+    bq_schemas \
+    --type bigquery \
+    --resolve drop \
+    --normalize-case
 
 # Generates a folder avro-data/
 $ ./scripts/mps-generate-avro-data.sh

scripts/mps-generate-avro-data-helper.py

@@ -81,7 +81,7 @@ def convert(data, schema):
 if not os.path.exists(outdir):
     os.makedirs(outdir)
 
-with open(f"avro/{document}.avro.json", "r") as f:
+with open(f"avro/{document}.schema.json", "r") as f:
     schema_data = f.read()
 schema = avro.schema.Parse(schema_data)
 

scripts/mps-generate-avro-schemas.sh → scripts/mps-generate-schemas.sh

@@ -9,7 +9,7 @@ if [[ ! -d "schemas/" ]]; then
 fi
 
 cargo build
-bin="target/debug/jsonschema_transpiler"
+bin="target/debug/jsonschema-transpiler"
 
 schemas=$(find schemas/ -name "*.schema.json")
 
@@ -18,16 +18,18 @@ outdir=${1:-"avro"}
 if [[ -d $outdir ]]; then
     rm -r $outdir
 fi
+shift;
+
 mkdir $outdir
 
 total=0
 failed=0
 for schema in $schemas; do
     namespace=$(basename $(dirname $(dirname $schema)))
-    schema_filename=$(basename $schema | sed 's/schema.json/avro.json/g')
+    schema_filename=$(basename $schema)
     outfile="$outdir/$namespace.$schema_filename"
 
-    if ! $bin -f "$schema" --type avro > $outfile; then
+    if ! $bin "$@" "$schema" > $outfile; then
         echo "Failed on $schema"
         rm $outfile
         ((failed++))