Introduce json schema validation

go.mod

@@ -10,5 +10,6 @@ require (
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/jf-tech/iohelper v1.0.3
 	github.com/stretchr/testify v1.6.1
+	github.com/xeipuuv/gojsonschema v1.2.0
 	golang.org/x/text v0.3.0
 )

go.sum

@@ -17,9 +17,16 @@ github.com/jf-tech/iohelper v1.0.3/go.mod h1:X28R+KF0lnKEhZ8Q0iBzLI9FKHJy/jXZ+ax
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

gogen.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+function green_printf () {
+  printf "\e[32m$@\e[m\n"
+}
+
+function red_printf () {
+  printf "\e[31m$@\e[m\n"
+}
+
+function panic () {
+  echo
+  red_printf "$@"
+  echo
+  exit 1
+}
+
+function panic_fail_op () {
+  panic "Operation failed! Exit."
+}
+
+SCRIPT_DIR=$(pwd `dirname "$0"`)
+
+cd $SCRIPT_DIR/omniparser/schemavalidate || panic_fail_op
+green_printf "go:generate in 'omniparser/schemavalidate'..."
+go generate || panic_fail_op
+
+cd $SCRIPT_DIR
+
+echo
+green_printf "go generate completed!\n"

omniparser/parser.go

@@ -12,6 +12,7 @@ import (
 	"github.com/jf-tech/omniparser/omniparser/errs"
 	"github.com/jf-tech/omniparser/omniparser/schemaplugin"
 	omniv2 "github.com/jf-tech/omniparser/omniparser/schemaplugin/omni/v2"
+	"github.com/jf-tech/omniparser/omniparser/schemavalidate"
 	"github.com/jf-tech/omniparser/omniparser/transformctx"
 )
 
@@ -53,12 +54,16 @@ func NewParser(schemaName string, schemaReader io.Reader, pluginConfigs ...Schem
 	if err != nil {
 		return nil, fmt.Errorf("unable to read schema '%s': %s", schemaName, err.Error())
 	}
-	var schemaHeader schemaplugin.Header
-	err = json.Unmarshal(schemaContent, &schemaHeader)
+	// validate the universal parser_settings header schema.
+	err = schemavalidate.SchemaValidate(schemaName, schemaContent, schemavalidate.JSONSchemaParserSettings)
 	if err != nil {
-		return nil, fmt.Errorf(
-			"unable to read schema '%s': corrupted header `parser_settings`: %s", schemaName, err)
+		// The err from schemavalidate.SchemaValidate is already context formatted.
+		return nil, err
 	}
+	var schemaHeader schemaplugin.Header
+	// parser_settings has just been json schema validated. so unmarshaling will not go wrong.
+	_ = json.Unmarshal(schemaContent, &schemaHeader)
+
 	var allPluginConfigs []SchemaPluginConfig
 	allPluginConfigs = append(allPluginConfigs, pluginConfigs...)
 	allPluginConfigs = append(allPluginConfigs, SchemaPluginConfig{

omniparser/parser_test.go

@@ -32,7 +32,7 @@ func TestNewParser(t *testing.T) {
 			name:        "fail to unmarshal schema header",
 			schema:      "[invalid",
 			pluginCfgs:  nil,
-			expectedErr: "unable to read schema 'test-schema': corrupted header `parser_settings`:.*",
+			expectedErr: "unable to perform schema validation: invalid character 'i' looking for beginning of value",
 		},
 		{
 			name:   "no supported schema plugin",
@@ -48,7 +48,13 @@ func TestNewParser(t *testing.T) {
 			expectedErr: errs.ErrSchemaNotSupported.Error(),
 		},
 		{
-			name:   "supported schema plugin found, but schema validation fails",
+			name:        "supported schema plugin found, but json schema validation for parser_settings failed",
+			schema:      `{"parser_settings": {"versionx": "9999", "file_format_type": "exe" }}`,
+			pluginCfgs:  nil,
+			expectedErr: "schema 'test-schema' validation failed:\nparser_settings: version is required\nparser_settings: Additional property versionx is not allowed",
+		},
+		{
+			name:   "supported schema plugin found, but schema validation failed",
 			schema: `{"parser_settings": {"version": "9999", "file_format_type": "exe" }}`,
 			pluginCfgs: []SchemaPluginConfig{
 				{
@@ -96,7 +102,7 @@ func TestNewParser(t *testing.T) {
 			plugin, err := NewParser("test-schema", schemaReader, test.pluginCfgs...)
 			if test.expectedErr != "" {
 				assert.Error(t, err)
-				assert.Regexp(t, test.expectedErr, err.Error())
+				assert.Equal(t, test.expectedErr, err.Error())
 				assert.Nil(t, plugin)
 			} else {
 				assert.NoError(t, err)

omniparser/schemaplugin/omni/v2/plugin.go

@@ -7,6 +7,7 @@ import (
 	"github.com/jf-tech/omniparser/omniparser/schemaplugin"
 	omniv2fileformat "github.com/jf-tech/omniparser/omniparser/schemaplugin/omni/v2/fileformat"
 	omniv2xml "github.com/jf-tech/omniparser/omniparser/schemaplugin/omni/v2/fileformat/xml"
+	"github.com/jf-tech/omniparser/omniparser/schemavalidate"
 	"github.com/jf-tech/omniparser/omniparser/transformctx"
 )
 
@@ -23,6 +24,12 @@ func ParseSchema(ctx *schemaplugin.ParseSchemaCtx) (schemaplugin.Plugin, error)
 	if ctx.Header.ParserSettings.Version != PluginVersion {
 		return nil, errs.ErrSchemaNotSupported
 	}
+	// Now do transform_declarations json schema validation
+	err := schemavalidate.SchemaValidate(ctx.Name, ctx.Content, schemavalidate.JSONSchemaTransformDeclarations)
+	if err != nil {
+		// err is always context formatted.
+		return nil, err
+	}
 	// If caller specifies a custom FileFormat, we'll use it (and it only);
 	// otherwise we'll use the builtin ones.
 	fileFormats := []omniv2fileformat.FileFormat{

omniparser/schemaplugin/omni/v2/plugin_test.go

@@ -76,12 +76,32 @@ func TestParseSchema_FormatNotSupported(t *testing.T) {
 					FileFormatType: "unknown",
 				},
 			},
+			Content: []byte(`{"transform_declarations": { "FINAL_OUTPUT": {} }}`),
 		})
 	assert.Error(t, err)
 	assert.Equal(t, errs.ErrSchemaNotSupported, err)
 	assert.Nil(t, p)
 }
 
+func TestParseSchema_TransformDeclarationsValidationFailed(t *testing.T) {
+	p, err := ParseSchema(
+		&schemaplugin.ParseSchemaCtx{
+			Name: "test-schema",
+			Header: schemaplugin.Header{
+				ParserSettings: schemaplugin.ParserSettings{
+					Version:        PluginVersion,
+					FileFormatType: "xml",
+				},
+			},
+			Content: []byte(`{"transform_declarations": {}}`),
+		})
+	assert.Error(t, err)
+	assert.Equal(t,
+		`schema 'test-schema' validation failed: transform_declarations: FINAL_OUTPUT is required`,
+		err.Error())
+	assert.Nil(t, p)
+}
+
 func TestParseSchema_CustomFileFormat_FormatNotSupported(t *testing.T) {
 	p, err := ParseSchema(
 		&schemaplugin.ParseSchemaCtx{
@@ -90,6 +110,7 @@ func TestParseSchema_CustomFileFormat_FormatNotSupported(t *testing.T) {
 					Version: PluginVersion,
 				},
 			},
+			Content: []byte(`{"transform_declarations": { "FINAL_OUTPUT": {} }}`),
 			PluginParams: &PluginParams{
 				CustomFileFormat: testFileFormat{
 					validateSchemaErr: errs.ErrSchemaNotSupported,
@@ -109,6 +130,7 @@ func TestParseSchema_CustomFileFormat_ValidationFailure(t *testing.T) {
 					Version: PluginVersion,
 				},
 			},
+			Content: []byte(`{"transform_declarations": { "FINAL_OUTPUT": {} }}`),
 			PluginParams: &PluginParams{
 				CustomFileFormat: testFileFormat{
 					validateSchemaErr: errors.New("validation failure"),
@@ -128,6 +150,7 @@ func TestParseSchema_CustomFileFormat_Success(t *testing.T) {
 					Version: PluginVersion,
 				},
 			},
+			Content: []byte(`{"transform_declarations": { "FINAL_OUTPUT": {} }}`),
 			PluginParams: &PluginParams{
 				CustomFileFormat: testFileFormat{
 					validateSchemaRuntime: "runtime data",

omniparser/schemavalidate/jsonschemas/gen.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"flag"
+	"io/ioutil"
+	"os"
+	"text/template"
+)
+
+const schemaTemplate = `// Code generated - DO NOT EDIT.
+
+package schemavalidate
+
+const (
+    {{.SchemaVarName}} =
+` + "\x60" /*\x60 is this char '`' :) */ + `
+{{.Schema}}
+` + "\x60" + `
+)
+`
+
+type templateVars struct {
+	SchemaVarName string
+	Schema        string
+}
+
+func main() {
+	var jsonFileName string
+	flag.StringVar(&jsonFileName, "json", "", "The name of json schema file.")
+	var tv templateVars
+	flag.StringVar(&tv.SchemaVarName, "varname", "", "The variable name of json schema string.")
+	flag.Parse()
+
+	jsonFileContent, err := ioutil.ReadFile("./" + jsonFileName)
+	if err != nil {
+		os.Exit(1)
+	}
+	tv.Schema = string(jsonFileContent)
+
+	err = template.Must(template.New("genjsonschema").Parse(schemaTemplate)).Execute(os.Stdout, tv)
+	if err != nil {
+		os.Exit(1)
+	}
+}

omniparser/schemavalidate/jsonschemas/parser_settings.json

@@ -0,0 +1,22 @@
+{
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "$id": "github.com/jf-tech/omniparser:parser_settings",
+    "title": "omniparser schema: parser_settings",
+    "type": "object",
+    "properties": {
+        "parser_settings": {
+            "type": "object",
+            "properties": {
+                "version": { "type": "string" },
+                "file_format_type": { "type": "string" },
+                "encoding": {
+                    "type": "string",
+                    "enum": [ "utf-8", "iso-8859-1", "windows-1252" ]
+                }
+            },
+            "required": [ "version", "file_format_type" ],
+            "additionalProperties": false
+        }
+    },
+    "required": [ "parser_settings" ]
+}