Skip to content

Commit

Permalink
Separate Schema from Conflate
Browse files Browse the repository at this point in the history 
A typical use case is to merge data then validate against a schema.
However, if you want to validate multiple data sets against a single
schema the API previously forced you to reparse the scema multiple
times, which is suboptimal. Now you can parse the schema once and
store it in an instance of Schema.
  • Loading branch information
@andy-miracl
andy-miracl committed Jul 25, 2018
1 parent 2eb4049 commit c72cb2b
Show file tree
Hide file tree
Showing 7 changed files with 141 additions and 157 deletions.
55 changes: 4 additions & 51 deletions conflate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ var Includes = "includes"
// Conflate contains a 'working' merged data set and optionally a JSON v4 schema
type Conflate struct {
data interface{}
schema interface{}
loader loader
}

Expand Down Expand Up @@ -108,55 +107,14 @@ func (c *Conflate) AddData(data ...[]byte) error {
return c.addData(fdata...)
}

// SetSchemaFile loads a JSON v4 schema from the given path
func (c *Conflate) SetSchemaFile(path string) error {
url, err := toURL(nil, path)
if err != nil {
return wrapError(err, "Failed to obtain url to schema file")
}
return c.SetSchemaURL(url)
}

// SetSchemaURL loads a JSON v4 schema from the given URL
func (c *Conflate) SetSchemaURL(url url.URL) error {
data, err := loadURL(url)
if err != nil {
return wrapError(err, "Failed to load schema file")
}
return c.SetSchemaData(data)
}

// SetSchemaData loads a JSON v4 schema from the given data
func (c *Conflate) SetSchemaData(data []byte) error {
var schema interface{}
err := JSONUnmarshal(data, &schema)
if err != nil {
return wrapError(err, "Schema is not valid json")
}
err = validateSchema(schema)
if err != nil {
return wrapError(err, "The schema is not valid against the meta-schema http://json-schema.org/draft-04/schema")
}
c.schema = schema
return nil
}

// ApplyDefaults sets any nil or missing values in the data, to the default values defined in the JSON v4 schema
func (c *Conflate) ApplyDefaults() error {
if c.schema == nil {
return makeError("Schema is not set")
}
err := applyDefaults(&c.data, c.schema)
return wrapError(err, "The defaults could not be applied")
func (c *Conflate) ApplyDefaults(s *Schema) error {
return s.ApplyDefaults(&c.data)
}

// Validate checks the data against the JSON v4 schema
func (c *Conflate) Validate() error {
if c.schema == nil {
return makeError("Schema is not set")
}
err := validate(&c.data, c.schema)
return wrapError(err, "Schema validation failed")
func (c *Conflate) Validate(s *Schema) error {
return s.Validate(c.data)
}

// Unmarshal extracts the data as a Golang object
Expand All @@ -179,11 +137,6 @@ func (c *Conflate) MarshalTOML() ([]byte, error) {
return tomlMarshal(c.data)
}

// MarshalSchema exports the schema as JSON
func (c *Conflate) MarshalSchema() ([]byte, error) {
return jsonMarshal(c.schema)
}

func (c *Conflate) addData(fdata ...filedata) error {
fdata, err := c.loader.loadDataRecursive(nil, fdata...)
if err != nil {
Expand Down
12 changes: 7 additions & 5 deletions conflate/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ func main() {

var data dataFlag
flag.Var(&data, "data", "The path/url of JSON/YAML/TOML data, or 'stdin' to read from standard input")
schema := flag.String("schema", "", "The path/url of a JSON v4 schema file")
schemaFile := flag.String("schema", "", "The path/url of a JSON v4 schema file")
defaults := flag.Bool("defaults", false, "Apply defaults from schema to data")
validate := flag.Bool("validate", false, "Validate the data against the schema")
format := flag.String("format", "", "Output format of the data JSON/YAML/TOML")
Expand Down Expand Up @@ -62,16 +62,18 @@ func main() {
}
}

if *schema != "" {
err := c.SetSchemaFile(*schema)
var schema *conflate.Schema
if *schemaFile != "" {
s, err := conflate.NewSchemaFile(*schemaFile)
failIfError(err)
schema = s
}
if *defaults {
err := c.ApplyDefaults()
err := c.ApplyDefaults(schema)
failIfError(err)
}
if *validate {
err := c.Validate()
err := c.Validate(schema)
failIfError(err)
}
if *format != "" {
Expand Down
85 changes: 11 additions & 74 deletions conflate_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,87 +171,48 @@ func TestFromFiles_ExpandError(t *testing.T) {
assert.Contains(t, err.Error(), "Failed to load url")
}

func TestFromFiles_SchemaBadUrl(t *testing.T) {
c, err := FromFiles("testdata/valid_parent.json")
assert.Nil(t, err)
err = c.SetSchemaFile(`!"£$%^&*()`)
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "Failed to obtain url to schema file")
}

func TestFromFiles_SchemaMissingError(t *testing.T) {
c, err := FromFiles("testdata/valid_parent.json")
assert.Nil(t, err)
err = c.SetSchemaFile("missing file")
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "Failed to load schema file")
}

func TestFromFiles_SchemaBadJsonError(t *testing.T) {
c, err := FromFiles("testdata/valid_parent.json")
assert.Nil(t, err)
err = c.SetSchemaFile("conflate.go")
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "Schema is not valid json")
}

func TestFromFiles_SchemaBadSchemaError(t *testing.T) {
c, err := FromFiles("testdata/valid_parent.json")
assert.Nil(t, err)
err = c.SetSchemaFile("testdata/bad.schema.json")
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "The schema is not valid against the meta-schema")
}

func TestFromFiles_Schema(t *testing.T) {
c, err := FromFiles("testdata/valid_parent.json")
assert.Nil(t, err)
err = c.SetSchemaFile("testdata/test.schema.json")
assert.Nil(t, err)
}

func TestFromFiles_ValidationNoSchemaErro(t *testing.T) {
c, err := FromFiles("testdata/valid_child.json")
assert.Nil(t, err)
err = c.Validate()
err = c.Validate(nil)
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "Schema is not set")
}

func TestFromFiles_ValidationError(t *testing.T) {
c, err := FromFiles("testdata/valid_child.json")
assert.Nil(t, err)
err = c.SetSchemaFile("testdata/test.schema.json")
s, err := NewSchemaFile("testdata/test.schema.json")
assert.Nil(t, err)
err = c.Validate()
err = c.Validate(s)
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "Schema validation failed")
}

func TestFromFiles_ValidationOk(t *testing.T) {
c, err := FromFiles("testdata/valid_parent.json")
assert.Nil(t, err)
err = c.SetSchemaFile("testdata/test.schema.json")
s, err := NewSchemaFile("testdata/test.schema.json")
assert.Nil(t, err)
err = c.Validate()
err = c.Validate(s)
assert.Nil(t, err)
}

func TestFromFiles_ApplyDefaultsNoSchema(t *testing.T) {
c, err := FromFiles("testdata/valid_parent.json")
assert.Nil(t, err)
err = c.ApplyDefaults()
err = c.ApplyDefaults(nil)
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "Schema is not set")
}

func TestFromFiles_ApplyDefaultsError(t *testing.T) {
c, err := FromFiles("testdata/valid_parent.json")
assert.Nil(t, err)
err = c.SetSchemaFile("testdata/test.schema.json")
s, err := NewSchemaFile("testdata/test.schema.json")
assert.Nil(t, err)
c.schema = []interface{}{"not a map"}
err = c.ApplyDefaults()
s.s = []interface{}{"not a map"}
err = c.ApplyDefaults(s)
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "The defaults could not be applied")
assert.Contains(t, err.Error(), "Schema section is not a map")
Expand All @@ -260,9 +221,9 @@ func TestFromFiles_ApplyDefaultsError(t *testing.T) {
func TestFromFiles_ApplyDefaults(t *testing.T) {
c, err := FromFiles()
assert.Nil(t, err)
err = c.SetSchemaFile("testdata/test.schema.json")
s, err := NewSchemaFile("testdata/test.schema.json")
assert.Nil(t, err)
err = c.ApplyDefaults()
err = c.ApplyDefaults(s)
assert.Nil(t, err)
testData := TestData{}
err = c.Unmarshal(&testData)
Expand Down Expand Up @@ -344,30 +305,6 @@ func TestConflate_MarshalTOML(t *testing.T) {
assert.Equal(t, testMarshalTOML, data)
}

func TestConflate_MarshalSchema(t *testing.T) {
c := New()
err := c.SetSchemaData([]byte("{}"))
assert.Nil(t, err)
data, err := c.MarshalSchema()
assert.Nil(t, err)
assert.Equal(t, "{}\n", string(data))
}

func TestConflate_MarshalSchemaNil(t *testing.T) {
c := New()
data, err := c.MarshalSchema()
assert.Nil(t, err)
assert.Equal(t, string(data), "null\n")
}

func TestConflate_MarshalSchemaError(t *testing.T) {
c := New()
c.schema = make(chan int, 1)
_, err := c.MarshalSchema()
assert.NotNil(t, err)
assert.Contains(t, err.Error(), "The data could not be marshalled")
}

func TestConflate_addDataError(t *testing.T) {
c := New()
err := c.AddData([]byte(`{"includes": ["missing"]}`))
Expand Down
6 changes: 3 additions & 3 deletions example/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,19 +37,19 @@ func main() {
return
}
// load a json schema
err = c.SetSchemaFile(path.Join(thisDir, "../testdata/test.schema.json"))
schema, err := conflate.NewSchemaFile(path.Join(thisDir, "../testdata/test.schema.json"))
if err != nil {
fmt.Println(err)
return
}
// apply defaults defined in schema to merged data
err = c.ApplyDefaults()
err = c.ApplyDefaults(schema)
if err != nil {
fmt.Println(err)
return
}
// validate merged data against schema
err = c.Validate()
err = c.Validate(schema)
if err != nil {
fmt.Println(err)
return
Expand Down
23 changes: 23 additions & 0 deletions filedata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,3 +135,26 @@ func expand(b []byte) ([]byte, int) {
return "$" + name
})), c
}

var getSchema = getDefaultSchema

func getDefaultSchema() map[string]interface{} {
return map[string]interface{}{
"anyOf": []interface{}{
map[string]interface{}{
"type": "object",
"properties": map[string]interface{}{
Includes: map[string]interface{}{
"type": "array",
"items": map[string]interface{}{
"type": "string",
},
},
},
},
map[string]interface{}{
"type": "null",
},
},
}
}

0 comments on commit c72cb2b

Please sign in to comment.