diff --git a/examples/ints/go.mod b/examples/ints/go.mod index 2ebac68f..65807250 100644 --- a/examples/ints/go.mod +++ b/examples/ints/go.mod @@ -1,6 +1,6 @@ module github.com/splunk/stef/examples/ints -go 1.23.2 +go 1.24.0 require ( github.com/splunk/stef/go/pkg v0.0.8 diff --git a/examples/ints/internal/ints/record.go b/examples/ints/internal/ints/record.go index 8b71b9fb..f0885161 100644 --- a/examples/ints/internal/ints/record.go +++ b/examples/ints/internal/ints/record.go @@ -320,9 +320,10 @@ func (e *RecordEncoder) CollectColumns(columnSet *pkg.WriteColumnSet) { // RecordDecoder implements decoding of Record type RecordDecoder struct { - buf pkg.BitsReader - column *pkg.ReadableColumn - fieldCount uint + buf pkg.BitsReader + column *pkg.ReadableColumn + fieldCount uint + uint64Decoder encoders.Uint64Decoder allocators *Allocators diff --git a/examples/jsonl/internal/jsonstef/jsonobject.go b/examples/jsonl/internal/jsonstef/jsonobject.go index 97264034..a0ebccb2 100644 --- a/examples/jsonl/internal/jsonstef/jsonobject.go +++ b/examples/jsonl/internal/jsonstef/jsonobject.go @@ -385,11 +385,13 @@ func (e *JsonObjectEncoder) encodeFull(list *JsonObject) { func (e *JsonObjectEncoder) CollectColumns(columnSet *pkg.WriteColumnSet) { columnSet.SetBytes(&e.buf) + colIdx := 0 if !e.isKeyRecursive { - e.keyEncoder.CollectColumns(columnSet.At(0)) + e.keyEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ } if !e.isValueRecursive { - e.valueEncoder.CollectColumns(columnSet.At(1)) + e.valueEncoder.CollectColumns(columnSet.At(colIdx)) } } diff --git a/examples/profile/internal/profile/labels.go b/examples/profile/internal/profile/labels.go index 81cdb85f..d75235a2 100644 --- a/examples/profile/internal/profile/labels.go +++ b/examples/profile/internal/profile/labels.go @@ -385,11 +385,13 @@ func (e *LabelsEncoder) encodeFull(list *Labels) { func (e *LabelsEncoder) CollectColumns(columnSet *pkg.WriteColumnSet) { columnSet.SetBytes(&e.buf) + colIdx := 0 if !e.isKeyRecursive { - e.keyEncoder.CollectColumns(columnSet.At(0)) + e.keyEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ } if !e.isValueRecursive { - e.valueEncoder.CollectColumns(columnSet.At(1)) + e.valueEncoder.CollectColumns(columnSet.At(colIdx)) } } diff --git a/go/otel/oteltef/attributes.go b/go/otel/oteltef/attributes.go index b307504a..aab0ee38 100644 --- a/go/otel/oteltef/attributes.go +++ b/go/otel/oteltef/attributes.go @@ -385,11 +385,13 @@ func (e *AttributesEncoder) encodeFull(list *Attributes) { func (e *AttributesEncoder) CollectColumns(columnSet *pkg.WriteColumnSet) { columnSet.SetBytes(&e.buf) + colIdx := 0 if !e.isKeyRecursive { - e.keyEncoder.CollectColumns(columnSet.At(0)) + e.keyEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ } if !e.isValueRecursive { - e.valueEncoder.CollectColumns(columnSet.At(1)) + e.valueEncoder.CollectColumns(columnSet.At(colIdx)) } } diff --git a/go/otel/oteltef/envelopeattributes.go b/go/otel/oteltef/envelopeattributes.go index 1e4f0e53..374ca856 100644 --- a/go/otel/oteltef/envelopeattributes.go +++ b/go/otel/oteltef/envelopeattributes.go @@ -382,11 +382,13 @@ func (e *EnvelopeAttributesEncoder) encodeFull(list *EnvelopeAttributes) { func (e *EnvelopeAttributesEncoder) CollectColumns(columnSet *pkg.WriteColumnSet) { columnSet.SetBytes(&e.buf) + colIdx := 0 if !e.isKeyRecursive { - e.keyEncoder.CollectColumns(columnSet.At(0)) + e.keyEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ } if !e.isValueRecursive { - e.valueEncoder.CollectColumns(columnSet.At(1)) + e.valueEncoder.CollectColumns(columnSet.At(colIdx)) } } diff --git a/go/otel/oteltef/keyvaluelist.go b/go/otel/oteltef/keyvaluelist.go index 038d4922..66fc1bbc 100644 --- a/go/otel/oteltef/keyvaluelist.go +++ b/go/otel/oteltef/keyvaluelist.go @@ -385,11 +385,13 @@ func (e *KeyValueListEncoder) encodeFull(list *KeyValueList) { func (e *KeyValueListEncoder) CollectColumns(columnSet *pkg.WriteColumnSet) { columnSet.SetBytes(&e.buf) + colIdx := 0 if !e.isKeyRecursive { - e.keyEncoder.CollectColumns(columnSet.At(0)) + e.keyEncoder.CollectColumns(columnSet.At(colIdx)) + colIdx++ } if !e.isValueRecursive { - e.valueEncoder.CollectColumns(columnSet.At(1)) + e.valueEncoder.CollectColumns(columnSet.At(colIdx)) } } diff --git a/go/pkg/schema/schema.go b/go/pkg/schema/schema.go index 935971d6..e08f0bfa 100644 --- a/go/pkg/schema/schema.go +++ b/go/pkg/schema/schema.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "sort" + "strings" ) // Schema is a STEF schema description, serializable in JSON format. @@ -606,6 +607,130 @@ func (d *Schema) markReachableFromFieldType( } } +// PrettyPrint outputs the schema in SDL format. +func (d *Schema) PrettyPrint() string { + var out []string + + // Print package declaration + out = append(out, fmt.Sprintf("package %s", strings.Join(d.PackageName, "."))) + + // Print enums + for _, enum := range sortedList(d.Enums) { + out = append(out, prettyPrintEnum(enum)) + } + + // Print multimaps + for _, mm := range sortedList(d.Multimaps) { + out = append(out, prettyPrintMultimap(mm)) + } + + // Print structs and oneofs + for _, s := range sortedList(d.Structs) { + out = append(out, prettyPrintStruct(s)) + } + + return strings.Join(out, "\n\n") +} + +func sortedList[T any](m map[string]*T) []*T { + var names []string + for k := range m { + names = append(names, k) + } + sort.Strings(names) + var out []*T + for _, n := range names { + out = append(out, m[n]) + } + return out +} + +func prettyPrintEnum(e *Enum) string { + out := fmt.Sprintf("enum %s {", e.Name) + for _, f := range e.Fields { + out += fmt.Sprintf("\n %s = %d", f.Name, f.Value) + } + out += "\n}" + return out +} + +func prettyPrintMultimap(m *Multimap) string { + out := fmt.Sprintf("multimap %s {\n", m.Name) + out += " key " + prettyPrintFieldType(&m.Key.Type) + if dict := m.Key.Type.DictName; dict != "" { + out += fmt.Sprintf(" dict(%s)", dict) + } + out += "\n" + out += " value " + prettyPrintFieldType(&m.Value.Type) + if dict := m.Value.Type.DictName; dict != "" { + out += fmt.Sprintf(" dict(%s)", dict) + } + out += "\n}" + return out +} + +func prettyPrintStruct(s *Struct) string { + var out string + if s.OneOf { + out = fmt.Sprintf("oneof %s {", s.Name) + } else { + out = fmt.Sprintf("struct %s", s.Name) + if s.DictName != "" { + out += fmt.Sprintf(" dict(%s)", s.DictName) + } + if s.IsRoot { + out += " root" + } + out += " {" + } + for _, f := range s.Fields { + out += "\n " + prettyPrintStructField(f) + } + out += "\n}" + return out +} + +func prettyPrintStructField(f *StructField) string { + ft := prettyPrintFieldType(&f.FieldType) + out := fmt.Sprintf("%s %s", f.Name, ft) + if f.DictName != "" { + out += fmt.Sprintf(" dict(%s)", f.DictName) + } + if f.Optional { + out += " optional" + } + return out +} + +func prettyPrintFieldType(ft *FieldType) string { + switch { + case ft.Primitive != nil: + switch ft.Primitive.Type { + case PrimitiveTypeInt64: + return "int64" + case PrimitiveTypeUint64: + return "uint64" + case PrimitiveTypeFloat64: + return "float64" + case PrimitiveTypeBool: + return "bool" + case PrimitiveTypeString: + return "string" + case PrimitiveTypeBytes: + return "bytes" + } + case ft.Array != nil: + return "[]" + prettyPrintFieldType(&ft.Array.ElemType) + case ft.Struct != "": + return ft.Struct + case ft.MultiMap != "": + return ft.MultiMap + case ft.Enum != "": + return ft.Enum + } + return "unknown" +} + type Struct struct { Name string `json:"name,omitempty"` OneOf bool `json:"oneof,omitempty"` diff --git a/go/pkg/schema/schema_prettyprint_test.go b/go/pkg/schema/schema_prettyprint_test.go new file mode 100644 index 00000000..afd9937d --- /dev/null +++ b/go/pkg/schema/schema_prettyprint_test.go @@ -0,0 +1,129 @@ +package schema + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPrettyPrint_SimpleStruct(t *testing.T) { + schema := &Schema{ + PackageName: []string{"com", "example", "test"}, + Structs: map[string]*Struct{ + "Person": { + Name: "Person", + Fields: []*StructField{ + {Name: "Name", FieldType: FieldType{Primitive: &PrimitiveType{Type: PrimitiveTypeString}}}, + {Name: "Age", FieldType: FieldType{Primitive: &PrimitiveType{Type: PrimitiveTypeUint64}}}, + }, + }, + }, + } + + expected := `package com.example.test + +struct Person { + Name string + Age uint64 +}` + actual := strings.TrimSpace(schema.PrettyPrint()) + require.Equal(t, expected, actual) +} + +func TestPrettyPrint_EnumAndMultimap(t *testing.T) { + schema := &Schema{ + PackageName: []string{"com", "example", "test"}, + Enums: map[string]*Enum{ + "MetricType": { + Name: "MetricType", + Fields: []EnumField{ + {Name: "Gauge", Value: 0}, + {Name: "Counter", Value: 1}, + }, + }, + }, + Multimaps: map[string]*Multimap{ + "Labels": { + Name: "Labels", + Key: MultimapField{ + Type: FieldType{ + Primitive: &PrimitiveType{Type: PrimitiveTypeString}, DictName: "LabelKeys", + }, + }, + Value: MultimapField{ + Type: FieldType{ + Primitive: &PrimitiveType{Type: PrimitiveTypeString}, DictName: "LabelValues", + }, + }, + }, + }, + } + + expected := `package com.example.test + +enum MetricType { + Gauge = 0 + Counter = 1 +} + +multimap Labels { + key string dict(LabelKeys) + value string dict(LabelValues) +}` + actual := strings.TrimSpace(schema.PrettyPrint()) + require.Equal(t, expected, actual) +} + +func TestPrettyPrint_OneofAndArray(t *testing.T) { + schema := &Schema{ + PackageName: []string{"example"}, + Structs: map[string]*Struct{ + "JsonValue": { + Name: "JsonValue", + OneOf: true, + Fields: []*StructField{ + {Name: "String", FieldType: FieldType{Primitive: &PrimitiveType{Type: PrimitiveTypeString}}}, + {Name: "Number", FieldType: FieldType{Primitive: &PrimitiveType{Type: PrimitiveTypeFloat64}}}, + {Name: "Array", FieldType: FieldType{Array: &ArrayType{ElemType: FieldType{Struct: "JsonValue"}}}}, + }, + }, + }, + } + expected := `package example + +oneof JsonValue { + String string + Number float64 + Array []JsonValue +}` + actual := strings.TrimSpace(schema.PrettyPrint()) + require.Equal(t, expected, actual) +} + +func TestPrettyPrint_OptionalAndDict(t *testing.T) { + schema := &Schema{ + PackageName: []string{"example"}, + Structs: map[string]*Struct{ + "User": { + Name: "User", + Fields: []*StructField{ + {Name: "Name", FieldType: FieldType{Primitive: &PrimitiveType{Type: PrimitiveTypeString}}}, + { + Name: "Email", + FieldType: FieldType{Primitive: &PrimitiveType{Type: PrimitiveTypeString}, DictName: "Emails"}, + Optional: true, + }, + }, + }, + }, + } + expected := `package example + +struct User { + Name string + Email string dict(Emails) optional +}` + actual := strings.TrimSpace(schema.PrettyPrint()) + require.Equal(t, expected, actual) +} diff --git a/java/src/main/java/com/example/oteltef/Attributes.java b/java/src/main/java/com/example/oteltef/Attributes.java index 6f6c330a..18ecc17a 100644 --- a/java/src/main/java/com/example/oteltef/Attributes.java +++ b/java/src/main/java/com/example/oteltef/Attributes.java @@ -135,6 +135,7 @@ public void append(StringValue k, AnyValue v) { elems[elemsLen-1] = elem; } + // setKey sets the key of the element at index i. public void setKey(int i, StringValue k) { if (!Types.StringEqual(elems[i].key, k)) { @@ -142,14 +143,8 @@ public void setKey(int i, StringValue k) { modifiedElems.markKeyModified(i); } } - - // setValue sets the value of the element at index i. - public void setValue(int i, AnyValue v) { - if (!elems[i].value.equals(v)) { - elems[i].value = v; - modifiedElems.markValModified(i); - } - } + + // byteSize returns approximate memory usage in bytes. Used to calculate // memory used by dictionaries. diff --git a/java/src/main/java/com/example/oteltef/AttributesEncoder.java b/java/src/main/java/com/example/oteltef/AttributesEncoder.java index 5c0ca261..6b2c5fc8 100644 --- a/java/src/main/java/com/example/oteltef/AttributesEncoder.java +++ b/java/src/main/java/com/example/oteltef/AttributesEncoder.java @@ -112,11 +112,13 @@ private void encodeFull(Attributes list) throws IOException { public void collectColumns(WriteColumnSet columnSet) { columnSet.setBytes(buf); + int colIdx = 0; if (!isKeyRecursive) { - keyEncoder.collectColumns(columnSet.at(0)); + keyEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; } if (!isValueRecursive) { - valueEncoder.collectColumns(columnSet.at(1)); + valueEncoder.collectColumns(columnSet.at(colIdx)); } } } diff --git a/java/src/main/java/com/example/oteltef/EnvelopeAttributes.java b/java/src/main/java/com/example/oteltef/EnvelopeAttributes.java index 33eecd33..dbef9a7b 100644 --- a/java/src/main/java/com/example/oteltef/EnvelopeAttributes.java +++ b/java/src/main/java/com/example/oteltef/EnvelopeAttributes.java @@ -131,6 +131,7 @@ public void append(StringValue k, BytesValue v) { elems[elemsLen-1] = elem; } + // setKey sets the key of the element at index i. public void setKey(int i, StringValue k) { if (!Types.StringEqual(elems[i].key, k)) { @@ -138,7 +139,8 @@ public void setKey(int i, StringValue k) { modifiedElems.markKeyModified(i); } } - + + // setValue sets the value of the element at index i. public void setValue(int i, BytesValue v) { if (!Types.BytesEqual(elems[i].value, v)) { @@ -146,6 +148,7 @@ public void setValue(int i, BytesValue v) { modifiedElems.markValModified(i); } } + // byteSize returns approximate memory usage in bytes. Used to calculate // memory used by dictionaries. diff --git a/java/src/main/java/com/example/oteltef/EnvelopeAttributesEncoder.java b/java/src/main/java/com/example/oteltef/EnvelopeAttributesEncoder.java index b3777896..02f5237b 100644 --- a/java/src/main/java/com/example/oteltef/EnvelopeAttributesEncoder.java +++ b/java/src/main/java/com/example/oteltef/EnvelopeAttributesEncoder.java @@ -106,11 +106,13 @@ private void encodeFull(EnvelopeAttributes list) throws IOException { public void collectColumns(WriteColumnSet columnSet) { columnSet.setBytes(buf); + int colIdx = 0; if (!isKeyRecursive) { - keyEncoder.collectColumns(columnSet.at(0)); + keyEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; } if (!isValueRecursive) { - valueEncoder.collectColumns(columnSet.at(1)); + valueEncoder.collectColumns(columnSet.at(colIdx)); } } } diff --git a/java/src/main/java/com/example/oteltef/KeyValueList.java b/java/src/main/java/com/example/oteltef/KeyValueList.java index 68193d75..0af59566 100644 --- a/java/src/main/java/com/example/oteltef/KeyValueList.java +++ b/java/src/main/java/com/example/oteltef/KeyValueList.java @@ -135,6 +135,7 @@ public void append(StringValue k, AnyValue v) { elems[elemsLen-1] = elem; } + // setKey sets the key of the element at index i. public void setKey(int i, StringValue k) { if (!Types.StringEqual(elems[i].key, k)) { @@ -142,14 +143,8 @@ public void setKey(int i, StringValue k) { modifiedElems.markKeyModified(i); } } - - // setValue sets the value of the element at index i. - public void setValue(int i, AnyValue v) { - if (!elems[i].value.equals(v)) { - elems[i].value = v; - modifiedElems.markValModified(i); - } - } + + // byteSize returns approximate memory usage in bytes. Used to calculate // memory used by dictionaries. diff --git a/java/src/main/java/com/example/oteltef/KeyValueListEncoder.java b/java/src/main/java/com/example/oteltef/KeyValueListEncoder.java index 0c6ceb05..f575eb43 100644 --- a/java/src/main/java/com/example/oteltef/KeyValueListEncoder.java +++ b/java/src/main/java/com/example/oteltef/KeyValueListEncoder.java @@ -112,11 +112,13 @@ private void encodeFull(KeyValueList list) throws IOException { public void collectColumns(WriteColumnSet columnSet) { columnSet.setBytes(buf); + int colIdx = 0; if (!isKeyRecursive) { - keyEncoder.collectColumns(columnSet.at(0)); + keyEncoder.collectColumns(columnSet.at(colIdx)); + colIdx++; } if (!isValueRecursive) { - valueEncoder.collectColumns(columnSet.at(1)); + valueEncoder.collectColumns(columnSet.at(colIdx)); } } } diff --git a/makefile b/makefile index 3d9bc267..8310e5d7 100644 --- a/makefile +++ b/makefile @@ -46,7 +46,7 @@ ifndef VERSION endif RELEASE_MODULES := go/pkg go/grpc go/otel go/pdata -ALL_MODULES += $(RELEASE_MODULES) stefc stefc/generator/testdata examples/jsonl examples/profile otelcol benchmarks +ALL_MODULES += $(RELEASE_MODULES) stefc stefc/generator/testdata examples/jsonl examples/profile examples/ints otelcol benchmarks .PHONY: gotidy gotidy: diff --git a/stefc/generator/compileschema.go b/stefc/generator/compileschema.go index 786d57ec..bc0f4752 100644 --- a/stefc/generator/compileschema.go +++ b/stefc/generator/compileschema.go @@ -52,11 +52,8 @@ func (s *genSchema) resolveType(typ genFieldTypeRef) error { } if ref, ok := typ.(*genArrayTypeRef); ok { - if refStr, ok := ref.ElemType.(*genStructTypeRef); ok { - refStr.Def = s.Structs[refStr.Name] - if refStr.Def == nil { - return fmt.Errorf("struct %s not found", refStr.Name) - } + if err := s.resolveType(ref.ElemType); err != nil { + return err } } @@ -70,6 +67,9 @@ func (s *genSchema) resolveType(typ genFieldTypeRef) error { ref, ok := typ.(*genPrimitiveTypeRef) if ok && ref.Enum != "" { ref.EnumDef = s.Enums[ref.Enum] + if ref.EnumDef == nil { + return fmt.Errorf("enum %s not found", ref.Enum) + } } return nil diff --git a/stefc/generator/generator_test.go b/stefc/generator/generator_test.go index db675862..cf5d888f 100644 --- a/stefc/generator/generator_test.go +++ b/stefc/generator/generator_test.go @@ -3,17 +3,81 @@ package generator import ( "bytes" "fmt" + "math/rand/v2" "os" "os/exec" "path" "path/filepath" + "strings" "testing" + "time" "github.com/stretchr/testify/require" "github.com/splunk/stef/go/pkg/idl" + "github.com/splunk/stef/go/pkg/schema" ) +func testSchema(t *testing.T, schemaContent []byte, schemaFileName string, failOnTest bool) { + // Parse the schema + lexer := idl.NewLexer(bytes.NewBuffer(schemaContent)) + parser := idl.NewParser(lexer, path.Base(schemaFileName)) + err := parser.Parse() + require.NoError(t, err) + + parsedSchema := parser.Schema() + + // Clean Go directory + goDir := path.Join("testdata", "out", path.Base(schemaFileName)) + err = os.RemoveAll(goDir) + require.NoError(t, err) + + // Generate the Go code + genGo := Generator{ + SchemaContent: schemaContent, + OutputDir: goDir, + Lang: LangGo, + genTools: true, // Generate testing tools + } + + err = genGo.GenFile(parsedSchema) + require.NoError(t, err) + + fmt.Printf("Testing generated code in %s\n", genGo.OutputDir) + + // Run tests in the generated code + cmd := exec.Command("go", "test", "-v", genGo.OutputDir+"/...") + cmd.Dir = genGo.OutputDir + stdoutStderr, err := cmd.CombinedOutput() + if err != nil { + fmt.Printf("%s\n", stdoutStderr) + if failOnTest { + t.Fatal(err) + } else { + t.Skipf("Warning: go test failed: %v\n", err) + return + } + } + + // Clean Java directory + javaDir := path.Join("../../java/src/test/java") + packageDir := path.Join(javaDir, strings.Join(parsedSchema.PackageName, "/")) + err = os.RemoveAll(packageDir) + require.NoError(t, err) + + // Generate the Java code + genJava := Generator{ + SchemaContent: schemaContent, + OutputDir: javaDir, + TestOutputDir: javaDir, + Lang: LangJava, + genTools: true, // Generate testing tools + } + + err = genJava.GenFile(parsedSchema) + require.NoError(t, err) +} + func TestGenerate(t *testing.T) { // Get the list of files in "testdata" directory files, err := filepath.Glob("testdata/*.stef") @@ -25,50 +89,189 @@ func TestGenerate(t *testing.T) { // Read the schema file schemaContent, err := os.ReadFile(file) require.NoError(t, err) + testSchema(t, schemaContent, file, true) + }, + ) + } +} - // Parse the schema - lexer := idl.NewLexer(bytes.NewBuffer(schemaContent)) - parser := idl.NewParser(lexer, path.Base(file)) - err = parser.Parse() - require.NoError(t, err) +type schemaGenerator struct { + multimapNames []string + structNames []string + enumNames []string + random *rand.Rand +} - parsedSchema := parser.Schema() +func (g *schemaGenerator) generate() *schema.Schema { + sch := &schema.Schema{} + sch.PackageName = []string{"com", "example", "gentest", "randomized"} - // Generate the Go code - genGo := Generator{ - SchemaContent: schemaContent, - OutputDir: path.Join("testdata", "out", path.Base(file)), - Lang: LangGo, - genTools: true, // Generate testing tools - } + // Generate multimap names + multiMapCount := g.random.IntN(5) + sch.Multimaps = map[string]*schema.Multimap{} + for i := 0; i < multiMapCount; i++ { + g.multimapNames = append(g.multimapNames, fmt.Sprintf("MultiMap%d", i+1)) + } - err = genGo.GenFile(parsedSchema) - require.NoError(t, err) + // Generate struct names + structCount := g.random.IntN(10) + 1 + sch.Structs = map[string]*schema.Struct{} + for i := 0; i < structCount; i++ { + g.structNames = append(g.structNames, fmt.Sprintf("Struct%d", i+1)) + } - fmt.Printf("Testing generated code in %s\n", genGo.OutputDir) - - // Run tests in the generated code - cmd := exec.Command("go", "test", "-v", genGo.OutputDir+"/...") - cmd.Dir = genGo.OutputDir - stdoutStderr, err := cmd.CombinedOutput() - if err != nil { - fmt.Printf("%s\n", stdoutStderr) - t.Fatal(err) - } - - // Generate the Java code - javaDir := path.Join("../../java/src/test/java") - genJava := Generator{ - SchemaContent: schemaContent, - OutputDir: javaDir, - TestOutputDir: javaDir, - Lang: LangJava, - genTools: true, // Generate testing tools - } - - err = genJava.GenFile(parsedSchema) - require.NoError(t, err) - }, - ) + // Generate enum names + enumCount := g.random.IntN(5) + sch.Enums = map[string]*schema.Enum{} + for i := 0; i < enumCount; i++ { + g.enumNames = append(g.enumNames, fmt.Sprintf("Enum%d", i+1)) + } + + // Generate multimaps + for _, mmName := range g.multimapNames { + mm := &schema.Multimap{ + Name: mmName, + Key: schema.MultimapField{Type: g.genRandomType(true)}, + Value: schema.MultimapField{Type: g.genRandomType(true)}, + } + sch.Multimaps[mmName] = mm + } + + // Generate structs + hasRoot := false + for _, structName := range g.structNames { + fieldCount := g.random.IntN(5) + 1 + fields := make([]*schema.StructField, fieldCount) + for i := 0; i < fieldCount; i++ { + typ := g.genRandomType(true) + field := &schema.StructField{ + Name: fmt.Sprintf("Field%d", i+1), + FieldType: typ, + Optional: g.random.IntN(2) == 0, + } + if typ.Struct != "" { + // TODO: detect and avoid cycles. For now mark it optional since cycles are + // allowed with optional fields. + field.Optional = true + } + fields[i] = field + } + str := &schema.Struct{ + Name: structName, + Fields: fields, + } + if g.random.IntN(structCount) == 0 { + str.IsRoot = true + hasRoot = true + } + sch.Structs[structName] = str + } + if !hasRoot { + // Ensure at least one root struct + for _, str := range sch.Structs { + str.IsRoot = true + break + } + } + + // Generate enums + for _, enumName := range g.enumNames { + fieldCount := g.random.IntN(5) + 1 + fields := make([]schema.EnumField, fieldCount) + for i := 0; i < fieldCount; i++ { + fields[i] = schema.EnumField{ + Name: fmt.Sprintf("Item%d", i+1), + Value: uint64(i), + } + } + sch.Enums[enumName] = &schema.Enum{ + Name: enumName, + Fields: fields, + } } + + return sch +} + +func (g *schemaGenerator) genRandomType(allowArray bool) schema.FieldType { + // Type categories: 0=primitive, 1=multimap, 2=enum, 3=array, 4=struct + choices := []int{0} // Always allow primitive + if allowArray { + choices = append(choices, 3) + } + if len(g.multimapNames) > 0 { + choices = append(choices, 1) + } + if len(g.enumNames) > 0 { + choices = append(choices, 2) + } + if len(g.structNames) > 0 { + choices = append(choices, 4) + } + cat := choices[g.random.IntN(len(choices))] + + switch cat { + case 0: // Primitive + prims := []schema.PrimitiveFieldType{ + schema.PrimitiveTypeInt64, + schema.PrimitiveTypeUint64, + schema.PrimitiveTypeFloat64, + schema.PrimitiveTypeBool, + schema.PrimitiveTypeString, + schema.PrimitiveTypeBytes, + } + prim := prims[g.random.IntN(len(prims))] + return schema.FieldType{Primitive: &schema.PrimitiveType{Type: prim}} + case 1: // Multimap + name := g.multimapNames[g.random.IntN(len(g.multimapNames))] + return schema.FieldType{MultiMap: name} + case 2: // Enum + name := g.enumNames[g.random.IntN(len(g.enumNames))] + return schema.FieldType{Enum: name} + case 3: // Array + // Limit array nesting to avoid deep recursion + if g.random.IntN(4) == 0 { // 25% chance to stop at primitive + prims := []schema.PrimitiveFieldType{ + schema.PrimitiveTypeInt64, + schema.PrimitiveTypeUint64, + schema.PrimitiveTypeFloat64, + schema.PrimitiveTypeBool, + schema.PrimitiveTypeString, + schema.PrimitiveTypeBytes, + } + prim := prims[g.random.IntN(len(prims))] + return schema.FieldType{Primitive: &schema.PrimitiveType{Type: prim}} + } + return schema.FieldType{Array: &schema.ArrayType{ElemType: g.genRandomType(false)}} + case 4: // Struct + name := g.structNames[g.random.IntN(len(g.structNames))] + return schema.FieldType{Struct: name} + } + // Fallback to primitive + return schema.FieldType{Primitive: &schema.PrimitiveType{Type: schema.PrimitiveTypeInt64}} +} + +func TestRandomizedSchema(t *testing.T) { + seed1 := uint64(time.Now().UnixNano()) + random := rand.New(rand.NewPCG(seed1, 0)) + + g := schemaGenerator{} + g.random = random + sch := g.generate() + + succeeded := false + defer func() { + if !succeeded { + fmt.Printf("Test failed with seed %v\n", seed1) + schemaContent := sch.PrettyPrint() + fmt.Printf("Schema:\n%s\n", schemaContent) + } + }() + + schemaContent := sch.PrettyPrint() + + // Test the schema. Don't fail if generated code tests fail, just report the seed for now. + testSchema(t, []byte(schemaContent), "randomized.stef", false) + + succeeded = true } diff --git a/stefc/generator/testdata/enum_array.stef b/stefc/generator/testdata/enum_array.stef new file mode 100644 index 00000000..d832dba6 --- /dev/null +++ b/stefc/generator/testdata/enum_array.stef @@ -0,0 +1,11 @@ +package com.example.gentest.enum_array + +enum Enum1 { + Item1 = 0 + Item2 = 1 + Item3 = 2 +} + +struct Struct1 root { + Field1 []Enum1 +} diff --git a/stefc/generator/testdata/multimap_key_recurse.stef b/stefc/generator/testdata/multimap_key_recurse.stef new file mode 100644 index 00000000..9a486703 --- /dev/null +++ b/stefc/generator/testdata/multimap_key_recurse.stef @@ -0,0 +1,35 @@ +package com.example.gentest.multimap_key_recurse + +enum Enum1 { + Item1 = 0 + Item2 = 1 + Item3 = 2 +} + +enum Enum2 { + Item1 = 0 + Item2 = 1 +} + +multimap MultiMap1 { + key MultiMap1 + value Enum1 +} + +struct Struct1 root { + Field1 Struct3 optional +} + +struct Struct2 { + Field1 []Struct1 optional + Field2 bool + Field3 string optional +} + +struct Struct3 { + Field1 bool optional + Field2 []Enum1 optional + Field3 MultiMap1 + Field4 Struct2 optional + Field5 Enum1 +} \ No newline at end of file diff --git a/stefc/go.mod b/stefc/go.mod index 13c8efda..95c1d4d7 100644 --- a/stefc/go.mod +++ b/stefc/go.mod @@ -9,7 +9,6 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/klauspost/compress v1.18.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/stefc/templates/go/array.go.tmpl b/stefc/templates/go/array.go.tmpl index fbf937cf..e952b35a 100644 --- a/stefc/templates/go/array.go.tmpl +++ b/stefc/templates/go/array.go.tmpl @@ -2,7 +2,7 @@ package {{ .PackageName }} import ( "math/rand/v2" - {{if not .ElemType.MustClone}} + {{if and (not .ElemType.MustClone) (not .ElemType.Enum)}} "slices" {{end}} "unsafe" @@ -71,11 +71,33 @@ func (e *{{.ArrayName}}) byteSize() uint { // of the array will be equal to the length of slice and elements of // the array will be assigned from elements of the slice. func (e* {{.ArrayName}}) CopyFromSlice(src []{{.ElemType.Exported}}) { +{{if .ElemType.Enum}} +{{/* Enums need special handling to convert from exported type to storage type.*/}} + differ := false + if len(e.elems) != len(src) { + differ = true + } else { + for i:=0; i