From bb11874848b448773f0ae8ecaeb0d3c6910c4f05 Mon Sep 17 00:00:00 2001 From: Xiwen Cheng Date: Fri, 10 Apr 2026 17:27:56 +0200 Subject: [PATCH 1/4] limited concurrency and better memory footprint --- README.md | 4 ++ default.yaml | 3 + lint/cache.go | 14 +++-- lint/config.go | 14 +++-- lint/config_test.go | 24 ++++++++ lint/lint.go | 10 ++++ lint/lint_rego.go | 9 ++- lint/options.go | 36 ++++++++++++ lint/options_test.go | 70 +++++++++++++++++++++++ mpr/mpr.go | 8 ++- resources/rules/001_0004_readfile_test.js | 2 +- 11 files changed, 180 insertions(+), 14 deletions(-) create mode 100644 lint/options.go create mode 100644 lint/options_test.go diff --git a/README.md b/README.md index b16e120..f5d349a 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,8 @@ lint: jsonFile: "" ignoreNoqa: false noCache: false + concurrency: 4 + regoTrace: false skip: example/doc: - rule: "001_002" @@ -102,6 +104,8 @@ Notes: - `rules.rulesets` are synchronized into `rules.path` before linting. - `lint.skip` supports skipping by document path (relative to `modelsource`) and rule number. - `lint.noCache` disables lint result cache when set to `true`. +- `lint.concurrency` limits how many rules are evaluated in parallel. Lower values reduce peak memory usage for large models. +- `lint.regoTrace` enables OPA tracing for Rego rules. Keep it `false` for normal runs to reduce memory overhead. --- diff --git a/default.yaml b/default.yaml index 04f5d2f..9feef1c 100644 --- a/default.yaml +++ b/default.yaml @@ -6,6 +6,9 @@ lint: xunitReport: "" jsonFile: "" ignoreNoqa: false + noCache: false + concurrency: 4 + regoTrace: false skip: {} cache: directory: .mendix-cache/mxlint diff --git a/lint/cache.go b/lint/cache.go index d36e7e3..0e51ecf 100644 --- a/lint/cache.go +++ b/lint/cache.go @@ -4,6 +4,7 @@ import ( "crypto/sha256" "encoding/json" "fmt" + "io" "os" "path/filepath" "strings" @@ -71,12 +72,18 @@ func getCachePath(cacheKey CacheKey) (string, error) { // computeFileHash computes SHA256 hash of a file's contents func computeFileHash(filePath string) (string, error) { - content, err := os.ReadFile(filePath) + file, err := os.Open(filePath) if err != nil { return "", err } - hash := sha256.Sum256(content) - return fmt.Sprintf("%x", hash), nil + defer file.Close() + + hasher := sha256.New() + if _, err := io.Copy(hasher, file); err != nil { + return "", err + } + + return fmt.Sprintf("%x", hasher.Sum(nil)), nil } // createCacheKey creates a cache key from rule and input file paths @@ -226,4 +233,3 @@ func GetCacheStats() (int, int64, error) { return fileCount, totalSize, err } - diff --git a/lint/config.go b/lint/config.go index 865a96d..0de5e5c 100644 --- a/lint/config.go +++ b/lint/config.go @@ -38,6 +38,9 @@ type ConfigLintSpec struct { XunitReport string `yaml:"xunitReport"` JSONFile string `yaml:"jsonFile"` IgnoreNoqa *bool `yaml:"ignoreNoqa"` + NoCache *bool `yaml:"noCache"` + Concurrency *int `yaml:"concurrency"` + RegoTrace *bool `yaml:"regoTrace"` Skip map[string][]ConfigSkipRule `yaml:"skip"` } @@ -293,11 +296,14 @@ func mergeConfig(base *Config, overlay *Config) { if overlay.Lint.IgnoreNoqa != nil { base.Lint.IgnoreNoqa = overlay.Lint.IgnoreNoqa } - if strings.TrimSpace(overlay.Cache.Directory) != "" { - base.Cache.Directory = strings.TrimSpace(overlay.Cache.Directory) + if overlay.Lint.NoCache != nil { + base.Lint.NoCache = overlay.Lint.NoCache } - if overlay.Cache.Enable != nil { - base.Cache.Enable = overlay.Cache.Enable + if overlay.Lint.Concurrency != nil { + base.Lint.Concurrency = overlay.Lint.Concurrency + } + if overlay.Lint.RegoTrace != nil { + base.Lint.RegoTrace = overlay.Lint.RegoTrace } if overlay.Serve.Port != nil { diff --git a/lint/config_test.go b/lint/config_test.go index fe640d4..472e3b1 100644 --- a/lint/config_test.go +++ b/lint/config_test.go @@ -326,3 +326,27 @@ func TestLoadMergedConfig_NormalizesSkipMapKeys(t *testing.T) { t.Fatalf("unexpected unnormalized skip key present: %#v", cfg.Lint.Skip) } } + +func TestLoadMergedConfig_LintConcurrencyAndTrace(t *testing.T) { + projectDir := t.TempDir() + setDefaultConfigForTest(t, "") + projectConfig := `lint: + concurrency: 2 + regoTrace: true +` + if err := os.WriteFile(filepath.Join(projectDir, "mxlint.yaml"), []byte(projectConfig), 0644); err != nil { + t.Fatalf("failed to write project config: %v", err) + } + + cfg, err := LoadMergedConfig(projectDir) + if err != nil { + t.Fatalf("LoadMergedConfig returned error: %v", err) + } + + if cfg.Lint.Concurrency == nil || *cfg.Lint.Concurrency != 2 { + t.Fatalf("expected lint.concurrency=2, got %#v", cfg.Lint.Concurrency) + } + if cfg.Lint.RegoTrace == nil || *cfg.Lint.RegoTrace != true { + t.Fatalf("expected lint.regoTrace=true, got %#v", cfg.Lint.RegoTrace) + } +} diff --git a/lint/lint.go b/lint/lint.go index 455dd42..f18cd47 100644 --- a/lint/lint.go +++ b/lint/lint.go @@ -48,11 +48,16 @@ func EvalAllWithResults(rulesPath string, modelSourcePath string, xunitReport st // Create a mutex to safely print testsuites var printMutex sync.Mutex + maxConcurrency := effectiveLintConcurrency(len(rules)) + sem := make(chan struct{}, maxConcurrency) + // Launch goroutines to evaluate rules in parallel for i, rule := range rules { + sem <- struct{}{} wg.Add(1) go func(index int, r Rule) { defer wg.Done() + defer func() { <-sem }() testsuite, err := evalTestsuite(r, modelSourcePath, ignoreNoqa, useCache) if err != nil { @@ -156,11 +161,16 @@ func EvalAll(rulesPath string, modelSourcePath string, xunitReport string, jsonF // Create a mutex to safely print testsuites var printMutex sync.Mutex + maxConcurrency := effectiveLintConcurrency(len(rules)) + sem := make(chan struct{}, maxConcurrency) + // Launch goroutines to evaluate rules in parallel for i, rule := range rules { + sem <- struct{}{} wg.Add(1) go func(index int, r Rule) { defer wg.Done() + defer func() { <-sem }() testsuite, err := evalTestsuite(r, modelSourcePath, ignoreNoqa, useCache) if err != nil { diff --git a/lint/lint_rego.go b/lint/lint_rego.go index f7e8b39..edea08b 100644 --- a/lint/lint_rego.go +++ b/lint/lint_rego.go @@ -51,12 +51,15 @@ func evalTestcase_Rego(rulePath string, queryString string, inputFilePath string ctx := context.Background() startTime := time.Now() - r := rego.New( + regoOptions := []func(*rego.Rego){ rego.Query(queryString), rego.Module(rulePath, regoContent), rego.Input(data), - rego.Trace(true), - ) + } + if regoTraceEnabled() { + regoOptions = append(regoOptions, rego.Trace(true)) + } + r := rego.New(regoOptions...) rs, err := r.Eval(ctx) if err != nil { diff --git a/lint/options.go b/lint/options.go new file mode 100644 index 0000000..062373e --- /dev/null +++ b/lint/options.go @@ -0,0 +1,36 @@ +package lint + +import "runtime" + +const defaultMaxLintConcurrency = 4 + +func effectiveLintConcurrency(ruleCount int) int { + if ruleCount <= 0 { + return 1 + } + + cfg := getConfig() + if cfg != nil && cfg.Lint.Concurrency != nil && *cfg.Lint.Concurrency > 0 { + if *cfg.Lint.Concurrency > ruleCount { + return ruleCount + } + return *cfg.Lint.Concurrency + } + + auto := runtime.GOMAXPROCS(0) + if auto < 1 { + auto = 1 + } + if auto > defaultMaxLintConcurrency { + auto = defaultMaxLintConcurrency + } + if auto > ruleCount { + auto = ruleCount + } + return auto +} + +func regoTraceEnabled() bool { + cfg := getConfig() + return cfg != nil && cfg.Lint.RegoTrace != nil && *cfg.Lint.RegoTrace +} diff --git a/lint/options_test.go b/lint/options_test.go new file mode 100644 index 0000000..5003e72 --- /dev/null +++ b/lint/options_test.go @@ -0,0 +1,70 @@ +package lint + +import "testing" + +func intPtr(v int) *int { + return &v +} + +func boolPtr(v bool) *bool { + return &v +} + +func TestEffectiveLintConcurrency_DefaultIsBounded(t *testing.T) { + SetConfig(&Config{}) + t.Cleanup(func() { + SetConfig(&Config{}) + }) + + value := effectiveLintConcurrency(100) + if value < 1 || value > defaultMaxLintConcurrency { + t.Fatalf("expected default concurrency within [1,%d], got %d", defaultMaxLintConcurrency, value) + } +} + +func TestEffectiveLintConcurrency_UsesConfigWhenProvided(t *testing.T) { + SetConfig(&Config{ + Lint: ConfigLintSpec{ + Concurrency: intPtr(2), + }, + }) + t.Cleanup(func() { + SetConfig(&Config{}) + }) + + value := effectiveLintConcurrency(10) + if value != 2 { + t.Fatalf("expected configured concurrency 2, got %d", value) + } +} + +func TestEffectiveLintConcurrency_CapsToRuleCount(t *testing.T) { + SetConfig(&Config{ + Lint: ConfigLintSpec{ + Concurrency: intPtr(8), + }, + }) + t.Cleanup(func() { + SetConfig(&Config{}) + }) + + value := effectiveLintConcurrency(3) + if value != 3 { + t.Fatalf("expected concurrency capped to rule count 3, got %d", value) + } +} + +func TestRegoTraceEnabled(t *testing.T) { + SetConfig(&Config{ + Lint: ConfigLintSpec{ + RegoTrace: boolPtr(true), + }, + }) + t.Cleanup(func() { + SetConfig(&Config{}) + }) + + if !regoTraceEnabled() { + t.Fatal("expected regoTraceEnabled to return true") + } +} diff --git a/mpr/mpr.go b/mpr/mpr.go index 920a55e..db8e487 100644 --- a/mpr/mpr.go +++ b/mpr/mpr.go @@ -12,8 +12,8 @@ import ( "strings" "sync" - "gopkg.in/yaml.v3" "go.mongodb.org/mongo-driver/bson" + "gopkg.in/yaml.v3" _ "github.com/glebarez/go-sqlite" ) @@ -98,7 +98,7 @@ func ExportModel(inputDirectory string, outputDirectory string, raw bool, appsto exportedCount := 0 if filter != "^Metadata$" { var err error - exportedCount, err = exportUnits(inputDirectory, tmpDir, raw, filter) + exportedCount, err = exportUnitsFromLoadedUnits(units, tmpDir, raw, filter) if err != nil { return fmt.Errorf("error exporting units: %v", err) } @@ -525,6 +525,10 @@ func exportUnits(inputDirectory string, outputDirectory string, raw bool, filter log.Errorf("Error getting units: %v", err) return 0, fmt.Errorf("error getting units: %v", err) } + return exportUnitsFromLoadedUnits(units, outputDirectory, raw, filter) +} + +func exportUnitsFromLoadedUnits(units []MxUnit, outputDirectory string, raw bool, filter string) (int, error) { folders, err := getMxFolders(units) if err != nil { return 0, fmt.Errorf("error getting folders: %v", err) diff --git a/resources/rules/001_0004_readfile_test.js b/resources/rules/001_0004_readfile_test.js index 1b2b88f..a283450 100644 --- a/resources/rules/001_0004_readfile_test.js +++ b/resources/rules/001_0004_readfile_test.js @@ -20,7 +20,7 @@ function rule(input = {}) { // Use mxlint.readfile to read the Settings$ProjectSettings.yaml file // which should be in the same directory as the input file try { - const settingsContent = mxlint.readfile("Settings$ProjectSettings.yaml"); + const settingsContent = mxlint.io.readfile("Settings$ProjectSettings.yaml"); // Check if the settings file contains expected content if (!settingsContent.includes("$Type:")) { From 7a8cbba73982a742be5ed3fe053ef63be89ccdaa Mon Sep 17 00:00:00 2001 From: Xiwen Cheng Date: Fri, 10 Apr 2026 22:11:51 +0200 Subject: [PATCH 2/4] memory efficient export --- mpr/export_stream.go | 391 +++++++++++++++++++++++++++++++++++++++++++ mpr/mpr.go | 16 +- 2 files changed, 400 insertions(+), 7 deletions(-) create mode 100644 mpr/export_stream.go diff --git a/mpr/export_stream.go b/mpr/export_stream.go new file mode 100644 index 0000000..ca6361d --- /dev/null +++ b/mpr/export_stream.go @@ -0,0 +1,391 @@ +package mpr + +import ( + "database/sql" + "encoding/base64" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + + _ "github.com/glebarez/go-sqlite" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/bson/primitive" +) + +var documentContainmentTypes = map[string]struct{}{ + "ProjectDocuments": {}, + "DomainModel": {}, + "ModuleSettings": {}, + "ModuleSecurity": {}, + "Documents": {}, +} + +type exportDocumentDescriptor struct { + UnitID string + Name string + Type string + ContainerID string + Path string +} + +type exportPlan struct { + Modules []MxModule + Documents []exportDocumentDescriptor + Load func(unitID string) (bson.M, error) + Close func() error +} + +func buildExportPlan(inputDirectory string) (*exportPlan, error) { + mprPath, err := getMprPath(inputDirectory) + if err != nil { + return nil, err + } + mprVersion, err := getMprVersion(mprPath) + if err != nil { + return nil, fmt.Errorf("error getting mpr version: %v", err) + } + if mprVersion == 2 { + return buildExportPlanV2(inputDirectory, mprPath) + } + return buildExportPlanV1(mprPath) +} + +func buildExportPlanV1(mprPath string) (*exportPlan, error) { + db, err := sql.Open("sqlite", mprPath) + if err != nil { + return nil, fmt.Errorf("error opening database: %v", err) + } + + rows, err := db.Query("SELECT UnitID, ContainerID, ContainmentName, Contents FROM Unit") + if err != nil { + _ = db.Close() + return nil, fmt.Errorf("error querying units: %v", err) + } + defer rows.Close() + + modules := make([]MxModule, 0) + folders := make([]MxFolder, 0) + documents := make([]exportDocumentDescriptor, 0) + + for rows.Next() { + var containmentName string + var unitID, containerID, contents []byte + if err := rows.Scan(&unitID, &containerID, &containmentName, &contents); err != nil { + _ = db.Close() + return nil, fmt.Errorf("error scanning unit: %v", err) + } + + var result bson.M + if err := bson.Unmarshal(contents, &result); err != nil { + _ = db.Close() + return nil, fmt.Errorf("error parsing unit: %v", err) + } + + unit := MxUnit{ + UnitID: base64.StdEncoding.EncodeToString(unitID), + ContainerID: base64.StdEncoding.EncodeToString(containerID), + ContainmentName: containmentName, + } + appendUnitDescriptor(unit, result, &modules, &folders, &documents) + } + if err := rows.Err(); err != nil { + _ = db.Close() + return nil, fmt.Errorf("error iterating units: %v", err) + } + + connectFolderParents(folders) + for i := range documents { + documents[i].Path = getMxDocumentPath(documents[i].ContainerID, folders) + } + + loadDocument := func(unitID string) (bson.M, error) { + rawUnitID, err := base64.StdEncoding.DecodeString(unitID) + if err != nil { + return nil, fmt.Errorf("failed decoding unit id %s: %w", unitID, err) + } + + var contents []byte + if err := db.QueryRow("SELECT Contents FROM Unit WHERE UnitID = ?", rawUnitID).Scan(&contents); err != nil { + return nil, fmt.Errorf("failed to query contents for unit %s: %w", unitID, err) + } + var result bson.M + if err := bson.Unmarshal(contents, &result); err != nil { + return nil, fmt.Errorf("failed to parse contents for unit %s: %w", unitID, err) + } + return result, nil + } + + return &exportPlan{ + Modules: modules, + Documents: documents, + Load: loadDocument, + Close: db.Close, + }, nil +} + +func buildExportPlanV2(inputDirectory string, mprPath string) (*exportPlan, error) { + units, err := getMxUnitsV2(mprPath) + if err != nil { + return nil, err + } + + unitHeaders := make(map[string]MxUnit, len(units)) + for _, unit := range units { + unitHeaders[unit.UnitID] = unit + } + + mxUnitPaths := make(map[string]string, len(units)) + modules := make([]MxModule, 0) + folders := make([]MxFolder, 0) + documents := make([]exportDocumentDescriptor, 0) + + mprContentsDirectory := filepath.Join(inputDirectory, "mprcontents") + err = filepath.Walk(mprContentsDirectory, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr + } + if strings.Contains(path, ".mendix-cache") { + return nil + } + if info.IsDir() || !strings.HasSuffix(info.Name(), ".mxunit") { + return nil + } + + contents, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("error reading file %s: %v", path, err) + } + var result bson.M + if err := bson.Unmarshal(contents, &result); err != nil { + return fmt.Errorf("unable to unmarshal BSON content for %s: %w", path, err) + } + + unitID, err := extractUnitIDFromRawContents(result, path) + if err != nil { + return err + } + header, exists := unitHeaders[unitID] + if !exists { + return fmt.Errorf("unable to find unit with ID: %s", unitID) + } + mxUnitPaths[unitID] = path + appendUnitDescriptor(header, result, &modules, &folders, &documents) + return nil + }) + if err != nil { + return nil, fmt.Errorf("error processing mprcontents: %v", err) + } + + connectFolderParents(folders) + for i := range documents { + documents[i].Path = getMxDocumentPath(documents[i].ContainerID, folders) + } + + loadDocument := func(unitID string) (bson.M, error) { + mxunitPath, ok := mxUnitPaths[unitID] + if !ok { + return nil, fmt.Errorf("mxunit path not found for unit %s", unitID) + } + + contents, err := os.ReadFile(mxunitPath) + if err != nil { + return nil, fmt.Errorf("failed to read mxunit %s: %w", mxunitPath, err) + } + var result bson.M + if err := bson.Unmarshal(contents, &result); err != nil { + return nil, fmt.Errorf("failed to unmarshal mxunit %s: %w", mxunitPath, err) + } + return result, nil + } + + return &exportPlan{ + Modules: modules, + Documents: documents, + Load: loadDocument, + Close: func() error { + return nil + }, + }, nil +} + +func appendUnitDescriptor(unit MxUnit, contents bson.M, modules *[]MxModule, folders *[]MxFolder, documents *[]exportDocumentDescriptor) { + if unit.ContainmentName == "Modules" { + name, _ := contents["Name"].(string) + *modules = append(*modules, MxModule{ + Name: name, + ID: unit.UnitID, + Attributes: contents, + }) + } + + if unit.ContainmentName == "Folders" || unit.ContainmentName == "Modules" || unit.ContainmentName == "" { + name := "" + if unit.ContainmentName != "" { + name, _ = contents["Name"].(string) + } + *folders = append(*folders, MxFolder{ + Name: name, + ID: unit.UnitID, + ParentID: unit.ContainerID, + Attributes: contents, + }) + } + + if _, ok := documentContainmentTypes[unit.ContainmentName]; ok { + name, _ := contents["Name"].(string) + docType, _ := contents["$Type"].(string) + *documents = append(*documents, exportDocumentDescriptor{ + UnitID: unit.UnitID, + Name: name, + Type: docType, + ContainerID: unit.ContainerID, + }) + } +} + +func connectFolderParents(folders []MxFolder) { + folderMap := make(map[string]*MxFolder) + for i := range folders { + folderMap[folders[i].ID] = &folders[i] + } + for i, folder := range folders { + if parent, exists := folderMap[folder.ParentID]; exists && folder.ParentID != folder.ID { + folders[i].Parent = parent + } + } +} + +func exportDocumentsFromPlan(plan *exportPlan, outputDirectory string, raw bool, filter string) (int, error) { + var err error + var filterRegex *regexp.Regexp + if filter != "" { + filterRegex, err = regexp.Compile(filter) + if err != nil { + return 0, fmt.Errorf("invalid filter regex pattern: %v", err) + } + log.Infof("Applying filter: %s", filter) + } + + exportedCount := 0 + for _, document := range plan.Documents { + if filterRegex != nil && !filterRegex.MatchString(document.Name) { + log.Debugf("Skipping document '%s' (does not match filter)", document.Name) + continue + } + + attributes, err := plan.Load(document.UnitID) + if err != nil { + return 0, fmt.Errorf("error loading document %s: %w", document.Name, err) + } + + if docType, _ := attributes["$Type"].(string); docType == microflowDocumentType { + enriched := enrichMicroflowDocument(MxDocument{ + Name: document.Name, + Type: document.Type, + Path: document.Path, + Attributes: attributes, + }) + attributes = enriched.Attributes + } + + if err := writeDocumentToDisk(document, outputDirectory, cleanData(attributes, raw)); err != nil { + return 0, err + } + exportedCount++ + } + + if filterRegex != nil { + log.Infof("Exported %d documents matching filter (out of %d total)", exportedCount, len(plan.Documents)) + } else { + log.Infof("Found %d documents", len(plan.Documents)) + } + return exportedCount, nil +} + +func writeDocumentToDisk(document exportDocumentDescriptor, outputDirectory string, attributes map[string]interface{}) error { + sanitizedPath := sanitizePath(document.Path) + if sanitizedPath != document.Path { + log.Warnf("Sanitized path: '%s' -> '%s'", document.Path, sanitizedPath) + } + + sanitizedName := sanitizePathComponent(document.Name) + sanitizedType := sanitizePathComponent(document.Type) + if sanitizedName != document.Name || sanitizedType != document.Type { + log.Debugf("Sanitized name: '%s' -> '%s', type: '%s' -> '%s'", document.Name, sanitizedName, document.Type, sanitizedType) + } + + fname := fmt.Sprintf("%s.%s.yaml", sanitizedName, sanitizedType) + if document.Name == "" { + fname = fmt.Sprintf("%s.yaml", sanitizedType) + } + + adjustedPath, adjustedFilename, err := validatePathLength(outputDirectory, sanitizedPath, fname) + if err != nil { + return fmt.Errorf("error adjusting path length: %v", err) + } + + directory := filepath.Join(outputDirectory, adjustedPath) + if _, err := os.Stat(directory); os.IsNotExist(err) { + if err := os.MkdirAll(directory, 0755); err != nil { + return fmt.Errorf("error creating directory: %v", err) + } + } + + if err := writeFile(filepath.Join(directory, adjustedFilename), attributes); err != nil { + log.Errorf("Error writing file: %v", err) + return err + } + return nil +} + +func extractUnitIDFromRawContents(data map[string]interface{}, path string) (string, error) { + idData, ok := data["$ID"] + if !ok { + return "", fmt.Errorf("missing $ID field in %s", path) + } + + switch id := idData.(type) { + case primitive.Binary: + if len(id.Data) >= 16 { + return base64.StdEncoding.EncodeToString(id.Data), nil + } + case map[string]interface{}: + if dataStr, ok := id["Data"].(string); ok && dataStr != "" { + return dataStr, nil + } + if dataVal, ok := id["data"]; ok { + switch dataBytes := dataVal.(type) { + case primitive.Binary: + if len(dataBytes.Data) >= 16 { + return base64.StdEncoding.EncodeToString(dataBytes.Data), nil + } + case []interface{}: + bytes := make([]byte, 0, len(dataBytes)) + for _, b := range dataBytes { + if num, ok := b.(float64); ok { + bytes = append(bytes, byte(num)) + continue + } + if num, ok := b.(int); ok { + bytes = append(bytes, byte(num)) + } + } + if len(bytes) >= 16 { + return base64.StdEncoding.EncodeToString(bytes), nil + } + case []byte: + if len(dataBytes) >= 16 { + return base64.StdEncoding.EncodeToString(dataBytes), nil + } + } + } + case string: + if id != "" { + return id, nil + } + } + + return "", fmt.Errorf("unable to extract unit id from %s", path) +} diff --git a/mpr/mpr.go b/mpr/mpr.go index db8e487..ce32832 100644 --- a/mpr/mpr.go +++ b/mpr/mpr.go @@ -83,13 +83,16 @@ func ExportModel(inputDirectory string, outputDirectory string, raw bool, appsto return fmt.Errorf("no MPR file found in directory: %s", inputDirectory) } - units, err := getMxUnits(inputDirectory) + plan, err := buildExportPlan(inputDirectory) if err != nil { - log.Errorf("Failed to parse MxUnits: %s", err) - return err + return fmt.Errorf("error building export plan: %v", err) } - - modules := getMxModules(units) + defer func() { + if closeErr := plan.Close(); closeErr != nil { + log.Warnf("Error closing export resources: %v", closeErr) + } + }() + modules := plan.Modules if err := exportMetadata(inputDirectory, tmpDir, modules); err != nil { return fmt.Errorf("error exporting metadata: %v", err) @@ -97,8 +100,7 @@ func ExportModel(inputDirectory string, outputDirectory string, raw bool, appsto exportedCount := 0 if filter != "^Metadata$" { - var err error - exportedCount, err = exportUnitsFromLoadedUnits(units, tmpDir, raw, filter) + exportedCount, err = exportDocumentsFromPlan(plan, tmpDir, raw, filter) if err != nil { return fmt.Errorf("error exporting units: %v", err) } From 2260db1b2fb54e8c3618b334ff8679d756816ac3 Mon Sep 17 00:00:00 2001 From: Xiwen Cheng Date: Fri, 10 Apr 2026 22:59:33 +0200 Subject: [PATCH 3/4] fix modules ordering --- mpr/mpr.go | 15 ++++++++++++++- mpr/mpr_test.go | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/mpr/mpr.go b/mpr/mpr.go index ce32832..2fe1be9 100644 --- a/mpr/mpr.go +++ b/mpr/mpr.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "regexp" + "sort" "strings" "sync" @@ -214,10 +215,22 @@ func exportMetadata(inputDirectory string, outputDirectory string, modules []MxM } // create metadata object + sortedModules := append([]MxModule(nil), modules...) + sort.Slice(sortedModules, func(i, j int) bool { + left := strings.ToLower(sortedModules[i].Name) + right := strings.ToLower(sortedModules[j].Name) + if left == right { + if sortedModules[i].Name == sortedModules[j].Name { + return sortedModules[i].ID < sortedModules[j].ID + } + return sortedModules[i].Name < sortedModules[j].Name + } + return left < right + }) metadataObj := MxMetadata{ ProductVersion: productVersion, BuildVersion: buildVersion, - Modules: modules, + Modules: sortedModules, } // write metadata to file diff --git a/mpr/mpr_test.go b/mpr/mpr_test.go index bc3dd20..6c8b973 100644 --- a/mpr/mpr_test.go +++ b/mpr/mpr_test.go @@ -8,8 +8,8 @@ import ( "testing" "time" - "gopkg.in/yaml.v3" "go.mongodb.org/mongo-driver/bson" + "gopkg.in/yaml.v3" ) func TestGetMprVersion(t *testing.T) { @@ -1453,3 +1453,48 @@ func TestGenerateAppYamlExcludesItself(t *testing.T) { t.Errorf("test.yaml should be included in app.yaml structure") } } + +func TestExportMetadata_SortsModulesByName(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "mpr-test-metadata-sort-*") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(tmpDir) + + modules := []MxModule{ + {Name: "zeta", ID: "2"}, + {Name: "Alpha", ID: "3"}, + {Name: "beta", ID: "1"}, + } + + if err := exportMetadata("./../resources/app-mpr-v1", tmpDir, modules); err != nil { + t.Fatalf("exportMetadata() unexpected error: %v", err) + } + + metadataFile, err := os.ReadFile(filepath.Join(tmpDir, "Metadata.yaml")) + if err != nil { + t.Fatalf("Failed to read metadata file: %v", err) + } + + var metadataObj MxMetadata + var node yaml.Node + if err := yaml.Unmarshal(metadataFile, &node); err != nil { + t.Fatalf("Failed to unmarshal metadata file: %v", err) + } + if err := node.Decode(&metadataObj); err != nil { + t.Fatalf("Failed to decode metadata file: %v", err) + } + + if len(metadataObj.Modules) != 3 { + t.Fatalf("Expected 3 modules, got %d", len(metadataObj.Modules)) + } + + if metadataObj.Modules[0].Name != "Alpha" || metadataObj.Modules[1].Name != "beta" || metadataObj.Modules[2].Name != "zeta" { + t.Fatalf( + "Expected module order [Alpha, beta, zeta], got [%s, %s, %s]", + metadataObj.Modules[0].Name, + metadataObj.Modules[1].Name, + metadataObj.Modules[2].Name, + ) + } +} From a094af830703e5594c57c438d5e31b04fec4d6de Mon Sep 17 00:00:00 2001 From: Xiwen Cheng Date: Fri, 10 Apr 2026 23:06:31 +0200 Subject: [PATCH 4/4] refresh modelsources --- resources/modelsource-v1/Metadata.yaml | 260 ++++++++++++------------- resources/modelsource-v2/Metadata.yaml | 248 +++++++++++------------ 2 files changed, 254 insertions(+), 254 deletions(-) diff --git a/resources/modelsource-v1/Metadata.yaml b/resources/modelsource-v1/Metadata.yaml index 760a576..4092208 100644 --- a/resources/modelsource-v1/Metadata.yaml +++ b/resources/modelsource-v1/Metadata.yaml @@ -1,68 +1,37 @@ ProductVersion: 10.18.3.58900 BuildVersion: 10.18.3.58900 Modules: - - Name: CommunityCommons - ID: CJwh54tYwk+hc1+bCPwjDg== + - Name: Administration + ID: TJ9xCCh3h0mdTVo+0PSB5A== Attributes: $ID: subtype: 0 data: + - 76 + - 159 + - 113 - 8 - - 156 - - 33 - - 231 - - 139 - - 88 - - 194 - - 79 - - 161 - - 115 - - 95 - - 155 - - 8 - - 252 - - 35 - - 14 - $Type: Projects$ModuleImpl - AppStoreGuid: 1f12fd41-98dc-4965-8473-ae46e16ad853 - AppStorePackageId: 170 - AppStoreVersion: 10.9.0 - AppStoreVersionGuid: 1f12fd41-98dc-4965-8473-ae46e16ad853 - FromAppStore: true - IsThemeModule: false - Name: CommunityCommons - NewSortIndex: 3 - - Name: NanoflowCommons - ID: EBEQuiOQm0CiZysqR/iiQw== - Attributes: - $ID: - subtype: 0 - data: - - 16 - - 17 - - 16 - - 186 - - 35 - - 144 - - 155 - - 64 - - 162 - - 103 - - 43 - - 42 - - 71 - - 248 - - 162 - - 67 + - 40 + - 119 + - 135 + - 73 + - 157 + - 77 + - 90 + - 62 + - 208 + - 244 + - 129 + - 228 $Type: Projects$ModuleImpl - AppStoreGuid: 75fda84f-6c3d-460d-98eb-50cdbfd3b53d - AppStorePackageId: 109515 - AppStoreVersion: 4.0.2 - AppStoreVersionGuid: 4bb5080c-8939-4001-a86d-e16d1711a693 + AppStoreGuid: 1d7b7a3f-84e6-4fc1-baf0-afbd0ee23b13 + AppStorePackageId: 23513 + AppStoreVersion: 4.1.0 + AppStoreVersionGuid: 08b6928f-4a67-4983-b98e-decffcec5a82 FromAppStore: true IsThemeModule: false - Name: NanoflowCommons - NewSortIndex: 2.75 + Name: Administration + NewSortIndex: -0.5 - Name: Atlas_Core ID: N+Ifuf/Of0mwB5I9DLbDOA== Attributes: @@ -94,37 +63,6 @@ Modules: IsThemeModule: true Name: Atlas_Core NewSortIndex: 38 - - Name: Administration - ID: TJ9xCCh3h0mdTVo+0PSB5A== - Attributes: - $ID: - subtype: 0 - data: - - 76 - - 159 - - 113 - - 8 - - 40 - - 119 - - 135 - - 73 - - 157 - - 77 - - 90 - - 62 - - 208 - - 244 - - 129 - - 228 - $Type: Projects$ModuleImpl - AppStoreGuid: 1d7b7a3f-84e6-4fc1-baf0-afbd0ee23b13 - AppStorePackageId: 23513 - AppStoreVersion: 4.1.0 - AppStoreVersionGuid: 08b6928f-4a67-4983-b98e-decffcec5a82 - FromAppStore: true - IsThemeModule: false - Name: Administration - NewSortIndex: -0.5 - Name: Atlas_Web_Content ID: ZSsTntE9bkWOq8+tPUe98w== Attributes: @@ -156,37 +94,37 @@ Modules: IsThemeModule: true Name: Atlas_Web_Content NewSortIndex: 39 - - Name: FeedbackModule - ID: anwcVtQfBESck7qCSr6wYA== + - Name: CommunityCommons + ID: CJwh54tYwk+hc1+bCPwjDg== Attributes: $ID: subtype: 0 data: - - 106 - - 124 - - 28 - - 86 - - 212 - - 31 - - 4 - - 68 + - 8 - 156 - - 147 - - 186 - - 130 - - 74 - - 190 - - 176 - - 96 + - 33 + - 231 + - 139 + - 88 + - 194 + - 79 + - 161 + - 115 + - 95 + - 155 + - 8 + - 252 + - 35 + - 14 $Type: Projects$ModuleImpl - AppStoreGuid: de8fdcee-11ea-4431-aead-8caba99a1c60 - AppStorePackageId: 205506 - AppStoreVersion: 1.4.0 - AppStoreVersionGuid: 8d3b9e71-ac72-4aab-ba3f-ee283bf4d81a + AppStoreGuid: 1f12fd41-98dc-4965-8473-ae46e16ad853 + AppStorePackageId: 170 + AppStoreVersion: 10.9.0 + AppStoreVersionGuid: 1f12fd41-98dc-4965-8473-ae46e16ad853 FromAppStore: true IsThemeModule: false - Name: FeedbackModule - NewSortIndex: 4 + Name: CommunityCommons + NewSortIndex: 3 - Name: DataWidgets ID: rOLa4hehRU2AzTf/EaZ+Hw== Attributes: @@ -218,37 +156,37 @@ Modules: IsThemeModule: true Name: DataWidgets NewSortIndex: 2 - - Name: WebActions - ID: tklS1TNqIECWUG8EyKUaDw== + - Name: FeedbackModule + ID: anwcVtQfBESck7qCSr6wYA== Attributes: $ID: subtype: 0 data: - - 182 - - 73 - - 82 - - 213 - - 51 - 106 - - 32 - - 64 - - 150 - - 80 - - 111 + - 124 + - 28 + - 86 + - 212 + - 31 - 4 - - 200 - - 165 - - 26 - - 15 + - 68 + - 156 + - 147 + - 186 + - 130 + - 74 + - 190 + - 176 + - 96 $Type: Projects$ModuleImpl - AppStoreGuid: 87f3694f-b0d1-432e-a150-df1753dbd0e8 - AppStorePackageId: 114337 - AppStoreVersion: 2.10.0 - AppStoreVersionGuid: 507824ec-9757-4b89-85b5-8d1f4fc4bc34 + AppStoreGuid: de8fdcee-11ea-4431-aead-8caba99a1c60 + AppStorePackageId: 205506 + AppStoreVersion: 1.4.0 + AppStoreVersionGuid: 8d3b9e71-ac72-4aab-ba3f-ee283bf4d81a FromAppStore: true IsThemeModule: false - Name: WebActions - NewSortIndex: 3 + Name: FeedbackModule + NewSortIndex: 4 - Name: MyFirstModule ID: xn10Fre2rkKqvyVdyirurw== Attributes: @@ -280,3 +218,65 @@ Modules: IsThemeModule: false Name: MyFirstModule NewSortIndex: 2 + - Name: NanoflowCommons + ID: EBEQuiOQm0CiZysqR/iiQw== + Attributes: + $ID: + subtype: 0 + data: + - 16 + - 17 + - 16 + - 186 + - 35 + - 144 + - 155 + - 64 + - 162 + - 103 + - 43 + - 42 + - 71 + - 248 + - 162 + - 67 + $Type: Projects$ModuleImpl + AppStoreGuid: 75fda84f-6c3d-460d-98eb-50cdbfd3b53d + AppStorePackageId: 109515 + AppStoreVersion: 4.0.2 + AppStoreVersionGuid: 4bb5080c-8939-4001-a86d-e16d1711a693 + FromAppStore: true + IsThemeModule: false + Name: NanoflowCommons + NewSortIndex: 2.75 + - Name: WebActions + ID: tklS1TNqIECWUG8EyKUaDw== + Attributes: + $ID: + subtype: 0 + data: + - 182 + - 73 + - 82 + - 213 + - 51 + - 106 + - 32 + - 64 + - 150 + - 80 + - 111 + - 4 + - 200 + - 165 + - 26 + - 15 + $Type: Projects$ModuleImpl + AppStoreGuid: 87f3694f-b0d1-432e-a150-df1753dbd0e8 + AppStorePackageId: 114337 + AppStoreVersion: 2.10.0 + AppStoreVersionGuid: 507824ec-9757-4b89-85b5-8d1f4fc4bc34 + FromAppStore: true + IsThemeModule: false + Name: WebActions + NewSortIndex: 3 diff --git a/resources/modelsource-v2/Metadata.yaml b/resources/modelsource-v2/Metadata.yaml index 0a22dcf..ac2a19e 100644 --- a/resources/modelsource-v2/Metadata.yaml +++ b/resources/modelsource-v2/Metadata.yaml @@ -1,99 +1,6 @@ ProductVersion: 10.24.9.81004 BuildVersion: 10.24.9.81004 Modules: - - Name: Atlas_Web_Content - ID: GMyJGqJY0U2f8EzUDi71Dg== - Attributes: - $ID: - subtype: 0 - data: - - 24 - - 204 - - 137 - - 26 - - 162 - - 88 - - 209 - - 77 - - 159 - - 240 - - 76 - - 212 - - 14 - - 46 - - 245 - - 14 - $Type: Projects$ModuleImpl - AppStoreGuid: 08060dc1-8f61-4969-8f90-707c74fc0cc1 - AppStorePackageId: 117183 - AppStoreVersion: 3.7.0 - AppStoreVersionGuid: c78ddb3b-ecb1-4970-b3f7-e1aa18334abb - FromAppStore: true - IsThemeModule: true - Name: Atlas_Web_Content - NewSortIndex: 39 - - Name: WebActions - ID: LrkbIDFi7UqJBl7WEXRFzg== - Attributes: - $ID: - subtype: 0 - data: - - 46 - - 185 - - 27 - - 32 - - 49 - - 98 - - 237 - - 74 - - 137 - - 6 - - 94 - - 214 - - 17 - - 116 - - 69 - - 206 - $Type: Projects$ModuleImpl - AppStoreGuid: 87f3694f-b0d1-432e-a150-df1753dbd0e8 - AppStorePackageId: 114337 - AppStoreVersion: 2.10.0 - AppStoreVersionGuid: 507824ec-9757-4b89-85b5-8d1f4fc4bc34 - FromAppStore: true - IsThemeModule: false - Name: WebActions - NewSortIndex: 3 - - Name: Module2 - ID: NR8NSYo2t0qz/m3jTEL67A== - Attributes: - $ID: - subtype: 0 - data: - - 53 - - 31 - - 13 - - 73 - - 138 - - 54 - - 183 - - 74 - - 179 - - 254 - - 109 - - 227 - - 76 - - 66 - - 250 - - 236 - $Type: Projects$ModuleImpl - AppStoreGuid: "" - AppStorePackageId: 0 - AppStoreVersion: "" - AppStoreVersionGuid: "" - FromAppStore: false - IsThemeModule: false - Name: Module2 - NewSortIndex: 3.86974074450023 - Name: Administration ID: O5j0GrEKKkmyWj3j8dSfVQ== Attributes: @@ -125,37 +32,6 @@ Modules: IsThemeModule: false Name: Administration NewSortIndex: -0.5 - - Name: DataWidgets - ID: jwnoFbjX9UKZzhRqfaFXdw== - Attributes: - $ID: - subtype: 0 - data: - - 143 - - 9 - - 232 - - 21 - - 184 - - 215 - - 245 - - 66 - - 153 - - 206 - - 20 - - 106 - - 125 - - 161 - - 87 - - 119 - $Type: Projects$ModuleImpl - AppStoreGuid: b0376c9a-37f9-4c96-bca9-491d20f3e975 - AppStorePackageId: 116540 - AppStoreVersion: 2.28.1 - AppStoreVersionGuid: 3ab7ec2f-84cd-4973-bc66-3e0f4d992b90 - FromAppStore: true - IsThemeModule: true - Name: DataWidgets - NewSortIndex: 2 - Name: Atlas_Core ID: po8inIuZIUaXMNeU8eApPQ== Attributes: @@ -187,6 +63,68 @@ Modules: IsThemeModule: true Name: Atlas_Core NewSortIndex: 38 + - Name: Atlas_Web_Content + ID: GMyJGqJY0U2f8EzUDi71Dg== + Attributes: + $ID: + subtype: 0 + data: + - 24 + - 204 + - 137 + - 26 + - 162 + - 88 + - 209 + - 77 + - 159 + - 240 + - 76 + - 212 + - 14 + - 46 + - 245 + - 14 + $Type: Projects$ModuleImpl + AppStoreGuid: 08060dc1-8f61-4969-8f90-707c74fc0cc1 + AppStorePackageId: 117183 + AppStoreVersion: 3.7.0 + AppStoreVersionGuid: c78ddb3b-ecb1-4970-b3f7-e1aa18334abb + FromAppStore: true + IsThemeModule: true + Name: Atlas_Web_Content + NewSortIndex: 39 + - Name: DataWidgets + ID: jwnoFbjX9UKZzhRqfaFXdw== + Attributes: + $ID: + subtype: 0 + data: + - 143 + - 9 + - 232 + - 21 + - 184 + - 215 + - 245 + - 66 + - 153 + - 206 + - 20 + - 106 + - 125 + - 161 + - 87 + - 119 + $Type: Projects$ModuleImpl + AppStoreGuid: b0376c9a-37f9-4c96-bca9-491d20f3e975 + AppStorePackageId: 116540 + AppStoreVersion: 2.28.1 + AppStoreVersionGuid: 3ab7ec2f-84cd-4973-bc66-3e0f4d992b90 + FromAppStore: true + IsThemeModule: true + Name: DataWidgets + NewSortIndex: 2 - Name: FeedbackModule ID: uiNyvoj+W0uB9ORByapJZw== Attributes: @@ -218,6 +156,37 @@ Modules: IsThemeModule: true Name: FeedbackModule NewSortIndex: 4 + - Name: Module2 + ID: NR8NSYo2t0qz/m3jTEL67A== + Attributes: + $ID: + subtype: 0 + data: + - 53 + - 31 + - 13 + - 73 + - 138 + - 54 + - 183 + - 74 + - 179 + - 254 + - 109 + - 227 + - 76 + - 66 + - 250 + - 236 + $Type: Projects$ModuleImpl + AppStoreGuid: "" + AppStorePackageId: 0 + AppStoreVersion: "" + AppStoreVersionGuid: "" + FromAppStore: false + IsThemeModule: false + Name: Module2 + NewSortIndex: 3.86974074450023 - Name: MyFirstModule ID: xn10Fre2rkKqvyVdyirurw== Attributes: @@ -280,3 +249,34 @@ Modules: IsThemeModule: false Name: NanoflowCommons NewSortIndex: 2.75 + - Name: WebActions + ID: LrkbIDFi7UqJBl7WEXRFzg== + Attributes: + $ID: + subtype: 0 + data: + - 46 + - 185 + - 27 + - 32 + - 49 + - 98 + - 237 + - 74 + - 137 + - 6 + - 94 + - 214 + - 17 + - 116 + - 69 + - 206 + $Type: Projects$ModuleImpl + AppStoreGuid: 87f3694f-b0d1-432e-a150-df1753dbd0e8 + AppStorePackageId: 114337 + AppStoreVersion: 2.10.0 + AppStoreVersionGuid: 507824ec-9757-4b89-85b5-8d1f4fc4bc34 + FromAppStore: true + IsThemeModule: false + Name: WebActions + NewSortIndex: 3