From 3920ab81bca2a0a3c00ea01045ab843c8a428611 Mon Sep 17 00:00:00 2001 From: marco Date: Tue, 23 Sep 2025 12:27:49 +0200 Subject: [PATCH 1/5] slicetools/Batch --- slicetools/batch.go | 36 +++++++++++++ slicetools/batch_test.go | 113 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 slicetools/batch.go create mode 100644 slicetools/batch_test.go diff --git a/slicetools/batch.go b/slicetools/batch.go new file mode 100644 index 0000000..836a188 --- /dev/null +++ b/slicetools/batch.go @@ -0,0 +1,36 @@ +package slicetools + +import "context" + +// a simple alternative to slices.Chunk (or slicetools.Chunks) +// if you need no allocations, context cancelation and no parallelism +// +// also: doesn't panic for batchSize = 0 + +// Batch applies fn to successive chunks of elements, each of size at most batchSize. +// A batchSize of 0 (or negative) processes all the elements in one chunk. +// Stops at the first error and returns it. +func Batch[T any](ctx context.Context, elems []T, batchSize int, fn func(context.Context, []T) error) error { + n := len(elems) + + if n == 0 { + return nil + } + + if batchSize <= 0 || batchSize > n { + batchSize = n + } + + for start := 0; start < n; start += batchSize { + if ctx.Err() != nil { + return ctx.Err() + } + + end := min(start+batchSize, n) + if err := fn(ctx, elems[start:end]); err != nil { + return err + } + } + + return nil +} diff --git a/slicetools/batch_test.go b/slicetools/batch_test.go new file mode 100644 index 0000000..4fda2f2 --- /dev/null +++ b/slicetools/batch_test.go @@ -0,0 +1,113 @@ +package slicetools_test + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/go-cs-lib/slicetools" +) + +func TestBatch(t *testing.T) { + type callRecord struct { + calledBatches [][]int + } + + tests := []struct { + name string + elems []int + batchSize int + cancelCtx bool + // number of batch where fn fails (0 = never) + fnErrorAt int + wantErr bool + wantBatches [][]int + }{ + { + name: "normal batching", + elems: []int{1, 2, 3, 4, 5}, + batchSize: 2, + wantBatches: [][]int{{1, 2}, {3, 4}, {5}}, + }, + { + name: "batchSize zero = all in one batch", + elems: []int{1, 2, 3}, + batchSize: 0, + wantBatches: [][]int{{1, 2, 3}}, + }, + { + name: "batchSize > len(elems)", + elems: []int{1, 2, 3}, + batchSize: 10, + wantBatches: [][]int{{1, 2, 3}}, + }, + { + name: "empty input", + elems: []int{}, + batchSize: 3, + wantBatches: nil, + }, + { + name: "nil input", + elems: nil, + batchSize: 3, + wantBatches: nil, + }, + { + name: "error in fn", + elems: []int{1, 2, 3, 4}, + batchSize: 2, + fnErrorAt: 2, + wantErr: true, + wantBatches: [][]int{{1, 2}}, + }, + { + name: "context canceled before loop", + elems: []int{1, 2, 3}, + batchSize: 2, + cancelCtx: true, + wantErr: true, + wantBatches: nil, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var rec callRecord + + ctx := t.Context() + + // not testing a cancel _between_ batches, this should be enough + if tc.cancelCtx { + canceled, cancel := context.WithCancel(ctx) + cancel() + + ctx = canceled + } + + err := slicetools.Batch(ctx, tc.elems, tc.batchSize, func(_ context.Context, batch []int) error { + if len(rec.calledBatches) == tc.fnErrorAt-1 { + return errors.New("simulated error") + } + + rec.calledBatches = append(rec.calledBatches, batch) + + return nil + }) + + switch { + case tc.wantErr && tc.cancelCtx: + require.ErrorContains(t, err, "context canceled") + case tc.wantErr: + require.ErrorContains(t, err, "simulated error") + default: + require.NoError(t, err) + } + + assert.Equal(t, tc.wantBatches, rec.calledBatches) + }) + } +} From d0c56bde30e0fbeb64d3ac77c05b3895daf01740 Mon Sep 17 00:00:00 2001 From: marco Date: Tue, 23 Sep 2025 12:40:35 +0200 Subject: [PATCH 2/5] lint --- .golangci.yml | 8 ++++---- cstest/aws.go | 5 ++++- cstime/cstime.go | 3 ++- cstime/duration.go | 5 +++++ cstime/duration_test.go | 5 +++++ csyaml/keys.go | 2 ++ csyaml/merge.go | 11 +++++++++++ csyaml/merge_test.go | 1 + csyaml/splityaml.go | 1 + downloader/download.go | 5 +++++ go.mod | 8 ++++---- go.sum | 12 ++++++------ 12 files changed, 50 insertions(+), 16 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 9d99f7a..f1ac5ad 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -11,7 +11,7 @@ linters: - inamedparam # reports interfaces with unnamed method parameters - wrapcheck # Checks that errors returned from external packages are wrapped - err113 # Go linter to check the errors handling expressions - #- noinlineerr + - noinlineerr - paralleltest # Detects missing usage of t.Parallel() method in your Go test - testpackage # linter that makes you use a separate _test package - exhaustruct # Checks if all structure fields are initialized @@ -104,15 +104,15 @@ linters: - 43 - name: defer disabled: true - #- name: enforce-switch-style - # disabled: true + - name: enforce-switch-style + disabled: true - name: flag-parameter disabled: true - name: function-length arguments: # lower this after refactoring - 104 - - 196 + - 198 - name: line-length-limit arguments: # lower this after refactoring diff --git a/cstest/aws.go b/cstest/aws.go index 990af90..1698c52 100644 --- a/cstest/aws.go +++ b/cstest/aws.go @@ -5,6 +5,7 @@ import ( "os" "strings" "testing" + "time" ) // SetAWSTestEnv sets the environment variables required to run tests against LocalStack, @@ -30,7 +31,9 @@ func SetAWSTestEnv(t *testing.T) string { t.Setenv("AWS_ACCESS_KEY_ID", "test") t.Setenv("AWS_SECRET_ACCESS_KEY", "test") - _, err := net.Dial("tcp", strings.TrimPrefix(endpoint, "http://")) + dialer := &net.Dialer{Timeout: 2 * time.Second} + + _, err := dialer.DialContext(t.Context(), "tcp", strings.TrimPrefix(endpoint, "http://")) if err != nil { t.Fatalf("%s: make sure localstack is running and retry", err) } diff --git a/cstime/cstime.go b/cstime/cstime.go index 653bf40..443bb3e 100644 --- a/cstime/cstime.go +++ b/cstime/cstime.go @@ -8,7 +8,7 @@ import ( "time" ) -// ParseDuration parses a string representing a duration, and supports +// ParseDurationWithDays parses a string representing a duration, and supports // days as a unit (e.g., "2d", "2d3h", "24h", "2h45m"). func ParseDurationWithDays(input string) (time.Duration, error) { var total time.Duration @@ -55,6 +55,7 @@ func ParseDurationWithDays(input string) (time.Duration, error) { if err != nil { return 0, err } + total += dur } diff --git a/cstime/duration.go b/cstime/duration.go index 3cb1de3..6725c93 100644 --- a/cstime/duration.go +++ b/cstime/duration.go @@ -20,11 +20,14 @@ type DurationWithDays time.Duration // UnmarshalText implements encoding.TextUnmarshaler (used by YAML/JSON libs). func (d *DurationWithDays) UnmarshalText(text []byte) error { s := string(text) + dur, err := ParseDurationWithDays(s) if err != nil { return err } + *d = DurationWithDays(dur) + return nil } @@ -39,6 +42,7 @@ func (d *DurationWithDays) UnmarshalJSON(b []byte) error { if err := json.Unmarshal(b, &s); err != nil { return err } + return d.UnmarshalText([]byte(s)) } @@ -48,6 +52,7 @@ func (d DurationWithDays) MarshalJSON() ([]byte, error) { if err != nil { return nil, err } + return json.Marshal(string(s)) } diff --git a/cstime/duration_test.go b/cstime/duration_test.go index e1952c0..d2e1b5b 100644 --- a/cstime/duration_test.go +++ b/cstime/duration_test.go @@ -59,11 +59,14 @@ func TestUnmarshalText(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { var d DurationWithDays + err := d.UnmarshalText([]byte(tc.input)) cstest.RequireErrorContains(t, err, tc.wantErr) + if tc.wantErr != "" { return } + assert.Equal(t, tc.want, time.Duration(d)) }) } @@ -115,6 +118,7 @@ func TestDuration_UnmarshalJSON(t *testing.T) { jsonInput := `{"timeout": "2d3h45m"}` var cfg Config + err := json.Unmarshal([]byte(jsonInput), &cfg) require.NoError(t, err) @@ -130,6 +134,7 @@ func TestDuration_UnmarshalYAML(t *testing.T) { yamlInput := "timeout: 2d3h45m" var cfg Config + err := yaml.Unmarshal([]byte(yamlInput), &cfg) require.NoError(t, err) diff --git a/csyaml/keys.go b/csyaml/keys.go index 1c3247a..ca23136 100644 --- a/csyaml/keys.go +++ b/csyaml/keys.go @@ -30,8 +30,10 @@ func GetDocumentKeys(r io.Reader) ([][]string, error) { if errors.Is(err, io.EOF) { break } + return nil, fmt.Errorf("position %d: %s", idx, yaml.FormatError(err, false, false)) } + keys := []string{} // Only mapping nodes become MapSlice with UseOrderedMap() diff --git a/csyaml/merge.go b/csyaml/merge.go index 7c9a547..044f3cc 100644 --- a/csyaml/merge.go +++ b/csyaml/merge.go @@ -22,7 +22,9 @@ import ( // Always runs in strict mode: type mismatches or duplicate keys cause an error. func Merge(inputs [][]byte) (*bytes.Buffer, error) { var merged any + hasContent := false + for idx, data := range inputs { dec := yaml.NewDecoder(bytes.NewReader(data), yaml.UseOrderedMap(), yaml.Strict()) @@ -31,8 +33,10 @@ func Merge(inputs [][]byte) (*bytes.Buffer, error) { if errors.Is(err, io.EOF) { continue } + return nil, fmt.Errorf("decoding document %d: %s", idx, yaml.FormatError(err, false, false)) } + hasContent = true mergedValue, err := mergeValue(merged, value) @@ -91,8 +95,10 @@ func mergeValue(into, from any) (any, error) { func mergeMap(into, from yaml.MapSlice) (yaml.MapSlice, error) { out := make(yaml.MapSlice, len(into)) copy(out, into) + for _, item := range from { matched := false + for i, existing := range out { if !reflect.DeepEqual(existing.Key, item.Key) { continue @@ -102,13 +108,16 @@ func mergeMap(into, from yaml.MapSlice) (yaml.MapSlice, error) { if err != nil { return nil, err } + out[i].Value = mergedVal matched = true } + if !matched { out = append(out, yaml.MapItem{Key: item.Key, Value: item.Value}) } } + return out, nil } @@ -126,8 +135,10 @@ func describe(i any) string { if isMapping(i) { return "mapping" } + if isSequence(i) { return "sequence" } + return "scalar" } diff --git a/csyaml/merge_test.go b/csyaml/merge_test.go index ceadb35..5ebecf9 100644 --- a/csyaml/merge_test.go +++ b/csyaml/merge_test.go @@ -68,6 +68,7 @@ func TestMergeYAML(t *testing.T) { buf, err := csyaml.Merge(bs) cstest.RequireErrorContains(t, err, tc.wantErr) + if tc.wantErr != "" { require.Nil(t, buf) return diff --git a/csyaml/splityaml.go b/csyaml/splityaml.go index 0923331..97a6239 100644 --- a/csyaml/splityaml.go +++ b/csyaml/splityaml.go @@ -34,6 +34,7 @@ func SplitDocumentsDecEnc(r io.Reader) ([][]byte, error) { enc := yaml.NewEncoder(&buf) enc.SetIndent(2) + if err := enc.Encode(&node); err != nil { return nil, fmt.Errorf("encode doc %d: %w", idx, err) } diff --git a/downloader/download.go b/downloader/download.go index 8ba368d..ddf09a1 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -282,6 +282,7 @@ func (d *Downloader) isLocalFresh(ctx context.Context, url string, modTime time. if !localIsOld { d.logger.Debugf("No last modified header, but local file is not old: %s", d.destPath) + return true, nil } @@ -298,6 +299,7 @@ func (d *Downloader) isLocalFresh(ctx context.Context, url string, modTime time. if modTime.After(lastAvailable) { d.logger.Debugf("Local file is newer than remote: %s (%s vs %s)", d.destPath, modTime, lastAvailable) + return true, nil } @@ -461,6 +463,7 @@ func compareFiles(file1, file2 string) (bool, error) { defer f2.Close() const bufSize = 4096 + buf1 := make([]byte, bufSize) buf2 := make([]byte, bufSize) @@ -580,6 +583,7 @@ func (d *Downloader) Download(ctx context.Context, url string) (bool, error) { } defer gzipReader.Close() + reader = gzipReader } @@ -601,6 +605,7 @@ func (d *Downloader) Download(ctx context.Context, url string) (bool, error) { } tmpFileName := tmpFile.Name() + defer func() { _ = tmpFile.Close() _ = os.Remove(tmpFileName) diff --git a/go.mod b/go.mod index 5bd53b6..50b9fd0 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,14 @@ module github.com/crowdsecurity/go-cs-lib -go 1.23 +go 1.24 require ( github.com/blackfireio/osinfo v1.1.0 github.com/coreos/go-systemd/v22 v22.5.0 github.com/goccy/go-yaml v1.18.0 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/cobra v1.9.1 - github.com/stretchr/testify v1.10.0 + github.com/spf13/cobra v1.10.1 + github.com/stretchr/testify v1.11.1 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -17,6 +17,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/spf13/pflag v1.0.6 // indirect + github.com/spf13/pflag v1.0.9 // indirect golang.org/x/sys v0.7.0 // indirect ) diff --git a/go.sum b/go.sum index 826f733..e08589b 100644 --- a/go.sum +++ b/go.sum @@ -16,14 +16,14 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= -github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From eab6f44a9b36dc9866ab05c33d8e94d901ae5347 Mon Sep 17 00:00:00 2001 From: marco Date: Tue, 23 Sep 2025 12:41:26 +0200 Subject: [PATCH 3/5] wip --- slicetools/batch.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/slicetools/batch.go b/slicetools/batch.go index 836a188..5b41d0b 100644 --- a/slicetools/batch.go +++ b/slicetools/batch.go @@ -3,7 +3,7 @@ package slicetools import "context" // a simple alternative to slices.Chunk (or slicetools.Chunks) -// if you need no allocations, context cancelation and no parallelism +// if you want no allocations, context cancelation and no parallelism // // also: doesn't panic for batchSize = 0 From 4d178cf38b1169b7c47fedc38cd28709143645a5 Mon Sep 17 00:00:00 2001 From: marco Date: Tue, 23 Sep 2025 12:48:35 +0200 Subject: [PATCH 4/5] wip --- slicetools/batch.go | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/slicetools/batch.go b/slicetools/batch.go index 5b41d0b..c7c46cd 100644 --- a/slicetools/batch.go +++ b/slicetools/batch.go @@ -1,36 +1,53 @@ package slicetools -import "context" +import ( + "context" + "slices" +) -// a simple alternative to slices.Chunk (or slicetools.Chunks) -// if you want no allocations, context cancelation and no parallelism +// a simple wrapper around slices.Chunk +// if you want context cancelation and don't need parallelism // -// also: doesn't panic for batchSize = 0 +// also: doesn't panic for size = 0 -// Batch applies fn to successive chunks of elements, each of size at most batchSize. -// A batchSize of 0 (or negative) processes all the elements in one chunk. +// Batch applies fn to successive chunks of at most "size" elements. +// A size of 0 (or negative) processes all the elements in one chunk. // Stops at the first error and returns it. -func Batch[T any](ctx context.Context, elems []T, batchSize int, fn func(context.Context, []T) error) error { +func Batch[T any](ctx context.Context, elems []T, size int, fn func(context.Context, []T) error) error { n := len(elems) if n == 0 { return nil } - if batchSize <= 0 || batchSize > n { - batchSize = n + if size <= 0 || size > n { + size = n } - for start := 0; start < n; start += batchSize { - if ctx.Err() != nil { - return ctx.Err() + // delegate to stdlib + + for part := range slices.Chunk(elems, size) { + if err := ctx.Err(); err != nil { + return err } - end := min(start+batchSize, n) - if err := fn(ctx, elems[start:end]); err != nil { + if err := fn(ctx, part); err != nil { return err } } + // we have stdlib at home + + // for start := 0; start < n; start += size { + // if ctx.Err() != nil { + // return ctx.Err() + // } + // + // end := min(start+size, n) + // if err := fn(ctx, elems[start:end]); err != nil { + // return err + // } + // } + return nil } From fa285b00d73cc4eb3518cb065550d0886e41361f Mon Sep 17 00:00:00 2001 From: marco Date: Tue, 23 Sep 2025 12:50:40 +0200 Subject: [PATCH 5/5] update golangci-lint --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 587bff9..e1a2abb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,6 +34,6 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0 with: - version: v2.1 + version: v2.4 args: --issues-exit-code=1 --timeout 10m only-new-issues: false