slicetools.Batch

.github/workflows/tests.yml

@@ -34,6 +34,6 @@ jobs:
     - name: golangci-lint
       uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0
       with:
-        version: v2.1
+        version: v2.4
         args: --issues-exit-code=1 --timeout 10m
         only-new-issues: false

.golangci.yml

@@ -11,7 +11,7 @@ linters:
     - inamedparam           # reports interfaces with unnamed method parameters
     - wrapcheck             # Checks that errors returned from external packages are wrapped
     - err113                # Go linter to check the errors handling expressions
-    #- noinlineerr
+    - noinlineerr
     - paralleltest          # Detects missing usage of t.Parallel() method in your Go test
     - testpackage           # linter that makes you use a separate _test package
     - exhaustruct           # Checks if all structure fields are initialized
@@ -104,15 +104,15 @@ linters:
             - 43
         - name: defer
           disabled: true
-        #- name: enforce-switch-style
-        #  disabled: true
+        - name: enforce-switch-style
+          disabled: true
         - name: flag-parameter
           disabled: true
         - name: function-length
           arguments:
             # lower this after refactoring
             - 104
-            - 196
+            - 198
         - name: line-length-limit
           arguments:
             # lower this after refactoring

cstest/aws.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"strings"
 	"testing"
+	"time"
 )
 
 // SetAWSTestEnv sets the environment variables required to run tests against LocalStack,
@@ -30,7 +31,9 @@ func SetAWSTestEnv(t *testing.T) string {
 	t.Setenv("AWS_ACCESS_KEY_ID", "test")
 	t.Setenv("AWS_SECRET_ACCESS_KEY", "test")
 
-	_, err := net.Dial("tcp", strings.TrimPrefix(endpoint, "http://"))
+	dialer := &net.Dialer{Timeout: 2 * time.Second}
+
+	_, err := dialer.DialContext(t.Context(), "tcp", strings.TrimPrefix(endpoint, "http://"))
 	if err != nil {
 		t.Fatalf("%s: make sure localstack is running and retry", err)
 	}

cstime/cstime.go

@@ -8,7 +8,7 @@ import (
 	"time"
 )
 
-// ParseDuration parses a string representing a duration, and supports
+// ParseDurationWithDays parses a string representing a duration, and supports
 // days as a unit (e.g., "2d", "2d3h", "24h", "2h45m").
 func ParseDurationWithDays(input string) (time.Duration, error) {
 	var total time.Duration
@@ -55,6 +55,7 @@ func ParseDurationWithDays(input string) (time.Duration, error) {
 		if err != nil {
 			return 0, err
 		}
+
 		total += dur
 	}
 

cstime/duration.go

@@ -20,11 +20,14 @@ type DurationWithDays time.Duration
 // UnmarshalText implements encoding.TextUnmarshaler (used by YAML/JSON libs).
 func (d *DurationWithDays) UnmarshalText(text []byte) error {
 	s := string(text)
+
 	dur, err := ParseDurationWithDays(s)
 	if err != nil {
 		return err
 	}
+
 	*d = DurationWithDays(dur)
+
 	return nil
 }
 
@@ -39,6 +42,7 @@ func (d *DurationWithDays) UnmarshalJSON(b []byte) error {
 	if err := json.Unmarshal(b, &s); err != nil {
 		return err
 	}
+
 	return d.UnmarshalText([]byte(s))
 }
 
@@ -48,6 +52,7 @@ func (d DurationWithDays) MarshalJSON() ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	return json.Marshal(string(s))
 }
 

cstime/duration_test.go

@@ -59,11 +59,14 @@ func TestUnmarshalText(t *testing.T) {
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
 			var d DurationWithDays
+
 			err := d.UnmarshalText([]byte(tc.input))
 			cstest.RequireErrorContains(t, err, tc.wantErr)
+
 			if tc.wantErr != "" {
 				return
 			}
+
 			assert.Equal(t, tc.want, time.Duration(d))
 		})
 	}
@@ -115,6 +118,7 @@ func TestDuration_UnmarshalJSON(t *testing.T) {
 	jsonInput := `{"timeout": "2d3h45m"}`
 
 	var cfg Config
+
 	err := json.Unmarshal([]byte(jsonInput), &cfg)
 	require.NoError(t, err)
 
@@ -130,6 +134,7 @@ func TestDuration_UnmarshalYAML(t *testing.T) {
 	yamlInput := "timeout: 2d3h45m"
 
 	var cfg Config
+
 	err := yaml.Unmarshal([]byte(yamlInput), &cfg)
 	require.NoError(t, err)
 

csyaml/keys.go

@@ -30,8 +30,10 @@ func GetDocumentKeys(r io.Reader) ([][]string, error) {
 			if errors.Is(err, io.EOF) {
 				break
 			}
+
 			return nil, fmt.Errorf("position %d: %s", idx, yaml.FormatError(err, false, false))
 		}
+
 		keys := []string{}
 
 		// Only mapping nodes become MapSlice with UseOrderedMap()

csyaml/merge.go

@@ -22,7 +22,9 @@ import (
 // Always runs in strict mode: type mismatches or duplicate keys cause an error.
 func Merge(inputs [][]byte) (*bytes.Buffer, error) {
 	var merged any
+
 	hasContent := false
+
 	for idx, data := range inputs {
 		dec := yaml.NewDecoder(bytes.NewReader(data), yaml.UseOrderedMap(), yaml.Strict())
 
@@ -31,8 +33,10 @@ func Merge(inputs [][]byte) (*bytes.Buffer, error) {
 			if errors.Is(err, io.EOF) {
 				continue
 			}
+
 			return nil, fmt.Errorf("decoding document %d: %s", idx, yaml.FormatError(err, false, false))
 		}
+
 		hasContent = true
 
 		mergedValue, err := mergeValue(merged, value)
@@ -91,8 +95,10 @@ func mergeValue(into, from any) (any, error) {
 func mergeMap(into, from yaml.MapSlice) (yaml.MapSlice, error) {
 	out := make(yaml.MapSlice, len(into))
 	copy(out, into)
+
 	for _, item := range from {
 		matched := false
+
 		for i, existing := range out {
 			if !reflect.DeepEqual(existing.Key, item.Key) {
 				continue
@@ -102,13 +108,16 @@ func mergeMap(into, from yaml.MapSlice) (yaml.MapSlice, error) {
 			if err != nil {
 				return nil, err
 			}
+
 			out[i].Value = mergedVal
 			matched = true
 		}
+
 		if !matched {
 			out = append(out, yaml.MapItem{Key: item.Key, Value: item.Value})
 		}
 	}
+
 	return out, nil
 }
 
@@ -126,8 +135,10 @@ func describe(i any) string {
 	if isMapping(i) {
 		return "mapping"
 	}
+
 	if isSequence(i) {
 		return "sequence"
 	}
+
 	return "scalar"
 }

csyaml/merge_test.go

@@ -68,6 +68,7 @@ func TestMergeYAML(t *testing.T) {
 
 			buf, err := csyaml.Merge(bs)
 			cstest.RequireErrorContains(t, err, tc.wantErr)
+
 			if tc.wantErr != "" {
 				require.Nil(t, buf)
 				return

csyaml/splityaml.go

@@ -34,6 +34,7 @@ func SplitDocumentsDecEnc(r io.Reader) ([][]byte, error) {
 
 		enc := yaml.NewEncoder(&buf)
 		enc.SetIndent(2)
+
 		if err := enc.Encode(&node); err != nil {
 			return nil, fmt.Errorf("encode doc %d: %w", idx, err)
 		}

downloader/download.go

@@ -282,6 +282,7 @@ func (d *Downloader) isLocalFresh(ctx context.Context, url string, modTime time.
 		if !localIsOld {
 			d.logger.Debugf("No last modified header, but local file is not old: %s",
 				d.destPath)
+
 			return true, nil
 		}
 
@@ -298,6 +299,7 @@ func (d *Downloader) isLocalFresh(ctx context.Context, url string, modTime time.
 	if modTime.After(lastAvailable) {
 		d.logger.Debugf("Local file is newer than remote: %s (%s vs %s)",
 			d.destPath, modTime, lastAvailable)
+
 		return true, nil
 	}
 
@@ -461,6 +463,7 @@ func compareFiles(file1, file2 string) (bool, error) {
 	defer f2.Close()
 
 	const bufSize = 4096
+
 	buf1 := make([]byte, bufSize)
 	buf2 := make([]byte, bufSize)
 
@@ -580,6 +583,7 @@ func (d *Downloader) Download(ctx context.Context, url string) (bool, error) {
 		}
 
 		defer gzipReader.Close()
+
 		reader = gzipReader
 	}
 
@@ -601,6 +605,7 @@ func (d *Downloader) Download(ctx context.Context, url string) (bool, error) {
 	}
 
 	tmpFileName := tmpFile.Name()
+
 	defer func() {
 		_ = tmpFile.Close()
 		_ = os.Remove(tmpFileName)

go.mod

@@ -1,14 +1,14 @@
 module github.com/crowdsecurity/go-cs-lib
 
-go 1.23
+go 1.24
 
 require (
 	github.com/blackfireio/osinfo v1.1.0
 	github.com/coreos/go-systemd/v22 v22.5.0
 	github.com/goccy/go-yaml v1.18.0
 	github.com/sirupsen/logrus v1.9.3
-	github.com/spf13/cobra v1.9.1
-	github.com/stretchr/testify v1.10.0
+	github.com/spf13/cobra v1.10.1
+	github.com/stretchr/testify v1.11.1
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -17,6 +17,6 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/spf13/pflag v1.0.9 // indirect
 	golang.org/x/sys v0.7.0 // indirect
 )

go.sum

@@ -16,14 +16,14 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
-github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
-github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
-github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
+github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
+github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
 golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

slicetools/batch.go

@@ -0,0 +1,53 @@
+package slicetools
+
+import (
+	"context"
+	"slices"
+)
+
+// a simple wrapper around slices.Chunk
+// if you want context cancelation and don't need parallelism
+//
+// also: doesn't panic for size = 0
+
+// Batch applies fn to successive chunks of at most "size" elements.
+// A size of 0 (or negative) processes all the elements in one chunk.
+// Stops at the first error and returns it.
+func Batch[T any](ctx context.Context, elems []T, size int, fn func(context.Context, []T) error) error {
+	n := len(elems)
+
+	if n == 0 {
+		return nil
+	}
+
+	if size <= 0 || size > n {
+		size = n
+	}
+
+	// delegate to stdlib
+
+	for part := range slices.Chunk(elems, size) {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+
+		if err := fn(ctx, part); err != nil {
+			return err
+		}
+	}
+
+	// we have stdlib at home
+
+	//	for start := 0; start < n; start += size {
+	//		if ctx.Err() != nil {
+	//			return ctx.Err()
+	//		}
+	//
+	//		end := min(start+size, n)
+	//		if err := fn(ctx, elems[start:end]); err != nil {
+	//			return err
+	//		}
+	//	}
+
+	return nil
+}