From e5447320978afae50ddd14e3ec5894d1ad534265 Mon Sep 17 00:00:00 2001 From: jiuker <2818723467@qq.com> Date: Fri, 17 May 2024 11:50:08 +0800 Subject: [PATCH 1/7] support ref resource support ref resource --- helm/operator/templates/job.min.io_jobs.yaml | 893 ++++++++++++++++++ pkg/apis/job.min.io/v1alpha1/types.go | 29 + pkg/controller/job-controller.go | 12 +- pkg/utils/miniojob/minioJob.go | 58 +- pkg/utils/miniojob/minioJob_test.go | 256 ++--- pkg/utils/miniojob/types.go | 177 ++-- resources/base/crds/job.min.io_miniojobs.yaml | 893 ++++++++++++++++++ 7 files changed, 2029 insertions(+), 289 deletions(-) diff --git a/helm/operator/templates/job.min.io_jobs.yaml b/helm/operator/templates/job.min.io_jobs.yaml index 08bc0d9d24..8ffc3bdc0e 100644 --- a/helm/operator/templates/job.min.io_jobs.yaml +++ b/helm/operator/templates/job.min.io_jobs.yaml @@ -47,10 +47,903 @@ spec: items: type: string type: array + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + items: + properties: + configMapRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array name: type: string op: type: string + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array required: - op type: object diff --git a/pkg/apis/job.min.io/v1alpha1/types.go b/pkg/apis/job.min.io/v1alpha1/types.go index 033a395b27..6515c3b44c 100644 --- a/pkg/apis/job.min.io/v1alpha1/types.go +++ b/pkg/apis/job.min.io/v1alpha1/types.go @@ -145,6 +145,35 @@ type CommandSpec struct { // DependsOn List of named `command` in this MinioJob that have to be scheduled and executed before this command runs // +optional DependsOn []string `json:"dependsOn,omitempty"` + + // Compute Resources required by this container. + // Cannot be updated. + // More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + // +optional + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // List of sources to populate environment variables in the container. + // The keys defined within a source must be a C_IDENTIFIER. All invalid keys + // will be reported as an event when the container is starting. When a key exists in multiple + // sources, the value associated with the last source will take precedence. + // Values defined by an Env with a duplicate key will take precedence. + // Cannot be updated. + // +optional + EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"` + // List of environment variables to set in the container. + // Cannot be updated. + // +optional + Env []corev1.EnvVar `json:"env,omitempty"` + + // Pod volumes to mount into the container's filesystem. + // Cannot be updated. + // +optional + VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"` + + // List of volumes that can be mounted by containers belonging to the pod. + // More info: https://kubernetes.io/docs/concepts/storage/volumes + // +optional + Volumes []corev1.Volume `json:"volumes,omitempty"` } // TenantRef Is the reference to the target tenant of the jobs diff --git a/pkg/controller/job-controller.go b/pkg/controller/job-controller.go index e9abffb629..dd8d0b07f9 100644 --- a/pkg/controller/job-controller.go +++ b/pkg/controller/job-controller.go @@ -304,15 +304,7 @@ func checkMinIOJob(jobCR *v1alpha1.MinIOJob) (intervalJob *miniojob.MinIOInterva return intervalJob, fmt.Errorf("serviceaccount name is empty") } for index, val := range jobCR.Spec.Commands { - mcCommand, found := miniojob.OperationAliasToMC(val.Operation) - if !found { - return intervalJob, fmt.Errorf("operation %s is not supported", val.Operation) - } - argsFuncs, found := miniojob.JobOperation[mcCommand] - if !found { - return intervalJob, fmt.Errorf("operation %s is not supported", mcCommand) - } - jobCommand, err := miniojob.GenerateMinIOIntervalJobCommand(mcCommand, index, val.DependsOn, val.Name, val.Args, argsFuncs) + jobCommand, err := miniojob.GenerateMinIOIntervalJobCommand(val, index) if err != nil { return intervalJob, err } @@ -321,7 +313,7 @@ func checkMinIOJob(jobCR *v1alpha1.MinIOJob) (intervalJob *miniojob.MinIOInterva } // check all dependon for _, command := range intervalJob.Command { - for _, dep := range command.DepnedsOn { + for _, dep := range command.CommandSpec.DependsOn { _, found := intervalJob.CommandMap[dep] if !found { return intervalJob, fmt.Errorf("dependent job %s not found", dep) diff --git a/pkg/utils/miniojob/minioJob.go b/pkg/utils/miniojob/minioJob.go index c5921d0f6e..70b2b56ff6 100644 --- a/pkg/utils/miniojob/minioJob.go +++ b/pkg/utils/miniojob/minioJob.go @@ -22,25 +22,9 @@ import ( "strings" ) -// ArgType - arg type -type ArgType int - -const ( - // ArgTypeKey - key=value print value - ArgTypeKey ArgType = iota - // ArgTypeFile - key=value print /temp/value.ext - ArgTypeFile - // ArgTypeKeyFile - key=value print key="/temp/value.ext" - ArgTypeKeyFile -) - // Arg - parse the arg result type Arg struct { - Command string - FileName string - FileExt string - FileContext string - ArgType ArgType + Command string } // FieldsFunc - alias function @@ -68,27 +52,6 @@ func Static(val string) FieldsFunc { } } -// File - fName is the the key, value is content, ext is the file ext -func File(fName string, ext string) FieldsFunc { - return func(args map[string]string) (out Arg, err error) { - if args == nil { - return out, fmt.Errorf("args is nil") - } - if val, ok := args[fName]; ok { - if val == "" { - return out, fmt.Errorf("value is empty") - } - out.FileName = fName - out.FileExt = ext - out.FileContext = strings.TrimSpace(val) - out.ArgType = ArgTypeFile - delete(args, fName) - return out, nil - } - return out, fmt.Errorf("file %s not found", fName) - } -} - // KeyValue - match key and putout the key, like endpoint="https://webhook-1.example.net" func KeyValue(key string) FieldsFunc { return func(args map[string]string) (out Arg, err error) { @@ -105,25 +68,6 @@ func KeyValue(key string) FieldsFunc { } } -// KeyFile - match key and putout the key, like client_cert="[here is content]" -func KeyFile(key string, ext string) FieldsFunc { - return func(args map[string]string) (out Arg, err error) { - if args == nil { - return out, fmt.Errorf("args is nil") - } - val, ok := args[key] - if !ok { - return out, fmt.Errorf("key %s not found", key) - } - out.FileName = key - out.FileExt = ext - out.FileContext = strings.TrimSpace(val) - out.ArgType = ArgTypeKeyFile - delete(args, key) - return out, nil - } -} - // Option - ignore the error func Option(opt FieldsFunc) FieldsFunc { return func(args map[string]string) (out Arg, err error) { diff --git a/pkg/utils/miniojob/minioJob_test.go b/pkg/utils/miniojob/minioJob_test.go index b6fb7a08c4..15977d407a 100644 --- a/pkg/utils/miniojob/minioJob_test.go +++ b/pkg/utils/miniojob/minioJob_test.go @@ -16,7 +16,11 @@ package miniojob -import "testing" +import ( + "testing" + + "github.com/minio/operator/pkg/apis/job.min.io/v1alpha1" +) func TestParser(t *testing.T) { args := map[string]string{ @@ -91,30 +95,6 @@ func TestParser(t *testing.T) { expect: Arg{Command: "test-static"}, expectError: false, }, - { - command: File("policy", "json"), - args: copyArgs(args), - expect: Arg{ - FileName: "policy", - FileExt: "json", - FileContext: `{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:*" - ], - "Resource": [ - "arn:aws:s3:::memes", - "arn:aws:s3:::memes/*" - ] - } - ] - }`, - }, - expectError: false, - }, { command: OneOf(KeyFormat("user", "--user"), KeyFormat("group", "--group")), args: copyArgs(args), @@ -152,59 +132,54 @@ func TestParser(t *testing.T) { if tc.expect.Command != "" && cmd.Command != tc.expect.Command { t.Fatalf("expectCommand %s, but got %s", tc.expect.Command, cmd.Command) } - if tc.expect.FileName != "" { - if tc.expect.FileContext != cmd.FileContext { - t.Fatalf("expectCommand %s, but got %s", tc.expect.FileContext, cmd.FileContext) - } - if tc.expect.FileExt != cmd.FileExt { - t.Fatalf("expectCommand %s, but got %s", tc.expect.FileExt, cmd.FileExt) - } - if tc.expect.FileName != cmd.FileName { - t.Fatalf("expectCommand %s, but got %s", tc.expect.FileName, cmd.FileName) - } - } } } } func TestAdminPolicyCreate(t *testing.T) { mcCommand := "admin/policy/create" - funcs := JobOperation[mcCommand] testCase := []struct { - name string - args map[string]string - expectError bool - expectCommand string - expectFileNumber int + name string + spec v1alpha1.CommandSpec + expectError bool + expectCommand string }{ { name: "testFull", - args: map[string]string{ - "name": "mypolicy", - "policy": "JsonContent", + spec: v1alpha1.CommandSpec{ + Operation: mcCommand, + Args: map[string]string{ + "name": "mypolicy", + "policy": "JsonContent", + }, }, - expectCommand: "myminio mypolicy /temp/policy.json", - expectFileNumber: 1, + expectCommand: "myminio mypolicy JsonContent", }, { name: "testError1", - args: map[string]string{ - "name": "mypolicy", + spec: v1alpha1.CommandSpec{ + Operation: mcCommand, + Args: map[string]string{ + "name": "mypolicy", + }, }, expectCommand: "", expectError: true, }, { name: "testError2", - args: map[string]string{ - "policy": "JsonContent", + spec: v1alpha1.CommandSpec{ + Operation: mcCommand, + Args: map[string]string{ + "policy": "JsonContent", + }, }, expectCommand: "", expectError: true, }, } for _, tc := range testCase { - command, err := GenerateMinIOIntervalJobCommand(mcCommand, 0, nil, "test", tc.args, funcs) + command, err := GenerateMinIOIntervalJobCommand(tc.spec, 0) if !tc.expectError { if err != nil { t.Fatal(err) @@ -222,94 +197,82 @@ func TestAdminPolicyCreate(t *testing.T) { func TestMCConfigSet(t *testing.T) { mcCommand := "admin/config/set" - funcs := JobOperation[mcCommand] testCase := []struct { - name string - args map[string]string - expectCommand string - expectError bool - expectFileNumber int + name string + spec v1alpha1.CommandSpec + expectCommand string + expectError bool }{ { name: "testFull", - args: map[string]string{ - "webhookName": "webhook1", - "endpoint": "endpoint1", - "auth_token": "token1", - "client_cert": "cert1", - "client_key": "key1", - }, - expectCommand: "myminio webhook1 endpoint=\"endpoint1\" client_key=\"/temp/client_key.key\" client_cert=\"/temp/client_cert.pem\" auth_token=\"token1\"", - expectFileNumber: 2, - }, - { - name: "testOptionFile", - args: map[string]string{ - "webhookName": "webhook1", - "endpoint": "endpoint1", - "auth_token": "token1", - "client_key": "key1", + spec: v1alpha1.CommandSpec{ + Operation: mcCommand, + Args: map[string]string{ + "webhookName": "webhook1", + "endpoint": "endpoint1", + "auth_token": "token1", + "client_cert": "cert1", + "client_key": "key1", + }, }, - expectCommand: "myminio webhook1 endpoint=\"endpoint1\" client_key=\"/temp/client_key.key\" auth_token=\"token1\"", - expectFileNumber: 1, - }, - { - name: "testOptionKeyValue", - args: map[string]string{ - "webhookName": "webhook1", - "endpoint": "endpoint1", - "client_key": "key1", - }, - expectCommand: "myminio webhook1 endpoint=\"endpoint1\" client_key=\"/temp/client_key.key\"", - expectFileNumber: 1, + expectCommand: "myminio webhook1 endpoint=\"endpoint1\" auth_token=\"token1\" client_cert=\"cert1\" client_key=\"key1\"", }, { name: "notify_mysql", - args: map[string]string{ - "webhookName": "notify_mysql", - "dsn_string": "username:password@tcp(mysql.example.com:3306)/miniodb", - "table": "minioevents", - "format": "namespace", + spec: v1alpha1.CommandSpec{ + Operation: mcCommand, + Args: map[string]string{ + "webhookName": "notify_mysql", + "dsn_string": "username:password@tcp(mysql.example.com:3306)/miniodb", + "table": "minioevents", + "format": "namespace", + }, }, - expectCommand: "myminio notify_mysql dsn_string=\"username:password@tcp(mysql.example.com:3306)/miniodb\" format=\"namespace\" table=\"minioevents\"", - expectFileNumber: 0, + expectCommand: "myminio notify_mysql dsn_string=\"username:password@tcp(mysql.example.com:3306)/miniodb\" format=\"namespace\" table=\"minioevents\"", }, { name: "notify_amqp", - args: map[string]string{ - "webhookName": "notify_amqp:primary", - "url": "user:password@amqp://amqp-endpoint.example.net:5672", + spec: v1alpha1.CommandSpec{ + Operation: mcCommand, + Args: map[string]string{ + "webhookName": "notify_amqp:primary", + "url": "user:password@amqp://amqp-endpoint.example.net:5672", + }, }, - expectCommand: "myminio notify_amqp:primary url=\"user:password@amqp://amqp-endpoint.example.net:5672\"", - expectFileNumber: 0, + expectCommand: "myminio notify_amqp:primary url=\"user:password@amqp://amqp-endpoint.example.net:5672\"", }, { name: "notify_elasticsearch", - args: map[string]string{ - "webhookName": "notify_elasticsearch:primary", - "url": "user:password@https://elasticsearch-endpoint.example.net:9200", - "index": "bucketevents", - "format": "namespace", + spec: v1alpha1.CommandSpec{ + Operation: mcCommand, + Args: map[string]string{ + "webhookName": "notify_elasticsearch:primary", + "url": "user:password@https://elasticsearch-endpoint.example.net:9200", + "index": "bucketevents", + "format": "namespace", + }, }, - expectCommand: "myminio notify_elasticsearch:primary format=\"namespace\" index=\"bucketevents\" url=\"user:password@https://elasticsearch-endpoint.example.net:9200\"", - expectFileNumber: 0, + expectCommand: "myminio notify_elasticsearch:primary format=\"namespace\" index=\"bucketevents\" url=\"user:password@https://elasticsearch-endpoint.example.net:9200\"", }, { name: "identity_ldap", - args: map[string]string{ - "webhookName": "identity_ldap", - "enabled": "true", - "server_addr": "ad-ldap.example.net/", - "lookup_bind_dn": "cn=miniolookupuser,dc=example,dc=net", - "lookup_bind_dn_password": "userpassword", - "user_dn_search_base_dn": "dc=example,dc=net", - "user_dn_search_filter": "(&(objectCategory=user)(sAMAccountName=%s))", + spec: v1alpha1.CommandSpec{ + Operation: mcCommand, + Args: map[string]string{ + "webhookName": "identity_ldap", + "enabled": "true", + "server_addr": "ad-ldap.example.net/", + "lookup_bind_dn": "cn=miniolookupuser,dc=example,dc=net", + "lookup_bind_dn_password": "userpassword", + "user_dn_search_base_dn": "dc=example,dc=net", + "user_dn_search_filter": "(&(objectCategory=user)(sAMAccountName=%s))", + }, }, expectCommand: "myminio identity_ldap enabled=\"true\" lookup_bind_dn=\"cn=miniolookupuser,dc=example,dc=net\" lookup_bind_dn_password=\"userpassword\" server_addr=\"ad-ldap.example.net/\" user_dn_search_base_dn=\"dc=example,dc=net\" user_dn_search_filter=\"(&(objectCategory=user)(sAMAccountName=%s))\"", }, } for _, tc := range testCase { - command, err := GenerateMinIOIntervalJobCommand(mcCommand, 0, nil, "test", tc.args, funcs) + command, err := GenerateMinIOIntervalJobCommand(tc.spec, 0) if !tc.expectError { if err != nil { t.Fatal(err) @@ -317,8 +280,67 @@ func TestMCConfigSet(t *testing.T) { if command.Command != tc.expectCommand { t.Fatalf("[%s] expectCommand %s, but got %s", tc.name, tc.expectCommand, command.Command) } - if tc.expectFileNumber != len(command.Files) { - t.Fatalf("[%s] expectFileNumber %d, but got %d", tc.name, tc.expectFileNumber, len(command.Files)) + } else { + if err == nil { + t.Fatalf("[%s] expectCommand error", tc.name) + } + } + } +} + +func TestSupportcallhome(t *testing.T) { + mcCommand := "support/callhome" + testCase := []struct { + name string + spec v1alpha1.CommandSpec + expectCommand string + expectError bool + }{ + { + name: "testEnable", + spec: v1alpha1.CommandSpec{ + Operation: mcCommand, + Args: map[string]string{ + "action": "enable", + "--logs": "", + "--diag": "", + }, + }, + expectCommand: "enable myminio --diag --logs", + }, + { + name: "testDisable", + spec: v1alpha1.CommandSpec{ + Operation: mcCommand, + Args: map[string]string{ + "action": "disable", + "--logs": "", + "--diag": "", + }, + }, + expectCommand: "disable myminio --diag --logs", + }, + { + name: "testNoAction", + spec: v1alpha1.CommandSpec{ + Operation: mcCommand, + Args: map[string]string{ + "--logs": "", + "--diag": "", + }, + }, + expectCommand: "", + expectError: true, + }, + } + for _, tc := range testCase { + command, err := GenerateMinIOIntervalJobCommand(tc.spec, 0) + if !tc.expectError { + if err != nil { + t.Fatal(err) + } + if command.Command != tc.expectCommand { + t.Fatalf("[%s] expectCommand %s, but got %s", tc.name, tc.expectCommand, command.Command) } } else { if err == nil { diff --git a/pkg/utils/miniojob/types.go b/pkg/utils/miniojob/types.go index 090e14eae4..95f1c0b16a 100644 --- a/pkg/utils/miniojob/types.go +++ b/pkg/utils/miniojob/types.go @@ -59,9 +59,11 @@ var operationAlias = map[string]string{ var JobOperation = map[string][]FieldsFunc{ "mb": {FLAGS(), Sanitize(ALIAS(), Static("/"), Key("name")), Static("--ignore-existing")}, "admin/user/add": {ALIAS(), Key("user"), Key("password")}, - "admin/policy/create": {ALIAS(), Key("name"), File("policy", "json")}, + "admin/policy/create": {ALIAS(), Key("name"), Key("policy")}, "admin/policy/attach": {ALIAS(), Key("policy"), OneOf(KeyFormat("user", "--user"), KeyFormat("group", "--group"))}, - "admin/config/set": {ALIAS(), Key("webhookName"), Option(KeyValue("endpoint")), Option(KeyFile("client_key", "key")), Option(KeyFile("client_cert", "pem")), OthersKeyValues()}, + "admin/config/set": {ALIAS(), Key("webhookName"), Option(KeyValue("endpoint")), OthersKeyValues()}, + "support/callhome": {Key("action"), ALIAS(), FLAGS()}, + "license/register": {ALIAS(), OthersKeyValues()}, } // OperationAliasToMC - convert operation to mc operation @@ -84,22 +86,13 @@ func OperationAliasToMC(operation string) (op string, found bool) { return "", false } -// MinIOIntervalJobCommandFile - Job run command need a file such as /temp/policy.json -type MinIOIntervalJobCommandFile struct { - Name string - Ext string - Dir string - Content string -} - // MinIOIntervalJobCommand - Job run command type MinIOIntervalJobCommand struct { mutex sync.RWMutex + CommandSpec v1alpha1.CommandSpec JobName string MCOperation string Command string - DepnedsOn []string - Files []MinIOIntervalJobCommandFile Succeeded bool Message string Created bool @@ -126,8 +119,8 @@ func (jobCommand *MinIOIntervalJobCommand) Success() bool { return jobCommand.Succeeded } -// CreateJob - create job -func (jobCommand *MinIOIntervalJobCommand) CreateJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob) error { +// createJob - create job +func (jobCommand *MinIOIntervalJobCommand) createJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob) (objs []client.Object) { if jobCommand == nil { return nil } @@ -148,11 +141,48 @@ func (jobCommand *MinIOIntervalJobCommand) CreateJob(ctx context.Context, k8sCli } } jobCommands = append(jobCommands, "--insecure") - objs := []client.Object{} mcImage := jobCR.Spec.MCImage if mcImage == "" { mcImage = DefaultMCImage } + baseVolumeMounts := []corev1.VolumeMount{ + { + Name: "config-dir", + MountPath: "/.mc", + }, + } + baseVolumeMounts = append(baseVolumeMounts, jobCommand.CommandSpec.VolumeMounts...) + baseVolumes := []corev1.Volume{ + { + Name: "config-dir", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } + baseVolumes = append(baseVolumes, jobCommand.CommandSpec.Volumes...) + baseEnvFrom := []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: fmt.Sprintf("%s-%s-cm", jobCR.Name, jobCommand.JobName), + }, + }, + }, + } + baseEnvFrom = append(baseEnvFrom, jobCommand.CommandSpec.EnvFrom...) + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("%s-%s-cm", jobCR.Name, jobCommand.JobName), + Namespace: jobCR.Namespace, + }, + Data: map[string]string{ + "MC_HOST_myminio": fmt.Sprintf("https://$(ACCESS_KEY):$(SECRET_KEY)@minio.%s.svc.cluster.local", jobCR.Namespace), + "MC_STS_ENDPOINT_myminio": fmt.Sprintf("https://sts.%s.svc.cluster.local:4223/sts/%s", miniov2.GetNSFromFile(), jobCR.Namespace), + "MC_WEB_IDENTITY_TOKEN_FILE_myminio": "/var/run/secrets/kubernetes.io/serviceaccount/token", + }, + } + objs = append(objs, cm) job := &batchjobv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", jobCR.Name, jobCommand.JobName), @@ -179,39 +209,16 @@ func (jobCommand *MinIOIntervalJobCommand) CreateJob(ctx context.Context, k8sCli Name: "mc", Image: mcImage, ImagePullPolicy: corev1.PullIfNotPresent, - Env: []corev1.EnvVar{ - { - Name: "MC_HOST_myminio", - Value: fmt.Sprintf("https://$(ACCESS_KEY):$(SECRET_KEY)@minio.%s.svc.cluster.local", jobCR.Namespace), - }, - { - Name: "MC_STS_ENDPOINT_myminio", - Value: fmt.Sprintf("https://sts.%s.svc.cluster.local:4223/sts/%s", miniov2.GetNSFromFile(), jobCR.Namespace), - }, - { - Name: "MC_WEB_IDENTITY_TOKEN_FILE_myminio", - Value: "/var/run/secrets/kubernetes.io/serviceaccount/token", - }, - }, + Env: jobCommand.CommandSpec.Env, + EnvFrom: baseEnvFrom, Command: jobCommands, SecurityContext: jobCR.Spec.ContainerSecurityContext, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "config-dir", - MountPath: "/.mc", - }, - }, + VolumeMounts: baseVolumeMounts, + Resources: jobCommand.CommandSpec.Resources, }, }, SecurityContext: jobCR.Spec.SecurityContext, - Volumes: []corev1.Volume{ - { - Name: "config-dir", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - }, + Volumes: baseVolumes, }, }, }, @@ -221,40 +228,13 @@ func (jobCommand *MinIOIntervalJobCommand) CreateJob(ctx context.Context, k8sCli } else { job.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyOnFailure } - if len(jobCommand.Files) > 0 { - cmName := fmt.Sprintf("%s-%s-cm", jobCR.Name, jobCommand.JobName) - job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ - Name: "file-volume", - ReadOnly: true, - MountPath: jobCommand.Files[0].Dir, - }) - job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{ - Name: "file-volume", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: cmName, - }, - }, - }, - }) - configMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: cmName, - Namespace: jobCR.Namespace, - Labels: map[string]string{ - "job.min.io/name": jobCR.Name, - }, - }, - Data: map[string]string{}, - } - for _, file := range jobCommand.Files { - configMap.Data[fmt.Sprintf("%s.%s", file.Name, file.Ext)] = file.Content - } - objs = append(objs, configMap) - } objs = append(objs, job) - for _, obj := range objs { + return objs +} + +// CreateJob - create job +func (jobCommand *MinIOIntervalJobCommand) CreateJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob) error { + for _, obj := range jobCommand.createJob(ctx, k8sClient, jobCR) { _, err := runtime.NewObjectSyncer(ctx, k8sClient, jobCR, func() error { return nil }, obj, runtime.SyncTypeCreateOrUpdate).Sync(ctx) @@ -327,14 +307,14 @@ func (intervalJob *MinIOIntervalJob) GetMinioJobStatus(ctx context.Context) v1al // CreateCommandJob - create command job func (intervalJob *MinIOIntervalJob) CreateCommandJob(ctx context.Context, k8sClient client.Client) error { for _, command := range intervalJob.Command { - if len(command.DepnedsOn) == 0 { + if len(command.CommandSpec.DependsOn) == 0 { err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR) if err != nil { return err } } else { allDepsSuccess := true - for _, dep := range command.DepnedsOn { + for _, dep := range command.CommandSpec.DependsOn { status, found := intervalJob.CommandMap[dep] if !found { return fmt.Errorf("dependent job %s not found", dep) @@ -356,44 +336,31 @@ func (intervalJob *MinIOIntervalJob) CreateCommandJob(ctx context.Context, k8sCl } // GenerateMinIOIntervalJobCommand - generate command -func GenerateMinIOIntervalJobCommand(mcCommand string, commandIndex int, dependsOn []string, jobName string, args map[string]string, argsFuncs []FieldsFunc) (*MinIOIntervalJobCommand, error) { +func GenerateMinIOIntervalJobCommand(commandSpec v1alpha1.CommandSpec, commandIndex int) (*MinIOIntervalJobCommand, error) { + mcCommand, found := OperationAliasToMC(commandSpec.Operation) + if !found { + return nil, fmt.Errorf("operation %s is not supported", commandSpec.Operation) + } + argsFuncs, found := JobOperation[mcCommand] + if !found { + return nil, fmt.Errorf("operation %s is not supported", mcCommand) + } commands := []string{} - files := []MinIOIntervalJobCommandFile{} for _, argsFunc := range argsFuncs { - jobArg, err := argsFunc(args) + jobArg, err := argsFunc(commandSpec.Args) if err != nil { return nil, err } - switch jobArg.ArgType { - case ArgTypeKey: - if jobArg.Command != "" { - commands = append(commands, jobArg.Command) - } - case ArgTypeFile: - files = append(files, MinIOIntervalJobCommandFile{ - Name: jobArg.FileName, - Ext: jobArg.FileExt, - Dir: CommandFilePath, - Content: jobArg.FileContext, - }) - commands = append(commands, fmt.Sprintf("%s/%s.%s", CommandFilePath, jobArg.FileName, jobArg.FileExt)) - case ArgTypeKeyFile: - files = append(files, MinIOIntervalJobCommandFile{ - Name: jobArg.FileName, - Ext: jobArg.FileExt, - Dir: CommandFilePath, - Content: jobArg.FileContext, - }) - commands = append(commands, fmt.Sprintf(`%s="%s/%s.%s"`, jobArg.FileName, CommandFilePath, jobArg.FileName, jobArg.FileExt)) + if jobArg.Command != "" { + commands = append(commands, jobArg.Command) } } jobCommand := &MinIOIntervalJobCommand{ - JobName: jobName, + JobName: commandSpec.Name, MCOperation: mcCommand, Command: strings.Join(commands, " "), - DepnedsOn: dependsOn, - Files: files, + CommandSpec: commandSpec, } // some commands need to have a empty name if jobCommand.JobName == "" { diff --git a/resources/base/crds/job.min.io_miniojobs.yaml b/resources/base/crds/job.min.io_miniojobs.yaml index 08bc0d9d24..8ffc3bdc0e 100644 --- a/resources/base/crds/job.min.io_miniojobs.yaml +++ b/resources/base/crds/job.min.io_miniojobs.yaml @@ -47,10 +47,903 @@ spec: items: type: string type: array + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + envFrom: + items: + properties: + configMapRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array name: type: string op: type: string + resources: + properties: + claims: + items: + properties: + name: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + volumes: + items: + properties: + awsElasticBlockStore: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + azureDisk: + properties: + cachingMode: + type: string + diskName: + type: string + diskURI: + type: string + fsType: + type: string + kind: + type: string + readOnly: + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + properties: + readOnly: + type: boolean + secretName: + type: string + shareName: + type: string + required: + - secretName + - shareName + type: object + cephfs: + properties: + monitors: + items: + type: string + type: array + path: + type: string + readOnly: + type: boolean + secretFile: + type: string + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - monitors + type: object + cinder: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + volumeID: + type: string + required: + - volumeID + type: object + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + csi: + properties: + driver: + type: string + fsType: + type: string + nodePublishSecretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + readOnly: + type: boolean + volumeAttributes: + additionalProperties: + type: string + type: object + required: + - driver + type: object + downwardAPI: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + emptyDir: + properties: + medium: + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + ephemeral: + properties: + volumeClaimTemplate: + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + properties: + accessModes: + items: + type: string + type: array + dataSource: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + properties: + apiGroup: + type: string + kind: + type: string + name: + type: string + namespace: + type: string + required: + - kind + - name + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + type: string + volumeAttributesClassName: + type: string + volumeMode: + type: string + volumeName: + type: string + type: object + required: + - spec + type: object + type: object + fc: + properties: + fsType: + type: string + lun: + format: int32 + type: integer + readOnly: + type: boolean + targetWWNs: + items: + type: string + type: array + wwids: + items: + type: string + type: array + type: object + flexVolume: + properties: + driver: + type: string + fsType: + type: string + options: + additionalProperties: + type: string + type: object + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + required: + - driver + type: object + flocker: + properties: + datasetName: + type: string + datasetUUID: + type: string + type: object + gcePersistentDisk: + properties: + fsType: + type: string + partition: + format: int32 + type: integer + pdName: + type: string + readOnly: + type: boolean + required: + - pdName + type: object + gitRepo: + properties: + directory: + type: string + repository: + type: string + revision: + type: string + required: + - repository + type: object + glusterfs: + properties: + endpoints: + type: string + path: + type: string + readOnly: + type: boolean + required: + - endpoints + - path + type: object + hostPath: + properties: + path: + type: string + type: + type: string + required: + - path + type: object + iscsi: + properties: + chapAuthDiscovery: + type: boolean + chapAuthSession: + type: boolean + fsType: + type: string + initiatorName: + type: string + iqn: + type: string + iscsiInterface: + type: string + lun: + format: int32 + type: integer + portals: + items: + type: string + type: array + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + targetPortal: + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + type: string + nfs: + properties: + path: + type: string + readOnly: + type: boolean + server: + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + properties: + claimName: + type: string + readOnly: + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + properties: + fsType: + type: string + pdID: + type: string + required: + - pdID + type: object + portworxVolume: + properties: + fsType: + type: string + readOnly: + type: boolean + volumeID: + type: string + required: + - volumeID + type: object + projected: + properties: + defaultMode: + format: int32 + type: integer + sources: + items: + properties: + clusterTrustBundle: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + name: + type: string + optional: + type: boolean + path: + type: string + signerName: + type: string + required: + - path + type: object + configMap: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + downwardAPI: + properties: + items: + items: + properties: + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + mode: + format: int32 + type: integer + path: + type: string + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + required: + - path + type: object + type: array + type: object + secret: + properties: + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + serviceAccountToken: + properties: + audience: + type: string + expirationSeconds: + format: int64 + type: integer + path: + type: string + required: + - path + type: object + type: object + type: array + type: object + quobyte: + properties: + group: + type: string + readOnly: + type: boolean + registry: + type: string + tenant: + type: string + user: + type: string + volume: + type: string + required: + - registry + - volume + type: object + rbd: + properties: + fsType: + type: string + image: + type: string + keyring: + type: string + monitors: + items: + type: string + type: array + pool: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + user: + type: string + required: + - image + - monitors + type: object + scaleIO: + properties: + fsType: + type: string + gateway: + type: string + protectionDomain: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + sslEnabled: + type: boolean + storageMode: + type: string + storagePool: + type: string + system: + type: string + volumeName: + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + storageos: + properties: + fsType: + type: string + readOnly: + type: boolean + secretRef: + properties: + name: + type: string + type: object + x-kubernetes-map-type: atomic + volumeName: + type: string + volumeNamespace: + type: string + type: object + vsphereVolume: + properties: + fsType: + type: string + storagePolicyID: + type: string + storagePolicyName: + type: string + volumePath: + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array required: - op type: object From b5f4e904f70183d2158e0d80fdfff09217274cad Mon Sep 17 00:00:00 2001 From: jiuker <2818723467@qq.com> Date: Fri, 17 May 2024 15:53:02 +0800 Subject: [PATCH 2/7] rename rename --- pkg/utils/miniojob/types.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/utils/miniojob/types.go b/pkg/utils/miniojob/types.go index 95f1c0b16a..4331639e3a 100644 --- a/pkg/utils/miniojob/types.go +++ b/pkg/utils/miniojob/types.go @@ -163,26 +163,26 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(ctx context.Context, k8sCli baseVolumes = append(baseVolumes, jobCommand.CommandSpec.Volumes...) baseEnvFrom := []corev1.EnvFromSource{ { - ConfigMapRef: &corev1.ConfigMapEnvSource{ + SecretRef: &corev1.SecretEnvSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: fmt.Sprintf("%s-%s-cm", jobCR.Name, jobCommand.JobName), + Name: fmt.Sprintf("%s-job-secret", jobCR.Name), }, }, }, } baseEnvFrom = append(baseEnvFrom, jobCommand.CommandSpec.EnvFrom...) - cm := &corev1.ConfigMap{ + secret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s-%s-cm", jobCR.Name, jobCommand.JobName), + Name: fmt.Sprintf("%s-job-secret", jobCR.Name), Namespace: jobCR.Namespace, }, - Data: map[string]string{ + StringData: map[string]string{ "MC_HOST_myminio": fmt.Sprintf("https://$(ACCESS_KEY):$(SECRET_KEY)@minio.%s.svc.cluster.local", jobCR.Namespace), "MC_STS_ENDPOINT_myminio": fmt.Sprintf("https://sts.%s.svc.cluster.local:4223/sts/%s", miniov2.GetNSFromFile(), jobCR.Namespace), "MC_WEB_IDENTITY_TOKEN_FILE_myminio": "/var/run/secrets/kubernetes.io/serviceaccount/token", }, } - objs = append(objs, cm) + objs = append(objs, secret) job := &batchjobv1.Job{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", jobCR.Name, jobCommand.JobName), From 6d5579523ecf92b847d38d399978d386bb7ce155 Mon Sep 17 00:00:00 2001 From: jiuker <2818723467@qq.com> Date: Tue, 21 May 2024 10:13:11 +0800 Subject: [PATCH 3/7] support port support port --- pkg/controller/job-controller.go | 2 +- pkg/utils/miniojob/types.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/controller/job-controller.go b/pkg/controller/job-controller.go index dd8d0b07f9..633495b479 100644 --- a/pkg/controller/job-controller.go +++ b/pkg/controller/job-controller.go @@ -258,7 +258,7 @@ func (c *JobController) SyncHandler(key string) (Result, error) { if err != nil { return WrapResult(Result{}, err) } - err = intervalJob.CreateCommandJob(ctx, c.k8sClient) + err = intervalJob.CreateCommandJob(ctx, c.k8sClient, STSDefaultPort) if err != nil { jobCR.Status.Phase = miniojob.MinioJobPhaseError jobCR.Status.Message = fmt.Sprintf("Create job error:%v", err) diff --git a/pkg/utils/miniojob/types.go b/pkg/utils/miniojob/types.go index 4331639e3a..40cca2d71e 100644 --- a/pkg/utils/miniojob/types.go +++ b/pkg/utils/miniojob/types.go @@ -120,7 +120,7 @@ func (jobCommand *MinIOIntervalJobCommand) Success() bool { } // createJob - create job -func (jobCommand *MinIOIntervalJobCommand) createJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob) (objs []client.Object) { +func (jobCommand *MinIOIntervalJobCommand) createJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob, port string) (objs []client.Object) { if jobCommand == nil { return nil } @@ -178,7 +178,7 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(ctx context.Context, k8sCli }, StringData: map[string]string{ "MC_HOST_myminio": fmt.Sprintf("https://$(ACCESS_KEY):$(SECRET_KEY)@minio.%s.svc.cluster.local", jobCR.Namespace), - "MC_STS_ENDPOINT_myminio": fmt.Sprintf("https://sts.%s.svc.cluster.local:4223/sts/%s", miniov2.GetNSFromFile(), jobCR.Namespace), + "MC_STS_ENDPOINT_myminio": fmt.Sprintf("https://sts.%s.svc.cluster.local:%s/sts/%s", miniov2.GetNSFromFile(), jobCR.Namespace, port), "MC_WEB_IDENTITY_TOKEN_FILE_myminio": "/var/run/secrets/kubernetes.io/serviceaccount/token", }, } @@ -233,8 +233,8 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(ctx context.Context, k8sCli } // CreateJob - create job -func (jobCommand *MinIOIntervalJobCommand) CreateJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob) error { - for _, obj := range jobCommand.createJob(ctx, k8sClient, jobCR) { +func (jobCommand *MinIOIntervalJobCommand) CreateJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob, port string) error { + for _, obj := range jobCommand.createJob(ctx, k8sClient, jobCR, port) { _, err := runtime.NewObjectSyncer(ctx, k8sClient, jobCR, func() error { return nil }, obj, runtime.SyncTypeCreateOrUpdate).Sync(ctx) @@ -305,10 +305,10 @@ func (intervalJob *MinIOIntervalJob) GetMinioJobStatus(ctx context.Context) v1al } // CreateCommandJob - create command job -func (intervalJob *MinIOIntervalJob) CreateCommandJob(ctx context.Context, k8sClient client.Client) error { +func (intervalJob *MinIOIntervalJob) CreateCommandJob(ctx context.Context, k8sClient client.Client, port string) error { for _, command := range intervalJob.Command { if len(command.CommandSpec.DependsOn) == 0 { - err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR) + err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR, port) if err != nil { return err } @@ -325,7 +325,7 @@ func (intervalJob *MinIOIntervalJob) CreateCommandJob(ctx context.Context, k8sCl } } if allDepsSuccess { - err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR) + err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR, port) if err != nil { return err } From 1ab6412f7173b0749e5edb10e6f87f6e0a8f368c Mon Sep 17 00:00:00 2001 From: jiuker <2818723467@qq.com> Date: Tue, 21 May 2024 10:14:40 +0800 Subject: [PATCH 4/7] rename rename --- pkg/utils/miniojob/types.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/utils/miniojob/types.go b/pkg/utils/miniojob/types.go index 40cca2d71e..6fdecfd68e 100644 --- a/pkg/utils/miniojob/types.go +++ b/pkg/utils/miniojob/types.go @@ -120,7 +120,7 @@ func (jobCommand *MinIOIntervalJobCommand) Success() bool { } // createJob - create job -func (jobCommand *MinIOIntervalJobCommand) createJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob, port string) (objs []client.Object) { +func (jobCommand *MinIOIntervalJobCommand) createJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob, stsPort string) (objs []client.Object) { if jobCommand == nil { return nil } @@ -178,7 +178,7 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(ctx context.Context, k8sCli }, StringData: map[string]string{ "MC_HOST_myminio": fmt.Sprintf("https://$(ACCESS_KEY):$(SECRET_KEY)@minio.%s.svc.cluster.local", jobCR.Namespace), - "MC_STS_ENDPOINT_myminio": fmt.Sprintf("https://sts.%s.svc.cluster.local:%s/sts/%s", miniov2.GetNSFromFile(), jobCR.Namespace, port), + "MC_STS_ENDPOINT_myminio": fmt.Sprintf("https://sts.%s.svc.cluster.local:%s/sts/%s", miniov2.GetNSFromFile(), jobCR.Namespace, stsPort), "MC_WEB_IDENTITY_TOKEN_FILE_myminio": "/var/run/secrets/kubernetes.io/serviceaccount/token", }, } @@ -233,8 +233,8 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(ctx context.Context, k8sCli } // CreateJob - create job -func (jobCommand *MinIOIntervalJobCommand) CreateJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob, port string) error { - for _, obj := range jobCommand.createJob(ctx, k8sClient, jobCR, port) { +func (jobCommand *MinIOIntervalJobCommand) CreateJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob, stsPort string) error { + for _, obj := range jobCommand.createJob(ctx, k8sClient, jobCR, stsPort) { _, err := runtime.NewObjectSyncer(ctx, k8sClient, jobCR, func() error { return nil }, obj, runtime.SyncTypeCreateOrUpdate).Sync(ctx) @@ -305,10 +305,10 @@ func (intervalJob *MinIOIntervalJob) GetMinioJobStatus(ctx context.Context) v1al } // CreateCommandJob - create command job -func (intervalJob *MinIOIntervalJob) CreateCommandJob(ctx context.Context, k8sClient client.Client, port string) error { +func (intervalJob *MinIOIntervalJob) CreateCommandJob(ctx context.Context, k8sClient client.Client, stsPort string) error { for _, command := range intervalJob.Command { if len(command.CommandSpec.DependsOn) == 0 { - err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR, port) + err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR, stsPort) if err != nil { return err } @@ -325,7 +325,7 @@ func (intervalJob *MinIOIntervalJob) CreateCommandJob(ctx context.Context, k8sCl } } if allDepsSuccess { - err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR, port) + err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR, stsPort) if err != nil { return err } From 2c6243ff524dc515f579b1586f2148479d6a95f1 Mon Sep 17 00:00:00 2001 From: jiuker <2818723467@qq.com> Date: Tue, 21 May 2024 10:17:36 +0800 Subject: [PATCH 5/7] ignore nil object ignore nil object --- pkg/utils/miniojob/types.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/utils/miniojob/types.go b/pkg/utils/miniojob/types.go index 6fdecfd68e..d264830e70 100644 --- a/pkg/utils/miniojob/types.go +++ b/pkg/utils/miniojob/types.go @@ -235,6 +235,9 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(ctx context.Context, k8sCli // CreateJob - create job func (jobCommand *MinIOIntervalJobCommand) CreateJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob, stsPort string) error { for _, obj := range jobCommand.createJob(ctx, k8sClient, jobCR, stsPort) { + if obj == nil { + continue + } _, err := runtime.NewObjectSyncer(ctx, k8sClient, jobCR, func() error { return nil }, obj, runtime.SyncTypeCreateOrUpdate).Sync(ctx) From bb4e2e18800ec155fbc658ecd3f5731117e509cb Mon Sep 17 00:00:00 2001 From: jiuker <2818723467@qq.com> Date: Tue, 21 May 2024 14:44:09 +0800 Subject: [PATCH 6/7] change order --- pkg/utils/miniojob/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/utils/miniojob/types.go b/pkg/utils/miniojob/types.go index d264830e70..03bf63c1ca 100644 --- a/pkg/utils/miniojob/types.go +++ b/pkg/utils/miniojob/types.go @@ -178,7 +178,7 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(ctx context.Context, k8sCli }, StringData: map[string]string{ "MC_HOST_myminio": fmt.Sprintf("https://$(ACCESS_KEY):$(SECRET_KEY)@minio.%s.svc.cluster.local", jobCR.Namespace), - "MC_STS_ENDPOINT_myminio": fmt.Sprintf("https://sts.%s.svc.cluster.local:%s/sts/%s", miniov2.GetNSFromFile(), jobCR.Namespace, stsPort), + "MC_STS_ENDPOINT_myminio": fmt.Sprintf("https://sts.%s.svc.cluster.local:%s/sts/%s", miniov2.GetNSFromFile(), stsPort, jobCR.Namespace), "MC_WEB_IDENTITY_TOKEN_FILE_myminio": "/var/run/secrets/kubernetes.io/serviceaccount/token", }, } From 322950daaca3b4981a197aa2975ba85e602c81d7 Mon Sep 17 00:00:00 2001 From: jiuker <2818723467@qq.com> Date: Wed, 22 May 2024 10:18:52 +0800 Subject: [PATCH 7/7] change type change type --- pkg/controller/sts.go | 7 ++++--- pkg/utils/miniojob/types.go | 8 ++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pkg/controller/sts.go b/pkg/controller/sts.go index 72bb70bc6d..5bf3beab17 100644 --- a/pkg/controller/sts.go +++ b/pkg/controller/sts.go @@ -4,6 +4,7 @@ import ( "context" "encoding/xml" "errors" + "fmt" "net/http" "os" "sync" @@ -34,8 +35,8 @@ const ( // STS API constants const ( - STSDefaultPort = "4223" - STSEndpoint = "/sts" + STSDefaultPort int = 4223 + STSEndpoint = "/sts" ) const ( @@ -263,7 +264,7 @@ func configureSTSServer(c *Controller) *http.Server { router.NotFoundHandler = http.NotFoundHandler() s := &http.Server{ - Addr: ":" + STSDefaultPort, + Addr: fmt.Sprintf(":%d", STSDefaultPort), Handler: router, ReadTimeout: time.Minute, WriteTimeout: time.Minute, diff --git a/pkg/utils/miniojob/types.go b/pkg/utils/miniojob/types.go index 03bf63c1ca..3eb7fd7346 100644 --- a/pkg/utils/miniojob/types.go +++ b/pkg/utils/miniojob/types.go @@ -120,7 +120,7 @@ func (jobCommand *MinIOIntervalJobCommand) Success() bool { } // createJob - create job -func (jobCommand *MinIOIntervalJobCommand) createJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob, stsPort string) (objs []client.Object) { +func (jobCommand *MinIOIntervalJobCommand) createJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob, stsPort int) (objs []client.Object) { if jobCommand == nil { return nil } @@ -178,7 +178,7 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(ctx context.Context, k8sCli }, StringData: map[string]string{ "MC_HOST_myminio": fmt.Sprintf("https://$(ACCESS_KEY):$(SECRET_KEY)@minio.%s.svc.cluster.local", jobCR.Namespace), - "MC_STS_ENDPOINT_myminio": fmt.Sprintf("https://sts.%s.svc.cluster.local:%s/sts/%s", miniov2.GetNSFromFile(), stsPort, jobCR.Namespace), + "MC_STS_ENDPOINT_myminio": fmt.Sprintf("https://sts.%s.svc.cluster.local:%d/sts/%s", miniov2.GetNSFromFile(), stsPort, jobCR.Namespace), "MC_WEB_IDENTITY_TOKEN_FILE_myminio": "/var/run/secrets/kubernetes.io/serviceaccount/token", }, } @@ -233,7 +233,7 @@ func (jobCommand *MinIOIntervalJobCommand) createJob(ctx context.Context, k8sCli } // CreateJob - create job -func (jobCommand *MinIOIntervalJobCommand) CreateJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob, stsPort string) error { +func (jobCommand *MinIOIntervalJobCommand) CreateJob(ctx context.Context, k8sClient client.Client, jobCR *v1alpha1.MinIOJob, stsPort int) error { for _, obj := range jobCommand.createJob(ctx, k8sClient, jobCR, stsPort) { if obj == nil { continue @@ -308,7 +308,7 @@ func (intervalJob *MinIOIntervalJob) GetMinioJobStatus(ctx context.Context) v1al } // CreateCommandJob - create command job -func (intervalJob *MinIOIntervalJob) CreateCommandJob(ctx context.Context, k8sClient client.Client, stsPort string) error { +func (intervalJob *MinIOIntervalJob) CreateCommandJob(ctx context.Context, k8sClient client.Client, stsPort int) error { for _, command := range intervalJob.Command { if len(command.CommandSpec.DependsOn) == 0 { err := command.CreateJob(ctx, k8sClient, intervalJob.JobCR, stsPort)