diff --git a/go.mod b/go.mod index 80cc14fa62..6335db4ff0 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( k8s.io/kube-scheduler v0.30.0-alpha.2 k8s.io/kubectl v0.30.0-alpha.2 k8s.io/kubelet v0.30.0-alpha.2 + k8s.io/pod-security-admission v0.30.0-alpha.2 ) require ( diff --git a/go.sum b/go.sum index a1b231e6ad..0351959dd9 100644 --- a/go.sum +++ b/go.sum @@ -1271,6 +1271,8 @@ k8s.io/kubectl v0.30.0-alpha.2 h1:fw+2Ijv4gqQdFgzYK1nJJR3MFopCdBAZEjnETcM+y4Y= k8s.io/kubectl v0.30.0-alpha.2/go.mod h1:74X1grqoxhb93ZLxjQo8FurqpWdSAgnNYiUhyYYiWoA= k8s.io/kubelet v0.30.0-alpha.2 h1:35RaAFKiBA3iBGg8fxFefDdKrBhcakTpbPxZdHMRml8= k8s.io/kubelet v0.30.0-alpha.2/go.mod h1:wP9oBhAREIMKYXiV7xK92P9OfeMi7TFHGm+EbYtn5ZQ= +k8s.io/pod-security-admission v0.30.0-alpha.2 h1:q2gKZJxHk4Uf0SBxnFLu34ZbbwW7Peml903Tw8jC7tA= +k8s.io/pod-security-admission v0.30.0-alpha.2/go.mod h1:v+SIoDPBLpZRM4yG0+rZRvbNXHYfPmpN1PcdZaOzZe8= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kernel.org/pub/linux/libs/security/libcap/cap v1.2.69 h1:N0m3tKYbkRMmDobh/47ngz+AWeV7PcfXMDi8xu3Vrag= diff --git a/hack/test/e2e.sh b/hack/test/e2e.sh index e83e002953..4cb3c3c1a6 100755 --- a/hack/test/e2e.sh +++ b/hack/test/e2e.sh @@ -150,7 +150,7 @@ function run_talos_integration_test { ;; esac - "${INTEGRATION_TEST}" -test.v -talos.failfast -talos.talosctlpath "${TALOSCTL}" -talos.kubectlpath "${KUBECTL}" -talos.provisioner "${PROVISIONER}" -talos.name "${CLUSTER_NAME}" "${EXTRA_TEST_ARGS[@]}" "${TEST_RUN[@]}" "${TEST_SHORT[@]}" + "${INTEGRATION_TEST}" -test.v -talos.failfast -talos.talosctlpath "${TALOSCTL}" -talos.kubectlpath "${KUBECTL}" -talos.provisioner "${PROVISIONER}" -talos.name "${CLUSTER_NAME}" -talos.image "${REGISTRY}/siderolabs/talos" "${EXTRA_TEST_ARGS[@]}" "${TEST_RUN[@]}" "${TEST_SHORT[@]}" } function run_talos_integration_test_docker { @@ -170,7 +170,7 @@ function run_talos_integration_test_docker { ;; esac - "${INTEGRATION_TEST}" -test.v -talos.talosctlpath "${TALOSCTL}" -talos.kubectlpath "${KUBECTL}" -talos.k8sendpoint 127.0.0.1:6443 -talos.provisioner "${PROVISIONER}" -talos.name "${CLUSTER_NAME}" "${EXTRA_TEST_ARGS[@]}" "${TEST_RUN[@]}" "${TEST_SHORT[@]}" + "${INTEGRATION_TEST}" -test.v -talos.talosctlpath "${TALOSCTL}" -talos.kubectlpath "${KUBECTL}" -talos.k8sendpoint 127.0.0.1:6443 -talos.provisioner "${PROVISIONER}" -talos.name "${CLUSTER_NAME}" -talos.image "${REGISTRY}/siderolabs/talos" "${EXTRA_TEST_ARGS[@]}" "${TEST_RUN[@]}" "${TEST_SHORT[@]}" } function run_kubernetes_conformance_test { diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_spec.go b/internal/app/machined/pkg/controllers/k8s/kubelet_spec.go index 807e08fbb9..ddb52d6b7e 100644 --- a/internal/app/machined/pkg/controllers/k8s/kubelet_spec.go +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_spec.go @@ -25,6 +25,7 @@ import ( kubeletconfig "k8s.io/kubelet/config/v1beta1" v1alpha1runtime "github.com/siderolabs/talos/internal/app/machined/pkg/runtime" + "github.com/siderolabs/talos/internal/pkg/cgroup" "github.com/siderolabs/talos/pkg/argsbuilder" "github.com/siderolabs/talos/pkg/machinery/constants" "github.com/siderolabs/talos/pkg/machinery/kubelet" @@ -275,9 +276,9 @@ func NewKubeletConfiguration(cfgSpec *k8s.KubeletConfigSpec, kubeletVersion comp config.Authorization = kubeletconfig.KubeletAuthorization{ Mode: kubeletconfig.KubeletAuthorizationModeWebhook, } - config.CgroupRoot = "/" - config.SystemCgroups = constants.CgroupSystem - config.KubeletCgroups = constants.CgroupKubelet + config.CgroupRoot = cgroup.Root() + config.SystemCgroups = cgroup.Path(constants.CgroupSystem) + config.KubeletCgroups = cgroup.Path(constants.CgroupKubelet) config.RotateCertificates = true config.ProtectKernelDefaults = true diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go index a5085f4ee0..46082967d9 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go @@ -245,9 +245,6 @@ func (*Sequencer) Boot(r runtime.Runtime) []runtime.Phase { r.State().Platform().Mode() != runtime.ModeContainer, "overlay", MountOverlayFilesystems, - ).Append( - "legacyCleanup", - CleanupLegacyStaticPodFiles, ).AppendWhen( r.State().Platform().Mode() != runtime.ModeContainer, "udevSetup", diff --git a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go index 6796510e9f..0970b874d7 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go +++ b/internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer_tasks.go @@ -53,6 +53,7 @@ import ( "github.com/siderolabs/talos/internal/app/machined/pkg/system" "github.com/siderolabs/talos/internal/app/machined/pkg/system/events" "github.com/siderolabs/talos/internal/app/machined/pkg/system/services" + "github.com/siderolabs/talos/internal/pkg/cgroup" "github.com/siderolabs/talos/internal/pkg/cri" "github.com/siderolabs/talos/internal/pkg/environment" "github.com/siderolabs/talos/internal/pkg/etcd" @@ -164,6 +165,11 @@ func CreateSystemCgroups(runtime.Sequence, any) (runtime.TaskExecutionFunc, stri } } + // Initialize cgroups root path. + if err = cgroup.InitRoot(); err != nil { + return fmt.Errorf("error initializing cgroups root path: %w", err) + } + groups := []struct { name string resources *cgroup2.Resources @@ -190,6 +196,10 @@ func CreateSystemCgroups(runtime.Sequence, any) (runtime.TaskExecutionFunc, stri name: constants.CgroupSystemRuntime, resources: &cgroup2.Resources{}, }, + { + name: constants.CgroupUdevd, + resources: &cgroup2.Resources{}, + }, { name: constants.CgroupPodRuntime, resources: &cgroup2.Resources{ @@ -228,7 +238,7 @@ func CreateSystemCgroups(runtime.Sequence, any) (runtime.TaskExecutionFunc, stri resources = &cgroup2.Resources{} } - cg, err := cgroup2.NewManager(constants.CgroupMountPath, c.name, resources) + cg, err := cgroup2.NewManager(constants.CgroupMountPath, cgroup.Path(c.name), resources) if err != nil { return fmt.Errorf("failed to create cgroup: %w", err) } @@ -2200,42 +2210,6 @@ func ForceCleanup(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { }, "forceCleanup" } -// CleanupLegacyStaticPodFiles removes legacy static pod files in the manifests directory. -// -// This part of transition to Talos 1.3.0, as Talos 1.3.0 serves static pods from internal web server. -func CleanupLegacyStaticPodFiles(runtime.Sequence, any) (runtime.TaskExecutionFunc, string) { - return func(ctx context.Context, logger *log.Logger, r runtime.Runtime) error { - manifestDir, err := os.Open(constants.ManifestsDirectory) - if err != nil { - return fmt.Errorf("error opening manifests directory: %w", err) - } - - defer manifestDir.Close() //nolint:errcheck - - manifests, err := manifestDir.Readdirnames(0) - if err != nil { - return fmt.Errorf("error listing manifests: %w", err) - } - - for _, manifest := range manifests { - // skip manifests not owned by Talos - if !strings.HasPrefix(manifest, constants.TalosManifestPrefix) { - continue - } - - podPath := filepath.Join(constants.ManifestsDirectory, manifest) - - logger.Printf("cleaning up legacy static pod file %q", podPath) - - if err = os.Remove(podPath); err != nil { - return fmt.Errorf("error cleaning up legacy static pod file: %w", err) - } - } - - return nil - }, "cleanupLegacyStaticPodFiles" -} - // ReloadMeta reloads META partition after disk mount, installer run, etc. // //nolint:gocyclo diff --git a/internal/app/machined/pkg/system/runner/containerd/containerd.go b/internal/app/machined/pkg/system/runner/containerd/containerd.go index 8a6ee0b033..5b0835dc14 100644 --- a/internal/app/machined/pkg/system/runner/containerd/containerd.go +++ b/internal/app/machined/pkg/system/runner/containerd/containerd.go @@ -22,6 +22,7 @@ import ( "github.com/siderolabs/talos/internal/app/machined/pkg/system/events" "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner" + "github.com/siderolabs/talos/internal/pkg/cgroup" ) // containerdRunner is a runner.Runner that runs container in containerd. @@ -319,7 +320,7 @@ func (c *containerdRunner) newOCISpecOpts(image oci.Image) []oci.SpecOpts { if c.opts.CgroupPath != "" { specOpts = append( specOpts, - oci.WithCgroup(c.opts.CgroupPath), + oci.WithCgroup(cgroup.Path(c.opts.CgroupPath)), ) } diff --git a/internal/app/machined/pkg/system/runner/process/process.go b/internal/app/machined/pkg/system/runner/process/process.go index 1957b15d43..47fc3bfea6 100644 --- a/internal/app/machined/pkg/system/runner/process/process.go +++ b/internal/app/machined/pkg/system/runner/process/process.go @@ -17,6 +17,7 @@ import ( "github.com/siderolabs/talos/internal/app/machined/pkg/system/events" "github.com/siderolabs/talos/internal/app/machined/pkg/system/runner" + "github.com/siderolabs/talos/internal/pkg/cgroup" "github.com/siderolabs/talos/pkg/machinery/constants" ) @@ -87,7 +88,7 @@ func (p *processRunner) build() (commandWrapper, error) { args := []string{ fmt.Sprintf("-name=%s", p.args.ID), fmt.Sprintf("-dropped-caps=%s", strings.Join(p.opts.DroppedCapabilities, ",")), - fmt.Sprintf("-cgroup-path=%s", p.opts.CgroupPath), + fmt.Sprintf("-cgroup-path=%s", cgroup.Path(p.opts.CgroupPath)), fmt.Sprintf("-oom-score=%d", p.opts.OOMScoreAdj), fmt.Sprintf("-uid=%d", p.opts.UID), } diff --git a/internal/app/machined/pkg/system/services/udevd.go b/internal/app/machined/pkg/system/services/udevd.go index dbc80c34d6..e6cabea4bc 100644 --- a/internal/app/machined/pkg/system/services/udevd.go +++ b/internal/app/machined/pkg/system/services/udevd.go @@ -83,7 +83,7 @@ func (c *Udevd) Runner(r runtime.Runtime) (runner.Runner, error) { debug, args, runner.WithLoggingManager(r.Logging()), - runner.WithCgroupPath(constants.CgroupSystemRuntime), + runner.WithCgroupPath(constants.CgroupUdevd), runner.WithDroppedCapabilities(constants.UdevdDroppedCapabilities), ), restart.WithType(restart.Forever), diff --git a/internal/integration/base/base.go b/internal/integration/base/base.go index 93c7d23169..de73df4b91 100644 --- a/internal/integration/base/base.go +++ b/internal/integration/base/base.go @@ -39,6 +39,8 @@ type TalosSuite struct { ExtensionsNvidia bool // TrustedBoot tells if the cluster is secure booted and disks are encrypted TrustedBoot bool + // TalosImage is the image name for 'talos' container. + TalosImage string discoveredNodes cluster.Info } diff --git a/internal/integration/base/k8s.go b/internal/integration/base/k8s.go index 3405b4d800..c123673ca0 100644 --- a/internal/integration/base/k8s.go +++ b/internal/integration/base/k8s.go @@ -7,9 +7,13 @@ package base import ( + "bufio" "bytes" "context" + "encoding/json" "fmt" + "io" + "slices" "time" "github.com/siderolabs/gen/xslices" @@ -18,14 +22,23 @@ import ( eventsv1 "k8s.io/api/events/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/yaml" + "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" + "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "k8s.io/client-go/tools/remotecommand" + watchtools "k8s.io/client-go/tools/watch" "k8s.io/kubectl/pkg/scheme" taloskubernetes "github.com/siderolabs/talos/pkg/kubernetes" @@ -39,6 +52,7 @@ type K8sSuite struct { DynamicClient dynamic.Interface DiscoveryClient *discovery.DiscoveryClient RestConfig *rest.Config + Mapper *restmapper.DeferredDiscoveryRESTMapper } // SetupSuite initializes Kubernetes client. @@ -68,6 +82,8 @@ func (k8sSuite *K8sSuite) SetupSuite() { k8sSuite.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(config) k8sSuite.Require().NoError(err) + + k8sSuite.Mapper = restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(k8sSuite.DiscoveryClient)) } // GetK8sNodeByInternalIP returns the kubernetes node by its internal ip or error if it is not found. @@ -246,3 +262,139 @@ func (k8sSuite *K8sSuite) GetPodsWithLabel(ctx context.Context, namespace, label return podList, nil } + +// ParseManifests parses YAML manifest bytes into unstructured objects. +func (k8sSuite *K8sSuite) ParseManifests(manifests []byte) []unstructured.Unstructured { + reader := yaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(manifests))) + + var parsedManifests []unstructured.Unstructured + + for { + yamlManifest, err := reader.Read() + if err != nil { + if err == io.EOF { + break + } + + k8sSuite.Require().NoError(err) + } + + yamlManifest = bytes.TrimSpace(yamlManifest) + + if len(yamlManifest) == 0 { + continue + } + + jsonManifest, err := yaml.ToJSON(yamlManifest) + if err != nil { + k8sSuite.Require().NoError(err, "error converting manifest to JSON") + } + + if bytes.Equal(jsonManifest, []byte("null")) || bytes.Equal(jsonManifest, []byte("{}")) { + // skip YAML docs which contain only comments + continue + } + + var obj unstructured.Unstructured + + if err = json.Unmarshal(jsonManifest, &obj); err != nil { + k8sSuite.Require().NoError(err, "error loading JSON manifest into unstructured") + } + + parsedManifests = append(parsedManifests, obj) + } + + return parsedManifests +} + +// ApplyManifests applies the given manifests to the Kubernetes cluster. +func (k8sSuite *K8sSuite) ApplyManifests(ctx context.Context, manifests []unstructured.Unstructured) { + for _, obj := range manifests { + mapping, err := k8sSuite.Mapper.RESTMapping(obj.GetObjectKind().GroupVersionKind().GroupKind(), obj.GetObjectKind().GroupVersionKind().Version) + if err != nil { + k8sSuite.Require().NoError(err, "error creating mapping for object %s", obj.GetName()) + } + + dr := k8sSuite.DynamicClient.Resource(mapping.Resource).Namespace(obj.GetNamespace()) + + _, err = dr.Create(ctx, &obj, metav1.CreateOptions{}) + k8sSuite.Require().NoError(err, "error creating object %s", obj.GetName()) + + k8sSuite.T().Logf("created object %s/%s/%s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName()) + } +} + +// DeleteManifests deletes the given manifests from the Kubernetes cluster. +func (k8sSuite *K8sSuite) DeleteManifests(ctx context.Context, manifests []unstructured.Unstructured) { + // process in reverse orderd + manifests = slices.Clone(manifests) + slices.Reverse(manifests) + + for _, obj := range manifests { + mapping, err := k8sSuite.Mapper.RESTMapping(obj.GetObjectKind().GroupVersionKind().GroupKind(), obj.GetObjectKind().GroupVersionKind().Version) + if err != nil { + k8sSuite.Require().NoError(err, "error creating mapping for object %s", obj.GetName()) + } + + dr := k8sSuite.DynamicClient.Resource(mapping.Resource).Namespace(obj.GetNamespace()) + + err = dr.Delete(ctx, obj.GetName(), metav1.DeleteOptions{}) + if errors.IsNotFound(err) { + continue + } + + k8sSuite.Require().NoError(err, "error deleting object %s", obj.GetName()) + + // wait for the object to be deleted + fieldSelector := fields.OneTermEqualSelector("metadata.name", obj.GetName()).String() + lw := &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + options.FieldSelector = fieldSelector + + return dr.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + options.FieldSelector = fieldSelector + + return dr.Watch(ctx, options) + }, + } + + preconditionFunc := func(store cache.Store) (bool, error) { + var exists bool + + _, exists, err = store.Get(&metav1.ObjectMeta{Namespace: obj.GetNamespace(), Name: obj.GetName()}) + if err != nil { + return true, err + } + + if !exists { + // since we're looking for it to disappear we just return here if it no longer exists + return true, nil + } + + return false, nil + } + + _, err = watchtools.UntilWithSync(ctx, lw, &unstructured.Unstructured{}, preconditionFunc, func(event watch.Event) (bool, error) { + return event.Type == watch.Deleted, nil + }) + + k8sSuite.Require().NoError(err, "error waiting for the object to be deleted %s", obj.GetName()) + + k8sSuite.T().Logf("deleted object %s/%s/%s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName()) + } +} + +// ToUnstructured converts the given runtime.Object to unstructured.Unstructured. +func (k8sSuite *K8sSuite) ToUnstructured(obj runtime.Object) unstructured.Unstructured { + unstructuredObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + k8sSuite.Require().NoError(err, "error converting object to unstructured") + } + + u := unstructured.Unstructured{Object: unstructuredObj} + u.SetGroupVersionKind(obj.GetObjectKind().GroupVersionKind()) + + return u +} diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go index f25ec556c1..0b1669dd86 100644 --- a/internal/integration/integration_test.go +++ b/internal/integration/integration_test.go @@ -22,6 +22,7 @@ import ( "github.com/siderolabs/talos/internal/integration/cli" "github.com/siderolabs/talos/internal/integration/k8s" provision_test "github.com/siderolabs/talos/internal/integration/provision" + "github.com/siderolabs/talos/pkg/images" clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config" "github.com/siderolabs/talos/pkg/machinery/constants" "github.com/siderolabs/talos/pkg/provision" @@ -49,6 +50,7 @@ var ( provisionerName string clusterName string stateDir string + talosImage string ) // TestIntegration ... @@ -99,6 +101,7 @@ func TestIntegration(t *testing.T) { ExtensionsQEMU: extensionsQEMU, ExtensionsNvidia: extensionsNvidia, TrustedBoot: trustedBoot, + TalosImage: talosImage, }) } @@ -158,6 +161,7 @@ func init() { flag.StringVar(&expectedGoVersion, "talos.go.version", constants.GoVersion, "expected Talos version") flag.StringVar(&talosctlPath, "talos.talosctlpath", "talosctl", "The path to 'talosctl' binary") flag.StringVar(&kubectlPath, "talos.kubectlpath", "kubectl", "The path to 'kubectl' binary") + flag.StringVar(&talosImage, "talos.image", images.DefaultTalosImageRepository, "The default 'talos' container image") flag.StringVar(&provision_test.DefaultSettings.CIDR, "talos.provision.cidr", provision_test.DefaultSettings.CIDR, "CIDR to use to provision clusters (provision tests only)") flag.Var(&provision_test.DefaultSettings.RegistryMirrors, "talos.provision.registry-mirror", "registry mirrors to use (provision tests only)") diff --git a/internal/integration/k8s/testdata/local-path-storage.yaml b/internal/integration/k8s/testdata/local-path-storage.yaml new file mode 100644 index 0000000000..5e992051f0 --- /dev/null +++ b/internal/integration/k8s/testdata/local-path-storage.yaml @@ -0,0 +1,191 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + pod-security.kubernetes.io/enforce: privileged + name: local-path-storage +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "true" + name: local-path +provisioner: rancher.io/local-path +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: local-path-provisioner-service-account + namespace: local-path-storage +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: local-path-provisioner-role + namespace: local-path-storage +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - create + - patch + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: local-path-provisioner-role +rules: +- apiGroups: + - "" + resources: + - nodes + - persistentvolumeclaims + - configmaps + - pods + - pods/log + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - patch + - update + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: local-path-provisioner-bind + namespace: local-path-storage +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: local-path-provisioner-role +subjects: +- kind: ServiceAccount + name: local-path-provisioner-service-account + namespace: local-path-storage +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: local-path-provisioner-bind +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: local-path-provisioner-role +subjects: +- kind: ServiceAccount + name: local-path-provisioner-service-account + namespace: local-path-storage +--- +apiVersion: v1 +data: + config.json: |- + { + "nodePathMap":[ + { + "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES", + "paths":["/var/local-path-provisioner"] + } + ] + } + helperPod.yaml: |- + apiVersion: v1 + kind: Pod + metadata: + name: helper-pod + spec: + priorityClassName: system-node-critical + tolerations: + - key: node.kubernetes.io/disk-pressure + operator: Exists + effect: NoSchedule + containers: + - name: helper-pod + image: busybox + imagePullPolicy: IfNotPresent + setup: |- + #!/bin/sh + set -eu + mkdir -m 0777 -p "$VOL_DIR" + teardown: |- + #!/bin/sh + set -eu + rm -rf "$VOL_DIR" +kind: ConfigMap +metadata: + name: local-path-config + namespace: local-path-storage +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: local-path-provisioner + namespace: local-path-storage +spec: + replicas: 1 + selector: + matchLabels: + app: local-path-provisioner + template: + metadata: + labels: + app: local-path-provisioner + spec: + containers: + - command: + - local-path-provisioner + - --debug + - start + - --config + - /etc/config/config.json + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: rancher/local-path-provisioner:v0.0.26 + imagePullPolicy: IfNotPresent + name: local-path-provisioner + volumeMounts: + - mountPath: /etc/config/ + name: config-volume + serviceAccountName: local-path-provisioner-service-account + volumes: + - configMap: + name: local-path-config + name: config-volume diff --git a/internal/integration/k8s/tink.go b/internal/integration/k8s/tink.go new file mode 100644 index 0000000000..c699446d0d --- /dev/null +++ b/internal/integration/k8s/tink.go @@ -0,0 +1,455 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +//go:build integration_k8s + +package k8s + +import ( + "context" + "crypto/tls" + _ "embed" + "fmt" + "net" + "net/netip" + "strconv" + "strings" + "testing" + "time" + + "github.com/siderolabs/gen/ensure" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-pointer" + "github.com/siderolabs/go-retry/retry" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + podsecurity "k8s.io/pod-security-admission/api" + + "github.com/siderolabs/talos/internal/integration/base" + "github.com/siderolabs/talos/pkg/cluster" + "github.com/siderolabs/talos/pkg/cluster/check" + machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" + "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/config/generate" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/version" +) + +// TinkSuite verifies Talos-in-Kubernetes. +type TinkSuite struct { + base.K8sSuite +} + +// SuiteName ... +func (suite *TinkSuite) SuiteName() string { + return "k8s.TinkSuite" +} + +//go:embed testdata/local-path-storage.yaml +var localPathStorageYAML []byte + +const ( + tinkK8sPort = "k8s-api" + tinkTalosPort = "talos-api" +) + +// TestDeploy verifies that tink can be deployed with a single control-plane node. +func (suite *TinkSuite) TestDeploy() { + if testing.Short() { + suite.T().Skip("skipping in short mode") + } + + if suite.Cluster == nil { + suite.T().Skip("without full cluster state reaching out to the node IP is not reliable") + } + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + suite.T().Cleanup(cancel) + + localPathStorage := suite.ParseManifests(localPathStorageYAML) + + suite.T().Cleanup(func() { + cleanUpCtx, cleanupCancel := context.WithTimeout(context.Background(), time.Minute) + defer cleanupCancel() + + suite.DeleteManifests(cleanUpCtx, localPathStorage) + }) + + suite.ApplyManifests(ctx, localPathStorage) + + const ( + namespace = "talos-in-talos" + service = "talos" + ss = "talos-cp" + ) + + talosImage := fmt.Sprintf("%s:%s", suite.TalosImage, version.Tag) + + suite.T().Logf("deploying Talos-in-Kubernetes from image %s", talosImage) + + tinkManifests := suite.getTinkManifests(namespace, service, ss, talosImage) + + suite.T().Cleanup(func() { + cleanUpCtx, cleanupCancel := context.WithTimeout(context.Background(), time.Minute) + defer cleanupCancel() + + suite.DeleteManifests(cleanUpCtx, tinkManifests) + }) + + suite.ApplyManifests(ctx, tinkManifests) + + // wait for the control-plane pod to be running + suite.Require().NoError(suite.WaitForPodToBeRunning(ctx, time.Minute, namespace, ss+"-0")) + + // read back Service to figure out the ports + svc, err := suite.Clientset.CoreV1().Services(namespace).Get(ctx, service, metav1.GetOptions{}) + suite.Require().NoError(err) + + var k8sPort, talosPort int + + for _, portSpec := range svc.Spec.Ports { + switch portSpec.Name { + case tinkK8sPort: + k8sPort = int(portSpec.NodePort) + case tinkTalosPort: + talosPort = int(portSpec.NodePort) + } + } + + suite.Require().NotZero(k8sPort) + suite.Require().NotZero(talosPort) + + // find pod IP + pod, err := suite.Clientset.CoreV1().Pods(namespace).Get(ctx, ss+"-0", metav1.GetOptions{}) + suite.Require().NoError(err) + + suite.Require().NotEmpty(pod.Status.PodIP) + + podIP := netip.MustParseAddr(pod.Status.PodIP) + + // grab any random lbNode IP + lbNode := suite.RandomDiscoveredNodeInternalIP() + + talosEndpoint := net.JoinHostPort(lbNode, strconv.Itoa(talosPort)) + + in, err := generate.NewInput(namespace, + fmt.Sprintf("https://%s", net.JoinHostPort(lbNode, strconv.Itoa(k8sPort))), + constants.DefaultKubernetesVersion, + generate.WithAdditionalSubjectAltNames([]string{lbNode}), + ) + suite.Require().NoError(err) + + // override pod/service subnets, as Talos-in-Talos would use it for "host" addresses + in.PodNet = []string{"192.168.0.0/20"} + in.ServiceNet = []string{"192.168.128.0/20"} + + cpCfg, err := in.Config(machine.TypeControlPlane) + suite.Require().NoError(err) + + cpCfgBytes, err := cpCfg.Bytes() + suite.Require().NoError(err) + + suite.waitForEndpointReady(talosEndpoint) + + insecureClient, err := client.New(ctx, + client.WithEndpoints(talosEndpoint), + client.WithTLSConfig(&tls.Config{InsecureSkipVerify: true}), + ) + suite.Require().NoError(err) + + suite.T().Log("applying initial configuration") + + _, err = insecureClient.ApplyConfiguration(ctx, &machineapi.ApplyConfigurationRequest{ + Data: cpCfgBytes, + Mode: machineapi.ApplyConfigurationRequest_AUTO, + }) + suite.Require().NoError(err) + + // bootstrap + talosconfig, err := in.Talosconfig() + suite.Require().NoError(err) + + talosconfig.Contexts[talosconfig.Context].Endpoints = []string{talosEndpoint} + talosconfig.Contexts[talosconfig.Context].Nodes = []string{podIP.String()} + + suite.T().Logf("talosconfig = %s", string(ensure.Value(talosconfig.Bytes()))) + + suite.waitForEndpointReady(talosEndpoint) + + talosClient, err := client.New(ctx, + client.WithConfigContext(talosconfig.Contexts[talosconfig.Context]), + ) + suite.Require().NoError(err) + + suite.T().Log("bootstrapping") + + suite.Require().NoError(talosClient.Bootstrap(ctx, &machineapi.BootstrapRequest{})) + + clusterAccess := &tinkClusterAccess{ + KubernetesClient: cluster.KubernetesClient{ + ClientProvider: &cluster.ConfigClientProvider{ + TalosConfig: talosconfig, + }, + }, + + nodeIP: podIP, + } + + suite.Require().NoError( + check.Wait( + ctx, + clusterAccess, + check.DefaultClusterChecks(), + check.StderrReporter(), + ), + ) +} + +type tinkClusterAccess struct { + cluster.KubernetesClient + + nodeIP netip.Addr +} + +func (access *tinkClusterAccess) Nodes() []cluster.NodeInfo { + return []cluster.NodeInfo{ + { + InternalIP: access.nodeIP, + IPs: []netip.Addr{access.nodeIP}, + }, + } +} + +func (access *tinkClusterAccess) NodesByType(typ machine.Type) []cluster.NodeInfo { + switch typ { + case machine.TypeControlPlane: + return []cluster.NodeInfo{ + { + InternalIP: access.nodeIP, + IPs: []netip.Addr{access.nodeIP}, + }, + } + case machine.TypeWorker, machine.TypeInit: + return nil + case machine.TypeUnknown: + fallthrough + default: + panic(fmt.Sprintf("unexpected machine type: %s", typ)) + } +} + +func (suite *TinkSuite) waitForEndpointReady(endpoint string) { + suite.Require().NoError(retry.Constant(30*time.Second, retry.WithUnits(10*time.Millisecond)).Retry(func() error { + c, err := tls.Dial("tcp", endpoint, + &tls.Config{ + InsecureSkipVerify: true, + }, + ) + + if c != nil { + c.Close() //nolint:errcheck + } + + return retry.ExpectedError(err) + })) +} + +func (suite *TinkSuite) getTinkManifests(namespace, serviceName, ssName, talosImage string) []unstructured.Unstructured { + labels := map[string]string{ + "app": "talos-cp", + } + + tinkManifests := []runtime.Object{ + &corev1.Namespace{ + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Labels: map[string]string{ + podsecurity.EnforceLevelLabel: string(podsecurity.LevelPrivileged), + }, + }, + }, + &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: serviceName, + Namespace: namespace, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeNodePort, + Selector: labels, + Ports: []corev1.ServicePort{ + { + Name: tinkK8sPort, + Protocol: corev1.ProtocolTCP, + Port: constants.DefaultControlPlanePort, + TargetPort: intstr.FromString(tinkK8sPort), + }, + { + Name: tinkTalosPort, + Protocol: corev1.ProtocolTCP, + Port: constants.ApidPort, + TargetPort: intstr.FromString(tinkTalosPort), + }, + }, + }, + }, + } + + statefulSet := &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "StatefulSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: ssName, + Namespace: namespace, + }, + Spec: appsv1.StatefulSetSpec{ + ServiceName: serviceName, + Replicas: pointer.To(int32(1)), + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "talos", + Image: talosImage, + ImagePullPolicy: corev1.PullAlways, + SecurityContext: &corev1.SecurityContext{ + Privileged: pointer.To(true), + ReadOnlyRootFilesystem: pointer.To(true), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeUnconfined, + }, + }, + Env: []corev1.EnvVar{ + { + Name: "PLATFORM", + Value: "container", + }, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("1Gi"), + corev1.ResourceCPU: resource.MustParse("750m"), + }, + }, + Ports: []corev1.ContainerPort{ + { + ContainerPort: constants.ApidPort, + Name: tinkTalosPort, + }, + { + ContainerPort: constants.DefaultControlPlanePort, + Name: tinkK8sPort, + }, + }, + }, + }, + }, + }, + }, + } + + for _, ephemeralMount := range []string{"run", "system", "tmp"} { + statefulSet.Spec.Template.Spec.Containers[0].VolumeMounts = append( + statefulSet.Spec.Template.Spec.Containers[0].VolumeMounts, + corev1.VolumeMount{ + MountPath: "/" + ephemeralMount, + Name: ephemeralMount, + }, + ) + + statefulSet.Spec.Template.Spec.Volumes = append( + statefulSet.Spec.Template.Spec.Volumes, + corev1.Volume{ + Name: ephemeralMount, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + ) + } + + type overlayMountSpec struct { + MountPoint string + Size string + } + + for _, overlayMount := range append( + []overlayMountSpec{ + { + MountPoint: constants.StateMountPoint, + Size: "100Mi", + }, + { + MountPoint: constants.EphemeralMountPoint, + Size: "6Gi", + }, + }, + xslices.Map( + constants.Overlays, + func(mountPath string) overlayMountSpec { + return overlayMountSpec{ + MountPoint: mountPath, + Size: "100Mi", + } + }, + )..., + ) { + name := strings.ReplaceAll(strings.TrimLeft(overlayMount.MountPoint, "/"), "/", "-") + + statefulSet.Spec.Template.Spec.Containers[0].VolumeMounts = append( + statefulSet.Spec.Template.Spec.Containers[0].VolumeMounts, + corev1.VolumeMount{ + MountPath: overlayMount.MountPoint, + Name: name, + }, + ) + + statefulSet.Spec.VolumeClaimTemplates = append( + statefulSet.Spec.VolumeClaimTemplates, + corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + corev1.ReadWriteOnce, + }, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(overlayMount.Size), + }, + }, + }, + }) + } + + tinkManifests = append(tinkManifests, statefulSet) + + return xslices.Map(tinkManifests, suite.ToUnstructured) +} + +func init() { + allSuites = append(allSuites, new(TinkSuite)) +} diff --git a/internal/pkg/cgroup/cgroup.go b/internal/pkg/cgroup/cgroup.go new file mode 100644 index 0000000000..a10f1652d1 --- /dev/null +++ b/internal/pkg/cgroup/cgroup.go @@ -0,0 +1,58 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package cgroup provides cgroup utilities to handle nested cgroups. +// +// When Talos runs in a container, it might either share or not share the host cgroup namespace. +// If the cgroup namespace is not shared, PID 1 will appear in cgroup '/', otherwise it will be +// part of some pre-existing cgroup hierarchy. +// +// When Talos is running in a non-container mode, it is always at the root of the cgroup hierarchy. +// +// This package provides a transparent way to handle nested cgroups by providing a Path() function +// which returns the correct cgroup path based on the cgroup hierarchy available. +package cgroup + +import ( + "path/filepath" + + "github.com/containerd/cgroups/v3" + "github.com/containerd/cgroups/v3/cgroup2" +) + +var root = "/" + +// InitRoot initializes the root cgroup path. +// +// This function should be called once at the beginning of the program, after the cgroup +// filesystem is mounted. +// +// This function only supports cgroupv2 nesting. +func InitRoot() error { + if cgroups.Mode() != cgroups.Unified { + return nil + } + + var err error + + root, err = cgroup2.NestedGroupPath("/") + + return err +} + +// Root returns the root cgroup path. +func Root() string { + return root +} + +// Path returns the path to the cgroup. +// +// This function handles the case when the cgroups are nested. +func Path(cgroupPath string) string { + if cgroups.Mode() != cgroups.Unified { + return cgroupPath + } + + return filepath.Join(root, cgroupPath) +} diff --git a/pkg/machinery/client/client.go b/pkg/machinery/client/client.go index afb19c85c3..f97b70d8c1 100644 --- a/pkg/machinery/client/client.go +++ b/pkg/machinery/client/client.go @@ -108,6 +108,10 @@ func (c *Client) GetEndpoints() []string { return c.options.endpointsOverride } + if c.options.configContext != nil { + return c.options.configContext.Endpoints + } + if c.options.config != nil { if err := c.resolveConfigContext(); err != nil { return nil diff --git a/pkg/machinery/constants/constants.go b/pkg/machinery/constants/constants.go index 8175e06b67..bbf8052038 100644 --- a/pkg/machinery/constants/constants.go +++ b/pkg/machinery/constants/constants.go @@ -650,6 +650,9 @@ const ( // CgroupSystemRuntime is the cgroup name for containerd runtime processes. CgroupSystemRuntime = CgroupSystem + "/runtime" + // CgroupUdevd is the cgroup name for udevd runtime processes. + CgroupUdevd = CgroupSystem + "/udevd" + // CgroupExtensions is the cgroup name for system extension processes. CgroupExtensions = CgroupSystem + "/extensions" diff --git a/pkg/provision/providers/docker/node.go b/pkg/provision/providers/docker/node.go index 24156ad765..2d1a2e88c0 100644 --- a/pkg/provision/providers/docker/node.go +++ b/pkg/provision/providers/docker/node.go @@ -101,8 +101,7 @@ func (p *provisioner) createNode(ctx context.Context, clusterReq provision.Clust }) } - // constants.UdevDir is in the list to support pre-1.4 Talos versions which had it in constants.Overlays - for _, path := range append(constants.Overlays, constants.UdevDir, "/var", "/system/state") { + for _, path := range append([]string{constants.EphemeralMountPoint, constants.StateMountPoint}, constants.Overlays...) { mounts = append(mounts, mount.Mount{ Type: mount.TypeVolume, Target: path, diff --git a/website/content/v1.7/talos-guides/install/local-platforms/docker.md b/website/content/v1.7/talos-guides/install/local-platforms/docker.md index 9939327202..ce986e321c 100644 --- a/website/content/v1.7/talos-guides/install/local-platforms/docker.md +++ b/website/content/v1.7/talos-guides/install/local-platforms/docker.md @@ -73,7 +73,6 @@ docker run --rm -it \ --mount type=volume,destination=/etc/cni \ --mount type=volume,destination=/etc/kubernetes \ --mount type=volume,destination=/usr/libexec/kubernetes \ - --mount type=volume,destination=/usr/etc/udev \ --mount type=volume,destination=/opt \ -e PLATFORM=container \ ghcr.io/siderolabs/talos:{{< release >}}