diff --git a/Dockerfile b/Dockerfile index d0ccc93c7..da5b2d113 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,11 +21,11 @@ COPY --from=builder /go/src/github.com/openshift/cluster-node-tuning-operator/_o COPY --from=builder /go/src/github.com/openshift/cluster-node-tuning-operator/_output/performance-profile-creator /usr/bin/ COPY --from=builder /go/src/github.com/openshift/cluster-node-tuning-operator/_output/gather-sysinfo /usr/bin/ COPY manifests/*.yaml manifests/image-references /manifests/ -ENV APP_ROOT=/var/lib/tuned +ENV APP_ROOT=/var/lib/ocp-tuned ENV PATH=${APP_ROOT}/bin:${PATH} ENV HOME=${APP_ROOT} +ENV SYSTEMD_IGNORE_CHROOT=1 WORKDIR ${APP_ROOT} -COPY --from=builder /go/src/github.com/openshift/cluster-node-tuning-operator/_output/openshift-tuned /usr/bin/ COPY --from=tuned /root/assets ${APP_ROOT} COPY --from=tuned /root/rpmbuild/RPMS/noarch /root/rpms RUN INSTALL_PKGS=" \ @@ -36,7 +36,8 @@ RUN INSTALL_PKGS=" \ rpm -V $INSTALL_PKGS && \ dnf --setopt=tsflags=nodocs -y install /root/rpms/*.rpm && \ find /root/rpms -name \*.rpm -exec basename {} .rpm \; | xargs rpm -e --justdb && \ - rm -rf /var/lib/tuned/tuned && \ + rm -rf /var/lib/ocp-tuned/{tuned,performanceprofile} && \ + sed -Ei 's|^#?\s*enable_unix_socket\s*=.*$|enable_unix_socket = 1|;s|^#?\s*rollback\s*=.*$|rollback = not_on_exit|' /etc/tuned/tuned-main.conf && \ touch /etc/sysctl.conf $APP_ROOT/provider && \ dnf clean all && \ rm -rf /var/cache/yum ~/patches /root/rpms && \ diff --git a/Dockerfile.rhel8 b/Dockerfile.rhel8 index 052b1f160..e64b3c907 100644 --- a/Dockerfile.rhel8 +++ b/Dockerfile.rhel8 @@ -8,12 +8,12 @@ COPY --from=builder /go/src/github.com/openshift/cluster-node-tuning-operator/_o COPY --from=builder /go/src/github.com/openshift/cluster-node-tuning-operator/_output/performance-profile-creator /usr/bin/ COPY --from=builder /go/src/github.com/openshift/cluster-node-tuning-operator/_output/gather-sysinfo /usr/bin/ COPY manifests/*.yaml manifests/image-references /manifests/ -ENV APP_ROOT=/var/lib/tuned +ENV APP_ROOT=/var/lib/ocp-tuned ENV PATH=${APP_ROOT}/bin:${PATH} ENV HOME=${APP_ROOT} +ENV SYSTEMD_IGNORE_CHROOT=1 WORKDIR ${APP_ROOT} COPY assets ${APP_ROOT} -COPY --from=builder /go/src/github.com/openshift/cluster-node-tuning-operator/_output/openshift-tuned /usr/bin/ RUN INSTALL_PKGS=" \ tuned tuned-profiles-atomic tuned-profiles-cpu-partitioning tuned-profiles-mssql tuned-profiles-nfv tuned-profiles-nfv-guest \ tuned-profiles-nfv-host tuned-profiles-openshift tuned-profiles-oracle tuned-profiles-postgresql tuned-profiles-realtime \ @@ -21,7 +21,8 @@ RUN INSTALL_PKGS=" \ nmap-ncat procps-ng pciutils" && \ mkdir -p /etc/grub.d/ /boot && \ dnf install --setopt=tsflags=nodocs -y $INSTALL_PKGS && \ - rm -rf /var/lib/tuned/tuned && \ + rm -rf /var/lib/ocp-tuned/{tuned,performanceprofile} && \ + sed -Ei 's|^#?\s*enable_unix_socket\s*=.*$|enable_unix_socket = 1|;s|^#?\s*rollback\s*=.*$|rollback = not_on_exit|' /etc/tuned/tuned-main.conf && \ touch /etc/sysctl.conf $APP_ROOT/provider && \ dnf clean all && \ rm -rf /var/cache/yum ~/patches /root/rpms && \ diff --git a/Dockerfile.rhel9 b/Dockerfile.rhel9 index 3f752bdf2..f26bccc6a 100644 --- a/Dockerfile.rhel9 +++ b/Dockerfile.rhel9 @@ -8,12 +8,12 @@ COPY --from=builder /go/src/github.com/openshift/cluster-node-tuning-operator/_o COPY --from=builder /go/src/github.com/openshift/cluster-node-tuning-operator/_output/performance-profile-creator /usr/bin/ COPY --from=builder /go/src/github.com/openshift/cluster-node-tuning-operator/_output/gather-sysinfo /usr/bin/ COPY manifests/*.yaml manifests/image-references /manifests/ -ENV APP_ROOT=/var/lib/tuned +ENV APP_ROOT=/var/lib/ocp-tuned ENV PATH=${APP_ROOT}/bin:${PATH} ENV HOME=${APP_ROOT} +ENV SYSTEMD_IGNORE_CHROOT=1 WORKDIR ${APP_ROOT} COPY assets ${APP_ROOT} -COPY --from=builder /go/src/github.com/openshift/cluster-node-tuning-operator/_output/openshift-tuned /usr/bin/ RUN INSTALL_PKGS=" \ tuned tuned-profiles-atomic tuned-profiles-cpu-partitioning tuned-profiles-mssql tuned-profiles-nfv tuned-profiles-nfv-guest \ tuned-profiles-nfv-host tuned-profiles-openshift tuned-profiles-oracle tuned-profiles-postgresql tuned-profiles-realtime \ @@ -21,7 +21,8 @@ RUN INSTALL_PKGS=" \ nmap-ncat procps-ng pciutils" && \ mkdir -p /etc/grub.d/ /boot && \ dnf install --setopt=tsflags=nodocs -y $INSTALL_PKGS && \ - rm -rf /var/lib/tuned/tuned && \ + rm -rf /var/lib/ocp-tuned/{tuned,performanceprofile} && \ + sed -Ei 's|^#?\s*enable_unix_socket\s*=.*$|enable_unix_socket = 1|;s|^#?\s*rollback\s*=.*$|rollback = not_on_exit|' /etc/tuned/tuned-main.conf && \ touch /etc/sysctl.conf $APP_ROOT/provider && \ dnf clean all && \ rm -rf /var/cache/yum ~/patches /root/rpms && \ diff --git a/Makefile b/Makefile index 075306161..6891ce924 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,6 @@ clone-tuned: build: $(BINDATA) pkg/generated build-performance-profile-creator build-gather-sysinfo $(GO_BUILD_RECIPE) - ln -sf $(PACKAGE_BIN) $(OUT_DIR)/openshift-tuned $(BINDATA): $(GOBINDATA_BIN) $(ASSETS) $(GOBINDATA_BIN) -mode 420 -modtime 1 -pkg manifests -o $(BINDATA) assets/... diff --git a/assets/bin/run b/assets/bin/run deleted file mode 100755 index d59721712..000000000 --- a/assets/bin/run +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -export SYSTEMD_IGNORE_CHROOT=1 - -set_tuned_main_opt() { - local key=$1 - local val=$2 - local tuned_main_conf=/etc/tuned/tuned-main.conf - - sed -Ei 's|^#?\s*'"$key"'\s*=.*$|'"$key"' = '"$val"'|' $tuned_main_conf -} - -start() { - # Tuned can take ~20s to reload/start when "ulimit -Sn == 1048576". - # See: - # - https://github.com/redhat-performance/tuned/issues/146 - # - https://www.python.org/dev/peps/pep-0446/#closing-all-open-file-descriptors - # - http://bugs.python.org/issue1663329 - ulimit -Sn 1024 # workaround for the issue above - - set_tuned_main_opt enable_unix_socket 1 - set_tuned_main_opt rollback not_on_exit - - exec /usr/bin/openshift-tuned \ - -v=0 -} - -stop() { - : -} - -$@ diff --git a/assets/tuned/manifests/default-cr-tuned.yaml b/assets/tuned/manifests/default-cr-tuned.yaml index 06c2f88e0..5c4672df6 100644 --- a/assets/tuned/manifests/default-cr-tuned.yaml +++ b/assets/tuned/manifests/default-cr-tuned.yaml @@ -9,7 +9,7 @@ spec: data: | [main] summary=Optimize systems running OpenShift (provider specific parent profile) - include=-provider-${f:exec:cat:/var/lib/tuned/provider},openshift + include=-provider-${f:exec:cat:/var/lib/ocp-tuned/provider},openshift recommend: - profile: "openshift-control-plane" priority: 30 diff --git a/assets/tuned/manifests/ds-tuned.yaml b/assets/tuned/manifests/ds-tuned.yaml index a1129a9bd..cbdaef77d 100644 --- a/assets/tuned/manifests/ds-tuned.yaml +++ b/assets/tuned/manifests/ds-tuned.yaml @@ -22,7 +22,7 @@ spec: spec: serviceAccountName: tuned containers: - - command: ["/var/lib/tuned/bin/run","start"] + - command: ["/usr/bin/cluster-node-tuning-operator","openshift-tuned","--in-cluster","-v=0"] resources: requests: cpu: 10m @@ -35,9 +35,6 @@ spec: terminationMessagePath: /dev/termination-log terminationMessagePolicy: FallbackToLogsOnError volumeMounts: - - mountPath: /var/lib/tuned/profiles-data - name: var-lib-tuned-profiles-data - mountPropagation: HostToContainer - mountPath: /etc/modprobe.d name: etc-modprobe-d mountPropagation: HostToContainer @@ -91,6 +88,8 @@ spec: value: "600" - name: RELEASE_VERSION value: "" + - name: CLUSTER_NODE_TUNED_IMAGE + value: ${CLUSTER_NODE_TUNED_IMAGE} volumes: - hostPath: path: /etc/modprobe.d @@ -136,14 +135,6 @@ spec: hostPath: path: / type: Directory - - configMap: - defaultMode: 0644 - items: - - key: tuned-profiles-data - path: tuned-profiles.yaml - name: tuned-profiles - optional: true - name: var-lib-tuned-profiles-data dnsPolicy: ClusterFirst nodeSelector: kubernetes.io/os: linux diff --git a/cmd/cluster-node-tuning-operator/main.go b/cmd/cluster-node-tuning-operator/main.go index c9d7106f7..b8697d3d8 100644 --- a/cmd/cluster-node-tuning-operator/main.go +++ b/cmd/cluster-node-tuning-operator/main.go @@ -6,7 +6,6 @@ import ( "flag" "fmt" "os" - "path/filepath" "runtime" "time" @@ -47,8 +46,7 @@ import ( "github.com/openshift/cluster-node-tuning-operator/pkg/metrics" "github.com/openshift/cluster-node-tuning-operator/pkg/operator" "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/cmd/render" - "github.com/openshift/cluster-node-tuning-operator/pkg/signals" - "github.com/openshift/cluster-node-tuning-operator/pkg/tuned" + "github.com/openshift/cluster-node-tuning-operator/pkg/tuned/cmd/operand" tunedrender "github.com/openshift/cluster-node-tuning-operator/pkg/tuned/cmd/render" "github.com/openshift/cluster-node-tuning-operator/pkg/util" "github.com/openshift/cluster-node-tuning-operator/version" @@ -93,13 +91,15 @@ var rootCmd = &cobra.Command{ }, } -var enableLeaderElection bool -var showVersionAndExit bool +var ( + enableLeaderElection bool + showVersionAndExit bool +) func prepareCommands() { - rootCmd.PersistentFlags().BoolVar(&enableLeaderElection, "enable-leader-election", true, + rootCmd.Flags().BoolVar(&enableLeaderElection, "enable-leader-election", true, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.") - rootCmd.PersistentFlags().BoolVar(&showVersionAndExit, "version", false, + rootCmd.Flags().BoolVar(&showVersionAndExit, "version", false, "Show program version and exit.") // Include the klog command line arguments @@ -110,6 +110,7 @@ func prepareCommands() { rootCmd.AddCommand(render.NewRenderCommand()) rootCmd.AddCommand(tunedrender.NewRenderBootCmdMCCommand()) } + rootCmd.AddCommand(operand.NewTunedCommand()) } func operatorRun() { @@ -205,17 +206,6 @@ func operatorRun() { } } -func tunedOperandRun() { - var boolVersion bool - flag.BoolVar(&boolVersion, "version", false, "show program version and exit") - - // flag.Parse is called from within tuned.Run -> parseCmdOpts - // but the version flag variable is inherited from here.. - - stopCh := signals.SetupSignalHandler() - tuned.Run(stopCh, &boolVersion, version.Version) -} - // Uninstall PAO OLM operator since PAO is shipped // as a core operator from 4.11. // This is relevant for any upgrade path of an OpenShift cluster @@ -376,15 +366,8 @@ func setupFeatureGates(ctx context.Context, config *rest.Config, operatorNamespa } func main() { - runAs := filepath.Base(os.Args[0]) - - switch runAs { - case version.OperatorFilename: - prepareCommands() - _ = rootCmd.Execute() - case version.OperandFilename: - tunedOperandRun() - default: - klog.Fatalf("application should be run as \"%s\" or \"%s\"", version.OperatorFilename, version.OperandFilename) + prepareCommands() + if err := rootCmd.Execute(); err != nil { + klog.Fatal(err) } } diff --git a/pkg/manifests/manifests.go b/pkg/manifests/manifests.go index cccdf9b70..39b0f157f 100644 --- a/pkg/manifests/manifests.go +++ b/pkg/manifests/manifests.go @@ -76,9 +76,11 @@ func TunedDaemonSet() *appsv1.DaemonSet { ds.Spec.Template.Spec.Containers[0].Image = imageTuned for i := range ds.Spec.Template.Spec.Containers[0].Env { - if ds.Spec.Template.Spec.Containers[0].Env[i].Name == "RELEASE_VERSION" { + switch ds.Spec.Template.Spec.Containers[0].Env[i].Name { + case "RELEASE_VERSION": ds.Spec.Template.Spec.Containers[0].Env[i].Value = os.Getenv("RELEASE_VERSION") - break + case "CLUSTER_NODE_TUNED_IMAGE": + ds.Spec.Template.Spec.Containers[0].Env[i].Value = os.Getenv("CLUSTER_NODE_TUNED_IMAGE") } } diff --git a/pkg/tuned/cmd/operand/cmd.go b/pkg/tuned/cmd/operand/cmd.go new file mode 100644 index 000000000..ada5882e7 --- /dev/null +++ b/pkg/tuned/cmd/operand/cmd.go @@ -0,0 +1,78 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package operand + +import ( + "flag" + + "github.com/openshift/cluster-node-tuning-operator/pkg/signals" + "github.com/openshift/cluster-node-tuning-operator/pkg/tuned" + "github.com/openshift/cluster-node-tuning-operator/version" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "k8s.io/klog/v2" +) + +type tunedOpts struct { + inCluster bool +} + +func NewTunedCommand() *cobra.Command { + tunedOpts := tunedOpts{} + + cmd := &cobra.Command{ + Use: version.OperandFilename, + Short: "Start NTO operand", + Run: func(cmd *cobra.Command, args []string) { + if err := tunedOpts.Validate(); err != nil { + klog.Fatal(err) + } + + if err := tunedOpts.Run(); err != nil { + klog.Fatal(err) + } + }, + } + + addKlogFlags(cmd) + tunedOpts.AddFlags(cmd.Flags()) + return cmd +} + +func (t *tunedOpts) AddFlags(fs *pflag.FlagSet) { + fs.BoolVar(&t.inCluster, "in-cluster", true, "In-cluster operand run.") +} + +func addKlogFlags(cmd *cobra.Command) { + fs := flag.NewFlagSet("", flag.PanicOnError) + klog.InitFlags(fs) + cmd.Flags().AddGoFlagSet(fs) +} + +func (t *tunedOpts) Validate() error { + return nil +} + +func (t *tunedOpts) Run() error { + return tunedOperandRun(t.inCluster) +} + +func tunedOperandRun(inCluster bool) error { + stopCh := signals.SetupSignalHandler() + return tuned.RunOperand(stopCh, version.Version, inCluster) +} diff --git a/pkg/tuned/controller.go b/pkg/tuned/controller.go index e3a4b7573..e2fab0a58 100644 --- a/pkg/tuned/controller.go +++ b/pkg/tuned/controller.go @@ -4,12 +4,10 @@ import ( "bufio" // scanner "bytes" // bytes.Buffer "context" // context.TODO() - "flag" // command-line options parsing "fmt" // Printf() "math" // math.Pow() "os" // os.Exit(), os.Stderr, ... "os/exec" // os.Exec() - "strconv" // strconv "strings" // strings.Join() "syscall" // syscall.SIGHUP, ... "time" // time.Second, ... @@ -73,9 +71,8 @@ const ( // TuneD logs before restarting TuneD and thus retrying the profile application. Keep this // reasonably low to workaround system/TuneD issues as soon as possible, but not too low // to increase the system load by retrying profile applications that can never succeed. - openshiftTunedHome = "/var/lib/tuned" + openshiftTunedHome = "/var/lib/ocp-tuned" openshiftTunedRunDir = "/run/" + programName - openshiftTunedPidFile = openshiftTunedRunDir + "/" + programName + ".pid" openshiftTunedProvider = openshiftTunedHome + "/provider" tunedInitialTimeout = 60 // timeout in seconds // With the less aggressive rate limiter, retries will happen at 100ms*2^(retry_n-1): @@ -108,6 +105,7 @@ type Daemon struct { // the TuneD profile we wish to be applied. recommendedProfile string } + type Controller struct { kubeconfig *restclient.Config kubeclient kubernetes.Interface @@ -150,19 +148,6 @@ type wqKey struct { name string // object name } -func parseCmdOpts() { - klog.InitFlags(nil) - flag.Usage = func() { - fmt.Fprintf(os.Stderr, "Usage: %s [options]\n", programName) - fmt.Fprintf(os.Stderr, "Example: %s\n\n", programName) - fmt.Fprintf(os.Stderr, "Options:\n") - - flag.PrintDefaults() - } - - flag.Parse() -} - // Get a client from kubelet's kubeconfig to write to the Node object. func getKubeClient() (kubernetes.Interface, error) { var ( @@ -478,21 +463,6 @@ func providerExtract(provider string) error { return nil } -func openshiftTunedPidFileWrite() error { - if err := util.Mkdir(openshiftTunedRunDir); err != nil { - return fmt.Errorf("failed to create %s run directory %q: %v", programName, openshiftTunedRunDir, err) - } - f, err := os.Create(openshiftTunedPidFile) - if err != nil { - return fmt.Errorf("failed to create %s pid file %q: %v", programName, openshiftTunedPidFile, err) - } - defer f.Close() - if _, err = f.WriteString(strconv.Itoa(os.Getpid())); err != nil { - return fmt.Errorf("failed to write %s pid file %q: %v", programName, openshiftTunedPidFile, err) - } - return nil -} - func TunedRecommendFileWrite(profileName string) error { klog.V(2).Infof("tunedRecommendFileWrite(): %s", profileName) if err := util.Mkdir(tunedRecommendDir); err != nil { @@ -1209,20 +1179,8 @@ func retryLoop(c *Controller) (err error) { } } -func Run(stopCh <-chan struct{}, boolVersion *bool, version string) { - klog.Infof("starting %s %s", programName, version) - parseCmdOpts() - - if *boolVersion { - fmt.Fprintf(os.Stderr, "%s %s\n", programName, version) - os.Exit(0) - } - - err := openshiftTunedPidFileWrite() - if err != nil { - // openshift-tuned PID file is not really used by anything, remove it in the future? - panic(err.Error()) - } +func RunOperand(stopCh <-chan struct{}, version string, inCluster bool) error { + klog.Infof("starting %s %s; in-cluster: %v", programName, version, inCluster) c, err := newController(stopCh) if err != nil { @@ -1230,8 +1188,5 @@ func Run(stopCh <-chan struct{}, boolVersion *bool, version string) { panic(err.Error()) } - err = retryLoop(c) - if err != nil { - panic(err.Error()) - } + return retryLoop(c) } diff --git a/pkg/tuned/run.go b/pkg/tuned/run.go index 9b52bc244..d4765a4b7 100644 --- a/pkg/tuned/run.go +++ b/pkg/tuned/run.go @@ -83,6 +83,7 @@ func TunedRunNoDaemon(cmd *exec.Cmd) error { onDaemonReload := func() {} return TunedRun(cmd, &daemon, onDaemonReload) } + func TunedRun(cmd *exec.Cmd, daemon *Daemon, onDaemonReload func()) error { klog.Infof("running cmd...") @@ -141,12 +142,6 @@ func TunedRun(cmd *exec.Cmd, daemon *Daemon, onDaemonReload func()) error { if daemon.reloaded { klog.V(2).Infof("profile applied or reload failed, stopping the TuneD watcher") onDaemonReload() - // c.tunedTimeout = tunedInitialTimeout // initialize the timeout - // c.daemon.status &= ^scTimeout // clear the scTimeout status bit - // c.tunedTicker.Stop() // profile applied or reload failed, stop the TuneD watcher - - // // Notify the event processor that the TuneD daemon finished reloading. - // c.wqTuneD.Add(wqKey{kind: wqKindDaemon}) } } }