From 00246981856d0fe50170f7fbf2e5cfeca60d05b4 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Wed, 5 Apr 2023 19:33:52 +0530 Subject: [PATCH 1/2] feat: move docker folder to container --- CHANGELOG.md | 10 +++++----- cmd/tools/generate/generate.go | 4 ++-- {docker => container}/containerd/README.md | 6 +++--- {docker => container}/k8/README.md | 0 {docker => container}/onePollerPerContainer/Dockerfile | 0 {docker => container}/onePollerPerContainer/README.md | 4 ++-- .../onePollerPerContainer/docker-compose.tmpl | 0 {docker => container}/podman/README.md | 0 {docker => container}/prometheus/alert_rules.yml | 0 {docker => container}/prometheus/ems_alert_rules.yml | 0 {docker => container}/prometheus/prometheus.yml | 0 deb/build-deb.sh | 2 +- docs/install/containers.md | 8 ++++---- docs/prometheus-exporter.md | 4 ++-- integration/Jenkinsfile | 6 +++--- integration/test/installer/prometheus.go | 2 +- jenkins/artifacts/jenkinsfile | 4 ++-- prom-stack.tmpl | 2 +- rpm/build-rpm.sh | 2 +- 19 files changed, 27 insertions(+), 27 deletions(-) rename {docker => container}/containerd/README.md (91%) rename {docker => container}/k8/README.md (100%) rename {docker => container}/onePollerPerContainer/Dockerfile (100%) rename {docker => container}/onePollerPerContainer/README.md (93%) rename {docker => container}/onePollerPerContainer/docker-compose.tmpl (100%) rename {docker => container}/podman/README.md (100%) rename {docker => container}/prometheus/alert_rules.yml (100%) rename {docker => container}/prometheus/ems_alert_rules.yml (100%) rename {docker => container}/prometheus/prometheus.yml (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index f722cd884..7b11f0d10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -742,7 +742,7 @@ The Unix collector is unable to monitor pollers running in containers. See [#249 - New `include_contains` plugin allows you to select a subset of objects. e.g. selecting only volumes with custom-defined ONTAP metadata -- We've included more out-of-the-box [Prometheus alerts](https://github.com/NetApp/harvest/blob/main/docker/prometheus/alert_rules.yml). Keep sharing your most useful alerts! +- We've included more out-of-the-box [Prometheus alerts](https://github.com/NetApp/harvest/blob/main/container/prometheus/alert_rules.yml). Keep sharing your most useful alerts! - 7mode workflows continue to be improved :heart: Harvest now collects Qtree and Quotas counters from 7mode filers (these are already collected in cDOT) @@ -775,7 +775,7 @@ The Unix collector is unable to monitor pollers running in containers. See [#249 - Harvest should provide a `include_contains` label agent plugin for filtering #735 Thanks to @chadpruden for reporting -- Improve Harvest's container compatibility with K8s via kompose. [#655](https://github.com/NetApp/harvest/pull/655) See [also](https://github.com/NetApp/harvest/tree/main/docker/k8) and [discussion](https://github.com/NetApp/harvest/discussions/827) +- Improve Harvest's container compatibility with K8s via kompose. [#655](https://github.com/NetApp/harvest/pull/655) See [also](https://github.com/NetApp/harvest/tree/main/container/k8) and [discussion](https://github.com/NetApp/harvest/discussions/827) - The ZAPI cli tool should include counter types when querying ZAPIs #663 @@ -786,13 +786,13 @@ The output of one plugin can be fed into the input of the next one. #736 Thanks - Harvest should collect Antivirus counters when ONTAP offbox vscan is configured [#346](https://github.com/NetApp/harvest/issues/346) Thanks to @burkl and @Falcon667 for reporting -- [Document](https://github.com/NetApp/harvest/tree/main/docker/containerd) how to run Harvest with `containerd` and `Rancher` +- [Document](https://github.com/NetApp/harvest/tree/main/container/containerd) how to run Harvest with `containerd` and `Rancher` - Qtree counters should be collected for 7-mode filers #766 Thanks to @jmg011 for raising this issue and iterating with us on a solution - Harvest admin node should work with pollers running in Docker compose [#678](https://github.com/NetApp/harvest/pull/678) -- [Document](https://github.com/NetApp/harvest/tree/main/docker/podman) how to run Harvest with Podman. Several RHEL customers asked about this since Podman ships as the default container runtime on RHEL8+. +- [Document](https://github.com/NetApp/harvest/tree/main/container/podman) how to run Harvest with Podman. Several RHEL customers asked about this since Podman ships as the default container runtime on RHEL8+. - Harvest should include a Systemd service file for the HTTP service discovery admin node [#656](https://github.com/NetApp/harvest/pull/656) @@ -1150,7 +1150,7 @@ On RHEL and Debian, the example Unix collector does not work at the moment due t - Improve client certificate authentication [documentation](https://github.com/NetApp/harvest/issues/314#issuecomment-882120238) -- Improve Docker deployment documentation [4019308](https://github.com/NetApp/harvest/tree/main/docker/onePollerPerContainer) +- Improve Docker deployment documentation [4019308](https://github.com/NetApp/harvest/tree/main/container/onePollerPerContainer) ### Fixes diff --git a/cmd/tools/generate/generate.go b/cmd/tools/generate/generate.go index d417be399..f6ed4b1c9 100644 --- a/cmd/tools/generate/generate.go +++ b/cmd/tools/generate/generate.go @@ -182,7 +182,7 @@ func generateDocker(path string, kind int) { filesd = append(filesd, fmt.Sprintf("- targets: ['%s:%d']", pollerInfo.ServiceName, pollerInfo.Port)) } - t, err := template.New("docker-compose.tmpl").ParseFiles("docker/onePollerPerContainer/docker-compose.tmpl") + t, err := template.New("docker-compose.tmpl").ParseFiles("container/onePollerPerContainer/docker-compose.tmpl") if err != nil { panic(err) } @@ -404,7 +404,7 @@ func init() { dFlags.BoolVarP(&opts.showPorts, "port", "p", false, "Expose poller ports to host machine") _ = dockerCmd.MarkPersistentFlagRequired("output") - fFlags.StringVar(&opts.filesdPath, "filesdpath", "docker/prometheus/harvest_targets.yml", + fFlags.StringVar(&opts.filesdPath, "filesdpath", "container/prometheus/harvest_targets.yml", "Prometheus file_sd target path. Written when the --output is set") fFlags.IntVar(&opts.promPort, "promPort", 9090, "Prometheus Port") fFlags.IntVar(&opts.grafanaPort, "grafanaPort", 3000, "Grafana Port") diff --git a/docker/containerd/README.md b/container/containerd/README.md similarity index 91% rename from docker/containerd/README.md rename to container/containerd/README.md index 2c0be6a25..75789c702 100644 --- a/docker/containerd/README.md +++ b/container/containerd/README.md @@ -30,12 +30,12 @@ After Rancher Desktop installs, start it `Cmd + Space` type: Rancher and wait fo ## Create Harvest Docker image -You only need to [create a new image](https://github.com/NetApp/harvest/tree/main/docker/onePollerPerContainer#building-harvest-docker-image) if you've made changes to Harvest. If you just want to use the latest version of Harvest, skip this step. +You only need to [create a new image](https://github.com/NetApp/harvest/tree/main/container/onePollerPerContainer#building-harvest-docker-image) if you've made changes to Harvest. If you just want to use the latest version of Harvest, skip this step. -These are the same steps outline on [Building Harvest Docker Image](https://github.com/NetApp/harvest/tree/main/docker/onePollerPerContainer#building-harvest-docker-image) except we replace `docker build` with `nerdctl` like so: +These are the same steps outline on [Building Harvest Docker Image](https://github.com/NetApp/harvest/tree/main/container/onePollerPerContainer#building-harvest-docker-image) except we replace `docker build` with `nerdctl` like so: ```sh -nerdctl build -f docker/onePollerPerContainer/Dockerfile -t harvest:latest . --no-cache +nerdctl build -f container/onePollerPerContainer/Dockerfile -t harvest:latest . --no-cache ``` ## Generate a Harvest compose file diff --git a/docker/k8/README.md b/container/k8/README.md similarity index 100% rename from docker/k8/README.md rename to container/k8/README.md diff --git a/docker/onePollerPerContainer/Dockerfile b/container/onePollerPerContainer/Dockerfile similarity index 100% rename from docker/onePollerPerContainer/Dockerfile rename to container/onePollerPerContainer/Dockerfile diff --git a/docker/onePollerPerContainer/README.md b/container/onePollerPerContainer/README.md similarity index 93% rename from docker/onePollerPerContainer/README.md rename to container/onePollerPerContainer/README.md index 70579b7b1..88783c8c2 100644 --- a/docker/onePollerPerContainer/README.md +++ b/container/onePollerPerContainer/README.md @@ -55,7 +55,7 @@ You only need to build a Harvest Docker image if you want a custom image. If ins Build Docker Image ``` -docker build -f docker/onePollerPerContainer/Dockerfile -t harvest:latest . --no-cache +docker build -f container/onePollerPerContainer/Dockerfile -t harvest:latest . --no-cache or -nerdctl build -f docker/onePollerPerContainer/Dockerfile -t harvest:latest . --no-cache +nerdctl build -f container/onePollerPerContainer/Dockerfile -t harvest:latest . --no-cache ``` diff --git a/docker/onePollerPerContainer/docker-compose.tmpl b/container/onePollerPerContainer/docker-compose.tmpl similarity index 100% rename from docker/onePollerPerContainer/docker-compose.tmpl rename to container/onePollerPerContainer/docker-compose.tmpl diff --git a/docker/podman/README.md b/container/podman/README.md similarity index 100% rename from docker/podman/README.md rename to container/podman/README.md diff --git a/docker/prometheus/alert_rules.yml b/container/prometheus/alert_rules.yml similarity index 100% rename from docker/prometheus/alert_rules.yml rename to container/prometheus/alert_rules.yml diff --git a/docker/prometheus/ems_alert_rules.yml b/container/prometheus/ems_alert_rules.yml similarity index 100% rename from docker/prometheus/ems_alert_rules.yml rename to container/prometheus/ems_alert_rules.yml diff --git a/docker/prometheus/prometheus.yml b/container/prometheus/prometheus.yml similarity index 100% rename from docker/prometheus/prometheus.yml rename to container/prometheus/prometheus.yml diff --git a/deb/build-deb.sh b/deb/build-deb.sh index 83bca49d5..d3dad0d15 100755 --- a/deb/build-deb.sh +++ b/deb/build-deb.sh @@ -23,7 +23,7 @@ mkdir -p "$BUILD/opt/harvest/bin/" cp -r "$SRC/grafana" "$SRC/conf" "$BUILD/opt/harvest/" cp "$SRC/harvest.yml" "$SRC/prom-stack.tmpl" "$SRC/harvest.cue" "$BUILD/opt/harvest/" cp -r "$SRC/.github/" "$BUILD/opt/harvest/" -cp -r "$SRC/pkg/" "$SRC/cmd/" "$SRC/docs/" "$SRC/docker/" "$BUILD/opt/harvest/" +cp -r "$SRC/pkg/" "$SRC/cmd/" "$SRC/docs/" "$SRC/container/" "$BUILD/opt/harvest/" cp -r "$SRC/rpm/" "$SRC/deb/" "$SRC/service/" "$SRC/cert/" "$SRC/autosupport/" "$SRC/.git" "$BUILD/opt/harvest/" cp "$SRC/Makefile" "$SRC/README.md" "$SRC/LICENSE" "$SRC/go.mod" "$SRC/go.sum" "$BUILD/opt/harvest/" if [ -d "$SRC/vendor" ]; then diff --git a/docs/install/containers.md b/docs/install/containers.md index 289e864cb..23eea0a3f 100644 --- a/docs/install/containers.md +++ b/docs/install/containers.md @@ -5,7 +5,7 @@ Harvest is container-ready and supports several deployment options: - [Stand-up Prometheus, Grafana, and Harvest via Docker Compose](#docker-compose). Choose this if you want to hit the ground running. Install, volume and network mounts automatically handled. -- [Poller-per-container model](https://github.com/NetApp/harvest/tree/main/docker/onePollerPerContainer) that offers +- [Poller-per-container model](https://github.com/NetApp/harvest/tree/main/container/onePollerPerContainer) that offers more flexibility in configuration. This deployment enables a broad range of orchestrators (Nomad, Mesosphere, Swarm, K8, etc.) since you pick-and-choose what gets built and how it's deployed, stronger familiarity with containers is recommended. @@ -15,10 +15,10 @@ Harvest is container-ready and supports several deployment options: stands up Harvest, Grafana, and Prometheus. - Want to run Harvest on a Mac - via [containerd and Racher Desktop](https://github.com/NetApp/harvest/tree/main/docker/containerd)? We got you + via [containerd and Racher Desktop](https://github.com/NetApp/harvest/tree/main/container/containerd)? We got you covered. -- [Local K8 Deployment](https://github.com/NetApp/harvest/blob/main/docker/k8/README.md) via Kompose +- [Local K8 Deployment](https://github.com/NetApp/harvest/blob/main/container/k8/README.md) via Kompose ## Docker Compose @@ -80,7 +80,7 @@ bin/harvest generate docker full --output harvest-compose.yml 1. Creates a Docker compose file with a container for each Harvest poller defined in your `harvest.yml` 2. Creates a matching Prometheus service discovery file for each Harvest poller (located - in `docker/prometheus/harvest_targets.yml`). Prometheus uses this file to scrape the Harvest pollers. + in `container/prometheus/harvest_targets.yml`). Prometheus uses this file to scrape the Harvest pollers. ### Start everything diff --git a/docs/prometheus-exporter.md b/docs/prometheus-exporter.md index b7cb85f62..c7bd1bfdd 100644 --- a/docs/prometheus-exporter.md +++ b/docs/prometheus-exporter.md @@ -303,8 +303,8 @@ Prometheus includes out-of-the-box support for simple alerting. Alert rules are file. Setup and details can be found in the Prometheus guide on [alerting](https://prometheus.io/docs/practices/alerting/). -Harvest also includes [ems alerts](https://github.com/NetApp/harvest/blob/main/docker/prometheus/ems_alert_rules.yml) -and [sample alerts](https://github.com/NetApp/harvest/blob/main/docker/prometheus/alert_rules.yml) for reference. +Harvest also includes [ems alerts](https://github.com/NetApp/harvest/blob/main/container/prometheus/ems_alert_rules.yml) +and [sample alerts](https://github.com/NetApp/harvest/blob/main/container/prometheus/alert_rules.yml) for reference. Refer [EMS Collector](https://github.com/NetApp/harvest/blob/main/cmd/collectors/ems/README.md) for more details about EMS events. diff --git a/integration/Jenkinsfile b/integration/Jenkinsfile index 08a925f8d..95bb9677d 100644 --- a/integration/Jenkinsfile +++ b/integration/Jenkinsfile @@ -52,7 +52,7 @@ pipeline { echo "Installing" export BUILD_PATH=$NATIVE export VERSION=$VERSION - cp /u/mpeg/harvest/harvest_targets.yml $WORKSPACE/harvest/docker/prometheus/ + cp /u/mpeg/harvest/harvest_targets.yml $WORKSPACE/harvest/container/prometheus/ bash $WORKSPACE/harvest/integration/test/test.sh install_native ''' } @@ -104,7 +104,7 @@ pipeline { echo "Installing" export BUILD_PATH=http://harvest-jenkins.rtp.openenglab.netapp.com:8080/job/harvest2_0/job/artifcats/lastSuccessfulBuild/artifact/harvest-21.11.0-1.x86_64.rpm export VERSION=$VERSION - cp /u/mpeg/harvest/harvest_targets.yml $WORKSPACE/harvest/docker/prometheus/ + cp /u/mpeg/harvest/harvest_targets.yml $WORKSPACE/harvest/container/prometheus/ bash $WORKSPACE/harvest/integration/test/test.sh install_rpm ''' } @@ -165,7 +165,7 @@ pipeline { ./bin/harvest stop ./bin/harvest generate docker full --port --output harvest-compose.yml docker login -u $DOCKER_USER -p $DOCKER_PWD - docker build -f docker/onePollerPerContainer/Dockerfile -t rahulguptajss/harvest:latest -t cr.netapp.io/harvest:latest -t ghcr.io/netapp/harvest:latest . --no-cache --build-arg VERSION=$VERSION + docker build -f container/onePollerPerContainer/Dockerfile -t rahulguptajss/harvest:latest -t cr.netapp.io/harvest:latest -t ghcr.io/netapp/harvest:latest . --no-cache --build-arg VERSION=$VERSION docker-compose -f prom-stack.yml -f harvest-compose.yml up -d --remove-orphans sleep 1m bash $WORKSPACE/harvest/integration/test/test.sh install_docker diff --git a/integration/test/installer/prometheus.go b/integration/test/installer/prometheus.go index 1d2602465..52f57da8c 100644 --- a/integration/test/installer/prometheus.go +++ b/integration/test/installer/prometheus.go @@ -28,7 +28,7 @@ func (p *Prometheus) Install() bool { ipAddress := utils.GetOutboundIP() cmd := exec.Command("docker", "run", "-d", "-p", utils.PrometheusPort+":"+utils.PrometheusPort, "--add-host=localhost:"+ipAddress, - "-v", path+"/../../docker/prometheus/:/etc/prometheus/", + "-v", path+"/../../container/prometheus/:/etc/prometheus/", p.image) cmd.Stdout = os.Stdout err := cmd.Start() diff --git a/jenkins/artifacts/jenkinsfile b/jenkins/artifacts/jenkinsfile index a822ba6ce..eeca4933c 100644 --- a/jenkins/artifacts/jenkinsfile +++ b/jenkins/artifacts/jenkinsfile @@ -132,7 +132,7 @@ pipeline { sh ''' targetLocation=$targetParentLocation$VERSION-$RELEASE-$BRANCH docker login - docker build -f docker/onePollerPerContainer/Dockerfile --build-arg VERSION=$VERSION --build-arg RELEASE=$RELEASE --build-arg ASUP_MAKE_TARGET=$ASUP_MAKE_TARGET --build-arg GIT_TOKEN=$GIT_TOKEN -t ${imageName}:latest -t ${imageName}:$VERSION-$RELEASE -t ${jfrogImagePrefix}:latest -t ${jfrogImagePrefix}:$VERSION-$RELEASE -t ${ghcrImage}:latest -t ${ghcrImage}:$VERSION-$RELEASE . --no-cache + docker build -f container/onePollerPerContainer/Dockerfile --build-arg VERSION=$VERSION --build-arg RELEASE=$RELEASE --build-arg ASUP_MAKE_TARGET=$ASUP_MAKE_TARGET --build-arg GIT_TOKEN=$GIT_TOKEN -t ${imageName}:latest -t ${imageName}:$VERSION-$RELEASE -t ${jfrogImagePrefix}:latest -t ${jfrogImagePrefix}:$VERSION-$RELEASE -t ${ghcrImage}:latest -t ${ghcrImage}:$VERSION-$RELEASE . --no-cache docker save -o ${targetLocation}/docker_harvest.tar ${imageName}:latest ''' } @@ -221,7 +221,7 @@ pipeline { git push https://$GIT_TOKEN@github.com/NetApp/harvest.git --delete nightly fi /opt/home/gh_2.8.0_linux_386/bin/gh release create nightly $targetLocation/*.rpm $targetLocation/*.deb $targetLocation/*.gz --notes "Nightly builds may include bugs and other issues. You might want to use the stable releases instead." --title "Harvest Nightly Release" --prerelease --target main - docker build -f docker/onePollerPerContainer/Dockerfile --build-arg VERSION=$VERSION --build-arg RELEASE=$RELEASE --build-arg ASUP_MAKE_TARGET=$ASUP_MAKE_TARGET --build-arg GIT_TOKEN=$GIT_TOKEN -t ${imageName}:latest -t ${imageName}:nightly -t ${jfrogImagePrefix}:latest -t ${jfrogImagePrefix}:nightly -t ${ghcrImage}:latest -t ${ghcrImage}:nightly . --no-cache + docker build -f container/onePollerPerContainer/Dockerfile --build-arg VERSION=$VERSION --build-arg RELEASE=$RELEASE --build-arg ASUP_MAKE_TARGET=$ASUP_MAKE_TARGET --build-arg GIT_TOKEN=$GIT_TOKEN -t ${imageName}:latest -t ${imageName}:nightly -t ${jfrogImagePrefix}:latest -t ${jfrogImagePrefix}:nightly -t ${ghcrImage}:latest -t ${ghcrImage}:nightly . --no-cache echo $GIT_TOKEN | docker login ghcr.io -u $GIT_USERNAME --password-stdin docker push ${ghcrImage}:nightly ''' diff --git a/prom-stack.tmpl b/prom-stack.tmpl index 691726730..316655d4f 100644 --- a/prom-stack.tmpl +++ b/prom-stack.tmpl @@ -15,7 +15,7 @@ services: container_name: prometheus image: prom/prometheus:v2.33.1 volumes: - - ./docker/prometheus/:/etc/prometheus/ + - ./container/prometheus/:/etc/prometheus/ - prometheus_data:/prometheus command: - '--config.file=/etc/prometheus/prometheus.yml' diff --git a/rpm/build-rpm.sh b/rpm/build-rpm.sh index 7ee60a087..28382c0f6 100755 --- a/rpm/build-rpm.sh +++ b/rpm/build-rpm.sh @@ -19,7 +19,7 @@ mkdir -p "$BUILD/harvest/bin" cp -r "$SRC/.git" "$BUILD/harvest" cp -r "$SRC/cmd/" "$BUILD/harvest/" cp -r "$SRC/conf/" "$BUILD/harvest/" -cp -r "$SRC/docker/" "$BUILD/harvest/" +cp -r "$SRC/container/" "$BUILD/harvest/" cp -r "$SRC/docs/" "$BUILD/harvest/" cp -r "$SRC/grafana/" "$BUILD/harvest/" cp -r "$SRC/pkg/" "$BUILD/harvest/" From 4d227f347fe29ac2e29e55b277018e41c39c1ed3 Mon Sep 17 00:00:00 2001 From: Rahul Gupta Date: Wed, 5 Apr 2023 20:05:52 +0530 Subject: [PATCH 2/2] feat: move docker folder to container --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8fb3db0ea..024150581 100644 --- a/Makefile +++ b/Makefile @@ -143,7 +143,7 @@ dist-tar: @rm -rf ${DIST} @mkdir ${TMP} @mkdir ${DIST} - @cp -r .git cmd bin conf docker docs grafana pkg service cert autosupport go.mod go.sum Makefile README.md LICENSE prom-stack.tmpl harvest.cue ${TMP} + @cp -r .git cmd bin conf container docs grafana pkg service cert autosupport go.mod go.sum Makefile README.md LICENSE prom-stack.tmpl harvest.cue ${TMP} @cp harvest.yml ${TMP}/harvest.yml @tar --directory /tmp --create --gzip --file ${DIST}/${HARVEST_PACKAGE}.tar.gz ${HARVEST_PACKAGE} @rm -rf ${TMP}