Merge pull request #222 from AnalogJ/influxdb

.github/ISSUE_TEMPLATE/bug_report.md

@@ -18,6 +18,7 @@ If applicable, add screenshots to help explain your problem.
 
 **Log Files**
 If related to missing devices or SMART data, please run the `collector` in DEBUG mode, and attach the log file.
+See [/docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md](docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md) for other troubleshooting tips. 
 
 ```
 docker run -it --rm -p 8080:8080 \
@@ -29,13 +30,12 @@ docker run -it --rm -p 8080:8080 \
 -e COLLECTOR_LOG_FILE=/tmp/collector.log \
 -e SCRUTINY_LOG_FILE=/tmp/web.log \
 --name scrutiny \
-analogj/scrutiny
+ghcr.io/analogj/scrutiny:master-omnibus
 
 # in another terminal trigger the collector
 docker exec scrutiny scrutiny-collector-metrics run
 
 # then use docker cp to copy the log files out of the container.
 docker cp scrutiny:/tmp/collector.log collector.log
 docker cp scrutiny:/tmp/web.log web.log
-
 ```

.github/workflows/build.yaml

@@ -7,6 +7,20 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     container: techknowlogick/xgo:go-1.13.x
+
+    # Service containers to run with `build` (Required for end-to-end testing)
+    services:
+      influxdb:
+        image: influxdb:2.2
+        env:
+          DOCKER_INFLUXDB_INIT_MODE: setup
+          DOCKER_INFLUXDB_INIT_USERNAME: admin
+          DOCKER_INFLUXDB_INIT_PASSWORD: password12345
+          DOCKER_INFLUXDB_INIT_ORG: scrutiny
+          DOCKER_INFLUXDB_INIT_BUCKET: metrics
+          DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: my-super-secret-auth-token
+        ports:
+          - 8086:8086
     env:
       PROJECT_PATH: /go/src/github.com/analogj/scrutiny
       CGO_ENABLED: 1
@@ -26,6 +40,13 @@ jobs:
 
           go mod vendor
           go test -race -coverprofile=coverage.txt -covermode=atomic -v -tags "static" $(go list ./... | grep -v /vendor/)
+      - name: Generate coverage report
+        uses: codecov/codecov-action@v2
+        with:
+          files: ${{ env.PROJECT_PATH }}/coverage.txt
+          flags: unittests
+          fail_ci_if_error: true
+          verbose: true
       - name: Build Binaries
         run: |
 
@@ -49,9 +70,13 @@ jobs:
             /build/scrutiny-collector-metrics-linux-arm-7
             /build/scrutiny-web-windows-4.0-amd64.exe
             /build/scrutiny-collector-metrics-windows-4.0-amd64.exe
+#            /build/scrutiny-web-darwin-arm64
+#            /build/scrutiny-collector-metrics-darwin-arm64
+#            /build/scrutiny-web-darwin-amd64
+#            /build/scrutiny-collector-metrics-darwin-amd64
 #            /build/scrutiny-web-freebsd-amd64
 #            /build/scrutiny-collector-metrics-freebsd-amd64
-      - uses: codecov/codecov-action@v1
+      - uses: codecov/codecov-action@v2
         with:
           file: ${{ env.PROJECT_PATH }}/coverage.txt
           flags: unittests

.github/workflows/docker-build.yaml

@@ -11,7 +11,6 @@ env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
 
-
 jobs:
   collector:
     runs-on: ubuntu-latest
@@ -22,6 +21,10 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
@@ -46,6 +49,7 @@ jobs:
       - name: Build and push Docker image
         uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
         with:
+          platforms: linux/amd64,linux/arm64
           context: .
           file: docker/Dockerfile.collector
           push: ${{ github.event_name != 'pull_request' }}
@@ -61,6 +65,10 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
@@ -85,6 +93,7 @@ jobs:
       - name: Build and push Docker image
         uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
         with:
+          platforms: linux/amd64,linux/arm64
           context: .
           file: docker/Dockerfile.web
           push: ${{ github.event_name != 'pull_request' }}
@@ -100,6 +109,10 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
       # Login against a Docker registry except on PR
       # https://github.com/docker/login-action
       - name: Log into registry ${{ env.REGISTRY }}
@@ -124,6 +137,7 @@ jobs:
       - name: Build and push Docker image
         uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
         with:
+          platforms: linux/amd64,linux/arm64
           context: .
           file: docker/Dockerfile
           push: ${{ github.event_name != 'pull_request' }}

.github/workflows/release.yaml

@@ -212,3 +212,44 @@ jobs:
           asset_path: /build/scrutiny-collector-metrics-windows-4.0-amd64.exe
           asset_name: scrutiny-collector-metrics-windows-4.0-amd64.exe
           asset_content_type: application/octet-stream
+#
+#      - name: Release Asset - Web - darwin-arm64
+#        id: upload-release-asset13
+#        uses: actions/upload-release-asset@v1
+#        env:
+#          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
+#        with:
+#          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+#          asset_path: /build/scrutiny-web-darwin-arm64
+#          asset_name: scrutiny-web-darwin-arm64
+#          asset_content_type: application/octet-stream
+#      - name: Release Asset - Collector - darwin-arm64
+#        id: upload-release-asset14
+#        uses: actions/upload-release-asset@v1
+#        env:
+#          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
+#        with:
+#          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+#          asset_path: /build/scrutiny-collector-metrics-darwin-arm64
+#          asset_name: scrutiny-collector-metrics-darwin-arm64
+#          asset_content_type: application/octet-stream
+#      - name: Release Asset - Web - darwin-amd64
+#        id: upload-release-asset15
+#        uses: actions/upload-release-asset@v1
+#        env:
+#          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
+#        with:
+#          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+#          asset_path: /build/scrutiny-web-darwin-amd64
+#          asset_name: scrutiny-web-darwin-amd64
+#          asset_content_type: application/octet-stream
+#      - name: Release Asset - Collector - darwin-amd64
+#        id: upload-release-asset16
+#        uses: actions/upload-release-asset@v1
+#        env:
+#          GITHUB_TOKEN: ${{ secrets.SCRUTINY_GITHUB_TOKEN }}
+#        with:
+#          upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
+#          asset_path: /build/scrutiny-collector-metrics-darwin-amd64
+#          asset_name: scrutiny-collector-metrics-darwin-amd64
+#          asset_content_type: application/octet-stream

CONTRIBUTING.md

@@ -12,7 +12,7 @@ docker run -it --rm -p 8080:8080 \
 --cap-add SYS_RAWIO \
 --device=/dev/sda \
 --device=/dev/sdb \
-analogj/scrutiny
+ghcr.io/analogj/scrutiny:master-omnibus
 /scrutiny/bin/scrutiny-collector-metrics run
 ```
 
@@ -54,7 +54,8 @@ web:
   src:
     frontend:
       path: ./dist
-
+  influxdb:
+    retention_policy: false
 
 log:
   file: 'web.log' #absolute or relative paths allowed, eg. web.log
@@ -71,6 +72,39 @@ go run webapp/backend/cmd/scrutiny/scrutiny.go start --config ./scrutiny.yaml
 Now visit http://localhost:8080
 
 
+If you'd like to populate the database with some test data,  you can run the following commands:
+
+> NOTE: you may need to update the `local_time` key within the JSON file, any timestamps older than ~3 weeks will be automatically ignored
+> (since the downsampling & retention policy takes effect at 2 weeks)
+> This is done automatically by the `webapp/backend/pkg/models/testdata/helper.go` script
+
+```
+docker run -p 8086:8086 --rm influxdb:2.2
+
+
+docker run --rm -p 8086:8086 \
+      -e DOCKER_INFLUXDB_INIT_MODE=setup \
+      -e DOCKER_INFLUXDB_INIT_USERNAME=admin \
+      -e DOCKER_INFLUXDB_INIT_PASSWORD=password12345 \
+      -e DOCKER_INFLUXDB_INIT_ORG=scrutiny \
+      -e DOCKER_INFLUXDB_INIT_BUCKET=metrics \
+      influxdb:2.2
+
+
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/web/testdata/register-devices-req.json localhost:8080/api/devices/register
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-ata.json localhost:8080/api/device/0x5000cca264eb01d7/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-ata-date.json localhost:8080/api/device/0x5000cca264eb01d7/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-ata-date2.json localhost:8080/api/device/0x5000cca264eb01d7/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-fail2.json localhost:8080/api/device/0x5000cca264ec3183/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-nvme.json localhost:8080/api/device/0x5002538e40a22954/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-scsi.json localhost:8080/api/device/0x5000cca252c859cc/smart
+# curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/models/testdata/smart-scsi2.json localhost:8080/api/device/0x5000cca264ebc248/smart
+go run webapp/backend/pkg/models/testdata/helper.go
+
+curl localhost:8080/api/summary
+
+```
+
 ### Collector
 ```
 brew install smartmontools

Makefile

@@ -9,7 +9,6 @@ BINARY=\
   linux/arm-7   \
   linux/arm64   \
 
-
 .PHONY: all $(BINARY)
 all: $(BINARY) windows/amd64
 

README.md

@@ -72,40 +72,48 @@ If you're using Docker, getting started is as simple as running the following co
 
 ```bash
 docker run -it --rm -p 8080:8080 \
+  -v `pwd`/scrutiny:/scrutiny/config \
+  -v `pwd`/influxdb2:/scrutiny/influxdb \
   -v /run/udev:/run/udev:ro \
   --cap-add SYS_RAWIO \
   --device=/dev/sda \
   --device=/dev/sdb \
   --name scrutiny \
-  analogj/scrutiny
+  ghcr.io/analogj/scrutiny:master-omnibus
 ```
 
 - `/run/udev` is necessary to provide the Scrutiny collector with access to your device metadata
 - `--cap-add SYS_RAWIO` is necessary to allow `smartctl` permission to query your device SMART data
     - NOTE: If you have **NVMe** drives, you must add `--cap-add SYS_ADMIN` as well. See issue [#26](https://github.com/AnalogJ/scrutiny/issues/26#issuecomment-696817130)
 - `--device` entries are required to ensure that your hard disk devices are accessible within the container.
-- `analogj/scrutiny` is a omnibus image, containing both the webapp server (frontend & api) as well as the S.M.A.R.T metric collector. (see below)
+- `ghcr.io/analogj/scrutiny:master-omnibus` is a omnibus image, containing both the webapp server (frontend & api) as well as the S.M.A.R.T metric collector. (see below)
 
 ### Hub/Spoke Deployment
 
 In addition to the Omnibus image (available under the `latest` tag) there are 2 other Docker images available:
 
-- `analogj/scrutiny:collector` - Contains the Scrutiny data collector, `smartctl` binary and cron-like scheduler. You can run one collector on each server.
-- `analogj/scrutiny:web` - Contains the Web UI, API and Database. Only one container necessary
+- `ghcr.io/analogj/scrutiny:master-collector` - Contains the Scrutiny data collector, `smartctl` binary and cron-like scheduler. You can run one collector on each server.
+- `ghcr.io/analogj/scrutiny:master-web` - Contains the Web UI, API and Database. Only one container necessary
 
 ```bash
-docker run -it --rm -p 8080:8080 \
+docker run --rm -p 8086:8086 \
+  -v `pwd`/influxdb2:/var/lib/influxdb2 \
+  --name scrutiny-influxdb \
+  influxdb:2.2
+
+docker run --rm -p 8080:8080 \
+  -v `pwd`/scrutiny:/scrutiny/config \
   --name scrutiny-web \
-  analogj/scrutiny:web
+  ghcr.io/analogj/scrutiny:master-web
 
-docker run -it --rm \
+docker run --rm \
   -v /run/udev:/run/udev:ro \
   --cap-add SYS_RAWIO \
   --device=/dev/sda \
   --device=/dev/sdb \
   -e SCRUTINY_API_ENDPOINT=http://SCRUTINY_WEB_IPADDRESS:8080 \
   --name scrutiny-collector \
-  analogj/scrutiny:collector
+  ghcr.io/analogj/scrutiny:master-collector
 ```
 
 ## Manual Installation (without-Docker)
@@ -140,6 +148,13 @@ There are two configuration files available:
 
 Neither file is required, however if provided, it allows you to configure how Scrutiny functions.
 
+## Cron Schedule
+Unfortunately the Cron schedule cannot be configured via the `collector.yaml` (as the collector binary needs to be trigged by a scheduler/cron).
+However, if you are using the official `ghcr.io/analogj/scrutiny:master-collector` or `ghcr.io/analogj/scrutiny:master-omnibus` docker images, 
+you can use the `COLLECTOR_CRON_SCHEDULE` environmental variable to override the default cron schedule (daily @ midnight - `0 0 * * *`).
+
+`docker run -e COLLECTOR_CRON_SCHEDULE="0 0 * * *" ...`
+
 ## Notifications
 
 Scrutiny supports sending SMART device failure notifications via the following services:

collector/pkg/collector/base.go

@@ -40,6 +40,7 @@ func (c *BaseCollector) postJson(url string, body interface{}, target interface{
 	return json.NewDecoder(r.Body).Decode(target)
 }
 
+// http://www.linuxguide.it/command_line/linux-manpage/do.php?file=smartctl#sect7
 func (c *BaseCollector) LogSmartctlExitCode(exitCode int) {
 	if exitCode&0x01 != 0 {
 		c.logger.Errorln("smartctl could not parse commandline")

collector/pkg/collector/metrics.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"github.com/analogj/scrutiny/collector/pkg/common"
+	"github.com/analogj/scrutiny/collector/pkg/common/shell"
 	"github.com/analogj/scrutiny/collector/pkg/config"
 	"github.com/analogj/scrutiny/collector/pkg/detect"
 	"github.com/analogj/scrutiny/collector/pkg/errors"
@@ -20,6 +20,7 @@ type MetricsCollector struct {
 	config config.Interface
 	BaseCollector
 	apiEndpoint *url.URL
+	shell       shell.Interface
 }
 
 func CreateMetricsCollector(appConfig config.Interface, logger *logrus.Entry, apiEndpoint string) (MetricsCollector, error) {
@@ -34,6 +35,7 @@ func CreateMetricsCollector(appConfig config.Interface, logger *logrus.Entry, ap
 		BaseCollector: BaseCollector{
 			logger: logger,
 		},
+		shell: shell.Create(),
 	}
 
 	return sc, nil
@@ -107,6 +109,10 @@ func (mc *MetricsCollector) Validate() error {
 //func (mc *MetricsCollector) Collect(wg *sync.WaitGroup, deviceWWN string, deviceName string, deviceType string) {
 func (mc *MetricsCollector) Collect(deviceWWN string, deviceName string, deviceType string) {
 	//defer wg.Done()
+	if len(deviceWWN) == 0 {
+		mc.logger.Errorf("no device WWN detected for %s. Skipping collection for this device (no data association possible).\n", deviceName)
+		return
+	}
 	mc.logger.Infof("Collecting smartctl results for %s\n", deviceName)
 
 	args := []string{"-x", "-j"}
@@ -116,7 +122,7 @@ func (mc *MetricsCollector) Collect(deviceWWN string, deviceName string, deviceT
 	}
 	args = append(args, fmt.Sprintf("%s%s", detect.DevicePrefix(), deviceName))
 
-	result, err := common.ExecCmd(mc.logger, "smartctl", args, "", os.Environ())
+	result, err := mc.shell.Command(mc.logger, "smartctl", args, "", os.Environ())
 	resultBytes := []byte(result)
 	if err != nil {
 		if exitError, ok := err.(*exec.ExitError); ok {

collector/pkg/common/shell/factory.go

@@ -0,0 +1,5 @@
+package shell
+
+func Create() Interface {
+	return new(localShell)
+}

collector/pkg/common/shell/interface.go

@@ -0,0 +1,11 @@
+package shell
+
+import (
+	"github.com/sirupsen/logrus"
+)
+
+// Create mock using:
+// mockgen -source=collector/pkg/common/shell/interface.go -destination=collector/pkg/common/shell/mock/mock_shell.go
+type Interface interface {
+	Command(logger *logrus.Entry, cmdName string, cmdArgs []string, workingDir string, environ []string) (string, error)
+}