Fix helm chart template value and add pre-commit hooks

.github/workflows/turing.yaml

@@ -39,7 +39,7 @@ on:
 env:
   ARTIFACT_RETENTION_DAYS: 7
   GO_VERSION: 1.18
-  GO_LINT_VERSION: v1.45.2
+  GO_LINT_VERSION: v1.48.0
   CLUSTER_NAME: turing-e2e
   ISTIO_VERSION: 1.9.9
   KNATIVE_VERSION: v0.18.3

.golangci.yml

@@ -12,12 +12,12 @@ linters:
     - gocyclo
     - gofmt
     - goimports
-    - golint
     - gosimple
     - govet
     - ineffassign
     - lll
     - misspell
+    - revive
     - staticcheck
     - structcheck
     - unused

.pre-commit-config.yaml

@@ -0,0 +1,77 @@
+repos:
+  - repo: https://github.com/norwoodj/helm-docs
+    rev: v1.11.0
+    hooks:
+      - id: helm-docs-built
+        name: 'Helm Docs for Turing Chart'
+        files: '^infra/charts/turing/'
+        args:
+          - --chart-search-root=infra/charts/turing
+          - --template-files=./README.md.gotmpl
+      - id: helm-docs-built
+        name: 'Helm Docs for Turing Init Chart'
+        files: '^infra/charts/turing-init/'
+        args:
+          - --chart-search-root=infra/charts/turing-init
+          - --template-files=./README.md.gotmpl
+  - repo: local
+    hooks:
+      - id: golangci-lint
+        name: 'Linter for API'
+        alias: golangci-lint-api
+        files: '^api/'
+        types: [go]
+        language: system
+        entry: bash -c 'cd api make lint'
+      - id: golangci-lint
+        name: 'Linter for the Router Engine'
+        alias: golangci-lint-router
+        files: '^engines/router/'
+        types: [go]
+        language: system
+        entry: bash -c 'cd engines/router make lint'
+      - id: golangci-lint
+        name: 'Linter for the Experiment Engine'
+        alias: golangci-lint-experiment
+        files: '^engines/experiment/'
+        types: [go]
+        language: system
+        entry: bash -c 'cd engines/experiment make lint'
+  - repo: local
+    hooks:
+      - id: prettier
+        name: 'Prettier for UI'
+        files: '^ui/'
+        types_or: [javascript, css]
+        language: system
+        entry: bash -c 'cd ui && yarn lint'
+  - repo: local
+    hooks:
+      - id: openapi-gen
+        name: 'OpenAPI Python client'
+        files: '^api/api/specs/'
+        language: system
+        entry: bash -c 'cd sdk && make gen-client'
+  - repo: local
+    hooks:
+      - id: black
+        name: 'Black formatter for the SDK'
+        alias: black-sdk
+        files: '^sdk/'
+        types: [python]
+        language: system
+        entry: bash -c 'cd sdk && make lint'
+      - id: black
+        name: 'Black formatter for Pyfunc Ensembler Job Engine'
+        alias: black-ensembler-job
+        files: '^engines/pyfunc-ensembler-job/'
+        types: [python]
+        language: system
+        entry: bash -c 'cd engines/pyfunc-ensembler-job && make lint'
+      - id: black
+        name: 'Black formatter for Pyfunc Ensembler Service Engine'
+        alias: black-ensembler-service
+        files: '^engines/pyfunc-ensembler-service/'
+        types: [python]
+        language: system
+        entry: bash -c 'cd engines/pyfunc-ensembler-service && make lint'

Makefile

@@ -24,3 +24,10 @@ build-image: version
 version:
 	$(eval VERSION=$(if $(OVERWRITE_VERSION),$(OVERWRITE_VERSION),v$(shell scripts/vertagen/vertagen.sh)))
 	@echo "turing version:" $$VERSION
+
+.PHONY: setup
+setup:
+	@echo "Setting up dev tools..."
+	@test -x "$(which pre-commit)" || pip install pre-commit
+	@pre-commit install
+	@pre-commit install-hooks

README.md

@@ -59,6 +59,7 @@ in these getting started guide.
 - [Go](https://golang.org/dl/) v1.18
 - [Node.js](https://nodejs.org/en/download/) v14 and yarn 1.22.x
 - [Kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) v1.22 (optional)
+- [Python](https://www.python.org/downloads/) >=3.7
 - Local ports 80, 8080, 8081, 8082, 8200, 5000, 6443 should be available
 
 ### Download Turing Source Code
@@ -71,6 +72,14 @@ git clone https://github.com/gojek/turing.git
 export TURING=$PWD/turing
 ```
 
+### Set up Development Tools
+
+Install the necessary pre-commit hooks when cloning the project for the first time. Subsequent changes to to hooks will be synced automatically.
+
+```bash
+make setup
+```
+
 ### Setup Local Infrastructure with Docker Compose
 
 Start all the required background services in Docker compose.

api/Makefile

@@ -15,7 +15,7 @@ clean:
 setup:
 	@echo "Setting up tools..."
 	@test -x $(shell go env GOPATH)/bin/golangci-lint || \
-		curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/v1.46.2/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.46.2
+		curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/v1.48.0/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.48.0
 
 .PHONY: fmt
 fmt:

api/e2e/test/helpers_api_test.go

@@ -159,7 +159,8 @@ func getPodLogs(t *testing.T, resp *http.Response) []service.PodLog {
 // function, that is passed as an argument
 //
 // cleanup - optional function, that can be used for cleaning up some resources from
-//           the cluster, after assertion of the router is done
+//
+//	the cluster, after assertion of the router is done
 func withDeployedRouter(
 	t *testing.T,
 	routerPayload []byte,

api/turing/api/pod_log_api.go

@@ -19,9 +19,9 @@ type PodLogController struct {
 
 // ListRouterPodLogs handles the HTTP request for getting Router Pod Logs
 // It supports 3 types of pods:
-//   1. Router
-//   2. Enricher
-//   3. Ensembler
+//  1. Router
+//  2. Enricher
+//  3. Ensembler
 func (c PodLogController) ListRouterPodLogs(_ *http.Request, vars RequestVars, _ interface{}) *Response {
 	// Parse input
 	var errResp *Response
@@ -88,9 +88,9 @@ func (c PodLogController) ListRouterPodLogs(_ *http.Request, vars RequestVars, _
 
 // ListEnsemblingJobPodLogs handles the HTTP request for getting Ensembling Pod Logs
 // It supports 3 types of pods:
-//   1. Image Builder
-//   2. Spark Driver
-//   3. Spark Executor
+//  1. Image Builder
+//  2. Spark Driver
+//  3. Spark Executor
 func (c PodLogController) ListEnsemblingJobPodLogs(_ *http.Request, vars RequestVars, _ interface{}) *Response {
 	var errResp *Response
 	var project *mlp.Project

api/turing/models/alert.go

@@ -26,7 +26,6 @@ import (
 //
 // Environment, Team, Service, Metric and Duration are required in order to generate the correct
 // query for the metric values and to direct the alert to the correct recipients.
-//
 type Alert struct {
 	Model
 

api/turing/models/ensembling_job.go

@@ -84,23 +84,29 @@ func (r *InfraConfig) Scan(value interface{}) error {
 // Status is the state of the finite machine ensembling job.
 // Possible statuses:
 // JobPending --▶ JobFailedSubmission
-//     |
-//     |
-//     |
-//     |
+//
+//	|
+//	|
+//	|
+//	|
+//
 // JobBuildingImage --▶ JobFailedBuildImage
-//     |
-//     |
-//     |
-//     |
-//     ▼
+//
+//	|
+//	|
+//	|
+//	|
+//	▼
+//
 // JobRunning --▶ JobFailed
-//     |
-//     |
-//     |--▶ JobTerminating --▶ JobTerminated
-//     |
-//     |
-//     ▼
+//
+//	|
+//	|
+//	|--▶ JobTerminating --▶ JobTerminated
+//	|
+//	|
+//	▼
+//
 // JobCompleted
 type Status string
 

api/turing/service/alert_service.go

@@ -70,7 +70,6 @@ type DashboardURLValue struct {
 // alert repository in GitLab. In most cases, the alerts persisted in the database will be in sync
 // with the alert files in GitLab (as long as the Git files are not manually modified i.e.
 // the alert files are only updated by calling this service).
-//
 func NewGitlabOpsAlertService(db *gorm.DB, config config.AlertConfig) (AlertService, error) {
 	if config.GitLab == nil {
 		return nil, errors.New("missing GitLab AlertConfig")

engines/experiment/Makefile

@@ -7,7 +7,7 @@ default: test
 setup:
 	@echo "Setting up tools..."
 	@test -x $(shell go env GOPATH)/bin/golangci-lint || \
-		curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/v1.46.2/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.46.2
+		curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/v1.48.0/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.48.0
 
 .PHONY: tidy
 tidy:

engines/experiment/factory.go

@@ -19,8 +19,9 @@ type EngineFactory interface {
 
 // NewEngineFactory is a constructor method that creates a new instance of EngineFactory
 // The concrete implementation of EngineFactory can be either:
-// 	- experiment/plugin/inproc/factory (for experiment engines implemented as compile-time plugins)
-//  - experiment/plugin/rpc/factory (for experiment engines implemented as external net/rpc plugins)
+//   - experiment/plugin/inproc/factory (for experiment engines implemented as compile-time plugins)
+//   - experiment/plugin/rpc/factory (for experiment engines implemented as external net/rpc plugins)
+//
 // The actual implementation is determined based on provided engine configuration (passed via `cfg`)
 func NewEngineFactory(name string, cfg map[string]interface{}, logger *zap.SugaredLogger) (EngineFactory, error) {
 	var engineCfg config.EngineConfig

engines/experiment/pkg/request/request.go

@@ -37,10 +37,13 @@ func GetFieldSource(srcString string) (FieldSource, error) {
 // reqHeader - request header
 // bodyBytes - request JSON payload
 // fieldSrc - source of data, where the given key will be looked in,
-//			  one of `PayloadFieldSource` | `HeaderFieldSource`
+//
+//	one of `PayloadFieldSource` | `HeaderFieldSource`
+//
 // field - if `fieldSrc` is `HeaderFieldSource` - name of request header
-//		   if `fieldSrc` is `PayloadFieldSource` - json path to the value that should
-//         be extracted from the request payload
+//
+//			   if `fieldSrc` is `PayloadFieldSource` - json path to the value that should
+//	        be extracted from the request payload
 func GetValueFromRequest(
 	reqHeader http.Header,
 	bodyBytes []byte,

engines/pyfunc-ensembler-job/Makefile

@@ -20,6 +20,12 @@ type-check:
 		--allow-untyped-globals \
 		ensembler
 
+.PHONY: lint
+lint:
+	@$(eval black_version=$(shell pip show black | grep Version | grep -oE '[0-9].*'))
+	@[ "${black_version}" == "22.6.0" ] || sh -c "pip install black==22.6.0"
+	@black .
+
 .PHONY: test
 test: type-check
 	@$(ACTIVATE_ENV) && \

engines/pyfunc-ensembler-job/ensembler/__init__.py

@@ -7,9 +7,10 @@
 
 
 def build_spark_session(
-        app_name: str,
-        spark_config: DefaultDict[str, str] = None,
-        hadoop_config: DefaultDict[str, str] = None) -> SparkSession:
+    app_name: str,
+    spark_config: DefaultDict[str, str] = None,
+    hadoop_config: DefaultDict[str, str] = None,
+) -> SparkSession:
     conf = SparkConf()
     if spark_config:
         conf.setAll(spark_config.items())
@@ -20,44 +21,38 @@ def build_spark_session(
         for k, v in hadoop_config.items():
             sc._jsc.hadoopConfiguration().set(k, v)
 
-    return SparkSession.builder \
-        .appName(app_name) \
-        .config(conf=sc.getConf()) \
-        .getOrCreate()
+    return (
+        SparkSession.builder.appName(app_name).config(conf=sc.getConf()).getOrCreate()
+    )
 
 
 class SparkApplication:
-    _ANNOTATION_GROUP_SPARK = 'spark'
-    _ANNOTATION_GROUP_HADOOP = 'hadoopConfiguration'
+    _ANNOTATION_GROUP_SPARK = "spark"
+    _ANNOTATION_GROUP_HADOOP = "hadoopConfiguration"
 
     def __init__(self, args):
         self._job, raw_config = BatchEnsemblingJob.from_yaml(args.job_spec)
-        self._logger = logging.getLogger('SparkApplication')
+        self._logger = logging.getLogger("SparkApplication")
         self._logger.debug(
-            'Job Specification:\n'
-            '===============================================================\n'
-            '%s\n'
-            '===============================================================\n',
-            raw_config
+            "Job Specification:\n"
+            "===============================================================\n"
+            "%s\n"
+            "===============================================================\n",
+            raw_config,
         )
 
     def run(self):
         annotations = {}
         for name, value in self._job.annotations().items():
-            group, *keys = name.split('/', 1)
-            annotations[group] = {
-                **annotations.get(group, {}),
-                ''.join(keys): value
-            }
+            group, *keys = name.split("/", 1)
+            annotations[group] = {**annotations.get(group, {}), "".join(keys): value}
         annotations.update()
         spark = build_spark_session(
             self._job.name(),
             annotations.get(self._ANNOTATION_GROUP_SPARK),
-            annotations.get(self._ANNOTATION_GROUP_HADOOP)
+            annotations.get(self._ANNOTATION_GROUP_HADOOP),
         )
         self._job.run(spark)
 
 
-__all__ = [
-    'SparkApplication'
-]
+__all__ = ["SparkApplication"]