Merge m3aggregator into monorepo

m3db · Oct 9, 2018 · 440b416 · 440b416
1 parent cb3119e
commit 440b416
Show file tree

Hide file tree

Showing 143 changed files with 421 additions and 1,400 deletions.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -1,4 +1,12 @@
 steps:
+  - name: "Codegen"
+    command: make clean install-vendor test-all-gen
+    env:
+      CGO_ENABLED: 0
+      GIMME_GO_VERSION: 1.10.x
+    plugins:
+        gopath-checkout#v1.0.1:
+          import: github.com/m3db/m3
   - name: "Unit"
     command: make clean install-vendor test-ci-unit
     plugins:
@@ -9,14 +17,6 @@ steps:
     plugins:
       docker-compose#v2.5.1:
         run: app
-  - name: "Integration (collector, m3em), Codegen"
-    command: make clean install-vendor test-ci-integration-collector test-ci-integration-m3em test-all-gen
-    env:
-      CGO_ENABLED: 0
-      GIMME_GO_VERSION: 1.10.x
-    plugins:
-        gopath-checkout#v1.0.1:
-          import: github.com/m3db/m3
   - name: "Integration (:docker:), Services, Tools, Metalint"
     command: make install-vendor metalint docker-integration-test tools services
     env:
@@ -25,21 +25,26 @@ steps:
     plugins:
         gopath-checkout#v1.0.1:
           import: github.com/m3db/m3
-  - label: "Integration (All Metadata) %n"
+  - label: "Integration (dbnode All Metadata) %n"
     parallelism: 2
     command: make clean install-vendor test-ci-integration-dbnode cache_policy=all_metadata
     plugins:
       docker-compose#v2.5.1:
         run: app
-  - name: "Integration (Recently Read) %n"
+  - name: "Integration (dbnode Recently Read) %n"
     parallelism: 2
     command: make clean install-vendor test-ci-integration-dbnode cache_policy=recently_read
     plugins:
       docker-compose#v2.5.1:
         run: app
-  - name: "Integration (LRU) %n"
+  - name: "Integration (dbnode LRU) %n"
     parallelism: 2
     command: make clean install-vendor test-ci-integration-dbnode cache_policy=lru
     plugins:
       docker-compose#v2.5.1:
         run: app
+  - label: "Integration (collector, m3em, agg)"
+    command: make clean install-vendor test-ci-integration-collector test-ci-integration-m3em test-ci-integration-aggregator
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
diff --git a/.codecov.yml b/.codecov.yml
@@ -8,6 +8,8 @@ coverage:
       default: on
       x:
         flags: x
+      aggregator:
+        flags: aggregator
       collector:
         flags: collector
       query:
@@ -29,6 +31,10 @@ coverage:
     collector:
       paths:
         - src/collector/
+    aggregator:
+      paths:
+        - src/aggregator/
+        - src/cmd/services/m3aggregator
     query:
       paths:
         - src/query/

diff --git a/.excludecoverage b/.excludecoverage
@@ -1,3 +1,4 @@
+generic_.*.go
 _mock.go
 _gen.go
 _matcher.go

diff --git a/.excludemetalint b/.excludemetalint
@@ -6,4 +6,4 @@ generated/
 mocks/
 vendor/
 src/m3ninx/x/bytes/slice_arraypool_gen.go
-src/m3ninx/index/segment/mem/ids_map_gen.go
+src/m3ninx/index/segment/mem/ids_map_gen.go
diff --git a/.metalinter.json b/.metalinter.json
@@ -1,19 +1,23 @@
 {
   "Linters": {
     "unused": {
-      "Command": "unused -tags 'integration big adhoc'",
+      "Command": "unused -tags 'integration big'",
       "Pattern": "PATH:LINE:COL:MESSAGE"
     },
     "gosimple": {
-      "Command": "gosimple -tags 'integration big adhoc'",
+      "Command": "gosimple -tags 'integration big'",
       "Pattern": "PATH:LINE:COL:MESSAGE"
     },
     "badtime": {
-      "Command": "badtime -tags 'integration big adhoc'",
+      "Command": "badtime -tags 'integration big'",
+      "Pattern": "PATH:LINE:COL:MESSAGE"
+    },
+    "varcheck": {
+      "Command": "varcheck -tags 'integration big'",
       "Pattern": "PATH:LINE:COL:MESSAGE"
     },
     "importorder": {
-      "Command": "importorder -patterns=\"STDLIB github.com/m3db EXTERNAL\"",
+      "Command": "importorder -patterns='STDLIB github.com/m3db EXTERNAL' -tags 'integration big'",
       "Pattern": "PATH:LINE:MESSAGE"
     }
   },
@@ -25,8 +29,8 @@
     "unconvert",
     "misspell",
     "golint",
-    "gosimple",
     "unused",
+    "gosimple",
     "badtime",
     "maligned",
     "importorder"

diff --git a/Makefile b/Makefile
@@ -41,6 +41,7 @@ GOMETALINT_VERSION   := v2.0.5
 SERVICES :=     \
 	m3dbnode      \
 	m3coordinator \
+	m3aggregator  \
 	m3query       \
 	m3em_agent    \
 	m3nsch_server \
@@ -55,15 +56,16 @@ SUBDIRS :=    \
 	m3em        \
 	m3nsch      \
 	m3ninx      \
-
-TOOLS :=            \
-	read_ids          \
-	read_index_ids    \
-	read_data_files   \
-	read_index_files  \
-	clone_fileset     \
-	dtest             \
-	verify_commitlogs \
+	aggregator  \
+
+TOOLS :=               \
+	read_ids             \
+	read_index_ids       \
+	read_data_files      \
+	read_index_files     \
+	clone_fileset        \
+	dtest                \
+	verify_commitlogs    \
 	verify_index_files
 
 .PHONY: setup
@@ -186,7 +188,7 @@ docs-deploy: docs-container
 
 .PHONY: docker-integration-test
 docker-integration-test:
-	@echo "Running Docker integration test"
+	@echo "--- Running Docker integration test"
 	@./scripts/docker-integration-tests/setup.sh
 	@./scripts/docker-integration-tests/simple/test.sh
 	@./scripts/docker-integration-tests/prometheus/test.sh
@@ -206,13 +208,6 @@ SUBDIR_TARGETS :=     \
 	all-gen             \
 	metalint
 
-define SUBDIR_TARGET_RULE
-.PHONY: $(SUBDIR_TARGET)
-$(SUBDIR_TARGET): $(foreach SUBDIR,$(SUBDIRS),$(SUBDIR_TARGET)-$(SUBDIR))
-endef
-
-$(foreach SUBDIR_TARGET,$(SUBDIR_TARGETS),$(eval $(SUBDIR_TARGET_RULE)))
-
 .PHONY: test-ci-unit
 test-ci-unit: test-base
 	$(process_coverfile) $(coverfile)
@@ -230,37 +225,37 @@ define SUBDIR_RULES
 
 .PHONY: mock-gen-$(SUBDIR)
 mock-gen-$(SUBDIR): install-codegen-tools install-mockgen
-	@echo Generating mocks $(SUBDIR)
+	@echo "--- Generating mocks $(SUBDIR)"
 	@[ ! -d src/$(SUBDIR)/$(mocks_rules_dir) ] || \
 		PATH=$(retool_bin_path):$(PATH) PACKAGE=$(m3db_package) $(auto_gen) src/$(SUBDIR)/$(mocks_output_dir) src/$(SUBDIR)/$(mocks_rules_dir)
 
 .PHONY: thrift-gen-$(SUBDIR)
 thrift-gen-$(SUBDIR): install-codegen-tools
-	@echo Generating thrift files $(SUBDIR)
+	@echo "--- Generating thrift files $(SUBDIR)"
 	@[ ! -d src/$(SUBDIR)/$(thrift_rules_dir) ] || \
 		PATH=$(retool_bin_path):$(PATH) PACKAGE=$(m3db_package) $(auto_gen) src/$(SUBDIR)/$(thrift_output_dir) src/$(SUBDIR)/$(thrift_rules_dir)
 
 .PHONY: proto-gen-$(SUBDIR)
 proto-gen-$(SUBDIR): install-codegen-tools
-	@echo Generating protobuf files $(SUBDIR)
+	@echo "--- Generating protobuf files $(SUBDIR)"
 	@[ ! -d src/$(SUBDIR)/$(proto_rules_dir) ] || \
 		PATH=$(retool_bin_path):$(PATH) PACKAGE=$(m3db_package) $(auto_gen) src/$(SUBDIR)/$(proto_output_dir) src/$(SUBDIR)/$(proto_rules_dir)
 
 .PHONY: asset-gen-$(SUBDIR)
 asset-gen-$(SUBDIR): install-codegen-tools
-	@echo Generating asset files $(SUBDIR)
+	@echo "--- Generating asset files $(SUBDIR)"
 	@[ ! -d src/$(SUBDIR)/$(assets_rules_dir) ] || \
 		PATH=$(retool_bin_path):$(PATH) PACKAGE=$(m3db_package) $(auto_gen) src/$(SUBDIR)/$(assets_output_dir) src/$(SUBDIR)/$(assets_rules_dir)
 
 .PHONY: genny-gen-$(SUBDIR)
 genny-gen-$(SUBDIR): install-codegen-tools
-	@echo Generating genny files $(SUBDIR)
+	@echo "--- Generating genny files $(SUBDIR)"
 	@[ ! -f $(SELF_DIR)/src/$(SUBDIR)/generated-source-files.mk ] || \
 		PATH=$(retool_bin_path):$(PATH) make -f $(SELF_DIR)/src/$(SUBDIR)/generated-source-files.mk genny-all
 
 .PHONY: license-gen-$(SUBDIR)
 license-gen-$(SUBDIR): install-codegen-tools
-	@echo Updating license in files $(SUBDIR)
+	@echo "--- Updating license in files $(SUBDIR)"
 	@find $(SELF_DIR)/src/$(SUBDIR) -name '*.go' | PATH=$(retool_bin_path):$(PATH) xargs -I{} update-license {}
 
 .PHONY: all-gen-$(SUBDIR)
@@ -269,12 +264,6 @@ license-gen-$(SUBDIR): install-codegen-tools
 # we make header changes.
 all-gen-$(SUBDIR): thrift-gen-$(SUBDIR) proto-gen-$(SUBDIR) asset-gen-$(SUBDIR) genny-gen-$(SUBDIR) mock-gen-$(SUBDIR) license-gen-$(SUBDIR)
 
-.PHONY: metalint-$(SUBDIR)
-metalint-$(SUBDIR): install-gometalinter install-linter-badtime install-linter-importorder
-	@echo metalinting $(SUBDIR)
-	@(PATH=$(retool_bin_path):$(PATH) $(metalint_check) \
-		$(metalint_config) $(metalint_exclude) src/$(SUBDIR))
-
 .PHONY: test-$(SUBDIR)
 test-$(SUBDIR):
 	@echo testing $(SUBDIR)
@@ -303,26 +292,53 @@ test-single-integration-$(SUBDIR):
 
 .PHONY: test-ci-unit-$(SUBDIR)
 test-ci-unit-$(SUBDIR):
-	@echo test-ci-unit $(SUBDIR)
+	@echo "--- test-ci-unit $(SUBDIR)"
 	SRC_ROOT=./src/$(SUBDIR) make test-base
+	@echo "--- uploading coverage report"
 	$(codecov_push) -f $(coverfile) -F $(SUBDIR)
 
 .PHONY: test-ci-big-unit-$(SUBDIR)
 test-ci-big-unit-$(SUBDIR):
-	@echo test-ci-big-unit $(SUBDIR)
+	@echo "--- test-ci-big-unit $(SUBDIR)"
 	SRC_ROOT=./src/$(SUBDIR) make test-big-base
+	@echo "--- uploading coverage report"
 	$(codecov_push) -f $(coverfile) -F $(SUBDIR)
 
 .PHONY: test-ci-integration-$(SUBDIR)
 test-ci-integration-$(SUBDIR):
-	@echo test-ci-integration $(SUBDIR)
+	@echo "--- test-ci-integration $(SUBDIR)"
 	SRC_ROOT=./src/$(SUBDIR) INTEGRATION_TIMEOUT=4m TEST_SERIES_CACHE_POLICY=$(cache_policy) make test-base-ci-integration
+	@echo "--- uploading coverage report"
 	$(codecov_push) -f $(coverfile) -F $(SUBDIR)
 
+.PHONY: metalint-$(SUBDIR)
+metalint-$(SUBDIR): install-gometalinter install-linter-badtime install-linter-importorder
+	@echo "--- metalinting $(SUBDIR)"
+	@(PATH=$(retool_bin_path):$(PATH) $(metalint_check) \
+		$(metalint_config) $(metalint_exclude) src/$(SUBDIR))
+
 endef
 
+# generate targets for each SUBDIR in SUBDIRS based on the rules specified above.
 $(foreach SUBDIR,$(SUBDIRS),$(eval $(SUBDIR_RULES)))
 
+define SUBDIR_TARGET_RULE
+.PHONY: $(SUBDIR_TARGET)
+$(SUBDIR_TARGET): $(foreach SUBDIR,$(SUBDIRS),$(SUBDIR_TARGET)-$(SUBDIR))
+endef
+
+# generate targets across SUBDIRS for each SUBDIR_TARGET. i.e. generate rules
+# which allow `make all-gen` to invoke `make all-gen-dbnode all-gen-coordinator ...`
+# NB: we skip metalint explicity as the default target below requires less invocations
+# of metalint and finishes faster.
+$(foreach SUBDIR_TARGET, $(filter-out metalint,$(SUBDIR_TARGETS)), $(eval $(SUBDIR_TARGET_RULE)))
+
+.PHONY: metalint
+metalint: install-gometalinter install-linter-badtime install-linter-importorder
+	@echo "--- metalinting src/"
+	@(PATH=$(retool_bin_path):$(PATH) $(metalint_check) \
+		$(metalint_config) $(metalint_exclude) src/)
+
 # Tests that all currently generated types match their contents if they were regenerated
 .PHONY: test-all-gen
 test-all-gen: all-gen

diff --git a/glide.lock b/glide.lock