more queues, and additional log

m-lab · Mar 28, 2018 · 040f64d · 040f64d
1 parent 65f02dd
commit 040f64d
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/cloud/tq/queuehandler.go b/cloud/tq/queuehandler.go
@@ -129,6 +129,7 @@ func (qh *ChannelQueueHandler) handleLoop(next api.BasicPipe, bucketOpts ...opti
 		// This ensures that the data has actually been processed, rather
 		// than just sitting in the queue or in the pipeline.
 		qh.waitForEmptyQueue()
+		log.Println(qh.Queue, "sending", prefix, "to dedup handler")
 		// This may block if previous hasn't finished.  Should be rare.
 		if n > 0 && next != nil {
 			next.Sink() <- prefix

diff --git a/k8s/data-processing-cluster/deployments/etl-gardener.yml b/k8s/data-processing-cluster/deployments/etl-gardener.yml
@@ -46,15 +46,15 @@ spec:
         - name: TASKFILE_BUCKET
           value: "archive-{{GCLOUD_PROJECT}}"
         - name: START_DATE
-          value: "20170501"
+          value: "20170523"
         - name: EXPERIMENTS
           value: "ndt"  # For example "ndt,sidestream,switch"
         - name: DATASET
           value: batch
         - name: QUEUE_BASE
           value: etl-ndt-batch-
         - name: NUM_QUEUES
-          value: "8"
+          value: "16"
         ports:
         - name: prometheus-port
           containerPort: 9090