From 6d1fc4d54203e00fde8a59f0dd0e7fb0d9754e0c Mon Sep 17 00:00:00 2001
From: Stephen Soltesz <stephen.soltesz@gmail.com>
Date: Mon, 10 Apr 2023 21:28:55 -0400
Subject: [PATCH 1/3] Enable historical processing of annotation & ndt data

---
 config/config.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/config/config.yml b/config/config.yml
index 4184cfe1..b5824fa5 100644
--- a/config/config.yml
+++ b/config/config.yml
@@ -13,7 +13,6 @@ sources:
   target_datasets:
     tmp: tmp_ndt
     raw: raw_ndt
-  daily_only: true
 - bucket: archive-{{NDT_SOURCE_PROJECT}}
   experiment: ndt
   datatype: ndt5
@@ -21,7 +20,6 @@ sources:
     tmp: tmp_ndt
     raw: raw_ndt
     join: ndt
-  daily_only: true
 - bucket: archive-{{NDT_SOURCE_PROJECT}}
   experiment: ndt
   datatype: ndt7
@@ -29,7 +27,6 @@ sources:
     tmp: tmp_ndt
     raw: raw_ndt
     join: ndt
-  daily_only: true
 - bucket: archive-{{NDT_SOURCE_PROJECT}}
   experiment: ndt
   datatype: pcap

From 3ba04683530f6866ad5259575db09c4eae8930be Mon Sep 17 00:00:00 2001
From: Stephen Soltesz <stephen.soltesz@gmail.com>
Date: Tue, 11 Apr 2023 07:32:18 -0400
Subject: [PATCH 2/3] Update staging and production sources

---
 apply-cluster.sh | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/apply-cluster.sh b/apply-cluster.sh
index 48618925..7b5d6df6 100755
--- a/apply-cluster.sh
+++ b/apply-cluster.sh
@@ -18,14 +18,16 @@ CLUSTER=${CLOUDSDK_CONTAINER_CLUSTER:?Please provide cluster name: $USAGE}
 DATE_SKIP=${DATE_SKIP:-"0"}  # Number of dates to skip between each processed date (for sandbox).
 TASK_FILE_SKIP=${TASK_FILE_SKIP:-"0"}  # Number of files to skip between each processed file (for sandbox).
 
-# Use sandbox in sandbox, measurement-lab in staging & oti.
-SOURCE_PROJECT=${PROJECT_ID/mlab-oti/measurement-lab}
-SOURCE_PROJECT=${SOURCE_PROJECT/mlab-staging/measurement-lab}
+# Source sandbox in sandbox.
+# Source annotations from staging in staging.
+# Source ndt from measurement-lab in staging & oti.
+ANNOTATION_SOURCE_PROJECT=${PROJECT_ID/mlab-oti/measurement-lab}
+NDT_SOURCE_PROJECT=${ANNOTATION_SOURCE_PROJECT/mlab-staging/measurement-lab}
 sed -i \
-    -e 's/{{ANNOTATION_SOURCE_PROJECT}}/'${SOURCE_PROJECT}'/g' \
+    -e 's/{{ANNOTATION_SOURCE_PROJECT}}/'${ANNOTATION_SOURCE_PROJECT}'/g' \
     config/config.yml
 sed -i \
-    -e 's/{{NDT_SOURCE_PROJECT}}/'${SOURCE_PROJECT}'/g' \
+    -e 's/{{NDT_SOURCE_PROJECT}}/'${NDT_SOURCE_PROJECT}'/g' \
     config/config.yml
 
 # Create the configmap

From 0c1e0d11eddc19639fa1e5ab2b2714d7bc499b2f Mon Sep 17 00:00:00 2001
From: Stephen Soltesz <stephen.soltesz@gmail.com>
Date: Tue, 11 Apr 2023 13:43:44 -0400
Subject: [PATCH 3/3] Revert changes to apply-cluster.sh

---
 apply-cluster.sh | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/apply-cluster.sh b/apply-cluster.sh
index 7b5d6df6..48618925 100755
--- a/apply-cluster.sh
+++ b/apply-cluster.sh
@@ -18,16 +18,14 @@ CLUSTER=${CLOUDSDK_CONTAINER_CLUSTER:?Please provide cluster name: $USAGE}
 DATE_SKIP=${DATE_SKIP:-"0"}  # Number of dates to skip between each processed date (for sandbox).
 TASK_FILE_SKIP=${TASK_FILE_SKIP:-"0"}  # Number of files to skip between each processed file (for sandbox).
 
-# Source sandbox in sandbox.
-# Source annotations from staging in staging.
-# Source ndt from measurement-lab in staging & oti.
-ANNOTATION_SOURCE_PROJECT=${PROJECT_ID/mlab-oti/measurement-lab}
-NDT_SOURCE_PROJECT=${ANNOTATION_SOURCE_PROJECT/mlab-staging/measurement-lab}
+# Use sandbox in sandbox, measurement-lab in staging & oti.
+SOURCE_PROJECT=${PROJECT_ID/mlab-oti/measurement-lab}
+SOURCE_PROJECT=${SOURCE_PROJECT/mlab-staging/measurement-lab}
 sed -i \
-    -e 's/{{ANNOTATION_SOURCE_PROJECT}}/'${ANNOTATION_SOURCE_PROJECT}'/g' \
+    -e 's/{{ANNOTATION_SOURCE_PROJECT}}/'${SOURCE_PROJECT}'/g' \
     config/config.yml
 sed -i \
-    -e 's/{{NDT_SOURCE_PROJECT}}/'${NDT_SOURCE_PROJECT}'/g' \
+    -e 's/{{NDT_SOURCE_PROJECT}}/'${SOURCE_PROJECT}'/g' \
     config/config.yml
 
 # Create the configmap