From 82c170e5fdea6c1584b9bf58f35754e48a9abae7 Mon Sep 17 00:00:00 2001
From: Benjamin Alan Weaver <benjamin.weaver@noirlab.edu>
Date: Mon, 3 Oct 2022 16:37:27 -0700
Subject: [PATCH] add notes on other data sets

---
 bin/desi_transfer_statistics.sh | 54 ++++++++++++++++++++++++++++-----
 1 file changed, 47 insertions(+), 7 deletions(-)

diff --git a/bin/desi_transfer_statistics.sh b/bin/desi_transfer_statistics.sh
index 3bf79a7..8070716 100755
--- a/bin/desi_transfer_statistics.sh
+++ b/bin/desi_transfer_statistics.sh
@@ -11,11 +11,11 @@ number_of_nights=0
 number_of_exposures=0
 total_data=0
 for n in ${DESI_SPECTRO_DATA}/${year}1[012][0-9][0-9] ${DESI_SPECTRO_DATA}/${next_year}0[0-9][0-9][0-9]; do
-    echo ${n}
+    # echo ${n}
     night=$(basename ${n})
     number_of_nights=$(( number_of_nights + 1 ))
     for e in ${DESI_SPECTRO_DATA}/${night}/*; do
-        echo ${e}
+        # echo ${e}
         expid=$(basename ${e})
         number_of_exposures=$(( number_of_exposures + 1 ))
         expid_size=$(du -k -s ${e} | awk '{print $1}')
@@ -33,18 +33,58 @@ if [[ "${year}" == "2021" ]]; then
     number_of_fire_exposures=0
     total_fire_data=0
     for n in ${DESI_SPECTRO_DATA}/${next_year}0[89][0-9][0-9]; do
-        echo ${n}
+        # echo ${n}
         night=$(basename ${n})
         number_of_fire_nights=$(( number_of_fire_nights + 1 ))
         for e in ${DESI_SPECTRO_DATA}/${night}/*; do
-            echo ${e}
+            # echo ${e}
             expid=$(basename ${e})
             expid_size=$(du -k -s ${e} | awk '{print $1}')
-            number_of_fire_exposures=$(( number_of_fore_exposures + 1 ))
-            total_data=$(( total_fire_data + expid_size ))
+            number_of_fire_exposures=$(( number_of_fire_exposures + 1 ))
+            total_fire_data=$(( total_fire_data + expid_size ))
     done
     done
     echo "Number of nights during fire recovery = ${number_of_fire_nights}."
     echo "Number of exposures during fire recovery = ${number_of_fire_exposures}."
     echo "Total data during fire recovery = ${total_fire_data} KB."
-fi
\ No newline at end of file
+fi
+#
+# Nightwatch data.
+#
+NIGHTWATCH=${DESI_ROOT}/spectro/nightwatch/kpno
+number_of_nightwatch_nights=0
+total_nightwatch_data=0
+for n in ${NIGHTWATCH}/${year}1[012][0-9][0-9] ${NIGHTWATCH}/${next_year}0[0-9][0-9][0-9]; do
+    # echo ${n}
+    # night=$(basename ${n})
+    number_of_nightwatch_nights=$(( number_of_nightwatch_nights + 1 ))
+    night_size=$(du -k -s ${n} | awk '{print $1}')
+    total_nightwatch_data=$(( total_nightwatch_data + night_size ))
+done
+echo "Number of nightwatch nights = ${number_of_nightwatch_nights}."
+# echo "Number of exposures = ${number_of_exposures}."
+echo "Total nightwatch data = ${total_nightwatch_data} KB."
+#
+# Engineering data.
+#
+# After the clean-up of the donut data, the focalplane directory is the
+# largest contributor. So even though as a whole the engineering data increased
+# by about 100 GB, the focalplane directory increased by about 1 TB, while the
+# donut directory shrank considerably.
+#
+# Reduced daily data transferred to Tucson.
+#
+# This is slightly tricky because we don't transfer *every* file.
+# But in practice we should also count KPNO nightwatch data transferred to
+# Tucson.
+# - The majority of daily reduction data is in the exposures and tiles
+#   directories.  This adds up to (102.9 - 46.6) + (51.7 - 20) = 88 TB.
+# - KPNO nightwatch data (57.5 - 28.5) = 29 TB at transferred.
+#
+# Fire recovery:
+#
+# - We are transferring raw data on disk.
+# - We are transferring a small amount of engineering files ~ 10 MB/night.
+# - We are *not* transferring nightwatch data.
+# - We are streaming to the database replica at NERSC.  This is the biggest share.
+#
\ No newline at end of file