https fallback with cURL

build/sb-mirror/download.sh

@@ -15,27 +15,29 @@ validate_file() {
 download() {
   curl -sL https://git.io/sb-dbapi-license -o "$MIRROR_DIR"/licence.md
   echo "Downloading from $URL"
-  if [ -n "$MIRROR_URL" ]; then rsync -rztvP --zc=lz4 --append rsync://"$URL"/sponsorblock "${MIRROR_DIR}"
-  else
-    # download from main server so get filenames
-    curl -sL sponsor.ajay.app/database.json?generate=false -o response.json
+  if [ -n "$MIRROR_URL" ]; then rsync -rztvP --zc=lz4 --append --contimeout=10 rsync://"$URL"/sponsorblock "${MIRROR_DIR}"
+  else # downloading from main mirror
+    # get filenames
+    curl -sL https://sponsor.ajay.app/database.json?generate=false -o response.json
     DUMP_DATE=$(jq .lastUpdated < response.json)
     # set $@ since posix doesn't have named variables
     set -- $(jq -r .links[].table < response.json)
     rm response.json
 
     for table in "$@"
-    do rsync -ztvP --zc=lz4 --append rsync://wiki.sponsor.ajay.app/sponsorblock/"${table}"_"${DUMP_DATE}".csv ${MIRROR_DIR}/${table}.csv; done
+    do
+      echo "Downloading $table.csv"
+      rsync -ztvP --zc=lz4 --append --contimeout=10 rsync://sponsor.ajay.app/sponsorblock/"${table}"_"${DUMP_DATE}".csv ${MIRROR_DIR}/${table}.csv ||
+        curl -L https://sponsor.ajay.app/database/"${table}".csv -o ${MIRROR_DIR}/"${table}".csv # fallback to CURL if rsync times out
+    done
     date -d@"$(echo "$DUMP_DATE" | cut -c 1-10)" +%F_%H-%M > "${MIRROR_DIR}"/lastUpdate.txt
   fi
 }
 
 validate() {
   echo "Validating Downloads"
   for file in "${MIRROR_DIR}"/*.csv
-  do validate_file "$file"
-  done
-  download
+  do validate_file "$file"; done
 }
 
 convert_sqlite() {
@@ -52,5 +54,5 @@ convert_sqlite() {
 download
 # if NO_VALIDATE, skip validation
 if [ -z "$NO_VALIDATE" ]; then validate; fi
-# if sqlite, merge all csvs into one .db file
+# if SQLITE, merge all csvs into one .db file
 if [ -n "$SQLITE" ]; then convert_sqlite; fi