yulWA-yfile

#!/usr/bin/env bash

DATE=$(date +"%Y%m%d")
LOG_FILE="/var/log/heritrix/yu-yfile-browsertrix.log"
CRAWL_CONFIG="/opt/web-archiving-cron-scripts/crawl-configs/yu-yfile.yaml"
CRAWL_DIR="/mnt/omega/web-archives/browsertrix"
COLLECTION_NAME="yu-yfile"
WORKERS=8
VERSION="1.1.0"

function start_crawl {
    cd "$CRAWL_DIR" || exit
    /usr/bin/docker run -v "$CRAWL_CONFIG:/app/crawl-config.yaml" -v "$CRAWL_DIR:/crawls" webrecorder/browsertrix-crawler:"$VERSION" crawl --config /app/crawl-config.yaml --collection "$COLLECTION_NAME" --allowHashUrls --workers "$WORKERS" --saveState always --statsFilename "$COLLECTION_NAME-stats-$DATE.json" "+YorkUniversityLibrariesCrawlerBot, diginit@yorku.ca" > "$LOG_FILE" 2>&1

    if [ $? -eq 0 ]; then
        echo "[INFO] $(date) - started" >> "$LOG_FILE"
    else
        echo "[ERROR] $(date)" >> "$LOG_FILE"
    fi
}

function stage_warcs {
    cd "$CRAWL_DIR/collections/$COLLECTION_NAME/archive" || exit
    rename -v 's/rec-/YU-YFILE-rec-/g' ./*.warc.gz
    echo "[INFO] $(date) - renamed files" >> "$LOG_FILE"
    mv -v ./*.warc.gz /mnt/omega/web-archives/import
    echo "[INFO] $(date) - staged files" >> "$LOG_FILE"
}

start_crawl
stage_warcs

docker container prune -f