run_pipeline_all.sh

#!/bin/bash
python run_all.py \
  --input_file=gs://httparchive/crawls_manifest/android-Sep_1_2023.txt \
  --runner=DataflowRunner \
  --project=httparchive \
  --temp_location=gs://httparchive-staging/experimental/temp \
  --staging_location=gs://httparchive-staging/experimental/staging \
  --region=us-west1 \
  --setup_file=./setup.py \
  --machine_type=n1-standard-32 \
  --worker_disk_type=compute.googleapis.com/projects//zones//diskTypes/pd-ssd \
  --noauth_local_webserver