Permalink
Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
executable file 29 lines (24 sloc) 562 Bytes
#!/usr/bin/env bash
timestamp() {
date +%F_%H-%M-%S
}
process_site() {
echo job: $1 site: $2 get: list of pages
curl -sL http://$2/system/sitemap.json | jq ".[]|.slug" | cut -d "\"" -f 2 > $1-$2.pages
while read q; do
process_page $1 $2 $q
done < $1-$2.pages
tar -czf $1-$2.pages.tgz $1-$2-*
rm -rf $1-$2-*
}
process_page() {
echo job: $1 site: $2 get: $3
curl -sL http://$2/$3.json > $1-$2-$3.json
}
mkdir -p pages && cd pages
while read p; do
mkdir -p $p && cd $p
echo $(timestamp)
process_site $(timestamp) $p
cd ..
done