Permalink
Browse files

Improvements to several bash scripts for testing/processing.

  • Loading branch information...
1 parent 620e7d1 commit d9f50c4ef489ff0debbbcee855b153a41dd3f7ea @GraylinKim GraylinKim committed Aug 21, 2012
Showing with 63 additions and 6 deletions.
  1. +9 −3 bin/create_env.sh
  2. +38 −3 bin/process.sh
  3. +16 −0 bin/reset_env.sh
View
@@ -1,10 +1,16 @@
#!/bin/bash
source $(dirname "$0")/utils.sh
+if [ "$1" = "" ]; then
+ env=".";
+else
+ env="$1";
+fi
+
# Create the specified base drive, primary folders, and work sub-folders
-mkdir -p $1 $1/work $1/json $1/lucene $1/processed $1/data
-mkdir $1/work/bills $1/work/calendars $1/work/agendas $1/work/transcripts
+mkdir -p $env $env/work $env/json $env/lucene $env/processed $env/data
+mkdir $env/work/bills $env/work/calendars $env/work/agendas $env/work/transcripts
# Create a shortcut link to the project repo for convenient access to scripts.
-ln -s $ROOTDIR/bin $1
+ln -s $ROOTDIR/bin $env
View
@@ -1,7 +1,15 @@
#!/bin/bash
source $(dirname "$0")/utils.sh
-USAGE="USAGE: `basename $0` --source SOURCE --work WORK --dest DEST --storage STORAGE --lucene LUCENE";
+USAGE="USAGE: `basename $0` --source SOURCE --work WORK --dest DEST --storage STORAGE --lucene LUCENE --basedir BASEDIR --dryrun";
+
+dryrun=0;
+basedir="";
+source="data";
+work="work";
+dest="processed";
+storage="json";
+lucene="lucene";
while [ $# -gt 0 ]; do
case "$1" in
@@ -10,11 +18,33 @@ while [ $# -gt 0 ]; do
--dest) shift; dest="$1" ;;
--storage) shift; storage="$1" ;;
--lucene) shift; lucene="$1" ;;
+ --basedir) shift; basedir="$1" ;;
+ --dryrun) shift; dryrun=1 ;;
*) echo $USAGE; exit 1 ;;
esac
shift
done
+if [ "$basedir" != "" ]; then
+ source="$basedir/$source";
+ work="$basedir/$work";
+ dest="$basedir/$dest";
+ storage="$basedir/$storage";
+ lucene="$basedir/$lucene";
+fi
+
+echo "Processing using: ";
+echo " Source: $source";
+echo " Work: $work";
+echo " Dest: $dest";
+echo " Storage: $storage";
+echo " Lucene: $lucene";
+
+
+if [ $dryrun -eq 1 ]; then
+ exit;
+fi
+
required_args=( "$source" "$work" "$dest" "$storage" "$lucene")
for required_arg in "${required_args[@]}"; do
if [ ! "$required_arg" ]; then
@@ -28,6 +58,7 @@ done
# Check to see if there is any work to be done
if [ `find $source -type f | wc -l` -eq 0 ]; then
+ echo "No files in $source to process."
exit 0;
fi
@@ -60,6 +91,10 @@ $BINDIR/run.sh push $storage --lucene $lucene --change-file $changelog &>>$error
if [ ! -r $dest/logs/ ]; then
mkdir $dest/logs/
fi
-mv $changelog $dest/logs/`date +D%Y%m%d.T%H%M%S.change.log`
-mv $errorlog $dest/logs/`date +D%Y%m%d.T%H%M%S.error.log`
+new_change_log=$dest/logs/`date +D%Y%m%d.T%H%M%S.change.log`;
+new_error_log=$dest/logs/`date +D%Y%m%d.T%H%M%S.error.log`;
+mv $changelog $new_change_log
+mv $errorlog $new_error_log
+
+echo "Finished processing changes to `cat $new_change_log | wc -l` objects.";
View
@@ -0,0 +1,16 @@
+#!/bin/bash
+source $(dirname "$0")/utils.sh
+
+if [ "$1" = "" ]; then
+ env=".";
+else
+ env="$1";
+fi
+
+if [ `find $env/processed -type f -name SOBI* | wc -l` -ne 0 ]; then
+ mv `find $env/processed -type f -name SOBI*` data/;
+fi
+
+rm -fr processed/*
+rm -fr json/*
+rm -fr lucene/*

0 comments on commit d9f50c4

Please sign in to comment.