Skip to content
Browse files

Simplify scripts, enforce required args, and enable logging to file.

  • Loading branch information...
1 parent 021178b commit 11c9e83abbd7a61cff5367e97ccbbf1f4f02db81 @GraylinKim GraylinKim committed Apr 12, 2012
View
19 bin/process.sh
@@ -18,6 +18,7 @@ done
required_args=( "$source" "$work" "$dest" "$storage" "$lucene")
for required_arg in "${required_args[@]}"; do
if [ ! "$required_arg" ]; then
+ echo "All options are required: source, work, dest, storage, lucene";
echo $USAGE; exit 1 ;
elif [ ! -r "$required_arg" ]; then
echo "$required_arg must exist on the filesystem.";
@@ -33,9 +34,12 @@ fi
# Stop immediately if any command below fails
set -e
+errorlog=$work/process.log
+changelog=$work/change.log
+
# Organize the change files and ingest to storage
-$BINDIR/run.sh collate $source $work;
-$BINDIR/run.sh ingest --source $work --storage $storage --change-file $work/change.log
+$BINDIR/run.sh collate $source $work &>$errorlog;
+$BINDIR/run.sh ingest $work $storage --change-file $changelog &>>$errorlog
# Migrate all processed files to $dest for archival purposes
year="`date +%Y`"
@@ -50,10 +54,11 @@ for file_type in "${file_types[@]}"; do
done
# Push the changes out from storage
-$BINDIR/run.sh push --lucene $lucene --storage $storage --change-file $work/change.log
+$BINDIR/run.sh push $storage --lucene $lucene --change-file $changelog &>>$errorlog
-# Move the change.log to $dest for archiving as well
-if [ ! -r $dest/changelogs/ ]; then
- mkdir $dest/changelogs/
+# Move the logs to $dest for archiving as well
+if [ ! -r $dest/logs/ ]; then
+ mkdir $dest/logs/
fi
-mv $work/change.log $dest/changelogs/`date +CHANGELOG.D%Y%m%d.T%H%M%S.TXT`
+mv $changelog $dest/logs/`date +D%Y%m%d.T%H%M%S.change.log`
+mv $errorlog $dest/logs/`date +D%Y%m%d.T%H%M%S.error.log`
View
6 src/main/java/gov/nysenate/openleg/scripts/Collate.java
@@ -18,8 +18,8 @@
public static void main(String[] args) {
if (args.length < 2) {
- System.out.println("USAGE: Collate source [... source] dest");
- System.exit(0);
+ System.err.println("USAGE: Collate source [... source] dest");
+ System.exit(1);
}
File destDirectory = new File(args[args.length-1]).getAbsoluteFile();
@@ -47,7 +47,7 @@ public static void main(String[] args) {
}
for (File file : sources) {
- logger.debug("Processing: "+file);
+ logger.info("Processing: "+file);
int inc = 1;
String in = null;
View
27 src/main/java/gov/nysenate/openleg/scripts/Ingest.java
@@ -4,7 +4,6 @@
import gov.nysenate.openleg.processors.BillProcessor;
import gov.nysenate.openleg.processors.CalendarProcessor;
import gov.nysenate.openleg.processors.TranscriptProcessor;
-import gov.nysenate.openleg.util.Config;
import gov.nysenate.openleg.util.Storage;
import gov.nysenate.openleg.util.Storage.Status;
import gov.nysenate.openleg.util.Timer;
@@ -20,7 +19,6 @@
import javax.xml.bind.UnmarshalException;
import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
@@ -43,36 +41,33 @@ public int compare(File a, File b) {
}
public static void main(String[] args) throws Exception {
+ String[] required = null;
CommandLine opts = null;
try {
Options options = new Options()
.addOption("h", "help", false, "Print this message")
+ .addOption("f", "change-file", true, "The path to store the changes");
//.addOption("dt", "document-type", true, "Type of document being indexed with -id (REQUIRED WITH -id).. (bill|calendar|agenda|transcript)")
//.addOption("id", "index-document", true, "Index JSON document specified by argument (path to file)")
- .addOption("t", "storage", true, "The path to the storage directory")
- .addOption("s", "source", true, "The path to the files to ingest")
- .addOption("f", "change-file", true, "The path to store the changes");
opts = new PosixParser().parse(options, args);
+ required = opts.getArgs();
if(opts.hasOption("-h")) {
- new HelpFormatter().printHelp("posix", options );
+ System.out.println("USAGE: Ingest SOURCE STORAGE [--change-file FILE]");
System.exit(0);
+
+ } else if (required.length != 2) {
+ System.err.println("Both source and storage directories are required.");
+ System.err.println("USAGE: Ingest SOURCE STORAGE [--change-file FILE]");
+ System.exit(1);
}
} catch (ParseException e) {
logger.fatal("Error parsing arguments: ", e);
System.exit(0);
}
- String sourceDir = opts.getOptionValue("source", Config.get("data.sobi"));
- String storageDir = opts.getOptionValue("storage", Config.get("data.json"));
-
- if( sourceDir == null || storageDir == null ) {
- throw new org.apache.commons.cli.ParseException("source and storage are both required parameters.");
- }
- Storage storage = new Storage(storageDir);
-
-
Timer timer = new Timer();
- Collection<File> files = FileUtils.listFiles(new File(sourceDir), null, true);
+ Storage storage = new Storage(required[1]);
+ Collection<File> files = FileUtils.listFiles(new File(required[0]), null, true);
Collections.sort((List<File>)files, new FileNameComparator());
// Process each file individually, flushing changes to storage as necessary
View
26 src/main/java/gov/nysenate/openleg/scripts/Push.java
@@ -3,7 +3,6 @@
import gov.nysenate.openleg.services.Lucene;
import gov.nysenate.openleg.services.ServiceBase;
import gov.nysenate.openleg.services.Varnish;
-import gov.nysenate.openleg.util.Config;
import gov.nysenate.openleg.util.Storage;
import java.io.File;
@@ -13,7 +12,6 @@
import java.util.HashMap;
import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
@@ -37,23 +35,29 @@
}
public static void main(String[] args) {
+ String[] required = null;
CommandLine opts = null;
try {
Options options = new Options()
- .addOption("t", "storage", true, "The path to the storage directory")
.addOption("l", "lucene", true, "Push changes to the Lucene service")
.addOption("v", "varnish", false, "Push changes to the Varnish service")
.addOption("f", "change-file", true, "Path of changeLog file.")
.addOption("c", "changes", true, "A newline delimited list of changes")
.addOption("h", "help", false, "Print this message");
opts = new PosixParser().parse(options, args);
+ required = opts.getArgs();
if(opts.hasOption("-h")) {
- new HelpFormatter().printHelp("posix", options );
+ System.err.println("USAGE: Push STORAGE [--lucene LOCATION] [--varnish] [--change-file FILE] [--changes CHANGES]");
System.exit(0);
+
+ } else if (required.length != 1) {
+ System.err.println("Storage is a required argument.");
+ System.err.println("USAGE: Push STORAGE [--lucene LOCATION] [--varnish] [--change-file FILE] [--changes CHANGES]");
+ System.exit(1);
}
} catch (ParseException e) {
logger.fatal("Error parsing arguments: ", e);
- System.exit(0);
+ System.exit(1);
}
// Parse the specified changes into a hash
@@ -63,19 +67,16 @@ public static void main(String[] args) {
File changeFile = new File(opts.getOptionValue("change-file"));
changes = parseChanges(FileUtils.readLines(changeFile, "UTF-8"));
} catch (IOException e) {
- logger.fatal("Error reading change-file: "+opts.getOptionValue("changes"), e);
- System.exit(0);
+ System.err.println("Error reading change-file: "+opts.getOptionValue("changes"));
+ System.exit(1);
}
} else if (opts.hasOption("changes")) {
changes = parseChanges(Arrays.asList(opts.getOptionValue("changes").split("\n")));
} else {
- logger.fatal("Nothing to do. Specify changes or a change-file");
- System.exit(0);
+ System.err.println("Changes to push must be specified with either --change-file or --changes");
+ System.exit(1);
}
- String storageDir = opts.getOptionValue("storage", Config.get("data.json"));
- Storage storage = new Storage(storageDir);
-
// Currently there is a Lucene hook and varnish hook, more to come
ArrayList<ServiceBase> services = new ArrayList<ServiceBase>();
if(opts.hasOption("lucene")) {
@@ -87,6 +88,7 @@ public static void main(String[] args) {
}
// Pass the change log through a set of service hooks
+ Storage storage = new Storage(required[0]);
for(ServiceBase service:services) {
try {
service.process(changes, storage);

0 comments on commit 11c9e83

Please sign in to comment.
Something went wrong with that request. Please try again.