Skip to content

Commit

Permalink
Simplify scripts, enforce required args, and enable logging to file.
Browse files Browse the repository at this point in the history
  • Loading branch information
@GraylinKim
GraylinKim committed Apr 12, 2012
1 parent 021178b commit 11c9e83
Show file tree
Hide file tree
Showing 4 changed files with 40 additions and 38 deletions.
19 changes: 12 additions & 7 deletions bin/process.sh
View file
Expand Up @@ -18,6 +18,7 @@ done
required_args=( "$source" "$work" "$dest" "$storage" "$lucene")
for required_arg in "${required_args[@]}"; do
if [ ! "$required_arg" ]; then
echo "All options are required: source, work, dest, storage, lucene";
echo $USAGE; exit 1 ;
elif [ ! -r "$required_arg" ]; then
echo "$required_arg must exist on the filesystem.";
Expand All @@ -33,9 +34,12 @@ fi
# Stop immediately if any command below fails
set -e

errorlog=$work/process.log
changelog=$work/change.log

# Organize the change files and ingest to storage
$BINDIR/run.sh collate $source $work;
$BINDIR/run.sh ingest --source $work --storage $storage --change-file $work/change.log
$BINDIR/run.sh collate $source $work &>$errorlog;
$BINDIR/run.sh ingest $work $storage --change-file $changelog &>>$errorlog

# Migrate all processed files to $dest for archival purposes
year="`date +%Y`"
Expand All @@ -50,10 +54,11 @@ for file_type in "${file_types[@]}"; do
done

# Push the changes out from storage
$BINDIR/run.sh push --lucene $lucene --storage $storage --change-file $work/change.log
$BINDIR/run.sh push $storage --lucene $lucene --change-file $changelog &>>$errorlog

# Move the change.log to $dest for archiving as well
if [ ! -r $dest/changelogs/ ]; then
mkdir $dest/changelogs/
# Move the logs to $dest for archiving as well
if [ ! -r $dest/logs/ ]; then
mkdir $dest/logs/
fi
mv $work/change.log $dest/changelogs/`date +CHANGELOG.D%Y%m%d.T%H%M%S.TXT`
mv $changelog $dest/logs/`date +D%Y%m%d.T%H%M%S.change.log`
mv $errorlog $dest/logs/`date +D%Y%m%d.T%H%M%S.error.log`
6 changes: 3 additions & 3 deletions src/main/java/gov/nysenate/openleg/scripts/Collate.java
View file
Expand Up @@ -18,8 +18,8 @@ public class Collate {

public static void main(String[] args) {
if (args.length < 2) {
System.out.println("USAGE: Collate source [... source] dest");
System.exit(0);
System.err.println("USAGE: Collate source [... source] dest");
System.exit(1);
}

File destDirectory = new File(args[args.length-1]).getAbsoluteFile();
Expand Down Expand Up @@ -47,7 +47,7 @@ public static void main(String[] args) {
}

for (File file : sources) {
logger.debug("Processing: "+file);
logger.info("Processing: "+file);

int inc = 1;
String in = null;
Expand Down
27 changes: 11 additions & 16 deletions src/main/java/gov/nysenate/openleg/scripts/Ingest.java
View file
Expand Up @@ -4,7 +4,6 @@
import gov.nysenate.openleg.processors.BillProcessor;
import gov.nysenate.openleg.processors.CalendarProcessor;
import gov.nysenate.openleg.processors.TranscriptProcessor;
import gov.nysenate.openleg.util.Config;
import gov.nysenate.openleg.util.Storage;
import gov.nysenate.openleg.util.Storage.Status;
import gov.nysenate.openleg.util.Timer;
Expand All @@ -20,7 +19,6 @@
import javax.xml.bind.UnmarshalException;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
Expand All @@ -43,36 +41,33 @@ public int compare(File a, File b) {
}

public static void main(String[] args) throws Exception {
String[] required = null;
CommandLine opts = null;
try {
Options options = new Options()
.addOption("h", "help", false, "Print this message")
.addOption("f", "change-file", true, "The path to store the changes");
//.addOption("dt", "document-type", true, "Type of document being indexed with -id (REQUIRED WITH -id).. (bill|calendar|agenda|transcript)")
//.addOption("id", "index-document", true, "Index JSON document specified by argument (path to file)")
.addOption("t", "storage", true, "The path to the storage directory")
.addOption("s", "source", true, "The path to the files to ingest")
.addOption("f", "change-file", true, "The path to store the changes");
opts = new PosixParser().parse(options, args);
required = opts.getArgs();
if(opts.hasOption("-h")) {
new HelpFormatter().printHelp("posix", options );
System.out.println("USAGE: Ingest SOURCE STORAGE [--change-file FILE]");
System.exit(0);

} else if (required.length != 2) {
System.err.println("Both source and storage directories are required.");
System.err.println("USAGE: Ingest SOURCE STORAGE [--change-file FILE]");
System.exit(1);
}
} catch (ParseException e) {
logger.fatal("Error parsing arguments: ", e);
System.exit(0);
}

String sourceDir = opts.getOptionValue("source", Config.get("data.sobi"));
String storageDir = opts.getOptionValue("storage", Config.get("data.json"));

if( sourceDir == null || storageDir == null ) {
throw new org.apache.commons.cli.ParseException("source and storage are both required parameters.");
}
Storage storage = new Storage(storageDir);


Timer timer = new Timer();
Collection<File> files = FileUtils.listFiles(new File(sourceDir), null, true);
Storage storage = new Storage(required[1]);
Collection<File> files = FileUtils.listFiles(new File(required[0]), null, true);
Collections.sort((List<File>)files, new FileNameComparator());

// Process each file individually, flushing changes to storage as necessary
Expand Down
26 changes: 14 additions & 12 deletions src/main/java/gov/nysenate/openleg/scripts/Push.java
View file
Expand Up @@ -3,7 +3,6 @@
import gov.nysenate.openleg.services.Lucene;
import gov.nysenate.openleg.services.ServiceBase;
import gov.nysenate.openleg.services.Varnish;
import gov.nysenate.openleg.util.Config;
import gov.nysenate.openleg.util.Storage;

import java.io.File;
Expand All @@ -13,7 +12,6 @@
import java.util.HashMap;

import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
Expand All @@ -37,23 +35,29 @@ public static HashMap<String, Storage.Status> parseChanges(Iterable<String> line
}

public static void main(String[] args) {
String[] required = null;
CommandLine opts = null;
try {
Options options = new Options()
.addOption("t", "storage", true, "The path to the storage directory")
.addOption("l", "lucene", true, "Push changes to the Lucene service")
.addOption("v", "varnish", false, "Push changes to the Varnish service")
.addOption("f", "change-file", true, "Path of changeLog file.")
.addOption("c", "changes", true, "A newline delimited list of changes")
.addOption("h", "help", false, "Print this message");
opts = new PosixParser().parse(options, args);
required = opts.getArgs();
if(opts.hasOption("-h")) {
new HelpFormatter().printHelp("posix", options );
System.err.println("USAGE: Push STORAGE [--lucene LOCATION] [--varnish] [--change-file FILE] [--changes CHANGES]");
System.exit(0);

} else if (required.length != 1) {
System.err.println("Storage is a required argument.");
System.err.println("USAGE: Push STORAGE [--lucene LOCATION] [--varnish] [--change-file FILE] [--changes CHANGES]");
System.exit(1);
}
} catch (ParseException e) {
logger.fatal("Error parsing arguments: ", e);
System.exit(0);
System.exit(1);
}

// Parse the specified changes into a hash
Expand All @@ -63,19 +67,16 @@ public static void main(String[] args) {
File changeFile = new File(opts.getOptionValue("change-file"));
changes = parseChanges(FileUtils.readLines(changeFile, "UTF-8"));
} catch (IOException e) {
logger.fatal("Error reading change-file: "+opts.getOptionValue("changes"), e);
System.exit(0);
System.err.println("Error reading change-file: "+opts.getOptionValue("changes"));
System.exit(1);
}
} else if (opts.hasOption("changes")) {
changes = parseChanges(Arrays.asList(opts.getOptionValue("changes").split("\n")));
} else {
logger.fatal("Nothing to do. Specify changes or a change-file");
System.exit(0);
System.err.println("Changes to push must be specified with either --change-file or --changes");
System.exit(1);
}

String storageDir = opts.getOptionValue("storage", Config.get("data.json"));
Storage storage = new Storage(storageDir);

// Currently there is a Lucene hook and varnish hook, more to come
ArrayList<ServiceBase> services = new ArrayList<ServiceBase>();
if(opts.hasOption("lucene")) {
Expand All @@ -87,6 +88,7 @@ public static void main(String[] args) {
}

// Pass the change log through a set of service hooks
Storage storage = new Storage(required[0]);
for(ServiceBase service:services) {
try {
service.process(changes, storage);
Expand Down

0 comments on commit 11c9e83

Please sign in to comment.