Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Simplify scripts, enforce required args, and enable logging to file.

  • Loading branch information...
commit 11c9e83abbd7a61cff5367e97ccbbf1f4f02db81 1 parent 021178b
Graylin Kim GraylinKim authored
19 bin/process.sh
@@ -18,6 +18,7 @@ done
18 18 required_args=( "$source" "$work" "$dest" "$storage" "$lucene")
19 19 for required_arg in "${required_args[@]}"; do
20 20 if [ ! "$required_arg" ]; then
  21 + echo "All options are required: source, work, dest, storage, lucene";
21 22 echo $USAGE; exit 1 ;
22 23 elif [ ! -r "$required_arg" ]; then
23 24 echo "$required_arg must exist on the filesystem.";
@@ -33,9 +34,12 @@ fi
33 34 # Stop immediately if any command below fails
34 35 set -e
35 36
  37 +errorlog=$work/process.log
  38 +changelog=$work/change.log
  39 +
36 40 # Organize the change files and ingest to storage
37   -$BINDIR/run.sh collate $source $work;
38   -$BINDIR/run.sh ingest --source $work --storage $storage --change-file $work/change.log
  41 +$BINDIR/run.sh collate $source $work &>$errorlog;
  42 +$BINDIR/run.sh ingest $work $storage --change-file $changelog &>>$errorlog
39 43
40 44 # Migrate all processed files to $dest for archival purposes
41 45 year="`date +%Y`"
@@ -50,10 +54,11 @@ for file_type in "${file_types[@]}"; do
50 54 done
51 55
52 56 # Push the changes out from storage
53   -$BINDIR/run.sh push --lucene $lucene --storage $storage --change-file $work/change.log
  57 +$BINDIR/run.sh push $storage --lucene $lucene --change-file $changelog &>>$errorlog
54 58
55   -# Move the change.log to $dest for archiving as well
56   -if [ ! -r $dest/changelogs/ ]; then
57   - mkdir $dest/changelogs/
  59 +# Move the logs to $dest for archiving as well
  60 +if [ ! -r $dest/logs/ ]; then
  61 + mkdir $dest/logs/
58 62 fi
59   -mv $work/change.log $dest/changelogs/`date +CHANGELOG.D%Y%m%d.T%H%M%S.TXT`
  63 +mv $changelog $dest/logs/`date +D%Y%m%d.T%H%M%S.change.log`
  64 +mv $errorlog $dest/logs/`date +D%Y%m%d.T%H%M%S.error.log`
6 src/main/java/gov/nysenate/openleg/scripts/Collate.java
@@ -18,8 +18,8 @@
18 18
19 19 public static void main(String[] args) {
20 20 if (args.length < 2) {
21   - System.out.println("USAGE: Collate source [... source] dest");
22   - System.exit(0);
  21 + System.err.println("USAGE: Collate source [... source] dest");
  22 + System.exit(1);
23 23 }
24 24
25 25 File destDirectory = new File(args[args.length-1]).getAbsoluteFile();
@@ -47,7 +47,7 @@ public static void main(String[] args) {
47 47 }
48 48
49 49 for (File file : sources) {
50   - logger.debug("Processing: "+file);
  50 + logger.info("Processing: "+file);
51 51
52 52 int inc = 1;
53 53 String in = null;
27 src/main/java/gov/nysenate/openleg/scripts/Ingest.java
@@ -4,7 +4,6 @@
4 4 import gov.nysenate.openleg.processors.BillProcessor;
5 5 import gov.nysenate.openleg.processors.CalendarProcessor;
6 6 import gov.nysenate.openleg.processors.TranscriptProcessor;
7   -import gov.nysenate.openleg.util.Config;
8 7 import gov.nysenate.openleg.util.Storage;
9 8 import gov.nysenate.openleg.util.Storage.Status;
10 9 import gov.nysenate.openleg.util.Timer;
@@ -20,7 +19,6 @@
20 19 import javax.xml.bind.UnmarshalException;
21 20
22 21 import org.apache.commons.cli.CommandLine;
23   -import org.apache.commons.cli.HelpFormatter;
24 22 import org.apache.commons.cli.Options;
25 23 import org.apache.commons.cli.ParseException;
26 24 import org.apache.commons.cli.PosixParser;
@@ -43,36 +41,33 @@ public int compare(File a, File b) {
43 41 }
44 42
45 43 public static void main(String[] args) throws Exception {
  44 + String[] required = null;
46 45 CommandLine opts = null;
47 46 try {
48 47 Options options = new Options()
49 48 .addOption("h", "help", false, "Print this message")
  49 + .addOption("f", "change-file", true, "The path to store the changes");
50 50 //.addOption("dt", "document-type", true, "Type of document being indexed with -id (REQUIRED WITH -id).. (bill|calendar|agenda|transcript)")
51 51 //.addOption("id", "index-document", true, "Index JSON document specified by argument (path to file)")
52   - .addOption("t", "storage", true, "The path to the storage directory")
53   - .addOption("s", "source", true, "The path to the files to ingest")
54   - .addOption("f", "change-file", true, "The path to store the changes");
55 52 opts = new PosixParser().parse(options, args);
  53 + required = opts.getArgs();
56 54 if(opts.hasOption("-h")) {
57   - new HelpFormatter().printHelp("posix", options );
  55 + System.out.println("USAGE: Ingest SOURCE STORAGE [--change-file FILE]");
58 56 System.exit(0);
  57 +
  58 + } else if (required.length != 2) {
  59 + System.err.println("Both source and storage directories are required.");
  60 + System.err.println("USAGE: Ingest SOURCE STORAGE [--change-file FILE]");
  61 + System.exit(1);
59 62 }
60 63 } catch (ParseException e) {
61 64 logger.fatal("Error parsing arguments: ", e);
62 65 System.exit(0);
63 66 }
64 67
65   - String sourceDir = opts.getOptionValue("source", Config.get("data.sobi"));
66   - String storageDir = opts.getOptionValue("storage", Config.get("data.json"));
67   -
68   - if( sourceDir == null || storageDir == null ) {
69   - throw new org.apache.commons.cli.ParseException("source and storage are both required parameters.");
70   - }
71   - Storage storage = new Storage(storageDir);
72   -
73   -
74 68 Timer timer = new Timer();
75   - Collection<File> files = FileUtils.listFiles(new File(sourceDir), null, true);
  69 + Storage storage = new Storage(required[1]);
  70 + Collection<File> files = FileUtils.listFiles(new File(required[0]), null, true);
76 71 Collections.sort((List<File>)files, new FileNameComparator());
77 72
78 73 // Process each file individually, flushing changes to storage as necessary
26 src/main/java/gov/nysenate/openleg/scripts/Push.java
@@ -3,7 +3,6 @@
3 3 import gov.nysenate.openleg.services.Lucene;
4 4 import gov.nysenate.openleg.services.ServiceBase;
5 5 import gov.nysenate.openleg.services.Varnish;
6   -import gov.nysenate.openleg.util.Config;
7 6 import gov.nysenate.openleg.util.Storage;
8 7
9 8 import java.io.File;
@@ -13,7 +12,6 @@
13 12 import java.util.HashMap;
14 13
15 14 import org.apache.commons.cli.CommandLine;
16   -import org.apache.commons.cli.HelpFormatter;
17 15 import org.apache.commons.cli.Options;
18 16 import org.apache.commons.cli.ParseException;
19 17 import org.apache.commons.cli.PosixParser;
@@ -37,23 +35,29 @@
37 35 }
38 36
39 37 public static void main(String[] args) {
  38 + String[] required = null;
40 39 CommandLine opts = null;
41 40 try {
42 41 Options options = new Options()
43   - .addOption("t", "storage", true, "The path to the storage directory")
44 42 .addOption("l", "lucene", true, "Push changes to the Lucene service")
45 43 .addOption("v", "varnish", false, "Push changes to the Varnish service")
46 44 .addOption("f", "change-file", true, "Path of changeLog file.")
47 45 .addOption("c", "changes", true, "A newline delimited list of changes")
48 46 .addOption("h", "help", false, "Print this message");
49 47 opts = new PosixParser().parse(options, args);
  48 + required = opts.getArgs();
50 49 if(opts.hasOption("-h")) {
51   - new HelpFormatter().printHelp("posix", options );
  50 + System.err.println("USAGE: Push STORAGE [--lucene LOCATION] [--varnish] [--change-file FILE] [--changes CHANGES]");
52 51 System.exit(0);
  52 +
  53 + } else if (required.length != 1) {
  54 + System.err.println("Storage is a required argument.");
  55 + System.err.println("USAGE: Push STORAGE [--lucene LOCATION] [--varnish] [--change-file FILE] [--changes CHANGES]");
  56 + System.exit(1);
53 57 }
54 58 } catch (ParseException e) {
55 59 logger.fatal("Error parsing arguments: ", e);
56   - System.exit(0);
  60 + System.exit(1);
57 61 }
58 62
59 63 // Parse the specified changes into a hash
@@ -63,19 +67,16 @@ public static void main(String[] args) {
63 67 File changeFile = new File(opts.getOptionValue("change-file"));
64 68 changes = parseChanges(FileUtils.readLines(changeFile, "UTF-8"));
65 69 } catch (IOException e) {
66   - logger.fatal("Error reading change-file: "+opts.getOptionValue("changes"), e);
67   - System.exit(0);
  70 + System.err.println("Error reading change-file: "+opts.getOptionValue("changes"));
  71 + System.exit(1);
68 72 }
69 73 } else if (opts.hasOption("changes")) {
70 74 changes = parseChanges(Arrays.asList(opts.getOptionValue("changes").split("\n")));
71 75 } else {
72   - logger.fatal("Nothing to do. Specify changes or a change-file");
73   - System.exit(0);
  76 + System.err.println("Changes to push must be specified with either --change-file or --changes");
  77 + System.exit(1);
74 78 }
75 79
76   - String storageDir = opts.getOptionValue("storage", Config.get("data.json"));
77   - Storage storage = new Storage(storageDir);
78   -
79 80 // Currently there is a Lucene hook and varnish hook, more to come
80 81 ArrayList<ServiceBase> services = new ArrayList<ServiceBase>();
81 82 if(opts.hasOption("lucene")) {
@@ -87,6 +88,7 @@ public static void main(String[] args) {
87 88 }
88 89
89 90 // Pass the change log through a set of service hooks
  91 + Storage storage = new Storage(required[0]);
90 92 for(ServiceBase service:services) {
91 93 try {
92 94 service.process(changes, storage);

0 comments on commit 11c9e83

Please sign in to comment.
Something went wrong with that request. Please try again.