diff --git a/csharp/Azavea.NijPredictivePolicing.AcsImporter/ImportJob.cs b/csharp/Azavea.NijPredictivePolicing.AcsImporter/ImportJob.cs index ec32adb..4e54f79 100644 --- a/csharp/Azavea.NijPredictivePolicing.AcsImporter/ImportJob.cs +++ b/csharp/Azavea.NijPredictivePolicing.AcsImporter/ImportJob.cs @@ -412,14 +412,7 @@ public bool ExecuteJob() manager.AddStrippedGEOIDcolumn = (!string.IsNullOrEmpty(this.AddStrippedGEOIDcolumn)); manager.AddGeometryAttributesToOutput = (!string.IsNullOrEmpty(this.AddGeometryAttributesToOutput)); manager.OutputFolder = FileUtilities.CleanPath(OutputFolder); - - - //if (!string.IsNullOrEmpty(this.WorkingFolder)) - //{ - // manager.WorkingPath = FileUtilities.SafePathEnsure(this.WorkingFolder); - //} manager.IncludeEmptyGridCells = (!string.IsNullOrEmpty(this.IncludeEmptyGridCells)); - //manager.SRID = Utilities.GetAs(this.ExportFilterSRID, -1); if (FileUtilities.SafePathEnsure(OutputFolder) != OutputFolder) @@ -617,30 +610,15 @@ public void SaveJobFile(string filename) val = val + " #" + Utilities.GetAs(val, BoundaryLevels.census_blockgroups); } - - - //sb.Append("#").Append(nl); sb.Append("# ").Append(arg.Description).Append(nl); - //sb.Append("#").Append(nl); sb.Append("-" + arg.Flag).Append(" ").Append(val).Append(nl); sb.Append(nl); - - - - //object defval = (prop.PropertyType == typeof(AcsState)) ? (object)AcsState.None : null; - //prop.GetValue(this, Utilities.GetAsType(prop.PropertyType, contents, defval), null); - - //break; } File.WriteAllText(filename, sb.ToString()); } - - - - public bool WorkOffline { get; set; } } } diff --git a/csharp/Azavea.NijPredictivePolicing.AcsImporter/Main.cs b/csharp/Azavea.NijPredictivePolicing.AcsImporter/Main.cs index 469d6ec..d7860d5 100644 --- a/csharp/Azavea.NijPredictivePolicing.AcsImporter/Main.cs +++ b/csharp/Azavea.NijPredictivePolicing.AcsImporter/Main.cs @@ -35,6 +35,9 @@ namespace Azavea.NijPredictivePolicing.ACSAlchemist public class Program { private static ILog _log = null; + + public const string DefaultConfigPath = "configs/AcsAlchemist.json.config"; + protected static void Init() { try @@ -99,10 +102,12 @@ public static void ShowCopyrightAndLicense() /// protected static void LoadConfigFile() { - Settings.ConfigFile = new Config(Path.Combine(Settings.ApplicationPath, "AcsAlchemist.json.config")); + string configName = Path.Combine(Settings.ApplicationPath, DefaultConfigPath); + Settings.ConfigFile = new Config(configName); if (Settings.ConfigFile.IsEmpty()) { - Settings.RestoreDefaults(); + _log.FatalFormat("Config file {0} missing or empty! Cannot continue, exiting", DefaultConfigPath); + Environment.Exit((int)Constants.ExitCodes.BadConfig); } //search for our 'per-year' config files that explain which paths to check, file naming, etc. diff --git a/csharp/Azavea.NijPredictivePolicing.AcsImporterGui/FormController.cs b/csharp/Azavea.NijPredictivePolicing.AcsImporterGui/FormController.cs index 29d814b..cef5959 100644 --- a/csharp/Azavea.NijPredictivePolicing.AcsImporterGui/FormController.cs +++ b/csharp/Azavea.NijPredictivePolicing.AcsImporterGui/FormController.cs @@ -147,16 +147,14 @@ public List AvailableProjections + #region Initialization Boilerplate + private static ILog _log = null; + public const string DefaultConfigPath = "configs/AcsAlchemist.json.config"; - - #region Initialization Boilerplate - - - private static ILog _log = null; internal void InitLogging(IAppender appenderObj) { if (_log != null) @@ -193,14 +191,9 @@ protected void ShowWelcomeScreen() _log.Debug("Welcome to ACS Alchemist"); _log.Debug(""); - _log.Debug(@"This project was supported by Award No. 2010-DE-BX-K004, awarded by the National Institute of Justice, Office of Justice Programs, U.S. Department of Justice. The opinions, findings, and conclusions or recommendations expressed in this software are those of the author(s) and do not necessarily reflect those of the Department of Justice or Temple University. The software was developed by Azavea in connection with the National Institute of Justice grant awarded to Jerry Ratcliffe and Ralph Taylor of Temple University's Center for Security and Crime Science. The source code is released under a GPLv3 license and is available at: https://github.com/azavea/acs-alchemist"); _log.Debug(""); - - //_log.Debug(@"This project was supported by Award No. 2010-DE-BX-K004, awarded by the National Institute of Justice, Office of Justice Programs, U.S. Department of Justice. The opinions, findings, and conclusions or recommendations expressed in this software are those of the author(s) and do not necessarily reflect those of the Department of Justice. The source code is released under a GPLv3 license and is available at: https://github.com/azavea/acs-alchemist"); - - //_log.Debug(""); } protected void ShowCopyrightAndLicense() @@ -220,10 +213,12 @@ protected void ShowCopyrightAndLicense() /// protected void LoadConfigFile() { - Settings.ConfigFile = new Config(Path.Combine(Settings.ApplicationPath, "AcsAlchemist.json.config")); + string configName = Path.Combine(Settings.ApplicationPath, DefaultConfigPath); + Settings.ConfigFile = new Config(configName); if (Settings.ConfigFile.IsEmpty()) { - Settings.RestoreDefaults(); + _log.FatalFormat("Config file {0} missing or empty! Cannot continue, exiting", DefaultConfigPath); + Environment.Exit((int)Constants.ExitCodes.BadConfig); } Settings.LoadYearConfigs(); @@ -235,27 +230,15 @@ protected void LoadConfigFile() /// internal void Initialize() { - //InitLogging(); - LoadConfigFile(); ShowWelcomeScreen(); ShowCopyrightAndLicense(); - - //this.JobInstance = new ImportJob(); NewDefaultJobInstance(); } #endregion Initialization Boilerplate - - - - - - - - /// /// Performs some sanity checks on our variables file /// (does it exist, can I read it, does it have at least one variable, etc.) diff --git a/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/AcsState.cs b/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/AcsState.cs index 517d27c..54e9b3c 100644 --- a/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/AcsState.cs +++ b/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/AcsState.cs @@ -39,7 +39,7 @@ public enum AcsState Colorado = 08, Connecticut = 09, Delaware = 10, - DistrictofColumbia = 11, + //DistrictofColumbia = 11, // This appears to have been removed from ACS downloads at some point Florida = 12, Georgia = 13, Hawaii = 15, @@ -86,8 +86,6 @@ public enum AcsState PuertoRico = 72, None = -1 - - } @@ -106,17 +104,6 @@ public static string StateToCensusName(AcsState state) return state.ToString(); } - ///// - ///// Shows how to use the settings - ///// - ///// - ///// - //public static string StateToCurrentBlockGroupFilename(AcsState state) - //{ - // return string.Concat(state.ToString(), Settings.BlockGroupsDataTableSuffix); - //} - - /// /// Get's a pretty name for a state /// diff --git a/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/FileFormats/ShapefileHelper.cs b/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/FileFormats/ShapefileHelper.cs index c57be9e..7bf768b 100644 --- a/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/FileFormats/ShapefileHelper.cs +++ b/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/FileFormats/ShapefileHelper.cs @@ -78,7 +78,6 @@ public static bool LoadShapefile(string filename, string tableName, IDataClient return false; } - //string workingDirectory = FileUtilities.SafePathEnsure(TempPath, Path.GetFileNameWithoutExtension(filename)); using (DbConnection conn = DbClient.GetConnection()) { if (!ShapefileHelper.ImportShapefile(conn, DbClient, filename, tableName, (int)CRS.AuthorityCode)) @@ -213,7 +212,7 @@ public static bool MakeOutputProjFile(string sourceProjectionFilename, string de if (File.Exists(sourceProjectionFilename)) { - File.Copy(sourceProjectionFilename, prjFileName, true); //File.WriteAllText(prjFileName, File.ReadAllText(wktProjFilename)); + File.Copy(sourceProjectionFilename, prjFileName, true); } else { @@ -244,18 +243,6 @@ public static GeometryFactory GetGeomFactory() return new GeometryFactory(new PrecisionModel(), 4269); } - ///// - ///// Helper function for cre - ///// - ///// - ///// - ///// - //public static System.Data.DataTable GetTable(string shapefilename, AcsState acsState) - //{ - // return Shapefile.CreateDataTable(shapefilename, acsState.ToString(), ShapefileHelper.GetGeomFactory()); - //} - - /// /// Consumes an ADO.net datatable and correctly initializes a Shapefile header object /// @@ -295,8 +282,6 @@ public static void AddColumn(DbaseFileHeader header, string columnName, Type t) } else if (t == typeof(DateTime)) { - // D stores only the date - //retVal.AddColumn(shapefileColumnName, 'D', 8, 0); header.AddColumn(columnName, 'C', 22, 0); } else if (t == typeof(float) || t == typeof(double) || t == typeof(decimal)) @@ -374,7 +359,11 @@ public static string GetRemoteShapefileURL(BoundaryLevels level, string stateFip /// public static bool IsForbiddenShapefileName(string name) { - string[] levels = new string[] { + if (string.IsNullOrEmpty(name)) + { + return true; + } + var levels = (new List { Settings.ShapeFileBlockGroupFilename, Settings.ShapeFileTractFilename, Settings.ShapeFileCountySubdivisionsFilename, @@ -382,7 +371,7 @@ public static bool IsForbiddenShapefileName(string name) Settings.ShapeFileThreeDigitZipsFilename, Settings.ShapeFileFiveDigitZipsFilename, Settings.ShapeFileCountiesFilename - }; + }).Where(s => !string.IsNullOrEmpty(s)); StringBuilder newRegex = new StringBuilder(512); foreach (string level in levels) diff --git a/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/Transfer/AcsDataManager.cs b/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/Transfer/AcsDataManager.cs index d936810..6a9fcc9 100644 --- a/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/Transfer/AcsDataManager.cs +++ b/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/Transfer/AcsDataManager.cs @@ -164,7 +164,6 @@ public AcsDataManager(AcsState aState, string workingFolder, string year) if (!string.IsNullOrEmpty(workingFolder)) { //override where we're storing temporary files - //Settings.AppDataDirectory = FileUtilities.SafePathEnsure(workingFolder, Settings.RequestedYear); Settings.AppDataDirectory = FileUtilities.SafePathEnsure(workingFolder); } _log.InfoFormat("Working directory is {0} ", workingFolder); @@ -415,12 +414,6 @@ public bool DownloadAndImportShapefile(string desiredUrl, string destFilepath, s var client = DbClient; using (var conn = client.GetConnection()) { - //if (DataClient.HasTable(conn, client, tablename)) - //{ - // _log.DebugFormat("{0} table already exists, skipping...", tablename); - // return true; - //} - //else if (DataClient.HasTable(conn, client, tablename)) { @@ -1186,13 +1179,6 @@ public GetGeometryRowKey GetGeometryRowKeyGenerator() foreach (DataRow row in wholeShapeTable.Rows) { - //string county = Utilities.GetAs(row["COUNTY"], "-1"); - //string tract = Utilities.GetAs(row["TRACT"], "-1"); - //string blkgroup = Utilities.GetAs(row["BLKGROUP"], "-1"); - //if (tract.Trim().Length != 6) - // tract += "00"; - //string key = string.Format("{0}_{1}_{2}", county, tract, blkgroup); - string key = keyDelegate(row); if (geomKeys.ContainsKey(key)) @@ -1270,8 +1256,6 @@ public bool IsIncluded(IGeometry geom, List filteringGeoms) var fg = filt.GetGeometryN(g); if (fg.Intersects(geom)) { - //double commonArea = fg.Intersection(geom).Area; - //if(commonArea > 0.01 * fg.Area || commonArea > 0.01 * geom.Area) return true; } } @@ -1306,12 +1290,6 @@ public List GetShapeFeaturesToExport(string tableName, bool spatialFilt { destCRS = Utilities.GetCoordinateSystemByWKTFile(this.OutputProjectionFilename); reprojector = Utilities.BuildTransformationObject(GeographicCoordinateSystem.WGS84, destCRS); - - //Reproject everything in this file to the requested projection... - //exportFeatures = Utilities.ReprojectFeaturesTo(exportFeatures, this.OutputProjectionFilename); - - //THESE MUST BE PROVIDED ALREADY PROJECTED! - //filteringGeoms = Utilities.ReprojectFeaturesTo(filteringGeoms, this.OutputProjectionFilename); } //TODO: @@ -1333,7 +1311,6 @@ public List GetShapeFeaturesToExport(string tableName, bool spatialFilt } } - GisSharpBlog.NetTopologySuite.IO.WKBReader binReader = new WKBReader( ShapefileHelper.GetGeomFactory()); @@ -1484,8 +1461,10 @@ public bool ExportShapefile(string tableName) DbaseFileHeader header = null; using (var conn = DbClient.GetConnection()) { - //Dictionary shapeDict = GetShapeRowsByLOGRECNO(conn); - var variablesDT = DataClient.GetMagicTable(conn, DbClient, string.Format("SELECT * FROM \"{0}\" where 0 = 1 ", tableName)); + var variablesDT = DataClient.GetMagicTable( + conn, + DbClient, + string.Format("SELECT * FROM \"{0}\" where 0 = 1 ", tableName)); header = ShapefileHelper.SetupHeader(variablesDT); } @@ -1538,9 +1517,10 @@ public bool ExportShapefile(string tableName) } catch (FileNotFoundException notFound) { - _log.Error("A needed file couldn't be found: " + notFound.FileName); + string msg = "A needed file couldn't be found: " + notFound.FileName; + _log.Error(msg); _log.Fatal("The export cannot continue. Exiting..."); - Environment.Exit(-1); + throw new ApplicationException(msg); } catch (Exception ex) { @@ -1550,8 +1530,6 @@ public bool ExportShapefile(string tableName) } - - /// /// Retrieves the contents of the specified table, and writes them to a shapefile /// @@ -1618,9 +1596,6 @@ public bool ExportGriddedShapefile(string tableName) var features = new List(exportFeatures.Count); - //var cellStepPoint = Utilities.GetCellFeetForProjection(GridCellWidthFeet, GridCellHeightFeet); - //double cellWidth = cellStepPoint.X; - //double cellHeight = cellStepPoint.Y; double cellWidth = GridCellWidth; double cellHeight = GridCellHeight; @@ -1780,9 +1755,10 @@ public bool ExportGriddedShapefile(string tableName) } catch (FileNotFoundException notFound) { - _log.Error("A needed file couldn't be found: " + notFound.FileName); + string msg = "A needed file couldn't be found: " + notFound.FileName; + _log.Error(msg); _log.Fatal("The export cannot continue. Exiting..."); - Environment.Exit(-1); + throw new ApplicationException(msg); } catch (Exception ex) { @@ -1817,13 +1793,6 @@ public Envelope GetGridEnvelope() } - - - - - - - public void Dispose() { if (DbClient != null) @@ -1874,8 +1843,6 @@ public void Cancel() _cancelled = true; } - - public bool WorkOffline { get; set; } } } \ No newline at end of file diff --git a/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/Transfer/FileDownloader.cs b/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/Transfer/FileDownloader.cs index e451be4..2a64ff4 100644 --- a/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/Transfer/FileDownloader.cs +++ b/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/Transfer/FileDownloader.cs @@ -141,9 +141,10 @@ public static bool GetFileByURL(string desiredURL, string filePath, ref bool can } catch (UnauthorizedAccessException cantWriteEx) { - _log.Error("The importer couldn't save the file, please run this application as administrator, or set the output directory."); + string msg = "The importer couldn't save the file, please run this application as administrator, or set the output directory."; + _log.Error(msg); _log.Fatal("The importer cannot continue. Exiting..."); - Environment.Exit(-1); + throw new ApplicationException(msg); } catch (Exception ex) { diff --git a/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/Transfer/FileLocator.cs b/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/Transfer/FileLocator.cs index 4b7a8e4..5a9eaa9 100644 --- a/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/Transfer/FileLocator.cs +++ b/csharp/Azavea.NijPredictivePolicing.AcsImporterLibrary/Transfer/FileLocator.cs @@ -150,16 +150,6 @@ public static bool ExpandZipFile(string sourceFile, string destPath) { try { - //if (Directory.Exists(destPath)) - //{ - // //var files = Directory.GetFiles(destPath); - // //if ((files != null) && (files.Length > 0)) - // //{ - // // _log.Debug("State file is already expanded"); - // // return true; - // //} - //} - return FileUtilities.UnzipFileTo(destPath, sourceFile); } catch (Exception ex) diff --git a/csharp/Azavea.NijPredictivePolicing.Common/Constants.cs b/csharp/Azavea.NijPredictivePolicing.Common/Constants.cs index d0cc875..6d7a34a 100644 --- a/csharp/Azavea.NijPredictivePolicing.Common/Constants.cs +++ b/csharp/Azavea.NijPredictivePolicing.Common/Constants.cs @@ -31,8 +31,9 @@ public static class Constants { public enum ExitCodes { - BAD_ARGUMENTS = -1, - OK = 0 + BadArguments = -1, + OK = 0, + BadConfig = 1 } public const string Warning_MissingProjection = @" @@ -48,7 +49,6 @@ public enum ExitCodes public static class DbConstants { - //public const string TABLE_Geographies = "geographies"; public const string TABLE_ColumnMappings = "columnMappings"; public const string TABLE_DesiredColumns = "desiredColumns";