Permalink
Browse files

Added support for loading 1:50000 Scale Gazetter Ordnance Survey data…

… file to SQL Server
  • Loading branch information...
1 parent 4140c06 commit 226101f812394fbec3d7a4d2122b364c5e829062 @AdaTheDev committed Mar 27, 2011
View
BIN OSCodePointDataImport.suo
Binary file not shown.
View
6 OSCodePointDataImport/CodePointArgParser.cs → ...ort/ArgumentParsers/CodePointArgParser.cs
@@ -5,8 +5,8 @@
namespace OSCodePointDataImport
{
- class CodePointArgParser : CommandLineArgs
- {
+ class CodePointArgParser : CommandLineArgParser<CodePointOptions>
+ {
/// <summary>
/// Directory containing the downloaded Ordnance Survey Code-Point CSV data files.
/// </summary>
@@ -20,7 +20,7 @@ class CodePointArgParser : CommandLineArgs
public override void Parse(string[] args)
{
base.Parse(args);
- if (args.Length >= 5) DataFileDirectory = args[5];
+ if (args.Length >= 6) ImportOptions.DataFileDirectory = args[5];
}
/// <summary>
View
14 OSCodePointDataImport/CommandLineArgs.cs → ...t/ArgumentParsers/CommandLineArgParser.cs
@@ -26,7 +26,7 @@ namespace OSCodePointDataImport
/// e.g. currently, importing of Code-Point data is supported - CodePointArgParser is used to parse any Code-Point
/// specific arguments in addition to this base parsing functionality.
/// </summary>
- abstract class CommandLineArgs
+ abstract class CommandLineArgParser<T> where T: Options,new()
{
/// <summary>
/// What type of data want to import:
@@ -54,7 +54,9 @@ abstract class CommandLineArgs
/// </summary>
public string TableName { get; set; }
-
+ protected T _options = new T();
+ public T ImportOptions { get { return _options;} }
+
/// <summary>
/// Parse the array of command line arguments.
/// </summary>
@@ -68,10 +70,10 @@ abstract class CommandLineArgs
public virtual void Parse(string[] args)
{
int numOfArgs = args.Length;
- if (numOfArgs >= 1) ServerName = args[1];
- if (numOfArgs >= 2) DBName = args[2];
- if (numOfArgs >= 3) SchemaName = args[3];
- if (numOfArgs >= 4) TableName = args[4];
+ if (numOfArgs >= 1) ImportOptions.ServerName = args[1];
+ if (numOfArgs >= 2) ImportOptions.DBName = args[2];
+ if (numOfArgs >= 3) ImportOptions.SchemaName = args[3];
+ if (numOfArgs >= 4) ImportOptions.TableName = args[4];
}
/// <summary>
View
12 OSCodePointDataImport/ArgumentParsers/IArgParser.cs
@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace OSCodePointDataImport
+{
+ interface IArgParser
+ {
+ Options ImportOptions { get; }
+ }
+}
View
29 OSCodePointDataImport/ArgumentParsers/ScaleGazetteerArgParser.cs
@@ -0,0 +1,29 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
+
+namespace OSCodePointDataImport
+{
+ class ScaleGazetteerArgParser : CommandLineArgParser<ScaleGazetteerOptions>
+ {
+
+ internal override void Validate()
+ {
+ base.Validate();
+ if (String.IsNullOrWhiteSpace(_options.CountyLookupTableName)) throw new ArgumentException("CountyLookupTableName argument must be supplied");
+ if (String.IsNullOrWhiteSpace(_options.FeatureLookupTableName)) throw new ArgumentException("FeatureLookupTableName argument must be supplied");
+ if (String.IsNullOrWhiteSpace(_options.DataFileName)) throw new ArgumentException("Datafile argument must be supplied");
+ if (!File.Exists(_options.DataFileName)) throw new ArgumentException("Datafile argument is not valid. File does not exist");
+ }
+
+ public override void Parse(string[] args)
+ {
+ base.Parse(args);
+ if (args.Length >= 6) _options.CountyLookupTableName = args[5];
+ if (args.Length >= 7) _options.FeatureLookupTableName = args[6];
+ if (args.Length >= 8) _options.DataFileName = args[7];
+ }
+ }
+}
View
36 OSCodePointDataImport/ImportFactory.cs
@@ -0,0 +1,36 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace OSCodePointDataImport
+{
+ class ImportFactory
+ {
+ //public static IArgParser GetArgParser(string importType)
+ //{
+ // switch (importType.ToUpperInvariant())
+ // {
+ // case "CODEPOINT":
+ // return new CodePointArgParser();
+ // case "GAZETTEER":
+ // return new ScaleGazetteerArgParser<ScaleGazetteerOptions>();
+ // default:
+ // throw new ArgumentException("Unsupported import type. Must be: CODEPOINT or GAZETTEER");
+ // }
+ //}
+
+ public static OSDataImporter GetDataImporter(string importType)
+ {
+ switch (importType.ToUpperInvariant())
+ {
+ case "CODEPOINT":
+ return new CodePointDataImporter();
+ case "GAZETTEER":
+ return new ScaleGazetteerDataImporter();
+ default:
+ throw new ArgumentException("Unsupported import type. Must be: CODEPOINT or GAZETTEER");
+ }
+ }
+ }
+}
View
69 ...ePointDataImport/CodePointDataImporter.cs → ...Import/Importers/CodePointDataImporter.cs
@@ -62,19 +62,25 @@ public int LoadData(string serverName, string databaseName, string schemaName, s
using (DataTable data = ReadDataFromFiles(dataFileDirectory))
{
// Now load the data to the DB
- LoadRowsToDatabase(connection, data, schemaName, tableName);
- result = data.Rows.Count;
+ result = LoadRowsToDatabase(connection, data, schemaName, tableName);
}
// Calculate and create rows for each postcode district (e.g. AB12) and sector (e.g. AB12 3), defining the Lon/Lat
// coordinates as the straight average of all the postcodes within that area.
CalculateDistrictsAndSectors(connection, schemaName, tableName);
// Now set the GeoLocation column (Geography type) based on the Latitude/Longitude
- SetGeoDataAndFinaliseTable(connection, schemaName, tableName);
+ SetGeoColumn(connection, schemaName, tableName);
+ // Set PK on table
+ SetPrimaryKey(connection, schemaName, tableName, new string[] { "OutwardCode", "InwardCode" });
}
return result;
}
+ public int LoadData(CodePointOptions options)
+ {
+ return LoadData(options.ServerName, options.DBName, options.SchemaName, options.TableName, options.DataFileDirectory);
+ }
+
/// <summary>
/// Creates a new table in the database ready to receive the Code-Point data.
/// </summary>
@@ -176,32 +182,7 @@ private DataTable ReadDataFromFiles(string directory)
Console.WriteLine("Done! {0} rows of data prepared", data.Rows.Count.ToString());
return data;
- }
-
- /// <summary>
- /// Loads the supplied data to the newly created table in the database.
- /// </summary>
- /// <param name="connection">SQL Server database connection</param>
- /// <param name="data">DataTable containing the Post Codes with their Lon/Lat coordinates</param>
- /// <param name="schemaName">name of schema the table belongs to</param>
- /// <param name="tableName">table to load the Post Code data to</param>
- private void LoadRowsToDatabase(SqlConnection connection, DataTable data, string schemaName, string tableName)
- {
- Console.WriteLine("Bulk loading the data into {0}.{1}...", schemaName, tableName);
-
- using (SqlBulkCopy bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.TableLock, null))
- {
- bulkCopy.BulkCopyTimeout = 60;
- bulkCopy.DestinationTableName = String.Format("[{0}].[{1}]", schemaName, tableName);
- bulkCopy.ColumnMappings.Add("OutwardCode", "OutwardCode");
- bulkCopy.ColumnMappings.Add("InwardCode", "InwardCode");
- bulkCopy.ColumnMappings.Add("Longitude", "Longitude");
- bulkCopy.ColumnMappings.Add("Latitude", "Latitude");
- bulkCopy.WriteToServer(data);
- }
-
- Console.WriteLine("Done!");
- }
+ }
/// <summary>
/// Calculates a simple average Lon/Lat for each postcode district (e.g. AB12) and sector (e.g. AB12 3),
@@ -234,34 +215,6 @@ UNION ALL
Console.WriteLine("Done!");
}
- }
-
- /// <summary>
- /// Updates the loaded table to set the GeoLocation column to a Point based on the loaded Latitude and Longitude data.
- /// Then makes the PostCode column a clustered primary key.
- /// </summary>
- /// <param name="connection">SQL Server database connection</param>
- /// <param name="schemaName">name of the schema the table is in</param>
- /// <param name="tableName">name of the table containing the data</param>
- private void SetGeoDataAndFinaliseTable(SqlConnection connection, string schemaName, string tableName)
- {
- using (SqlCommand cmd = new SqlCommand())
- {
- cmd.Connection = connection;
- cmd.CommandTimeout = 600;
- Console.WriteLine("Setting GeoLocation GEOGRAPHY column to point based on Latitude and Longitude...");
- cmd.CommandText =
- String.Format(@"UPDATE [{0}].[{1}] SET GeoLocation = geography::Point([Latitude], [Longitude], 4326);",
- schemaName, tableName);
- cmd.ExecuteNonQuery();
- Console.WriteLine("Done!");
-
- Console.WriteLine("Setting PRIMARY KEY on table...");
- cmd.CommandText = String.Format("ALTER TABLE [{0}].[{1}] ADD CONSTRAINT [PK_{1}] PRIMARY KEY CLUSTERED ([OutwardCode], [InwardCode]);",
- schemaName, tableName);
- cmd.ExecuteNonQuery();
- Console.WriteLine("Done!");
- }
- }
+ }
}
}
View
121 OSCodePointDataImport/Importers/OSDataImporter.cs
@@ -0,0 +1,121 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using TDPG.GeoCoordConversion;
+using System.Data.SqlClient;
+using System.Data;
+
+namespace OSCodePointDataImport
+{
+ abstract class OSDataImporter
+ {
+ /// <summary>
+ /// Converts northing and easting coordinates into Longitude/Latitude coordinates in the WGS84 coordinate system.
+ /// </summary>
+ /// <param name="northing">northing coordinate</param>
+ /// <param name="easting">easting coordinate</param>
+ /// <returns>converted coordinates</returns>
+ protected PolarGeoCoordinate ConvertToLonLat(double northing, double easting)
+ {
+ // Use GeoCoordConversion DLL to convert the Eastings & Northings to Lon/Lat
+ // coordinates in the WGS84 system. The DLL was pulled from: http://code.google.com/p/geocoordconversion/
+ // and is available under the GNU General Public License v3: http://www.gnu.org/licenses/gpl.html
+ GridReference gridRef = new GridReference((long)easting, (long)northing);
+ PolarGeoCoordinate polarCoord = GridReference.ChangeToPolarGeo(gridRef);
+ return PolarGeoCoordinate.ChangeCoordinateSystem(polarCoord, CoordinateSystems.WGS84);
+ }
+
+ /// <summary>
+ /// Loads the supplied data to the newly created table in the database.
+ /// </summary>
+ /// <param name="connection">SQL Server database connection</param>
+ /// <param name="data">DataTable containing the data to load</param>
+ /// <param name="schemaName">name of schema the table belongs to</param>
+ /// <param name="tableName">table to load the data to</param>
+ protected int LoadRowsToDatabase(SqlConnection connection, DataTable data, string schemaName, string tableName)
+ {
+ Console.WriteLine("Bulk loading the data into {0}.{1}...", schemaName, tableName);
+
+ using (SqlBulkCopy bulkCopy = new SqlBulkCopy(connection, SqlBulkCopyOptions.TableLock, null))
+ {
+ bulkCopy.BulkCopyTimeout = 600;
+ bulkCopy.DestinationTableName = String.Format("[{0}].[{1}]", schemaName, tableName);
+ foreach(DataColumn column in data.Columns)
+ {
+ bulkCopy.ColumnMappings.Add(column.ColumnName, column.ColumnName);
+ }
+ bulkCopy.WriteToServer(data);
+ }
+
+ Console.WriteLine("Done! {0} rows inserted", data.Rows.Count.ToString());
+ return data.Rows.Count;
+
+ }
+
+ /// <summary>
+ /// Executes the supplied SQL.
+ /// </summary>
+ /// <param name="connection">db connection to execute against</param>
+ /// <param name="commandText">SQL to execute</param>
+ private void ExecCommand(SqlConnection connection, string commandText)
+ {
+ using (SqlCommand cmd = new SqlCommand())
+ {
+ cmd.Connection = connection;
+ cmd.CommandTimeout = 600;
+
+ cmd.CommandText = commandText;
+ cmd.ExecuteNonQuery();
+ Console.WriteLine("Done!");
+ }
+ }
+
+ /// <summary>
+ /// Set the GeoLocation column in the supplied table, to a GEOGRAPHY point based on the Latitude and Longitude columns
+ /// </summary>
+ /// <param name="connection">db connection to use</param>
+ /// <param name="schemaName">name of schema the table belongs to</param>
+ /// <param name="tableName">name of table to update</param>
+ protected void SetGeoColumn(SqlConnection connection, string schemaName, string tableName)
+ {
+ Console.WriteLine("Setting GeoLocation GEOGRAPHY column to point based on Latitude and Longitude...");
+
+ ExecCommand(connection, String.Format(@"UPDATE [{0}].[{1}] SET GeoLocation = geography::Point([Latitude], [Longitude], 4326);",
+ schemaName, tableName));
+ }
+
+ /// <summary>
+ /// Sets the primary key on the supplied table
+ /// </summary>
+ /// <param name="connection">db connection to use</param>
+ /// <param name="schemaName">name of schema the table belongs to</param>
+ /// <param name="tableName">name of the table to add the PK constraint on</param>
+ /// <param name="pkColumnNames">column or columns that form the PK</param>
+ protected void SetPrimaryKey(SqlConnection connection, string schemaName, string tableName, string[] pkColumnNames)
+ {
+ string pkCols = "[" + String.Join("],[", pkColumnNames) + "]";
+ Console.WriteLine("Setting PRIMARY KEY on table: {0}...", tableName);
+
+ ExecCommand(connection, String.Format("ALTER TABLE [{0}].[{1}] ADD CONSTRAINT [PK_{1}] PRIMARY KEY CLUSTERED ({2});",
+ schemaName, tableName, pkCols));
+ }
+
+ /// <summary>
+ /// Adds a foreign key constraint.
+ /// </summary>
+ /// <param name="connection">db connection to run the command against</param>
+ /// <param name="schemaName">schema name</param>
+ /// <param name="primaryKeyTableName">name of the primary key table</param>
+ /// <param name="primaryKeyColumnName">name of the primary key column</param>
+ /// <param name="foreignKeyTableName">name of the foreign key table</param>
+ /// <param name="foreignKeyColumnName">name of the foreign key column</param>
+ protected void SetForeignKey(SqlConnection connection, string schemaName, string primaryKeyTableName, string primaryKeyColumnName, string foreignKeyTableName, string foreignKeyColumnName)
+ {
+ Console.WriteLine("Setting FOREIGN KEY from {0} to {1}...", foreignKeyTableName, primaryKeyTableName);
+
+ ExecCommand(connection, String.Format("ALTER TABLE [{0}].[{1}] ADD CONSTRAINT [FK_{1}_{2}] FOREIGN KEY ([{3}]) REFERENCES {2}({4});",
+ schemaName, foreignKeyTableName, primaryKeyTableName, foreignKeyColumnName, primaryKeyColumnName));
+ }
+ }
+}
View
244 OSCodePointDataImport/Importers/ScaleGazetteerDataImporter.cs
@@ -0,0 +1,244 @@
+/*
+ Copyright 2010 - Adrian Hills
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.IO;
+using System.Data.SqlClient;
+using System.Data;
+using System.Globalization;
+using System.Collections;
+using TDPG.GeoCoordConversion;
+
+namespace OSCodePointDataImport
+{
+ /// <summary>
+ /// Functionality to load Ordnance Survey 1:50000 Scale Gazetteer data files to SQL Server.
+ /// The data files are available for download from: https://www.ordnancesurvey.co.uk/opendatadownload/products.html
+ /// </summary>
+ class ScaleGazetteerDataImporter : OSDataImporter
+ {
+ /// <summary>
+ /// Loads Scale Gazetteer CSV data file into a new table in SQL Server, creating 2 related lookup tables also
+ /// for the county codes and feature codes
+ /// </summary>
+ /// <param name="options">options</param>
+ /// <returns>number of rows loaded</returns>
+ public int LoadData(ScaleGazetteerOptions options)
+ {
+ return LoadData(options.ServerName, options.DBName, options.SchemaName, options.TableName, options.DataFileName, options.CountyLookupTableName, options.FeatureLookupTableName);
+ }
+
+ public int LoadData(string serverName, string databaseName, string schemaName, string tableName, string dataFile,
+ string countyLookupTableName, string featureLookupTableName)
+ {
+ // Basic validation
+ if (String.IsNullOrWhiteSpace(serverName)) throw new ArgumentException("ServerName must be supplied", "serverName");
+ if (String.IsNullOrWhiteSpace(databaseName)) throw new ArgumentException("DatabaseName must be supplied", "databaseName");
+ if (String.IsNullOrWhiteSpace(schemaName)) throw new ArgumentException("SchemaName must be supplied", "schemaName");
+ if (String.IsNullOrWhiteSpace(tableName)) throw new ArgumentException("TableName must be supplied", "tableName");
+ if (String.IsNullOrWhiteSpace(dataFile)) throw new ArgumentException("DataFile must be supplied", "dataFile");
+ if (String.IsNullOrWhiteSpace(countyLookupTableName)) throw new ArgumentException("CountyLookupTableName must be supplied", "countyLookupTableName");
+ if (String.IsNullOrWhiteSpace(featureLookupTableName)) throw new ArgumentException("FeatureLookupTableName must be supplied", "featureLookupTableName");
+ if (!File.Exists(dataFile)) throw new DirectoryNotFoundException("DataFile does not exist");
+ int result = 0;
+
+ using (SqlConnection connection = new SqlConnection(String.Format("Data Source={0};Initial Catalog={1};Integrated Security=SSPI;", serverName, databaseName)))
+ {
+ connection.Open();
+ // Create the table to import the data into. If it already exists, or the schema is not valid this will throw an exception.
+ PrepareTable(connection, schemaName, tableName, countyLookupTableName, featureLookupTableName);
+ // Read all the data in from the data files into a single DataTable in memory
+ using (DataSet data = ReadDataFromFile(dataFile))
+ {
+ // Load the point data to the DB
+ result = LoadRowsToDatabase(connection, data.Tables["Point"], schemaName, tableName);
+
+ // Load the County reference data to the DB
+ LoadRowsToDatabase(connection, data.Tables["County"], schemaName, countyLookupTableName);
+
+ // Load the Feature code reference data to the DB
+ LoadRowsToDatabase(connection, data.Tables["Feature"], schemaName, featureLookupTableName);
+ }
+ // Now set the GeoLocation column (Geography type) based on the Latitude/Longitude
+ SetGeoColumn(connection, schemaName, tableName);
+ // Set PK on the point data table.
+ SetPrimaryKey(connection, schemaName, tableName, new string[] { "SeqNo" });
+ // Set PK on the County lookup table
+ SetPrimaryKey(connection, schemaName, countyLookupTableName, new string[] { "Code" });
+ // Set PK on the Feature lookup table
+ SetPrimaryKey(connection, schemaName, featureLookupTableName, new string[] { "Code" });
+ // Set FK on point data table -> County lookup table
+ SetForeignKey(connection, schemaName, countyLookupTableName, "Code", tableName, "CountyCode");
+ // Set FK on point data table -> Feature lookup table
+ SetForeignKey(connection, schemaName, featureLookupTableName, "Code", tableName, "FeatureCode");
+
+ }
+
+ return result;
+ }
+
+ /// <summary>
+ /// Reads the data in from the Scale Gazetteer data file into a DataSet containing 3 datatables:
+ /// 1) main point data
+ /// 2) county lookup data
+ /// 3) feature lookup data
+ /// </summary>
+ /// <param name="dataFile">1:50000 scale gazetteer data file as supplied by Ordance Survey at
+ /// https://www.ordnancesurvey.co.uk/opendatadownload/products.html</param>
+ /// <returns>Dataset</returns>
+ private DataSet ReadDataFromFile(string dataFile)
+ {
+ DataSet ds = new DataSet();
+ DataTable pointData = new DataTable("Point");
+ DataTable countyData = new DataTable("County");
+ DataTable featureData = new DataTable("Feature");
+
+ pointData.Columns.Add("SeqNo", typeof(int));
+ pointData.Columns.Add("PlaceName", typeof(string));
+ pointData.Columns.Add("CountyCode", typeof(string));
+ pointData.Columns.Add("FeatureCode", typeof(string));
+ pointData.Columns.Add("Longitude", typeof(double));
+ pointData.Columns.Add("Latitude", typeof(double));
+
+ countyData.Columns.Add("Code", typeof(string));
+ countyData.Columns.Add("Name", typeof(string));
+
+ featureData.Columns.Add("Code", typeof(string));
+ featureData.Columns.Add("Description", typeof(string));
+
+ Console.WriteLine("Loading scale gazetteer data from file into memory...");
+ int sequenceNumber;
+ string placeName, featureCode, gmt;
+ double latDegrees, latMinutes, lonDegrees, lonMinutes, latitude, longitude;
+
+ DataRow row;
+ string[] lineData;
+ Dictionary<string, string> counties = new Dictionary<string, string>();
+ string countyCode, countyName;
+
+ using (StreamReader stream = new StreamReader(dataFile))
+ {
+ while (stream.Peek() >= 0)
+ {
+ lineData = stream.ReadLine().Split(':');
+ sequenceNumber = int.Parse(lineData[0]);
+ placeName = lineData[2];
+ latDegrees = double.Parse(lineData[4]);
+ latMinutes = double.Parse(lineData[5]);
+ lonDegrees = double.Parse(lineData[6]);
+ lonMinutes = double.Parse(lineData[7]);
+ gmt = lineData[10];
+ countyCode = lineData[11];
+ countyName = lineData[13];
+ featureCode = lineData[14];
+
+ latitude = latDegrees + ((latMinutes != 0) ? latMinutes / 60.0 : 0.0);
+ longitude = lonDegrees + ((lonMinutes != 0) ? lonMinutes / 60.0 : 0.0);
+
+ row = pointData.NewRow();
+ row["SeqNo"] = int.Parse(lineData[0]);
+ row["PlaceName"] = lineData[2];
+ row["CountyCode"] = countyCode;
+ row["FeatureCode"] = lineData[14];
+ row["Latitude"] = latitude;
+ row["Longitude"] = (String.Compare(gmt,"W") == 0) ? -longitude : longitude;
+ pointData.Rows.Add(row);
+
+ // Maintain a distinct list of county codes & names
+ if (!counties.ContainsKey(countyCode))
+ counties.Add(countyCode, countyName);
+ }
+ }
+
+ // Create datatable of counties
+ foreach (var county in counties)
+ {
+ countyData.Rows.Add(county.Key, county.Value);
+ }
+
+ // Create datatable of feature codes & descriptions from resx - taken from Ordance Survey download documentation
+ var featureResource = Resources.FeatureCodes.ResourceManager.GetResourceSet(CultureInfo.InvariantCulture, true, true);
+ foreach (DictionaryEntry resourceItem in featureResource)
+ {
+ featureData.Rows.Add(resourceItem.Key, resourceItem.Value);
+ }
+ ds.Tables.Add(pointData);
+ ds.Tables.Add(countyData);
+ ds.Tables.Add(featureData);
+ Console.WriteLine("Done! {0} point data rows and {1} county data rows of data prepared", pointData.Rows.Count.ToString(), countyData.Rows.Count.ToString());
+
+ return ds;
+ }
+
+ /// <summary>
+ /// Creates a new table in the database ready to receive the Code-Point data.
+ /// </summary>
+ /// <param name="connection">database connection to SQL Server</param>
+ /// <param name="schemaName">schema in which to create the table</param>
+ /// <param name="tableName">name of table to create</param>
+ private void PrepareTable(SqlConnection connection, string schemaName, string tableName, string countyTableName, string featureTableName)
+ {
+ try
+ {
+ using (SqlCommand cmd = new SqlCommand())
+ {
+ cmd.Connection = connection;
+ cmd.CommandText =
+ String.Format(@"IF NOT EXISTS(SELECT * FROM sys.tables t JOIN sys.schemas s ON t.schema_id = s.schema_id WHERE t.name IN (@TableName, @CountyTableName, @FeatureTableName) AND s.name = @SchemaName)
+ BEGIN
+ CREATE TABLE [{0}].[{1}]
+ (
+ SeqNo INTEGER NOT NULL,
+ PlaceName VARCHAR(60) NOT NULL,
+ CountyCode CHAR(2) NOT NULL,
+ FeatureCode VARCHAR(3) NOT NULL,
+ Longitude FLOAT,
+ Latitude FLOAT,
+ GeoLocation GEOGRAPHY
+ );
+ CREATE TABLE [{0}].[{2}]
+ (
+ Code CHAR(2) NOT NULL,
+ Name VARCHAR(60) NOT NULL
+ );
+ CREATE TABLE [{0}].[{3}]
+ (
+ Code VARCHAR(3) NOT NULL,
+ Description VARCHAR(50) NOT NULL
+ )
+ SET @Created = 1
+ END;", schemaName, tableName, countyTableName, featureTableName);
+ cmd.Parameters.Add("@TableName", System.Data.SqlDbType.NVarChar, 128).Value = tableName;
+ cmd.Parameters.Add("@SchemaName", System.Data.SqlDbType.NVarChar, 128).Value = schemaName;
+ cmd.Parameters.Add("@CountyTableName", System.Data.SqlDbType.NVarChar, 128).Value = countyTableName;
+ cmd.Parameters.Add("@FeatureTableName", System.Data.SqlDbType.NVarChar, 128).Value = featureTableName;
+ cmd.Parameters.Add("@Created", System.Data.SqlDbType.Bit, 1).Direction = ParameterDirection.Output;
+ cmd.ExecuteNonQuery();
+ if (cmd.Parameters["@Created"].Value == DBNull.Value)
+ {
+ throw new Exception("Cannot create new table to load to because the table already exists");
+ }
+ }
+ }
+ catch (Exception ex)
+ {
+ throw new Exception(String.Format("Error attemping to create table to load data into [{0}].[{1}]: {2}", schemaName, tableName, ex.Message), ex);
+ }
+ }
+ }
+}
View
26 OSCodePointDataImport/OSCodePointDataImport.csproj
@@ -47,14 +47,32 @@
<Reference Include="System.Xml" />
</ItemGroup>
<ItemGroup>
- <Compile Include="CodePointArgParser.cs" />
- <Compile Include="CodePointDataImporter.cs" />
- <Compile Include="CommandLineArgs.cs">
+ <Compile Include="ArgumentParsers\CodePointArgParser.cs" />
+ <Compile Include="ArgumentParsers\IArgParser.cs" />
+ <Compile Include="ImportFactory.cs" />
+ <Compile Include="Importers\CodePointDataImporter.cs" />
+ <Compile Include="ArgumentParsers\CommandLineArgParser.cs">
<SubType>Code</SubType>
</Compile>
- <Compile Include="OSDataImporter.cs" />
+ <Compile Include="Options\CodePointOptions.cs" />
+ <Compile Include="Options\Options.cs" />
+ <Compile Include="Options\ScaleGazetteerOptions.cs" />
+ <Compile Include="Importers\OSDataImporter.cs" />
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
+ <Compile Include="ArgumentParsers\ScaleGazetteerArgParser.cs" />
+ <Compile Include="Importers\ScaleGazetteerDataImporter.cs" />
+ <Compile Include="Resources\FeatureCodes.Designer.cs">
+ <AutoGen>True</AutoGen>
+ <DesignTime>True</DesignTime>
+ <DependentUpon>FeatureCodes.resx</DependentUpon>
+ </Compile>
+ </ItemGroup>
+ <ItemGroup>
+ <EmbeddedResource Include="Resources\FeatureCodes.resx">
+ <Generator>ResXFileCodeGenerator</Generator>
+ <LastGenOutput>FeatureCodes.Designer.cs</LastGenOutput>
+ </EmbeddedResource>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
View
27 OSCodePointDataImport/OSDataImporter.cs
@@ -1,27 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using TDPG.GeoCoordConversion;
-
-namespace OSCodePointDataImport
-{
- abstract class OSDataImporter
- {
- /// <summary>
- /// Converts northing and easting coordinates into Longitude/Latitude coordinates in the WGS84 coordinate system.
- /// </summary>
- /// <param name="northing">northing coordinate</param>
- /// <param name="easting">easting coordinate</param>
- /// <returns>converted coordinates</returns>
- protected PolarGeoCoordinate ConvertToLonLat(double northing, double easting)
- {
- // Use GeoCoordConversion DLL to convert the Eastings & Northings to Lon/Lat
- // coordinates in the WGS84 system. The DLL was pulled from: http://code.google.com/p/geocoordconversion/
- // and is available under the GNU General Public License v3: http://www.gnu.org/licenses/gpl.html
- GridReference gridRef = new GridReference((long)easting, (long)northing);
- PolarGeoCoordinate polarCoord = GridReference.ChangeToPolarGeo(gridRef);
- return PolarGeoCoordinate.ChangeCoordinateSystem(polarCoord, CoordinateSystems.WGS84);
- }
- }
-}
View
15 OSCodePointDataImport/Options/CodePointOptions.cs
@@ -0,0 +1,15 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace OSCodePointDataImport
+{
+ class CodePointOptions : Options
+ {
+ /// <summary>
+ /// Directory containing the downloaded Ordnance Survey Code-Point CSV data files.
+ /// </summary>
+ public string DataFileDirectory { get; set; }
+ }
+}
View
37 OSCodePointDataImport/Options/Options.cs
@@ -0,0 +1,37 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace OSCodePointDataImport
+{
+ abstract class Options
+ {
+ /// <summary>
+ /// What type of data want to import:
+ /// CODEPOINT
+ /// </summary>
+ public string DataType { get; set; }
+
+ /// <summary>
+ /// SQL Server name.
+ /// </summary>
+ public string ServerName { get; set; }
+
+ /// <summary>
+ /// Name of schema the table should belong to.
+ /// </summary>
+ public string SchemaName { get; set; }
+
+ /// <summary>
+ /// Database name.
+ /// </summary>
+ public string DBName { get; set; }
+
+ /// <summary>
+ /// Name of table to load data to.
+ /// </summary>
+ public string TableName { get; set; }
+ }
+
+}
View
25 OSCodePointDataImport/Options/ScaleGazetteerOptions.cs
@@ -0,0 +1,25 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace OSCodePointDataImport
+{
+ class ScaleGazetteerOptions : Options
+ {
+ /// <summary>
+ /// Scale Gazetteer data file
+ /// </summary>
+ public string DataFileName { get; set; }
+
+ /// <summary>
+ /// Name of the lookup table to create to store the county codes/names
+ /// </summary>
+ public string CountyLookupTableName { get; set; }
+
+ /// <summary>
+ /// Name of the lookup table to create to store the feature codes/names
+ /// </summary>
+ public string FeatureLookupTableName { get; set; }
+ }
+}
View
35 OSCodePointDataImport/Program.cs
@@ -23,11 +23,12 @@ namespace OSCodePointDataImport
/// <summary>
/// Console app.
/// Prerequisites:
- /// - download the Code-Point data from: https://www.ordnancesurvey.co.uk/opendatadownload/products.html
+ /// - download the Code-Point / 1:50000 Scale Gazetteer data from: https://www.ordnancesurvey.co.uk/opendatadownload/products.html
/// and extract the files to a directory on your machine.
/// </summary>
/// <example>
/// OSCodePointDataImport.exe CODEPOINT MySqlServerName MyDbName dbo PostCodeData "C:\OS Code-Point Data"
+ /// OSCodePointDataImport.exe GAZETTEER SQLServerA MyDatabase dbo Gazetteer County Feature "C:\OSGazetteerData\50kgaz2010.txt"
/// </example>
class Program
{
@@ -37,18 +38,19 @@ static void Main(string[] args)
{
if (args == null || args.Length == 0) throw new Exception("No arguments were provided");
-
- // Currently, just for importing Code-Point data files. In future, could import other Ordnance Survey
- // data sets such as Boundary Data dependent on command line params.
- switch (args[0].ToUpperInvariant())
+ string importType = args[0];
+ switch (importType.ToUpperInvariant())
{
case "CODEPOINT":
RunCodePointDataImport(args);
- break;
+ break;
+ case "GAZETTEER":
+ RunGazetteerDataImport(args);
+ break;
default:
- throw new Exception("Invalid first argument: import type must be CODEPOINT");
- }
-
+ throw new Exception("Invalid first argument. Expected: CODEPOINT, GAZZETTEER");
+ }
+
Console.WriteLine("The import process is complete. Press any key to exit.");
Console.ReadKey();
}
@@ -59,6 +61,19 @@ static void Main(string[] args)
Console.ReadKey();
}
}
+
+ /// <summary>
+ /// Import 1:50000 Scale Gazetteer data.
+ /// </summary>
+ /// <param name="args">command line args</param>
+ static void RunGazetteerDataImport(string[] args)
+ {
+ ScaleGazetteerArgParser argParser = new ScaleGazetteerArgParser();
+ argParser.Parse(args);
+
+ ScaleGazetteerDataImporter importer = new ScaleGazetteerDataImporter();
+ importer.LoadData(argParser.ImportOptions);
+ }
/// <summary>
/// Import Code-Point data.
@@ -71,7 +86,7 @@ static void RunCodePointDataImport(string[] args)
argParser.Parse(args);
CodePointDataImporter importer = new CodePointDataImporter();
- importer.LoadData(argParser.ServerName, argParser.DBName, argParser.SchemaName, argParser.TableName, argParser.DataFileDirectory);
+ importer.LoadData(argParser.ImportOptions);
}
}
}
View
39 README.txt
@@ -1,9 +1,13 @@
This project requires .NET 4.0.
-You will need to have downloaded and extracted the Code Point data to your machine prior to running. They
-are available from: https://www.ordnancesurvey.co.uk/opendatadownload/products.html
+You will need to have downloaded and extracted the appropriate Ordnance Survey data file(s) to your machine prior
+to running. They are available from: https://www.ordnancesurvey.co.uk/opendatadownload/products.html
+==============
Example usage:
-
+==============
+------------------
+CODEPOINT data
+------------------
To load code point data that has been downloaded and extracted to C:\OSCodePointData, into
dbo.PostCodeData table in MyDatabase on server SQLServerA, run the following from the command prompt:
@@ -19,5 +23,34 @@ GeoLocation GEOGRAPHY
It will also calculate a basic average location for each post code district (e.g. AB12) and sector (e.g. AB12 3)
and create rows for those. For districts, the InwardCode will be an empty string.
+-----------------------------
+1:50000 SCALE GAZETTEER data
+-----------------------------
+To load the 1:50000 scale gazetteer data file that has been downloaded and extracted to C:\OSGazetteerData\50kgaz2010.txt,
+into dbo.Gazetteer table in MyDatabase on server SQLServerA, and load county codes into table: County and feature codes
+into table: Feature, run the following from the command prompt:
+
+OSCodePointDataImport.exe GAZETTEER SQLServerA MyDatabase dbo Gazetteer County Feature "C:\OSGazetteerData\50kgaz2010.txt"
+
+The Gazetteer database table will consist of the following columns:
+SeqNo INTEGER PRIMARY KEY
+PlaceName VARCHAR(60)
+CountyCode CHAR(2) FK to County.Code
+FeatureCode VARCHAR(3) FK to Feature.Code
+Longitude FLOAT
+Latitude FLOAT
+GeoLocation GEOGRAPHY
+
+The County lookup table will consist of the following columns:
+Code CHAR(2) PRIMARY KEY
+Name VARCHAR(60)
+
+The Feature lookup table will consist of the following columns:
+Code VARCHAR(3) PRIMARY KEY
+Description VARCHAR(50)
+
+==========
+Background
+==========
A quick background to this project is on my blog post:
http://www.adathedev.co.uk/2011/01/gb-post-code-geographic-data-load-to.html

0 comments on commit 226101f

Please sign in to comment.