-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[#46] Enrich IndexManager with postalcode
# Scenario Summary Along with the reverse geocoding response, we should provide also the information of postalCode # Proposed Solution Enrich the `ReverseGeocodingResponse` returned by the reverse geocoding API with the postalCode, adding the logic into the `IndexManager`. So when the `IndexManager` will start, it will load also all available postalCode for each country.
- Loading branch information
1 parent
f54b717
commit 7cf062b
Showing
15 changed files
with
369 additions
and
14 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
| #!/bin/bash | ||
|
|
||
| # This script allow to extract postal codes from a given osm.pbf. | ||
| # Example usage: ./create_postalcode_shapefile.sh italy-latest.osm.pbf output italy_postalcode.shp | ||
|
|
||
| usage() { echo "Usage: $0 -f <osm.pbf file path> -o <shape destination folder> -n <shape file name(must end with .shp)> " 1>&2; exit 1; } | ||
|
|
||
| while getopts ":f:o:n:" o; do | ||
| case "${o}" in | ||
| f) | ||
| OSM_PBF_PATH=${OPTARG} | ||
| ;; | ||
| o) | ||
| OUTPUT_PATH=${OPTARG} | ||
| ;; | ||
| n) | ||
| OUTPUT_FILENAME=${OPTARG} | ||
| ;; | ||
| *) | ||
| usage | ||
| ;; | ||
| esac | ||
| done | ||
| shift $((OPTIND-1)) | ||
|
|
||
| OSM_PBF_EXT="osm.pbf" | ||
|
|
||
| if [ -z "${OSM_PBF_PATH}" ] || [ -z "${OUTPUT_PATH}" ] || [ -z "${OUTPUT_FILENAME}" ]; then | ||
| usage | ||
| fi | ||
|
|
||
| if [[ ! -f "${OSM_PBF_PATH}" ]] | ||
| then | ||
| echo "Error: ${OSM_PBF_PATH} doesn't exists on your filesystem." | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [ "${OSM_PBF_PATH#*.}" != "$OSM_PBF_EXT" ] | ||
| then | ||
| echo "Error: ${OSM_PBF_PATH} is not a valid osm.pbf file" | ||
| exit 1 | ||
| fi | ||
|
|
||
| if [[ "${OUTPUT_FILENAME}" != *.shp ]] | ||
| then | ||
| echo "Error: ${OUTPUT_FILENAME} is not a valid shape filename" | ||
| exit 1 | ||
| fi | ||
|
|
||
| OSM_PBF_ABS_PATH=$(readlink -m "$OSM_PBF_PATH") | ||
| OUTPUT_PATH_ABS_PATH=$(readlink -m "$OUTPUT_PATH") | ||
| OSM_PBF_FILENAME=$(basename "$OSM_PBF_PATH") | ||
| O5M_FILENAME_WITHOUT_EXT=$(basename "$OSM_PBF_PATH" .$OSM_PBF_EXT) | ||
| O5M_FILENAME=$O5M_FILENAME_WITHOUT_EXT".o5m" | ||
| OSM_FILTERED_FILENAME=$O5M_FILENAME_WITHOUT_EXT"-filtered.osm" | ||
| TMP_FOLDER="tmp_$(date +%Y%m%d)" | ||
| SHAPEFILE_EXT="shp" | ||
| TRANSFORMED_SHAPE_FILE="transformed" | ||
|
|
||
| rm -rf $TMP_FOLDER | ||
| mkdir $TMP_FOLDER | ||
| cd $TMP_FOLDER | ||
|
|
||
| echo "Uncompressing $OSM_PBF_FILENAME" | ||
| osmconvert $OSM_PBF_ABS_PATH -o=$O5M_FILENAME | ||
|
|
||
| if [ $? -eq 0 ] | ||
| then | ||
| echo "Successfully created $O5M_FILENAME" | ||
| else | ||
| echo $1 | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo "Creating filtered osm" | ||
| osmfilter $O5M_FILENAME --keep="addr:city addr:postcode" --keep-tags="all addr:city= addr:postcode=" --drop-tags="all" --ignore-dependencies --drop-version --drop-author > $OSM_FILTERED_FILENAME | ||
|
|
||
| if [ $? -eq 0 ] | ||
| then | ||
| echo "Successfully created $OSM_FILTERED_FILENAME" | ||
| else | ||
| echo $1 | ||
| exit 1 | ||
| fi | ||
|
|
||
|
|
||
| echo "Creating shape file from filtered file" | ||
| ogr2ogr -f "ESRI Shapefile" -skip shape $OSM_FILTERED_FILENAME | ||
|
|
||
| if [ $? -eq 0 ] | ||
| then | ||
| echo "Successfully created shapefile" | ||
| else | ||
| echo $1 | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo "Extracting tags from shape file" | ||
| cd shape | ||
| ogr2ogr -sql "select hstore_get_value(other_tags,'addr:postcode') as cap, hstore_get_value(other_tags,'addr:city') as city from points" $TRANSFORMED_SHAPE_FILE.$SHAPEFILE_EXT points.shp | ||
|
|
||
| if [ $? -eq 0 ] | ||
| then | ||
| echo "Successfully extracted tags from shapefile" | ||
| else | ||
| echo $1 | ||
| exit 1 | ||
| fi | ||
|
|
||
|
|
||
| echo "Creating final shape file" | ||
| mkdir -p final | ||
| ogr2ogr -dialect sqlite -sql "select min(cap), city, max(geometry) from $TRANSFORMED_SHAPE_FILE where cap is not NULL group by city " final/$OUTPUT_FILENAME $TRANSFORMED_SHAPE_FILE.$SHAPEFILE_EXT | ||
|
|
||
| if [ $? -eq 0 ] | ||
| then | ||
| echo "Successfully created final shapefile named $OUTPUT_FILENAME" | ||
| else | ||
| echo $1 | ||
| exit 1 | ||
| fi | ||
|
|
||
| echo "Copying $OUTPUT_FILENAME into $OUTPUT_PATH" | ||
| mkdir -p $OUTPUT_PATH_ABS_PATH | ||
| cp -R final/* $OUTPUT_PATH_ABS_PATH | ||
|
|
||
| echo "Removing tmp dir" | ||
| cd ../.. | ||
| rm -rf $TMP_FOLDER | ||
|
|
||
| echo "Terminated with $?" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
35 changes: 35 additions & 0 deletions
35
src/main/scala/it/agilelab/bigdata/gis/domain/loader/OSMPostalCodeLoader.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| package it.agilelab.bigdata.gis.domain.loader | ||
|
|
||
| import com.typesafe.config.Config | ||
| import com.vividsolutions.jts.geom.Geometry | ||
| import it.agilelab.bigdata.gis.core.loader.Loader | ||
| import it.agilelab.bigdata.gis.domain.managers.PathManager | ||
| import it.agilelab.bigdata.gis.domain.models.OSMPostalCode | ||
|
|
||
|
|
||
| case class OSMPostalCodeLoader(config: Config, pathManager: PathManager) extends Loader[OSMPostalCode] { | ||
|
|
||
| override def loadFile(source: String): Iterator[(Array[AnyRef], Geometry)] = { | ||
|
|
||
| ShapeFileReader.readPointFeatures(source).map { case (point, list) => | ||
| (list.toArray) -> point | ||
| }.toIterator | ||
|
|
||
| } | ||
|
|
||
| protected def objectMapping(fields: Array[AnyRef], line: Geometry): OSMPostalCode = { | ||
|
|
||
| val postalCodeValue = fields(1).toString | ||
| val cityValue = fields(2).toString | ||
|
|
||
| OSMPostalCode( | ||
| point = line, | ||
| postalCode = postalCodeValue, | ||
| city = Some(cityValue) | ||
| ) | ||
| } | ||
|
|
||
| protected def parseStringName(string: String): String = { | ||
| new String(string.getBytes("ISO-8859-1"), "UTF-8") | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.