From 84ca84982da284f990f7726f59e9a40a283dc6f1 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 18 Mar 2012 16:12:41 +0100 Subject: [PATCH 1/6] remove array_agg shipped with postgresql in all supported versions --- sql/functions.sql | 7 ------- 1 file changed, 7 deletions(-) diff --git a/sql/functions.sql b/sql/functions.sql index 6ad13892c..688be0206 100644 --- a/sql/functions.sql +++ b/sql/functions.sql @@ -2393,13 +2393,6 @@ END; $$ LANGUAGE plpgsql; -CREATE AGGREGATE array_agg(INT[]) -( - sfunc = array_cat, - stype = INT[], - initcond = '{}' -); - CREATE OR REPLACE FUNCTION tigger_create_interpolation(linegeo GEOMETRY, in_startnumber INTEGER, in_endnumber INTEGER, interpolationtype TEXT, in_street TEXT, in_isin TEXT, in_postcode TEXT) RETURNS INTEGER From 57ef20cc7ce56a94ae4fa62bd3fd3e13a7e400cf Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 18 Mar 2012 16:15:45 +0100 Subject: [PATCH 2/6] only re-index addressable places exclude places from hierarchical updates when they are only addressed through their parent and do not have a name (thus no entry in search name) --- sql/functions.sql | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sql/functions.sql b/sql/functions.sql index 688be0206..a44fee9e9 100644 --- a/sql/functions.sql +++ b/sql/functions.sql @@ -1117,9 +1117,9 @@ BEGIN -- RAISE WARNING 'placex poly insert: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type; -- work around bug in postgis update placex set indexed_status = 2 where (ST_Contains(NEW.geometry, placex.geometry) OR ST_Intersects(NEW.geometry, placex.geometry)) - AND rank_search > NEW.rank_search and indexed_status = 0 and ST_geometrytype(placex.geometry) = 'ST_Point'; + AND rank_search > NEW.rank_search and indexed_status = 0 and ST_geometrytype(placex.geometry) = 'ST_Point' and (rank_search < 28 or name is not null); update placex set indexed_status = 2 where (ST_Contains(NEW.geometry, placex.geometry) OR ST_Intersects(NEW.geometry, placex.geometry)) - AND rank_search > NEW.rank_search and indexed_status = 0 and ST_geometrytype(placex.geometry) != 'ST_Point'; + AND rank_search > NEW.rank_search and indexed_status = 0 and ST_geometrytype(placex.geometry) != 'ST_Point' and (rank_search < 28 or name is not null); END IF; ELSE -- mark nearby items for re-indexing, where 'nearby' depends on the features rank_search and is a complete guess :( @@ -1144,7 +1144,7 @@ BEGIN END IF; IF diameter > 0 THEN -- RAISE WARNING 'placex point insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,diameter; - update placex set indexed_status = 2 where indexed_status = 0 and rank_search > NEW.rank_search and ST_DWithin(placex.geometry, NEW.geometry, diameter); + update placex set indexed_status = 2 where indexed_status = 0 and rank_search > NEW.rank_search and ST_DWithin(placex.geometry, NEW.geometry, diameter) and (rank_search < 28 or name is not null); END IF; END IF; @@ -1792,12 +1792,12 @@ BEGIN update placex set indexed_status = 2 where indexed_status = 0 and (ST_Contains(NEW.geometry, placex.geometry) OR ST_Intersects(NEW.geometry, placex.geometry)) AND NOT (ST_Contains(existinggeometry, placex.geometry) OR ST_Intersects(existinggeometry, placex.geometry)) - AND rank_search > existingplacex.rank_search; + AND rank_search > existingplacex.rank_search AND (rank_search < 28 or name is not null); update placex set indexed_status = 2 where indexed_status = 0 and (ST_Contains(existinggeometry, placex.geometry) OR ST_Intersects(existinggeometry, placex.geometry)) AND NOT (ST_Contains(NEW.geometry, placex.geometry) OR ST_Intersects(NEW.geometry, placex.geometry)) - AND rank_search > existingplacex.rank_search; + AND rank_search > existingplacex.rank_search AND (rank_search < 28 or name is not null); END IF; @@ -1817,7 +1817,8 @@ BEGIN IF st_area(NEW.geometry) < 0.5 THEN UPDATE placex set indexed_status = 2 from place_addressline where address_place_id = existingplacex.place_id - and placex.place_id = place_addressline.place_id and indexed_status = 0; + and placex.place_id = place_addressline.place_id and indexed_status = 0 + and (rank_search < 28 or name is not null); END IF; END IF; From 4d7e0d97b2a644cf5479c3301eb20f5f46d6509e Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 18 Mar 2012 17:58:19 +0100 Subject: [PATCH 3/6] return a 500 on database error --- lib/lib.php | 25 +++++++++++++++++++++++++ website/details.php | 9 ++++----- website/reverse.php | 9 +++------ website/search.php | 24 ++++++++---------------- 4 files changed, 40 insertions(+), 27 deletions(-) diff --git a/lib/lib.php b/lib/lib.php index a7c805f7d..d8ff2f64f 100644 --- a/lib/lib.php +++ b/lib/lib.php @@ -1,5 +1,30 @@

Internal Server Error

"; + echo '

Nominatim has encountered an internal error while processing your request. This is most likely because of a bug in the software.

'; + echo "

Details: ".$sError,"

"; + echo '

Feel free to report the bug in the OSM bug database. Please include the error message above an the URL you used.

'; + if (CONST_Debug) + { + echo "

Debugging Information


"; + if ($sSQL) { + echo "

SQL query

".$sSQL.""; + } + if ($vDumpVar) { + echo "

Result

"; + var_dump($vDumpVar); + echo ""; + } + } + echo "\n\n"; + exit; + + } + function fail($sError, $sUserError = false) { if (!$sUserError) $sUserError = $sError; diff --git a/website/details.php b/website/details.php index 110fdc07a..eeb486568 100755 --- a/website/details.php +++ b/website/details.php @@ -48,11 +48,11 @@ $sSQL .= " ST_GeometryType(geometry) in ('ST_Polygon','ST_MultiPolygon') as isarea,ST_GeometryType(geometry) as geotype, ST_Y(ST_Centroid(geometry)) as lat,ST_X(ST_Centroid(geometry)) as lon "; $sSQL .= " from placex where place_id = $iPlaceID"; $aPointDetails = $oDB->getRow($sSQL); - IF (PEAR::IsError($aPointDetails)) + if (PEAR::IsError($aPointDetails)) { - var_dump($aPointDetails); - exit; + failInternalError("Could not get details of place object.", $sSQL, $aPointDetails); } + $aPointDetails['localname'] = $aPointDetails['localname']?$aPointDetails['localname']:$aPointDetails['housenumber']; $fLon = $aPointDetails['lon']; $fLat = $aPointDetails['lat']; @@ -77,8 +77,7 @@ $aPointPolygon = $oDB->getRow($sSQL); IF (PEAR::IsError($aPointPolygon)) { - var_dump($aPointPolygon); - exit; + failInternalError("Could not get bounding box of place object.", $sSQL, $aPointPolygon); } if (preg_match('#POLYGON\\(\\(([- 0-9.,]+)#',$aPointPolygon['outlinestring'],$aMatch)) { diff --git a/website/reverse.php b/website/reverse.php index f569ffac1..d9ecc09ef 100755 --- a/website/reverse.php +++ b/website/reverse.php @@ -105,8 +105,7 @@ $iPlaceID = $aPlace['place_id']; if (PEAR::IsError($iPlaceID)) { - var_Dump($sSQL, $iPlaceID); - exit; + failInternalError("Could not determine closest place.", $sSQL, $iPlaceID); } } @@ -118,8 +117,7 @@ $iPlaceID = $oDB->getOne($sSQL); if (PEAR::IsError($iPlaceID)) { - var_Dump($sSQL, $iPlaceID); - exit; + failInternalError("Could not get parent for place.", $sSQL, $iPlaceID); } if ($iPlaceID && $aPlace['place_id'] && $iMaxRank < 28) @@ -129,8 +127,7 @@ $iPlaceID = $oDB->getOne($sSQL); if (PEAR::IsError($iPlaceID)) { - var_Dump($sSQL, $iPlaceID); - exit; + failInternalError("Could not get larger parent for place.", $sSQL, $iPlaceID); } } if (!$iPlaceID) diff --git a/website/search.php b/website/search.php index d9adf6b84..50d009e1a 100755 --- a/website/search.php +++ b/website/search.php @@ -163,8 +163,7 @@ $sViewboxSmallSQL = $oDB->getOne($sSQL); if (PEAR::isError($sViewboxSmallSQL)) { - var_dump($sViewboxSmallSQL); - exit; + failInternalError("Could not get small viewbox.", $sSQL, $sViewboxSmallSQL); } $sViewboxSmallSQL = "'".$sViewboxSmallSQL."'::geometry"; @@ -172,8 +171,7 @@ $sViewboxLargeSQL = $oDB->getOne($sSQL); if (PEAR::isError($sViewboxLargeSQL)) { - var_dump($sViewboxLargeSQL); - exit; + failInternalError("Could not get large viewbox.", $sSQL, $sViewboxLargeSQL); } $sViewboxLargeSQL = "'".$sViewboxLargeSQL."'::geometry"; } @@ -324,8 +322,7 @@ $aDatabaseWords = array(); if (PEAR::IsError($aDatabaseWords)) { - var_dump($sSQL, $aDatabaseWords); - exit; + failInternalError("Could not get word tokens.", $sSQL, $aDatabaseWords); } $aPossibleMainWordIDs = array(); foreach($aDatabaseWords as $aToken) @@ -789,8 +786,7 @@ $aViewBoxPlaceIDs = $oDB->getAll($sSQL); if (PEAR::IsError($aViewBoxPlaceIDs)) { - var_dump($sSQL, $aViewBoxPlaceIDs); - exit; + failInternalError("Could not get places for search terms.", $sSQL, $aViewBoxPlaceIDs); } //var_dump($aViewBoxPlaceIDs); // Did we have an viewbox matches? @@ -971,8 +967,7 @@ if (PEAR::IsError($aPlaceIDs)) { - var_dump($sSQL, $aPlaceIDs); - exit; + failInternalError("Could not get place IDs from tokens." ,$sSQL, $aPlaceIDs); } if (CONST_Debug) var_Dump($aPlaceIDs); @@ -1048,8 +1043,7 @@ if (PEAR::IsError($aSearchResults)) { - var_dump($sSQL, $aSearchResults); - exit; + failInternalError("Could not get details for place.", $sSQL, $aSearchResults); } } } // end if ($sQuery) @@ -1117,8 +1111,7 @@ if (PEAR::IsError($aSearchResults)) { - var_dump($sSQL, $aSearchResults); - exit; + failInternalError("Could not get details for place (near).", $sSQL, $aSearchResults); } } } @@ -1149,8 +1142,7 @@ $aPointPolygon = $oDB->getRow($sSQL); if (PEAR::IsError($aPointPolygon)) { - var_dump($sSQL, $aPointPolygon); - exit; + failInternalError("Could not get outline.", $sSQL, $aPointPolygon); } if ($aPointPolygon['place_id']) { From b1e10bce0d0b0cae4f401305608154c898311755 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sun, 18 Mar 2012 18:18:33 +0100 Subject: [PATCH 4/6] use X/Y max/min functions instead of box making There is a strange bug in postgis where for some coordinates it does not make a proper box around a point. --- website/details.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/details.php b/website/details.php index eeb486568..a4cb8b960 100755 --- a/website/details.php +++ b/website/details.php @@ -71,8 +71,8 @@ // Get the bounding box and outline polygon $sSQL = "select ST_AsText(geometry) as outlinestring,"; - $sSQL .= "ST_Y(ST_PointN(ExteriorRing(ST_Box2D(geometry)),4)) as minlat,ST_Y(ST_PointN(ExteriorRing(ST_Box2D(geometry)),2)) as maxlat,"; - $sSQL .= "ST_X(ST_PointN(ExteriorRing(ST_Box2D(geometry)),1)) as minlon,ST_X(ST_PointN(ExteriorRing(ST_Box2D(geometry)),3)) as maxlon"; + $sSQL .= "ST_YMin(geometry) as minlat,ST_YMax(geometry) as maxlat,"; + $sSQL .= "ST_XMin(geometry) as minlon,ST_XMax(geometry) as maxlon"; $sSQL .= " from placex where place_id = $iPlaceID"; $aPointPolygon = $oDB->getRow($sSQL); IF (PEAR::IsError($aPointPolygon)) From 248b6ec90fe72bb2b6817a5569d1888983b5eb2d Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 21 Mar 2012 00:40:35 +0100 Subject: [PATCH 5/6] Black/whitelisting for special phrases Allows to restrict the special phrases imported from the wiki. Blacklist allows to exclude certain class/type combinations. Whitelist allows to define an allowed subset of types for a class. Adjust to your liking in settings/phrase_settings.php before running the specialphrases script. --- settings/phrase_settings.php | 51 ++++++++++++++++++++++++++++++++++++ utils/specialphrases.php | 46 +++++++++----------------------- 2 files changed, 63 insertions(+), 34 deletions(-) create mode 100644 settings/phrase_settings.php diff --git a/settings/phrase_settings.php b/settings/phrase_settings.php new file mode 100644 index 000000000..4456316e3 --- /dev/null +++ b/settings/phrase_settings.php @@ -0,0 +1,51 @@ + array('administrative'), + 'place' => array('house', 'houses'), +); +# If a class is in the white list then all types will +# be ignored except the ones given in the list. +# Also use this list to exclude an entire class from +# special phrases. +$aTagsWhitelist = array( + 'highway' => array('bus_stop', 'rest_area', 'raceway'), + 'building' => array(), +); diff --git a/utils/specialphrases.php b/utils/specialphrases.php index 1c3eff5b4..b2f336c76 100755 --- a/utils/specialphrases.php +++ b/utils/specialphrases.php @@ -15,39 +15,6 @@ ); getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); - $aLanguageIn = array( - 'af', - 'ar', - 'br', - 'ca', - 'cs', - 'de', - 'en', - 'es', - 'et', - 'eu', - 'fa', - 'fi', - 'fr', - 'gl', - 'hr', - 'hu', - 'ia', - 'is', - 'it', - 'ja', - 'mk', - 'nl', - 'no', - 'pl', - 'ps', - 'pt', - 'ru', - 'sk', - 'sv', - 'uk', - 'vi', - ); if ($aCMDResult['countries']) { echo "select getorcreate_country(make_standard_name('uk'), 'gb');\n"; @@ -63,6 +30,7 @@ if ($aCMDResult['wiki-import']) { + include(CONST_BasePath.'/settings/phrase_settings.php'); $aPairs = array(); foreach($aLanguageIn as $sLanguage) @@ -84,7 +52,17 @@ preg_match('/^\\w+$/', $sType) < 1) { trigger_error("Bad class/type for language $sLanguage: $sClass=$sType"); exit; - } + } + # blacklisting: disallow certain class/type combinations + if (isset($aTagsBlacklist[$sClass]) && in_array($sType, $aTagsBlacklist[$sClass])) { + # fwrite(STDERR, "Blacklisted: ".$sClass."/".$sType."\n"); + continue; + } + # whitelisting: if class is in whitelist, allow only tags in the list + if (isset($aTagsWhitelist[$sClass]) && !in_array($sType, $aTagsWhitelist[$sClass])) { + # fwrite(STDERR, "Non-Whitelisted: ".$sClass."/".$sType."\n"); + continue; + } $aPairs[$sClass.'|'.$sType] = array($sClass, $sType); switch(trim($aMatch[4])) From b6db7c17f9c6733e228f4c8111ed1301e4687147 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 21 Mar 2012 20:59:35 +0100 Subject: [PATCH 6/6] countries need language list as well --- utils/specialphrases.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/utils/specialphrases.php b/utils/specialphrases.php index b2f336c76..b8c49da02 100755 --- a/utils/specialphrases.php +++ b/utils/specialphrases.php @@ -15,6 +15,8 @@ ); getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); + include(CONST_BasePath.'/settings/phrase_settings.php'); + if ($aCMDResult['countries']) { echo "select getorcreate_country(make_standard_name('uk'), 'gb');\n"; @@ -30,7 +32,6 @@ if ($aCMDResult['wiki-import']) { - include(CONST_BasePath.'/settings/phrase_settings.php'); $aPairs = array(); foreach($aLanguageIn as $sLanguage)