diff --git a/README.md b/README.md index 95be409f8..ce1b0cbde 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,9 @@ among others. It can also be used for [spatial analysis](docs/analysis.md) or In addition to the standard [pgsql](docs/pgsql.md) backend designed for rendering there is also the [gazetteer](docs/gazetteer.md) database for geocoding, principally with [Nominatim](http://www.nominatim.org/), and the -null backend for testing. +null backend for testing. For flexibility a new [multi](docs/multi.md) +backend is also avialable which allows the configuration of custom +postgres tables instead of those provided in the pgsql backend. Any questions should be directed at the osm dev list http://wiki.openstreetmap.org/index.php/Mailing_lists diff --git a/building.lua b/building.lua new file mode 100644 index 000000000..93e002fee --- /dev/null +++ b/building.lua @@ -0,0 +1,124 @@ +tags = { 'building', 'shop', 'amenity' } + +function filter_tags_generic(keyvalues, nokeys) + filter = 0 + tagcount = 0 + + --if there were no tags passed in, ie keyvalues is empty + if nokeys == 0 then + filter = 1 + return filter, keyvalues + end + + --remove anything we dont care about + for i,k in ipairs(tags) do + if keyvalues[k] then + tagcount = tagcount + 1 + end + end + + --if we didnt find any tags we care about + if tagcount == 0 then + filter = 1 + end + + --tell the caller whether we think we want this feature or not and give back the modified tags + return filter, keyvalues +end + +function nodes_proc (keyvalues, nokeys) + --we dont care about nodes at all so filter all of them + filter = 1 + return filter, keyvalues +end + +function rels_proc (keyvalues, nokeys) + --let the generic filtering do its job + filter, keyvalues = filter_tags_generic(keyvalues, nokeys) + if filter == 1 then + return filter, keyvalues + end + + --dont keep any relations with types other than multipolygon + if keyvalues["type"] ~= "multipolygon" then + filter = 1 + return filter, keyvalues + end + + --let the caller know if its a keeper or not and give back the modified tags + return filter, keyvalues +end + +function ways_proc (keyvalues, nokeys) + filter = 0 + + --let the generic filtering do its job + filter, keyvalues = filter_tags_generic(keyvalues, nokeys) + poly = (filter + 1) % 2 + roads = 0 + + --let the caller know if its a keeper or not and give back the modified tags + --also tell it whether or not its a polygon or road + return filter, keyvalues, poly, roads +end + +function rel_members_proc (keyvalues, keyvaluemembers, roles, membercount) + + filter = 0 + boundary = 0 + polygon = 0 + roads = 0 + + --mark each way of the relation to tell the caller if its going + --to be used in the relation or by itself as its own standalone way + --we start by assuming each way will not be used as part of the relation + membersuperseeded = {} + for i = 1, membercount do + membersuperseeded[i] = 0 + end + + --remember the type on the relation and erase it from the tags + type = keyvalues["type"] + keyvalues["type"] = nil + + if (type == "multipolygon") and keyvalues["boundary"] == nil then + --check if this relation has tags we care about + polygon = 1 + filter, keyvalues = filter_tags_generic(keyvalues, 1) + + --if the relation didn't have the tags we need go grab the tags from + --any members that are marked as outers of the multipolygon + if (filter == 1) then + for i = 1,membercount do + if (roles[i] == "outer") then + for j,k in ipairs(tags) do + v = keyvaluemembers[i][k] + if v then + keyvalues[k] = v + filter = 0 + end + end + end + end + end + if filter == 1 then + return filter, keyvalues, membersuperseeded, boundary, polygon, roads + end + + --for each tag of each member if the relation have the tag or has a non matching value for it + --then we say the member will not be used in the relation and is there for not superseeded + --ie it is kept as a standalone way + for i = 1,membercount do + superseeded = 1 + for k,v in pairs(keyvaluemembers[i]) do + if ((keyvalues[k] == nil) or (keyvalues[k] ~= v)) then + superseeded = 0; + break + end + end + membersuperseeded[i] = superseeded + end + end + + return filter, keyvalues, membersuperseeded, boundary, polygon, roads +end diff --git a/docs/multi.md b/docs/multi.md new file mode 100644 index 000000000..ebffd196b --- /dev/null +++ b/docs/multi.md @@ -0,0 +1,55 @@ +# Multi Backend # + +The multi backend is designed for custom table structures as an alternative +to the standard [pgsql](pgsql.md) backend tables. It is intended to allow +the configuration of a custom set of tables with hopefully fewer rows and fewer +columns. This can be beneficial to queries in which some context (eg. zoom level) +could limit the number of tables that need to be queried. Addtionaly it would +allow more tables to be queried in parallel. + +## Database Layout ## +It connects to a PostgreSQL database and stores the data in one or more tables. +Each table is configured in a way similar to that of the `pgsql` backend. +That is essentially why it was named `multi` because it's basically multiple +`pgsql` backends each with its own set of options and only a single table. + +## Table Configuration ## +As sample configuration may resemble the following: + + [ + { + "name": "building", + "type": "polygon", + "tagtransform": "building.lua", + "tagtransform-node-function": "nodes_proc", + "tagtransform-way-function": "ways_proc", + "tagtransform-relation-function": "rels_proc", + "tagtransform-relation-member-function": "rel_members_proc", + "tags": [ + {"name": "building", "type": "text"}, + {"name": "shop", "type": "text"}, + {"name": "amenity", "type": "text"} + ] + }, + ... + ] + +Note that each table has a `name` and can target a single type of geometry +by setting the `type` to one of `point`, `line` or `polygon`. `tagtransform` +is used to set the name of the lua script to be used for custom tag processing. +Within the lua script you may define several methods that will be called +when processing various tags, these can be named via +`tagtransform-node-function`, `tagtransform-way-function`, +`tagtransform-relation-function`, and `tagtransform-relation-member-function`. +As with the normal top level options within osm2pgsql you can specify any of the +following: `tablespace-index`, `tablespace-data`, `enable-hstore`, +`enable-hstore-index`, `enable-multi`, `hstore-match-only`. Hstore colum names +may be specified via an array of strings named `hstores`. Finally standard columns +may be specified via an array of objects named `tags` with each object containing +a `name` and a postgres `type`. Note you may also set `flags` on each tag as with +the standard osm2pgsql style file.`flags` is formated exactly as in the style file +as a string of flag names seprated by commas. + +## Importing ## + +See: [Importing](pgsql.md#importing). diff --git a/options.cpp b/options.cpp index 3a0a8777f..6691ca2a5 100644 --- a/options.cpp +++ b/options.cpp @@ -184,9 +184,11 @@ namespace #endif printf("\ -O|--output Output backend.\n\ - pgsql - Output to a PostGIS database. (default)\n\ + pgsql - Output to a PostGIS database (default)\n\ + multi - Multiple Custom Table Output to a PostGIS \n\ + database (requires style file for configuration)\n\ gazetteer - Output to a PostGIS database for Nominatim\n\ - null - No output. Useful for testing.\n"); + null - No output. Useful for testing\n"); #ifdef HAVE_LUA printf("\ --tag-transform-script Specify a lua script to handle tag filtering and normalisation\n\ diff --git a/output-multi.cpp b/output-multi.cpp index eae1799ed..85b8fd824 100644 --- a/output-multi.cpp +++ b/output-multi.cpp @@ -7,14 +7,6 @@ #include #include -namespace { - -std::string mk_column_name(const std::string &name, const options_t &options) { - return (boost::format("%1%_%2%") % options.prefix % name).str(); -} - -} // anonymous namespace - output_multi_t::output_multi_t(const std::string &name, boost::shared_ptr processor_, const struct export_list &export_list_, @@ -25,7 +17,7 @@ output_multi_t::output_multi_t(const std::string &name, m_processor(processor_), //TODO: we could in fact have something that is interested in nodes and ways.. m_osm_type(m_processor->interests(geometry_processor::interest_node) ? OSMTYPE_NODE : OSMTYPE_WAY), - m_table(new table_t(m_options.conninfo, mk_column_name(name, m_options), m_processor->column_type(), + m_table(new table_t(m_options.conninfo, name, m_processor->column_type(), m_export_list->normal_columns(m_osm_type), m_options.hstore_columns, m_processor->srid(), m_options.scale, m_options.append, m_options.slim, m_options.droptemp, diff --git a/sample.multi.json b/sample.multi.json new file mode 100644 index 000000000..d671a9171 --- /dev/null +++ b/sample.multi.json @@ -0,0 +1,16 @@ + [ + { + "name": "building", + "type": "polygon", + "tagtransform": "building.lua", + "tagtransform-node-function": "nodes_proc", + "tagtransform-way-function": "ways_proc", + "tagtransform-relation-function": "rels_proc", + "tagtransform-relation-member-function": "rel_members_proc", + "tags": [ + {"name": "building", "type": "text"}, + {"name": "shop", "type": "text"}, + {"name": "amenity", "type": "text"} + ] + } + ] diff --git a/tests/test-output-multi-line.cpp b/tests/test-output-multi-line.cpp index c55fd1ceb..0681a5658 100644 --- a/tests/test-output-multi-line.cpp +++ b/tests/test-output-multi-line.cpp @@ -62,7 +62,6 @@ int main(int argc, char *argv[]) { options_t options; options.conninfo = db->conninfo().c_str(); options.num_procs = 1; - options.prefix = "osm2pgsql_test"; options.tblsslim_index = "tablespacetest"; options.tblsslim_data = "tablespacetest"; options.slim = 1; @@ -92,27 +91,27 @@ int main(int argc, char *argv[]) { // start a new connection to run tests on pg::conn_ptr test_conn = pg::conn::connect(db->conninfo()); - check_count(test_conn, 1, "select count(*) from pg_catalog.pg_class where relname = 'osm2pgsql_test_foobar_highways'"); - check_count(test_conn, 2753, "select count(*) from osm2pgsql_test_foobar_highways"); + check_count(test_conn, 1, "select count(*) from pg_catalog.pg_class where relname = 'foobar_highways'"); + check_count(test_conn, 2753, "select count(*) from foobar_highways"); //check that we have the right spread - check_count(test_conn, 13, "select count(*) from osm2pgsql_test_foobar_highways where highway='bridleway'"); - check_count(test_conn, 3, "select count(*) from osm2pgsql_test_foobar_highways where highway='construction'"); - check_count(test_conn, 96, "select count(*) from osm2pgsql_test_foobar_highways where highway='cycleway'"); - check_count(test_conn, 249, "select count(*) from osm2pgsql_test_foobar_highways where highway='footway'"); - check_count(test_conn, 18, "select count(*) from osm2pgsql_test_foobar_highways where highway='living_street'"); - check_count(test_conn, 171, "select count(*) from osm2pgsql_test_foobar_highways where highway='path'"); - check_count(test_conn, 6, "select count(*) from osm2pgsql_test_foobar_highways where highway='pedestrian'"); - check_count(test_conn, 81, "select count(*) from osm2pgsql_test_foobar_highways where highway='primary'"); - check_count(test_conn, 842, "select count(*) from osm2pgsql_test_foobar_highways where highway='residential'"); - check_count(test_conn, 3, "select count(*) from osm2pgsql_test_foobar_highways where highway='road'"); - check_count(test_conn, 90, "select count(*) from osm2pgsql_test_foobar_highways where highway='secondary'"); - check_count(test_conn, 1, "select count(*) from osm2pgsql_test_foobar_highways where highway='secondary_link'"); - check_count(test_conn, 352, "select count(*) from osm2pgsql_test_foobar_highways where highway='service'"); - check_count(test_conn, 34, "select count(*) from osm2pgsql_test_foobar_highways where highway='steps'"); - check_count(test_conn, 33, "select count(*) from osm2pgsql_test_foobar_highways where highway='tertiary'"); - check_count(test_conn, 597, "select count(*) from osm2pgsql_test_foobar_highways where highway='track'"); - check_count(test_conn, 164, "select count(*) from osm2pgsql_test_foobar_highways where highway='unclassified'"); + check_count(test_conn, 13, "select count(*) from foobar_highways where highway='bridleway'"); + check_count(test_conn, 3, "select count(*) from foobar_highways where highway='construction'"); + check_count(test_conn, 96, "select count(*) from foobar_highways where highway='cycleway'"); + check_count(test_conn, 249, "select count(*) from foobar_highways where highway='footway'"); + check_count(test_conn, 18, "select count(*) from foobar_highways where highway='living_street'"); + check_count(test_conn, 171, "select count(*) from foobar_highways where highway='path'"); + check_count(test_conn, 6, "select count(*) from foobar_highways where highway='pedestrian'"); + check_count(test_conn, 81, "select count(*) from foobar_highways where highway='primary'"); + check_count(test_conn, 842, "select count(*) from foobar_highways where highway='residential'"); + check_count(test_conn, 3, "select count(*) from foobar_highways where highway='road'"); + check_count(test_conn, 90, "select count(*) from foobar_highways where highway='secondary'"); + check_count(test_conn, 1, "select count(*) from foobar_highways where highway='secondary_link'"); + check_count(test_conn, 352, "select count(*) from foobar_highways where highway='service'"); + check_count(test_conn, 34, "select count(*) from foobar_highways where highway='steps'"); + check_count(test_conn, 33, "select count(*) from foobar_highways where highway='tertiary'"); + check_count(test_conn, 597, "select count(*) from foobar_highways where highway='track'"); + check_count(test_conn, 164, "select count(*) from foobar_highways where highway='unclassified'"); return 0; } catch (const std::exception &e) { diff --git a/tests/test-output-multi-point-multi-table.cpp b/tests/test-output-multi-point-multi-table.cpp index ee43b9e53..0744a8f94 100644 --- a/tests/test-output-multi-point-multi-table.cpp +++ b/tests/test-output-multi-point-multi-table.cpp @@ -111,25 +111,25 @@ int main(int argc, char *argv[]) { check_count(test_conn, 1, (boost::format("select count(*) from pg_catalog.pg_class " - "where relname = 'osm2pgsql_test_foobar_%d'") + "where relname = 'foobar_%d'") % i).str()); check_count(test_conn, 244, - (boost::format("select count(*) from osm2pgsql_test_foobar_%d") + (boost::format("select count(*) from foobar_%d") % i).str()); check_count(test_conn, 36, - (boost::format("select count(*) from osm2pgsql_test_foobar_%d " + (boost::format("select count(*) from foobar_%d " "where amenity='parking'") % i).str()); check_count(test_conn, 34, - (boost::format("select count(*) from osm2pgsql_test_foobar_%d " + (boost::format("select count(*) from foobar_%d " "where amenity='bench'") % i).str()); check_count(test_conn, 1, - (boost::format("select count(*) from osm2pgsql_test_foobar_%d " + (boost::format("select count(*) from foobar_%d " "where amenity='vending_machine'") % i).str()); } diff --git a/tests/test-output-multi-point.cpp b/tests/test-output-multi-point.cpp index 799692271..47d4cd847 100644 --- a/tests/test-output-multi-point.cpp +++ b/tests/test-output-multi-point.cpp @@ -94,19 +94,19 @@ int main(int argc, char *argv[]) { check_count(test_conn, 1, "select count(*) from pg_catalog.pg_class " - "where relname = 'osm2pgsql_test_foobar_amenities'"); + "where relname = 'foobar_amenities'"); check_count(test_conn, 244, - "select count(*) from osm2pgsql_test_foobar_amenities"); + "select count(*) from foobar_amenities"); check_count(test_conn, 36, - "select count(*) from osm2pgsql_test_foobar_amenities where amenity='parking'"); + "select count(*) from foobar_amenities where amenity='parking'"); check_count(test_conn, 34, - "select count(*) from osm2pgsql_test_foobar_amenities where amenity='bench'"); + "select count(*) from foobar_amenities where amenity='bench'"); check_count(test_conn, 1, - "select count(*) from osm2pgsql_test_foobar_amenities where amenity='vending_machine'"); + "select count(*) from foobar_amenities where amenity='vending_machine'"); return 0; diff --git a/tests/test-output-multi-polygon.cpp b/tests/test-output-multi-polygon.cpp index 63eeac1d7..be3c48fdf 100644 --- a/tests/test-output-multi-polygon.cpp +++ b/tests/test-output-multi-polygon.cpp @@ -91,28 +91,28 @@ int main(int argc, char *argv[]) { // start a new connection to run tests on pg::conn_ptr test_conn = pg::conn::connect(db->conninfo()); - check_count(test_conn, 1, "select count(*) from pg_catalog.pg_class where relname = 'osm2pgsql_test_foobar_buildings'"); - check_count(test_conn, 0, "select count(*) from osm2pgsql_test_foobar_buildings where building is null"); - check_count(test_conn, 3723, "select count(*) from osm2pgsql_test_foobar_buildings"); + check_count(test_conn, 1, "select count(*) from pg_catalog.pg_class where relname = 'foobar_buildings'"); + check_count(test_conn, 0, "select count(*) from foobar_buildings where building is null"); + check_count(test_conn, 3723, "select count(*) from foobar_buildings"); //check that we have the right spread - check_count(test_conn, 1, "select count(*) from osm2pgsql_test_foobar_buildings where building='barn'"); - check_count(test_conn, 1, "select count(*) from osm2pgsql_test_foobar_buildings where building='chapel'"); - check_count(test_conn, 5, "select count(*) from osm2pgsql_test_foobar_buildings where building='church'"); - check_count(test_conn, 3, "select count(*) from osm2pgsql_test_foobar_buildings where building='commercial'"); - check_count(test_conn, 6, "select count(*) from osm2pgsql_test_foobar_buildings where building='farm'"); - check_count(test_conn, 1, "select count(*) from osm2pgsql_test_foobar_buildings where building='garage'"); - check_count(test_conn, 2, "select count(*) from osm2pgsql_test_foobar_buildings where building='glasshouse'"); - check_count(test_conn, 1, "select count(*) from osm2pgsql_test_foobar_buildings where building='greenhouse'"); - check_count(test_conn, 153, "select count(*) from osm2pgsql_test_foobar_buildings where building='house'"); - check_count(test_conn, 4, "select count(*) from osm2pgsql_test_foobar_buildings where building='hut'"); - check_count(test_conn, 8, "select count(*) from osm2pgsql_test_foobar_buildings where building='industrial'"); - check_count(test_conn, 200, "select count(*) from osm2pgsql_test_foobar_buildings where building='residential'"); - check_count(test_conn, 6, "select count(*) from osm2pgsql_test_foobar_buildings where building='roof'"); - check_count(test_conn, 4, "select count(*) from osm2pgsql_test_foobar_buildings where building='school'"); - check_count(test_conn, 2, "select count(*) from osm2pgsql_test_foobar_buildings where building='station'"); - check_count(test_conn, 3, "select count(*) from osm2pgsql_test_foobar_buildings where building='warehouse'"); - check_count(test_conn, 3323, "select count(*) from osm2pgsql_test_foobar_buildings where building='yes'"); + check_count(test_conn, 1, "select count(*) from foobar_buildings where building='barn'"); + check_count(test_conn, 1, "select count(*) from foobar_buildings where building='chapel'"); + check_count(test_conn, 5, "select count(*) from foobar_buildings where building='church'"); + check_count(test_conn, 3, "select count(*) from foobar_buildings where building='commercial'"); + check_count(test_conn, 6, "select count(*) from foobar_buildings where building='farm'"); + check_count(test_conn, 1, "select count(*) from foobar_buildings where building='garage'"); + check_count(test_conn, 2, "select count(*) from foobar_buildings where building='glasshouse'"); + check_count(test_conn, 1, "select count(*) from foobar_buildings where building='greenhouse'"); + check_count(test_conn, 153, "select count(*) from foobar_buildings where building='house'"); + check_count(test_conn, 4, "select count(*) from foobar_buildings where building='hut'"); + check_count(test_conn, 8, "select count(*) from foobar_buildings where building='industrial'"); + check_count(test_conn, 200, "select count(*) from foobar_buildings where building='residential'"); + check_count(test_conn, 6, "select count(*) from foobar_buildings where building='roof'"); + check_count(test_conn, 4, "select count(*) from foobar_buildings where building='school'"); + check_count(test_conn, 2, "select count(*) from foobar_buildings where building='station'"); + check_count(test_conn, 3, "select count(*) from foobar_buildings where building='warehouse'"); + check_count(test_conn, 3323, "select count(*) from foobar_buildings where building='yes'"); return 0; } catch (const std::exception &e) {