From 189fcbbf2c74a08e4c3bf928da7c0cd82fe8b0b2 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Fri, 14 May 2021 09:04:42 +0200 Subject: [PATCH 1/5] Test more of the flex Lua config files --- flex-config/with-schema.lua | 3 +++ tests/CMakeLists.txt | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/flex-config/with-schema.lua b/flex-config/with-schema.lua index 085da784d..a223cc3de 100644 --- a/flex-config/with-schema.lua +++ b/flex-config/with-schema.lua @@ -2,6 +2,9 @@ -- This configuration for the flex output shows how to define a table in -- a PostgreSQL schema. +-- +-- This config file expects that you have a schema called `myschema` in +-- your database (created with something like `CREATE SCHEMA myschema;`). local dtable = osm2pgsql.define_way_table('data', { { column = 'tags', type = 'hstore' }, diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4b39d9d1c..e877b5e89 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -98,8 +98,8 @@ if (HAVE_LUA) set_test(test-output-flex-way-relation-del) set_test(test-output-flex-example-configs) - set(FLEX_EXAMPLE_CONFIGS "compatible,data-types,generic,geometries,route-relations,simple,unitable") - # places.lua not tested because it needs dkjson package + set(FLEX_EXAMPLE_CONFIGS "attributes,compatible,data-types,generic,geometries,places,route-relations,simple,unitable") + # with-schema.lua is not tested because it needs the schema created in the database set_tests_properties(test-output-flex-example-configs PROPERTIES ENVIRONMENT "EXAMPLE_FILES=${FLEX_EXAMPLE_CONFIGS}") endif() From 444bbeb64b19796a7dcbbd2aac662e1abfcbfbd0 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Fri, 14 May 2021 09:15:35 +0200 Subject: [PATCH 2/5] Switch flex config files from hstore to jsonb The jsonb data type is built into PostgreSQL, no hstore extension is needed. It is more flexible and there are more functions to extract data from it. --- flex-config/attributes.lua | 6 +++--- flex-config/data-types.lua | 4 ++-- flex-config/generic.lua | 12 ++++++------ flex-config/geometries.lua | 10 +++++----- flex-config/route-relations.lua | 4 ++-- flex-config/simple.lua | 10 +++++----- flex-config/unitable.lua | 4 ++-- flex-config/with-schema.lua | 2 +- 8 files changed, 26 insertions(+), 26 deletions(-) diff --git a/flex-config/attributes.lua b/flex-config/attributes.lua index 64487a6c2..af6e19144 100644 --- a/flex-config/attributes.lua +++ b/flex-config/attributes.lua @@ -11,7 +11,7 @@ local srid = 4326 local tables = {} tables.points = osm2pgsql.define_node_table('points', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, { column = 'geom', type = 'point', projection = srid }, { column = 'version', type = 'int' }, { column = 'changeset', type = 'int' }, @@ -21,7 +21,7 @@ tables.points = osm2pgsql.define_node_table('points', { }) tables.lines = osm2pgsql.define_way_table('lines', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, { column = 'geom', type = 'linestring', projection = srid }, { column = 'version', type = 'int' }, { column = 'changeset', type = 'int' }, @@ -31,7 +31,7 @@ tables.lines = osm2pgsql.define_way_table('lines', { }) tables.relations = osm2pgsql.define_relation_table('relations', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, { column = 'version', type = 'int' }, { column = 'changeset', type = 'int' }, { column = 'created', type = 'timestamp' }, diff --git a/flex-config/data-types.lua b/flex-config/data-types.lua index 5afe65f30..5df495486 100644 --- a/flex-config/data-types.lua +++ b/flex-config/data-types.lua @@ -1,6 +1,6 @@ -- This config example file is released into the Public Domain. --- This is a very simple Lua config for the Flex Backend not intended for +-- This is a very simple Lua config for the Flex output not intended for -- real-world use. Look at and understand "simple.lua" first, before looking -- at this file. This file demonstrates some column data type options. @@ -19,7 +19,7 @@ local highways = osm2pgsql.define_way_table('highways', { -- type "bool" is special, see below { column = 'lit', type = 'bool' }, - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, -- also available: 'json', 'hstore' -- an PostgreSQL array type, not specially handled by osm2pgsql, see below { column = 'nodes', type = 'int8[]' }, diff --git a/flex-config/generic.lua b/flex-config/generic.lua index 9b2d1d4bc..d1ca9b7ef 100644 --- a/flex-config/generic.lua +++ b/flex-config/generic.lua @@ -2,7 +2,7 @@ -- This is a generic configuration that is a good starting point for -- real-world projects. Data is split into tables according to geometry type --- and most tags are stored in hstore columns. +-- and most tags are stored in jsonb columns. -- Set this to the projection you want to use local srid = 3857 @@ -10,28 +10,28 @@ local srid = 3857 local tables = {} tables.points = osm2pgsql.define_node_table('points', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, { column = 'geom', type = 'point', projection = srid }, }) tables.lines = osm2pgsql.define_way_table('lines', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, { column = 'geom', type = 'linestring', projection = srid }, }) tables.polygons = osm2pgsql.define_area_table('polygons', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, { column = 'geom', type = 'geometry', projection = srid }, { column = 'area', type = 'area' }, }) tables.routes = osm2pgsql.define_relation_table('routes', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, { column = 'geom', type = 'multilinestring', projection = srid }, }) tables.boundaries = osm2pgsql.define_relation_table('boundaries', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, { column = 'geom', type = 'multilinestring', projection = srid }, }) diff --git a/flex-config/geometries.lua b/flex-config/geometries.lua index b651ab739..52615ee91 100644 --- a/flex-config/geometries.lua +++ b/flex-config/geometries.lua @@ -1,6 +1,6 @@ -- This config example file is released into the Public Domain. --- This is a very simple Lua config for the Flex Backend not intended for +-- This is a very simple Lua config for the Flex output not intended for -- real-world use. Look at and understand "simple.lua" first, before looking -- at this file. This file will show some options around geometry processing. -- After you have understood this file, go on to "data-types.lua". @@ -8,21 +8,21 @@ local tables = {} tables.pois = osm2pgsql.define_node_table('pois', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, -- Create a geometry column for point geometries. The geometry will be -- in web mercator, EPSG 3857. { column = 'geom', type = 'point' }, }) tables.ways = osm2pgsql.define_way_table('ways', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, -- Create a geometry column for linestring geometries. The geometry will -- be in latlong (WGS84), EPSG 4326. { column = 'geom', type = 'linestring', projection = 4326 }, }) tables.polygons = osm2pgsql.define_area_table('polygons', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, { column = 'geom', type = 'geometry' }, -- The 'area' type is used to store the calculated area of a polygon -- feature. This can be used in style sheets to only render larger polygons @@ -34,7 +34,7 @@ tables.polygons = osm2pgsql.define_area_table('polygons', { tables.boundaries = osm2pgsql.define_relation_table('boundaries', { { column = 'type', type = 'text' }, - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, -- Boundaries will be stitched together from relation members into long -- linestrings. This is a multilinestring column because sometimes the -- boundaries are not contiguous. diff --git a/flex-config/route-relations.lua b/flex-config/route-relations.lua index 61a095b0d..0129f8463 100644 --- a/flex-config/route-relations.lua +++ b/flex-config/route-relations.lua @@ -12,7 +12,7 @@ local tables = {} tables.highways = osm2pgsql.define_way_table('highways', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, { column = 'rel_refs', type = 'text' }, -- for the refs from the relations { column = 'rel_ids', type = 'int8[]' }, -- array with integers (for relation IDs) { column = 'geom', type = 'linestring' }, @@ -20,7 +20,7 @@ tables.highways = osm2pgsql.define_way_table('highways', { -- Tables don't have to have a geometry column tables.routes = osm2pgsql.define_relation_table('routes', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, }) -- This will be used to store information about relations queryable by member diff --git a/flex-config/simple.lua b/flex-config/simple.lua index a9f1d5035..c2d33101a 100644 --- a/flex-config/simple.lua +++ b/flex-config/simple.lua @@ -1,6 +1,6 @@ -- This config example file is released into the Public Domain. --- This is a very simple Lua config for the Flex Backend not intended for +-- This is a very simple Lua config for the Flex output not intended for -- real-world use. Use it do understand the basic principles of the -- configuration. After reading and understanding this, have a look at -- "geometries.lua". @@ -24,7 +24,7 @@ local tables = {} -- "append" mode, osm2pgsql will automatically update this table using the node -- ids. tables.pois = osm2pgsql.define_node_table('pois', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, { column = 'geom', type = 'point' }, -- will be something like `GEOMETRY(Point, 4326)` in SQL }) @@ -40,7 +40,7 @@ tables.restaurants = osm2pgsql.define_node_table('restaurants', { -- contain a "way_id" column. When running in "append" mode, osm2pgsql will -- automatically update this table using the way ids. tables.ways = osm2pgsql.define_way_table('ways', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, { column = 'geom', type = 'linestring' }, }) @@ -50,7 +50,7 @@ tables.ways = osm2pgsql.define_way_table('ways', { -- running in "append" mode, osm2pgsql will automatically update this table -- using the way/relation ids. tables.polygons = osm2pgsql.define_area_table('polygons', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, -- The type of the `geom` column is `geometry`, because we need to store -- polygons AND multipolygons { column = 'geom', type = 'geometry' }, @@ -95,7 +95,7 @@ function osm2pgsql.process_node(object) }) else tables.pois:add_row({ - -- We know `tags` is of type `hstore` so this will do the + -- We know `tags` is of type `jsonb` so this will do the -- right thing. tags = object.tags }) diff --git a/flex-config/unitable.lua b/flex-config/unitable.lua index 715b5612e..c0a18b39c 100644 --- a/flex-config/unitable.lua +++ b/flex-config/unitable.lua @@ -12,8 +12,8 @@ local dtable = osm2pgsql.define_table{ -- "osm_type CHAR(1)" for the type of object: N(ode), W(way), R(relation) ids = { type = 'any', id_column = 'osm_id', type_column = 'osm_type' }, columns = { - { column = 'attrs', type = 'hstore' }, - { column = 'tags', type = 'hstore' }, + { column = 'attrs', type = 'jsonb' }, + { column = 'tags', type = 'jsonb' }, { column = 'geom', type = 'geometry' }, } } diff --git a/flex-config/with-schema.lua b/flex-config/with-schema.lua index a223cc3de..851e3e8e1 100644 --- a/flex-config/with-schema.lua +++ b/flex-config/with-schema.lua @@ -7,7 +7,7 @@ -- your database (created with something like `CREATE SCHEMA myschema;`). local dtable = osm2pgsql.define_way_table('data', { - { column = 'tags', type = 'hstore' }, + { column = 'tags', type = 'jsonb' }, { column = 'geom', type = 'geometry' }, }, { schema = 'myschema' }) From 5838a261d655e04e6fd512800af8b36a594569b3 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Fri, 14 May 2021 11:52:18 +0200 Subject: [PATCH 3/5] date formatting --- flex-config/attributes.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flex-config/attributes.lua b/flex-config/attributes.lua index af6e19144..0377c3888 100644 --- a/flex-config/attributes.lua +++ b/flex-config/attributes.lua @@ -48,7 +48,7 @@ function osm2pgsql.process_node(object) tags = object.tags, version = object.version, changeset = object.changeset, - created = os.date('!%Y-%m-%dT%TZ', object.timestamp), + created = os.date('!%Y-%m-%dT%H:%M:%S', object.timestamp) + 'Z', uid = object.uid, user = object.user }) @@ -59,7 +59,7 @@ function osm2pgsql.process_way(object) tags = object.tags, version = object.version, changeset = object.changeset, - created = os.date('!%Y-%m-%dT%TZ', object.timestamp), + created = os.date('!%Y-%m-%dT%T', object.timestamp) + 'Z', uid = object.uid, user = object.user }) @@ -70,7 +70,7 @@ function osm2pgsql.process_relation(object) tags = object.tags, version = object.version, changeset = object.changeset, - created = os.date('!%Y-%m-%dT%TZ', object.timestamp), + created = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), uid = object.uid, user = object.user }) From 79f98733704831c7cf0d9f50329cb29b307e5043 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Fri, 14 May 2021 15:24:11 +0200 Subject: [PATCH 4/5] string concat fix --- flex-config/attributes.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flex-config/attributes.lua b/flex-config/attributes.lua index 0377c3888..5a83064d1 100644 --- a/flex-config/attributes.lua +++ b/flex-config/attributes.lua @@ -48,7 +48,7 @@ function osm2pgsql.process_node(object) tags = object.tags, version = object.version, changeset = object.changeset, - created = os.date('!%Y-%m-%dT%H:%M:%S', object.timestamp) + 'Z', + created = os.date('!%Y-%m-%dT%H:%M:%S', object.timestamp) .. 'Z', uid = object.uid, user = object.user }) @@ -59,7 +59,7 @@ function osm2pgsql.process_way(object) tags = object.tags, version = object.version, changeset = object.changeset, - created = os.date('!%Y-%m-%dT%T', object.timestamp) + 'Z', + created = os.date('!%Y-%m-%dT%T', object.timestamp) .. 'Z', uid = object.uid, user = object.user }) From ea6fdd30cc7e200ae91201014df2b73a1ae61c36 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Fri, 14 May 2021 15:51:52 +0200 Subject: [PATCH 5/5] fix --- flex-config/attributes.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flex-config/attributes.lua b/flex-config/attributes.lua index 5a83064d1..b84fede0c 100644 --- a/flex-config/attributes.lua +++ b/flex-config/attributes.lua @@ -48,7 +48,7 @@ function osm2pgsql.process_node(object) tags = object.tags, version = object.version, changeset = object.changeset, - created = os.date('!%Y-%m-%dT%H:%M:%S', object.timestamp) .. 'Z', + created = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), uid = object.uid, user = object.user }) @@ -59,7 +59,7 @@ function osm2pgsql.process_way(object) tags = object.tags, version = object.version, changeset = object.changeset, - created = os.date('!%Y-%m-%dT%T', object.timestamp) .. 'Z', + created = os.date('!%Y-%m-%dT%H:%M:%SZ', object.timestamp), uid = object.uid, user = object.user })