diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b01a361..93c5546 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v3 diff --git a/CHANGES.md b/CHANGES.md index 834237f..79d0b16 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -24,6 +24,7 @@ * renamed `timvt.db.table_index` to `timvt.dbmodel.get_table_index` * default to only view tables within the `public` schema * renamed *base exception class* to `TiMVTError` +* remove python 3.7 support ## 0.7.0 (2022-06-09) diff --git a/pyproject.toml b/pyproject.toml index 2beac0f..f173e7f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "timvt" description = "A lightweight PostGIS based dynamic vector tile server." readme = "README.md" -requires-python = ">=3.7" +requires-python = ">=3.8" license = {file = "LICENSE"} authors = [ {name = "Vincent Sarago", email = "vincent@developmentseed.com"}, @@ -12,10 +12,10 @@ classifiers = [ "Intended Audience :: Information Technology", "Intended Audience :: Science/Research", "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Scientific/Engineering :: GIS", ] dynamic = ["version"] @@ -27,6 +27,7 @@ dependencies = [ "morecantile>=3.1,<4.0", "starlette-cramjam>=0.3,<0.4", "importlib_resources>=1.1.0; python_version < '3.9'", + "typing_extensions; python_version < '3.9.2'", ] [project.optional-dependencies] diff --git a/timvt/dbmodel.py b/timvt/dbmodel.py index 1ff0f7b..b0186ba 100644 --- a/timvt/dbmodel.py +++ b/timvt/dbmodel.py @@ -5,6 +5,8 @@ from buildpg import asyncpg from pydantic import BaseModel, Field +from timvt.settings import TableSettings + class Column(BaseModel): """Model for database Column.""" @@ -16,38 +18,51 @@ class Column(BaseModel): @property def json_type(self) -> str: """Return JSON field type.""" - pgtype = self.type - - if pgtype.endswith("[]"): + if self.type.endswith("[]"): return "array" - if any( - [ - pgtype.startswith("int"), - pgtype.startswith("num"), - pgtype.startswith("float"), - ] - ): + if self.type in [ + "smallint", + "integer", + "bigint", + "decimal", + "numeric", + "real", + "double precision", + "smallserial", + "serial", + "bigserial", + # Float8 is not a Postgres type name but is the name we give + # internally do Double Precision type + # ref: https://github.com/developmentseed/tifeatures/pull/60/files#r1011863866 + "float8", + ]: return "number" - if pgtype.startswith("bool"): + if self.type.startswith("bool"): return "boolean" - if any([pgtype.startswith("json"), pgtype.startswith("geo")]): + if any([self.type.startswith("json"), self.type.startswith("geo")]): return "object" return "string" -class GeometryColumn(BaseModel): +class GeometryColumn(Column): """Model for PostGIS geometry/geography column.""" - name: str - bounds: List[float] - srid: int + bounds: List[float] = [-180, -90, 180, 90] + srid: int = 4326 geometry_type: str +class DatetimeColumn(Column): + """Model for PostGIS geometry/geography column.""" + + min: Optional[str] + max: Optional[str] + + class Table(BaseModel): """Model for DB Table.""" @@ -55,30 +70,39 @@ class Table(BaseModel): table: str dbschema: str = Field(..., alias="schema") description: Optional[str] - id_column: str - geometry_columns: List[GeometryColumn] - properties: List[Column] + properties: List[Column] = [] + id_column: Optional[str] + geometry_columns: List[GeometryColumn] = [] + datetime_columns: List[DatetimeColumn] = [] + geometry_column: Optional[GeometryColumn] + datetime_column: Optional[DatetimeColumn] + + def get_datetime_column(self, name: Optional[str] = None) -> Optional[Column]: + """Return the Column for either the passed in tstz column or the first tstz column.""" + if not self.datetime_columns: + return None - @property - def datetime_columns(self) -> List[Column]: - """Return the name of all timestamptz columns.""" - return [p for p in self.properties if p.type.startswith("timestamp")] + if name is None: + return self.datetime_column - def datetime_column(self, name: Optional[str] = None) -> Optional[Column]: - """Return the Column for either the passed in tstz column or the first tstz column.""" for col in self.datetime_columns: - if name is None or col.name == name: + if col.name == name: return col return None - def geometry_column(self, name: Optional[str] = None) -> Optional[GeometryColumn]: + def get_geometry_column( + self, name: Optional[str] = None + ) -> Optional[GeometryColumn]: """Return the name of the first geometry column.""" - if name and name.lower() == "none": + if (not self.geometry_columns) or (name and name.lower() == "none"): return None + if name is None: + return self.geometry_column + for col in self.geometry_columns: - if name is None or col.name == name: + if col.name == name: return col return None @@ -94,10 +118,10 @@ def columns(self, properties: Optional[List[str]] = None) -> List[str]: """Return table columns optionally filtered to only include columns from properties.""" cols = [c.name for c in self.properties] if properties is not None: - if self.id_column not in properties: + if self.id_column and self.id_column not in properties: properties.append(self.id_column) - geom_col = self.geometry_column() + geom_col = self.get_geometry_column() if geom_col: properties.append(geom_col.name) @@ -129,130 +153,197 @@ async def get_table_index( """Fetch Table index.""" query = """ - WITH t AS ( + WITH table_columns AS ( SELECT - nspname as schemaname, - relname as tablename, + nspname, + relname, format('%I.%I', nspname, relname) as id, c.oid as t_oid, obj_description(c.oid, 'pg_class') as description, - ( - SELECT - attname - FROM - pg_attribute a - LEFT JOIN - pg_index i - ON ( - a.attrelid = i.indrelid - AND a.attnum = ANY(i.indkey) - ) - WHERE - a.attrelid = c.oid - ORDER BY - i.indisprimary DESC NULLS LAST, - i.indisunique DESC NULLS LAST, - attname ~* E'id$' DESC NULLS LAST - LIMIT 1 - ) as pk, - ( - SELECT - jsonb_agg( - jsonb_build_object( - 'name', attname, - 'type', format_type(atttypid, null), - 'description', col_description(attrelid, attnum) - ) - ) - FROM - pg_attribute - WHERE - attnum>0 - AND attrelid=c.oid - AND NOT attisdropped - ) as columns, - ( - SELECT - coalesce(jsonb_agg( - jsonb_build_object( - 'name', f_geometry_column, - 'srid', srid, - 'geometry_type', type, - 'bounds', - CASE WHEN srid IS NOT NULL AND srid != 0 THEN - ( - SELECT - ARRAY[ - ST_XMin(extent.geom), - ST_YMin(extent.geom), - ST_XMax(extent.geom), - ST_YMax(extent.geom) - ] - FROM ( - SELECT - coalesce( - ST_Transform( - ST_SetSRID( - ST_EstimatedExtent(f_table_schema, f_table_name, f_geometry_column), - srid - ), - 4326 - ), - ST_MakeEnvelope(-180, -90, 180, 90, 4326) - ) as geom - ) AS extent - ) - ELSE ARRAY[-180,-90,180,90] - END - ) - ),'[]'::jsonb) - FROM - ( - SELECT f_table_schema, f_table_name, f_geometry_column, srid, type - FROM geometry_columns - UNION ALL - SELECT f_table_schema, f_table_name, f_geography_column, 4326, type - FROM geography_columns - ) as geo - WHERE - f_table_schema = n.nspname - AND f_table_name = c.relname - ) as geometry_columns + attname, + atttypmod, + replace(replace(replace(replace(format_type(atttypid, null),'character varying','text'),'double precision','float8'),'timestamp with time zone','timestamptz'),'timestamp without time zone','timestamp') as "type", + col_description(attrelid, attnum) FROM pg_class c - JOIN pg_namespace n ON (c.relnamespace=n.oid) + JOIN pg_namespace n on (c.relnamespace=n.oid) + JOIN pg_attribute a on (attnum>0 and attrelid=c.oid and not attisdropped) WHERE - relkind in ('r','v', 'm', 'f', 'p') + relkind IN ('r','v', 'm', 'f', 'p') + AND has_table_privilege(c.oid, 'SELECT') + AND has_column_privilege(c.oid,a.attnum, 'SELECT') AND n.nspname NOT IN ('pg_catalog', 'information_schema') - AND c.relname NOT IN ('spatial_ref_sys','geometry_columns') - AND (:schemas::text[] IS NULL OR n.nspname = ANY (:schemas)) + AND c.relname NOT IN ('spatial_ref_sys','geometry_columns','geography_columns') + AND (:schemas::text[] IS NULL OR n.nspname = ANY (:schemas)) AND (:tables::text[] IS NULL OR c.relname = ANY (:tables)) - + ), + grouped as + (SELECT + nspname, + relname, + id, + t_oid, + description, + ( + SELECT attname + FROM + pg_attribute a + LEFT JOIN + pg_index i + ON ( + a.attrelid = i.indrelid + AND a.attnum = ANY(i.indkey) + ) + WHERE + a.attrelid = t_oid + AND + i.indnatts = 1 + ORDER BY + i.indisprimary DESC NULLS LAST, + i.indisunique DESC NULLS LAST + LIMIT 1 + ) as id_column, + coalesce(jsonb_agg( + jsonb_build_object( + 'name', attname, + 'type', "type", + 'geometry_type', postgis_typmod_type(atttypmod), + 'srid', postgis_typmod_srid(atttypmod), + 'description', description, + 'bounds', + CASE WHEN postgis_typmod_srid(atttypmod) IS NOT NULL AND postgis_typmod_srid(atttypmod) != 0 THEN + ( + SELECT + ARRAY[ + ST_XMin(extent.geom), + ST_YMin(extent.geom), + ST_XMax(extent.geom), + ST_YMax(extent.geom) + ] + FROM ( + SELECT + coalesce( + ST_Transform( + ST_SetSRID( + ST_EstimatedExtent(nspname, relname, attname), + postgis_typmod_srid(atttypmod) + ), + 4326 + ), + ST_MakeEnvelope(-180, -90, 180, 90, 4326) + ) as geom + ) AS extent + ) + ELSE ARRAY[-180,-90,180,90] + END + ) + ) FILTER (WHERE "type" IN ('geometry','geography')), '[]'::jsonb) as geometry_columns, + coalesce(jsonb_agg( + jsonb_build_object( + 'name', attname, + 'type', "type", + 'description', description + ) + ) FILTER (WHERE type LIKE 'timestamp%'), '[]'::jsonb) as datetime_columns, + coalesce(jsonb_agg( + jsonb_build_object( + 'name', attname, + 'type', "type", + 'description', description + ) + ),'[]'::jsonb) as properties + FROM + table_columns + GROUP BY 1,2,3,4,5,6 ORDER BY 1,2 ) SELECT - id, - schemaname as dbschema, - tablename as tablename, - geometry_columns, - pk as id_col, - columns as properties, - description - FROM t + id, + relname as table, + nspname as dbschema, + description, + id_column, + geometry_columns, + datetime_columns, + properties + FROM grouped WHERE :spatial = FALSE OR jsonb_array_length(geometry_columns)>=1 ; + """ async with db_pool.acquire() as conn: rows = await conn.fetch_b( - query, schemas=schemas, tables=tables, spatial=spatial + query, + schemas=schemas, + tables=tables, + spatial=spatial, ) - keys = [ - "id", - "schema", - "table", - "geometry_columns", - "id_column", - "properties", - "description", - ] - return {row["id"]: dict(zip(keys, tuple(row))) for row in rows} + + catalog = {} + table_settings = TableSettings() + table_confs = table_settings.table_config + fallback_key_names = table_settings.fallback_key_names + + for table in rows: + id = table["id"] + confid = id.replace(".", "_") + table_conf = table_confs.get(confid, {}) + + # Make sure that any properties set in conf exist in table + properties = table.get("properties", []) + properties_setting = table_conf.get("properties", []) + if properties_setting: + properties = [p for p in properties if p["name"] in properties_setting] + + property_names = [p["name"] for p in properties] + + # ID Column + id_column = table_conf.get("pk") or table["id_column"] + if not id_column and fallback_key_names: + for p in properties: + if p["name"] in fallback_key_names: + id_column = p["name"] + break + + # Datetime Column + datetime_columns = [ + c + for c in table.get("datetime_columns", []) + if c["name"] in property_names + ] + + datetime_column = None + for col in datetime_columns: + if table_conf.get("datetimecol") == col["name"]: + datetime_column = col + + if not datetime_column and datetime_columns: + datetime_column = datetime_columns[0] + + # Geometry Column + geometry_columns = [ + c + for c in table.get("geometry_columns", []) + if c["name"] in property_names + ] + geometry_column = None + for col in geometry_columns: + if table_conf.get("geomcol") == col["name"]: + geometry_column = col + if not geometry_column and geometry_columns: + geometry_column = geometry_columns[0] + + catalog[id] = { + "id": id, + "table": table["table"], + "schema": table["dbschema"], + "description": table["description"], + "id_column": id_column, + "geometry_columns": geometry_columns, + "datetime_columns": datetime_columns, + "properties": properties, + "datetime_column": datetime_column, + "geometry_column": geometry_column, + } + + return catalog diff --git a/timvt/layer.py b/timvt/layer.py index c1739a5..d3d8e19 100644 --- a/timvt/layer.py +++ b/timvt/layer.py @@ -37,6 +37,9 @@ class Layer(BaseModel, metaclass=abc.ABCMeta): id: str bounds: List[float] = [-180, -90, 180, 90] + crs: str = "http://www.opengis.net/def/crs/EPSG/0/4326" + title: Optional[str] + description: Optional[str] minzoom: int = tile_settings.default_minzoom maxzoom: int = tile_settings.default_maxzoom default_tms: str = tile_settings.default_tms @@ -91,7 +94,11 @@ def bounds_default(cls, values): """Get default bounds from the first geometry columns.""" geoms = values.get("geometry_columns") if geoms: - values["bounds"] = geoms[0].bounds + # Get the Extent of all the bounds + minx, miny, maxx, maxy = zip(*[geom.bounds for geom in geoms]) + values["bounds"] = [min(minx), min(miny), max(maxx), max(maxy)] + values["crs"] = f"http://www.opengis.net/def/crs/EPSG/0/{geoms[0].srid}" + return values async def get_tile( @@ -127,7 +134,7 @@ async def get_tile( ) geom = kwargs.get("geom", None) - geometry_column = self.geometry_column(geom) + geometry_column = self.get_geometry_column(geom) if not geometry_column: raise InvalidGeometryColumnName(f"Invalid Geometry Column: {geom}.") diff --git a/timvt/settings.py b/timvt/settings.py index cf2feb6..7ace7e7 100644 --- a/timvt/settings.py +++ b/timvt/settings.py @@ -1,15 +1,46 @@ """ TiMVT config. -TiVTiler uses pydantic.BaseSettings to either get settings from `.env` or environment variables +TiMVT uses pydantic.BaseSettings to either get settings from `.env` or environment variables see: https://pydantic-docs.helpmanual.io/usage/settings/ """ +import sys from functools import lru_cache from typing import Any, Dict, List, Optional import pydantic +# Pydantic does not support older versions of typing.TypedDict +# https://github.com/pydantic/pydantic/pull/3374 +if sys.version_info < (3, 9, 2): + from typing_extensions import TypedDict +else: + from typing import TypedDict + + +class TableConfig(TypedDict, total=False): + """Configuration to add table options with env variables.""" + + geomcol: Optional[str] + datetimecol: Optional[str] + pk: Optional[str] + properties: Optional[List[str]] + + +class TableSettings(pydantic.BaseSettings): + """Table configuration settings""" + + fallback_key_names: List[str] = ["ogc_fid", "id", "pkey", "gid"] + table_config: Dict[str, TableConfig] = {} + + class Config: + """model config""" + + env_prefix = "TIMVT_" + env_file = ".env" + env_nested_delimiter = "__" + class _ApiSettings(pydantic.BaseSettings): """API settings"""