Skip to content

Commit

Permalink
fix(queries): wrap sql in slonik sql template strings (#12)
Browse files Browse the repository at this point in the history 
* fix(queries): wrap sql in slonik sql template strings

fix(docs): add instructions for preventing SQL injection
fix(example): same as readme for filters
fix(queries): document functions

* fix(tiler): fix rollup issue with slonik
  • Loading branch information
@bartolkaruza
bartolkaruza committed Oct 30, 2019
1 parent 181f820 commit 81a738e
Show file tree
Hide file tree
Showing 9 changed files with 670 additions and 110 deletions.
28 changes: 17 additions & 11 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,17 +8,24 @@ Clusterbuster is designed to be used in a NodeJS server connected to a PostgreSQ

```Javascript
const { TileServer } = require('clusterbuster');

TileServer({
// types/TileServerConfig.ts
maxZoomLevel,
resolution: 512,
attributes: ['status', 'speed'],
filtersToWhere: filters => {
// You are responsible for protecting against SQL injection in this function. Because there are many ways to filter, it depends on the filter type on how to approach this.

// For example a number can be safely used by passing it thorugh parseFloat
const whereStatements = [];
if (filters.status) {

// The below statement checks that filters.status is one of 'free' or 'busy' to prevent a potential SQL injection
if (filters.status && (filters.status === 'free' || filters.status === 'busy')) {

whereStatements.push(`status = '${filters.status}'`);
}
if (filters.speed) {
if (filters.speed && (filters.speed === 'fast' || filters.speed === 'slow')) {
whereStatements.push(`speed = '${filters.speed}'`);
}
return whereStatements;
Expand All @@ -36,7 +43,7 @@ TileServer({
});
```

See the [express.js example](/example) for a fully functioning server that exposes the above tile server on a REST endpoint.
See the [express.js example](/example) for a fully functioning server that exposes the above tile server on a REST endpoint.

See the [TileServerConfig](/types/TileServerConfig.ts) for the initial configuration options, to configure the cache, connection pool, filters, etc..

Expand Down Expand Up @@ -94,15 +101,14 @@ The main performance bottleneck for clusterbuster is the PostgreSQL server as th

All of these tile servers and tile generators offer some subset of the functionality we required, but lacked atleast one, which is our motivation for making clusterbuster.

| Tiler | dynamic data | filtering | clustering |
| Tiler | dynamic data | filtering | clustering |
| ------------- | ------------ | --------- | ---------- |
| clusterbuster | ✓ | ✓ | ✓ |
| Martin | ✓ | ✓ | x |
| Tilestrata | ✓ | x | x |
| Tegola | ✓ | x | x |
| Tippecanoe | x | x | ✓ |
| supertiler | x | x | ✓ |

| clusterbuster | ✓ | ✓ | ✓ |
| Martin | ✓ | ✓ | x |
| Tilestrata | ✓ | x | x |
| Tegola | ✓ | x | x |
| Tippecanoe | x | x | ✓ |
| supertiler | x | x | ✓ |

## When not to use clusterbuster

Expand Down
17 changes: 14 additions & 3 deletions example/express.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,23 @@ TileServer({
maxZoomLevel,
resolution: 512,
attributes: ['status', 'speed'],
filtersToWhere: filters => {
filtersToWhere: (filters = { status: undefined, speed: undefined }) => {
// You are responsible for protecting against SQL injection in this function. Because there are many ways to filter, it depends on the filter type on how to approach this.

// For example a number can be safely used by passing it thorugh parseFloat
const whereStatements = [];
if (!!filters && !!filters.status) {

// The below statement checks that filters.status is one of 'free' or 'busy' to prevent a potential SQL injection
if (
filters.status &&
(filters.status === 'free' || filters.status === 'busy')
) {
whereStatements.push(`status = '${filters.status}'`);
}
if (!!filters && !!filters.speed) {
if (
filters.speed &&
(filters.speed === 'fast' || filters.speed === 'slow')
) {
whereStatements.push(`speed = '${filters.speed}'`);
}
return whereStatements;
Expand Down
102 changes: 79 additions & 23 deletions lib/__test__/__snapshots__/queries.test.ts.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,21 @@ exports[`createClusterQuery should create a clustered Query 1`] = `
"
with filtered AS
(SELECT public.stations.wkb_geometry, 1 as theCount , a
(SELECT $1.$2, 1 as theCount $3
FROM public.stations
WHERE ST_Intersects(TileBBox(1, 0, 1, 3857), ST_Transform(public.stations.wkb_geometry, 3857))
AND status = status AND [1, 2] @> [2, 3]
WHERE ST_Intersects(TileBBox($4, $5, $6, 3857), ST_Transform($7.$8, 3857))
$9
),
clustered_10 AS
(SELECT wkb_geometry as center,
clustered_$10 AS
(SELECT $11 as center,
theCount,
ST_ClusterDBSCAN(wkb_geometry, 0.009765625, 1) over () as clusters , a
ST_ClusterDBSCAN($12, $13, 1) over () as clusters $14
FROM filtered),
grouped_clusters_10 AS
grouped_clusters_$15 AS
(SELECT SUM(theCount) as theCount,
FIRST(a) as a,
$16
ST_Centroid(ST_Collect(center)) as center
FROM clustered_10
FROM clustered_$17
GROUP BY clusters),
Expand Down Expand Up @@ -132,33 +132,89 @@ with filtered AS
tiled as
(SELECT center,
theCount , a
FROM grouped_clusters_1
WHERE ST_Intersects(TileBBox(1, 0, 1, 3857), ST_Transform(center, 3857))),
theCount $18
FROM grouped_clusters_$19
WHERE ST_Intersects(TileBBox($20, $21, $22, 3857), ST_Transform(center, 3857))),
q as
(SELECT 1 as c1,
ST_AsMVTGeom(ST_Transform(center, 3857), TileBBox(1, 0, 1, 3857), 512, 10, false) AS geom,
jsonb_build_object('count', theCount, 'a', a) as attributes
ST_AsMVTGeom(ST_Transform(center, 3857), TileBBox($23, $24, $25, 3857), $26, 10, false) AS geom,
jsonb_build_object('count', theCount$27) as attributes
FROM tiled)
SELECT ST_AsMVT(q, 'points', 512, 'geom') as mvt
SELECT ST_AsMVT(q, '$28', $29, 'geom') as mvt
from q
"
`;

exports[`createClusterQuery should create a clustered Query 2`] = `
Array [
"public.stations",
"wkb_geometry",
", a",
1,
0,
1,
"public.stations",
"wkb_geometry",
"AND status = status AND [1, 2] @> [2, 3]",
10,
"wkb_geometry",
"wkb_geometry",
0.009765625,
", a",
10,
"FIRST(a) as a,",
10,
", a",
1,
1,
0,
1,
1,
0,
1,
512,
", 'a', a",
"points",
512,
]
`;

exports[`createClusterQuery should create a unclustered Query 1`] = `
"
Object {
"sql": "
WITH filtered AS
(SELECT public.stations.wkb_geometry , a
(SELECT $1.$2 $3
FROM public.stations
WHERE ST_Intersects(TileBBox(15, 0, 1, 3857), ST_Transform(public.stations.wkb_geometry, 3857))
WHERE ST_Intersects(TileBBox($4, $5, $6, 3857), ST_Transform($7.$8, 3857))
$9
),
q as
(SELECT 1 as c1,
ST_AsMVTGeom(ST_Transform(wkb_geometry, 3857), TileBBox(15, 0, 1, 3857), 512, 10, false) AS geom,
jsonb_build_object('count', 1, 'a', a) as attributes
ST_AsMVTGeom(ST_Transform($10, 3857), TileBBox($11, $12, $13, 3857), $14, 10, false) AS geom,
jsonb_build_object('count', 1$15) as attributes
FROM filtered)
SELECT ST_AsMVT(q, 'points', 512, 'geom') as mvt
SELECT ST_AsMVT(q, '$16', $17, 'geom') as mvt
from q
"
",
"type": "SLONIK_TOKEN_SQL",
"values": Array [
"public.stations",
"wkb_geometry",
", a",
15,
0,
1,
"public.stations",
"wkb_geometry",
"",
"wkb_geometry",
15,
0,
1,
512,
", 'a', a",
"points",
512,
],
}
`;
28 changes: 16 additions & 12 deletions lib/__test__/queries.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,23 @@ const { createQueryForTile } = require('../queries');

describe('createClusterQuery', () => {
it('should create a clustered Query', () => {
const query = createQueryForTile({
z: 1,
x: 0,
y: 1,
table: 'public.stations',
geometry: 'wkb_geometry',
sourceLayer: 'points',
maxZoomLevel: 10,
resolution: 512,
attributes: ['a'],
query: ['status = status', '[1, 2] @> [2, 3]'],
});
expect(
createQueryForTile({
z: 1,
x: 0,
y: 1,
table: 'public.stations',
geometry: 'wkb_geometry',
sourceLayer: 'points',
maxZoomLevel: 10,
resolution: 512,
attributes: ['a'],
query: ['status = status', '[1, 2] @> [2, 3]'],
})
query.sql
).toMatchSnapshot();
expect(
query.values
).toMatchSnapshot();
});

Expand Down
Loading

0 comments on commit 81a738e

Please sign in to comment.