Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
813 changes: 327 additions & 486 deletions demo/js/index.js

Large diffs are not rendered by default.

528 changes: 528 additions & 0 deletions demo/js/ml-datasets.js

Large diffs are not rendered by default.

15 changes: 15 additions & 0 deletions demo/ml-datasets.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>React TypeScript Webpack App</title>
<link href="/assets/govuk-frontend.min.css" rel="stylesheet" media="all">
<link href="./index.css" rel="stylesheet" media="all">
</head>
<body style="padding: 20px">
<script>document.body.classList.add('im-is-loading')</script>
<div id="map"></div>
<script src="./ml-datasets.js"></script>
</body>
</html>
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,6 @@ import { MAX_TILE_ZOOM, hashString } from '../layerIds.js'
import { anchorToMaplibre } from '../../../../../../../providers/maplibre/src/utils/symbolImages.js'

export class MapLibreDataset extends Dataset {
get isDynamicSource () {
return typeof this.geojson === 'string' && !!this.idProperty && typeof this.transformRequest === 'function'
}

get visibility () { return this.visible ? 'visible' : 'none' }

get fillLayerId () {
Expand Down Expand Up @@ -72,19 +68,22 @@ export class MapLibreDataset extends Dataset {

get sourceId () {
if (this.isSublayer) { return this.parent.sourceId }
if (this.hasDynamicGeoJSON) { return this.dynamicGeoJSON.sourceId }
if (this.tiles) {
const tilesKey = Array.isArray(this.tiles) ? this.tiles.join(',') : this.tiles
return `tiles-${hashString(tilesKey)}`
}
if (this.geojson) {
if (this.isDynamicSource) { return `geojson-dynamic-${this.id}` }
if (typeof this.geojson === 'string') { return `geojson-${hashString(this.geojson)}` }
return `geojson-${this.id}`
}
return `source-${this.id}`
}

get source () {
if (this.hasDynamicGeoJSON) {
return this.dynamicGeoJSON.source
}
if (this.tiles) {
return {
type: 'vector',
Expand All @@ -94,8 +93,7 @@ export class MapLibreDataset extends Dataset {
}
}
if (this.geojson) {
const data = this.isDynamicSource ? { type: 'FeatureCollection', features: [] } : this.geojson
return { type: 'geojson', data, generateId: true }
return { type: 'geojson', data: this.geojson, generateId: true }
}
return null
}
Expand Down Expand Up @@ -147,7 +145,7 @@ export class MapLibreDataset extends Dataset {
}

get _hiddenFeaturesIdExpression () {
return this.idProperty ? ['to-string', ['get', this.idProperty]] : ['to-string', ['id']]
return this.hasDynamicGeoJSON ? this.dynamicGeoJSON.hiddenFeaturesIdExpression : ['to-string', ['id']]
}

get _hiddenFeaturesFilter () {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,10 @@ describe('MapLibreDataset', () => {
'ds-bare': { id: 'ds-bare' },
'ds-no-id-prop': { id: 'ds-no-id-prop', geojson: 'https://example.com/data', transformRequest: () => {} },
'ds-no-transform': { id: 'ds-no-transform', geojson: 'https://example.com/data', idProperty: 'id' },
// shared: dynamic geojson — used by isDynamicSource, sourceId, and source tests
'ds-dynamic': { id: 'ds-dynamic', geojson: 'https://example.com/data', idProperty: 'gid', transformRequest: () => {} },
'ds-static-url': { id: 'ds-static-url', geojson: 'https://example.com/static.geojson' },
// shared: tiles with no zoom — used by source minzoom and maxzoom fallback tests
'ds-tiles-no-zoom': { id: 'ds-tiles-no-zoom', tiles: ['https://example.com/{z}/{x}/{y}'] },
// getSymbolSource/getFillSource/getStrokeSource 'has filter' tests use historic-monuments-prehistoric and existing-fields from demo data
// _hiddenFeaturesIdExpression: ds-dynamic reused (has idProperty: 'gid')
// _hiddenFeaturesFilter — ds-hf-123 also reused in filter describe
'ds-hf-123': { id: 'ds-hf-123', hiddenFeatures: [1, 2, 3] },
'ds-hf-neg1': { id: 'ds-hf-neg1', hiddenFeatures: [-1, 5] },
Expand Down Expand Up @@ -97,24 +94,6 @@ describe('MapLibreDataset', () => {
})
})

describe('isDynamicSource', () => {
it('returns true when geojson is a string, idProperty is set, and transformRequest is a function', () => {
expect(datasetRegistry.getDataset('ds-dynamic').isDynamicSource).toBe(true)
})

it('returns false when geojson is an object', () => {
expect(datasetRegistry.getDataset('historic-monuments').isDynamicSource).toBe(false)
})

it('returns false when idProperty is missing', () => {
expect(datasetRegistry.getDataset('ds-no-id-prop').isDynamicSource).toBe(false)
})

it('returns false when transformRequest is not a function', () => {
expect(datasetRegistry.getDataset('ds-no-transform').isDynamicSource).toBe(false)
})
})

describe('visibility', () => {
it('returns "visible" when visible is true', () => {
expect(datasetRegistry.getDataset('existing-fields').visibility).toBe('visible')
Expand Down Expand Up @@ -154,7 +133,7 @@ describe('MapLibreDataset', () => {
expect(result).toEqual([{
layerIds: ['land-covers-130-131', 'land-covers-130-131-stroke'],
filter: ['all',
['!', ['in', ['to-string', ['id']], ['literal', ['42']]]],
['!', ['in', ['to-string', ['get', 'id']], ['literal', ['42']]]],
['in', ['get', 'dominant_land_cover'], ['literal', ['130', '131']]]]
}])
})
Expand Down Expand Up @@ -192,7 +171,7 @@ describe('MapLibreDataset', () => {
})

it('returns geojson-dynamic-{id} for a dynamic geojson source', () => {
expect(datasetRegistry.getDataset('ds-dynamic').sourceId).toBe('geojson-dynamic-ds-dynamic')
expect(datasetRegistry.getDataset('land-covers').sourceId).toBe('geojson-dynamic-land-covers')
})

it('returns geojson-{hash} for a static string geojson url', () => {
Expand Down Expand Up @@ -229,7 +208,7 @@ describe('MapLibreDataset', () => {
})

it('returns a geojson source with empty FeatureCollection for a dynamic geojson source', () => {
expect(datasetRegistry.getDataset('ds-dynamic').source).toEqual({
expect(datasetRegistry.getDataset('land-covers').source).toEqual({
type: 'geojson',
data: { type: 'FeatureCollection', features: [] },
generateId: true
Expand Down Expand Up @@ -370,7 +349,7 @@ describe('MapLibreDataset', () => {

describe('_hiddenFeaturesIdExpression', () => {
it('uses get(idProperty) when idProperty is set', () => {
expect(datasetRegistry.getDataset('ds-dynamic')._hiddenFeaturesIdExpression).toEqual(['to-string', ['get', 'gid']])
expect(datasetRegistry.getDataset('land-covers')._hiddenFeaturesIdExpression).toEqual(['to-string', ['get', 'id']])
})

it('uses the feature id when idProperty is not set', () => {
Expand Down
6 changes: 3 additions & 3 deletions plugins/beta/datasets/src/datasets.js
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ export const createDatasets = ({
datasetRegistry.attach(mappedDatasets)
adapter.init(mapStyle).then(() => {
datasetRegistry.forEachDataset(registryDataset => {
if (!registryDataset.hasDynamicSource) {
if (!registryDataset.hasDynamicGeoJSON) {
return
}

const { dynamicGeoJSON } = registryDataset
const dynamicSource = createDynamicSource({
dataset: registryDataset,
dynamicGeoJSON,
map: mapProvider.map,
onUpdate: (datasetId, geojson) => adapter.setData(datasetId, geojson)
})
Expand Down
16 changes: 8 additions & 8 deletions plugins/beta/datasets/src/fetch/createDynamicSource.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,14 @@ const EVICTION_THRESHOLD = 1.2 // Trigger eviction at 120% of maxFeatures
/**
* Create a dynamic GeoJSON source that fetches data based on viewport
* @param {Object} options
* @param {Object} options.dataset - Dataset configuration
* @param {Object} options.registryDataset - registryDataset instance
* @param {Object} options.map - Map instance
* @param {string} options.sourceId - Source ID for the map
* @param {Function} options.onUpdate - Callback when source data should be updated
* @returns {Object} { destroy, clear, refresh }
*/
export const createDynamicSource = ({ dataset, map, onUpdate }) => {
const { geojson: baseUrl, idProperty, transformRequest, maxFeatures, minZoom = 0 } = dataset
export const createDynamicSource = ({ dynamicGeoJSON, map, onUpdate }) => {
const { url: baseUrl, idProperty, transformRequest, maxFeatures, minZoom = 0 } = dynamicGeoJSON

// Feature cache: id → { feature, bbox, lastSeenAt }
const features = new Map()
Expand Down Expand Up @@ -112,7 +112,7 @@ export const createDynamicSource = ({ dataset, map, onUpdate }) => {
currentController = new AbortController()

try {
const context = { bbox: currentBbox, zoom, dataset }
const context = { bbox: currentBbox, zoom }
const data = await fetchGeoJSON(baseUrl, context, transformRequest, currentController.signal)

const now = Date.now()
Expand Down Expand Up @@ -144,12 +144,12 @@ export const createDynamicSource = ({ dataset, map, onUpdate }) => {
}

// Update map source
onUpdate(dataset.id, toFeatureCollection())
onUpdate(dynamicGeoJSON.id, toFeatureCollection())
} catch (error) {
if (error.name === 'AbortError') {
return
}
console.error(`Failed to fetch dynamic GeoJSON for ${dataset.id}:`, error)
console.error(`Failed to fetch dynamic GeoJSON for ${dynamicGeoJSON.id}:`, error)
}
}

Expand Down Expand Up @@ -184,7 +184,7 @@ export const createDynamicSource = ({ dataset, map, onUpdate }) => {
clear () {
features.clear()
fetchedBbox = null
onUpdate(dataset.id, { type: 'FeatureCollection', features: [] })
onUpdate(dynamicGeoJSON.id, { type: 'FeatureCollection', features: [] })
},

/**
Expand All @@ -208,7 +208,7 @@ export const createDynamicSource = ({ dataset, map, onUpdate }) => {
*/
reapply () {
if (features.size > 0) {
onUpdate(dataset.id, toFeatureCollection())
onUpdate(dynamicGeoJSON.id, toFeatureCollection())
}
}
}
Expand Down
7 changes: 6 additions & 1 deletion plugins/beta/datasets/src/reducers/__data__/demoDatasets.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,12 @@ export const datasets = [
{
id: 'land-covers',
label: 'Land covers',
geojson: `${process.env.FARMING_API_URL}/api/collections/parcels/items?sbi=106325052`, // 106200212
dynamicGeoJSON: {
idProperty: 'id', // required - the ID that identifies individual features
url: `${process.env.FARMING_API_URL}/api/collections/parcels/items?sbi=106325052`, // required
transformRequest: (url) => url + 'TRANSFORMED', // Required
maxFeatures: 50000 // Optional: evict distant features when exceeded
},
hiddenFeatures: [42],
query: {},
maxFeatures: 50000, // Optional: evict distant features when exceeded
Expand Down
16 changes: 13 additions & 3 deletions plugins/beta/datasets/src/registry/dataset.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { datasetRegistry } from './datasetRegistry.js'
import { hasCustomVisualStyle } from '../defaults.js'
import { hasPattern } from '../../../../../src/utils/patternUtils.js'
import { DynamicGeoJson } from './dynamicGeoJson.js'

export class Dataset {
constructor (dataset) {
Expand All @@ -16,7 +17,8 @@ export class Dataset {
get tiles () { return this._datasetDefinition.tiles }
get geojson () { return this._datasetDefinition.geojson }
get idProperty () { return this._datasetDefinition.idProperty }
get transformRequest () { return this._datasetDefinition.transformRequest }
// TODO - handle transformRequest for non-dynamicGeoJSON as well (e.g. to add auth headers) --- IGNORE ---
// get transformRequest () { return this._datasetDefinition.transformRequest }
get parentId () { return this._datasetDefinition.parentId }
get minZoom () { return this._datasetDefinition.minZoom || this.parent?.minZoom }
get maxZoom () { return this._datasetDefinition.maxZoom || this.parent?.maxZoom }
Expand All @@ -26,8 +28,16 @@ export class Dataset {
return this.style.opacity === undefined ? 1 : this.style.opacity
}

get hasDynamicSource () {
return typeof this.geojson === 'string' && !!this.idProperty && typeof this.transformRequest === 'function'
get hasDynamicGeoJSON () {
return Boolean(this._datasetDefinition.dynamicGeoJSON)
}

get dynamicGeoJSON () {
return new DynamicGeoJson({
...this._datasetDefinition.dynamicGeoJSON,
id: this.id,
minZoom: this.minZoom
})
}

get hiddenFeatures () { return this._datasetDefinition.hiddenFeatures }
Expand Down
30 changes: 6 additions & 24 deletions plugins/beta/datasets/src/registry/dataset.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ describe('Dataset class', () => {
})
})

describe('tiles, geojson, idProperty, transformRequest, parentId', () => {
describe('tiles, geojson, idProperty, parentId', () => {
it('returns tiles from the definition', () => {
const tiles = ['https://example.com/{z}/{x}/{y}']
const dataset = new Dataset({ tiles })
Expand All @@ -215,12 +215,6 @@ describe('Dataset class', () => {
expect(dataset.idProperty).toBe('gid')
})

it('returns transformRequest from the definition', () => {
const fn = () => {}
const dataset = new Dataset({ transformRequest: fn })
expect(dataset.transformRequest).toBe(fn)
})

it('returns parentId from the definition', () => {
const dataset = new Dataset({ parentId: 'parent' })
expect(dataset.parentId).toBe('parent')
Expand Down Expand Up @@ -270,25 +264,13 @@ describe('Dataset class', () => {
})
})

describe('hasDynamicSource', () => {
it('returns true when geojson is a string, idProperty is set, and transformRequest is a function', () => {
const dataset = new Dataset({ geojson: 'https://example.com/data', idProperty: 'id', transformRequest: () => {} })
expect(dataset.hasDynamicSource).toBe(true)
})

it('returns false when geojson is an object (not a string)', () => {
const dataset = new Dataset({ geojson: { type: 'FeatureCollection', features: [] }, idProperty: 'id', transformRequest: () => {} })
expect(dataset.hasDynamicSource).toBe(false)
})

it('returns false when idProperty is not set', () => {
const dataset = new Dataset({ geojson: 'https://example.com/data', transformRequest: () => {} })
expect(dataset.hasDynamicSource).toBe(false)
describe('hasDynamicGeoJSON', () => {
it('returns true when definition has a dynamicGeoJSON object', () => {
expect(datasetRegistry.getDataset('land-covers').hasDynamicGeoJSON).toBe(true)
})

it('returns false when transformRequest is not a function', () => {
const dataset = new Dataset({ geojson: 'https://example.com/data', idProperty: 'id' })
expect(dataset.hasDynamicSource).toBe(false)
it('returns false when definition does not have a dynamicGeoJSON object', () => {
expect(datasetRegistry.getDataset('historic-monuments').hasDynamicGeoJSON).toBe(false)
})
})

Expand Down
17 changes: 17 additions & 0 deletions plugins/beta/datasets/src/registry/dynamicGeoJson.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
export class DynamicGeoJson {
constructor (dynamicGeoJSONDefinition) {
this.id = dynamicGeoJSONDefinition.id
this.url = dynamicGeoJSONDefinition.url
this.transformRequest = dynamicGeoJSONDefinition.transformRequest
this.maxFeatures = dynamicGeoJSONDefinition.maxFeatures
this.minZoom = dynamicGeoJSONDefinition.minZoom
this.idProperty = dynamicGeoJSONDefinition.idProperty
this.hiddenFeaturesIdExpression = ['to-string', ['get', this.idProperty]]
this.sourceId = `geojson-dynamic-${this.id}`
this.source = {
type: 'geojson',
data: { type: 'FeatureCollection', features: [] },
generateId: true
}
}
}
1 change: 1 addition & 0 deletions webpack.dev.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ export default {
planning: path.join(__dirname, 'demo/js/planning.js'),
'planning-ol': path.join(__dirname, 'demo/js/planning-ol.js'),
gep: path.join(__dirname, 'demo/js/gep.js'),
'ml-datasets': path.join(__dirname, 'demo/js/ml-datasets.js'),
multimap: path.join(__dirname, 'demo/js/multimap.js')
},
output: {
Expand Down
Loading