From b758be6fc518c1024093d536f101547596ff34ca Mon Sep 17 00:00:00 2001 From: Jay Date: Mon, 5 Feb 2024 22:58:14 +0530 Subject: [PATCH 1/4] feat: add script to parse gdb to geoJSONs --- server/package.json | 2 ++ server/scripts/gdb_to_geojson.js | 51 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 server/scripts/gdb_to_geojson.js diff --git a/server/package.json b/server/package.json index 9eb2bfd..dcb9074 100644 --- a/server/package.json +++ b/server/package.json @@ -18,7 +18,9 @@ "@types/swagger-ui-express": "^4.1.4", "bun": "^1.0.7", "express": "^4.18.2", + "fgdb": "^1.0.0", "geojson-rbush": "^3.2.0", + "jszip": "^3.10.1", "maxmind": "^4.3.16", "maxmind-db-reader": "^0.2.1", "mmdb-lib": "^2.0.2", diff --git a/server/scripts/gdb_to_geojson.js b/server/scripts/gdb_to_geojson.js new file mode 100644 index 0000000..67d5261 --- /dev/null +++ b/server/scripts/gdb_to_geojson.js @@ -0,0 +1,51 @@ +import fgdb from 'fgdb'; +import * as fs from 'fs/promises'; +import * as path from "path"; +import JSZip from "jszip"; + +const inputDir = `${import.meta.dir}/../geojson-data/indian_village_boundaries.zip`; +const outputDir = `${import.meta.dir}/../geojson-data/indian_village_boundaries`; + +function isZipFile(filename) { + return filename.endsWith('.zip'); +} + +const convertGDBtoGeoJSON = async (gdbFileBuffer, outputFilePath) => { + try { + const geoJSON = await fgdb(gdbFileBuffer); + const contentToWrite = JSON.stringify(geoJSON); + await fs.writeFile(outputFilePath, contentToWrite, 'utf8'); + console.log("Converted and saved:", outputFilePath); + + } catch (error) { + console.error("Error converting GDB to GeoJSON:", error); + } +}; + + +const processZipFiles = async () => { + let new_zip = new JSZip(); + let parent_file_data = await fs.readFile(inputDir); + let parent_folder = await new_zip.loadAsync(parent_file_data); + + await fs.rm(outputDir, {recursive: true, force: true}); + await fs.mkdir(outputDir); + + + for (const fileName of Object.keys(parent_folder.files)) { + const file = parent_folder.files[fileName]; + + if (!isZipFile(fileName)) { + await fs.rm(path.join(outputDir, fileName), {recursive: true, force: true}); + await fs.mkdir(path.join(outputDir, fileName), {recursive: true}); + } else if (isZipFile(fileName)) { + const fileBuffer = await parent_folder.file(fileName).async('nodebuffer'); + await convertGDBtoGeoJSON(fileBuffer, path.join(outputDir, fileName.replace(/\.zip$/i, '.geoJSON'))); + } + } +} + + +processZipFiles().then(r => { + console.log("Converted all gdbs to geJSON"); +}); \ No newline at end of file From cfd1d9081013ead82fd99a6a5899d451c08a3a31 Mon Sep 17 00:00:00 2001 From: Jay Kumar <70096901+35C4n0r@users.noreply.github.com> Date: Mon, 12 Feb 2024 14:09:32 +0530 Subject: [PATCH 2/4] Update package.json --- server/package.json | 1 - 1 file changed, 1 deletion(-) diff --git a/server/package.json b/server/package.json index dcb9074..590d6fa 100644 --- a/server/package.json +++ b/server/package.json @@ -20,7 +20,6 @@ "express": "^4.18.2", "fgdb": "^1.0.0", "geojson-rbush": "^3.2.0", - "jszip": "^3.10.1", "maxmind": "^4.3.16", "maxmind-db-reader": "^0.2.1", "mmdb-lib": "^2.0.2", From 89cccf427b9f4bb941b2cb8c3afdf8cb6a608668 Mon Sep 17 00:00:00 2001 From: Jay Kumar <70096901+35C4n0r@users.noreply.github.com> Date: Mon, 12 Feb 2024 14:10:09 +0530 Subject: [PATCH 3/4] Update gdb_to_geojson.js --- server/scripts/gdb_to_geojson.js | 44 +++++++++++++++----------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/server/scripts/gdb_to_geojson.js b/server/scripts/gdb_to_geojson.js index 67d5261..a3696e9 100644 --- a/server/scripts/gdb_to_geojson.js +++ b/server/scripts/gdb_to_geojson.js @@ -1,15 +1,20 @@ import fgdb from 'fgdb'; import * as fs from 'fs/promises'; import * as path from "path"; -import JSZip from "jszip"; -const inputDir = `${import.meta.dir}/../geojson-data/indian_village_boundaries.zip`; -const outputDir = `${import.meta.dir}/../geojson-data/indian_village_boundaries`; +const inputDir = `${import.meta.dir}/../geojson-data/indian_villages_boundaries.zip`; +const outputDir = `${import.meta.dir}/../geojson-data/indian_villages_boundaries`; function isZipFile(filename) { return filename.endsWith('.zip'); } +async function createIfAbsent(filePath) { + if (!await fs.exists(filePath)) { + await fs.mkdir(filePath, {recursive: true}); + } +} + const convertGDBtoGeoJSON = async (gdbFileBuffer, outputFilePath) => { try { const geoJSON = await fgdb(gdbFileBuffer); @@ -22,30 +27,23 @@ const convertGDBtoGeoJSON = async (gdbFileBuffer, outputFilePath) => { } }; +const processFiles = async () => { + await createIfAbsent(outputDir); + const items = await fs.readdir(inputDir, {withFileTypes: true, recursive: true}); -const processZipFiles = async () => { - let new_zip = new JSZip(); - let parent_file_data = await fs.readFile(inputDir); - let parent_folder = await new_zip.loadAsync(parent_file_data); - - await fs.rm(outputDir, {recursive: true, force: true}); - await fs.mkdir(outputDir); + for (const item of items) { + const itemPath = path.join(inputDir, item.name); - - for (const fileName of Object.keys(parent_folder.files)) { - const file = parent_folder.files[fileName]; - - if (!isZipFile(fileName)) { - await fs.rm(path.join(outputDir, fileName), {recursive: true, force: true}); - await fs.mkdir(path.join(outputDir, fileName), {recursive: true}); - } else if (isZipFile(fileName)) { - const fileBuffer = await parent_folder.file(fileName).async('nodebuffer'); - await convertGDBtoGeoJSON(fileBuffer, path.join(outputDir, fileName.replace(/\.zip$/i, '.geoJSON'))); + if (!isZipFile(item.name)) { + await createIfAbsent(path.join(outputDir, item.name)); + } else if (isZipFile(item.name)) { + const fileBuffer = await fs.readFile(itemPath); + await convertGDBtoGeoJSON(fileBuffer, path.join(outputDir, item.name.replace(/\.zip$/i, '.geoJSON'))); } } -} +}; -processZipFiles().then(r => { +processFiles().then(r => { console.log("Converted all gdbs to geJSON"); -}); \ No newline at end of file +}); From a811c89575f7369df80648cb5bc7e02b962921c6 Mon Sep 17 00:00:00 2001 From: Jay Kumar <70096901+35C4n0r@users.noreply.github.com> Date: Wed, 14 Feb 2024 18:56:38 +0530 Subject: [PATCH 4/4] Update gdb_to_geojson.js ref: refactored output destination --- server/scripts/gdb_to_geojson.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/scripts/gdb_to_geojson.js b/server/scripts/gdb_to_geojson.js index a3696e9..bf1b774 100644 --- a/server/scripts/gdb_to_geojson.js +++ b/server/scripts/gdb_to_geojson.js @@ -3,7 +3,7 @@ import * as fs from 'fs/promises'; import * as path from "path"; const inputDir = `${import.meta.dir}/../geojson-data/indian_villages_boundaries.zip`; -const outputDir = `${import.meta.dir}/../geojson-data/indian_villages_boundaries`; +const outputDir = `${import.meta.dir}/../geojson-data/indian_village_boundaries`; function isZipFile(filename) { return filename.endsWith('.zip');