diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..0bf64679 --- /dev/null +++ b/.env.example @@ -0,0 +1,15 @@ +# # postgres +# DS_TYPE=postgres +# DS_DATABASE=ba000001 +# DS_HOST=localhost +# DS_PORT=5432 +# DS_USERNAME=postgres +# DS_PASSWORD=postgres + +# # mysql +# DS_TYPE=mysql +# DS_DATABASE=ba000001 +# DS_HOST=127.0.0.1 +# DS_PORT=3306 +# DS_USERNAME=root +# DS_PASSWORD=password diff --git a/.gitignore b/.gitignore index 9159f5e5..cb71d64f 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ spec/*.sqlite *.tgz .pnpm-store/ pnpm-lock.yaml -.vscode/result.* \ No newline at end of file +.vscode/result.* +.env diff --git a/.vscode/launch.json b/.vscode/launch.json index 60f67e66..b89566f1 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -185,6 +185,34 @@ ], "console": "integratedTerminal", "internalConsoleOptions": "neverOpen", - } + }, + { + "type": "node", + "request": "launch", + "name": "test", + "program": "${workspaceFolder}/node_modules/jest/bin/jest", + "preLaunchTask": "npm: build", + "runtimeArgs": [ + "--require", + "ts-node/register" + ], + "args": [ + ], + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen", + }, + { + "type": "node", + "request": "launch", + "name": "test:system", + "program": "${workspaceFolder}/spec/execution-test.spec.mjs", + "preLaunchTask": "npm: build", + "runtimeArgs": [ + "--require", + "ts-node/register" + ], + "console": "integratedTerminal", + "internalConsoleOptions": "neverOpen", + }, ] } \ No newline at end of file diff --git a/__mocks__/typeorm.ts b/__mocks__/typeorm.ts new file mode 100644 index 00000000..a010b920 --- /dev/null +++ b/__mocks__/typeorm.ts @@ -0,0 +1,52 @@ +// import { jest } from '@jest/globals'; +// import 'reflect-metadata'; + +export const QueryRunner = jest.fn().mockImplementation( + () => { + return { + isTransactionActive: false, + connect: jest.fn().mockImplementation(() => { + return Promise.resolve(); + }), + startTransaction: jest.fn(() => { + return Promise.resolve(); + }), + connection: { + query: jest.fn(() => { + return Promise.resolve(); + }) + }, + commitTransaction: jest.fn(() => { + return Promise.resolve(); + }), + rollbackTransaction: jest.fn(() => { + return Promise.resolve(); + }), + release: jest.fn(() => { + return Promise.resolve(); + }) + } + } +); + +export const DataSource = jest.fn().mockImplementation( + () => { + return { + initialize: jest.fn(() => { + return Promise.resolve(this); + }), + query: jest.fn().mockImplementation(() => { + return Promise.resolve([]); + }), + createQueryRunner: jest.fn().mockImplementation(() => { + return new QueryRunner(); + }), + destroy: jest.fn(() => { + return Promise.resolve(); + }), + options: { + type: 'better-sqlite3' + } + }; + } +); diff --git a/jest.config.js b/jest.config.js index 01b30ec5..2eeb0781 100644 --- a/jest.config.js +++ b/jest.config.js @@ -25,6 +25,7 @@ module.exports = { 'node:fs': '/__mocks__/fs.ts', 'node:os': '/__mocks__/os.ts', 'better-sqlite3': '/__mocks__/better-sqlite3.ts', + 'typeorm': '/__mocks__/typeorm.ts', 'csv-parser': '/__mocks__/csv-parser.ts', ...pathsToModuleNameMapper(compilerOptions.paths), }, diff --git a/package.json b/package.json index aa938c67..10167699 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,8 @@ "test:ci": "jest --runInBand", "test:system": "npm run build && node spec/execution-test.spec.mjs", "lint": "eslint . --ext .ts", - "lint-and-fix": "eslint . --ext .ts --fix" + "lint-and-fix": "eslint . --ext .ts --fix", + "typeorm": "typeorm-ts-node-commonjs" }, "devDependencies": { "@jest/globals": "29.7.0", @@ -69,14 +70,18 @@ "cli-infinity-progress": "^0.6.7", "cli-progress": "^3.12.0", "csv-parser": "^3.0.0", + "dotenv": "^16.3.2", "http-status-codes": "^2.3.0", "i18next": "^23.5.1", "lru-cache": "^10.1.0", + "mysql2": "^3.7.1", "node-stream-zip": "^1.15.0", + "pg": "^8.11.3", "proj4": "^2.9.1", "reflect-metadata": "^0.1.13", "string-hash": "^1.1.3", "tsyringe": "^4.8.0", + "typeorm": "^0.3.19", "undici": "^5.26.3", "winston": "^3.11.0", "yargs": "^17.7.2" diff --git a/src/abrg-data-source.ts b/src/abrg-data-source.ts new file mode 100644 index 00000000..a6fc49ff --- /dev/null +++ b/src/abrg-data-source.ts @@ -0,0 +1,46 @@ +import 'reflect-metadata'; +import { join } from 'path'; +import { DataSource, DataSourceOptions } from 'typeorm'; +import 'dotenv/config'; + +export const commonOptions = { + synchronize: false, + logging: false, + entities: [join(__dirname, 'entity', '*.{ts,js}')], + migrations: [join(__dirname, 'migration', '*.{ts,js}')], + migrationsRun: true, +}; + +const sqliteOptions: DataSourceOptions = { + ...commonOptions, + type: 'better-sqlite3', + // src/interface-adapter/providers/provide-data-source.ts 内で絶対パスに置換 + database: 'ba000001.sqlite', +}; + +let options: DataSourceOptions = sqliteOptions; +if (process.env.DS_TYPE === 'postgres') { + options = { + ...commonOptions, + type: 'postgres', + host: process.env.DS_HOST, + port: Number(process.env.DS_PORT), + username: process.env.DS_USERNAME, + password: process.env.DS_PASSWORD, + database: process.env.DS_DATABASE, + parseInt8: true, + }; +} else if (process.env.DS_TYPE === 'mysql') { + options = { + ...commonOptions, + type: 'mysql', + host: process.env.DS_HOST, + port: Number(process.env.DS_PORT), + username: process.env.DS_USERNAME, + password: process.env.DS_PASSWORD, + database: process.env.DS_DATABASE, + charset: 'utf8mb4', + }; +} + +export const AbrgDataSource = new DataSource(options); diff --git a/src/controller/download/__tests__/download-dataset.spec.ts b/src/controller/download/__tests__/download-dataset.spec.ts index 9184ac84..325d9f81 100644 --- a/src/controller/download/__tests__/download-dataset.spec.ts +++ b/src/controller/download/__tests__/download-dataset.spec.ts @@ -23,7 +23,6 @@ */ import { describe, expect, it, jest } from '@jest/globals'; import mockedFs from '@mock/fs'; -import { default as BetterSqlite3 } from 'better-sqlite3'; import { downloadDataset } from '../download-dataset'; import { downloadProcess } from '../process/download-process'; import { extractDatasetProcess } from '../process/extract-dataset-process'; @@ -32,7 +31,7 @@ import { DOWNLOAD_DATASET_RESULT } from '../download-dataset-result'; jest.mock('fs'); jest.mock('node:fs'); jest.mock('winston'); -jest.mock('better-sqlite3'); +// jest.mock('typeorm.DataSource'); jest.mock('@interface-adapter/setup-container'); jest.mock('@usecase/ckan-downloader/ckan-downloader'); jest.mock('@domain/key-store/get-value-with-key') diff --git a/src/controller/download/download-dataset.ts b/src/controller/download/download-dataset.ts index e8878072..3586d945 100644 --- a/src/controller/download/download-dataset.ts +++ b/src/controller/download/download-dataset.ts @@ -28,7 +28,7 @@ import { AbrgMessage } from '@abrg-message/abrg-message'; import { saveKeyAndValue } from '@domain/key-store/save-key-and-value'; import { setupContainer } from '@interface-adapter/setup-container'; import { DI_TOKEN } from '@interface-adapter/tokens'; -import { Database } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; import fs from 'node:fs'; import path from 'node:path'; import { Logger } from 'winston'; @@ -70,9 +70,9 @@ export const downloadDataset = async ({ return DOWNLOAD_DATASET_RESULT.CAN_NOT_ACCESS_TO_DATASET_ERROR; } - const db = container.resolve(DI_TOKEN.DATABASE); + const ds = container.resolve(DI_TOKEN.DATASOURCE); const datasetHistory = await loadDatasetHistory({ - db, + ds, }); // -------------------------------------- @@ -92,7 +92,7 @@ export const downloadDataset = async ({ logger?.info( AbrgMessage.toString(AbrgMessage.ERROR_NO_UPDATE_IS_AVAILABLE) ); - db.close(); + await ds.destroy(); // 展開したzipファイルのディレクトリを削除 await fs.promises.rm(extractDir, { recursive: true }); @@ -103,18 +103,18 @@ export const downloadDataset = async ({ logger?.info(AbrgMessage.toString(AbrgMessage.LOADING_INTO_DATABASE)); await loadDatasetProcess({ - db, + ds, csvFiles, container, }); - saveKeyAndValue({ - db, + await saveKeyAndValue({ + ds, key: ckanId, value: downloadInfo.metadata.toString(), }); - db.close(); + await ds.destroy(); // 展開したzipファイルのディレクトリを削除 await fs.promises.rm(extractDir, { recursive: true }); diff --git a/src/controller/download/process/__mocks__/load-dataset-history.ts b/src/controller/download/process/__mocks__/load-dataset-history.ts index 6430c8f6..e08c928d 100644 --- a/src/controller/download/process/__mocks__/load-dataset-history.ts +++ b/src/controller/download/process/__mocks__/load-dataset-history.ts @@ -23,7 +23,7 @@ */ import { DatasetRow } from "@domain/dataset/dataset-row"; import { jest } from '@jest/globals'; -import { Database } from "better-sqlite3"; +import { DataSource } from "typeorm"; export const expectedResult = new Map([ ['mt_city_all.csv', new DatasetRow({ @@ -85,7 +85,7 @@ export const expectedResult = new Map([ ]); export const loadDatasetHistory = jest.fn(async (params: { - db: Database, + ds: DataSource, }): Promise> => { return Promise.resolve(expectedResult); }); \ No newline at end of file diff --git a/src/controller/download/process/__tests__/load-dataset-history.spec.ts b/src/controller/download/process/__tests__/load-dataset-history.spec.ts index 7c731b82..7b92f563 100644 --- a/src/controller/download/process/__tests__/load-dataset-history.spec.ts +++ b/src/controller/download/process/__tests__/load-dataset-history.spec.ts @@ -22,18 +22,21 @@ * SOFTWARE. */ -import MockedDB from '@mock/better-sqlite3'; +import { DataSource as MockedDS } from '@mock/typeorm'; import { describe, expect, it, jest } from "@jest/globals"; import { loadDatasetHistory } from '../load-dataset-history'; import { expectedResult } from '../__mocks__/load-dataset-history'; -jest.mock('better-sqlite3'); +jest.mock('typeorm'); jest.dontMock('../load-dataset-history') describe('load-dataset-history', () => { it('should return a Map', async () => { - const db = new MockedDB('dummy database'); - db.prepare.mockImplementation(() => ({ - all: jest.fn().mockReturnValue([ + const ds = new MockedDS({ + type: 'better-sqlite3', + database: 'dummy database', + }); + ds.query.mockImplementation(() => ( + Promise.resolve([ { key: 'mt_city_all.csv', type: 'city', @@ -97,13 +100,13 @@ describe('load-dataset-history', () => { crc32: 4236985285, last_modified: 1674556138000, } - ]), - })); + ]) + )); const results = await loadDatasetHistory({ - db, + ds, }); expect(results).toEqual(expectedResult); }) -}) \ No newline at end of file +}) diff --git a/src/controller/download/process/__tests__/load-dataset-process.spec.ts b/src/controller/download/process/__tests__/load-dataset-process.spec.ts index 3054604d..bc974eec 100644 --- a/src/controller/download/process/__tests__/load-dataset-process.spec.ts +++ b/src/controller/download/process/__tests__/load-dataset-process.spec.ts @@ -25,7 +25,7 @@ import { DummyCsvFile } from "@domain/dataset/__mocks__/dummy-csv.skip"; import { setupContainer } from "@interface-adapter/__mocks__/setup-container"; import { describe, expect, it, jest } from "@jest/globals"; -import MockedDB from '@mock/better-sqlite3'; +import { DataSource as MockedDS } from '@mock/typeorm'; import Stream from "node:stream"; import { DependencyContainer } from "tsyringe"; import { loadDatasetProcess } from "../load-dataset-process"; @@ -39,7 +39,7 @@ jest.mock("@domain/dataset/town-pos-dataset-file"); jest.mock("@domain/dataset/rsdtdsp-blk-pos-file"); jest.mock("@domain/dataset/rsdtdsp-rsdt-pos-file"); jest.mock('@interface-adapter/setup-container'); -jest.mock('better-sqlite3'); +jest.mock('typeorm'); jest.mock('csv-parser'); jest.dontMock('../load-dataset-process') @@ -310,9 +310,14 @@ describe('load-dataset-process', () => { const container = setupContainer() as DependencyContainer; it('should return csv file list', async () => { - const db = new MockedDB("dummy db"); + const ds = new MockedDS({ + type: 'better-sqlite3', + database: 'dummy db', + }); + const qr = ds.createQueryRunner(); + jest.spyOn(ds, 'createQueryRunner').mockReturnValue(qr); await loadDatasetProcess({ - db, + ds, container, csvFiles: [ mt_city_all_csv(), @@ -325,42 +330,42 @@ describe('load-dataset-process', () => { mt_town_pos_pref01_csv(), ], }); - expect(db.exec).toBeCalledWith("BEGIN"); - expect(db.prepare).toBeCalledWith("CityDatasetFile sql"); - expect(db.prepare).toBeCalledWith("PrefDatasetFile sql"); - expect(db.prepare).toBeCalledWith("RsdtdspBlkFile sql"); - expect(db.prepare).toBeCalledWith("RsdtdspBlkPosFile sql"); - expect(db.prepare).toBeCalledWith("RsdtdspRsdtFile sql"); - expect(db.prepare).toBeCalledWith("RsdtdspRsdtPosFile sql"); - expect(db.prepare).toBeCalledWith("TownDatasetFile sql"); - expect(db.prepare).toBeCalledWith("TownPosDatasetFile sql"); - expect(db.exec).toBeCalledWith("COMMIT"); + expect(qr.connect).toBeCalled(); + expect(qr.startTransaction).toBeCalled(); + expect(qr.connection.query).toBeCalledWith("PrefDatasetFile sql", []); + expect(qr.connection.query).toBeCalledWith("RsdtdspBlkFile sql", []); + expect(qr.connection.query).toBeCalledWith("RsdtdspBlkPosFile sql", []); + expect(qr.connection.query).toBeCalledWith("RsdtdspRsdtFile sql", []); + expect(qr.connection.query).toBeCalledWith("RsdtdspRsdtPosFile sql", []); + expect(qr.connection.query).toBeCalledWith("TownDatasetFile sql", []); + expect(qr.connection.query).toBeCalledWith("TownPosDatasetFile sql", []); + expect(qr.commitTransaction).toBeCalled(); + expect(qr.release).toBeCalled(); }) it('should rollback if an error has been occurred during the process', async () => { - const db = new MockedDB("dummy db"); - db.inTransaction = true; - db.prepare.mockImplementation((sql: string) => { - return { - run: jest.fn().mockImplementation(() => { - if (sql !== 'PrefDatasetFile sql') { - return; - } - throw new Error('Error!'); - }) + const ds = new MockedDS("dummy db"); + const qr = ds.createQueryRunner(); + jest.spyOn(ds, 'createQueryRunner').mockReturnValue(qr); + qr.isTransactionActive = true; + qr.connection.query.mockImplementation((sql: string, params: string[]) => { + if (sql !== 'PrefDatasetFile sql') { + return Promise.resolve(); } + return Promise.reject(new Error('Error!')); }) await expect(loadDatasetProcess({ - db, + ds, container, csvFiles: [ mt_pref_all_csv(), ], })).rejects.toThrow(); - expect(db.exec).toBeCalledWith("BEGIN"); - expect(db.prepare).toBeCalledWith("PrefDatasetFile sql"); - expect(db.exec).not.toBeCalledWith("COMMIT"); - expect(db.exec).toBeCalledWith("ROLLBACK"); + expect(qr.connect).toBeCalled(); + expect(qr.startTransaction).toBeCalled(); + expect(qr.connection.query).toBeCalledWith("PrefDatasetFile sql", []); + expect(qr.commitTransaction).not.toBeCalled(); + expect(qr.rollbackTransaction).toBeCalled(); }) }); diff --git a/src/controller/download/process/download-process.ts b/src/controller/download/process/download-process.ts index 151be514..6cbd7ce4 100644 --- a/src/controller/download/process/download-process.ts +++ b/src/controller/download/process/download-process.ts @@ -27,7 +27,7 @@ import { CkanDownloader, CkanDownloaderEvent, } from '@usecase/ckan-downloader/ckan-downloader'; -import { Database } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; import { SingleBar } from 'cli-progress'; import { DependencyContainer } from 'tsyringe'; @@ -43,7 +43,7 @@ export const downloadProcess = async ({ metadata: DatasetMetadata; downloadFilePath: string | null; }> => { - const db = container.resolve(DI_TOKEN.DATABASE); + const ds = container.resolve(DI_TOKEN.DATASOURCE); const userAgent = container.resolve(DI_TOKEN.USER_AGENT); const datasetUrl = container.resolve(DI_TOKEN.DATASET_URL); const progress = container.resolve( @@ -51,7 +51,7 @@ export const downloadProcess = async ({ ); const downloader = new CkanDownloader({ - db, + ds, userAgent, datasetUrl, ckanId, diff --git a/src/controller/download/process/load-dataset-history.ts b/src/controller/download/process/load-dataset-history.ts index d241f8a4..22e72498 100644 --- a/src/controller/download/process/load-dataset-history.ts +++ b/src/controller/download/process/load-dataset-history.ts @@ -22,14 +22,14 @@ * SOFTWARE. */ import { DatasetRow } from '@domain/dataset/dataset-row'; -import { Database } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; export const loadDatasetHistory = async ({ - db, + ds, }: { - db: Database; + ds: DataSource; }): Promise> => { - const rows = db.prepare('select * from dataset').all() as { + const rows = (await ds.query('select * from dataset')) as { key: string; type: string; content_length: number; diff --git a/src/controller/download/process/load-dataset-process.ts b/src/controller/download/process/load-dataset-process.ts index 30c49c64..94557c1d 100644 --- a/src/controller/download/process/load-dataset-process.ts +++ b/src/controller/download/process/load-dataset-process.ts @@ -33,19 +33,21 @@ import { TownDatasetFile } from '@domain/dataset/town-dataset-file'; import { TownPosDatasetFile } from '@domain/dataset/town-pos-dataset-file'; import { IStreamReady } from '@domain/istream-ready'; import { DI_TOKEN } from '@interface-adapter/tokens'; -import { Database } from 'better-sqlite3'; +import { DataSource, QueryRunner } from 'typeorm'; import { MultiBar } from 'cli-progress'; import csvParser from 'csv-parser'; import { Stream } from 'node:stream'; import { DependencyContainer } from 'tsyringe'; import { Logger } from 'winston'; +import { prepareSqlAndParamKeys } from '@domain/prepare-sql-and-param-keys'; +import { replaceSqlInsertValues } from '@domain/replace-sql-insert-values'; export const loadDatasetProcess = async ({ - db, + ds, container, csvFiles, }: { - db: Database; + ds: DataSource; csvFiles: IStreamReady[]; container: DependencyContainer; }) => { @@ -53,6 +55,7 @@ export const loadDatasetProcess = async ({ const multiProgressBar = container.resolve( DI_TOKEN.MULTI_PROGRESS_BAR ); + const BULK_INSERT_SIZE = 500; // _pos_ ファイルのSQL が updateになっているので、 // それ以外の基本的な情報を先に insert する必要がある。 @@ -134,20 +137,49 @@ export const loadDatasetProcess = async ({ }, }); + const processBulkInsertOrEachUpdate = async ( + ds: DataSource, + datasetFile: DatasetFile, + queryRunner: QueryRunner, + preparedSql: string, + paramsList: Array> + ) => { + if (!datasetFile.type.includes('pos')) { + const sql = replaceSqlInsertValues( + ds, + preparedSql, + paramsList[0].length, + paramsList.length + ); + await queryRunner.connection.query(sql, paramsList.flat()); + } else { + const tasks = paramsList.map(params => { + return queryRunner.connection.query(preparedSql, params); + }); + await Promise.all(tasks); + } + }; + const loadDataProgress = multiProgressBar?.create(csvFiles.length, 0, { filename: 'loading...', }); const loadDataStream = new Stream.Writable({ objectMode: true, - write(datasetFile: DatasetFile, encoding, callback) { + async write(datasetFile: DatasetFile, encoding, callback) { // 1ファイルごと transform() が呼び出される + const queryRunner = ds.createQueryRunner(); + await queryRunner.connect(); + // CSVファイルの読み込み - const statement = db.prepare(datasetFile.sql); + const { preparedSql, paramKeys } = prepareSqlAndParamKeys( + ds, + datasetFile.sql + ); - const errorHandler = (error: unknown) => { - if (db.inTransaction) { - db.exec('ROLLBACK'); + const errorHandler = async (error: unknown) => { + if (queryRunner.isTransactionActive) { + await queryRunner.rollbackTransaction(); } if (error instanceof Error) { @@ -160,7 +192,10 @@ export const loadDatasetProcess = async ({ }; // DBに登録 - db.exec('BEGIN'); + await queryRunner.startTransaction(); + + let paramsList: Array> = []; + datasetFile.csvFile.getStream().then(fileStream => { fileStream .pipe( @@ -171,49 +206,85 @@ export const loadDatasetProcess = async ({ .pipe( new Stream.Writable({ objectMode: true, - write(chunk, encoding, next) { + async write(chunk, encoding, next) { try { const processed = datasetFile.process(chunk); - statement.run(processed); + paramsList.push(paramKeys.map(key => processed[key])); + if (paramsList.length === BULK_INSERT_SIZE) { + await processBulkInsertOrEachUpdate( + ds, + datasetFile, + queryRunner, + preparedSql, + paramsList + ); + paramsList = []; + } next(null); } catch (error) { - errorHandler(error); + await errorHandler(error); } }, }) ) - .on('finish', () => { - db.exec('COMMIT'); - - db.prepare( - `INSERT OR REPLACE INTO "dataset" - ( - key, - type, - content_length, - crc32, - last_modified - ) values ( - @key, - @type, - @content_length, - @crc32, - @last_modified - )` - ).run({ + .on('finish', async () => { + if (paramsList.length > 0) { + try { + await processBulkInsertOrEachUpdate( + ds, + datasetFile, + queryRunner, + preparedSql, + paramsList + ); + paramsList = []; + } catch (error) { + await errorHandler(error); + await queryRunner.release(); + return; + } + } + await queryRunner.commitTransaction(); + const params: { [key: string]: string | number } = { key: datasetFile.filename, type: datasetFile.type, content_length: datasetFile.csvFile.contentLength, crc32: datasetFile.csvFile.crc32, last_modified: datasetFile.csvFile.lastModified, - }); + }; + const { + preparedSql: datasetPreparedSql, + paramKeys: datasetParamKeys, + } = prepareSqlAndParamKeys( + ds, + `INSERT OR REPLACE INTO "dataset" + ( + "key", + type, + content_length, + crc32, + last_modified + ) values ( + @key, + @type, + @content_length, + @crc32, + @last_modified + )` + ); + await ds.query( + datasetPreparedSql, + datasetParamKeys.map(key => params[key]) + ); + await queryRunner.release(); loadDataProgress?.increment(); loadDataProgress?.updateETA(); callback(null); }) - .on('error', (error: Error) => { - errorHandler(error); + .on('error', async (error: Error) => { + await errorHandler(error); + await queryRunner.release(); }); }); }, diff --git a/src/controller/geocode/__tests__/step3b-transform.spec.ts b/src/controller/geocode/__tests__/step3b-transform.spec.ts index 9ac118c8..c5b5d9e9 100644 --- a/src/controller/geocode/__tests__/step3b-transform.spec.ts +++ b/src/controller/geocode/__tests__/step3b-transform.spec.ts @@ -27,25 +27,29 @@ import { MatchLevel } from '@domain/match-level'; import { PrefectureName } from '@domain/prefecture-name'; import { Query } from '@domain/query'; import { describe, expect, it, jest } from '@jest/globals'; -import Database from 'better-sqlite3'; +import 'reflect-metadata'; import Stream from 'node:stream'; import { pipeline } from 'node:stream/promises'; import { GeocodingStep3B } from '../step3b-transform'; import { WritableStreamToArray } from './stream-to-array.skip'; +import { DataSource } from 'typeorm'; jest.mock('@usecase/geocode/address-finder-for-step3and5'); -jest.mock('better-sqlite3'); +// jest.mock('typeorm'); describe('step3b-transform', () => { it('複数の都道府県名にマッチする場合は、町名まで正規化して都道府県名を判別する', async () => { // 東京都府中市と にマッチする const dummyCallback = jest.fn(); - const db = new Database('dummy'); + const ds = new DataSource({ + type: 'better-sqlite3', + database: ':memory:', + }); const wildcardHelper = (address: string) => address; const finder = new AddressFinderForStep3and5({ - db, + ds, wildcardHelper, }); const target = new GeocodingStep3B(finder); diff --git a/src/controller/geocode/__tests__/step5-transform.spec.ts b/src/controller/geocode/__tests__/step5-transform.spec.ts index dd592d6c..78a92c38 100644 --- a/src/controller/geocode/__tests__/step5-transform.spec.ts +++ b/src/controller/geocode/__tests__/step5-transform.spec.ts @@ -25,25 +25,28 @@ import { AddressFinderForStep3and5 } from '@usecase/geocode/address-finder-for-s import { MatchLevel } from '@domain/match-level'; import { PrefectureName } from '@domain/prefecture-name'; import { Query } from '@domain/query'; -import { beforeAll, describe, expect, it, jest } from '@jest/globals'; -import Database from 'better-sqlite3'; +import { beforeAll, afterAll, describe, expect, it, jest } from '@jest/globals'; +import { DataSource } from 'typeorm'; import Stream, { PassThrough } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import { GeocodingStep5 } from '../step5-transform'; import { WritableStreamToArray } from './stream-to-array.skip'; jest.mock('@usecase/geocode/address-finder-for-step3and5'); -jest.mock('better-sqlite3'); +// jest.mock('typeorm'); const createWriteStream = () => { return new WritableStreamToArray(); }; const createAddressFinder = () => { - const db = new Database('dummy'); + const ds = new DataSource({ + type: 'better-sqlite3', + database: ':memory:', + }); const wildcardHelper = (address: string) => address; const finder = new AddressFinderForStep3and5({ - db, + ds, wildcardHelper, }); return finder; diff --git a/src/controller/geocode/__tests__/stream-geocoder.spec.ts b/src/controller/geocode/__tests__/stream-geocoder.spec.ts index 9e96e686..590f8035 100644 --- a/src/controller/geocode/__tests__/stream-geocoder.spec.ts +++ b/src/controller/geocode/__tests__/stream-geocoder.spec.ts @@ -28,7 +28,6 @@ jest.mock('@domain/geocode/get-prefecture-regex-patterns'); jest.mock('@domain/geocode/get-prefectures-from-db'); jest.mock('@settings/patch-patterns'); jest.mock('node:stream'); -jest.mock('better-sqlite3'); jest.mock('../step1-transform'); jest.mock('../step2-transform'); jest.mock('../step3-transform'); @@ -47,8 +46,8 @@ import { getCityPatternsForEachPrefecture as getCityP } from '@domain/geocode/ge import { getPrefectureRegexPatterns as getPreRegP } from '@domain/geocode/get-prefecture-regex-patterns'; import { getPrefecturesFromDB as getPrefs } from '@domain/geocode/get-prefectures-from-db'; import { getSameNamedPrefecturePatterns as getSamePrefs } from '@domain/geocode/get-same-named-prefecture-patterns'; -import { default as MockedBetterSqlite3 } from '@mock/better-sqlite3'; -import { default as BetterSqlite3 } from 'better-sqlite3'; +import { DataSource as MockedDataSource } from '@mock/typeorm'; +import { DataSource } from 'typeorm'; import { Readable } from 'node:stream';import * as PATCHES from '@settings/patch-patterns'; import { StreamGeocoder } from '../stream-geocoder'; @@ -113,9 +112,9 @@ mockedReadable.prototype.pipe.mockReturnValue({ const mockedPatches = PATCHES as jest.MockedObject; mockedPatches.default = []; - -const createDB = () => { - return new MockedBetterSqlite3(''); +// FIXME: TypeORM +const createDS = () => { + return new MockedDataSource(''); } describe('StreamGeocoder', () => { @@ -129,8 +128,8 @@ describe('StreamGeocoder', () => { }); it('Fuzzyを指定しない場合、同じ値を返す', async () => { - const mockedDB = createDB(); - await StreamGeocoder.create(mockedDB); + const mockedDS = createDS(); + await StreamGeocoder.create(mockedDS); expect(mockedGetPreRegP).toHaveBeenCalled(); const args = mockedGetPreRegP.mock.calls[0]; const wildcardHelper = args[0].wildcardHelper; @@ -139,8 +138,8 @@ describe('StreamGeocoder', () => { it('Fuzzyを指定した場合、正規表現を書き換える', async () => { const fuzzy = '●'; - const mockedDB = createDB(); - await StreamGeocoder.create(mockedDB, fuzzy); + const mockedDS = createDS(); + await StreamGeocoder.create(mockedDS, fuzzy); expect(mockedGetPreRegP).toHaveBeenCalled(); const args = mockedGetPreRegP.mock.calls[0]; const wildcardHelper = args[0].wildcardHelper; diff --git a/src/controller/geocode/geocode.ts b/src/controller/geocode/geocode.ts index 5fa28a0b..bde78960 100644 --- a/src/controller/geocode/geocode.ts +++ b/src/controller/geocode/geocode.ts @@ -24,7 +24,7 @@ // reflect-metadata is necessary for DI import 'reflect-metadata'; -import { Database } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; import byline from 'byline'; import fs from 'node:fs'; import path from 'node:path'; @@ -68,11 +68,11 @@ export const geocode = async ({ ckanId, }); - // データベースのインスタンスを取得 - const db: Database = container.resolve(DI_TOKEN.DATABASE); + // データソースのインスタンスを取得 + const ds: DataSource = container.resolve(DI_TOKEN.DATASOURCE); // Geocodingを行うメイン部分 - const geocoder = await StreamGeocoder.create(db, fuzzy); + const geocoder = await StreamGeocoder.create(ds, fuzzy); // Streamを1行単位にしてくれる TransformStream const lineStream = byline.createStream(); @@ -127,7 +127,7 @@ export const geocode = async ({ formatter, outputStream ); - db.close(); + await ds.destroy(); return GEOCODE_RESULT.SUCCESS; }; diff --git a/src/controller/geocode/stream-geocoder.ts b/src/controller/geocode/stream-geocoder.ts index 95c57173..5500cd35 100644 --- a/src/controller/geocode/stream-geocoder.ts +++ b/src/controller/geocode/stream-geocoder.ts @@ -35,7 +35,7 @@ import { RegExpEx } from '@domain/reg-exp-ex'; import PATCH_PATTERNS from '@settings/patch-patterns'; import { AddressFinderForStep3and5 } from '@usecase/geocode/address-finder-for-step3and5'; import { AddressFinderForStep7 } from '@usecase/geocode/address-finder-for-step7'; -import { Database } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; import { Readable, Transform, Writable } from 'node:stream'; import { TransformCallback } from 'stream'; import { GeocodingStep1 } from './step1-transform'; @@ -75,14 +75,14 @@ export class StreamGeocoder extends Transform { } static create = async ( - database: Database, + dataSource: DataSource, fuzzy?: string ): Promise => { /** * 都道府県とそれに続く都市名を取得する */ const prefectures: IPrefecture[] = await getPrefecturesFromDB({ - db: database, + ds: dataSource, }); /** @@ -167,7 +167,7 @@ export class StreamGeocoder extends Transform { // step3はデータベースを使って都道府県と市町村を特定するため、処理が複雑になる // なので、さらに別のストリームで処理を行う const addressFinderForStep3and5 = new AddressFinderForStep3and5({ - db: database, + ds: dataSource, wildcardHelper, }); @@ -301,7 +301,7 @@ export class StreamGeocoder extends Transform { // } // /* eslint-enable no-irregular-whitespace */ - const addressFinderForStep7 = new AddressFinderForStep7(database); + const addressFinderForStep7 = new AddressFinderForStep7(dataSource); const step7 = new GeocodingStep7(addressFinderForStep7); // {SPACE} と {DASH} をもとに戻す diff --git a/src/controller/update-check/update-check.ts b/src/controller/update-check/update-check.ts index e9344e77..fc677a68 100644 --- a/src/controller/update-check/update-check.ts +++ b/src/controller/update-check/update-check.ts @@ -28,7 +28,7 @@ import { AbrgMessage } from '@abrg-message/abrg-message'; import { setupContainer } from '@interface-adapter/setup-container'; import { DI_TOKEN } from '@interface-adapter/tokens'; import { CkanDownloader } from '@usecase/ckan-downloader/ckan-downloader'; -import { Database } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; import { Logger } from 'winston'; import { UPDATE_CHECK_RESULT } from './update-check-result'; @@ -46,12 +46,12 @@ export const updateCheck = async ({ }); const logger = container.resolve(DI_TOKEN.LOGGER); - const db = container.resolve(DI_TOKEN.DATABASE); + const ds = container.resolve(DI_TOKEN.DATASOURCE); const datasetUrl = container.resolve(DI_TOKEN.DATASET_URL); const userAgent = container.resolve(DI_TOKEN.USER_AGENT); const downloader = new CkanDownloader({ - db, + ds, userAgent, datasetUrl, ckanId, diff --git a/src/domain/dataset/__tests__/town-dataset-file.spec.ts b/src/domain/dataset/__tests__/town-dataset-file.spec.ts index bd436134..0c2fb75c 100644 --- a/src/domain/dataset/__tests__/town-dataset-file.spec.ts +++ b/src/domain/dataset/__tests__/town-dataset-file.spec.ts @@ -79,7 +79,6 @@ describe('TownDatasetFile', () => { DataField.OAZA_TOWN_ALT_NAME_FLG, DataField.KOAZA_FRN_LTRS_FLG, DataField.OAZA_FRN_LTRS_FLG, - DataField.KOAZA_FRN_LTRS_FLG, DataField.STATUS_FLG, DataField.WAKE_NUM_FLG, DataField.EFCT_DATE, @@ -137,7 +136,6 @@ describe('TownDatasetFile', () => { [DataField.OAZA_TOWN_ALT_NAME_FLG.csv] : '0', [DataField.KOAZA_FRN_LTRS_FLG.csv] : '0', [DataField.OAZA_FRN_LTRS_FLG.csv] : '0', - [DataField.KOAZA_FRN_LTRS_FLG.csv] : '0', [DataField.STATUS_FLG.csv] : '0', [DataField.WAKE_NUM_FLG.csv] : '0', [DataField.EFCT_DATE.csv] : '1947-04-17', diff --git a/src/domain/dataset/town-dataset-file.ts b/src/domain/dataset/town-dataset-file.ts index d75b615b..e8e7d22e 100644 --- a/src/domain/dataset/town-dataset-file.ts +++ b/src/domain/dataset/town-dataset-file.ts @@ -61,7 +61,6 @@ export class TownDatasetFile DataField.OAZA_TOWN_ALT_NAME_FLG, DataField.KOAZA_FRN_LTRS_FLG, DataField.OAZA_FRN_LTRS_FLG, - DataField.KOAZA_FRN_LTRS_FLG, DataField.STATUS_FLG, DataField.WAKE_NUM_FLG, DataField.EFCT_DATE, @@ -112,7 +111,6 @@ export class TownDatasetFile ${DataField.OAZA_TOWN_ALT_NAME_FLG.dbColumn}, ${DataField.KOAZA_FRN_LTRS_FLG.dbColumn}, ${DataField.OAZA_FRN_LTRS_FLG.dbColumn}, - ${DataField.KOAZA_FRN_LTRS_FLG.dbColumn}, ${DataField.STATUS_FLG.dbColumn}, ${DataField.WAKE_NUM_FLG.dbColumn}, ${DataField.EFCT_DATE.dbColumn}, @@ -152,7 +150,6 @@ export class TownDatasetFile @${DataField.OAZA_TOWN_ALT_NAME_FLG.dbColumn}, @${DataField.KOAZA_FRN_LTRS_FLG.dbColumn}, @${DataField.OAZA_FRN_LTRS_FLG.dbColumn}, - @${DataField.KOAZA_FRN_LTRS_FLG.dbColumn}, @${DataField.STATUS_FLG.dbColumn}, @${DataField.WAKE_NUM_FLG.dbColumn}, @${DataField.EFCT_DATE.dbColumn}, @@ -161,6 +158,8 @@ export class TownDatasetFile @${DataField.POST_CODE.dbColumn}, @${DataField.REMARKS.dbColumn} ) + -- ON CONFLICT (${DataField.LG_CODE.dbColumn}, ${DataField.TOWN_ID.dbColumn}) + -- DO NOTHING `; return new TownDatasetFile({ ...params, diff --git a/src/domain/geocode/__tests__/get-prefectures-from-db.spec.ts b/src/domain/geocode/__tests__/get-prefectures-from-db.spec.ts index 9aab5386..fe819329 100644 --- a/src/domain/geocode/__tests__/get-prefectures-from-db.spec.ts +++ b/src/domain/geocode/__tests__/get-prefectures-from-db.spec.ts @@ -25,44 +25,42 @@ import { City } from '@domain/city'; import { Prefecture } from '@domain/prefecture'; import { PrefectureName } from '@domain/prefecture-name'; import { describe, expect, it, jest } from '@jest/globals'; -import { default as BetterSqlite3, default as Database } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; import { getPrefecturesFromDB } from '../get-prefectures-from-db'; -jest.mock('better-sqlite3'); +jest.mock('typeorm'); -const MockedDB = Database as unknown as jest.Mock; +const MockedDS = DataSource as unknown as jest.Mock; -MockedDB.mockImplementation(() => { +MockedDS.mockImplementation(() => { return { - prepare: (sql: string) => { - return { - all: (params: { - prefecture?: PrefectureName; - city?: string; - town?: string; - }) => { - return [ - { - 'name': '佐賀県', - 'cities': '[{"name":"佐賀市","lg_code":"412015"},{"name":"藤津郡太良町","lg_code":"414417"}]', - }, + query: (sql: string, params: string[]) => { + return Promise.resolve([ + { + 'name': '佐賀県', + 'cities': '[{"name":"佐賀市","lg_code":"412015"},{"name":"藤津郡太良町","lg_code":"414417"}]', + }, - { - 'name': '富山県', - 'cities': '[{"name":"富山市","lg_code":"162019"},{"name":"下新川郡朝日町","lg_code":"163431"}]', - } - ]; + { + 'name': '富山県', + 'cities': '[{"name":"富山市","lg_code":"162019"},{"name":"下新川郡朝日町","lg_code":"163431"}]', } - } + ]); + }, + options: { + type: 'better-sqlite3', }, }; }); describe('getPrefecturesFromDB', () => { it('should return prefectures as Prefecture[]', async () => { - const mockedDB = new Database(''); + const mockedDS = new DataSource({ + type: 'better-sqlite3', + database: ':memory:', + }); const prefectures = await getPrefecturesFromDB({ - db: mockedDB, + ds: mockedDS, }); expect(prefectures).toEqual([ new Prefecture({ diff --git a/src/domain/geocode/get-prefectures-from-db.ts b/src/domain/geocode/get-prefectures-from-db.ts index a681b89d..019fef86 100644 --- a/src/domain/geocode/get-prefectures-from-db.ts +++ b/src/domain/geocode/get-prefectures-from-db.ts @@ -25,14 +25,15 @@ import { City, ICity } from '@domain/city'; import { DataField } from '@domain/dataset/data-field'; import { IPrefecture, Prefecture } from '@domain/prefecture'; import { PrefectureName } from '@domain/prefecture-name'; -import { Database, Statement } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; +import { prepareSqlAndParamKeys } from '@domain/prepare-sql-and-param-keys'; export const getPrefecturesFromDB = async ({ - db, + ds, }: { - db: Database; + ds: DataSource; }): Promise => { - const statement: Statement = db.prepare(` + const sql: string = ` SELECT pref_name AS "name", json_group_array(json_object( @@ -45,14 +46,18 @@ export const getPrefecturesFromDB = async ({ )) AS "cities" FROM city GROUP BY pref_name - `); + `; + const { preparedSql } = prepareSqlAndParamKeys(ds, sql); - const prefectures = statement.all() as { + const prefectures = (await ds.query(preparedSql)) as { name: string; cities: string; }[]; return prefectures.map(value => { - const townRawValues: ICity[] = JSON.parse(value.cities); + const townRawValues: ICity[] = + typeof value.cities === 'string' + ? JSON.parse(value.cities) + : value.cities; const cities = townRawValues.map(value => { return new City(value); }); diff --git a/src/domain/key-store/__tests__/get-value-with-key.spec.ts b/src/domain/key-store/__tests__/get-value-with-key.spec.ts index 44e75ad6..db455377 100644 --- a/src/domain/key-store/__tests__/get-value-with-key.spec.ts +++ b/src/domain/key-store/__tests__/get-value-with-key.spec.ts @@ -1,32 +1,33 @@ import { describe, expect, it, jest } from '@jest/globals'; -import { default as BetterSqlite3, default as Database } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; import { getValueWithKey } from '../get-value-with-key'; -jest.mock('better-sqlite3'); +// jest.mock('typeorm'); jest.mock('string-hash') -const MockedDB = Database as unknown as jest.Mock; +const MockedDS = DataSource as unknown as jest.Mock; describe("getValueWithKey()", () => { it.concurrent("should return expected value", async () => { - MockedDB.mockImplementation(() => { + MockedDS.mockImplementation(() => { return { - prepare: (sql: string) => { - return { - get: (key: number): { [key: string]: string } | undefined => { - return { - key: 'ba000001', - value: '1234', - } - }, - }; + query: (sql: string, params: string[]) => { + return Promise.resolve([{ + value: '1234', + }]); + }, + options: { + type: 'better-sqlite3', }, }; }); - const mockedDB = new Database(''); + const mockedDS = new DataSource({ + type: 'better-sqlite3', + database: ':memory:', + }); - const receivedValue = getValueWithKey({ - db: mockedDB, + const receivedValue = await getValueWithKey({ + ds: mockedDS, key: 'ba000001', }); @@ -34,21 +35,23 @@ describe("getValueWithKey()", () => { }); it.concurrent("should return ", async () => { - MockedDB.mockImplementation(() => { + MockedDS.mockImplementation(() => { return { - prepare: (sql: string) => { - return { - get: (key: number): { [key: string]: string } | undefined => { - return undefined - }, - }; + query: (sql: string, params: string[]) => { + return Promise.resolve(undefined); + }, + options: { + type: 'better-sqlite3', }, }; }); - const mockedDB = new Database(''); + const mockedDS = new DataSource({ + type: 'better-sqlite3', + database: ':memory:', + }); - const receivedValue = getValueWithKey({ - db: mockedDB, + const receivedValue = await getValueWithKey({ + ds: mockedDS, key: 'ba000001', }); diff --git a/src/domain/key-store/__tests__/save-key-and-value.spec.ts b/src/domain/key-store/__tests__/save-key-and-value.spec.ts index 0a4d3673..2ec60ec7 100644 --- a/src/domain/key-store/__tests__/save-key-and-value.spec.ts +++ b/src/domain/key-store/__tests__/save-key-and-value.spec.ts @@ -1,41 +1,45 @@ import { beforeEach, describe, expect, it, jest } from '@jest/globals'; -import { default as BetterSqlite3, default as Database } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; import {saveKeyAndValue} from '../save-key-and-value'; -jest.mock('better-sqlite3'); +// jest.mock('typeorm'); jest.mock('string-hash') -const MockedDB = Database as unknown as jest.Mock; +const MockedDS = DataSource as unknown as jest.Mock; const mockRunMethod = jest.fn(); -MockedDB.mockImplementation(() => { +MockedDS.mockImplementation(() => { return { - prepare: (sql: string) => { - return { - run: mockRunMethod, - }; + query: mockRunMethod, // (sql: string, params: string[]) => {} + options: { + type: 'better-sqlite3', }, }; }); describe("saveKeyAndValue()", () => { - const mockedDB = new Database(''); + const mockedDS = new DataSource({ + type: 'better-sqlite3', + database: ':memory:', + }); beforeEach(() => { mockRunMethod.mockClear(); - MockedDB.mockClear(); + MockedDS.mockClear(); }); - it("should save value correctly", () => { + it("should save value correctly", async () => { - saveKeyAndValue({ - db: mockedDB, + await saveKeyAndValue({ + ds: mockedDS, key: 'ba000001', value: '1234', }); expect(mockRunMethod).toBeCalled(); - const receivedValue = mockRunMethod.mock.calls[0][0]; - expect(receivedValue).toEqual({"key": 'ba000001', "value": "1234"}); + const receivedValue1 = mockRunMethod.mock.calls[0][0]; + const receivedValue2 = mockRunMethod.mock.calls[0][1]; + expect(receivedValue1).toEqual('insert or replace into metadata values(?, ?)'); + expect(receivedValue2).toEqual(['ba000001', '1234']); }); }); \ No newline at end of file diff --git a/src/domain/key-store/get-value-with-key.ts b/src/domain/key-store/get-value-with-key.ts index 13143103..4fef0681 100644 --- a/src/domain/key-store/get-value-with-key.ts +++ b/src/domain/key-store/get-value-with-key.ts @@ -21,26 +21,29 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -import { Database } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; +import { prepareSqlAndParamKeys } from '@domain/prepare-sql-and-param-keys'; -export const getValueWithKey = ({ - db, +export const getValueWithKey = async ({ + ds, key, }: { - db: Database; + ds: DataSource; key: string; -}): string | undefined => { - const result = db - .prepare('select value from metadata where key = @key limit 1') - .get({ - key, - }) as - | { - value: string; - } - | undefined; - if (!result) { +}): Promise => { + const { preparedSql, paramKeys } = prepareSqlAndParamKeys( + ds, + 'select value from metadata where "key" = @key limit 1' + ); + const params: { [key: string]: string | number } = { + key, + }; + const result = (await ds.query( + preparedSql, + paramKeys.map(key => params[key]) + )) as { value: string }[]; + if (!result || result.length === 0) { return; } - return result.value; + return result[0].value; }; diff --git a/src/domain/key-store/save-key-and-value.ts b/src/domain/key-store/save-key-and-value.ts index c13dbfd4..d21993f2 100644 --- a/src/domain/key-store/save-key-and-value.ts +++ b/src/domain/key-store/save-key-and-value.ts @@ -21,19 +21,28 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -import { Database } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; +import { prepareSqlAndParamKeys } from '@domain/prepare-sql-and-param-keys'; -export const saveKeyAndValue = ({ - db, +export const saveKeyAndValue = async ({ + ds, key, value, }: { - db: Database; + ds: DataSource; key: string; value: string; }) => { - db.prepare('insert or replace into metadata values(@key, @value)').run({ + const { preparedSql, paramKeys } = prepareSqlAndParamKeys( + ds, + 'insert or replace into metadata values(@key, @value)' + ); + const params: { [key: string]: string | number } = { key, value, - }); + }; + await ds.query( + preparedSql, + paramKeys.map(key => params[key]) + ); }; diff --git a/src/domain/prepare-sql-and-param-keys.ts b/src/domain/prepare-sql-and-param-keys.ts new file mode 100644 index 00000000..733fdf73 --- /dev/null +++ b/src/domain/prepare-sql-and-param-keys.ts @@ -0,0 +1,55 @@ +import { DataSource } from 'typeorm'; + +export const prepareSqlAndParamKeys = ( + ds: DataSource, + sql: string +): { + preparedSql: string; + paramKeys: string[]; +} => { + let tempSql = sql; + const keys = []; + const dsType = ds.options.type; + const matchedParams = tempSql.match(/@[a-zA-Z_0-9]+/g); + if (matchedParams) { + for (let i = 0; i < matchedParams.length; i++) { + const matchedParam = matchedParams[i]; + const paramName = matchedParam.replace('@', ''); + keys.push(paramName); + const placeHolder = dsType === 'postgres' ? `$${i + 1}` : '?'; + tempSql = tempSql.replace(matchedParam, placeHolder); + } + } + const matchedInsertOrReplace = tempSql.match(/^INSERT OR REPLACE INTO/i); + if (matchedInsertOrReplace) { + if (dsType === 'postgres') { + tempSql = tempSql.replace(matchedInsertOrReplace[0], 'INSERT INTO'); + tempSql = tempSql.replace(/-- /g, ''); + } else if (dsType === 'mysql') { + tempSql = tempSql.replace(matchedInsertOrReplace[0], 'REPLACE INTO'); + } + } + const matchedJSONFunctions = tempSql.match( + /(json_group_array|json_object)/gi + ); + if (matchedJSONFunctions) { + for (let i = 0; i < matchedJSONFunctions.length; i++) { + const matchedJSONFucntion = matchedJSONFunctions[i]; + if (matchedJSONFucntion.toLowerCase() === 'json_group_array') { + if (dsType === 'postgres') { + tempSql = tempSql.replace(matchedJSONFucntion, 'json_agg'); + } else if (dsType === 'mysql') { + tempSql = tempSql.replace(matchedJSONFucntion, 'json_arrayagg'); + } + } else if (matchedJSONFucntion.toLowerCase() === 'json_object') { + if (dsType === 'postgres') { + tempSql = tempSql.replace(matchedJSONFucntion, 'json_build_object'); + } + } + } + } + return { + preparedSql: tempSql, + paramKeys: keys, + }; +}; diff --git a/src/domain/replace-sql-insert-values.ts b/src/domain/replace-sql-insert-values.ts new file mode 100644 index 00000000..33664649 --- /dev/null +++ b/src/domain/replace-sql-insert-values.ts @@ -0,0 +1,35 @@ +import { DataSource } from 'typeorm'; + +export const replaceSqlInsertValues = ( + ds: DataSource, + sql: string, + paramsCount: number, + rowCount: number +): string => { + let tempSql = sql; + const dsType = ds.options.type; + const rows = []; + const pattern = + dsType === 'postgres' + ? /VALUES[\s\n]*\([\s\n$0-9,]+\)/im + : /VALUES[\s\n]*\([\s\n?,]+\)/im; + const matchedInsertValues = tempSql.match(pattern); + if (matchedInsertValues) { + for (let rowIdx = 0; rowIdx < rowCount; rowIdx++) { + const placeHolders = []; + for (let paramIdx = 0; paramIdx < paramsCount; paramIdx++) { + const placeHolder = + dsType === 'postgres' + ? `$${rowIdx * paramsCount + paramIdx + 1}` + : '?'; + placeHolders.push(placeHolder); + } + rows.push(`(${placeHolders.join(',')})`); + } + tempSql = tempSql.replace( + matchedInsertValues[0], + 'VALUES ' + rows.join(',') + ); + } + return tempSql; +}; diff --git a/src/entity/city.ts b/src/entity/city.ts new file mode 100644 index 00000000..c00f0e2b --- /dev/null +++ b/src/entity/city.ts @@ -0,0 +1,52 @@ +import { Entity, PrimaryColumn, Column } from 'typeorm'; + +@Entity() +export class City { + @PrimaryColumn('varchar', { length: '6', comment: '全国地方公共団体コード' }) + lg_code!: string; + + @Column('text', { comment: '都道府県名' }) + pref_name!: string; + + @Column('text', { comment: '都道府県名_カナ' }) + pref_name_kana!: string; + + @Column('text', { comment: '都道府県名_英字' }) + pref_name_roma!: string; + + @Column('text', { nullable: true, comment: '郡名' }) + county_name!: string; + + @Column('text', { nullable: true, comment: '郡名_カナ' }) + county_name_kana!: string; + + @Column('text', { nullable: true, comment: '郡名_英字' }) + county_name_roma!: string; + + @Column('text', { comment: '市区町村名' }) + city_name!: string; + + @Column('text', { comment: '市区町村名_カナ' }) + city_name_kana!: string; + + @Column('text', { comment: '市区町村名_英字' }) + city_name_roma!: string; + + @Column('text', { nullable: true, comment: '政令市区名' }) + od_city_name!: string; + + @Column('text', { nullable: true, comment: '政令市区名_カナ' }) + od_city_name_kana!: string; + + @Column('text', { nullable: true, comment: '政令市区名_英字' }) + od_city_name_roma!: string; + + @Column('text', { nullable: true, comment: '効力発生日' }) + efct_date!: string; + + @Column('text', { nullable: true, comment: '廃止日' }) + ablt_date!: string; + + @Column('text', { nullable: true, comment: '備考' }) + remarks!: string; +} diff --git a/src/entity/dataset.ts b/src/entity/dataset.ts new file mode 100644 index 00000000..0505a2a8 --- /dev/null +++ b/src/entity/dataset.ts @@ -0,0 +1,19 @@ +import { Entity, PrimaryColumn, Column } from 'typeorm'; + +@Entity() +export class Dataset { + @PrimaryColumn('varchar', { length: '255' }) + key!: string; + + @Column('text') + type!: string; + + @Column('bigint') + content_length!: number; + + @Column('bigint') + crc32!: number; + + @Column('bigint') + last_modified!: number; +} diff --git a/src/entity/metadata.ts b/src/entity/metadata.ts new file mode 100644 index 00000000..14ca3681 --- /dev/null +++ b/src/entity/metadata.ts @@ -0,0 +1,10 @@ +import { Entity, PrimaryColumn, Column } from 'typeorm'; + +@Entity() +export class Metadata { + @PrimaryColumn('varchar', { length: '255' }) + key!: string; + + @Column('text') + value!: string; +} diff --git a/src/entity/pref.ts b/src/entity/pref.ts new file mode 100644 index 00000000..db686fec --- /dev/null +++ b/src/entity/pref.ts @@ -0,0 +1,25 @@ +import { Entity, PrimaryColumn, Column } from 'typeorm'; + +@Entity() +export class Pref { + @PrimaryColumn('varchar', { length: '6', comment: '全国地方公共団体コード' }) + lg_code!: string; + + @Column('text', { comment: '都道府県名' }) + pref_name!: string; + + @Column('text', { comment: '都道府県名_カナ' }) + pref_name_kana!: string; + + @Column('text', { comment: '都道府県名_英字' }) + pref_name_roma!: string; + + @Column('text', { nullable: true, comment: '効力発生日' }) + efct_date!: string; + + @Column('text', { nullable: true, comment: '廃止日' }) + ablt_date!: string; + + @Column('text', { nullable: true, comment: '備考' }) + remarks!: string; +} diff --git a/src/entity/rsdtdsp-blk.ts b/src/entity/rsdtdsp-blk.ts new file mode 100644 index 00000000..afdf2a2f --- /dev/null +++ b/src/entity/rsdtdsp-blk.ts @@ -0,0 +1,65 @@ +import { Entity, PrimaryColumn, Column } from 'typeorm'; + +@Entity() +export class RsdtdspBlk { + @PrimaryColumn('varchar', { length: '6', comment: '全国地方公共団体コード' }) + lg_code!: string; + + @PrimaryColumn('varchar', { length: '7', comment: '町字ID' }) + town_id!: string; + + @PrimaryColumn('varchar', { length: '3', comment: '街区ID' }) + blk_id!: string; + + @Column('text', { comment: '市区町村名' }) + city_name!: string; + + @Column('text', { nullable: true, comment: '政令市区名' }) + od_city_name!: string; + + @Column('text', { comment: '大字・町名' }) + oaza_town_name!: string; + + @Column('text', { nullable: true, comment: '丁目名' }) + chome_name!: string; + + @Column('text', { nullable: true, comment: '小字名' }) + koaza_name!: string; + + @Column('text', { nullable: true, comment: '街区符号' }) + blk_num!: string; + + @Column('smallint', { comment: '住居表示フラグ' }) + rsdt_addr_flg!: number; + + @Column('smallint', { comment: '住居表示方式コード' }) + rsdt_addr_mtd_code!: number; + + @Column('text', { nullable: true, comment: '大字・町_外字フラグ' }) + oaza_frn_ltrs_flg!: string; + + @Column('text', { nullable: true, comment: '小字_外字フラグ' }) + koaza_frn_ltrs_flg!: string; + + @Column('text', { nullable: true, comment: '状態フラグ' }) + status_flg!: string; + + @Column('text', { nullable: true, comment: '効力発生日' }) + efct_date!: string; + + @Column('text', { nullable: true, comment: '廃止日' }) + ablt_date!: string; + + @Column('smallint', { nullable: true, comment: '原典資料コード' }) + src_code!: number; + + @Column('text', { nullable: true, comment: '備考' }) + remarks!: string; + + // 住居表示-街区位置参照(mt_rsdtdsp_blk_pos_prefXX)から結合 + @Column('double precision', { nullable: true, comment: '代表点_経度' }) + rep_pnt_lon!: number; + + @Column('double precision', { nullable: true, comment: '代表点_緯度' }) + rep_pnt_lat!: number; +} diff --git a/src/entity/rsdtdsp-rsdt.ts b/src/entity/rsdtdsp-rsdt.ts new file mode 100644 index 00000000..f7287bc1 --- /dev/null +++ b/src/entity/rsdtdsp-rsdt.ts @@ -0,0 +1,84 @@ +import { Entity, PrimaryColumn, Column } from 'typeorm'; + +@Entity() +export class RsdtdspRsdt { + @PrimaryColumn('varchar', { length: '6', comment: '全国地方公共団体コード' }) + lg_code!: string; + + @PrimaryColumn('varchar', { length: '7', comment: '町字ID' }) + town_id!: string; + + @PrimaryColumn('varchar', { length: '3', comment: '街区ID' }) + blk_id!: string; + + @PrimaryColumn('varchar', { length: '3', comment: '住居ID' }) + addr_id!: string; + + @PrimaryColumn('varchar', { + length: '5', + // nullable: true, // インデックスに含まれるため、nullableにできない + comment: '住居2ID', + }) + addr2_id!: string; + + @Column('text', { comment: '市区町村名' }) + city_name!: string; + + @Column('text', { nullable: true, comment: '政令市区名' }) + od_city_name!: string; + + @Column('text', { comment: '大字・町名' }) + oaza_town_name!: string; + + @Column('text', { nullable: true, comment: '丁目名' }) + chome_name!: string; + + @Column('text', { nullable: true, comment: '小字名' }) + koaza_name!: string; + + @Column('text', { nullable: true, comment: '街区符号' }) + blk_num!: string; + + @Column('text', { comment: '住居番号' }) + rsdt_num!: string; + + @Column('text', { nullable: true, comment: '住居番号2' }) + rsdt_num2!: string; + + @Column('text', { nullable: true, comment: '基礎番号・住居番号区分' }) + basic_rsdt_div!: number; + + @Column('smallint', { comment: '住居表示フラグ' }) + rsdt_addr_flg!: number; + + @Column('smallint', { comment: '住居表示方式コード' }) + rsdt_addr_mtd_code!: number; + + @Column('text', { nullable: true, comment: '大字・町_外字フラグ' }) + oaza_frn_ltrs_flg!: string; + + @Column('text', { nullable: true, comment: '小字_外字フラグ' }) + koaza_frn_ltrs_flg!: string; + + @Column('text', { nullable: true, comment: '状態フラグ' }) + status_flg!: string; + + @Column('text', { nullable: true, comment: '効力発生日' }) + efct_date!: string; + + @Column('text', { nullable: true, comment: '廃止日' }) + ablt_date!: string; + + @Column('smallint', { nullable: true, comment: '原典資料コード' }) + src_code!: number; + + @Column('text', { nullable: true, comment: '備考' }) + remarks!: string; + + // 住居表示-住居位置参照(mt_rsdtdsp_rsdt_pos_prefXX)から結合 + @Column('double precision', { nullable: true, comment: '代表点_経度' }) + rep_pnt_lon!: number; + + @Column('double precision', { nullable: true, comment: '代表点_緯度' }) + rep_pnt_lat!: number; +} diff --git a/src/entity/town.ts b/src/entity/town.ts new file mode 100644 index 00000000..8a6ed962 --- /dev/null +++ b/src/entity/town.ts @@ -0,0 +1,122 @@ +import { Entity, PrimaryColumn, Column } from 'typeorm'; + +@Entity() +export class Town { + @PrimaryColumn('varchar', { length: '6', comment: '全国地方公共団体コード' }) + lg_code!: string; + + @PrimaryColumn('varchar', { length: '7', comment: '町字ID' }) + town_id!: string; + + @Column('smallint', { comment: '町字区分コード' }) + town_code!: number; + + @Column('text', { comment: '都道府県名' }) + pref_name!: string; + + @Column('text', { comment: '都道府県名_カナ' }) + pref_name_kana!: string; + + @Column('text', { comment: '都道府県名_英字' }) + pref_name_roma!: string; + + @Column('text', { nullable: true, comment: '郡名' }) + county_name!: string; + + @Column('text', { nullable: true, comment: '郡名_カナ' }) + county_name_kana!: string; + + @Column('text', { nullable: true, comment: '郡名_英字' }) + county_name_roma!: string; + + @Column('text', { comment: '市区町村名' }) + city_name!: string; + + @Column('text', { comment: '市区町村名_カナ' }) + city_name_kana!: string; + + @Column('text', { comment: '市区町村名_英字' }) + city_name_roma!: string; + + @Column('text', { nullable: true, comment: '政令市区名' }) + od_city_name!: string; + + @Column('text', { nullable: true, comment: '政令市区名_カナ' }) + od_city_name_kana!: string; + + @Column('text', { nullable: true, comment: '政令市区名_英字' }) + od_city_name_roma!: string; + + @Column('text', { nullable: true, comment: '大字・町名' }) + oaza_town_name!: string; + + @Column('text', { nullable: true, comment: '大字・町名_カナ' }) + oaza_town_name_kana!: string; + + @Column('text', { nullable: true, comment: '大字・町名_英字' }) + oaza_town_name_roma!: string; + + @Column('text', { nullable: true, comment: '丁目名' }) + chome_name!: string; + + @Column('text', { nullable: true, comment: '丁目名_カナ' }) + chome_name_kana!: string; + + @Column('text', { nullable: true, comment: '丁目名_数字' }) + chome_name_number!: string; + + @Column('text', { nullable: true, comment: '小字名' }) + koaza_name!: string; + + @Column('text', { nullable: true, comment: '小字名_カナ' }) + koaza_name_kana!: string; + + @Column('text', { nullable: true, comment: '小字名_英字' }) + koaza_name_roma!: string; + + @Column('smallint', { comment: '住居表示フラグ' }) + rsdt_addr_flg!: number; + + @Column('smallint', { nullable: true, comment: '住居表示方式コード' }) + rsdt_addr_mtd_code!: number; + + @Column('smallint', { nullable: true, comment: '大字・町_通称フラグ' }) + oaza_town_alt_name_flg!: number; + + @Column('smallint', { nullable: true, comment: '小字_通称フラグ' }) + koaza_alt_name_flg!: number; + + @Column('text', { nullable: true, comment: '大字・町_外字フラグ' }) + oaza_frn_ltrs_flg!: string; + + @Column('text', { nullable: true, comment: '小字_外字フラグ' }) + koaza_frn_ltrs_flg!: string; + + @Column('text', { nullable: true, comment: '状態フラグ' }) + status_flg!: string; + + @Column('smallint', { nullable: true, comment: '起番フラグ' }) + wake_num_flg!: number; + + @Column('text', { nullable: true, comment: '効力発生日' }) + efct_date!: string; + + @Column('text', { nullable: true, comment: '廃止日' }) + ablt_date!: string; + + @Column('smallint', { nullable: true, comment: '原典資料コード' }) + src_code!: number; + + @Column('text', { nullable: true, comment: '郵便番号' }) + post_code!: string; + + @Column('text', { nullable: true, comment: '備考' }) + remarks!: string; + + // 町字位置参照(mt_town_pos_prefXX)から結合 + @Column('double precision', { nullable: true, comment: '代表点_経度' }) + rep_pnt_lon!: number; + + @Column('double precision', { nullable: true, comment: '代表点_緯度' }) + rep_pnt_lat!: number; +} diff --git a/src/interface-adapter/__mocks__/setup-container.ts b/src/interface-adapter/__mocks__/setup-container.ts index c5f38a15..0d52cc0a 100644 --- a/src/interface-adapter/__mocks__/setup-container.ts +++ b/src/interface-adapter/__mocks__/setup-container.ts @@ -24,14 +24,11 @@ import { jest } from '@jest/globals'; import { DI_TOKEN } from '../tokens'; import { PassThrough } from 'node:stream'; -import { default as Database } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; // __mocks__/winston jest.mock('winston'); -// __mocks__/better-sqlite3 -jest.mock('better-sqlite3'); - export const setupContainer = jest.fn().mockImplementation(() => { return { resolve: (target: DI_TOKEN) => { @@ -39,8 +36,11 @@ export const setupContainer = jest.fn().mockImplementation(() => { case DI_TOKEN.LOGGER: return undefined; - case DI_TOKEN.DATABASE: - return new Database('dummy'); + case DI_TOKEN.DATASOURCE: + return new DataSource({ + type: 'better-sqlite3', + database: ':memory:', + }); case DI_TOKEN.DATASET_URL: return 'dataset_url'; diff --git a/src/interface-adapter/providers/provide-database.ts b/src/interface-adapter/providers/provide-data-source.ts similarity index 63% rename from src/interface-adapter/providers/provide-database.ts rename to src/interface-adapter/providers/provide-data-source.ts index 3b536dc8..b5a0df5e 100644 --- a/src/interface-adapter/providers/provide-database.ts +++ b/src/interface-adapter/providers/provide-data-source.ts @@ -21,25 +21,27 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -import type BetterSqlite3 from 'better-sqlite3'; -import Database from 'better-sqlite3'; -import fs from 'node:fs'; +import { DataSource, DataSourceOptions } from 'typeorm'; +import { AbrgDataSource, commonOptions } from '../../abrg-data-source'; -export const provideDatabase = async ({ +export const provideDataSource = async ({ sqliteFilePath, - schemaFilePath, }: { sqliteFilePath: string; - schemaFilePath: string; -}): Promise => { - const schemaSQL = await fs.promises.readFile(schemaFilePath, 'utf8'); - const db = new Database(sqliteFilePath); - - // We use these dangerous settings to improve performance, because if data is corrupted, - // we can always just regenerate the database. - // ref: https://qastack.jp/programming/1711631/improve-insert-per-second-performance-of-sqlite - db.pragma('journal_mode = MEMORY'); - db.pragma('synchronous = OFF'); - db.exec(schemaSQL); - return db; +}): Promise => { + if (AbrgDataSource.options.type === 'better-sqlite3') { + const options: DataSourceOptions = { + ...commonOptions, + type: 'better-sqlite3', + database: sqliteFilePath, + statementCacheSize: 150, + prepareDatabase: db => { + db.pragma('journal_mode = MEMORY'); + db.pragma('synchronous = OFF'); + }, + }; + return await new DataSource(options).initialize(); + } else { + return await AbrgDataSource.initialize(); + } }; diff --git a/src/interface-adapter/setup-container.ts b/src/interface-adapter/setup-container.ts index b67df532..6794e63f 100644 --- a/src/interface-adapter/setup-container.ts +++ b/src/interface-adapter/setup-container.ts @@ -37,7 +37,7 @@ import { GeoJsonTransform } from './formatters/geo-json-transform'; import { JsonTransform } from './formatters/json-transform'; import { NdGeoJsonTransform } from './formatters/nd-geo-json-transform'; import { NdJsonTransform } from './formatters/nd-json-transform'; -import { provideDatabase } from './providers/provide-database'; +import { provideDataSource } from './providers/provide-data-source'; import { provideInifinityProgressBar } from './providers/provide-inifinity-progress-bar'; import { provideLogger } from './providers/provide-logger'; import { provideMultiProgressBar } from './providers/provide-multi-progress-bar'; @@ -80,13 +80,12 @@ export const setupContainer = async ({ }); } - // アプリケーション全体を通して使用するデータベース - const db = await provideDatabase({ - schemaFilePath, + // アプリケーション全体を通して使用するデータソース + const ds = await provideDataSource({ sqliteFilePath, }); - myContainer.register(DI_TOKEN.DATABASE, { - useValue: db, + myContainer.register(DI_TOKEN.DATASOURCE, { + useValue: ds, }); // ロガー diff --git a/src/interface-adapter/tokens.ts b/src/interface-adapter/tokens.ts index 745e098d..f0136968 100644 --- a/src/interface-adapter/tokens.ts +++ b/src/interface-adapter/tokens.ts @@ -24,7 +24,7 @@ export enum DI_TOKEN { USER_AGENT = 'user_agent', DATASET_URL = 'dataset_url', - DATABASE = 'database', + DATASOURCE = 'datasource', LOGGER = 'logger', INFINITY_PROGRESS_BAR = 'infinity_progress_bar', PROGRESS_BAR = 'progress_bar', diff --git a/src/migration/1705589170557-InitialSchema.ts b/src/migration/1705589170557-InitialSchema.ts new file mode 100644 index 00000000..7e95acb9 --- /dev/null +++ b/src/migration/1705589170557-InitialSchema.ts @@ -0,0 +1,251 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class InitialSchema1705589170557 implements MigrationInterface { + name = 'InitialSchema1705589170557'; + + public async up(queryRunner: QueryRunner): Promise { + let needsOldDataMigration = false; + if (queryRunner.connection.options.type === 'better-sqlite3') { + const hasOldMetadataIdx = + ( + await queryRunner.query( + "SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = 'metadata_key'" + ) + )?.length > 0; + if (hasOldMetadataIdx) { + needsOldDataMigration = true; + } + } + if (needsOldDataMigration) { + // TypeORM対応以前からのマイグレーションの場合は、元テーブルをリネームしておく + await queryRunner.query('ALTER TABLE "pref" RENAME TO "pref_old"'); + await queryRunner.query('ALTER TABLE "city" RENAME TO "city_old"'); + await queryRunner.query('ALTER TABLE "town" RENAME TO "town_old"'); + await queryRunner.query( + 'ALTER TABLE "rsdtdsp_blk" RENAME TO "rsdtdsp_blk_old"' + ); + await queryRunner.query( + 'ALTER TABLE "rsdtdsp_rsdt" RENAME TO "rsdtdsp_rsdt_old"' + ); + await queryRunner.query( + 'ALTER TABLE "metadata" RENAME TO "metadata_old"' + ); + await queryRunner.query('ALTER TABLE "dataset" RENAME TO "dataset_old"'); + } + + await queryRunner.query(` + CREATE TABLE "pref" ( + "lg_code" varchar(6) PRIMARY KEY NOT NULL, + "pref_name" text NOT NULL, + "pref_name_kana" text NOT NULL, + "pref_name_roma" text NOT NULL, + "efct_date" text, + "ablt_date" text, + "remarks" text + ) + `); + + await queryRunner.query(` + CREATE TABLE "city" ( + "lg_code" varchar(6) PRIMARY KEY NOT NULL, + "pref_name" text NOT NULL, + "pref_name_kana" text NOT NULL, + "pref_name_roma" text NOT NULL, + "county_name" text, + "county_name_kana" text, + "county_name_roma" text, + "city_name" text NOT NULL, + "city_name_kana" text NOT NULL, + "city_name_roma" text NOT NULL, + "od_city_name" text, + "od_city_name_kana" text, + "od_city_name_roma" text, + "efct_date" text, + "ablt_date" text, + "remarks" text + ) + `); + + await queryRunner.query(` + CREATE TABLE "town" ( + "lg_code" varchar(6) NOT NULL, + "town_id" varchar(7) NOT NULL, + "town_code" smallint NOT NULL, + "pref_name" text NOT NULL, + "pref_name_kana" text NOT NULL, + "pref_name_roma" text NOT NULL, + "county_name" text, + "county_name_kana" text, + "county_name_roma" text, + "city_name" text NOT NULL, + "city_name_kana" text NOT NULL, + "city_name_roma" text NOT NULL, + "od_city_name" text, + "od_city_name_kana" text, + "od_city_name_roma" text, + "oaza_town_name" text, + "oaza_town_name_kana" text, + "oaza_town_name_roma" text, + "chome_name" text, + "chome_name_kana" text, + "chome_name_number" text, + "koaza_name" text, + "koaza_name_kana" text, + "koaza_name_roma" text, + "rsdt_addr_flg" smallint NOT NULL, + "rsdt_addr_mtd_code" smallint, + "oaza_town_alt_name_flg" smallint, + "koaza_alt_name_flg" smallint, + "oaza_frn_ltrs_flg" text, + "koaza_frn_ltrs_flg" text, + "status_flg" text, + "wake_num_flg" smallint, + "efct_date" text, + "ablt_date" text, + "src_code" smallint, + "post_code" text, + "remarks" text, + "rep_pnt_lon" double precision, + "rep_pnt_lat" double precision, + PRIMARY KEY ("lg_code", "town_id") + ) + `); + + await queryRunner.query(` + CREATE TABLE "rsdtdsp_blk" ( + "lg_code" varchar(6) NOT NULL, + "town_id" varchar(7) NOT NULL, + "blk_id" varchar(3) NOT NULL, + "city_name" text NOT NULL, + "od_city_name" text, + "oaza_town_name" text NOT NULL, + "chome_name" text, + "koaza_name" text, + "blk_num" text, + "rsdt_addr_flg" smallint NOT NULL, + "rsdt_addr_mtd_code" smallint NOT NULL, + "oaza_frn_ltrs_flg" text, + "koaza_frn_ltrs_flg" text, + "status_flg" text, + "efct_date" text, + "ablt_date" text, + "src_code" smallint, + "remarks" text, + "rep_pnt_lon" double precision, + "rep_pnt_lat" double precision, + PRIMARY KEY ("lg_code", "town_id", "blk_id") + ) + `); + + await queryRunner.query(` + CREATE TABLE "rsdtdsp_rsdt" ( + "lg_code" varchar(6) NOT NULL, + "town_id" varchar(7) NOT NULL, + "blk_id" varchar(3) NOT NULL, + "addr_id" varchar(3) NOT NULL, + "addr2_id" varchar(5) NOT NULL, + "city_name" text NOT NULL, + "od_city_name" text, + "oaza_town_name" text NOT NULL, + "chome_name" text, + "koaza_name" text, + "blk_num" text, + "rsdt_num" text NOT NULL, + "rsdt_num2" text, + "basic_rsdt_div" text, + "rsdt_addr_flg" smallint NOT NULL, + "rsdt_addr_mtd_code" smallint NOT NULL, + "oaza_frn_ltrs_flg" text, + "koaza_frn_ltrs_flg" text, + "status_flg" text, + "efct_date" text, + "ablt_date" text, + "src_code" smallint, + "remarks" text, + "rep_pnt_lon" double precision, + "rep_pnt_lat" double precision, + PRIMARY KEY ( + "lg_code", + "town_id", + "blk_id", + "addr_id", + "addr2_id" + ) + ) + `); + + await queryRunner.query(` + CREATE TABLE "metadata" ( + "key" varchar(255) PRIMARY KEY NOT NULL, + "value" text NOT NULL + ) + `); + + await queryRunner.query(` + CREATE TABLE "dataset" ( + "key" varchar(255) PRIMARY KEY NOT NULL, + "type" text NOT NULL, + "content_length" bigint NOT NULL, + "crc32" bigint NOT NULL, + "last_modified" bigint NOT NULL + ) + `); + + if (needsOldDataMigration) { + // 元テーブルからデータをコピー後、元テーブル・インデックスを削除 + await queryRunner.query('INSERT INTO "pref" SELECT * FROM "pref_old"'); + await queryRunner.query('INSERT INTO "city" SELECT * FROM "city_old"'); + await queryRunner.query('INSERT INTO "town" SELECT * FROM "town_old"'); + await queryRunner.query( + 'INSERT INTO "rsdtdsp_blk" SELECT * FROM "rsdtdsp_blk_old"' + ); + await queryRunner.query( + 'INSERT INTO "rsdtdsp_rsdt" SELECT * FROM "rsdtdsp_rsdt_old"' + ); + await queryRunner.query( + 'INSERT INTO "metadata" SELECT * FROM "metadata_old"' + ); + await queryRunner.query( + 'INSERT INTO "dataset" SELECT * FROM "dataset_old"' + ); + await queryRunner.query('DROP INDEX IF EXISTS "dataset_key"'); + await queryRunner.query('DROP TABLE IF EXISTS "dataset_old"'); + await queryRunner.query('DROP INDEX IF EXISTS "metadata_key"'); + await queryRunner.query('DROP TABLE IF EXISTS "metadata_old"'); + await queryRunner.query('DROP INDEX IF EXISTS "rsdtdsp_rsdt_code"'); + await queryRunner.query('DROP TABLE IF EXISTS "rsdtdsp_rsdt_old"'); + await queryRunner.query('DROP INDEX IF EXISTS "rsdtdsp_blk_code"'); + await queryRunner.query('DROP TABLE IF EXISTS "rsdtdsp_blk_old"'); + await queryRunner.query('DROP INDEX IF EXISTS "town_code"'); + await queryRunner.query('DROP TABLE IF EXISTS "town_old"'); + await queryRunner.query('DROP INDEX IF EXISTS "city_code"'); + await queryRunner.query('DROP TABLE IF EXISTS "city_old"'); + await queryRunner.query('DROP INDEX IF EXISTS "pref_code"'); + await queryRunner.query('DROP TABLE IF EXISTS "pref_old"'); + } + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + DROP TABLE "dataset" + `); + await queryRunner.query(` + DROP TABLE "metadata" + `); + await queryRunner.query(` + DROP TABLE "rsdtdsp_rsdt" + `); + await queryRunner.query(` + DROP TABLE "rsdtdsp_blk" + `); + await queryRunner.query(` + DROP TABLE "city" + `); + await queryRunner.query(` + DROP TABLE "town" + `); + await queryRunner.query(` + DROP TABLE "pref" + `); + } +} diff --git a/src/usecase/ckan-downloader/__tests__/ckan-downloader.spec.ts b/src/usecase/ckan-downloader/__tests__/ckan-downloader.spec.ts index 30d6cb1b..d5da41a5 100644 --- a/src/usecase/ckan-downloader/__tests__/ckan-downloader.spec.ts +++ b/src/usecase/ckan-downloader/__tests__/ckan-downloader.spec.ts @@ -22,9 +22,8 @@ * SOFTWARE. */ import { beforeEach, describe, expect, it, jest } from '@jest/globals'; -import { default as BetterSqlite3, default as Database } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; -jest.mock('better-sqlite3'); jest.mock('@domain/http/head-request'); import { CkanDownloader } from '../ckan-downloader'; // adjust this import according to your project structure @@ -36,7 +35,10 @@ describe('CkanDownloader', () => { ckanDownloader = new CkanDownloader({ userAgent: 'testUserAgent', datasetUrl: 'testDatasetUrl', - db: new Database('dummy'), // mock this according to your Database implementation + ds: new DataSource({ + type: 'better-sqlite3', + database: ':memory:', + }), // mock this according to your DataSource implementation ckanId: 'testCkanId', dstDir: 'testDstDir', }); diff --git a/src/usecase/ckan-downloader/ckan-downloader.ts b/src/usecase/ckan-downloader/ckan-downloader.ts index 24e09329..209db9c5 100644 --- a/src/usecase/ckan-downloader/ckan-downloader.ts +++ b/src/usecase/ckan-downloader/ckan-downloader.ts @@ -30,7 +30,7 @@ import { getRequest } from '@domain/http/get-request'; import { headRequest } from '@domain/http/head-request'; import { getValueWithKey } from '@domain/key-store/get-value-with-key'; import { saveKeyAndValue } from '@domain/key-store/save-key-and-value'; -import { Database } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; import { StatusCodes } from 'http-status-codes'; import EventEmitter from 'node:events'; import fs from 'node:fs'; @@ -42,7 +42,7 @@ import { verifyPartialDownloadedFile } from './verify-partial-downloaded-file'; export interface CkanDownloaderParams { userAgent: string; datasetUrl: string; - db: Database; + ds: DataSource; ckanId: string; dstDir: string; } @@ -56,7 +56,7 @@ export enum CkanDownloaderEvent { export class CkanDownloader extends EventEmitter { private readonly userAgent: string; private readonly datasetUrl: string; - private readonly db: Database; + private readonly ds: DataSource; private readonly ckanId: string; private readonly dstDir: string; private cacheMetadata: DatasetMetadata | null = null; @@ -64,14 +64,14 @@ export class CkanDownloader extends EventEmitter { constructor({ userAgent, datasetUrl, - db, + ds, ckanId, dstDir, }: CkanDownloaderParams) { super(); this.userAgent = userAgent; this.datasetUrl = datasetUrl; - this.db = db; + this.ds = ds; this.ckanId = ckanId; this.dstDir = dstDir; } @@ -118,9 +118,9 @@ export class CkanDownloader extends EventEmitter { }); } - const csvMeta = (() => { - const csvMetaStr = getValueWithKey({ - db: this.db, + const csvMeta = await (async () => { + const csvMetaStr = await getValueWithKey({ + ds: this.ds, key: this.ckanId, }); if (!csvMetaStr) { @@ -167,8 +167,8 @@ export class CkanDownloader extends EventEmitter { } async updateCheck(): Promise { - const csvMetaStr = getValueWithKey({ - db: this.db, + const csvMetaStr = await getValueWithKey({ + ds: this.ds, key: this.ckanId, }); if (!csvMetaStr) { @@ -255,7 +255,7 @@ export class CkanDownloader extends EventEmitter { lastModified: headers['last-modified'] as string, }); saveKeyAndValue({ - db: this.db, + ds: this.ds, key: this.ckanId, value: newCsvMeta.toString(), }); @@ -281,7 +281,7 @@ export class CkanDownloader extends EventEmitter { }); saveKeyAndValue({ - db: this.db, + ds: this.ds, key: this.ckanId, value: newCsvMeta.toString(), }); diff --git a/src/usecase/geocode/__tests__/address-finder-for-step3and5.spec.ts b/src/usecase/geocode/__tests__/address-finder-for-step3and5.spec.ts index 92e977be..be4ed8ea 100644 --- a/src/usecase/geocode/__tests__/address-finder-for-step3and5.spec.ts +++ b/src/usecase/geocode/__tests__/address-finder-for-step3and5.spec.ts @@ -24,12 +24,12 @@ import { PrefectureName } from '@domain/prefecture-name'; import { beforeEach, describe, expect, it, jest } from '@jest/globals'; import { DASH } from '@settings/constant-values'; -import { default as BetterSqlite3, default as Database } from 'better-sqlite3'; import { AddressFinderForStep3and5, TownRow } from '../address-finder-for-step3and5'; +import { DataSource } from 'typeorm'; -jest.mock('better-sqlite3'); +jest.mock('typeorm'); -const MockedDB = Database as unknown as jest.Mock; +const MockedDS = DataSource as unknown as jest.Mock; const tokyoTowns: TownRow[] = [ { lg_code: '132063', @@ -132,26 +132,25 @@ const kyotoTowns: TownRow[] = [ }, ]; -MockedDB.mockImplementation(() => { +MockedDS.mockImplementation(() => { return { - prepare: (sql: string) => { - return { - all: (params: { prefecture: PrefectureName; city: string }) => { - switch (params.prefecture) { - case PrefectureName.TOKYO: - return tokyoTowns; - - case PrefectureName.KYOTO: - return kyotoTowns; - - case PrefectureName.SHIZUOKA: - return []; - - default: - throw new Error(`Unexpected prefecture : ${params.prefecture}`); - } - }, - }; + query: (sql: string, params: string[]) => { + switch (params[0]) { + case PrefectureName.TOKYO: + return Promise.resolve(tokyoTowns); + + case PrefectureName.KYOTO: + return Promise.resolve(kyotoTowns); + + case PrefectureName.SHIZUOKA: + return Promise.resolve([]); + + default: + throw new Error(`Unexpected prefecture : ${params[0]}`); + } + }, + options: { + type: 'better-sqlite3', }, }; }); @@ -160,15 +159,18 @@ const wildcardHelper = (address: string) => { return address; }; describe('AddressFinderForStep3and5', () => { - const mockedDB = new Database(''); + const mockedDS = new DataSource({ + type: 'better-sqlite3', + database: ':memory:', + }); const instance = new AddressFinderForStep3and5({ - db: mockedDB, + ds: mockedDS, wildcardHelper, }); beforeEach(() => { - MockedDB.mockClear(); + MockedDS.mockClear(); }); it.concurrent('特定できるはずのケース', async () => { diff --git a/src/usecase/geocode/__tests__/address-finder-for-step7.spec.ts b/src/usecase/geocode/__tests__/address-finder-for-step7.spec.ts index c92507e6..968c0f99 100644 --- a/src/usecase/geocode/__tests__/address-finder-for-step7.spec.ts +++ b/src/usecase/geocode/__tests__/address-finder-for-step7.spec.ts @@ -26,77 +26,75 @@ import { PrefectureName } from '@domain/prefecture-name'; import { Query } from '@domain/query'; import { describe, expect, it, jest } from '@jest/globals'; import { DASH, SINGLE_DASH_ALTERNATIVE, SPACE } from '@settings/constant-values'; -import { default as BetterSqlite3, default as Database } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; import dummyBlockList from './dummyBlockList.json'; import dummyRsdtList2 from './dummyRsdtList2.json'; import dummySmallBlockListIwate from './dummySmallBlockListIwate.json'; import dummySmallBlockListMiyagi from './dummySmallBlockListMiyagi.json'; import { AddressFinderForStep7 } from '../address-finder-for-step7'; -jest.mock('better-sqlite3'); +jest.mock('typeorm'); -const MockedDB = Database as unknown as jest.Mock; +const MockedDS = DataSource as unknown as jest.Mock; -MockedDB.mockImplementation(() => { +MockedDS.mockImplementation(() => { return { - prepare: (sql: string) => { - return { - all: (params: { - prefecture?: PrefectureName; - city?: string; - town?: string; - }) => { - // statementに合わせてデータを返す - if (sql.includes('/* unit test: getBlockListStatement */')) { - switch(params.prefecture) { - case PrefectureName.TOKYO: - return dummyBlockList; - - default: - return []; - } - } - if (sql.includes('/* unit test: getRsdtListStatement2 */')) { - return dummyRsdtList2; - } - if (sql.includes('/* unit test: getSmallBlockListStatement */')) { + query: (sql: string, params: string[]) => { + // statementに合わせてデータを返す + if (sql.includes('/* unit test: getBlockListSql */')) { + switch(params[1]) { + case PrefectureName.TOKYO: + return Promise.resolve(dummyBlockList); + + default: + return Promise.resolve([]); + } + } + if (sql.includes('/* unit test: getRsdtList2Sql */')) { + return Promise.resolve(dummyRsdtList2); + } + if (sql.includes('/* unit test: getSmallBlockListSql */')) { - switch (params.prefecture) { - case PrefectureName.MIYAGI: - return dummySmallBlockListMiyagi; + switch (params[1]) { + case PrefectureName.MIYAGI: + return Promise.resolve(dummySmallBlockListMiyagi); - case PrefectureName.IWATE: - return dummySmallBlockListIwate; - - case PrefectureName.FUKUSHIMA: - return [ - { - "lg_code": "072044", - "town_id": "0113116", - "pref": "福島県", - "city": "いわき市", - "town": "山玉町", - "koaza_name": "脇川", - "lat": 36.901176, - "lon": 140.725118 - } - ]; + case PrefectureName.IWATE: + return Promise.resolve(dummySmallBlockListIwate); + + case PrefectureName.FUKUSHIMA: + return Promise.resolve([ + { + "lg_code": "072044", + "town_id": "0113116", + "pref": "福島県", + "city": "いわき市", + "town": "山玉町", + "koaza_name": "脇川", + "lat": 36.901176, + "lon": 140.725118 + } + ]); - default: - return []; - } - } - throw new Error('Unexpected sql was given'); + default: + return Promise.resolve([]); } } + throw new Error('Unexpected sql was given'); }, - }; + options: { + type: 'better-sqlite3' + } + } }); // TODO: カバレッジ100%になるテストケースを考える describe('AddressFinderForStep7', () => { - const mockedDB = new Database(''); - const addressFinder = new AddressFinderForStep7(mockedDB); + const mockedDS = new DataSource({ + type: 'better-sqlite3', + database: ':memory:', + }); + const addressFinder = new AddressFinderForStep7(mockedDS); describe('find', () => { it.concurrent('住居表示の街区までマッチするケース1', async () => { diff --git a/src/usecase/geocode/address-finder-for-step3and5.ts b/src/usecase/geocode/address-finder-for-step3and5.ts index 0d90b7b7..98294926 100644 --- a/src/usecase/geocode/address-finder-for-step3and5.ts +++ b/src/usecase/geocode/address-finder-for-step3and5.ts @@ -35,7 +35,8 @@ import { J_DASH, KANJI_1to10_SYMBOLS, } from '@settings/constant-values'; -import { Database, Statement } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; +import { prepareSqlAndParamKeys } from '@domain/prepare-sql-and-param-keys'; export type TownRow = { lg_code: string; @@ -63,20 +64,25 @@ export type FindParameters = { * 実質的にジオコーディングしている部分 */ export class AddressFinderForStep3and5 { - private readonly getTownStatement: Statement; + private readonly ds: DataSource; + private readonly getTownSql: string; + private readonly getTownParamKeys: string[]; private readonly wildcardHelper: (address: string) => string; constructor({ - db, + ds, wildcardHelper, }: { - db: Database; + ds: DataSource; wildcardHelper: (address: string) => string; }) { + this.ds = ds; this.wildcardHelper = wildcardHelper; - // getTownList() で使用するSQLをstatementにしておく + // getTownList() で使用するSQL・パラメータキーを用意しておく // "name"の文字数が長い順にソートする - this.getTownStatement = db.prepare(` + const { preparedSql, paramKeys } = prepareSqlAndParamKeys( + ds, + ` select "town".${DataField.LG_CODE.dbColumn}, "town"."${DataField.TOWN_ID.dbColumn}", @@ -96,7 +102,10 @@ export class AddressFinderForStep3and5 { ) = @city AND "${DataField.TOWN_CODE.dbColumn}" <> 3 order by length("name") desc; - `); + ` + ); + this.getTownSql = preparedSql; + this.getTownParamKeys = paramKeys; } async find({ @@ -383,10 +392,14 @@ export class AddressFinderForStep3and5 { prefecture: PrefectureName; city: string; }): Promise { - const results = this.getTownStatement.all({ + const params: { [key: string]: string } = { prefecture, city, - }) as TownRow[]; + }; + const results = (await this.ds.query( + this.getTownSql, + this.getTownParamKeys.map(key => params[key]) + )) as TownRow[]; return Promise.resolve( results.map(townRow => { diff --git a/src/usecase/geocode/address-finder-for-step7.ts b/src/usecase/geocode/address-finder-for-step7.ts index 011b8393..dd839f99 100644 --- a/src/usecase/geocode/address-finder-for-step7.ts +++ b/src/usecase/geocode/address-finder-for-step7.ts @@ -30,7 +30,8 @@ import { RegExpEx } from '@domain/reg-exp-ex'; import { Trie } from '@domain/trie'; import { zen2HankakuNum } from '@domain/zen2hankaku-num'; import { DASH, SPACE } from '@settings/constant-values'; -import { Database, Statement } from 'better-sqlite3'; +import { DataSource } from 'typeorm'; +import { prepareSqlAndParamKeys } from '@domain/prepare-sql-and-param-keys'; export type TownSmallBlock = { lg_code: string; @@ -75,13 +76,18 @@ export type RsdtAddr = { * 実質的にジオコーディングしている部分 */ export class AddressFinderForStep7 { - private readonly getBlockListStatement: Statement; - private readonly getRsdtListStatement2: Statement; - private readonly getSmallBlockListStatement: Statement; - - constructor(db: Database) { - this.getBlockListStatement = db.prepare(` - /* unit test: getBlockListStatement */ + private readonly ds: DataSource; + private readonly getBlockListSql: string; + private readonly getBlockListParamKeys: string[]; + private readonly getRsdtList2Sql: string; + private readonly getRsdtList2ParamKeys: string[]; + private readonly getSmallBlockListSql: string; + private readonly getSmallBlockListParamKeys: string[]; + + constructor(ds: DataSource) { + this.ds = ds; + const blockListSql = ` + /* unit test: getBlockListSql */ select "blk".${DataField.LG_CODE.dbColumn}, @@ -117,10 +123,14 @@ export class AddressFinderForStep7 { "city".${DataField.OD_CITY_NAME.dbColumn} ) = @city and blk.${DataField.BLK_NUM.dbColumn} is not null - `); + `; + const { preparedSql: blockListPreparedSql, paramKeys: blockListParamKeys } = + prepareSqlAndParamKeys(ds, blockListSql); + this.getBlockListSql = blockListPreparedSql; + this.getBlockListParamKeys = blockListParamKeys; - this.getRsdtListStatement2 = db.prepare(` - /* unit test: getRsdtListStatement2 */ + const rsdtList2Sql = ` + /* unit test: getRsdtList2Sql */ select ${DataField.ADDR_ID.dbColumn} as "addr1_id", @@ -138,10 +148,10 @@ export class AddressFinderForStep7 { order by ${DataField.RSDT_NUM.dbColumn} desc, ${DataField.RSDT_NUM2.dbColumn} desc - `); + `; - this.getSmallBlockListStatement = db.prepare(` - /* unit test: getSmallBlockListStatement */ + const smallBlockListSql = ` + /* unit test: getSmallBlockListSql */ select "town".${DataField.LG_CODE.dbColumn}, @@ -174,7 +184,18 @@ export class AddressFinderForStep7 { "town".${DataField.KOAZA_NAME.dbColumn} like @koaza order by town.${DataField.KOAZA_NAME.dbColumn} desc - `); + `; + + const { preparedSql: rsdtList2PreparedSql, paramKeys: rsdtList2ParamKeys } = + prepareSqlAndParamKeys(ds, rsdtList2Sql); + this.getRsdtList2Sql = rsdtList2PreparedSql; + this.getRsdtList2ParamKeys = rsdtList2ParamKeys; + const { + preparedSql: smallBlockListPreparedSql, + paramKeys: smallBlockListParamKeys, + } = prepareSqlAndParamKeys(ds, smallBlockListSql); + this.getSmallBlockListSql = smallBlockListPreparedSql; + this.getSmallBlockListParamKeys = smallBlockListParamKeys; } // 小字を検索する @@ -474,12 +495,16 @@ export class AddressFinderForStep7 { town: string; koaza: string; }): Promise { - const results = (await this.getSmallBlockListStatement.all({ + const params: { [key: string]: string } = { prefecture, city, town, koaza: `${koaza}%`, - })) as TownSmallBlock[]; + }; + const results = (await this.ds.query( + this.getSmallBlockListSql, + this.getSmallBlockListParamKeys.map(key => params[key]) + )) as TownSmallBlock[]; return Promise.resolve( results.map(smallBlock => { @@ -508,11 +533,15 @@ export class AddressFinderForStep7 { city: string; town: string; }): Promise { - const results = (await this.getBlockListStatement.all({ + const params: { [key: string]: string } = { prefecture, city, town, - })) as TownBlock[]; + }; + const results = (await this.ds.query( + this.getBlockListSql, + this.getBlockListParamKeys.map(key => params[key]) + )) as TownBlock[]; return Promise.resolve( results.map(town => { @@ -531,11 +560,15 @@ export class AddressFinderForStep7 { town_id: string; block_id: string; }): Promise { - const results = this.getRsdtListStatement2.all({ + const params: { [key: string]: string } = { lg_code, town_id, block_id, - }) as RsdtAddr[]; + }; + const results = (await this.ds.query( + this.getRsdtList2Sql, + this.getRsdtList2ParamKeys.map(key => params[key]) + )) as RsdtAddr[]; // better-sqlite3自体はasyncではないが、将来的にTypeORMに変更したいので // asyncで関数を作っておく