From 5457dc4acbf6379a5bc464bbd992060b883789bc Mon Sep 17 00:00:00 2001 From: Aram Al-Sabti Date: Fri, 4 Feb 2022 17:23:14 +0100 Subject: [PATCH 1/4] Batch bulk import create. Update missing --- .../bulk-import/bulk-import.component.ts | 265 ++++++++++++++---- .../bulk-import/bulk-import.model.ts | 2 +- .../iot-devices/iot-device.model.ts | 103 +++---- .../iot-devices/iot-device.service.ts | 20 +- src/app/shared/error-message.service.ts | 5 +- src/app/shared/helpers/array.helper.ts | 11 + .../shared/services/bulk-import.service.ts | 11 + 7 files changed, 300 insertions(+), 117 deletions(-) create mode 100644 src/app/shared/helpers/array.helper.ts create mode 100644 src/app/shared/services/bulk-import.service.ts diff --git a/src/app/applications/bulk-import/bulk-import.component.ts b/src/app/applications/bulk-import/bulk-import.component.ts index e28d2506..b7172325 100644 --- a/src/app/applications/bulk-import/bulk-import.component.ts +++ b/src/app/applications/bulk-import/bulk-import.component.ts @@ -2,24 +2,38 @@ import { HttpErrorResponse } from '@angular/common/http'; import { Component, OnInit } from '@angular/core'; import { Title } from '@angular/platform-browser'; import { ActivatedRoute } from '@angular/router'; +import { + IotDevicesImportResponse, + IotDeviceImportRequest, +} from '@applications/iot-devices/iot-device.model'; import { IoTDeviceService } from '@applications/iot-devices/iot-device.service'; import { faDownload, faTrash } from '@fortawesome/free-solid-svg-icons'; import { TranslateService } from '@ngx-translate/core'; import { ErrorMessageService } from '@shared/error-message.service'; +import { splitList } from '@shared/helpers/array.helper'; import { Download } from '@shared/helpers/download.helper'; +import { BulkImportService } from '@shared/services/bulk-import.service'; import { DownloadService } from '@shared/services/download.service'; import { Papa } from 'ngx-papaparse'; import { Observable } from 'rxjs'; +import { takeWhile } from 'rxjs/operators'; import { BulkImport } from './bulk-import.model'; import { BulkMapping } from './bulkMapping'; +let timeoutId: any = null; + @Component({ selector: 'app-bulk-import', templateUrl: './bulk-import.component.html', - styleUrls: ['./bulk-import.component.scss'] + styleUrls: ['./bulk-import.component.scss'], }) export class BulkImportComponent implements OnInit { - displayedColumns: string[] = ['name', 'type', 'importStatus', 'errorMessages']; + displayedColumns: string[] = [ + 'name', + 'type', + 'importStatus', + 'errorMessages', + ]; isLoading = false; bulkImport: BulkImport[]; bulkImportResult: BulkImport[]; @@ -28,10 +42,19 @@ export class BulkImportComponent implements OnInit { faTrash = faTrash; faDownload = faDownload; samples = [ - { name: 'generic-http-sample.csv', url: '../../../assets/docs/iotdevice_generichttp.csv' }, - { name: 'lorawan-otaa-sample.csv', url: '../../../assets/docs/iotdevice_lorawan_otaa.csv' }, - { name: 'lorawan-abp-sample.csv', url: '../../../assets/docs/iotdevice_lorawan_abp.csv' }, - ] + { + name: 'generic-http-sample.csv', + url: '../../../assets/docs/iotdevice_generichttp.csv', + }, + { + name: 'lorawan-otaa-sample.csv', + url: '../../../assets/docs/iotdevice_lorawan_otaa.csv', + }, + { + name: 'lorawan-abp-sample.csv', + url: '../../../assets/docs/iotdevice_lorawan_abp.csv', + }, + ]; download$: Observable; private bulkMapper = new BulkMapping(); public backButtonTitle: string; @@ -44,25 +67,23 @@ export class BulkImportComponent implements OnInit { private titleService: Title, private translate: TranslateService, private downloads: DownloadService, - private errorMessageService: ErrorMessageService + private errorMessageService: ErrorMessageService, + private bulkImportService: BulkImportService ) { this.translate.use('da'); - } + } ngOnInit(): void { - this.translate.get(['TITLE.BULKIMPORT']) - .subscribe(translations => { - this.titleService.setTitle(translations['TITLE.BULKIMPORT']); - }); + this.translate.get(['TITLE.BULKIMPORT']).subscribe((translations) => { + this.titleService.setTitle(translations['TITLE.BULKIMPORT']); + }); this.applicationId = +this.route.snapshot.paramMap.get('id'); - } - download({ name, url }: { name: string, url: string }) { + download({ name, url }: { name: string; url: string }) { this.download$ = this.downloads.download(url, name); } - deleteAttachment(index) { this.files.splice(index, 1); } @@ -76,13 +97,14 @@ export class BulkImportComponent implements OnInit { } this.bulkImport = []; this.bulkImportResult = []; - for (let index = 0; index < evt.length; index++) { - const element = evt[index]; + + for (const element of evt) { this.files.push(element.name); } + // handle csv data this.isLoading = true; - const files = evt; // File List object + const files = evt; // File List object const file = files[0]; const reader = new FileReader(); reader.readAsText(file); @@ -91,7 +113,7 @@ export class BulkImportComponent implements OnInit { this.papa.parse(csv, { skipEmptyLines: true, header: true, - complete: results => { + complete: (results) => { this.mapData(results.data); // this step ensures material can read from the array - should be fixed. this.bulkImportResult = this.bulkImport; @@ -100,9 +122,8 @@ export class BulkImportComponent implements OnInit { } else { return this.bulkImport; } - } - } - ); + }, + }); this.isLoading = false; }; } @@ -118,43 +139,187 @@ export class BulkImportComponent implements OnInit { private mapData(data: any[]) { data.forEach((device) => { - const mappedDevice = this.bulkMapper.dataMapper(device, this.applicationId); + const mappedDevice = this.bulkMapper.dataMapper( + device, + this.applicationId + ); if (mappedDevice) { this.bulkImport.push(new BulkImport(mappedDevice)); } else { - this.translate.get(['ERROR.SEMANTIC']) - .subscribe(translations => { - this.bulkImport.push(new BulkImport(null, [translations['ERROR.SEMANTIC']])); - }); + this.translate.get(['ERROR.SEMANTIC']).subscribe((translations) => { + this.bulkImport.push( + new BulkImport(null, [translations['ERROR.SEMANTIC']]) + ); + }); } }); } addIoTDevice() { - this.bulkImportResult.forEach((requestItem) => { - if (requestItem.device?.id) { - this.iotDeviceService.updateIoTDevice(requestItem.device, requestItem.device.id).subscribe( - (response) => { - console.log(response); - requestItem.importStatus = 'success'; - }, - (error: HttpErrorResponse) => { - requestItem.errorMessages = this.errorMessageService.handleErrorMessageWithFields(error).errorMessages; - requestItem.importStatus = 'Failed'; - } - ); - } else if (requestItem.device) { - this.iotDeviceService.createIoTDevice(requestItem.device).subscribe( - (res: any) => { - console.log(res); - requestItem.importStatus = 'success'; - }, - (error) => { - requestItem.errorMessages = this.errorMessageService.handleErrorMessage(error); - requestItem.importStatus = 'Failed'; - } - ); + const times: { label: string; duration: number }[] = []; + + // Subscribe to subject in service, Emit the index of next item in the array to be previous + // The emit will activate the subscription which should call the updateIoTDevice + + const { newDevices, updatedDevices } = this.splitDevices(); + + // Reset the index of the next list to process + this.bulkImportService.nextIotDeviceListIndex$.next(0); + this.bulkImportService.nextIotDeviceListIndex$ + .pipe(takeWhile((value) => value in newDevices)) + .subscribe( + (nextIndex) => { + console.log("next index is ", nextIndex); + const timeNow = performance.now(); + + const requestItems = newDevices[nextIndex]; + // TODO: Only subscribe to devices if there's actually data to send + const devices: IotDeviceImportRequest = { + data: requestItems.map((bulkResult) => bulkResult.device), + }; + this.iotDeviceService.createIoTDevices(devices).subscribe( + (response) => { + times.push({ + label: `${nextIndex} - Success`, + duration: performance.now() - timeNow, + }); + this.onSuccessfulImport(response, requestItems); + // TODO: Update nextIoTDeviceIndex? + this.bulkImportService.nextIotDeviceListIndex$.next(nextIndex + 1); + }, + (error: HttpErrorResponse) => { + times.push({ + label: `${requestItems[0].device.name} - Failed`, + duration: performance.now() - timeNow, + }); + requestItems.forEach((item) => { + item.errorMessages = this.errorMessageService.handleErrorMessageWithFields( + error + ).errorMessages; + item.importStatus = 'Failed'; + }); + } + ); + }, + (error: HttpErrorResponse) => {} + ); + + // for (const requestItem of this.bulkImportResult) { + // if (requestItem.device?.id) { + // // TODO: This doesn't work. Subscribe returns immediately. Then again, a request is made for every item at the same time.. + // const timeNow = performance.now(); + // this.iotDeviceService + // .updateIoTDevice(requestItem.device, requestItem.device.id) + // .subscribe( + // (response) => { + // times.push({ + // label: `${requestItem.device.name} - Success`, + // duration: performance.now() - timeNow, + // }); + // // console.log(response); + // requestItem.importStatus = 'success'; + // }, + // (error: HttpErrorResponse) => { + // times.push({ + // label: `${requestItem.device.name} - Failed`, + // duration: performance.now() - timeNow, + // }); + // requestItem.errorMessages = this.errorMessageService.handleErrorMessageWithFields( + // error + // ).errorMessages; + // requestItem.importStatus = 'Failed'; + // } + // ); + // } else if (requestItem.device) { + // const timeNow = performance.now(); + // this.iotDeviceService.createIoTDevice(requestItem.device).subscribe( + // (res: any) => { + // // console.log(res); + // times.push({ + // label: `${requestItem.device.name} - Success`, + // duration: performance.now() - timeNow, + // }); + // requestItem.importStatus = 'success'; + // }, + // (error) => { + // times.push({ + // label: `${requestItem.device.name} - Failed`, + // duration: performance.now() - timeNow, + // }); + // requestItem.errorMessages = this.errorMessageService.handleErrorMessage( + // error + // ); + // requestItem.importStatus = 'Failed'; + // } + // ); + // } + // } + + timeoutId = setInterval(() => { + if (times.length < this.bulkImportResult.length / 30) { + console.log('Timer still going...'); + return; + } + clearInterval(timeoutId); + console.log( + 'times', + times.sort((a, b) => b.duration - a.duration) + ); + }, 10 * 1000); + + // TODO: Edge case - a device's status was not set. This indicates a flaw in the above logic. Set them to error + + // Clear the subject. Or should we? TODO: + // this.bulkImportService.nextIotDeviceIndex$.next(0); + } + + private onSuccessfulImport( + response: IotDevicesImportResponse[], + requestItems: BulkImport[] + ) { + response.forEach((responseItem) => { + const match = requestItems.find( + ({ device }) => + device.name === responseItem.data.name && + device.applicationId === responseItem.data.application?.id + ); + if (!match) { + return; + } + + if (responseItem.error && match) { + match.errorMessages = this.errorMessageService.handleErrorMessageWithFields( + { error: responseItem.error } + ).errorMessages; + match.importStatus = 'Failed'; + } else { + match.errorMessages = []; + match.importStatus = 'Success'; } }); } + + private splitDevices() { + const { updatedDevices, newDevices } = this.bulkImportResult.reduce( + ( + res: { + newDevices: BulkImport[]; + updatedDevices: BulkImport[]; + }, + curr + ) => { + if (curr.device.id) { + res.updatedDevices.push(curr); + } else if (curr.device) { + res.newDevices.push(curr); + } + return res; + }, + { updatedDevices: [], newDevices: [] } + ); + return { + newDevices: splitList(newDevices), + updatedDevices: splitList(updatedDevices), + }; + } } diff --git a/src/app/applications/bulk-import/bulk-import.model.ts b/src/app/applications/bulk-import/bulk-import.model.ts index 05d07be5..904569f7 100644 --- a/src/app/applications/bulk-import/bulk-import.model.ts +++ b/src/app/applications/bulk-import/bulk-import.model.ts @@ -2,7 +2,7 @@ import { IotDevice } from '@applications/iot-devices/iot-device.model'; export class BulkImport { public device: IotDevice; - public errorMessages = []; + public errorMessages: unknown[] = []; public importStatus = ''; constructor(device: IotDevice, errorMessages = [], importStatus = '') { this.device = device; diff --git a/src/app/applications/iot-devices/iot-device.model.ts b/src/app/applications/iot-devices/iot-device.model.ts index 114e3d85..a3c762f0 100644 --- a/src/app/applications/iot-devices/iot-device.model.ts +++ b/src/app/applications/iot-devices/iot-device.model.ts @@ -8,70 +8,59 @@ import { ReceivedMessageMetadata } from '@shared/models/received-message-metadat import { LatestReceivedMessage } from './latestReceivedMessage.model'; export class IotDevice { - name: string; - application?: Application; - location: JsonLocation; - commentOnLocation: string; - comment: string; - type: DeviceType = DeviceType.GENERICHTTP; - receivedMessagesMetadata: ReceivedMessageMetadata[]; - metadata?: JSON; - apiKey?: string; - id: number; - createdAt: Date; - updatedAt: Date; - createdBy: number; - updatedBy: number; - createdByName: string; - updatedByName: string; - applicationId: number; - longitude = 0; - latitude = 0; - deviceModelId?: number; - latestReceivedMessage: LatestReceivedMessage; - lorawanSettings = new LorawanSettings(); - sigfoxSettings = new SigfoxSettings(); - deviceModel?: DeviceModel; + name: string; + application?: Application; + location: JsonLocation; + commentOnLocation: string; + comment: string; + type: DeviceType = DeviceType.GENERICHTTP; + receivedMessagesMetadata: ReceivedMessageMetadata[]; + metadata?: JSON; + apiKey?: string; + id: number; + createdAt: Date; + updatedAt: Date; + createdBy: number; + updatedBy: number; + createdByName: string; + updatedByName: string; + applicationId: number; + longitude = 0; + latitude = 0; + deviceModelId?: number; + latestReceivedMessage: LatestReceivedMessage; + lorawanSettings = new LorawanSettings(); + sigfoxSettings = new SigfoxSettings(); + deviceModel?: DeviceModel; } -export class IotDeviceResponse { - name: string; - application?: Application; - location: JsonLocation; - commentOnLocation: string; - comment: string; - type: DeviceType = DeviceType.GENERICHTTP; - receivedMessagesMetadata: ReceivedMessageMetadata[]; - metadata?: JSON; - apiKey?: string; - id: number; - createdAt: Date; - updatedAt: Date; - applicationId: number; - longitude = 0; - latitude = 0; - deviceModelId?: DeviceModel; - latestReceivedMessage: LatestReceivedMessage; - lorawanSettings = new LorawanSettings(); - sigfoxSettings = new SigfoxSettings(); -} +export class IotDeviceResponse extends IotDevice {} export interface IotDevicesResponse { - data: IotDevice[]; - ok?: boolean; - count?: number; + data: IotDevice[]; + ok?: boolean; + count?: number; +} + +export interface IotDeviceImportRequest { + data: IotDevice[]; +} + +export interface IotDevicesImportResponse { + data: IotDevice; + error?: Omit; } export class IoTDeviceMinimal { - id: number; - name: string; - canRead: boolean; - organizationId: number; - applicationId: number; - lastActiveTime: Date; + id: number; + name: string; + canRead: boolean; + organizationId: number; + applicationId: number; + lastActiveTime: Date; } export class IoTDevicesMinimalResponse { - data: IoTDeviceMinimal[]; - count: number; -} \ No newline at end of file + data: IoTDeviceMinimal[]; + count: number; +} diff --git a/src/app/applications/iot-devices/iot-device.service.ts b/src/app/applications/iot-devices/iot-device.service.ts index bedbf529..d01709e1 100644 --- a/src/app/applications/iot-devices/iot-device.service.ts +++ b/src/app/applications/iot-devices/iot-device.service.ts @@ -1,6 +1,6 @@ import { Injectable } from '@angular/core'; -import { Observable } from 'rxjs'; -import { IotDevice, IoTDevicesMinimalResponse, IotDevicesResponse } from './iot-device.model'; +import { Observable} from 'rxjs'; +import { IotDevice, IoTDevicesMinimalResponse, IotDevicesImportResponse, IotDeviceImportRequest } from './iot-device.model'; import { RestService } from 'src/app/shared/services/rest.service'; import { map } from 'rxjs/operators'; import { UserMinimalService } from '@app/admin/users/user-minimal.service'; @@ -14,17 +14,25 @@ export class IoTDeviceService { constructor( private restService: RestService, - private userMinimalService: UserMinimalService + private userMinimalService: UserMinimalService ) { } - createIoTDevice(body: IotDevice): Observable { + createIoTDevice(body: IotDevice): Observable { return this.restService.post(this.BASEURL, body); } - updateIoTDevice(body: IotDevice, id: number): Observable { + updateIoTDevice(body: IotDevice, id: number): Observable { return this.restService.put(this.BASEURL, body, id, { observe: 'response' }); } + createIoTDevices(body: IotDeviceImportRequest): Observable { + return this.restService.post(`${this.BASEURL}/createMany`, body); + } + + updateIoTDevices(body: IotDeviceImportRequest): Observable { + return this.restService.put(`${this.BASEURL}/updateMany`, body, undefined, { observe: 'response' }); + } + getIoTDevice(id: number): Observable { return this.restService.get(this.BASEURL, {}, id).pipe( map( @@ -60,7 +68,7 @@ export class IoTDeviceService { } getIoTDevicesUsingPayloadDecoderMinimal(payloadDecoderId: number, limit: number, offset: number): Observable { - return this.restService.get(`iot-device/minimalByPayloadDecoder`, {limit: limit, offset: offset}, payloadDecoderId) + return this.restService.get(`${this.BASEURL}/minimalByPayloadDecoder`, {limit, offset}, payloadDecoderId); } deleteIoTDevice(id: number) { diff --git a/src/app/shared/error-message.service.ts b/src/app/shared/error-message.service.ts index 7446ab11..bd69cc41 100644 --- a/src/app/shared/error-message.service.ts +++ b/src/app/shared/error-message.service.ts @@ -13,8 +13,7 @@ export class ErrorMessageService { errorMessages.push(err.error.message); } else if (err.error.chirpstackError) { errorMessages.push(err.error.chirpstackError.message); - } - else { + } else { err.error.message.forEach( (err) => { if (err.property === 'lorawanSettings') { err.children.forEach( (element) => { @@ -32,7 +31,7 @@ export class ErrorMessageService { return errorMessages; } - public handleErrorMessageWithFields(error: HttpErrorResponse): ErrorMessage { + public handleErrorMessageWithFields(error: HttpErrorResponse | Pick): ErrorMessage { const errors: ErrorMessage = {errorFields: [], errorMessages: []}; if (typeof error.error.message === 'string') { errors.errorMessages.push(error.error.message); diff --git a/src/app/shared/helpers/array.helper.ts b/src/app/shared/helpers/array.helper.ts new file mode 100644 index 00000000..2d953202 --- /dev/null +++ b/src/app/shared/helpers/array.helper.ts @@ -0,0 +1,11 @@ +export const splitList = ( + data: T, + batchSize = 50 +): T[] => { + const dataBatches: typeof data[] = []; + for (let i = 0; i < data.length; i += batchSize) { + dataBatches.push(data.slice(i, i + batchSize) as T); + } + + return dataBatches; +}; diff --git a/src/app/shared/services/bulk-import.service.ts b/src/app/shared/services/bulk-import.service.ts new file mode 100644 index 00000000..08227be5 --- /dev/null +++ b/src/app/shared/services/bulk-import.service.ts @@ -0,0 +1,11 @@ +import { Injectable } from '@angular/core'; +import { BehaviorSubject } from 'rxjs'; + +@Injectable({ + providedIn: 'root' +}) +export class BulkImportService { + public readonly nextIotDeviceListIndex$: BehaviorSubject = new BehaviorSubject(0); + + constructor() { } +} From 5d9370a03d16d6efb7d39a2e4675b4192820255e Mon Sep 17 00:00:00 2001 From: Aram Al-Sabti Date: Mon, 7 Feb 2022 17:07:47 +0100 Subject: [PATCH 2/4] Reworked update many and cleanup --- .../bulk-import/bulk-import.component.ts | 198 ++++++++---------- .../iot-devices/iot-device.model.ts | 4 + .../iot-devices/iot-device.service.ts | 4 +- src/app/shared/error-message.service.ts | 8 +- src/app/shared/helpers/array.helper.ts | 8 +- .../shared/services/bulk-import.service.ts | 9 +- src/assets/i18n/da.json | 5 +- 7 files changed, 110 insertions(+), 126 deletions(-) diff --git a/src/app/applications/bulk-import/bulk-import.component.ts b/src/app/applications/bulk-import/bulk-import.component.ts index b7172325..8fcae01b 100644 --- a/src/app/applications/bulk-import/bulk-import.component.ts +++ b/src/app/applications/bulk-import/bulk-import.component.ts @@ -3,8 +3,8 @@ import { Component, OnInit } from '@angular/core'; import { Title } from '@angular/platform-browser'; import { ActivatedRoute } from '@angular/router'; import { - IotDevicesImportResponse, IotDeviceImportRequest, + IotDevicesImportResponse, } from '@applications/iot-devices/iot-device.model'; import { IoTDeviceService } from '@applications/iot-devices/iot-device.service'; import { faDownload, faTrash } from '@fortawesome/free-solid-svg-icons'; @@ -15,13 +15,11 @@ import { Download } from '@shared/helpers/download.helper'; import { BulkImportService } from '@shared/services/bulk-import.service'; import { DownloadService } from '@shared/services/download.service'; import { Papa } from 'ngx-papaparse'; -import { Observable } from 'rxjs'; +import { Observable, Subject } from 'rxjs'; import { takeWhile } from 'rxjs/operators'; import { BulkImport } from './bulk-import.model'; import { BulkMapping } from './bulkMapping'; -let timeoutId: any = null; - @Component({ selector: 'app-bulk-import', templateUrl: './bulk-import.component.html', @@ -156,121 +154,71 @@ export class BulkImportComponent implements OnInit { } addIoTDevice() { - const times: { label: string; duration: number }[] = []; - // Subscribe to subject in service, Emit the index of next item in the array to be previous // The emit will activate the subscription which should call the updateIoTDevice - const { newDevices, updatedDevices } = this.splitDevices(); - // Reset the index of the next list to process - this.bulkImportService.nextIotDeviceListIndex$.next(0); - this.bulkImportService.nextIotDeviceListIndex$ - .pipe(takeWhile((value) => value in newDevices)) - .subscribe( - (nextIndex) => { - console.log("next index is ", nextIndex); - const timeNow = performance.now(); + this.postBulkImportPayload( + newDevices, + this.bulkImportService.nextCreateIotDeviceBatchIndex$, + this.iotDeviceService.createIoTDevices.bind(this.iotDeviceService) + ); + this.postBulkImportPayload( + updatedDevices, + this.bulkImportService.nextUpdateDeviceBatchIndex$, + this.iotDeviceService.updateIoTDevices.bind(this.iotDeviceService) + ); + } - const requestItems = newDevices[nextIndex]; - // TODO: Only subscribe to devices if there's actually data to send - const devices: IotDeviceImportRequest = { - data: requestItems.map((bulkResult) => bulkResult.device), - }; - this.iotDeviceService.createIoTDevices(devices).subscribe( - (response) => { - times.push({ - label: `${nextIndex} - Success`, - duration: performance.now() - timeNow, - }); - this.onSuccessfulImport(response, requestItems); - // TODO: Update nextIoTDeviceIndex? - this.bulkImportService.nextIotDeviceListIndex$.next(nextIndex + 1); - }, - (error: HttpErrorResponse) => { - times.push({ - label: `${requestItems[0].device.name} - Failed`, - duration: performance.now() - timeNow, - }); - requestItems.forEach((item) => { - item.errorMessages = this.errorMessageService.handleErrorMessageWithFields( - error - ).errorMessages; - item.importStatus = 'Failed'; - }); - } - ); - }, - (error: HttpErrorResponse) => {} - ); + private postBulkImportPayload( + bulkDevices: BulkImport[][], + batchIndex$: Subject, + importDevices: ( + payload: IotDeviceImportRequest + ) => Observable + ): void { + if (!bulkDevices.length) { + return; + } - // for (const requestItem of this.bulkImportResult) { - // if (requestItem.device?.id) { - // // TODO: This doesn't work. Subscribe returns immediately. Then again, a request is made for every item at the same time.. - // const timeNow = performance.now(); - // this.iotDeviceService - // .updateIoTDevice(requestItem.device, requestItem.device.id) - // .subscribe( - // (response) => { - // times.push({ - // label: `${requestItem.device.name} - Success`, - // duration: performance.now() - timeNow, - // }); - // // console.log(response); - // requestItem.importStatus = 'success'; - // }, - // (error: HttpErrorResponse) => { - // times.push({ - // label: `${requestItem.device.name} - Failed`, - // duration: performance.now() - timeNow, - // }); - // requestItem.errorMessages = this.errorMessageService.handleErrorMessageWithFields( - // error - // ).errorMessages; - // requestItem.importStatus = 'Failed'; - // } - // ); - // } else if (requestItem.device) { - // const timeNow = performance.now(); - // this.iotDeviceService.createIoTDevice(requestItem.device).subscribe( - // (res: any) => { - // // console.log(res); - // times.push({ - // label: `${requestItem.device.name} - Success`, - // duration: performance.now() - timeNow, - // }); - // requestItem.importStatus = 'success'; - // }, - // (error) => { - // times.push({ - // label: `${requestItem.device.name} - Failed`, - // duration: performance.now() - timeNow, - // }); - // requestItem.errorMessages = this.errorMessageService.handleErrorMessage( - // error - // ); - // requestItem.importStatus = 'Failed'; - // } - // ); - // } - // } + let batchIndex = 0; - timeoutId = setInterval(() => { - if (times.length < this.bulkImportResult.length / 30) { - console.log('Timer still going...'); - return; + batchIndex$.pipe(takeWhile(() => batchIndex in bulkDevices)).subscribe( + () => { + const requestItems = bulkDevices[batchIndex]; + const devices: IotDeviceImportRequest = { + data: requestItems.map((bulkResult) => bulkResult.device), + }; + importDevices(devices).subscribe( + (response) => { + this.onSuccessfulImport(response, requestItems); + ++batchIndex; + batchIndex$.next(); + }, + (error: HttpErrorResponse) => { + requestItems.forEach((item) => { + item.errorMessages = this.errorMessageService.handleErrorMessageWithFields( + error + ).errorMessages; + item.importStatus = 'Failed'; + }); + // Unsubscribe from subject while still allowing future subscriptions + ++batchIndex; + batchIndex$.next(); + } + ); + }, + (_error: HttpErrorResponse) => { + // Should not happen + }, + () => { + // All devices have been processed. Check if some devices' status hasn't been set + this.onCompleteImport(bulkDevices); } - clearInterval(timeoutId); - console.log( - 'times', - times.sort((a, b) => b.duration - a.duration) - ); - }, 10 * 1000); - - // TODO: Edge case - a device's status was not set. This indicates a flaw in the above logic. Set them to error + ); - // Clear the subject. Or should we? TODO: - // this.bulkImportService.nextIotDeviceIndex$.next(0); + // Trigger our listener + batchIndex$.next(); } private onSuccessfulImport( @@ -280,8 +228,8 @@ export class BulkImportComponent implements OnInit { response.forEach((responseItem) => { const match = requestItems.find( ({ device }) => - device.name === responseItem.data.name && - device.applicationId === responseItem.data.application?.id + device.name === responseItem.idMetadata.name && + device.applicationId === responseItem.idMetadata.applicationId ); if (!match) { return; @@ -299,7 +247,31 @@ export class BulkImportComponent implements OnInit { }); } - private splitDevices() { + private onCompleteImport(devicesBulk: BulkImport[][]) { + for (const bulk of devicesBulk) { + for (const device of bulk) { + if (!device.importStatus) { + device.importStatus = 'Failed'; + device.errorMessages = this.errorMessageService.handleErrorMessageWithFields( + { + error: { + message: 'MESSAGE.FAILED-TO-CREATE-OR-UPDATE-IOT-DEVICE', + }, + } + ).errorMessages; + } + } + } + } + + private splitDevices(): { + newDevices: BulkImport[][]; + updatedDevices: BulkImport[][]; + } { + if (!this.bulkImportResult) { + return { newDevices: [], updatedDevices: [] }; + } + const { updatedDevices, newDevices } = this.bulkImportResult.reduce( ( res: { diff --git a/src/app/applications/iot-devices/iot-device.model.ts b/src/app/applications/iot-devices/iot-device.model.ts index a3c762f0..039647fd 100644 --- a/src/app/applications/iot-devices/iot-device.model.ts +++ b/src/app/applications/iot-devices/iot-device.model.ts @@ -48,6 +48,10 @@ export interface IotDeviceImportRequest { export interface IotDevicesImportResponse { data: IotDevice; + idMetadata: { + name: string; + applicationId: number; + }; error?: Omit; } diff --git a/src/app/applications/iot-devices/iot-device.service.ts b/src/app/applications/iot-devices/iot-device.service.ts index d01709e1..ff2d5fb5 100644 --- a/src/app/applications/iot-devices/iot-device.service.ts +++ b/src/app/applications/iot-devices/iot-device.service.ts @@ -1,5 +1,5 @@ import { Injectable } from '@angular/core'; -import { Observable} from 'rxjs'; +import { Observable } from 'rxjs'; import { IotDevice, IoTDevicesMinimalResponse, IotDevicesImportResponse, IotDeviceImportRequest } from './iot-device.model'; import { RestService } from 'src/app/shared/services/rest.service'; import { map } from 'rxjs/operators'; @@ -30,7 +30,7 @@ export class IoTDeviceService { } updateIoTDevices(body: IotDeviceImportRequest): Observable { - return this.restService.put(`${this.BASEURL}/updateMany`, body, undefined, { observe: 'response' }); + return this.restService.post(`${this.BASEURL}/updateMany`, body); } getIoTDevice(id: number): Observable { diff --git a/src/app/shared/error-message.service.ts b/src/app/shared/error-message.service.ts index bd69cc41..0e76e396 100644 --- a/src/app/shared/error-message.service.ts +++ b/src/app/shared/error-message.service.ts @@ -33,7 +33,11 @@ export class ErrorMessageService { public handleErrorMessageWithFields(error: HttpErrorResponse | Pick): ErrorMessage { const errors: ErrorMessage = {errorFields: [], errorMessages: []}; - if (typeof error.error.message === 'string') { + if (typeof error.error === 'string') { + errors.errorMessages.push(error.error); + } else if (typeof error.error?.error === 'string') { + errors.errorMessages.push(error.error.error); + } else if (typeof error.error?.message === 'string') { errors.errorMessages.push(error.error.message); } else { error.error.message.forEach((err) => { @@ -64,7 +68,7 @@ export class ErrorMessageService { } else if (err.message) { errors.errorFields.push(err.field); errors.errorMessages.push(err.message); - } else { + } else if (err.constraints) { errors.errorFields.push(err.property); errors.errorMessages = errors.errorMessages.concat( Object.values(err.constraints) diff --git a/src/app/shared/helpers/array.helper.ts b/src/app/shared/helpers/array.helper.ts index 2d953202..779ffa89 100644 --- a/src/app/shared/helpers/array.helper.ts +++ b/src/app/shared/helpers/array.helper.ts @@ -1,10 +1,10 @@ -export const splitList = ( - data: T, +export const splitList = ( + data: T[], batchSize = 50 -): T[] => { +): typeof data[] => { const dataBatches: typeof data[] = []; for (let i = 0; i < data.length; i += batchSize) { - dataBatches.push(data.slice(i, i + batchSize) as T); + dataBatches.push(data.slice(i, i + batchSize)); } return dataBatches; diff --git a/src/app/shared/services/bulk-import.service.ts b/src/app/shared/services/bulk-import.service.ts index 08227be5..b093be00 100644 --- a/src/app/shared/services/bulk-import.service.ts +++ b/src/app/shared/services/bulk-import.service.ts @@ -1,11 +1,12 @@ import { Injectable } from '@angular/core'; -import { BehaviorSubject } from 'rxjs'; +import { Subject } from 'rxjs'; @Injectable({ - providedIn: 'root' + providedIn: 'root', }) export class BulkImportService { - public readonly nextIotDeviceListIndex$: BehaviorSubject = new BehaviorSubject(0); + public readonly nextCreateIotDeviceBatchIndex$: Subject = new Subject(); + public readonly nextUpdateDeviceBatchIndex$: Subject = new Subject(); - constructor() { } + constructor() {} } diff --git a/src/assets/i18n/da.json b/src/assets/i18n/da.json index 2381c4c3..bdbf99bb 100644 --- a/src/assets/i18n/da.json +++ b/src/assets/i18n/da.json @@ -708,7 +708,10 @@ "OTAA-INFO-MISSING": "OTAA nøgle mangler eller er ikke gyldig.", "ABP-INFO-MISSING": "ABP nøgle mangler eller er ikke gyldig.", "DIFFERENT-SERVICE-PROFILE": "Dine devices har forskellige service profiles. De skal have den samme service profile!", - "WRONG-SERVICE-PROFILE": "Dine devices har forkert service profile. Vælg devices som har samme service profile som din multicast." + "WRONG-SERVICE-PROFILE": "Dine devices har forkert service profile. Vælg devices som har samme service profile som din multicast.", + "ID-DOES-NOT-EXIST": "Id'et findes ikke", + "APPLICATION-DOES-NOT-EXIST": "Den tilhørende applikation findes ikke", + "FAILED-TO-CREATE-OR-UPDATE-IOT-DEVICE": "Enheden kunne ikke oprettes eller opdateres" }, "PROFILES": { "NAME": "LoRaWAN profiler", From 46a2af8807fc185253c376f02d777807d7d97eaf Mon Sep 17 00:00:00 2001 From: Aram Al-Sabti Date: Wed, 9 Feb 2022 17:50:23 +0100 Subject: [PATCH 3/4] Comment on bulk import --- src/app/applications/bulk-import/bulk-import.component.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/app/applications/bulk-import/bulk-import.component.ts b/src/app/applications/bulk-import/bulk-import.component.ts index 8fcae01b..5d7a6293 100644 --- a/src/app/applications/bulk-import/bulk-import.component.ts +++ b/src/app/applications/bulk-import/bulk-import.component.ts @@ -183,6 +183,7 @@ export class BulkImportComponent implements OnInit { let batchIndex = 0; + // takeWhile() will unsubscribe once the condition is false batchIndex$.pipe(takeWhile(() => batchIndex in bulkDevices)).subscribe( () => { const requestItems = bulkDevices[batchIndex]; @@ -202,7 +203,7 @@ export class BulkImportComponent implements OnInit { ).errorMessages; item.importStatus = 'Failed'; }); - // Unsubscribe from subject while still allowing future subscriptions + // Continue processing the next batches ++batchIndex; batchIndex$.next(); } @@ -212,7 +213,7 @@ export class BulkImportComponent implements OnInit { // Should not happen }, () => { - // All devices have been processed. Check if some devices' status hasn't been set + // Process any devices whose status hasn't been set and mark them as errors. this.onCompleteImport(bulkDevices); } ); From 78f4ca511da38fc0ae54fcf874952e0dbd0f10a2 Mon Sep 17 00:00:00 2001 From: Aram Al-Sabti Date: Wed, 16 Feb 2022 17:32:07 +0100 Subject: [PATCH 4/4] Added device model error codes --- src/assets/i18n/da.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/assets/i18n/da.json b/src/assets/i18n/da.json index bdbf99bb..c899c2ad 100644 --- a/src/assets/i18n/da.json +++ b/src/assets/i18n/da.json @@ -711,7 +711,9 @@ "WRONG-SERVICE-PROFILE": "Dine devices har forkert service profile. Vælg devices som har samme service profile som din multicast.", "ID-DOES-NOT-EXIST": "Id'et findes ikke", "APPLICATION-DOES-NOT-EXIST": "Den tilhørende applikation findes ikke", - "FAILED-TO-CREATE-OR-UPDATE-IOT-DEVICE": "Enheden kunne ikke oprettes eller opdateres" + "FAILED-TO-CREATE-OR-UPDATE-IOT-DEVICE": "Enheden kunne ikke oprettes eller opdateres", + "DEVICE-MODEL-ORGANIZATION-DOES-NOT-MATCH": "Organisationsid'et på device modellen matcher ikke den tilhørende applikation", + "DEVICE-MODEL-DOES-NOT-EXIST": "Device model findes ikke" }, "PROFILES": { "NAME": "LoRaWAN profiler",