Feature/iot 1530 datatarget log

src/config/configuration.ts

@@ -18,6 +18,7 @@ export default (): any => {
     backend: {
       baseurl: process.env.BACKEND_BASEURL || "https://localhost:3000",
       deviceStatsIntervalInDays: parseInt(process.env.DEVICE_STATS_INTERVAL_IN_DAYS, 10) || 29,
+      datatargetLogMaxEvents: process.env.DATATARGET_LOG_MAX_EVENTS || 250,
     },
     kombit: {
       entryPoint:

src/controllers/admin-controller/data-target-log.controller.ts

@@ -0,0 +1,33 @@
+import { ComposeAuthGuard } from "@auth/compose-auth.guard";
+import { Read } from "@auth/roles.decorator";
+import { RolesGuard } from "@auth/roles.guard";
+import { ApiAuth } from "@auth/swagger-auth-decorator";
+import { DatatargetLog } from "@entities/datatarget-log.entity";
+import { Controller, Get, Param, ParseIntPipe, UseGuards } from "@nestjs/common";
+import { ApiForbiddenResponse, ApiTags, ApiUnauthorizedResponse } from "@nestjs/swagger";
+import { InjectRepository } from "@nestjs/typeorm";
+import { Repository } from "typeorm";
+
+@ApiTags("Data Target Logs")
+@Controller("datatarget-log")
+@UseGuards(ComposeAuthGuard, RolesGuard)
+@ApiAuth()
+@Read()
+@ApiForbiddenResponse()
+@ApiUnauthorizedResponse()
+export class DatatargetLogController {
+  constructor(
+    @InjectRepository(DatatargetLog)
+    private datatargetLogRepository: Repository<DatatargetLog>
+  ) {}
+
+  @Get(":datatargetId")
+  async getDatatargetLogs(@Param("datatargetId", new ParseIntPipe()) datatargetId: number): Promise<DatatargetLog[]> {
+    return await this.datatargetLogRepository.find({
+      where: {
+        datatarget: { id: datatargetId },
+      },
+      relations: ["iotDevice"],
+    });
+  }
+}

src/controllers/admin-controller/data-target.controller.ts

@@ -75,7 +75,7 @@ export class DataTargetController {
   @ApiOperation({ summary: "Find DataTarget by id" })
   async findOne(@Req() req: AuthenticatedRequest, @Param("id", new ParseIntPipe()) id: number): Promise<DataTarget> {
     try {
-      const dataTarget = await this.dataTargetService.findOne(id);
+      const dataTarget = await this.dataTargetService.findOneWithHasRecentError(id);
       checkIfUserHasAccessToApplication(req, dataTarget.application.id, ApplicationAccessScope.Read);
       return dataTarget;
     } catch (err) {

src/entities/data-target.entity.ts

@@ -19,6 +19,9 @@ export abstract class DataTarget extends DbBaseEntity {
   @Column()
   name: string;
 
+  @Column({ nullable: true })
+  lastMessageDate?: Date;
+
   @ManyToOne(
     // eslint-disable-next-line @typescript-eslint/no-unused-vars
     type => Application,

src/entities/datatarget-log.entity.ts

@@ -0,0 +1,31 @@
+import { SendStatus } from "@enum/send-status.enum";
+import { Column, Entity, Index, JoinColumn, ManyToOne } from "typeorm";
+import { DbBaseEntity } from "./base.entity";
+import { DataTarget } from "./data-target.entity";
+import { IoTDevice } from "./iot-device.entity";
+import { PayloadDecoder } from "./payload-decoder.entity";
+
+@Entity("datatarget-log")
+@Index(["datatarget", "createdAt"])
+export class DatatargetLog extends DbBaseEntity {
+  @ManyToOne(() => DataTarget, { onDelete: "CASCADE" })
+  @JoinColumn()
+  datatarget: DataTarget;
+
+  @Column()
+  type: SendStatus;
+
+  @Column({ nullable: true })
+  statusCode?: number;
+
+  @Column({ nullable: true })
+  message?: string;
+
+  @ManyToOne(() => IoTDevice, { onDelete: "SET NULL", nullable: true })
+  @JoinColumn()
+  iotDevice?: IoTDevice;
+
+  @ManyToOne(() => PayloadDecoder, { onDelete: "SET NULL", nullable: true })
+  @JoinColumn()
+  payloadDecoder?: PayloadDecoder;
+}

src/entities/dto/list-all-data-targets-response.dto.ts

@@ -1,4 +1,8 @@
 import { ListAllEntitiesResponseDto } from "@dto/list-all-entities-response.dto";
 import { DataTarget } from "@entities/data-target.entity";
 
-export class ListAllDataTargetsResponseDto extends ListAllEntitiesResponseDto<DataTarget> {}
+export type DataTargetDto = DataTarget & {
+  hasRecentErrors?: boolean;
+};
+
+export class ListAllDataTargetsResponseDto extends ListAllEntitiesResponseDto<DataTargetDto> {}

src/entities/interfaces/data-target-send-status.interface.ts

@@ -3,4 +3,6 @@ import { SendStatus } from "@enum/send-status.enum";
 export interface DataTargetSendStatus {
   errorMessage?: string;
   status: SendStatus;
+  statusCode?: number;
+  statusText?: string;
 }

src/migration/1726653509863-datatarget-log.ts

@@ -0,0 +1,22 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class DatatargetLog1726653509863 implements MigrationInterface {
+    name = 'DatatargetLog1726653509863'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "data_target" ADD "lastMessageDate" TIMESTAMP`);
+        await queryRunner.query(`CREATE TABLE "datatarget-log" ("id" SERIAL NOT NULL, "createdAt" TIMESTAMP NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP NOT NULL DEFAULT now(), "type" character varying NOT NULL, "statusCode" integer, "message" character varying, "createdById" integer, "updatedById" integer, "datatargetId" integer, "iotDeviceId" integer, "payloadDecoderId" integer, CONSTRAINT "PK_f853c6517b8a428fa350221a0f2" PRIMARY KEY ("id"))`);
+        await queryRunner.query(`CREATE INDEX "IDX_1d333574ef086bee54ff78b2e0" ON "datatarget-log" ("datatargetId", "createdAt") `);
+        await queryRunner.query(`ALTER TABLE "datatarget-log" ADD CONSTRAINT "FK_2f62220b03e524c7a056890fb3b" FOREIGN KEY ("createdById") REFERENCES "user"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`);
+        await queryRunner.query(`ALTER TABLE "datatarget-log" ADD CONSTRAINT "FK_7bcebf642504f26aa8c03074cca" FOREIGN KEY ("updatedById") REFERENCES "user"("id") ON DELETE NO ACTION ON UPDATE NO ACTION`);
+        await queryRunner.query(`ALTER TABLE "datatarget-log" ADD CONSTRAINT "FK_0e1507ac15e7d6755273691345e" FOREIGN KEY ("datatargetId") REFERENCES "data_target"("id") ON DELETE CASCADE ON UPDATE NO ACTION`);
+        await queryRunner.query(`ALTER TABLE "datatarget-log" ADD CONSTRAINT "FK_4fd74ac0a22db2f38e6b420a657" FOREIGN KEY ("iotDeviceId") REFERENCES "iot_device"("id") ON DELETE SET NULL ON UPDATE NO ACTION`);
+        await queryRunner.query(`ALTER TABLE "datatarget-log" ADD CONSTRAINT "FK_28c126089e5dac0360926acf819" FOREIGN KEY ("payloadDecoderId") REFERENCES "payload_decoder"("id") ON DELETE SET NULL ON UPDATE NO ACTION`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "data_target" DROP COLUMN "lastMessageDate"`);
+        await queryRunner.query(`DROP TABLE "datatarget-log"`);
+    }
+
+}

src/modules/device-management/data-target.module.ts

@@ -1,3 +1,4 @@
+import { DatatargetLogController } from "@admin-controller/data-target-log.controller";
 import { DataTargetController } from "@admin-controller/data-target.controller";
 import configuration from "@config/configuration";
 import { ApplicationModule } from "@modules/device-management/application.module";
@@ -8,6 +9,7 @@ import { ConfigModule } from "@nestjs/config";
 import { DataTargetService } from "@services/data-targets/data-target.service";
 import { OS2IoTMail } from "@services/os2iot-mail.service";
 import { CLIENT_SECRET_PROVIDER, PlainTextClientSecretProvider } from "../../helpers/fiware-token.helper";
+import { DataTargetLogService } from "@services/data-targets/data-target-log.service";
 
 @Module({
   imports: [
@@ -16,10 +18,11 @@ import { CLIENT_SECRET_PROVIDER, PlainTextClientSecretProvider } from "../../hel
     OrganizationModule,
     ConfigModule.forRoot({ load: [configuration] }),
   ],
-  exports: [DataTargetService],
-  controllers: [DataTargetController],
+  exports: [DataTargetService, DataTargetLogService],
+  controllers: [DataTargetController, DatatargetLogController],
   providers: [
     DataTargetService,
+    DataTargetLogService,
     OS2IoTMail,
     {
       provide: CLIENT_SECRET_PROVIDER,

src/modules/shared.module.ts

@@ -1,46 +1,48 @@
 import { Module } from "@nestjs/common";
 import { TypeOrmModule } from "@nestjs/typeorm";
 
+import { ApiKey } from "@entities/api-key.entity";
+import { ApplicationDeviceType } from "@entities/application-device-type.entity";
 import { Application } from "@entities/application.entity";
+import { ControlledProperty } from "@entities/controlled-property.entity";
 import { DataTarget } from "@entities/data-target.entity";
+import { DatatargetLog } from "@entities/datatarget-log.entity";
+import { DeviceModel } from "@entities/device-model.entity";
+import { Downlink } from "@entities/downlink.entity";
+import { FiwareDataTarget } from "@entities/fiware-data-target.entity";
+import { GatewayStatusHistory } from "@entities/gateway-status-history.entity";
+import { Gateway } from "@entities/gateway.entity";
 import { GenericHTTPDevice } from "@entities/generic-http-device.entity";
 import { HttpPushDataTarget } from "@entities/http-push-data-target.entity";
-import { FiwareDataTarget } from "@entities/fiware-data-target.entity";
 import { IoTDevicePayloadDecoderDataTargetConnection } from "@entities/iot-device-payload-decoder-data-target-connection.entity";
 import { IoTDevice } from "@entities/iot-device.entity";
 import { LoRaWANDevice } from "@entities/lorawan-device.entity";
+import { LorawanMulticastDefinition } from "@entities/lorawan-multicast.entity";
+import { MqttDataTarget } from "@entities/mqtt-data-target.entity";
+import { MQTTExternalBrokerDevice } from "@entities/mqtt-external-broker-device.entity";
+import { MQTTInternalBrokerDevice } from "@entities/mqtt-internal-broker-device.entity";
+import { Multicast } from "@entities/multicast.entity";
+import { OpenDataDkDataset } from "@entities/open-data-dk-dataset.entity";
+import { OpenDataDkDataTarget } from "@entities/open-data-dk-push-data-target.entity";
 import { Organization } from "@entities/organization.entity";
 import { PayloadDecoder } from "@entities/payload-decoder.entity";
+import { PermissionTypeEntity } from "@entities/permissions/permission-type.entity";
 import { Permission } from "@entities/permissions/permission.entity";
-import { ReceivedMessage } from "@entities/received-message.entity";
 import { ReceivedMessageMetadata } from "@entities/received-message-metadata.entity";
+import { ReceivedMessageSigFoxSignals } from "@entities/received-message-sigfox-signals.entity";
+import { ReceivedMessage } from "@entities/received-message.entity";
 import { SigFoxDevice } from "@entities/sigfox-device.entity";
 import { SigFoxGroup } from "@entities/sigfox-group.entity";
 import { User } from "@entities/user.entity";
-import { DeviceModel } from "@entities/device-model.entity";
-import { OpenDataDkDataset } from "@entities/open-data-dk-dataset.entity";
 import { AuditLog } from "@services/audit-log.service";
-import { ApiKey } from "@entities/api-key.entity";
-import { Multicast } from "@entities/multicast.entity";
-import { LorawanMulticastDefinition } from "@entities/lorawan-multicast.entity";
-import { ControlledProperty } from "@entities/controlled-property.entity";
-import { ApplicationDeviceType } from "@entities/application-device-type.entity";
-import { ReceivedMessageSigFoxSignals } from "@entities/received-message-sigfox-signals.entity";
-import { MqttDataTarget } from "@entities/mqtt-data-target.entity";
-import { PermissionTypeEntity } from "@entities/permissions/permission-type.entity";
-import { GatewayStatusHistory } from "@entities/gateway-status-history.entity";
-import { OpenDataDkDataTarget } from "@entities/open-data-dk-push-data-target.entity";
-import { MQTTInternalBrokerDevice } from "@entities/mqtt-internal-broker-device.entity";
-import { MQTTExternalBrokerDevice } from "@entities/mqtt-external-broker-device.entity";
-import { Gateway } from "@entities/gateway.entity";
-import { Downlink } from "@entities/downlink.entity";
 
 @Module({
   imports: [
     TypeOrmModule.forFeature([
       User,
       Application,
       DataTarget,
+      DatatargetLog,
       GenericHTTPDevice,
       HttpPushDataTarget,
       FiwareDataTarget,
@@ -70,7 +72,7 @@ import { Downlink } from "@entities/downlink.entity";
       MQTTInternalBrokerDevice,
       MQTTExternalBrokerDevice,
       Gateway,
-      Downlink
+      Downlink,
     ]),
   ],
   providers: [AuditLog],

src/services/data-targets/base-data-target.service.ts

@@ -10,18 +10,20 @@ import { DataTarget } from "@entities/data-target.entity";
 export abstract class BaseDataTargetService {
   protected readonly baseLogger = new Logger(BaseDataTargetService.name);
 
-  success(receiver: string): DataTargetSendStatus {
+  success(receiver: string, statusCode?: number, statusText?: string): DataTargetSendStatus {
     this.baseLogger.debug(`Send to ${receiver} sucessful!`);
-    return { status: SendStatus.OK };
+    return { status: SendStatus.OK, statusCode, statusText };
   }
 
-  failure(receiver: string, errorMessage: string, dataTarget: DataTarget): DataTargetSendStatus {
+  failure(receiver: string, errorMessage: string, dataTarget: DataTarget, statusCode?: number, statusText?: string): DataTargetSendStatus {
     this.baseLogger.error(
       `Datatarget {Id: ${dataTarget.id}, Name: ${dataTarget.name}} Send to ${receiver} failed with error ${errorMessage}`
     );
     return {
       status: SendStatus.ERROR,
       errorMessage: errorMessage.toString(),
+      statusCode,
+      statusText
     };
   }
 }

src/services/data-targets/data-target-kafka-listener.service.ts

@@ -4,14 +4,14 @@ import { IoTDevice } from "@entities/iot-device.entity";
 import { DataTargetType } from "@enum/data-target-type.enum";
 import { KafkaTopic } from "@enum/kafka-topic.enum";
 import { DataTargetSendStatus } from "@interfaces/data-target-send-status.interface";
-import { Injectable, Logger, NotImplementedException } from "@nestjs/common";
+import { Injectable, Logger } from "@nestjs/common";
 import { DataTargetService } from "@services/data-targets/data-target.service";
 import { HttpPushDataTargetService } from "@services/data-targets/http-push-data-target.service";
-import { IoTDevicePayloadDecoderDataTargetConnectionService } from "@services/device-management/iot-device-payload-decoder-data-target-connection.service";
 import { IoTDeviceService } from "@services/device-management/iot-device.service";
 import { AbstractKafkaConsumer } from "@services/kafka/kafka.abstract.consumer";
 import { CombinedSubscribeTo } from "@services/kafka/kafka.decorator";
 import { KafkaPayload } from "@services/kafka/kafka.message";
+import { DataTargetLogService } from "./data-target-log.service";
 import { FiwareDataTargetService } from "./fiware-data-target.service";
 import { MqttDataTargetService } from "./mqtt-data-target.service";
 
@@ -25,7 +25,7 @@ export class DataTargetKafkaListenerService extends AbstractKafkaConsumer {
     private httpPushDataTargetService: HttpPushDataTargetService,
     private fiwareDataTargetService: FiwareDataTargetService,
     private mqttDataTargetService: MqttDataTargetService,
-    private ioTDevicePayloadDecoderDataTargetConnectionService: IoTDevicePayloadDecoderDataTargetConnectionService
+    private dataTargetLogService: DataTargetLogService
   ) {
     super();
   }
@@ -72,22 +72,22 @@ export class DataTargetKafkaListenerService extends AbstractKafkaConsumer {
       if (target.type == DataTargetType.HttpPush) {
         try {
           const status = await this.httpPushDataTargetService.send(target, dto);
-          this.logger.debug(`Sent to HttpPush target: ${JSON.stringify(status)}`);
+          await this.onSendDone(status, DataTargetType.HttpPush, target, dto);
         } catch (err) {
-          this.logger.error(`Error while sending to Http Push DataTarget: ${err}`);
+          await this.onSendError(err, DataTargetType.HttpPush, target, dto);
         }
       } else if (target.type == DataTargetType.Fiware) {
         try {
           const status = await this.fiwareDataTargetService.send(target, dto);
-          this.logger.debug(`Sent to FIWARE target: ${JSON.stringify(status)}`);
+          await this.onSendDone(status, DataTargetType.Fiware, target, dto);
         } catch (err) {
-          this.logger.error(`Error while sending to FIWARE DataTarget: ${err}`);
+          await this.onSendError(err, DataTargetType.Fiware, target, dto);
         }
       } else if (target.type === DataTargetType.MQTT) {
         try {
-          this.mqttDataTargetService.send(target, dto, this.onSendDone);
+          this.mqttDataTargetService.send(target, dto, this.onSendDone, this.onSendError);
         } catch (err) {
-          this.logger.error(`Error while sending to MQTT DataTarget: ${err}`);
+          await this.onSendError(err, DataTargetType.MQTT, target, dto);
         }
       } else if (target.type === DataTargetType.OpenDataDK) {
         // OpenDataDk data targets are handled uniquely and ignored here.
@@ -97,7 +97,24 @@ export class DataTargetKafkaListenerService extends AbstractKafkaConsumer {
     });
   }
 
-  private onSendDone = (status: DataTargetSendStatus, targetType: DataTargetType) => {
+  private onSendDone = async (
+    status: DataTargetSendStatus,
+    targetType: DataTargetType,
+    datatarget: DataTarget,
+    payloadDto: TransformedPayloadDto
+  ) => {
     this.logger.debug(`Sent to ${targetType} target: ${JSON.stringify(status)}`);
+    await this.dataTargetService.updateLastMessageDate(datatarget.id);
+    await this.dataTargetLogService.onSendDone(status, datatarget, payloadDto);
+  };
+  private onSendError = async (
+    err: Error,
+    targetType: DataTargetType,
+    datatarget: DataTarget,
+    payloadDto: TransformedPayloadDto
+  ) => {
+    this.logger.error(`Error while sending to ${targetType} DataTarget: ${err}`);
+    await this.dataTargetService.updateLastMessageDate(datatarget.id);
+    await this.dataTargetLogService.onSendError(err, datatarget, payloadDto);
   };
 }