fix

.env.production

@@ -0,0 +1,2 @@
+ENVIRONMENT=production
+DIALECT_SDK_ENVIRONMENT=production

.github/workflows/main.yaml

@@ -63,7 +63,7 @@ jobs:
       image_tag: ${{ steps.get-image-tag.outputs.image_tag }}
 
   cd_production:
-    if: ${{ contains(github.ref, 'heads/master') }}
+#    if: ${{ contains(github.ref, 'heads/master') }}
     needs:
       - docker-image
     uses: ./.github/workflows/cd.yaml

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:16-alpine
+FROM node:20-alpine3.18
 
 WORKDIR /app
 

package.json

@@ -13,6 +13,7 @@
     "start:dev": "dotenv -e .env.dev -e .env.service -- nest start --watch",
     "start:dev:debug": "dotenv -e .env.dev -e .env.service -- nest start --debug --watch",
     "start:local-dev": "dotenv -e .env.local-dev -e .env.service -- nest start --watch",
+    "start:production": "dotenv -e .env.production -e .env.service -- nest start --watch",
     "start:local-dev:debug": "dotenv -e .env.local-dev -e .env.service -- nest start --debug --watch",
     "client:start:dev": "dotenv -e .env.dev -e .env.client -- ts-node test/client.ts",
     "client:start:local-dev": "dotenv -e .env.local-dev -e .env.client -- ts-node test/client.ts"
@@ -24,14 +25,16 @@
     "@nestjs/common": "^9.0.0",
     "@nestjs/config": "^2.2.0",
     "@nestjs/core": "^9.0.0",
+    "@nestjs/event-emitter": "^1.3.1",
     "@nestjs/platform-express": "^9.0.0",
     "@nestjs/schedule": "^2.1.0",
     "@nestjs/terminus": "^9.1.0",
-    "@nestjs/event-emitter": "^1.3.1",
-    "@solana/spl-governance": "^0.0.34",
-    "@solana/spl-token": "0.1.8",
+    "@solana/spl-governance": "^0.3.28",
+    "@solana/spl-token": "^0.4.3",
     "@solana/spl-token-registry": "^0.2.3775",
-    "bn.js": "^5.2.1",
+    "bn.js": "^5.1.3",
+    "borsh": "^0.3.1",
+    "bs58": "^4.0.1",
     "lodash": "^4.17.21",
     "luxon": "^3.0.1",
     "nestjs-pino": "^2.6.0",
@@ -47,6 +50,7 @@
     "@nestjs/schematics": "^9.0.0",
     "@nestjs/testing": "^9.0.0",
     "@types/bn.js": "^5.1.0",
+    "@types/bs58": "^4.0.4",
     "@types/cron": "^2.0.0",
     "@types/express": "^4.17.13",
     "@types/jest": "28.1.4",
@@ -56,6 +60,7 @@
     "@types/supertest": "^2.0.11",
     "@typescript-eslint/eslint-plugin": "^5.0.0",
     "@typescript-eslint/parser": "^5.0.0",
+    "dotenv-cli": "^7.4.1",
     "eslint": "^8.0.1",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-prettier": "^4.0.0",

src/app.module.ts

@@ -11,7 +11,7 @@ import { DialectSdk } from './dialect-sdk';
 import { ConfigModule } from '@nestjs/config';
 import { RealmsRestService } from './realms-rest-service';
 import { RealmsService } from './realms.service';
-import { RealmsRepository } from './realms-repository';
+import { RealmsCache } from './realms-cache';
 import { ScheduleModule } from '@nestjs/schedule';
 import { NewProposalsMonitoringService } from './new-proposals-monitoring.service';
 import { ProposalStateChangeMonitoringService } from './proposal-state-monitoring.service';
@@ -45,9 +45,9 @@ import { EventEmitterModule } from '@nestjs/event-emitter';
   ],
   controllers: [HealthController],
   providers: [
+    RealmsCache,
     CachingHealth,
     RealmsRestService,
-    RealmsRepository,
     RealmsService,
     NewProposalsMonitoringService,
     ProposalStateChangeMonitoringService,

src/caching.health.ts

@@ -4,61 +4,80 @@ import {
   HealthIndicatorResult,
 } from '@nestjs/terminus';
 import { Injectable, Logger } from '@nestjs/common';
-import { OnEvent } from '@nestjs/event-emitter';
-
-export interface CachingEvent {
-  type: CachingEventType;
-}
-
-export interface CachingStartedEvent extends CachingEvent {
-  type: CachingEventType.Started;
-  timeStarted: number;
-}
-
-export interface CachingFinishedEvent extends CachingEvent {
-  type: CachingEventType.Finished;
-}
-
-export enum CachingEventType {
-  Started = 'caching.started',
-  Finished = 'caching.finished',
-}
+import { RealmsCache } from './realms-cache';
 
 @Injectable()
 export class CachingHealth extends HealthIndicator {
-  private static readonly MAX_CACHING_EXECUTION_TIME_MILLIS = process.env
-    .MAX_CACHING_EXECUTION_TIME_MILLIS
-    ? parseInt(process.env.MAX_CACHING_EXECUTION_TIME_MILLIS, 10)
-    : 600000;
+  private static MAX_STATIC_ACCOUNT_CACHE_AGE_MILLIS =
+    process.env.MAX_CACHE_AGE_MILLIS ?? 3 * 60 * 60 * 1000;
+
+  private static MAX_DYNAMIC_ACCOUNT_CACHE_AGE_MILLIS =
+    process.env.MAX_CACHE_AGE_MILLIS ?? 30 * 60 * 1000;
+
   private readonly logger = new Logger(CachingHealth.name);
-  private lastStartedCaching: number;
-  private cachingInProgress = false;
+
+  constructor(private readonly realmsCache: RealmsCache) {
+    super();
+  }
 
   public isHealthy(): HealthIndicatorResult {
-    const isHealthy = this.cachingInProgress
-      ? Date.now() - this.lastStartedCaching <
-        CachingHealth.MAX_CACHING_EXECUTION_TIME_MILLIS
-      : true;
-    if (isHealthy) {
-      return this.getStatus('caching', isHealthy);
+    if (this.realmsCache.initializationError) {
+      this.logger.error(
+        'Caching health check failed. Service needs to be restarted, because initialization failed',
+      );
+      throw new HealthCheckError(
+        'Caching failed',
+        this.getStatus('caching', false),
+      );
     }
-    this.logger.error(
-      'Caching health check failed. Service needs to be restarted',
-    );
-    throw new HealthCheckError(
-      'Caching failed',
-      this.getStatus('caching', isHealthy),
-    );
-  }
 
-  @OnEvent(CachingEventType.Started)
-  onCachingStarted({ timeStarted }: CachingStartedEvent) {
-    this.lastStartedCaching = timeStarted;
-    this.cachingInProgress = true;
-  }
+    if (!this.realmsCache.isInitialized) {
+      return this.getStatus('caching', true);
+    }
+
+    if (
+      !this.realmsCache.lastDynamicAccountCachingSuccessFinishedAt ||
+      !this.realmsCache.lastStaticAccountCachingSuccessFinishedAt
+    ) {
+      this.logger.error(
+        `Some of the cache ages are not initialized, this should not happen. 
+  Static cache age: ${this.realmsCache.lastStaticAccountCachingSuccessFinishedAt}, dynamic cache age: ${this.realmsCache.lastDynamicAccountCachingSuccessFinishedAt}
+  Service needs to be restarted`,
+      );
+      throw new HealthCheckError(
+        'Caching failed',
+        this.getStatus('caching', false),
+      );
+    }
+
+    const staticCacheAge =
+      Date.now() -
+      this.realmsCache.lastStaticAccountCachingSuccessFinishedAt.getTime();
+
+    if (staticCacheAge > CachingHealth.MAX_STATIC_ACCOUNT_CACHE_AGE_MILLIS) {
+      this.logger.error(
+        `Static cache age is too old: ${staticCacheAge}, service needs to be restarted`,
+      );
+      throw new HealthCheckError(
+        'Caching failed',
+        this.getStatus('caching', false),
+      );
+    }
+
+    const dynamicCacheAge =
+      Date.now() -
+      this.realmsCache.lastDynamicAccountCachingSuccessFinishedAt.getTime();
+
+    if (dynamicCacheAge > CachingHealth.MAX_DYNAMIC_ACCOUNT_CACHE_AGE_MILLIS) {
+      this.logger.error(
+        `Dynamic cache age is too old: ${dynamicCacheAge}, service needs to be restarted`,
+      );
+      throw new HealthCheckError(
+        'Caching failed',
+        this.getStatus('caching', false),
+      );
+    }
 
-  @OnEvent(CachingEventType.Finished)
-  onCachingFinished() {
-    this.cachingInProgress = false;
+    return this.getStatus('caching', true);
   }
 }

src/dialect-sdk.ts

@@ -8,9 +8,9 @@ import {
 } from '@dialectlabs/sdk';
 
 export abstract class DialectSdk implements IDialectSdk {
-  readonly dapps: Dapps;
-  readonly info: DialectSdkInfo;
-  readonly threads: Messaging;
-  readonly wallet: Wallets;
-  readonly identity: IdentityResolver;
+  readonly dapps!: Dapps;
+  readonly info!: DialectSdkInfo;
+  readonly threads!: Messaging;
+  readonly wallet!: Wallets;
+  readonly identity!: IdentityResolver;
 }

src/health.controller.ts

@@ -10,12 +10,12 @@ import { CachingHealth } from './caching.health';
 export class HealthController {
   constructor(
     private health: HealthCheckService,
-    private readonly dataIngestionHealth: CachingHealth,
+    private readonly cachingHealth: CachingHealth,
   ) {}
 
   @Get()
   @HealthCheck()
   check() {
-    return this.health.check([() => this.dataIngestionHealth.isHealthy()]);
+    return this.health.check([() => this.cachingHealth.isHealthy()]);
   }
 }

src/main.ts

@@ -1,14 +1,17 @@
 import { NestFactory } from '@nestjs/core';
 import { AppModule } from './app.module';
 import { Logger } from 'nestjs-pino';
-import { VersioningType } from '@nestjs/common';
+import { INestApplication, VersioningType } from '@nestjs/common';
+import { Server } from 'http';
 
-export const NOTIF_TYPE_ID_PROPOSALS = '04827917-dde4-48c7-bf1b-780b77895e97'
+export const NOTIF_TYPE_ID_PROPOSALS = '04827917-dde4-48c7-bf1b-780b77895e97';
 
 async function bootstrap() {
   const app = await NestFactory.create(AppModule, {
     logger: ['log', 'warn', 'error'],
   });
+  configureHttpServer(app);
+  configureUnhandledErrorsHandling();
   app.setGlobalPrefix('api');
   app.enableVersioning({
     type: VersioningType.URI,
@@ -24,4 +27,22 @@ async function bootstrap() {
   await app.listen(process.env.PORT ?? 0);
 }
 
+function configureHttpServer(app: INestApplication) {
+  // https://shuheikagawa.com/blog/2019/04/25/keep-alive-timeout/
+  // ALB has default timeout of 60 seconds
+  const httpAdapter = app.getHttpAdapter();
+  const server: Server = httpAdapter.getHttpServer();
+  server.keepAliveTimeout = 61 * 1000;
+  server.headersTimeout = 65 * 1000;
+}
+
+function configureUnhandledErrorsHandling() {
+  process.on('unhandledRejection', (error) => {
+    console.error(error);
+  });
+  process.on('uncaughtException', (error) => {
+    console.error(error);
+  });
+}
+
 bootstrap();