feat(grpc): add syncing protocol buffers

docker-compose.ci.yml

@@ -22,11 +22,11 @@ services:
     environment:
       DB_TYPE: "${DB_TYPE}"
       CACHE_TYPE: "${CACHE_TYPE}"
+      SERVICE_PROXY_TYPE: "${SERVICE_PROXY_TYPE}"
     container_name: server-ci
     ports:
       - 3123:3000
       - 3125:3104
-      - 50051:50051
     volumes:
       - ./logs:/var/lib/server/logs
     networks:

docker/docker-entrypoint.sh

@@ -14,6 +14,10 @@ if [ -z "$SYNCING_SERVER_PORT" ]; then
   export SYNCING_SERVER_PORT=3101
 fi
 
+if [ -z "$SYNCING_SERVER_GRPC_PORT" ]; then
+  export SYNCING_SERVER_GRPC_PORT=50052
+fi
+
 if [ -z "$AUTH_SERVER_PORT" ]; then
   export AUTH_SERVER_PORT=3103
 fi
@@ -356,6 +360,7 @@ export API_GATEWAY_NODE_ENV=production
 export API_GATEWAY_VERSION=local
 
 export API_GATEWAY_SYNCING_SERVER_JS_URL=http://localhost:$SYNCING_SERVER_PORT
+export API_GATEWAY_SYNCING_SERVER_GRPC_URL=0.0.0.0:$SYNCING_SERVER_GRPC_PORT
 export API_GATEWAY_AUTH_SERVER_URL=http://localhost:$AUTH_SERVER_PORT
 export API_GATEWAY_AUTH_SERVER_GRPC_URL=0.0.0.0:$AUTH_SERVER_GRPC_PORT
 export API_GATEWAY_REVISIONS_SERVER_URL=http://localhost:$REVISIONS_SERVER_PORT

packages/api-gateway/.env.sample

@@ -6,6 +6,7 @@ VERSION=development
 PORT=3000
 
 SYNCING_SERVER_JS_URL=http://syncing_server_js:3000
+SYNCING_SERVER_GRPC_URL=http://syncing_server_js:50052
 AUTH_SERVER_URL=http://auth:3000
 AUTH_SERVER_GRPC_URL=http://auth:50051
 WEB_SOCKET_SERVER_URL=http://websockets:3000

packages/api-gateway/package.json

@@ -26,7 +26,7 @@
     "start": "yarn node dist/bin/server.js"
   },
   "dependencies": {
-    "@grpc/grpc-js": "^1.9.10",
+    "@grpc/grpc-js": "^1.9.11",
     "@standardnotes/domain-core": "workspace:^",
     "@standardnotes/domain-events": "workspace:*",
     "@standardnotes/domain-events-infra": "workspace:*",

packages/api-gateway/src/Bootstrap/Container.ts

@@ -16,14 +16,25 @@ import { RedisCrossServiceTokenCache } from '../Infra/Redis/RedisCrossServiceTok
 import { WebSocketAuthMiddleware } from '../Controller/WebSocketAuthMiddleware'
 import { InMemoryCrossServiceTokenCache } from '../Infra/InMemory/InMemoryCrossServiceTokenCache'
 import { DirectCallServiceProxy } from '../Service/DirectCall/DirectCallServiceProxy'
-import { ServiceContainerInterface } from '@standardnotes/domain-core'
+import { MapperInterface, ServiceContainerInterface } from '@standardnotes/domain-core'
 import { EndpointResolverInterface } from '../Service/Resolver/EndpointResolverInterface'
 import { EndpointResolver } from '../Service/Resolver/EndpointResolver'
 import { RequiredCrossServiceTokenMiddleware } from '../Controller/RequiredCrossServiceTokenMiddleware'
 import { OptionalCrossServiceTokenMiddleware } from '../Controller/OptionalCrossServiceTokenMiddleware'
 import { Transform } from 'stream'
-import { ISessionsClient, SessionsClient } from '@standardnotes/grpc'
+import {
+  ISessionsClient,
+  ISyncingClient,
+  SessionsClient,
+  SyncRequest,
+  SyncResponse,
+  SyncingClient,
+} from '@standardnotes/grpc'
 import { GRPCServiceProxy } from '../Service/gRPC/GRPCServiceProxy'
+import { GRPCSyncingServerServiceProxy } from '../Service/gRPC/GRPCSyncingServerServiceProxy'
+import { SyncResponseHttpRepresentation } from '../Mapping/Sync/Http/SyncResponseHttpRepresentation'
+import { SyncRequestGRPCMapper } from '../Mapping/Sync/GRPC/SyncRequestGRPCMapper'
+import { SyncResponseGRPCMapper } from '../Mapping/Sync/GRPC/SyncResponseGRPCMapper'
 
 export class ContainerConfigLoader {
   async load(configuration?: {
@@ -145,6 +156,7 @@ export class ContainerConfigLoader {
       const isConfiguredForGRPCProxy = env.get('SERVICE_PROXY_TYPE', true) === 'grpc'
       if (isConfiguredForGRPCProxy) {
         container.bind(TYPES.ApiGateway_AUTH_SERVER_GRPC_URL).toConstantValue(env.get('AUTH_SERVER_GRPC_URL'))
+        container.bind(TYPES.ApiGateway_SYNCING_SERVER_GRPC_URL).toConstantValue(env.get('SYNCING_SERVER_GRPC_URL'))
         const grpcAgentKeepAliveTimeout = env.get('GRPC_AGENT_KEEP_ALIVE_TIMEOUT', true)
           ? +env.get('GRPC_AGENT_KEEP_ALIVE_TIMEOUT', true)
           : 8_000
@@ -158,6 +170,35 @@ export class ContainerConfigLoader {
             },
           ),
         )
+        container.bind<ISyncingClient>(TYPES.ApiGateway_GRPCSyncingClient).toConstantValue(
+          new SyncingClient(
+            container.get<string>(TYPES.ApiGateway_SYNCING_SERVER_GRPC_URL),
+            grpc.credentials.createInsecure(),
+            {
+              'grpc.keepalive_time_ms': grpcAgentKeepAliveTimeout * 2,
+              'grpc.keepalive_timeout_ms': grpcAgentKeepAliveTimeout,
+            },
+          ),
+        )
+
+        container
+          .bind<MapperInterface<Record<string, unknown>, SyncRequest>>(TYPES.Mapper_SyncRequestGRPCMapper)
+          .toConstantValue(new SyncRequestGRPCMapper())
+        container
+          .bind<MapperInterface<SyncResponse, SyncResponseHttpRepresentation>>(TYPES.Mapper_SyncResponseGRPCMapper)
+          .toConstantValue(new SyncResponseGRPCMapper())
+
+        container
+          .bind<GRPCSyncingServerServiceProxy>(TYPES.ApiGateway_GRPCSyncingServerServiceProxy)
+          .toConstantValue(
+            new GRPCSyncingServerServiceProxy(
+              container.get<ISyncingClient>(TYPES.ApiGateway_GRPCSyncingClient),
+              container.get<MapperInterface<Record<string, unknown>, SyncRequest>>(TYPES.Mapper_SyncRequestGRPCMapper),
+              container.get<MapperInterface<SyncResponse, SyncResponseHttpRepresentation>>(
+                TYPES.Mapper_SyncResponseGRPCMapper,
+              ),
+            ),
+          )
         container
           .bind<ServiceProxyInterface>(TYPES.ApiGateway_ServiceProxy)
           .toConstantValue(
@@ -175,6 +216,7 @@ export class ContainerConfigLoader {
               container.get<winston.Logger>(TYPES.ApiGateway_Logger),
               container.get<TimerInterface>(TYPES.ApiGateway_Timer),
               container.get<ISessionsClient>(TYPES.ApiGateway_GRPCSessionsClient),
+              container.get<GRPCSyncingServerServiceProxy>(TYPES.ApiGateway_GRPCSyncingServerServiceProxy),
             ),
           )
       } else {

packages/api-gateway/src/Bootstrap/Types.ts

@@ -6,6 +6,7 @@ export const TYPES = {
   ApiGateway_SYNCING_SERVER_JS_URL: Symbol.for('ApiGateway_SYNCING_SERVER_JS_URL'),
   ApiGateway_AUTH_SERVER_URL: Symbol.for('ApiGateway_AUTH_SERVER_URL'),
   ApiGateway_AUTH_SERVER_GRPC_URL: Symbol.for('ApiGateway_AUTH_SERVER_GRPC_URL'),
+  ApiGateway_SYNCING_SERVER_GRPC_URL: Symbol.for('ApiGateway_SYNCING_SERVER_GRPC_URL'),
   ApiGateway_PAYMENTS_SERVER_URL: Symbol.for('ApiGateway_PAYMENTS_SERVER_URL'),
   ApiGateway_FILES_SERVER_URL: Symbol.for('ApiGateway_FILES_SERVER_URL'),
   ApiGateway_REVISIONS_SERVER_URL: Symbol.for('ApiGateway_REVISIONS_SERVER_URL'),
@@ -24,10 +25,15 @@ export const TYPES = {
   ApiGateway_OptionalCrossServiceTokenMiddleware: Symbol.for('ApiGateway_OptionalCrossServiceTokenMiddleware'),
   ApiGateway_WebSocketAuthMiddleware: Symbol.for('ApiGateway_WebSocketAuthMiddleware'),
   ApiGateway_SubscriptionTokenAuthMiddleware: Symbol.for('ApiGateway_SubscriptionTokenAuthMiddleware'),
+  // Mapping
+  Mapper_SyncRequestGRPCMapper: Symbol.for('Mapper_SyncRequestGRPCMapper'),
+  Mapper_SyncResponseGRPCMapper: Symbol.for('Mapper_SyncResponseGRPCMapper'),
   // Services
+  ApiGateway_GRPCSyncingServerServiceProxy: Symbol.for('ApiGateway_GRPCSyncingServerServiceProxy'),
   ApiGateway_ServiceProxy: Symbol.for('ApiGateway_ServiceProxy'),
   ApiGateway_CrossServiceTokenCache: Symbol.for('ApiGateway_CrossServiceTokenCache'),
   ApiGateway_Timer: Symbol.for('ApiGateway_Timer'),
   ApiGateway_EndpointResolver: Symbol.for('ApiGateway_EndpointResolver'),
   ApiGateway_GRPCSessionsClient: Symbol.for('ApiGateway_GRPCSessionsClient'),
+  ApiGateway_GRPCSyncingClient: Symbol.for('ApiGateway_GRPCSyncingClient'),
 }

packages/api-gateway/src/Mapping/Sync/GRPC/SyncRequestGRPCMapper.ts

@@ -0,0 +1,94 @@
+import { MapperInterface, Validator } from '@standardnotes/domain-core'
+import { ItemHash, SyncRequest } from '@standardnotes/grpc'
+
+export class SyncRequestGRPCMapper implements MapperInterface<Record<string, unknown>, SyncRequest> {
+  toDomain(_projection: SyncRequest): Record<string, unknown> {
+    throw new Error('Method not implemented.')
+  }
+
+  toProjection(domain: Record<string, unknown>): SyncRequest {
+    const syncRequest = new SyncRequest()
+    if ('items' in domain) {
+      syncRequest.setItemsList((domain.items as Record<string, unknown>[]).map((item) => this.createItemHash(item)))
+    }
+
+    if ('shared_vault_uuids' in domain) {
+      const sharedVaultUuidsValidation = Validator.isNotEmpty(domain.shared_vault_uuids)
+      if (!sharedVaultUuidsValidation.isFailed()) {
+        syncRequest.setSharedVaultUuidsList(domain.shared_vault_uuids as string[])
+      }
+    }
+
+    if ('compute_integrity' in domain) {
+      syncRequest.setComputeIntegrity(!!domain.compute_integrity)
+    }
+
+    if ('sync_token' in domain) {
+      syncRequest.setSyncToken(domain.sync_token as string)
+    }
+
+    if ('cursor_token' in domain) {
+      syncRequest.setCursorToken(domain.cursor_token as string)
+    }
+
+    if ('limit' in domain) {
+      syncRequest.setLimit(domain.limit as number)
+    }
+
+    if ('content_type' in domain) {
+      syncRequest.setContentType(domain.content_type as string)
+    }
+
+    if ('api' in domain) {
+      syncRequest.setApiVersion(domain.api as string)
+    }
+
+    return syncRequest
+  }
+
+  private createItemHash(record: Record<string, unknown>): ItemHash {
+    const itemHash = new ItemHash()
+    itemHash.setUuid(record.uuid as string)
+    if (record.content) {
+      itemHash.setContent(record.content as string)
+    }
+    if (record.content_type) {
+      itemHash.setContentType(record.content_type as string)
+    }
+    if (record.deleted !== undefined) {
+      itemHash.setDeleted(!!record.deleted)
+    }
+    if (record.duplicate_of) {
+      itemHash.setDuplicateOf(record.duplicate_of as string)
+    }
+    if (record.auth_hash) {
+      itemHash.setAuthHash(record.auth_hash as string)
+    }
+    if (record.enc_item_key) {
+      itemHash.setEncItemKey(record.enc_item_key as string)
+    }
+    if (record.items_key_id) {
+      itemHash.setItemsKeyId(record.items_key_id as string)
+    }
+    if (record.key_system_identifier) {
+      itemHash.setKeySystemIdentifier(record.key_system_identifier as string)
+    }
+    if (record.shared_vault_uuid) {
+      itemHash.setSharedVaultUuid(record.shared_vault_uuid as string)
+    }
+    if (record.created_at) {
+      itemHash.setCreatedAt(record.created_at as string)
+    }
+    if (record.created_at_timestamp) {
+      itemHash.setCreatedAtTimestamp(record.created_at_timestamp as number)
+    }
+    if (record.updated_at) {
+      itemHash.setUpdatedAt(record.updated_at as string)
+    }
+    if (record.updated_at_timestamp) {
+      itemHash.setUpdatedAtTimestamp(record.updated_at_timestamp as number)
+    }
+
+    return itemHash
+  }
+}