-
Notifications
You must be signed in to change notification settings - Fork 0
/
index.ts
382 lines (327 loc) · 10.6 KB
/
index.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
import { PubSub as GooglePubSub } from '@google-cloud/pubsub'
import { JSON, base64ToParsedJSON } from './json'
type Base64String = string
type MessageId = string
type MessageKey = string
/**
* This type represents a message whose data has
* not yet been decoded
*
* At this point, you don't yet know if the base64 encoded
* data conforms to the shape you expect for a particular subscription
*
* Secondly, you don't know if the subscription is recognizeable
*/
export interface UnprocessedPubSubMessage {
subscription: string
message: {
messageId: MessageId
data: Base64String
}
}
/**
* This type represents a valid pubsub message
*
* A valid pubsub message has a recognized subscription name, and
* the data conforms to a schema (and whose data came from JSON)
*/
export interface PubSubMessage<T> {
messageId: MessageId
data: T
}
export enum SubscriptionError {
InvalidSubscription = 'invalid_subscription',
InvalidEventData = 'invalid_event_data',
MissingHandlerForSubscription = 'missing_handler_for_subscription',
HandlerFailedToProcessMessage = 'handler_failed_to_process_message',
}
export enum EventStatus {
InProgress = 'in_progress',
Completed = 'completed',
Failed = 'failed',
}
export interface PubSubEvent {
idempotency_key: MessageKey
last_run_at: Date
created_at: Date
attempt_statuses: EventStatus[]
last_failure_reason?: string
subscription: string
base64_event_data: Base64String
}
/**
* This interface represents any state manager
*
* If you want to handle state in a more robust way than
* in-memory, then you would implement this interface:
*
* ```typescript
* class PostgresStateManager implements StateManager { ... }
* ```
*/
export interface StateManager {
getPubSubEvent(
messageId: MessageId,
subscription: string,
): Promise<PubSubEvent | undefined>
recordMessageReceived(
rawMessage: UnprocessedPubSubMessage,
subscription: string,
cachedEvent?: PubSubEvent,
): Promise<PubSubEvent>
recordMessageProcessingOutcome(
cachedEvent: PubSubEvent,
outcome: EventStatus,
failureReason?: FailureReason,
): Promise<void>
}
/**
* Exported for testing purposes, you do not need to ever import this class
* as it is being used automatically if you instantiate PubSub without your own
* state manager (the third argument in the PubSub constructor)
*/
export class InMemoryStateManager implements StateManager {
private cache: Map<MessageKey, PubSubEvent>
constructor() {
if (process.env.NODE_ENV !== 'test') {
const warning = [
'[a1pubsub]',
'Currently using the `InMemoryStateManager`',
'This state manager is only suitable for single-instance applications',
].join(' - ')
console.info(warning)
}
this.cache = new Map()
}
buildKey(messageId: MessageId, subscription: string): MessageKey {
return [messageId, '-', subscription].join('')
}
async getPubSubEvent(
messageId: MessageId,
subscription: string,
): Promise<PubSubEvent | undefined> {
return this.cache.get(this.buildKey(messageId, subscription))
}
async recordMessageReceived(
rawMessage: UnprocessedPubSubMessage,
subscription: string,
cachedEvent?: PubSubEvent,
): Promise<PubSubEvent> {
const today = new Date()
if (cachedEvent) {
const event = {
...cachedEvent,
// eslint-disable-next-line
last_run_at: today,
}
this.cache.set(cachedEvent.idempotency_key, event)
return event
} else {
/* eslint-disable @typescript-eslint/camelcase */
const event = {
idempotency_key: this.buildKey(rawMessage.message.messageId, subscription),
last_run_at: today,
created_at: today,
attempt_statuses: [EventStatus.InProgress],
subscription: subscription,
base64_event_data: rawMessage.message.data,
}
/* eslint-enable */
this.cache.set(event.idempotency_key, event)
return event
}
}
async recordMessageProcessingOutcome(
cachedEvent: PubSubEvent,
outcome: EventStatus,
failureReason?: FailureReason,
): Promise<void> {
/* eslint-disable @typescript-eslint/camelcase */
this.cache.set(cachedEvent.idempotency_key, {
...cachedEvent,
attempt_statuses: cachedEvent.attempt_statuses.concat(outcome),
last_failure_reason: failureReason || cachedEvent.last_failure_reason,
})
/* eslint-enable */
}
}
/**
* Represents the outcome of your `handler` inside of a `SubscriptionHandler`
*/
export enum HandlerResult {
FailedToProcess = 'failed_to_process',
Success = 'success',
}
type FailureReason = string
/**
* Takes deserialized and untyped JSON and either returns data that conforms
* to a schema or nothing (to represent failed validation)
*
* Under the hood, the `a1pubsub` module will catch any errors within this function,
* so you do not need to implement error handling.
*/
type Validator<T> = (json: JSON) => T | undefined
/**
* The function that actually processes your validated pubsub event
*
* If this function returns `HandlerResult.FailedToProcess` then the retry
* mechanism as described in the GCP PubSub documentation will kick into place
*/
export type MessageHandler<D extends {}> = (
event: PubSubEvent,
subscription: string,
data: PubSubMessage<D>,
) => Promise<[HandlerResult] | [HandlerResult, FailureReason]>
export interface SubscriptionHandler<D extends {} = {}> {
validator: Validator<D>
handler: MessageHandler<D>
}
/* eslint-disable @typescript-eslint/no-empty-interface, @typescript-eslint/no-explicit-any */
export interface SubscriptionMap extends Record<string, SubscriptionHandler<any>> {}
const subscriptionRe = /^projects\/[a-z-]+\d*\/subscriptions\/(.+)$/
// converts projects/myproject/subscriptions/mysubscription into mysubscription
export const getSubscription = (rawSubscription: string): string | null => {
const parsed = subscriptionRe.exec(rawSubscription)
if (!parsed) {
return null
}
return parsed[1]
}
type Either<T, E> = { type: 'ok'; data: T } | { type: 'error'; error: E }
// the developer could throw an exception
// hence we call the validator in a try / catch context
const handleValidator = <T>(
validator: Validator<T>,
data: JSON,
): Either<T, SubscriptionError.InvalidEventData> => {
try {
const validatedData = validator(data)
if (validatedData) {
return { type: 'ok', data: validatedData }
} else {
return { type: 'error', error: SubscriptionError.InvalidEventData }
}
} catch (e) {
return { type: 'error', error: SubscriptionError.InvalidEventData }
}
}
export class PubSub {
private projectId: string
private stateManager: StateManager
private decoders: SubscriptionMap
constructor(
projectId: string,
decodingTable: SubscriptionMap,
customStateManager?: StateManager,
) {
this.projectId = projectId
this.stateManager = customStateManager
? customStateManager
: new InMemoryStateManager()
this.decoders = decodingTable
}
/**
* Publish an event to a topic
*
* This method assumes that your data is serializeable to JSON
*
* @param topic the event stream in GCP PubSub
* @param data the data associated with this event stream
*/
async publish<J extends {}>(topic: string, data: J): Promise<void> {
// GCP client libraries use a strategy called Application Default Credentials (ADC)
// to find the application's credentials.
// more info: https://cloud.google.com/docs/authentication/production
const pubSub = new GooglePubSub({
projectId: this.projectId,
})
await pubSub.topic(topic).publishJSON(data)
}
/**
* This is the entrypoint for all pubsub subscription events
*
* @param rawMsg An unprocessed pubsub message that must first be validated
*/
async handlePubSubMessage(
rawMsg: UnprocessedPubSubMessage,
): Promise<{ error: SubscriptionError; reason?: string } | undefined> {
const subscription = getSubscription(rawMsg.subscription)
if (!subscription) {
return {
error: SubscriptionError.InvalidSubscription,
reason: `Subscription "${subscription}" doesn't follow the "projects/<GCP_PROJECT_NAME>/subscriptions/<SUBSCRIPTION_NAME>" pattern`,
}
}
const cachedMessage = await this.stateManager.getPubSubEvent(
rawMsg.message.messageId,
subscription,
)
if (cachedMessage && cachedMessage.attempt_statuses.length > 0) {
const lastIndex = cachedMessage.attempt_statuses.length - 1
const mostRecentStatus = cachedMessage.attempt_statuses[lastIndex]
if (mostRecentStatus === EventStatus.Completed) {
// idempotency :)
return
}
}
const updatedCachedMessage = await this.stateManager.recordMessageReceived(
rawMsg,
subscription,
cachedMessage,
)
const subscriptionHandler = this.decoders[subscription]
if (!subscriptionHandler) {
return {
error: SubscriptionError.MissingHandlerForSubscription,
reason: `Subscription "${subscription}" doesn't have a corresponding handler`,
}
}
const { validator, handler } = subscriptionHandler
const untypedJsonData = base64ToParsedJSON(rawMsg.message.data)
const decodedMessage = handleValidator(validator, untypedJsonData)
if (decodedMessage.type === 'error') {
return {
error: decodedMessage.error,
reason: `Data for subscription "${subscription}" did not pass validation. Data: ${untypedJsonData}`,
}
}
const pubSubMessage = {
messageId: rawMsg.message.messageId,
data: decodedMessage.data,
}
const [handlerResult, failureReason] = await handler(
updatedCachedMessage,
subscription,
pubSubMessage,
).catch((err: unknown) => {
// catching in case handler didn't catch its own errors
const errorMessage = [
'uncaught promise rejection within handler',
`Error: ${err}`,
].join(' - ')
return [HandlerResult.FailedToProcess, errorMessage] as [
HandlerResult,
FailureReason,
]
})
if (handlerResult === HandlerResult.Success) {
this.stateManager.recordMessageProcessingOutcome(
updatedCachedMessage,
EventStatus.Completed,
)
return
} else {
this.stateManager.recordMessageProcessingOutcome(
updatedCachedMessage,
EventStatus.Failed,
failureReason,
)
return {
error: SubscriptionError.HandlerFailedToProcessMessage,
reason: `Handler for subscription "${subscription}" failed to process successfully. `.concat(
failureReason ? `Failure reason: ${failureReason}` : '',
),
}
}
}
}