Skip to content
This repository has been archived by the owner on Apr 13, 2023. It is now read-only.

Commit

Permalink
feat: add config options to DdbToEsSync (#108)
Browse files Browse the repository at this point in the history 
config options allow:
- overriding the ES client,
- disabling index/alias creation
- using custom alias/index names
  • Loading branch information
@carvantes
carvantes committed Aug 30, 2021
1 parent fbe19c7 commit f9c9414
Show file tree
Hide file tree
Showing 6 changed files with 456 additions and 138 deletions.
33 changes: 8 additions & 25 deletions src/ddbToEs/ddbToEsHelper.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,14 +214,14 @@ describe('DdbToEsHelper', () => {
};

// TEST
const result: ESBulkCommand = ddbToEsHelper.createBulkESDelete(ddbImage);
const result: ESBulkCommand = ddbToEsHelper.createBulkESDelete(ddbImage, 'someAlias');
// VALIDATE
const expectedOutput: ESBulkCommand = {
id: compositeId,
type: 'delete',
bulkCommand: [
{
delete: { _index: `${resourceType.toLowerCase()}-alias`, _id: compositeId },
delete: { _index: `someAlias`, _id: compositeId },
},
],
};
Expand All @@ -244,13 +244,13 @@ describe('DdbToEsHelper', () => {
const ddbImageCopy = { ...ddbImage, documentStatus: 'AVAILABLE' };

// TEST
const result: ESBulkCommand | null = ddbToEsHelper.createBulkESUpsert(ddbImageCopy);
const result: ESBulkCommand | null = ddbToEsHelper.createBulkESUpsert(ddbImageCopy, 'someAlias');
// VALIDATE
const expectedOutput: ESBulkCommand = {
id: compositeId,
type: 'upsert-AVAILABLE',
bulkCommand: [
{ update: { _index: `${resourceType.toLowerCase()}-alias`, _id: compositeId } },
{ update: { _index: 'someAlias', _id: compositeId } },
{ doc: ddbImageCopy, doc_as_upsert: true },
],
};
Expand All @@ -261,13 +261,13 @@ describe('DdbToEsHelper', () => {
const ddbImageCopy = { ...ddbImage, documentStatus: 'DELETED' };

// TEST
const result: ESBulkCommand | null = ddbToEsHelper.createBulkESUpsert(ddbImageCopy);
const result: ESBulkCommand | null = ddbToEsHelper.createBulkESUpsert(ddbImageCopy, 'someAlias');
// VALIDATE
const expectedOutput: ESBulkCommand = {
id: compositeId,
type: 'upsert-DELETED',
bulkCommand: [
{ update: { _index: `${resourceType.toLowerCase()}-alias`, _id: compositeId } },
{ update: { _index: 'someAlias', _id: compositeId } },
{ doc: ddbImageCopy, doc_as_upsert: true },
],
};
Expand All @@ -278,7 +278,7 @@ describe('DdbToEsHelper', () => {
const ddbImageCopy = { ...ddbImage, documentStatus: 'PENDING' };

// TEST
const result: ESBulkCommand | null = ddbToEsHelper.createBulkESUpsert(ddbImageCopy);
const result: ESBulkCommand | null = ddbToEsHelper.createBulkESUpsert(ddbImageCopy, 'someAlias');
// VALIDATE
expect(result).toBeNull();
});
Expand All @@ -287,7 +287,7 @@ describe('DdbToEsHelper', () => {
const ddbImageCopy = { ...ddbImage, documentStatus: 'LOCKED' };

// TEST
const result: ESBulkCommand | null = ddbToEsHelper.createBulkESUpsert(ddbImageCopy);
const result: ESBulkCommand | null = ddbToEsHelper.createBulkESUpsert(ddbImageCopy, 'someAlias');
// VALIDATE
expect(result).toBeNull();
});
Expand Down Expand Up @@ -512,21 +512,4 @@ describe('DdbToEsHelper', () => {
expect(ddbToEsHelper.isRemoveResource(modifyRecord)).toBeFalsy();
});
});

describe('generateAlias', () => {
it('Simple resource', () => {
const testResource = {
resourceType: 'Patient',
};
expect(ddbToEsHelper.generateAlias(testResource)).toEqual('patient-alias');
});

it('Resource with tenantId', () => {
const testResource = {
resourceType: 'Patient',
_tenantId: 'tenant1',
};
expect(ddbToEsHelper.generateAlias(testResource)).toEqual('patient-alias-tenant-tenant1');
});
});
});
64 changes: 26 additions & 38 deletions src/ddbToEs/ddbToEsHelper.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,27 +35,31 @@ const formatDocument = (ddbImage: any): any => {
return ddbImage;
};

export default class DdbToEsHelper {
public ElasticSearch: Client;
const getDefaultESClientFromEnvVars: () => Client = () => {
let ES_DOMAIN_ENDPOINT = ELASTICSEARCH_DOMAIN_ENDPOINT || 'https://fake-es-endpoint.com';
if (IS_OFFLINE === 'true') {
const { ACCESS_KEY, SECRET_KEY, AWS_REGION, OFFLINE_ELASTICSEARCH_DOMAIN_ENDPOINT } = process.env;

AWS.config.update({
region: AWS_REGION || 'us-west-2',
accessKeyId: ACCESS_KEY,
secretAccessKey: SECRET_KEY,
});
ES_DOMAIN_ENDPOINT = OFFLINE_ELASTICSEARCH_DOMAIN_ENDPOINT || 'https://fake-es-endpoint.com';
}

constructor() {
let ES_DOMAIN_ENDPOINT = ELASTICSEARCH_DOMAIN_ENDPOINT || 'https://fake-es-endpoint.com';
if (IS_OFFLINE === 'true') {
const { ACCESS_KEY, SECRET_KEY, AWS_REGION, OFFLINE_ELASTICSEARCH_DOMAIN_ENDPOINT } = process.env;
return new Client({
node: ES_DOMAIN_ENDPOINT,
Connection: AmazonConnection,
Transport: AmazonTransport,
});
};

AWS.config.update({
region: AWS_REGION || 'us-west-2',
accessKeyId: ACCESS_KEY,
secretAccessKey: SECRET_KEY,
});
ES_DOMAIN_ENDPOINT = OFFLINE_ELASTICSEARCH_DOMAIN_ENDPOINT || 'https://fake-es-endpoint.com';
}
export default class DdbToEsHelper {
public ElasticSearch: Client;

this.ElasticSearch = new Client({
node: ES_DOMAIN_ENDPOINT,
Connection: AmazonConnection,
Transport: AmazonTransport,
});
constructor({ esClient = getDefaultESClientFromEnvVars() }: { esClient?: Client } = {}) {
this.ElasticSearch = esClient;
}

// async createIndexAndAliasIfNotExist(resourceTypes: Set<string>) {
Expand Down Expand Up @@ -165,29 +169,13 @@ export default class DdbToEsHelper {
return `${id}_${vid}`;
}

// eslint-disable-next-line class-methods-use-this
generateAlias(ddbImage: any) {
const { resourceType, _tenantId } = ddbImage;
const lowercaseResourceType = resourceType.toLowerCase();
if (_tenantId) {
return `${lowercaseResourceType}-alias-tenant-${_tenantId}`;
}
return `${lowercaseResourceType}-alias`;
}

// eslint-disable-next-line class-methods-use-this
generateIndexName(ddbImage: any) {
const { resourceType } = ddbImage;
return resourceType.toLowerCase();
}

// Getting promise params for actual deletion of the record from ES
createBulkESDelete(ddbResourceImage: any): ESBulkCommand {
createBulkESDelete(ddbResourceImage: any, alias: string): ESBulkCommand {
const compositeId = this.generateFullId(ddbResourceImage);
return {
bulkCommand: [
{
delete: { _index: this.generateAlias(ddbResourceImage), _id: compositeId },
delete: { _index: alias, _id: compositeId },
},
],
id: compositeId,
Expand All @@ -196,7 +184,7 @@ export default class DdbToEsHelper {
}

// Getting promise params for inserting a new record or editing a record
createBulkESUpsert(newImage: any): ESBulkCommand | null {
createBulkESUpsert(newImage: any, alias: string): ESBulkCommand | null {
// We only perform operations on records with documentStatus === AVAILABLE || DELETED
if (
newImage[DOCUMENT_STATUS_FIELD] !== DOCUMENT_STATUS.AVAILABLE &&
Expand All @@ -213,7 +201,7 @@ export default class DdbToEsHelper {
return {
id: compositeId,
bulkCommand: [
{ update: { _index: this.generateAlias(newImage), _id: compositeId } },
{ update: { _index: alias, _id: compositeId } },
{ doc: formatDocument(newImage), doc_as_upsert: true },
],
type,
Expand Down
Loading

0 comments on commit f9c9414

Please sign in to comment.