Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore: migrate AWS SDK for JavaScript v2 APIs to v3 #7008

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

chore: migrate AWS SDK for JavaScript v2 APIs to v3 #7008

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3f9114a
chore: migrate AWS SDK for JavaScript v2 APIs to v3
trivikr Feb 19, 2024
9682b05
chore: manually update config and remove .promise() calls
trivikr Feb 19, 2024
b9fac49
chore: update configuration for JS SDK v3 S3 client
trivikr Feb 19, 2024
a68007b
chore(deps): replace JS SDK v2 with v3
trivikr Feb 19, 2024
6d48a29
chore: remove extra newlines
trivikr Feb 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 3 additions & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@
"node": ">=10.12"
},
"dependencies": {
"@aws-sdk/client-cloudsearch": "3.515.0",
"@aws-sdk/client-cloudsearch-domain": "3.515.0",
"@aws-sdk/client-s3": "3.515.0",
"@azure/storage-blob": "12.12.0",
"@exlinc/keycloak-passport": "1.0.2",
"@joplin/turndown-plugin-gfm": "1.0.45",
Expand All @@ -51,7 +54,6 @@
"apollo-server-express": "2.25.2",
"asciidoctor": "2.2.6",
"auto-load": "3.0.4",
"aws-sdk": "2.1309.0",
"azure-search-client": "3.1.5",
"bcryptjs-then": "1.0.1",
"bluebird": "3.7.2",
Expand Down
63 changes: 33 additions & 30 deletions server/modules/search/aws/engine.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
const _ = require('lodash')
const AWS = require('aws-sdk')
const { CloudSearch } = require('@aws-sdk/client-cloudsearch');
const { CloudSearchDomain } = require('@aws-sdk/client-cloudsearch-domain');
const stream = require('stream')
const Promise = require('bluebird')
const pipeline = Promise.promisify(stream.pipeline)
Expand All @@ -18,17 +19,19 @@ module.exports = {
*/
async init() {
WIKI.logger.info(`(SEARCH/AWS) Initializing...`)
this.client = new AWS.CloudSearch({
apiVersion: '2013-01-01',
accessKeyId: this.config.accessKeyId,
secretAccessKey: this.config.secretAccessKey,
this.client = new CloudSearch({
credentials: {
accessKeyId: this.config.accessKeyId,
secretAccessKey: this.config.secretAccessKey
},
region: this.config.region
})
this.clientDomain = new AWS.CloudSearchDomain({
apiVersion: '2013-01-01',
this.clientDomain = new CloudSearchDomain({
endpoint: this.config.endpoint,
accessKeyId: this.config.accessKeyId,
secretAccessKey: this.config.secretAccessKey,
credentials: {
accessKeyId: this.config.accessKeyId,
secretAccessKey: this.config.secretAccessKey
},
region: this.config.region
})

Expand All @@ -38,7 +41,7 @@ module.exports = {
const schemes = await this.client.describeAnalysisSchemes({
DomainName: this.config.domain,
AnalysisSchemeNames: ['default_anlscheme']
}).promise()
})
if (_.get(schemes, 'AnalysisSchemes', []).length < 1) {
WIKI.logger.info(`(SEARCH/AWS) Defining Analysis Scheme...`)
await this.client.defineAnalysisScheme({
Expand All @@ -47,14 +50,14 @@ module.exports = {
AnalysisSchemeLanguage: this.config.AnalysisSchemeLang,
AnalysisSchemeName: 'default_anlscheme'
}
}).promise()
})
rebuildIndex = true
}

// -> Define Index Fields
const fields = await this.client.describeIndexFields({
DomainName: this.config.domain
}).promise()
})
if (_.get(fields, 'IndexFields', []).length < 1) {
WIKI.logger.info(`(SEARCH/AWS) Defining Index Fields...`)
await this.client.defineIndexField({
Expand All @@ -63,21 +66,21 @@ module.exports = {
IndexFieldName: 'id',
IndexFieldType: 'literal'
}
}).promise()
})
await this.client.defineIndexField({
DomainName: this.config.domain,
IndexField: {
IndexFieldName: 'path',
IndexFieldType: 'literal'
}
}).promise()
})
await this.client.defineIndexField({
DomainName: this.config.domain,
IndexField: {
IndexFieldName: 'locale',
IndexFieldType: 'literal'
}
}).promise()
})
await this.client.defineIndexField({
DomainName: this.config.domain,
IndexField: {
Expand All @@ -88,7 +91,7 @@ module.exports = {
AnalysisScheme: 'default_anlscheme'
}
}
}).promise()
})
await this.client.defineIndexField({
DomainName: this.config.domain,
IndexField: {
Expand All @@ -99,7 +102,7 @@ module.exports = {
AnalysisScheme: 'default_anlscheme'
}
}
}).promise()
})
await this.client.defineIndexField({
DomainName: this.config.domain,
IndexField: {
Expand All @@ -110,15 +113,15 @@ module.exports = {
AnalysisScheme: 'default_anlscheme'
}
}
}).promise()
})
rebuildIndex = true
}

// -> Define suggester
const suggesters = await this.client.describeSuggesters({
DomainName: this.config.domain,
SuggesterNames: ['default_suggester']
}).promise()
})
if (_.get(suggesters, 'Suggesters', []).length < 1) {
WIKI.logger.info(`(SEARCH/AWS) Defining Suggester...`)
await this.client.defineSuggester({
Expand All @@ -130,7 +133,7 @@ module.exports = {
FuzzyMatching: 'high'
}
}
}).promise()
})
rebuildIndex = true
}

Expand All @@ -139,7 +142,7 @@ module.exports = {
WIKI.logger.info(`(SEARCH/AWS) Requesting Index Rebuild...`)
await this.client.indexDocuments({
DomainName: this.config.domain
}).promise()
})
}

WIKI.logger.info(`(SEARCH/AWS) Initialization completed.`)
Expand All @@ -157,13 +160,13 @@ module.exports = {
query: q,
partial: true,
size: 50
}).promise()
})
if (results.hits.found < 5) {
const suggestResults = await this.clientDomain.suggest({
query: q,
suggester: 'default_suggester',
size: 5
}).promise()
})
suggestions = suggestResults.suggest.suggestions.map(s => s.suggestion)
}
return {
Expand Down Expand Up @@ -203,7 +206,7 @@ module.exports = {
}
}
])
}).promise()
})
},
/**
* UPDATE
Expand All @@ -226,7 +229,7 @@ module.exports = {
}
}
])
}).promise()
})
},
/**
* DELETE
Expand All @@ -242,7 +245,7 @@ module.exports = {
id: page.hash
}
])
}).promise()
})
},
/**
* RENAME
Expand All @@ -258,7 +261,7 @@ module.exports = {
id: page.hash
}
])
}).promise()
})
await this.clientDomain.uploadDocuments({
contentType: 'application/json',
documents: JSON.stringify([
Expand All @@ -274,7 +277,7 @@ module.exports = {
}
}
])
}).promise()
})
},
/**
* REBUILD INDEX
Expand Down Expand Up @@ -340,7 +343,7 @@ module.exports = {
content: WIKI.models.pages.cleanHTML(doc.render)
}
})))
}).promise()
})
} catch (err) {
WIKI.logger.warn('(SEARCH/AWS) Failed to send batch to AWS CloudSearch: ', err)
}
Expand All @@ -363,7 +366,7 @@ module.exports = {
WIKI.logger.info(`(SEARCH/AWS) Requesting Index Rebuild...`)
await this.client.indexDocuments({
DomainName: this.config.domain
}).promise()
})

WIKI.logger.info(`(SEARCH/AWS) Index rebuilt successfully.`)
}
Expand Down
37 changes: 17 additions & 20 deletions server/modules/storage/s3/common.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
const S3 = require('aws-sdk/clients/s3')
const { S3 } = require('@aws-sdk/client-s3');
const stream = require('stream')
const Promise = require('bluebird')
const pipeline = Promise.promisify(stream.pipeline)
Expand Down Expand Up @@ -34,10 +34,7 @@ module.exports = class S3CompatibleStorage {
WIKI.logger.info(`(STORAGE/${this.storageName}) Initializing...`)
const { accessKeyId, secretAccessKey, bucket } = this.config
const s3Config = {
accessKeyId,
secretAccessKey,
params: { Bucket: bucket },
apiVersions: '2006-03-01'
credentials: { accessKeyId, secretAccessKey }
}

if (!_.isNil(this.config.region)) {
Expand All @@ -47,37 +44,37 @@ module.exports = class S3CompatibleStorage {
s3Config.endpoint = this.config.endpoint
}
if (!_.isNil(this.config.sslEnabled)) {
s3Config.sslEnabled = this.config.sslEnabled
s3Config.tls = this.config.sslEnabled
}
if (!_.isNil(this.config.s3ForcePathStyle)) {
s3Config.s3ForcePathStyle = this.config.s3ForcePathStyle
s3Config.forcePathStyle = this.config.s3ForcePathStyle
}
if (!_.isNil(this.config.s3BucketEndpoint)) {
s3Config.s3BucketEndpoint = this.config.s3BucketEndpoint
s3Config.bucketEndpoint = this.config.s3BucketEndpoint
}

this.s3 = new S3(s3Config)
this.bucketName = bucket

// determine if a bucket exists and you have permission to access it
await this.s3.headBucket().promise()
await this.s3.headBucket({ Bucket: this.bucketName })

WIKI.logger.info(`(STORAGE/${this.storageName}) Initialization completed.`)
}
async created(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Creating file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.putObject({ Key: filePath, Body: page.injectMetadata() }).promise()
await this.s3.putObject({ Bucket: this.bucketName, Key: filePath, Body: page.injectMetadata() })
}
async updated(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Updating file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.putObject({ Key: filePath, Body: page.injectMetadata() }).promise()
await this.s3.putObject({ Bucket: this.bucketName, Key: filePath, Body: page.injectMetadata() })
}
async deleted(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Deleting file ${page.path}...`)
const filePath = getFilePath(page, 'path')
await this.s3.deleteObject({ Key: filePath }).promise()
await this.s3.deleteObject({ Bucket: this.bucketName, Key: filePath })
}
async renamed(page) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file ${page.path} to ${page.destinationPath}...`)
Expand All @@ -91,8 +88,8 @@ module.exports = class S3CompatibleStorage {
destinationFilePath = `${page.destinationLocaleCode}/${destinationFilePath}`
}
}
await this.s3.copyObject({ CopySource: `${this.bucketName}/${sourceFilePath}`, Key: destinationFilePath }).promise()
await this.s3.deleteObject({ Key: sourceFilePath }).promise()
await this.s3.copyObject({ Bucket: this.bucketName, CopySource: `${this.bucketName}/${sourceFilePath}`, Key: destinationFilePath })
await this.s3.deleteObject({ Bucket: this.bucketName, Key: sourceFilePath })
}
/**
* ASSET UPLOAD
Expand All @@ -101,7 +98,7 @@ module.exports = class S3CompatibleStorage {
*/
async assetUploaded (asset) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Creating new file ${asset.path}...`)
await this.s3.putObject({ Key: asset.path, Body: asset.data }).promise()
await this.s3.putObject({ Bucket: this.bucketName, Key: asset.path, Body: asset.data })
}
/**
* ASSET DELETE
Expand All @@ -110,7 +107,7 @@ module.exports = class S3CompatibleStorage {
*/
async assetDeleted (asset) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Deleting file ${asset.path}...`)
await this.s3.deleteObject({ Key: asset.path }).promise()
await this.s3.deleteObject({ Bucket: this.bucketName, Key: asset.path })
}
/**
* ASSET RENAME
Expand All @@ -119,8 +116,8 @@ module.exports = class S3CompatibleStorage {
*/
async assetRenamed (asset) {
WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file from ${asset.path} to ${asset.destinationPath}...`)
await this.s3.copyObject({ CopySource: `${this.bucketName}/${asset.path}`, Key: asset.destinationPath }).promise()
await this.s3.deleteObject({ Key: asset.path }).promise()
await this.s3.copyObject({ Bucket: this.bucketName, CopySource: `${this.bucketName}/${asset.path}`, Key: asset.destinationPath })
await this.s3.deleteObject({ Bucket: this.bucketName, Key: asset.path })
}
async getLocalLocation () {

Expand All @@ -141,7 +138,7 @@ module.exports = class S3CompatibleStorage {
transform: async (page, enc, cb) => {
const filePath = getFilePath(page, 'path')
WIKI.logger.info(`(STORAGE/${this.storageName}) Adding page ${filePath}...`)
await this.s3.putObject({ Key: filePath, Body: pageHelper.injectPageMetadata(page) }).promise()
await this.s3.putObject({ Bucket: this.bucketName, Key: filePath, Body: pageHelper.injectPageMetadata(page) })
cb()
}
})
Expand All @@ -157,7 +154,7 @@ module.exports = class S3CompatibleStorage {
transform: async (asset, enc, cb) => {
const filename = (asset.folderId && asset.folderId > 0) ? `${_.get(assetFolders, asset.folderId)}/${asset.filename}` : asset.filename
WIKI.logger.info(`(STORAGE/${this.storageName}) Adding asset ${filename}...`)
await this.s3.putObject({ Key: filename, Body: asset.data }).promise()
await this.s3.putObject({ Bucket: this.bucketName, Key: filename, Body: asset.data })
cb()
}
})
Expand Down