Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "@contentstack/datasync-manager",
"author": "Contentstack LLC <support@contentstack.com>",
"version": "2.1.1",
"version": "2.1.2",
"description": "The primary module of Contentstack DataSync. Syncs Contentstack data with your server using Contentstack Sync API",
"main": "dist/index.js",
"dependencies": {
Expand Down
54 changes: 46 additions & 8 deletions src/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,14 @@ import { readFileSync } from './util/fs'

const debug = Debug('api')
let MAX_RETRY_LIMIT
let RETRY_DELAY_BASE = 200 // Default base delay in milliseconds
let TIMEOUT = 30000 // Default timeout in milliseconds
let Contentstack

/**
* @description Initialize sync utilities API requests
* @param {Object} contentstack - Contentstack configuration details
*/
*/
export const init = (contentstack) => {
const packageInfo: any = JSON.parse(readFileSync(join(__dirname, '..', 'package.json')))
Contentstack = contentstack
Expand All @@ -35,6 +37,14 @@ export const init = (contentstack) => {
if (Contentstack.MAX_RETRY_LIMIT) {
MAX_RETRY_LIMIT = Contentstack.MAX_RETRY_LIMIT
}

if (Contentstack.RETRY_DELAY_BASE) {
RETRY_DELAY_BASE = Contentstack.RETRY_DELAY_BASE
}

if (Contentstack.TIMEOUT) {
TIMEOUT = Contentstack.TIMEOUT
}
}

/**
Expand All @@ -60,13 +70,14 @@ export const get = (req, RETRY = 1) => {
path: sanitizeUrl(encodeURI(req.path)),
port: Contentstack.port,
protocol: Contentstack.protocol,
timeout: TIMEOUT, // Configurable timeout to prevent socket hang ups
}

try {
debug(`${options.method.toUpperCase()}: ${options.path}`)
let timeDelay
let body = ''
request(options, (response) => {
const httpRequest = request(options, (response) => {

response
.setEncoding('utf-8')
Expand All @@ -76,8 +87,8 @@ export const get = (req, RETRY = 1) => {
if (response.statusCode >= 200 && response.statusCode <= 399) {
return resolve(JSON.parse(body))
} else if (response.statusCode === 429) {
timeDelay = Math.pow(Math.SQRT2, RETRY) * 200
debug(`API rate limit exceeded. Retrying ${options.path} with ${timeDelay} sec delay`)
timeDelay = Math.pow(Math.SQRT2, RETRY) * RETRY_DELAY_BASE
debug(`API rate limit exceeded. Retrying ${options.path} with ${timeDelay} ms delay`)

return setTimeout(() => {
return get(req, RETRY)
Expand All @@ -86,8 +97,8 @@ export const get = (req, RETRY = 1) => {
}, timeDelay)
} else if (response.statusCode >= 500) {
// retry, with delay
timeDelay = Math.pow(Math.SQRT2, RETRY) * 200
debug(`Retrying ${options.path} with ${timeDelay} sec delay`)
timeDelay = Math.pow(Math.SQRT2, RETRY) * RETRY_DELAY_BASE
debug(`Retrying ${options.path} with ${timeDelay} ms delay`)
RETRY++

return setTimeout(() => {
Expand All @@ -102,8 +113,35 @@ export const get = (req, RETRY = 1) => {
}
})
})
.on('error', reject)
.end()

// Set socket timeout to handle socket hang ups
httpRequest.setTimeout(options.timeout, () => {
debug(`Request timeout for ${options.path || 'unknown'}`)
httpRequest.destroy()
reject(new Error('Request timeout'))
})

// Enhanced error handling for socket hang ups and connection resets
httpRequest.on('error', (error: any) => {
debug(`Request error for ${options.path || 'unknown'}: ${error?.message || 'Unknown error'} (${error?.code || 'NO_CODE'})`)

// Handle socket hang up and connection reset errors with retry
if ((error?.code === 'ECONNRESET' || error?.message?.includes('socket hang up')) && RETRY <= MAX_RETRY_LIMIT) {
timeDelay = Math.pow(Math.SQRT2, RETRY) * RETRY_DELAY_BASE
debug(`Socket hang up detected. Retrying ${options.path || 'unknown'} with ${timeDelay} ms delay (attempt ${RETRY}/${MAX_RETRY_LIMIT})`)
RETRY++

return setTimeout(() => {
return get(req, RETRY)
.then(resolve)
.catch(reject)
}, timeDelay)
}

return reject(error)
})

httpRequest.end()
} catch (error) {
return reject(error)
}
Expand Down
2 changes: 2 additions & 0 deletions src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ export const config = {
},
contentstack: {
MAX_RETRY_LIMIT: 6,
TIMEOUT: 30000, // 30 seconds - can be overridden by user config
RETRY_DELAY_BASE: 200, // Base delay for retry logic - can be overridden by user config
actions: {
delete: 'delete',
publish: 'publish',
Expand Down
5 changes: 4 additions & 1 deletion src/core/inet.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,10 @@ export const checkNetConnectivity = () => {
}

export const netConnectivityIssues = (error) => {
if (error.code === 'ENOTFOUND' || error.code === 'ETIMEDOUT') {
// Include socket hang up and connection reset errors as network connectivity issues
const networkErrorCodes = ['ENOTFOUND', 'ETIMEDOUT', 'ECONNRESET', 'EPIPE', 'EHOSTUNREACH']

if (networkErrorCodes.includes(error.code) || error.message?.includes('socket hang up')) {
return true
}

Expand Down
Loading