Skip to content

feat: add retry resilience to imsApiCall #1438

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ravverma merged 3 commits into from
Mar 16, 2026
Merged

feat: add retry resilience to imsApiCall #1438

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0473047
feat(ims-client): add retry resilience to imsApiCall
ravverma Mar 13, 2026
1977312
fix(ims): address PR review feedback on retry resilience
ravverma Mar 13, 2026
692339c
Merge branch 'main' into feat/ims-api-call-resilience
ravverma Mar 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 87 additions & 17 deletions packages/spacecat-shared-ims-client/src/clients/ims-base-client.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,12 @@ import {
import { createFormData } from '../utils.js';

export default class ImsBaseClient {
// Retry at most 2 times (3 total attempts) to avoid flooding IMS with requests.
static #MAX_RETRIES = 2;

// Base delay in ms; doubles each attempt: 1 000 ms → 2 000 ms.
static #RETRY_BASE_DELAY_MS = 1000;

/**
* Creates a new Ims client
*
Expand All @@ -27,6 +33,19 @@ export default class ImsBaseClient {
constructor(config, log) {
this.config = config;
this.log = log;
// Exposed as an instance property so tests can override it without fake timers.
this.retryBaseDelayMs = ImsBaseClient.#RETRY_BASE_DELAY_MS;
}

static #isRetryableStatus(status) {
// Retry on server errors and rate limiting; never on client errors.
return status === 429 || status >= 500;
}

static #sleep(ms) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}

/**
Expand Down Expand Up @@ -90,26 +109,77 @@ export default class ImsBaseClient {
options = {},
) {
const startTime = process.hrtime.bigint();
const maxAttempts = ImsBaseClient.#MAX_RETRIES + 1;
const callerName = new Error().stack.split('\n')[2].trim().split(' ')[1];

const headers = await this.#prepareImsRequestHeaders(options);
const url = createUrl(`https://${this.config.imsHost}${endpoint}`, queryString);
const fetchOptions = {
...(isObject(body) ? { method: 'POST' } : { method: 'GET' }),
headers,
...(isObject(body) ? { body: createFormData(body) } : {}),
};

for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
try {
// eslint-disable-next-line no-await-in-loop
const response = await tracingFetch(url, fetchOptions);

const shouldRetry = ImsBaseClient.#isRetryableStatus(response.status)
&& attempt < ImsBaseClient.#MAX_RETRIES;
if (shouldRetry) {
const baseDelay = this.retryBaseDelayMs * (2 ** attempt);
// Respect Retry-After header (value in seconds) for 429 rate-limit responses
const retryAfterHeader = response.headers.get('Retry-After');
const retryAfterMs = retryAfterHeader ? parseInt(retryAfterHeader, 10) * 1000 : 0;
// ±20% jitter to spread retries and avoid thundering herd
const jitter = 0.8 + Math.random() * 0.4;
const delay = Math.round(Math.max(baseDelay, retryAfterMs) * jitter);
this.log.warn(
`IMS ${endpoint} request returned status ${response.status} `
+ `(attempt ${attempt + 1}/${maxAttempts}). `
+ `Retrying in ${delay}ms...`,
);
// Drain the response body to avoid socket leaks before discarding the response
// eslint-disable-next-line no-await-in-loop
await response.body?.cancel?.();
// eslint-disable-next-line no-await-in-loop
await ImsBaseClient.#sleep(delay);
continue; // eslint-disable-line no-continue
}

try {
const response = await tracingFetch(
createUrl(`https://${this.config.imsHost}${endpoint}`, queryString),
{
...(isObject(body) ? { method: 'POST' } : { method: 'GET' }),
headers,
...(isObject(body) ? { body: createFormData(body) } : {}),
},
);

const callerName = new Error().stack.split('\n')[2].trim().split(' ')[1];
this.#logDuration(`IMS ${callerName} request`, startTime);

return response;
} catch (error) {
this.log.error('Error while fetching data from IMS API: ', error.message);
throw error;
this.#logDuration(`IMS ${callerName} request`, startTime);

if (attempt > 0) {
this.log.info(
`IMS ${endpoint} request succeeded on attempt ${attempt + 1}/${maxAttempts} `
+ `(used ${attempt} ${attempt === 1 ? 'retry' : 'retries'}).`,
);
}

return response;
} catch (error) {
if (attempt < ImsBaseClient.#MAX_RETRIES) {
const baseDelay = this.retryBaseDelayMs * (2 ** attempt);
// ±20% jitter to spread retries and avoid thundering herd
const jitter = 0.8 + Math.random() * 0.4;
const delay = Math.round(baseDelay * jitter);
this.log.warn(
`IMS ${endpoint} request failed with network error `
+ `(attempt ${attempt + 1}/${maxAttempts}): ${error.message}. `
+ `Retrying in ${delay}ms...`,
);
// eslint-disable-next-line no-await-in-loop
await ImsBaseClient.#sleep(delay);
} else {
this.log.error(
`IMS ${endpoint} request failed after ${maxAttempts} attempts: ${error.message}`,
);
throw error;
}
}
}
/* c8 ignore next */
throw new Error('imsApiCall: retry loop exited unexpectedly');
}
}
136 changes: 133 additions & 3 deletions packages/spacecat-shared-ims-client/test/clients/ims-client.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ describe('ImsClient', () => {

beforeEach(() => {
client = ImsClient.createFrom(mockContext);
client.retryBaseDelayMs = 0;
});

it('should throw an error for invalid imsOrgId', async () => {
Expand Down Expand Up @@ -144,9 +145,10 @@ describe('ImsClient', () => {

it('should handle IMS service token request failures', async () => {
nock(`https://${DUMMY_HOST}`)
// Mock the token request, with a 500 server error response
// Mock the token request, with a 500 server error response — must cover all retry attempts
.post('/ims/token/v4')
.query(true)
.times(3)
.reply(500);

await expect(client.getImsOrganizationDetails('123456@AdobeOrg')).to.be.rejectedWith('IMS getServiceAccessToken request failed with status: 500');
Expand Down Expand Up @@ -186,9 +188,10 @@ describe('ImsClient', () => {

it('should handle IMS service token v3 request failures', async () => {
nock(`https://${DUMMY_HOST}`)
// Mock the token request, with a 500 server error response
// Mock the token request, with a 500 server error response — must cover all retry attempts
.post('/ims/token/v3')
.query(true)
.times(3)
.reply(500);

await expect(client.getServiceAccessTokenV3()).to.be.rejectedWith('IMS getServiceAccessTokenV3 request failed with status: 500');
Expand Down Expand Up @@ -232,9 +235,10 @@ describe('ImsClient', () => {
.get(`/ims/organizations/${testOrgId}/v2`)
.query(true)
.reply(200, IMS_FETCH_ORG_DETAILS_RESPONSE)
// Mock the request for group members in 123456789
// Mock the request for group members in 123456789 — 500 must cover all retry attempts
.get(`/ims/organizations/${testOrgId}/groups/${GROUP_1_ID}/members`)
.query(true)
.times(3)
.reply(500);

await expect(client.getImsOrganizationDetails(testOrgId)).to.be.rejectedWith('IMS getUsersByImsGroupId request failed with status: 500');
Expand Down Expand Up @@ -343,23 +347,28 @@ describe('ImsClient', () => {

beforeEach(() => {
client = ImsClient.createFrom(mockContext);
client.retryBaseDelayMs = 0;
});

it('throws error if no access token is provided', async () => {
await expect(client.getImsUserOrganizations(null)).to.be.rejectedWith('imsAccessToken param is required.');
});

it('throws error if fetch throws error', async () => {
// Network errors are retried — mock all 3 attempts
nock(`https://${DUMMY_HOST}`)
.get('/ims/organizations/v6')
.times(3)
.replyWithError('test error');

await expect(client.getImsUserOrganizations('some-token')).to.be.rejectedWith('test error');
});

it('throws error if request fails', async () => {
// 5xx responses are retried — mock all 3 attempts
nock(`https://${DUMMY_HOST}`)
.get('/ims/organizations/v6')
.times(3)
.reply(500, {
error: 'server_error',
error_description: 'Boom',
Expand Down Expand Up @@ -412,15 +421,18 @@ describe('ImsClient', () => {

beforeEach(() => {
client = ImsClient.createFrom(mockContext);
client.retryBaseDelayMs = 0;
});

it('throws error if no access token is provided', async () => {
await expect(client.validateAccessToken('')).to.be.rejectedWith('imsAccessToken param is required.');
});

it('throws error if request fails', async () => {
// 5xx responses are retried — mock all 3 attempts
nock(`https://${DUMMY_HOST}`)
.post('/ims/validate_token/v1')
.times(3)
.reply(500, {
error: 'server_error',
error_description: 'Boom',
Expand Down Expand Up @@ -605,6 +617,7 @@ describe('ImsClient', () => {

beforeEach(() => {
client = ImsClient.createFrom(mockContext);
client.retryBaseDelayMs = 0;
});

it('throws error when accessToken is not provided', async () => {
Expand Down Expand Up @@ -633,10 +646,12 @@ describe('ImsClient', () => {
});

it('throws error when account cluster request fails', async () => {
// 5xx responses are retried — mock all 3 attempts
nock(`https://${DUMMY_HOST}`)
.get('/ims/account_cluster/v2')
.query({ client_id: mockContext.env.IMS_CLIENT_ID })
.matchHeader('Authorization', (val) => val === `Bearer ${testAccessToken}`)
.times(3)
.reply(500, {
error: 'server_error',
error_description: 'Internal server error',
Expand Down Expand Up @@ -702,32 +717,38 @@ describe('ImsClient', () => {
});

it('handles non-JSON error response gracefully', async () => {
// 5xx responses are retried — mock all 3 attempts
nock(`https://${DUMMY_HOST}`)
.get('/ims/account_cluster/v2')
.query({ client_id: mockContext.env.IMS_CLIENT_ID })
.matchHeader('Authorization', (val) => val === `Bearer ${testAccessToken}`)
.times(3)
.reply(500, 'Internal Server Error');

await expect(client.getAccountCluster(testAccessToken))
.to.be.rejectedWith('IMS getAccountCluster request failed with status: 500');
});

it('handles empty error response body gracefully', async () => {
// 5xx responses are retried — mock all 3 attempts
nock(`https://${DUMMY_HOST}`)
.get('/ims/account_cluster/v2')
.query({ client_id: mockContext.env.IMS_CLIENT_ID })
.matchHeader('Authorization', (val) => val === `Bearer ${testAccessToken}`)
.times(3)
.reply(503, {});

await expect(client.getAccountCluster(testAccessToken))
.to.be.rejectedWith('IMS getAccountCluster request failed with status: 503');
});

it('handles error response with no error or message fields', async () => {
// 429 is retried — mock all 3 attempts
nock(`https://${DUMMY_HOST}`)
.get('/ims/account_cluster/v2')
.query({ client_id: mockContext.env.IMS_CLIENT_ID })
.matchHeader('Authorization', (val) => val === `Bearer ${testAccessToken}`)
.times(3)
.reply(429, {
retryAfter: 60,
code: 'rate_limit_exceeded',
Expand All @@ -737,4 +758,113 @@ describe('ImsClient', () => {
.to.be.rejectedWith('IMS getAccountCluster request failed with status: 429');
});
});

describe('imsApiCall retry behavior', () => {
let client;
let warnSpy;

beforeEach(() => {
client = ImsClient.createFrom(mockContext);
// Zero delay avoids real sleep in tests while still exercising retry paths
client.retryBaseDelayMs = 0;
warnSpy = sandbox.spy(console, 'warn');
});

it('retries on 5xx and succeeds on second attempt', async () => {
nock(`https://${DUMMY_HOST}`)
.get('/ims/organizations/v6')
.reply(503) // attempt 1: server error
.get('/ims/organizations/v6')
.reply(200, []); // attempt 2: success

const result = await client.getImsUserOrganizations('some-token');

expect(result).to.deep.equal([]);
expect(warnSpy.calledOnce).to.be.true;
expect(warnSpy.firstCall.args[0]).to.include('/ims/organizations/v6');
expect(warnSpy.firstCall.args[0]).to.include('503');
expect(warnSpy.firstCall.args[0]).to.include('attempt 1/3');
});

it('retries on 429 rate limiting and succeeds on second attempt', async () => {
nock(`https://${DUMMY_HOST}`)
.get('/ims/organizations/v6')
.reply(429) // attempt 1: rate limited
.get('/ims/organizations/v6')
.reply(200, []); // attempt 2: success

const result = await client.getImsUserOrganizations('some-token');

expect(result).to.deep.equal([]);
expect(warnSpy.calledOnce).to.be.true;
expect(warnSpy.firstCall.args[0]).to.include('429');
expect(warnSpy.firstCall.args[0]).to.include('attempt 1/3');
});

it('respects Retry-After header on 429 response', async () => {
// Retry-After: 0 keeps the test instant while still exercising the header path
nock(`https://${DUMMY_HOST}`)
.get('/ims/organizations/v6')
.reply(429, '', { 'Retry-After': '0' }) // attempt 1: rate limited with header
.get('/ims/organizations/v6')
.reply(200, []); // attempt 2: success

const result = await client.getImsUserOrganizations('some-token');

expect(result).to.deep.equal([]);
expect(warnSpy.calledOnce).to.be.true;
expect(warnSpy.firstCall.args[0]).to.include('429');
});

it('retries on network error and succeeds on third attempt', async () => {
nock(`https://${DUMMY_HOST}`)
.get('/ims/organizations/v6')
.replyWithError('ECONNRESET') // attempt 1: network error
.get('/ims/organizations/v6')
.replyWithError('ECONNRESET') // attempt 2: network error
.get('/ims/organizations/v6')
.reply(200, []); // attempt 3: success

const result = await client.getImsUserOrganizations('some-token');

expect(result).to.deep.equal([]);
expect(warnSpy.calledTwice).to.be.true;
expect(warnSpy.firstCall.args[0]).to.include('attempt 1/3');
expect(warnSpy.secondCall.args[0]).to.include('attempt 2/3');
});

it('exhausts all retries on network error and throws', async () => {
nock(`https://${DUMMY_HOST}`)
.get('/ims/organizations/v6')
.times(3)
.replyWithError('ECONNRESET');

await expect(client.getImsUserOrganizations('some-token')).to.be.rejectedWith('ECONNRESET');

// Two warn logs (attempts 1 and 2), no warn on final attempt — error log instead
expect(warnSpy.calledTwice).to.be.true;
});

it('returns last bad response when all 5xx retries exhausted', async () => {
nock(`https://${DUMMY_HOST}`)
.get('/ims/organizations/v6')
.times(3)
.reply(503);

await expect(client.getImsUserOrganizations('some-token')).to.be.rejectedWith('IMS getImsUserOrganizations request failed with status: 503');

// Two warn logs for the two retries; no warn on last attempt (just returns the response)
expect(warnSpy.calledTwice).to.be.true;
});

it('does not retry on 4xx client errors', async () => {
nock(`https://${DUMMY_HOST}`)
.get('/ims/organizations/v6')
.reply(401, { error: 'unauthorized' });

await expect(client.getImsUserOrganizations('some-token')).to.be.rejectedWith('IMS getImsUserOrganizations request failed with status: 401');

expect(warnSpy.called).to.be.false;
});
});
});
Loading