feat: merge main

.gitignore

@@ -135,3 +135,6 @@ junit
 
 # Files deliberately excluded from version control
 *DO-NOT-COMMIT*
+
+# Mac OS
+.DS_Store

package.json

@@ -32,14 +32,14 @@
     "@semantic-release/changelog": "6.0.3",
     "@semantic-release/git": "10.0.1",
     "@semantic-release/npm": "12.0.1",
-    "@typescript-eslint/eslint-plugin": "7.12.0",
-    "@typescript-eslint/parser": "7.12.0",
+    "@typescript-eslint/eslint-plugin": "7.13.0",
+    "@typescript-eslint/parser": "7.13.0",
     "ajv": "8.16.0",
-    "c8": "10.1.1",
+    "c8": "10.1.2",
     "eslint": "8.57.0",
     "husky": "9.0.11",
     "jsdoc-to-markdown": "8.0.1",
-    "lint-staged": "15.2.5",
+    "lint-staged": "15.2.7",
     "mocha": "10.4.0",
     "mocha-multi-reporters": "1.5.1",
     "nock": "13.5.4",

packages/spacecat-shared-ahrefs-client/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [@adobe/spacecat-shared-ahrefs-client-v1.3.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-ahrefs-client-v1.2.6...@adobe/spacecat-shared-ahrefs-client-v1.3.0) (2024-06-19)
+
+
+### Features
+
+* introduce organic keywords for ahrefs client and top keyword in top pages ([#257](https://github.com/adobe/spacecat-shared/issues/257)) ([371f1c4](https://github.com/adobe/spacecat-shared/commit/371f1c475870fd2aac833f925236237a8b25c026))
+
 # [@adobe/spacecat-shared-ahrefs-client-v1.2.6](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-ahrefs-client-v1.2.5...@adobe/spacecat-shared-ahrefs-client-v1.2.6) (2024-06-11)
 
 

packages/spacecat-shared-ahrefs-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-ahrefs-client",
-  "version": "1.2.6",
+  "version": "1.3.0",
   "description": "Shared modules of the Spacecat Services - Ahrefs Client",
   "type": "module",
   "main": "src/index.js",

packages/spacecat-shared-ahrefs-client/src/index.d.ts

@@ -70,4 +70,10 @@ export default class AhrefsAPIClient {
      */
   getBacklinks(url: string, limit?: number):
       Promise<{ result: object, fullAuditRef: string }>;
+
+  /**
+   * Asynchronous method to get organic keywords.
+   */
+  getOrganicKeywords(url: string, country?: string, keywordFilter?: string[], limit?: number):
+      Promise<{ result: object, fullAuditRef: string }>;
 }

packages/spacecat-shared-ahrefs-client/src/index.js

@@ -10,7 +10,7 @@
  * governing permissions and limitations under the License.
  */
 
-import { isValidUrl } from '@adobe/spacecat-shared-utils';
+import { hasText, isValidUrl, isArray } from '@adobe/spacecat-shared-utils';
 import { context as h2, h1 } from '@adobe/fetch';
 
 /* c8 ignore next 3 */
@@ -120,6 +120,7 @@ export default class AhrefsAPIClient {
       select: [
         'url',
         'sum_traffic',
+        'top_keyword',
       ].join(','),
       order_by: 'sum_traffic',
       date: new Date().toISOString().split('T')[0],
@@ -174,4 +175,45 @@ export default class AhrefsAPIClient {
 
     return this.sendRequest('/site-explorer/metrics-history', queryParams);
   }
+
+  async getOrganicKeywords(url, country = 'us', keywordFilter = [], limit = 200) {
+    if (!hasText(url)) {
+      throw new Error(`Invalid URL: ${url}`);
+    }
+    if (!hasText(country)) {
+      throw new Error(`Invalid country: ${country}`);
+    }
+    if (!isArray(keywordFilter)) {
+      throw new Error(`Invalid keyword filter: ${keywordFilter}`);
+    }
+    if (!Number.isInteger(limit) || limit < 1) {
+      throw new Error(`Invalid limit: ${limit}`);
+    }
+    const queryParams = {
+      country,
+      date: new Date().toISOString().split('T')[0],
+      select: [
+        'keyword',
+        'sum_traffic',
+        'best_position_url',
+      ].join(','),
+      order_by: 'sum_traffic:desc',
+      target: url,
+      limit: getLimit(limit, 2000),
+      mode: 'prefix',
+      output: 'json',
+    };
+    if (keywordFilter.length > 0) {
+      try {
+        queryParams.where = JSON.stringify({
+          or: keywordFilter.map((keyword) => ({ field: 'keyword', is: ['iphrase_match', keyword] })),
+        });
+      } catch (e) {
+        this.log.error(`Error parsing keyword filter: ${e.message}`);
+        throw new Error(`Error parsing keyword filter: ${e.message}`);
+      }
+    }
+
+    return this.sendRequest('/site-explorer/organic-keywords', queryParams);
+  }
 }

packages/spacecat-shared-ahrefs-client/test/index.test.js

@@ -52,10 +52,12 @@ describe('AhrefsAPIClient', () => {
       {
         url: 'page-url-1',
         sum_traffic: 100,
+        top_keyword: 'keyword1',
       },
       {
         url: 'page-url-2',
         sum_traffic: 300,
+        top_keyword: 'keyword2',
       },
     ],
   };
@@ -79,6 +81,21 @@ describe('AhrefsAPIClient', () => {
     ],
   };
 
+  const organicKeywordsResponse = {
+    keywords: [
+      {
+        keyword: 'keyword1',
+        sum_traffic: 100,
+        best_position_url: 'url1',
+      },
+      {
+        keyword: 'keyword2',
+        sum_traffic: 200,
+        best_position_url: 'url2',
+      },
+    ],
+  };
+
   before('setup', function () {
     this.clock = sandbox.useFakeTimers({
       now: new Date(mockDate).getTime(),
@@ -203,6 +220,7 @@ describe('AhrefsAPIClient', () => {
         select: [
           'url',
           'sum_traffic',
+          'top_keyword',
         ].join(','),
         where: JSON.stringify(filter),
         order_by: 'sum_traffic',
@@ -221,7 +239,7 @@ describe('AhrefsAPIClient', () => {
       const result = await client.getTopPages(target, specifiedLimit);
       expect(result).to.deep.equal({
         result: topPagesResponse,
-        fullAuditRef: `https://example.com/site-explorer/top-pages?select=url%2Csum_traffic&order_by=sum_traffic&date=${date}&target=${target}&limit=${specifiedLimit}&mode=prefix&output=json&where=%7B%22and%22%3A%5B%7B%22field%22%3A%22sum_traffic%22%2C%22is%22%3A%5B%22gt%22%2C0%5D%7D%5D%7D`,
+        fullAuditRef: `https://example.com/site-explorer/top-pages?select=url%2Csum_traffic%2Ctop_keyword&order_by=sum_traffic&date=${date}&target=${target}&limit=${specifiedLimit}&mode=prefix&output=json&where=%7B%22and%22%3A%5B%7B%22field%22%3A%22sum_traffic%22%2C%22is%22%3A%5B%22gt%22%2C0%5D%7D%5D%7D`,
       });
     });
   });
@@ -288,4 +306,97 @@ describe('AhrefsAPIClient', () => {
       });
     });
   });
+
+  describe('getOrganicKeywords', () => {
+    it('sends API request with appropriate endpoint query params', async () => {
+      nock(config.apiBaseUrl)
+        .get('/site-explorer/organic-keywords')
+        .query({
+          country: 'us',
+          date: new Date().toISOString().split('T')[0],
+          select: [
+            'keyword',
+            'sum_traffic',
+            'best_position_url',
+          ].join(','),
+          order_by: 'sum_traffic:desc',
+          target: 'test-site.com',
+          limit: 200,
+          mode: 'prefix',
+          output: 'json',
+          where: JSON.stringify({
+            or: [
+              { field: 'keyword', is: ['iphrase_match', 'keyword1'] },
+              { field: 'keyword', is: ['iphrase_match', 'keyword2'] },
+            ],
+          }),
+        })
+        .reply(200, organicKeywordsResponse);
+
+      const result = await client.getOrganicKeywords('test-site.com', 'us', ['keyword1', 'keyword2']);
+
+      expect(result)
+        .to
+        .deep
+        .equal({
+          result: organicKeywordsResponse,
+          fullAuditRef: 'https://example.com/site-explorer/organic-keywords?country=us&date=2023-03-12&select=keyword%2Csum_traffic%2Cbest_position_url&order_by=sum_traffic%3Adesc&target=test-site.com&limit=200&mode=prefix&output=json&where=%7B%22or%22%3A%5B%7B%22field%22%3A%22keyword%22%2C%22is%22%3A%5B%22iphrase_match%22%2C%22keyword1%22%5D%7D%2C%7B%22field%22%3A%22keyword%22%2C%22is%22%3A%5B%22iphrase_match%22%2C%22keyword2%22%5D%7D%5D%7D',
+        });
+    });
+
+    it('sends API request with no keyword filter if none are specified', async () => {
+      nock(config.apiBaseUrl)
+        .get('/site-explorer/organic-keywords')
+        .query({
+          country: 'us',
+          date: new Date().toISOString().split('T')[0],
+          select: [
+            'keyword',
+            'sum_traffic',
+            'best_position_url',
+          ].join(','),
+          order_by: 'sum_traffic:desc',
+          target: 'test-site.com',
+          limit: 200,
+          mode: 'prefix',
+          output: 'json',
+        })
+        .reply(200, organicKeywordsResponse);
+
+      const result = await client.getOrganicKeywords('test-site.com');
+
+      expect(result)
+        .to
+        .deep
+        .equal({
+          result: organicKeywordsResponse,
+          fullAuditRef: 'https://example.com/site-explorer/organic-keywords?country=us&date=2023-03-12&select=keyword%2Csum_traffic%2Cbest_position_url&order_by=sum_traffic%3Adesc&target=test-site.com&limit=200&mode=prefix&output=json',
+        });
+    });
+
+    it('throws error when keyword filter does not contain appropriate keyword items', async () => {
+      const result = client.getOrganicKeywords('test-site.com', 'us', [BigInt(123)]);
+      await expect(result).to.be.rejectedWith('Error parsing keyword filter: Do not know how to serialize a BigInt');
+    });
+
+    it('throws error when keyword filter is not an array', async () => {
+      const result = client.getOrganicKeywords('test-site.com', 'us', 'keyword1');
+      await expect(result).to.be.rejectedWith('Invalid keyword filter: keyword1');
+    });
+
+    it('throws error when url is not a string', async () => {
+      const result = client.getOrganicKeywords(123);
+      await expect(result).to.be.rejectedWith('Invalid URL: 123');
+    });
+
+    it('throws error when country is not a string', async () => {
+      const result = client.getOrganicKeywords('test-site.com', 123);
+      await expect(result).to.be.rejectedWith('Invalid country: 123');
+    });
+
+    it('throws error when limit is not an integer', async () => {
+      const result = client.getOrganicKeywords('test-site.com', 'us', [], 1.5);
+      await expect(result).to.be.rejectedWith('Invalid limit: 1.5');
+    });
+  });
 });

packages/spacecat-shared-data-access/CHANGELOG.md

@@ -1,3 +1,38 @@
+# [@adobe/spacecat-shared-data-access-v1.32.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v1.31.0...@adobe/spacecat-shared-data-access-v1.32.0) (2024-06-20)
+
+
+### Features
+
+* **audit-configs:** ess audits disabled by default ([#271](https://github.com/adobe/spacecat-shared/issues/271)) ([de6f83d](https://github.com/adobe/spacecat-shared/commit/de6f83db75135820b236d55d7d2b477f29ca5e43))
+
+# [@adobe/spacecat-shared-data-access-v1.31.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v1.30.0...@adobe/spacecat-shared-data-access-v1.31.0) (2024-06-20)
+
+
+### Features
+
+* overwrite broken-backlinks ([#270](https://github.com/adobe/spacecat-shared/issues/270)) ([62256f2](https://github.com/adobe/spacecat-shared/commit/62256f2f750a97ada3d6b624b0ec50865589f39f))
+
+# [@adobe/spacecat-shared-data-access-v1.30.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v1.29.2...@adobe/spacecat-shared-data-access-v1.30.0) (2024-06-19)
+
+
+### Features
+
+* introduce organic keywords for ahrefs client and top keyword in top pages ([#257](https://github.com/adobe/spacecat-shared/issues/257)) ([371f1c4](https://github.com/adobe/spacecat-shared/commit/371f1c475870fd2aac833f925236237a8b25c026))
+
+# [@adobe/spacecat-shared-data-access-v1.29.2](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v1.29.1...@adobe/spacecat-shared-data-access-v1.29.2) (2024-06-18)
+
+
+### Bug Fixes
+
+* [#268](https://github.com/adobe/spacecat-shared/issues/268) Import URL entities need a REDIRECT status, and additional properties for reporting ([#269](https://github.com/adobe/spacecat-shared/issues/269)) ([88bb491](https://github.com/adobe/spacecat-shared/commit/88bb49105a9b0a9cd297fe9667aff1df24bcee03))
+
+# [@adobe/spacecat-shared-data-access-v1.29.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v1.29.0...@adobe/spacecat-shared-data-access-v1.29.1) (2024-06-15)
+
+
+### Bug Fixes
+
+* **deps:** update external fixes ([#266](https://github.com/adobe/spacecat-shared/issues/266)) ([a89ab83](https://github.com/adobe/spacecat-shared/commit/a89ab83e1c108c10044f6d098526bafed67b88b2))
+
 # [@adobe/spacecat-shared-data-access-v1.29.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v1.28.0...@adobe/spacecat-shared-data-access-v1.29.0) (2024-06-13)
 
 

packages/spacecat-shared-data-access/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-data-access",
-  "version": "1.29.0",
+  "version": "1.32.0",
   "description": "Shared modules of the Spacecat Services - Data Access",
   "type": "module",
   "main": "src/index.js",
@@ -31,8 +31,8 @@
   "dependencies": {
     "@adobe/spacecat-shared-dynamo": "1.2.5",
     "@adobe/spacecat-shared-utils": "1.2.0",
-    "@aws-sdk/client-dynamodb": "3.592.0",
-    "@aws-sdk/lib-dynamodb": "3.592.0",
+    "@aws-sdk/client-dynamodb": "3.598.0",
+    "@aws-sdk/lib-dynamodb": "3.598.0",
     "@types/joi": "17.2.3",
     "joi": "17.13.1",
     "uuid": "10.0.0"

packages/spacecat-shared-data-access/src/dto/import-url.js

@@ -26,6 +26,9 @@ export const ImportUrlDto = {
     jobId: importUrl.getJobId(),
     url: importUrl.getUrl(),
     status: importUrl.getStatus(),
+    reason: importUrl.getReason(),
+    path: importUrl.getPath(),
+    file: importUrl.getFile(),
   }),
 
   /**
@@ -37,6 +40,9 @@ export const ImportUrlDto = {
       jobId: dynamoItem.jobId,
       url: dynamoItem.url,
       status: dynamoItem.status,
+      reason: dynamoItem.reason,
+      path: dynamoItem.path,
+      file: dynamoItem.file,
     };
     return createImportUrl(importUrlData);
   },

packages/spacecat-shared-data-access/src/dto/site-top-page.js

@@ -26,6 +26,7 @@ export const SiteTopPageDto = {
     siteId: siteTopPage.getSiteId(),
     url: siteTopPage.getURL(),
     traffic: siteTopPage.getTraffic(),
+    topKeyword: siteTopPage.getTopKeyword(),
     source: siteTopPage.getSource(),
     geo: siteTopPage.getGeo(),
     importedAt: siteTopPage.getImportedAt(),
@@ -34,13 +35,16 @@ export const SiteTopPageDto = {
 
   /**
    * Converts a DynamoDB item into a SiteTopPage object.
-   * @param {{siteId, url, traffic, source, geo, importedAt, SK: string}} item - DynamoDB item.
+   * @param {
+   *   {siteId, url, traffic, topKeyword, source, geo, importedAt, SK: string}
+   * } item - DynamoDB item.
    * @returns {SiteTopPage}
    */
   fromDynamoItem: (item) => createSiteTopPage({
     siteId: item.siteId,
     url: item.url,
     traffic: item.traffic,
+    topKeyword: item.topKeyword,
     source: item.source,
     geo: item.geo,
     importedAt: item.importedAt,

packages/spacecat-shared-data-access/src/index.d.ts

@@ -304,6 +304,12 @@ export interface SiteTopPage {
    */
   getTraffic: () => number;
 
+  /**
+   * Retrieves the keyword that brings the most organic traffic to the page.
+   * @returns {string} The keyword.
+   */
+  getTopKeyword: () => string;
+
   /**
    * Retrieves the source of the site top page.
    * @returns {string} The source.

packages/spacecat-shared-data-access/src/models/audit.js

@@ -22,6 +22,8 @@ export const AUDIT_TYPE_ORGANIC_TRAFFIC = 'organic-traffic';
 export const AUDIT_TYPE_CWV = 'cwv';
 export const AUDIT_TYPE_LHS_DESKTOP = 'lhs-desktop';
 export const AUDIT_TYPE_LHS_MOBILE = 'lhs-mobile';
+export const AUDIT_TYPE_EXPERIMENTATION_ESS_MONTHLY = 'experimentation-ess-monthly';
+export const AUDIT_TYPE_EXPERIMENTATION_ESS_DAILY = 'experimentation-ess-daily';
 
 const EXPIRES_IN_DAYS = 30;