Merge branch 'main' into overwrite-broken-backlinks

packages/spacecat-shared-ahrefs-client/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [@adobe/spacecat-shared-ahrefs-client-v1.3.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-ahrefs-client-v1.2.6...@adobe/spacecat-shared-ahrefs-client-v1.3.0) (2024-06-19)
+
+
+### Features
+
+* introduce organic keywords for ahrefs client and top keyword in top pages ([#257](https://github.com/adobe/spacecat-shared/issues/257)) ([371f1c4](https://github.com/adobe/spacecat-shared/commit/371f1c475870fd2aac833f925236237a8b25c026))
+
 # [@adobe/spacecat-shared-ahrefs-client-v1.2.6](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-ahrefs-client-v1.2.5...@adobe/spacecat-shared-ahrefs-client-v1.2.6) (2024-06-11)
 
 

packages/spacecat-shared-ahrefs-client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-ahrefs-client",
-  "version": "1.2.6",
+  "version": "1.3.0",
   "description": "Shared modules of the Spacecat Services - Ahrefs Client",
   "type": "module",
   "main": "src/index.js",

packages/spacecat-shared-ahrefs-client/src/index.d.ts

@@ -70,4 +70,10 @@ export default class AhrefsAPIClient {
      */
   getBacklinks(url: string, limit?: number):
       Promise<{ result: object, fullAuditRef: string }>;
+
+  /**
+   * Asynchronous method to get organic keywords.
+   */
+  getOrganicKeywords(url: string, country?: string, keywordFilter?: string[], limit?: number):
+      Promise<{ result: object, fullAuditRef: string }>;
 }

packages/spacecat-shared-ahrefs-client/src/index.js

@@ -10,7 +10,7 @@
  * governing permissions and limitations under the License.
  */
 
-import { isValidUrl } from '@adobe/spacecat-shared-utils';
+import { hasText, isValidUrl, isArray } from '@adobe/spacecat-shared-utils';
 import { context as h2, h1 } from '@adobe/fetch';
 
 /* c8 ignore next 3 */
@@ -120,6 +120,7 @@ export default class AhrefsAPIClient {
       select: [
         'url',
         'sum_traffic',
+        'top_keyword',
       ].join(','),
       order_by: 'sum_traffic',
       date: new Date().toISOString().split('T')[0],
@@ -174,4 +175,45 @@ export default class AhrefsAPIClient {
 
     return this.sendRequest('/site-explorer/metrics-history', queryParams);
   }
+
+  async getOrganicKeywords(url, country = 'us', keywordFilter = [], limit = 200) {
+    if (!hasText(url)) {
+      throw new Error(`Invalid URL: ${url}`);
+    }
+    if (!hasText(country)) {
+      throw new Error(`Invalid country: ${country}`);
+    }
+    if (!isArray(keywordFilter)) {
+      throw new Error(`Invalid keyword filter: ${keywordFilter}`);
+    }
+    if (!Number.isInteger(limit) || limit < 1) {
+      throw new Error(`Invalid limit: ${limit}`);
+    }
+    const queryParams = {
+      country,
+      date: new Date().toISOString().split('T')[0],
+      select: [
+        'keyword',
+        'sum_traffic',
+        'best_position_url',
+      ].join(','),
+      order_by: 'sum_traffic:desc',
+      target: url,
+      limit: getLimit(limit, 2000),
+      mode: 'prefix',
+      output: 'json',
+    };
+    if (keywordFilter.length > 0) {
+      try {
+        queryParams.where = JSON.stringify({
+          or: keywordFilter.map((keyword) => ({ field: 'keyword', is: ['iphrase_match', keyword] })),
+        });
+      } catch (e) {
+        this.log.error(`Error parsing keyword filter: ${e.message}`);
+        throw new Error(`Error parsing keyword filter: ${e.message}`);
+      }
+    }
+
+    return this.sendRequest('/site-explorer/organic-keywords', queryParams);
+  }
 }

packages/spacecat-shared-ahrefs-client/test/index.test.js

@@ -52,10 +52,12 @@ describe('AhrefsAPIClient', () => {
       {
         url: 'page-url-1',
         sum_traffic: 100,
+        top_keyword: 'keyword1',
       },
       {
         url: 'page-url-2',
         sum_traffic: 300,
+        top_keyword: 'keyword2',
       },
     ],
   };
@@ -79,6 +81,21 @@ describe('AhrefsAPIClient', () => {
     ],
   };
 
+  const organicKeywordsResponse = {
+    keywords: [
+      {
+        keyword: 'keyword1',
+        sum_traffic: 100,
+        best_position_url: 'url1',
+      },
+      {
+        keyword: 'keyword2',
+        sum_traffic: 200,
+        best_position_url: 'url2',
+      },
+    ],
+  };
+
   before('setup', function () {
     this.clock = sandbox.useFakeTimers({
       now: new Date(mockDate).getTime(),
@@ -203,6 +220,7 @@ describe('AhrefsAPIClient', () => {
         select: [
           'url',
           'sum_traffic',
+          'top_keyword',
         ].join(','),
         where: JSON.stringify(filter),
         order_by: 'sum_traffic',
@@ -221,7 +239,7 @@ describe('AhrefsAPIClient', () => {
       const result = await client.getTopPages(target, specifiedLimit);
       expect(result).to.deep.equal({
         result: topPagesResponse,
-        fullAuditRef: `https://example.com/site-explorer/top-pages?select=url%2Csum_traffic&order_by=sum_traffic&date=${date}&target=${target}&limit=${specifiedLimit}&mode=prefix&output=json&where=%7B%22and%22%3A%5B%7B%22field%22%3A%22sum_traffic%22%2C%22is%22%3A%5B%22gt%22%2C0%5D%7D%5D%7D`,
+        fullAuditRef: `https://example.com/site-explorer/top-pages?select=url%2Csum_traffic%2Ctop_keyword&order_by=sum_traffic&date=${date}&target=${target}&limit=${specifiedLimit}&mode=prefix&output=json&where=%7B%22and%22%3A%5B%7B%22field%22%3A%22sum_traffic%22%2C%22is%22%3A%5B%22gt%22%2C0%5D%7D%5D%7D`,
       });
     });
   });
@@ -288,4 +306,97 @@ describe('AhrefsAPIClient', () => {
       });
     });
   });
+
+  describe('getOrganicKeywords', () => {
+    it('sends API request with appropriate endpoint query params', async () => {
+      nock(config.apiBaseUrl)
+        .get('/site-explorer/organic-keywords')
+        .query({
+          country: 'us',
+          date: new Date().toISOString().split('T')[0],
+          select: [
+            'keyword',
+            'sum_traffic',
+            'best_position_url',
+          ].join(','),
+          order_by: 'sum_traffic:desc',
+          target: 'test-site.com',
+          limit: 200,
+          mode: 'prefix',
+          output: 'json',
+          where: JSON.stringify({
+            or: [
+              { field: 'keyword', is: ['iphrase_match', 'keyword1'] },
+              { field: 'keyword', is: ['iphrase_match', 'keyword2'] },
+            ],
+          }),
+        })
+        .reply(200, organicKeywordsResponse);
+
+      const result = await client.getOrganicKeywords('test-site.com', 'us', ['keyword1', 'keyword2']);
+
+      expect(result)
+        .to
+        .deep
+        .equal({
+          result: organicKeywordsResponse,
+          fullAuditRef: 'https://example.com/site-explorer/organic-keywords?country=us&date=2023-03-12&select=keyword%2Csum_traffic%2Cbest_position_url&order_by=sum_traffic%3Adesc&target=test-site.com&limit=200&mode=prefix&output=json&where=%7B%22or%22%3A%5B%7B%22field%22%3A%22keyword%22%2C%22is%22%3A%5B%22iphrase_match%22%2C%22keyword1%22%5D%7D%2C%7B%22field%22%3A%22keyword%22%2C%22is%22%3A%5B%22iphrase_match%22%2C%22keyword2%22%5D%7D%5D%7D',
+        });
+    });
+
+    it('sends API request with no keyword filter if none are specified', async () => {
+      nock(config.apiBaseUrl)
+        .get('/site-explorer/organic-keywords')
+        .query({
+          country: 'us',
+          date: new Date().toISOString().split('T')[0],
+          select: [
+            'keyword',
+            'sum_traffic',
+            'best_position_url',
+          ].join(','),
+          order_by: 'sum_traffic:desc',
+          target: 'test-site.com',
+          limit: 200,
+          mode: 'prefix',
+          output: 'json',
+        })
+        .reply(200, organicKeywordsResponse);
+
+      const result = await client.getOrganicKeywords('test-site.com');
+
+      expect(result)
+        .to
+        .deep
+        .equal({
+          result: organicKeywordsResponse,
+          fullAuditRef: 'https://example.com/site-explorer/organic-keywords?country=us&date=2023-03-12&select=keyword%2Csum_traffic%2Cbest_position_url&order_by=sum_traffic%3Adesc&target=test-site.com&limit=200&mode=prefix&output=json',
+        });
+    });
+
+    it('throws error when keyword filter does not contain appropriate keyword items', async () => {
+      const result = client.getOrganicKeywords('test-site.com', 'us', [BigInt(123)]);
+      await expect(result).to.be.rejectedWith('Error parsing keyword filter: Do not know how to serialize a BigInt');
+    });
+
+    it('throws error when keyword filter is not an array', async () => {
+      const result = client.getOrganicKeywords('test-site.com', 'us', 'keyword1');
+      await expect(result).to.be.rejectedWith('Invalid keyword filter: keyword1');
+    });
+
+    it('throws error when url is not a string', async () => {
+      const result = client.getOrganicKeywords(123);
+      await expect(result).to.be.rejectedWith('Invalid URL: 123');
+    });
+
+    it('throws error when country is not a string', async () => {
+      const result = client.getOrganicKeywords('test-site.com', 123);
+      await expect(result).to.be.rejectedWith('Invalid country: 123');
+    });
+
+    it('throws error when limit is not an integer', async () => {
+      const result = client.getOrganicKeywords('test-site.com', 'us', [], 1.5);
+      await expect(result).to.be.rejectedWith('Invalid limit: 1.5');
+    });
+  });
 });

packages/spacecat-shared-data-access/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [@adobe/spacecat-shared-data-access-v1.30.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v1.29.2...@adobe/spacecat-shared-data-access-v1.30.0) (2024-06-19)
+
+
+### Features
+
+* introduce organic keywords for ahrefs client and top keyword in top pages ([#257](https://github.com/adobe/spacecat-shared/issues/257)) ([371f1c4](https://github.com/adobe/spacecat-shared/commit/371f1c475870fd2aac833f925236237a8b25c026))
+
 # [@adobe/spacecat-shared-data-access-v1.29.2](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-data-access-v1.29.1...@adobe/spacecat-shared-data-access-v1.29.2) (2024-06-18)
 
 

packages/spacecat-shared-data-access/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-data-access",
-  "version": "1.29.2",
+  "version": "1.30.0",
   "description": "Shared modules of the Spacecat Services - Data Access",
   "type": "module",
   "main": "src/index.js",

packages/spacecat-shared-data-access/src/dto/site-top-page.js

@@ -26,6 +26,7 @@ export const SiteTopPageDto = {
     siteId: siteTopPage.getSiteId(),
     url: siteTopPage.getURL(),
     traffic: siteTopPage.getTraffic(),
+    topKeyword: siteTopPage.getTopKeyword(),
     source: siteTopPage.getSource(),
     geo: siteTopPage.getGeo(),
     importedAt: siteTopPage.getImportedAt(),
@@ -34,13 +35,16 @@ export const SiteTopPageDto = {
 
   /**
    * Converts a DynamoDB item into a SiteTopPage object.
-   * @param {{siteId, url, traffic, source, geo, importedAt, SK: string}} item - DynamoDB item.
+   * @param {
+   *   {siteId, url, traffic, topKeyword, source, geo, importedAt, SK: string}
+   * } item - DynamoDB item.
    * @returns {SiteTopPage}
    */
   fromDynamoItem: (item) => createSiteTopPage({
     siteId: item.siteId,
     url: item.url,
     traffic: item.traffic,
+    topKeyword: item.topKeyword,
     source: item.source,
     geo: item.geo,
     importedAt: item.importedAt,

packages/spacecat-shared-data-access/src/index.d.ts

@@ -347,6 +347,12 @@ export interface SiteTopPage {
    */
   getTraffic: () => number;
 
+  /**
+   * Retrieves the keyword that brings the most organic traffic to the page.
+   * @returns {string} The keyword.
+   */
+  getTopKeyword: () => string;
+
   /**
    * Retrieves the source of the site top page.
    * @returns {string} The source.

packages/spacecat-shared-data-access/src/models/site-top-page.js

@@ -23,6 +23,7 @@ const SiteTopPage = (data = {}) => {
   self.getSiteId = () => self.state.siteId;
   self.getURL = () => self.state.url;
   self.getTraffic = () => self.state.traffic;
+  self.getTopKeyword = () => self.state.topKeyword;
   self.getSource = () => self.state.source.toLowerCase();
   self.getGeo = () => self.state.geo;
   self.getImportedAt = () => self.state.importedAt;

packages/spacecat-shared-data-access/test/it/db.test.js

@@ -83,6 +83,7 @@ function checkSiteTopPage(siteTopPage) {
   expect(siteTopPage.getSiteId()).to.be.a('string');
   expect(siteTopPage.getURL()).to.be.a('string');
   expect(siteTopPage.getTraffic()).to.be.a('number');
+  expect(siteTopPage.getTopKeyword()).to.be.a('string');
   expect(siteTopPage.getSource()).to.be.a('string');
   expect(siteTopPage.getGeo()).to.be.a('string');
   expect(isIsoDate(siteTopPage.getImportedAt())).to.be.true;
@@ -768,6 +769,7 @@ describe('DynamoDB Integration Test', async () => {
       siteId,
       url: 'https://example12345.com/page-12345',
       traffic: 360420000,
+      topKeyword: 'keyword12345',
       source: 'rum',
       geo: 'au',
       importedAt: new Date().toISOString(),

packages/spacecat-shared-data-access/test/it/generateSampleData.js

@@ -324,6 +324,7 @@ export default async function generateSampleData(
       SK: `ahrefs#global#${String(traffic).padStart(12, '0')}`,
       url: `${sites[i % numberOfSites].baseURL}/page-${i}`,
       traffic,
+      topKeyword: `keyword-${i}`,
       source: 'ahrefs',
       geo: 'global',
       importedAt: nowIso,

packages/spacecat-shared-data-access/test/unit/models/site-top-page.test.js

@@ -19,6 +19,7 @@ const validData = {
   siteId: 'site123',
   url: 'https://www.example.com',
   traffic: 1000,
+  topKeyword: 'keyword',
   source: 'rum',
   geo: 'au',
   importedAt: new Date().toISOString(),

packages/spacecat-shared-data-access/test/unit/service/site-top-pages/index.test.js

@@ -75,6 +75,7 @@ describe('Site Top Pages Access Pattern Tests', () => {
         url: 'https://www.example.com',
         traffic: 1000,
         source: 'gsc',
+        topKeyword: 'keyword',
         geo: 'au',
         importedAt: new Date().toISOString(),
       };
@@ -93,6 +94,7 @@ describe('Site Top Pages Access Pattern Tests', () => {
       expect(result[0].getSiteId()).to.equal(siteTopPageData.siteId);
       expect(result[0].getURL()).to.equal(siteTopPageData.url);
       expect(result[0].getTraffic()).to.equal(siteTopPageData.traffic);
+      expect(result[0].getTopKeyword()).to.equal(siteTopPageData.topKeyword);
       expect(result[0].getSource()).to.equal(siteTopPageData.source);
       expect(result[0].getGeo()).to.equal(siteTopPageData.geo);
       expect(result[0].getImportedAt()).to.equal(siteTopPageData.importedAt);
@@ -103,6 +105,7 @@ describe('Site Top Pages Access Pattern Tests', () => {
         siteId: 'site123',
         url: 'https://www.example.com',
         traffic: 1000,
+        topKeyword: 'keyword',
         source: 'gsc',
         geo: 'au',
         importedAt: new Date().toISOString(),
@@ -122,6 +125,7 @@ describe('Site Top Pages Access Pattern Tests', () => {
       expect(result[0].getSiteId()).to.equal(siteTopPageData.siteId);
       expect(result[0].getURL()).to.equal(siteTopPageData.url);
       expect(result[0].getTraffic()).to.equal(siteTopPageData.traffic);
+      expect(result[0].getTopKeyword()).to.equal(siteTopPageData.topKeyword);
       expect(result[0].getSource()).to.equal(siteTopPageData.source);
       expect(result[0].getGeo()).to.equal(siteTopPageData.geo);
       expect(result[0].getImportedAt()).to.equal(siteTopPageData.importedAt);
@@ -147,6 +151,7 @@ describe('Site Top Pages Access Pattern Tests', () => {
         siteId: 'site123',
         url: 'https://www.example.com',
         traffic: 1000,
+        topKeyword: 'keyword',
         source: 'rum',
         geo: 'us',
         importedAt: new Date().toISOString(),
@@ -162,6 +167,7 @@ describe('Site Top Pages Access Pattern Tests', () => {
       expect(result.getSiteId()).to.equal(siteTopPageData.siteId);
       expect(result.getURL()).to.equal(siteTopPageData.url);
       expect(result.getTraffic()).to.equal(siteTopPageData.traffic);
+      expect(result.getTopKeyword()).to.equal(siteTopPageData.topKeyword);
       expect(result.getSource()).to.equal(siteTopPageData.source);
       expect(result.getGeo()).to.equal(siteTopPageData.geo);
       expect(result.getImportedAt()).to.equal(siteTopPageData.importedAt);
@@ -172,6 +178,7 @@ describe('Site Top Pages Access Pattern Tests', () => {
         siteId: 'site123',
         url: 'https://www.example.com',
         traffic: 1000,
+        topKeyword: 'keyword',
         source: 'rum',
         geo: 'us',
         importedAt: new Date().toISOString(),