feat(specs): import crawler and usage specs from new world repo (#…

…2598)

scripts/buildSpecs.ts

@@ -52,11 +52,6 @@ async function transformBundle({
       }
 
       const docMethod = bundledDocSpec.paths[pathKey][method];
-      if (docMethod.summary) {
-        // Remove dot at the end of summary for better sidebar display
-        docMethod.summary = docMethod.summary.replace(/\.$/gm, '');
-      }
-
       if (!docMethod.tags) {
         continue;
       }

scripts/common.ts

@@ -73,7 +73,11 @@ export const GENERATORS = Object.entries(clientsConfig).reduce(
 
 export const LANGUAGES = [...new Set(Object.values(GENERATORS).map((gen) => gen.language))];
 
-export const CLIENTS = [...new Set(Object.values(GENERATORS).map((gen) => gen.client))];
+export const CLIENTS = [
+  ...new Set(Object.values(GENERATORS).map((gen) => gen.client)),
+  'usage',
+  'crawler',
+];
 
 export async function run(
   command: string,

specs/common/responses/Forbidden.yml

@@ -0,0 +1,35 @@
+description: Invalid credentials.
+content:
+  application/json:
+    schema:
+      type: object
+      properties:
+        error:
+          type: object
+          properties:
+            code:
+              type: string
+            message:
+              type: string
+            errors:
+              type: array
+              items:
+                type: object
+                properties:
+                  code:
+                    type: string
+                  message:
+                    type: string
+                  line:
+                    type: integer
+                  position:
+                    type: integer
+                required:
+                  - message
+                example:
+                  message: 'url is not defined'
+                  line: 5
+          example:
+            code: malformed_id
+      required:
+        - error

specs/common/responses/InvalidRequest.yml

@@ -0,0 +1,35 @@
+description: Invalid request.
+content:
+  application/json:
+    schema:
+      type: object
+      properties:
+        error:
+          type: object
+          properties:
+            code:
+              type: string
+            message:
+              type: string
+            errors:
+              type: array
+              items:
+                type: object
+                properties:
+                  code:
+                    type: string
+                  message:
+                    type: string
+                  line:
+                    type: integer
+                  position:
+                    type: integer
+                required:
+                  - message
+                example:
+                  message: 'url is not defined'
+                  line: 5
+          example:
+            code: malformed_id
+      required:
+        - error

specs/crawler/common/parameters.yml

@@ -0,0 +1,159 @@
+CrawlerIdParameter:
+  name: id
+  in: path
+  description: The Id of the targeted Crawler.
+  required: true
+  schema:
+    type: string
+
+TaskIdParameter:
+  name: tid
+  in: path
+  description: The Id of the targeted Task.
+  required: true
+  schema:
+    type: string
+
+CrawlerVersionParameter:
+  name: version
+  in: path
+  description: The version of the targeted Crawler revision.
+  required: true
+  schema:
+    type: integer
+
+CrawlerId:
+  type: string
+  description: The unique id of the Crawler.
+  example: 'e0f6db8a-24f5-4092-83a4-1b2c6cb6d809'
+
+CrawlerName:
+  type: string
+  maxLength: 64
+  description: The name of the Crawler.
+  example: 'My Crawler'
+
+Configuration:
+  type: object
+  description: A Crawler configuration object. See the Crawler documentation to have more details about it.
+  properties:
+    appId:
+      type: string
+      example: ABC9DEFGHI
+    apiKey:
+      type: string
+      example: c69564c68bad256f8d11399bf2048f82
+    indexPrefix:
+      type: string
+      example: crawler_
+    rateLimit:
+      type: number
+      example: 8
+    startUrls:
+      type: array
+      items:
+        type: string
+        example: https://www.algolia.com
+    actions:
+      type: array
+      items:
+        type: object
+        properties:
+          indexName:
+            type: string
+            example: algolia_website
+          pathsToMatch:
+            type: array
+            items:
+              type: string
+              example: https://www.algolia.com/**
+          selectorsToMatch:
+            type: array
+            items:
+              type: string
+            example: ['.products', '!.featured']
+          fileTypesToMatch:
+            type: array
+            items:
+              type: string
+            example: ['html', 'pdf']
+          recordExtractor:
+            type: object
+            properties:
+              __type:
+                $ref: '#/configurationRecordExtractorType'
+              source:
+                type: string
+                example: '() => {}'
+        required:
+          - indexName
+          - recordExtractor
+
+configurationRecordExtractorType:
+  type: string
+  enum:
+    - function
+
+UrlsCrawledGroup:
+  type: object
+  description: Represent a group of URLs that have been crawled and have the same final state.
+  properties:
+    status:
+      $ref: '#/urlsCrawledGroupStatus'
+    reason:
+      type: string
+      description: The code of the reason why when ended up in this status.
+    category:
+      $ref: '#/urlsCrawledGroupCategory'
+    count:
+      type: integer
+      description: Number of URLs belonging to this group.
+    readable:
+      type: string
+      description: Human redeable version of the error.
+  example:
+    status: SKIPPED
+    reason: forbidden_by_robotstxt
+    category: fetch
+    nbUrls: 3
+    readable: Forbidden by robots.txt
+
+urlsCrawledGroupStatus:
+  type: string
+  description: A string corresponding to the status of the group.
+  enum:
+    - DONE
+    - SKIPPED
+    - FAILED
+
+urlsCrawledGroupCategory:
+  type: string
+  description: In case of error, will be set to the step where the error occurred, otherwise will be set to 'success'.
+  enum: 
+    - fetch
+    - extraction
+    - indexing
+    - success
+
+Pagination:
+  type: object
+  description: Represent a group of items and pagination information.
+  properties:
+    items:
+      type: array
+      items:
+        type: object
+    itemsPerPage:
+      type: integer
+      description: The maximum number of items returned by this request.
+      default: 20
+      example: 20
+    page:
+      type: integer
+      description: The current page browsed by this request.
+      default: 1
+      example: 1
+    total:
+      type: integer
+      description: The total number of items.
+      example: 100

specs/crawler/common/schemas/crawlerConfigVersionsResponse.yml

@@ -0,0 +1,19 @@
+allOf:
+  - $ref: '../parameters.yml#/Pagination'
+  - type: object
+    properties:
+      items:
+        type: array
+        items:
+          type: object
+          properties:
+            version:
+              type: integer
+              minimum: 1
+            createdAt:
+              type: string
+            authorId:
+              type: string
+          required:
+            - version
+            - createdAt

specs/crawler/common/schemas/crawlersResponse.yml

@@ -0,0 +1,13 @@
+allOf:
+  - $ref: '../parameters.yml#/Pagination'
+  - type: object
+    properties:
+      items:
+        type: array
+        items:
+          type: object
+          properties:
+            id:
+              $ref: '../parameters.yml#/CrawlerId'
+            name:
+              $ref: '../parameters.yml#/CrawlerName'

specs/crawler/common/schemas/domainsResponse.yml

@@ -0,0 +1,18 @@
+allOf:
+  - $ref: '../parameters.yml#/Pagination'
+  - type: object
+    properties:
+      items:
+        type: array
+        items:
+          type: object
+          properties:
+            appId:
+              type: string
+              description: The app for which the domain is registered.
+            domain:
+              type: string
+              description: The registered domain name.
+            validated:
+              type: boolean
+              description: Indicates whether the domain is validated or not.

specs/crawler/common/schemas/responses.yml

@@ -0,0 +1,37 @@
+ActionAcknowledged:
+  description: The request has been acknowledged by the crawler.
+  content:
+    application/json:
+      schema:
+        type: object
+        properties:
+          taskId:
+            type: string
+            example: 'e0f6db8a-24f5-4092-83a4-1b2c6cb6d809'
+        required:
+          - taskId
+
+NoRightsOnCrawler:
+  description: The user doesn't have enough rights on the specified Crawler, or it doesn't exists.
+
+DocSearchCreateApp:
+  description: A new DocSearch app has been created.
+  content:
+    application/json:
+      schema:
+        additionalProperties: false
+        type: object
+        properties:
+          appId:
+            type: string
+          id:
+            type: number
+          name:
+            type: string
+        required:
+          - appId
+          - id
+          - name
+
+MissingAuthorization:
+  description: Authorization information is missing or invalid.