From 0553512d391148e7439e768de73bb39d43a4e08f Mon Sep 17 00:00:00 2001 From: Ishaan Shah <70190533+ishaan812@users.noreply.github.com> Date: Tue, 22 Aug 2023 21:50:42 +0530 Subject: [PATCH 1/5] Update README.md (#1) * Update README.md * FEATURE: Added Elastic Search Runbook to README * Update Readme.md --- README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/README.md b/README.md index 366cdae..7d74505 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ + # GitHub OpenAPI Search The goal of this project is to provide a robust yet easy way to search Github for Swagger and OpenAPI definitions. Understanding that there is a lot of noise available, that we only care about OpenAPIs that validate, and that the Github API has rate limits that require you to automate the crawling over time. Providing a robust open-source solution that will crawl public Github repositories for machine-readable API definitions. @@ -10,4 +11,32 @@ The project will consist of developing an open-source API that allows you to pas - Octokit.JS - Jest +## Dev Runbook +Dependancies: NodeJS 18, npm, GithubAPIKey +How to get a Github API Key: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens + + 1. Pull the repository to your local setup + 2. Run `npm i` + 3. Make a `.env` file in the directory and add the variables: + **PORT**= **(port number you want to host the api)** + **GITHUB_API_KEY**= **(Github API key)** +4. Run `npm run build:watch` on one terminal. +5. On another terminal, run `npm run start` to start the server on the port specified on. +6. Now the nodejs server should be running! To test it just go to `localhost:{{PORT}}` and the text `TypeScript With Express` should be returned. + +## Setting up ElasticSearch Runbook (Dev/Local) +-**docker pull docker.elastic.co/elasticsearch/elasticsearch:8.8.2** +-**docker network create elastic** +-**docker run \ + -p 9200:9200 \ + -p 9300:9300 \ + -e "discovery.type=single-node" \ + -e "xpack.security.enabled=false" \ + docker.elastic.co/elasticsearch/elasticsearch:8.8.2** + +## API Endpoints +[![Run in Postman](https://run.pstmn.io/button.svg)](https://app.getpostman.com/run-collection/19841716-f1801bb7-b189-429b-a875-91b115d349a2?action=collection%2Ffork&source=rip_markdown&collection-url=entityId%3D19841716-f1801bb7-b189-429b-a875-91b115d349a2%26entityType%3Dcollection%26workspaceId%3D5ebe19fb-61d4-47a7-9cae-de3834853f6b) + - **/search?prompt={{prompt}}** : The active search endpoint. Add the search prompt in the prompt variable in query params. + - **/passive?q={}** : The passive search endpoint. Add the search prompt in the query param (q) to query the database. + 🚧Under Construction From 561555ae992a0bffec4dd09dc11de40affc4f994 Mon Sep 17 00:00:00 2001 From: Ishaan Shah <70190533+ishaan812@users.noreply.github.com> Date: Sat, 26 Aug 2023 11:59:23 +0530 Subject: [PATCH 2/5] Update README.md (#5) --- README.md | 59 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 7d74505..931d6e3 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,59 @@ + # GitHub OpenAPI Search -The goal of this project is to provide a robust yet easy way to search Github for Swagger and OpenAPI definitions. Understanding that there is a lot of noise available, that we only care about OpenAPIs that validate, and that the Github API has rate limits that require you to automate the crawling over time. Providing a robust open-source solution that will crawl public Github repositories for machine-readable API definitions. -The project will consist of developing an open-source API that allows you to pass in search parameters and then utilize the GitHub API to perform the search, helping simplify the search interface, make rate limits visible as part of the response, and handle conducting a search in an asynchronous way, allowing the user to make a call to initiate, but then separate calls to receive results over time as results come in, helping show outcomes over time. +The goal of this project is to provide a robust yet easy way to search Github for OpenAPI and Swagger definitions. Understanding that there is a lot of noise available, that we only care about OpenAPIs that validate, and that the Github API has rate limits that require you to automate the crawling over time. Providing a robust open-source solution that will crawl public Github repositories for machine-readable API definitions. +The project will consist of developing an open-source API that allows you to pass in search parameters and then utilize the GitHub API to perform the search, helping simplify the search interface, and handle conducting a search in an asynchronous way, allowing the user to make a call to initiate, but then separate calls to receive results over time as results come in, helping show outcomes over time. ## Tech Stack - - Node JS/Express JS - Typescript - Octokit.JS -- Jest +- Jest (For testing) +- Docker +- Python (Scripting) -## Dev Runbook -Dependancies: NodeJS 18, npm, GithubAPIKey +## Dev Runbook +Dependancies: NodeJS 19, npm, Github APIKey How to get a Github API Key: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens - 1. Pull the repository to your local setup +## Setting up OpenAPI Search with Docker Compose + +1. Clone the repository to your local setup +2. Make sure you have Docker installed locally. +3. Run `docker compose up` +4. Two Containers - Elasticsearch (The database container) and an instance of the server should have started. + +## Setting up the server manually + + 1. Clone the repository to your local setup 2. Run `npm i` 3. Make a `.env` file in the directory and add the variables: **PORT**= **(port number you want to host the api)** - **GITHUB_API_KEY**= **(Github API key)** + **GITHUB_API_KEY**= **(github API key)** + **ES_HOST**= **(determines location of elasticsearch db)** 4. Run `npm run build:watch` on one terminal. 5. On another terminal, run `npm run start` to start the server on the port specified on. -6. Now the nodejs server should be running! To test it just go to `localhost:{{PORT}}` and the text `TypeScript With Express` should be returned. - -## Setting up ElasticSearch Runbook (Dev/Local) --**docker pull docker.elastic.co/elasticsearch/elasticsearch:8.8.2** --**docker network create elastic** --**docker run \ - -p 9200:9200 \ - -p 9300:9300 \ - -e "discovery.type=single-node" \ - -e "xpack.security.enabled=false" \ - docker.elastic.co/elasticsearch/elasticsearch:8.8.2** +6. Now the nodejs server should be running! To test it just go to `localhost:{{PORT}}` and then you will be able to see the admin panel through which you can inference with some of the API's +7. Now to load the database with OpenAPI Files, run +`python scripts/seed_script.py` from the root of the folder. (Takes around 2-3hrs) + +## Setting up ElasticSearch locally (Manually) + 1. docker pull docker.elastic.co/elasticsearch/elasticsearch:8.8.2 + 2. docker network create elastic + 3. docker run \ + -p 9200:9200 \ + -p 9300:9300 \ + -e "discovery.type=single-node" \ + -e "xpack.security.enabled=false" \ + docker.elastic.co/elasticsearch/elasticsearch:8.8.2 + +## Loading Details +Currently, we are only indexing OpenAPI Files from the top 1000 most popular organisations from Github (Based on stars). Although more organisations can be indexed by adding them to the `scripts/assets/organisations.txt` file. + ## API Endpoints [![Run in Postman](https://run.pstmn.io/button.svg)](https://app.getpostman.com/run-collection/19841716-f1801bb7-b189-429b-a875-91b115d349a2?action=collection%2Ffork&source=rip_markdown&collection-url=entityId%3D19841716-f1801bb7-b189-429b-a875-91b115d349a2%26entityType%3Dcollection%26workspaceId%3D5ebe19fb-61d4-47a7-9cae-de3834853f6b) - - **/search?prompt={{prompt}}** : The active search endpoint. Add the search prompt in the prompt variable in query params. - - **/passive?q={}** : The passive search endpoint. Add the search prompt in the query param (q) to query the database. + 🚧Under Construction From f5e400aeb445a7e059de03b81d2eb08877cc86e2 Mon Sep 17 00:00:00 2001 From: Ishaan Shah <70190533+ishaan812@users.noreply.github.com> Date: Mon, 28 Aug 2023 18:29:10 +0530 Subject: [PATCH 3/5] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 931d6e3..1e13cf1 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ The project will consist of developing an open-source API that allows you to pas - Jest (For testing) - Docker - Python (Scripting) +- ElasticSearch ## Dev Runbook Dependancies: NodeJS 19, npm, Github APIKey @@ -23,6 +24,9 @@ How to get a Github API Key: https://docs.github.com/en/authentication/keeping-y 2. Make sure you have Docker installed locally. 3. Run `docker compose up` 4. Two Containers - Elasticsearch (The database container) and an instance of the server should have started. +5. Now to load the database with OpenAPI Files, run +`python scripts/seed_script.py` from the root of the folder. (Takes around 2-3hrs) +(More configuration of organisation list you can edit the scripts/assets/organisations1.txt, scripts/assets/organisations2.txt is for the next 1000 organisations) ## Setting up the server manually From 986f3a4efd3af31d389656d095e118a7557c84b8 Mon Sep 17 00:00:00 2001 From: Ishaan Shah <70190533+ishaan812@users.noreply.github.com> Date: Sun, 5 Nov 2023 14:11:57 +0530 Subject: [PATCH 4/5] FEATURE: Added PassiveSearch v1 (#2) * FEATURE: Added PassiveSearch v1 - Added Connection to ElasticSearch - Added 2 API's - /ping and /passive - Keyword Search using /passive - Added Storing to DB capability when using /search * Fix: Added error handling -added error handling for passivesearch api endpoint * FEATURE: Added Simple Query String - Changed ElasticSearch Search type to Simple Query String * REFACTOR: Improved Active Search -Made Active Search make better use of downtimes (Rate Limits or Validation Downtime) * FIX: Issues with Active Search -Fixed errors with parrallel processing --- package-lock.json | 901 ++++++++++++++++++++++++++++++--- package.json | 6 +- src/DB/dbutils.ts | 23 + src/app.ts | 50 +- src/routes/user.routes.ts | 10 - src/searchtools/search.ts | 107 +++- src/searchtools/searchutils.ts | 81 ++- 7 files changed, 1048 insertions(+), 130 deletions(-) create mode 100644 src/DB/dbutils.ts delete mode 100644 src/routes/user.routes.ts diff --git a/package-lock.json b/package-lock.json index 99963a1..d67d26b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,8 +9,12 @@ "version": "0.0.0", "license": "Apache-2.0", "dependencies": { + "@octokit/plugin-retry": "^6.0.0", + "@octokit/plugin-throttling": "^7.0.0", "dotenv": "^16.1.4", + "elasticsearch": "^16.7.3", "express": "^4.18.2", + "fs": "^0.0.1-security", "oas-normalize": "^8.4.1", "octokit": "^2.0.19", "tslib": "~2.5" @@ -18,7 +22,7 @@ "devDependencies": { "@types/express": "^4.17.17", "@types/jest": "~29.5", - "@types/node": "~18", + "@types/node": "^18.16.19", "@types/supertest": "^2.0.12", "@typescript-eslint/eslint-plugin": "~5.59", "@typescript-eslint/parser": "~5.59", @@ -1281,6 +1285,44 @@ "node": ">= 14" } }, + "node_modules/@octokit/app/node_modules/@octokit/auth-token": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.4.tgz", + "integrity": "sha512-TWFX7cZF2LXoCvdmJWY7XVPi74aSY0+FfBZNSXEXFkMpjcqsQwDSYVv5FhRFaI0V1ECnwbz4j59T/G+rXNWaIQ==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/@octokit/app/node_modules/@octokit/core": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.4.tgz", + "integrity": "sha512-rYKilwgzQ7/imScn3M9/pFfUf4I1AZEH3KhyJmtPdE2zfaXAn2mFfUy4FbKewzc2We5y/LlKLj36fWJLKC2SIQ==", + "dependencies": { + "@octokit/auth-token": "^3.0.0", + "@octokit/graphql": "^5.0.0", + "@octokit/request": "^6.0.0", + "@octokit/request-error": "^3.0.0", + "@octokit/types": "^9.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@octokit/app/node_modules/@octokit/graphql": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.6.tgz", + "integrity": "sha512-Fxyxdy/JH0MnIB5h+UQ3yCoh1FG4kWXfFKkpWqjZHw/p+Kc8Y44Hu/kCgNBT6nU1shNumEchmW/sUO1JuQnPcw==", + "dependencies": { + "@octokit/request": "^6.0.0", + "@octokit/types": "^9.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/@octokit/auth-app": { "version": "4.0.13", "resolved": "https://registry.npmjs.org/@octokit/auth-app/-/auth-app-4.0.13.tgz", @@ -1356,11 +1398,12 @@ } }, "node_modules/@octokit/auth-token": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.4.tgz", - "integrity": "sha512-TWFX7cZF2LXoCvdmJWY7XVPi74aSY0+FfBZNSXEXFkMpjcqsQwDSYVv5FhRFaI0V1ECnwbz4j59T/G+rXNWaIQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", + "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==", + "peer": true, "engines": { - "node": ">= 14" + "node": ">= 18" } }, "node_modules/@octokit/auth-unauthenticated": { @@ -1376,20 +1419,89 @@ } }, "node_modules/@octokit/core": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.1.tgz", - "integrity": "sha512-tEDxFx8E38zF3gT7sSMDrT1tGumDgsw5yPG6BBh/X+5ClIQfMH/Yqocxz1PnHx6CHyF6pxmovUTOfZAUvQ0Lvw==", - "dependencies": { - "@octokit/auth-token": "^3.0.0", - "@octokit/graphql": "^5.0.0", - "@octokit/request": "^6.0.0", - "@octokit/request-error": "^3.0.0", - "@octokit/types": "^9.0.0", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.0.0.tgz", + "integrity": "sha512-YbAtMWIrbZ9FCXbLwT9wWB8TyLjq9mxpKdgB3dUNxQcIVTf9hJ70gRPwAcqGZdY6WdJPZ0I7jLaaNDCiloGN2A==", + "peer": true, + "dependencies": { + "@octokit/auth-token": "^4.0.0", + "@octokit/graphql": "^7.0.0", + "@octokit/request": "^8.0.2", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.0.0", "before-after-hook": "^2.2.0", "universal-user-agent": "^6.0.0" }, "engines": { - "node": ">= 14" + "node": ">= 18" + } + }, + "node_modules/@octokit/core/node_modules/@octokit/endpoint": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.0.tgz", + "integrity": "sha512-szrQhiqJ88gghWY2Htt8MqUDO6++E/EIXqJ2ZEp5ma3uGS46o7LZAzSLt49myB7rT+Hfw5Y6gO3LmOxGzHijAQ==", + "peer": true, + "dependencies": { + "@octokit/types": "^11.0.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/core/node_modules/@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==", + "peer": true + }, + "node_modules/@octokit/core/node_modules/@octokit/request": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.1.0.tgz", + "integrity": "sha512-0gg/NwewU0iXctYBale0VVcCPqOtoW5lsog8cNBJgzV/CyTHa2gicUBOlNnzOk6pJkuwXI34qkq+uRm40PmD4A==", + "peer": true, + "dependencies": { + "@octokit/endpoint": "^9.0.0", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.1.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/core/node_modules/@octokit/request-error": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.0.tgz", + "integrity": "sha512-1ue0DH0Lif5iEqT52+Rf/hf0RmGO9NWFjrzmrkArpG9trFfDM/efx00BJHdLGuro4BR/gECxCU2Twf5OKrRFsQ==", + "peer": true, + "dependencies": { + "@octokit/types": "^11.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/core/node_modules/@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "peer": true, + "dependencies": { + "@octokit/openapi-types": "^18.0.0" + } + }, + "node_modules/@octokit/core/node_modules/is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "peer": true, + "engines": { + "node": ">=0.10.0" } }, "node_modules/@octokit/endpoint": { @@ -1414,16 +1526,85 @@ } }, "node_modules/@octokit/graphql": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.6.tgz", - "integrity": "sha512-Fxyxdy/JH0MnIB5h+UQ3yCoh1FG4kWXfFKkpWqjZHw/p+Kc8Y44Hu/kCgNBT6nU1shNumEchmW/sUO1JuQnPcw==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.0.1.tgz", + "integrity": "sha512-T5S3oZ1JOE58gom6MIcrgwZXzTaxRnxBso58xhozxHpOqSTgDS6YNeEUvZ/kRvXgPrRz/KHnZhtb7jUMRi9E6w==", + "peer": true, "dependencies": { - "@octokit/request": "^6.0.0", - "@octokit/types": "^9.0.0", + "@octokit/request": "^8.0.1", + "@octokit/types": "^11.0.0", "universal-user-agent": "^6.0.0" }, "engines": { - "node": ">= 14" + "node": ">= 18" + } + }, + "node_modules/@octokit/graphql/node_modules/@octokit/endpoint": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.0.tgz", + "integrity": "sha512-szrQhiqJ88gghWY2Htt8MqUDO6++E/EIXqJ2ZEp5ma3uGS46o7LZAzSLt49myB7rT+Hfw5Y6gO3LmOxGzHijAQ==", + "peer": true, + "dependencies": { + "@octokit/types": "^11.0.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/graphql/node_modules/@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==", + "peer": true + }, + "node_modules/@octokit/graphql/node_modules/@octokit/request": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.1.0.tgz", + "integrity": "sha512-0gg/NwewU0iXctYBale0VVcCPqOtoW5lsog8cNBJgzV/CyTHa2gicUBOlNnzOk6pJkuwXI34qkq+uRm40PmD4A==", + "peer": true, + "dependencies": { + "@octokit/endpoint": "^9.0.0", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.1.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/graphql/node_modules/@octokit/request-error": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.0.tgz", + "integrity": "sha512-1ue0DH0Lif5iEqT52+Rf/hf0RmGO9NWFjrzmrkArpG9trFfDM/efx00BJHdLGuro4BR/gECxCU2Twf5OKrRFsQ==", + "peer": true, + "dependencies": { + "@octokit/types": "^11.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/graphql/node_modules/@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "peer": true, + "dependencies": { + "@octokit/openapi-types": "^18.0.0" + } + }, + "node_modules/@octokit/graphql/node_modules/is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "peer": true, + "engines": { + "node": ">=0.10.0" } }, "node_modules/@octokit/oauth-app": { @@ -1445,6 +1626,44 @@ "node": ">= 14" } }, + "node_modules/@octokit/oauth-app/node_modules/@octokit/auth-token": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.4.tgz", + "integrity": "sha512-TWFX7cZF2LXoCvdmJWY7XVPi74aSY0+FfBZNSXEXFkMpjcqsQwDSYVv5FhRFaI0V1ECnwbz4j59T/G+rXNWaIQ==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/@octokit/oauth-app/node_modules/@octokit/core": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.4.tgz", + "integrity": "sha512-rYKilwgzQ7/imScn3M9/pFfUf4I1AZEH3KhyJmtPdE2zfaXAn2mFfUy4FbKewzc2We5y/LlKLj36fWJLKC2SIQ==", + "dependencies": { + "@octokit/auth-token": "^3.0.0", + "@octokit/graphql": "^5.0.0", + "@octokit/request": "^6.0.0", + "@octokit/request-error": "^3.0.0", + "@octokit/types": "^9.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@octokit/oauth-app/node_modules/@octokit/graphql": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.6.tgz", + "integrity": "sha512-Fxyxdy/JH0MnIB5h+UQ3yCoh1FG4kWXfFKkpWqjZHw/p+Kc8Y44Hu/kCgNBT6nU1shNumEchmW/sUO1JuQnPcw==", + "dependencies": { + "@octokit/request": "^6.0.0", + "@octokit/types": "^9.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/@octokit/oauth-authorization-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/@octokit/oauth-authorization-url/-/oauth-authorization-url-5.0.0.tgz", @@ -1503,33 +1722,73 @@ } }, "node_modules/@octokit/plugin-retry": { - "version": "4.1.6", - "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-4.1.6.tgz", - "integrity": "sha512-obkYzIgEC75r8+9Pnfiiqy3y/x1bc3QLE5B7qvv9wi9Kj0R5tGQFC6QMBg1154WQ9lAVypuQDGyp3hNpp15gQQ==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-6.0.0.tgz", + "integrity": "sha512-a1/A4A+PB1QoAHQfLJxGHhLfSAT03bR1jJz3GgQJZvty2ozawFWs93MiBQXO7SL2YbO7CIq0Goj4qLOBj8JeMQ==", "dependencies": { - "@octokit/types": "^9.0.0", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.0.0", "bottleneck": "^2.15.3" }, "engines": { - "node": ">= 14" + "node": ">= 18" }, "peerDependencies": { - "@octokit/core": ">=3" + "@octokit/core": ">=5" + } + }, + "node_modules/@octokit/plugin-retry/node_modules/@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==" + }, + "node_modules/@octokit/plugin-retry/node_modules/@octokit/request-error": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.0.tgz", + "integrity": "sha512-1ue0DH0Lif5iEqT52+Rf/hf0RmGO9NWFjrzmrkArpG9trFfDM/efx00BJHdLGuro4BR/gECxCU2Twf5OKrRFsQ==", + "dependencies": { + "@octokit/types": "^11.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/plugin-retry/node_modules/@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "dependencies": { + "@octokit/openapi-types": "^18.0.0" } }, "node_modules/@octokit/plugin-throttling": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-5.2.3.tgz", - "integrity": "sha512-C9CFg9mrf6cugneKiaI841iG8DOv6P5XXkjmiNNut+swePxQ7RWEdAZRp5rJoE1hjsIqiYcKa/ZkOQ+ujPI39Q==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-7.0.0.tgz", + "integrity": "sha512-KL2k/d0uANc8XqP5S64YcNFCudR3F5AaKO39XWdUtlJIjT9Ni79ekWJ6Kj5xvAw87udkOMEPcVf9xEge2+ahew==", "dependencies": { - "@octokit/types": "^9.0.0", + "@octokit/types": "^11.0.0", "bottleneck": "^2.15.3" }, "engines": { - "node": ">= 14" + "node": ">= 18" }, "peerDependencies": { - "@octokit/core": "^4.0.0" + "@octokit/core": "^5.0.0" + } + }, + "node_modules/@octokit/plugin-throttling/node_modules/@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==" + }, + "node_modules/@octokit/plugin-throttling/node_modules/@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "dependencies": { + "@octokit/openapi-types": "^18.0.0" } }, "node_modules/@octokit/request": { @@ -1936,9 +2195,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "18.15.13", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.13.tgz", - "integrity": "sha512-N+0kuo9KgrUQ1Sn/ifDXsvg0TTleP7rIy4zOBGECxAljqvqfqpTfzx0Q1NUedOixRMBfe2Whhb056a42cWs26Q==" + "version": "18.16.19", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.19.tgz", + "integrity": "sha512-IXl7o+R9iti9eBW4Wg2hx1xQDig183jj7YLn8F7udNceyfkbn1ZxmzZXuak20gR40D7pIkIY1kYGx5VIGbaHKA==" }, "node_modules/@types/prettier": { "version": "2.7.2", @@ -2260,6 +2519,17 @@ "node": ">=0.4.0" } }, + "node_modules/agentkeepalive": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz", + "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 4.0.0" + } + }, "node_modules/aggregate-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", @@ -4805,6 +5075,78 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, + "node_modules/elasticsearch": { + "version": "16.7.3", + "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.3.tgz", + "integrity": "sha512-e9kUNhwnIlu47fGAr4W6yZJbkpsgQJB0TqNK8rCANe1J4P65B1sGnbCFTgcKY3/dRgCWnuP1AJ4obvzW604xEQ==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "dependencies": { + "agentkeepalive": "^3.4.1", + "chalk": "^1.0.0", + "lodash": "^4.17.10" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/elasticsearch/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/elasticsearch/node_modules/ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/elasticsearch/node_modules/chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "dependencies": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/elasticsearch/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/elasticsearch/node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/elasticsearch/node_modules/supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/electron-to-chromium": { "version": "1.4.368", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.368.tgz", @@ -5640,6 +5982,11 @@ } ] }, + "node_modules/fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==" + }, "node_modules/fs-readdir-recursive": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fs-readdir-recursive/-/fs-readdir-recursive-1.1.0.tgz", @@ -5873,7 +6220,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", - "dev": true, "dependencies": { "ansi-regex": "^2.0.0" }, @@ -5885,7 +6231,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -6055,6 +6400,14 @@ "node": ">=10.17.0" } }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "dependencies": { + "ms": "^2.0.0" + } + }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -8098,6 +8451,74 @@ "node": ">= 14" } }, + "node_modules/octokit/node_modules/@octokit/auth-token": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.4.tgz", + "integrity": "sha512-TWFX7cZF2LXoCvdmJWY7XVPi74aSY0+FfBZNSXEXFkMpjcqsQwDSYVv5FhRFaI0V1ECnwbz4j59T/G+rXNWaIQ==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/octokit/node_modules/@octokit/core": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.4.tgz", + "integrity": "sha512-rYKilwgzQ7/imScn3M9/pFfUf4I1AZEH3KhyJmtPdE2zfaXAn2mFfUy4FbKewzc2We5y/LlKLj36fWJLKC2SIQ==", + "dependencies": { + "@octokit/auth-token": "^3.0.0", + "@octokit/graphql": "^5.0.0", + "@octokit/request": "^6.0.0", + "@octokit/request-error": "^3.0.0", + "@octokit/types": "^9.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/octokit/node_modules/@octokit/graphql": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.6.tgz", + "integrity": "sha512-Fxyxdy/JH0MnIB5h+UQ3yCoh1FG4kWXfFKkpWqjZHw/p+Kc8Y44Hu/kCgNBT6nU1shNumEchmW/sUO1JuQnPcw==", + "dependencies": { + "@octokit/request": "^6.0.0", + "@octokit/types": "^9.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/octokit/node_modules/@octokit/plugin-retry": { + "version": "4.1.6", + "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-4.1.6.tgz", + "integrity": "sha512-obkYzIgEC75r8+9Pnfiiqy3y/x1bc3QLE5B7qvv9wi9Kj0R5tGQFC6QMBg1154WQ9lAVypuQDGyp3hNpp15gQQ==", + "dependencies": { + "@octokit/types": "^9.0.0", + "bottleneck": "^2.15.3" + }, + "engines": { + "node": ">= 14" + }, + "peerDependencies": { + "@octokit/core": ">=3" + } + }, + "node_modules/octokit/node_modules/@octokit/plugin-throttling": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-5.2.3.tgz", + "integrity": "sha512-C9CFg9mrf6cugneKiaI841iG8DOv6P5XXkjmiNNut+swePxQ7RWEdAZRp5rJoE1hjsIqiYcKa/ZkOQ+ujPI39Q==", + "dependencies": { + "@octokit/types": "^9.0.0", + "bottleneck": "^2.15.3" + }, + "engines": { + "node": ">= 14" + }, + "peerDependencies": { + "@octokit/core": "^4.0.0" + } + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -11498,6 +11919,37 @@ "@octokit/plugin-paginate-rest": "^6.0.0", "@octokit/types": "^9.0.0", "@octokit/webhooks": "^10.0.0" + }, + "dependencies": { + "@octokit/auth-token": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.4.tgz", + "integrity": "sha512-TWFX7cZF2LXoCvdmJWY7XVPi74aSY0+FfBZNSXEXFkMpjcqsQwDSYVv5FhRFaI0V1ECnwbz4j59T/G+rXNWaIQ==" + }, + "@octokit/core": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.4.tgz", + "integrity": "sha512-rYKilwgzQ7/imScn3M9/pFfUf4I1AZEH3KhyJmtPdE2zfaXAn2mFfUy4FbKewzc2We5y/LlKLj36fWJLKC2SIQ==", + "requires": { + "@octokit/auth-token": "^3.0.0", + "@octokit/graphql": "^5.0.0", + "@octokit/request": "^6.0.0", + "@octokit/request-error": "^3.0.0", + "@octokit/types": "^9.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/graphql": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.6.tgz", + "integrity": "sha512-Fxyxdy/JH0MnIB5h+UQ3yCoh1FG4kWXfFKkpWqjZHw/p+Kc8Y44Hu/kCgNBT6nU1shNumEchmW/sUO1JuQnPcw==", + "requires": { + "@octokit/request": "^6.0.0", + "@octokit/types": "^9.0.0", + "universal-user-agent": "^6.0.0" + } + } } }, "@octokit/auth-app": { @@ -11562,9 +12014,10 @@ } }, "@octokit/auth-token": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.4.tgz", - "integrity": "sha512-TWFX7cZF2LXoCvdmJWY7XVPi74aSY0+FfBZNSXEXFkMpjcqsQwDSYVv5FhRFaI0V1ECnwbz4j59T/G+rXNWaIQ==" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", + "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==", + "peer": true }, "@octokit/auth-unauthenticated": { "version": "3.0.5", @@ -11576,17 +12029,76 @@ } }, "@octokit/core": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.1.tgz", - "integrity": "sha512-tEDxFx8E38zF3gT7sSMDrT1tGumDgsw5yPG6BBh/X+5ClIQfMH/Yqocxz1PnHx6CHyF6pxmovUTOfZAUvQ0Lvw==", - "requires": { - "@octokit/auth-token": "^3.0.0", - "@octokit/graphql": "^5.0.0", - "@octokit/request": "^6.0.0", - "@octokit/request-error": "^3.0.0", - "@octokit/types": "^9.0.0", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.0.0.tgz", + "integrity": "sha512-YbAtMWIrbZ9FCXbLwT9wWB8TyLjq9mxpKdgB3dUNxQcIVTf9hJ70gRPwAcqGZdY6WdJPZ0I7jLaaNDCiloGN2A==", + "peer": true, + "requires": { + "@octokit/auth-token": "^4.0.0", + "@octokit/graphql": "^7.0.0", + "@octokit/request": "^8.0.2", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.0.0", "before-after-hook": "^2.2.0", "universal-user-agent": "^6.0.0" + }, + "dependencies": { + "@octokit/endpoint": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.0.tgz", + "integrity": "sha512-szrQhiqJ88gghWY2Htt8MqUDO6++E/EIXqJ2ZEp5ma3uGS46o7LZAzSLt49myB7rT+Hfw5Y6gO3LmOxGzHijAQ==", + "peer": true, + "requires": { + "@octokit/types": "^11.0.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==", + "peer": true + }, + "@octokit/request": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.1.0.tgz", + "integrity": "sha512-0gg/NwewU0iXctYBale0VVcCPqOtoW5lsog8cNBJgzV/CyTHa2gicUBOlNnzOk6pJkuwXI34qkq+uRm40PmD4A==", + "peer": true, + "requires": { + "@octokit/endpoint": "^9.0.0", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.1.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/request-error": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.0.tgz", + "integrity": "sha512-1ue0DH0Lif5iEqT52+Rf/hf0RmGO9NWFjrzmrkArpG9trFfDM/efx00BJHdLGuro4BR/gECxCU2Twf5OKrRFsQ==", + "peer": true, + "requires": { + "@octokit/types": "^11.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "peer": true, + "requires": { + "@octokit/openapi-types": "^18.0.0" + } + }, + "is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "peer": true + } } }, "@octokit/endpoint": { @@ -11607,13 +12119,72 @@ } }, "@octokit/graphql": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.6.tgz", - "integrity": "sha512-Fxyxdy/JH0MnIB5h+UQ3yCoh1FG4kWXfFKkpWqjZHw/p+Kc8Y44Hu/kCgNBT6nU1shNumEchmW/sUO1JuQnPcw==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.0.1.tgz", + "integrity": "sha512-T5S3oZ1JOE58gom6MIcrgwZXzTaxRnxBso58xhozxHpOqSTgDS6YNeEUvZ/kRvXgPrRz/KHnZhtb7jUMRi9E6w==", + "peer": true, "requires": { - "@octokit/request": "^6.0.0", - "@octokit/types": "^9.0.0", + "@octokit/request": "^8.0.1", + "@octokit/types": "^11.0.0", "universal-user-agent": "^6.0.0" + }, + "dependencies": { + "@octokit/endpoint": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.0.tgz", + "integrity": "sha512-szrQhiqJ88gghWY2Htt8MqUDO6++E/EIXqJ2ZEp5ma3uGS46o7LZAzSLt49myB7rT+Hfw5Y6gO3LmOxGzHijAQ==", + "peer": true, + "requires": { + "@octokit/types": "^11.0.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==", + "peer": true + }, + "@octokit/request": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.1.0.tgz", + "integrity": "sha512-0gg/NwewU0iXctYBale0VVcCPqOtoW5lsog8cNBJgzV/CyTHa2gicUBOlNnzOk6pJkuwXI34qkq+uRm40PmD4A==", + "peer": true, + "requires": { + "@octokit/endpoint": "^9.0.0", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.1.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/request-error": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.0.tgz", + "integrity": "sha512-1ue0DH0Lif5iEqT52+Rf/hf0RmGO9NWFjrzmrkArpG9trFfDM/efx00BJHdLGuro4BR/gECxCU2Twf5OKrRFsQ==", + "peer": true, + "requires": { + "@octokit/types": "^11.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "peer": true, + "requires": { + "@octokit/openapi-types": "^18.0.0" + } + }, + "is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "peer": true + } } }, "@octokit/oauth-app": { @@ -11630,6 +12201,37 @@ "@types/aws-lambda": "^8.10.83", "fromentries": "^1.3.1", "universal-user-agent": "^6.0.0" + }, + "dependencies": { + "@octokit/auth-token": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.4.tgz", + "integrity": "sha512-TWFX7cZF2LXoCvdmJWY7XVPi74aSY0+FfBZNSXEXFkMpjcqsQwDSYVv5FhRFaI0V1ECnwbz4j59T/G+rXNWaIQ==" + }, + "@octokit/core": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.4.tgz", + "integrity": "sha512-rYKilwgzQ7/imScn3M9/pFfUf4I1AZEH3KhyJmtPdE2zfaXAn2mFfUy4FbKewzc2We5y/LlKLj36fWJLKC2SIQ==", + "requires": { + "@octokit/auth-token": "^3.0.0", + "@octokit/graphql": "^5.0.0", + "@octokit/request": "^6.0.0", + "@octokit/request-error": "^3.0.0", + "@octokit/types": "^9.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/graphql": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.6.tgz", + "integrity": "sha512-Fxyxdy/JH0MnIB5h+UQ3yCoh1FG4kWXfFKkpWqjZHw/p+Kc8Y44Hu/kCgNBT6nU1shNumEchmW/sUO1JuQnPcw==", + "requires": { + "@octokit/request": "^6.0.0", + "@octokit/types": "^9.0.0", + "universal-user-agent": "^6.0.0" + } + } } }, "@octokit/oauth-authorization-url": { @@ -11672,21 +12274,62 @@ } }, "@octokit/plugin-retry": { - "version": "4.1.6", - "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-4.1.6.tgz", - "integrity": "sha512-obkYzIgEC75r8+9Pnfiiqy3y/x1bc3QLE5B7qvv9wi9Kj0R5tGQFC6QMBg1154WQ9lAVypuQDGyp3hNpp15gQQ==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-6.0.0.tgz", + "integrity": "sha512-a1/A4A+PB1QoAHQfLJxGHhLfSAT03bR1jJz3GgQJZvty2ozawFWs93MiBQXO7SL2YbO7CIq0Goj4qLOBj8JeMQ==", "requires": { - "@octokit/types": "^9.0.0", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.0.0", "bottleneck": "^2.15.3" + }, + "dependencies": { + "@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==" + }, + "@octokit/request-error": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.0.tgz", + "integrity": "sha512-1ue0DH0Lif5iEqT52+Rf/hf0RmGO9NWFjrzmrkArpG9trFfDM/efx00BJHdLGuro4BR/gECxCU2Twf5OKrRFsQ==", + "requires": { + "@octokit/types": "^11.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "requires": { + "@octokit/openapi-types": "^18.0.0" + } + } } }, "@octokit/plugin-throttling": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-5.2.3.tgz", - "integrity": "sha512-C9CFg9mrf6cugneKiaI841iG8DOv6P5XXkjmiNNut+swePxQ7RWEdAZRp5rJoE1hjsIqiYcKa/ZkOQ+ujPI39Q==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-7.0.0.tgz", + "integrity": "sha512-KL2k/d0uANc8XqP5S64YcNFCudR3F5AaKO39XWdUtlJIjT9Ni79ekWJ6Kj5xvAw87udkOMEPcVf9xEge2+ahew==", "requires": { - "@octokit/types": "^9.0.0", + "@octokit/types": "^11.0.0", "bottleneck": "^2.15.3" + }, + "dependencies": { + "@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==" + }, + "@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "requires": { + "@octokit/openapi-types": "^18.0.0" + } + } } }, "@octokit/request": { @@ -12053,9 +12696,9 @@ "dev": true }, "@types/node": { - "version": "18.15.13", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.13.tgz", - "integrity": "sha512-N+0kuo9KgrUQ1Sn/ifDXsvg0TTleP7rIy4zOBGECxAljqvqfqpTfzx0Q1NUedOixRMBfe2Whhb056a42cWs26Q==" + "version": "18.16.19", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.19.tgz", + "integrity": "sha512-IXl7o+R9iti9eBW4Wg2hx1xQDig183jj7YLn8F7udNceyfkbn1ZxmzZXuak20gR40D7pIkIY1kYGx5VIGbaHKA==" }, "@types/prettier": { "version": "2.7.2", @@ -12274,6 +12917,14 @@ "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", "dev": true }, + "agentkeepalive": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz", + "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==", + "requires": { + "humanize-ms": "^1.2.1" + } + }, "aggregate-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", @@ -14402,6 +15053,58 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, + "elasticsearch": { + "version": "16.7.3", + "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.3.tgz", + "integrity": "sha512-e9kUNhwnIlu47fGAr4W6yZJbkpsgQJB0TqNK8rCANe1J4P65B1sGnbCFTgcKY3/dRgCWnuP1AJ4obvzW604xEQ==", + "requires": { + "agentkeepalive": "^3.4.1", + "chalk": "^1.0.0", + "lodash": "^4.17.10" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==" + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==" + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==" + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==" + } + } + }, "electron-to-chromium": { "version": "1.4.368", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.368.tgz", @@ -15041,6 +15744,11 @@ "resolved": "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz", "integrity": "sha512-cHEpEQHUg0f8XdtZCc2ZAhrHzKzT0MrFUTcvx+hfxYu7rGMDc5SKoXFh+n4YigxsHXRzc6OrCshdR1bWH6HHyg==" }, + "fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==" + }, "fs-readdir-recursive": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fs-readdir-recursive/-/fs-readdir-recursive-1.1.0.tgz", @@ -15215,7 +15923,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", - "dev": true, "requires": { "ansi-regex": "^2.0.0" }, @@ -15223,8 +15930,7 @@ "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "dev": true + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==" } } }, @@ -15354,6 +16060,14 @@ "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", "dev": true }, + "humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "requires": { + "ms": "^2.0.0" + } + }, "iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -16918,6 +17632,55 @@ "@octokit/plugin-retry": "^4.1.3", "@octokit/plugin-throttling": "^5.2.2", "@octokit/types": "^9.2.2" + }, + "dependencies": { + "@octokit/auth-token": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.4.tgz", + "integrity": "sha512-TWFX7cZF2LXoCvdmJWY7XVPi74aSY0+FfBZNSXEXFkMpjcqsQwDSYVv5FhRFaI0V1ECnwbz4j59T/G+rXNWaIQ==" + }, + "@octokit/core": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.4.tgz", + "integrity": "sha512-rYKilwgzQ7/imScn3M9/pFfUf4I1AZEH3KhyJmtPdE2zfaXAn2mFfUy4FbKewzc2We5y/LlKLj36fWJLKC2SIQ==", + "requires": { + "@octokit/auth-token": "^3.0.0", + "@octokit/graphql": "^5.0.0", + "@octokit/request": "^6.0.0", + "@octokit/request-error": "^3.0.0", + "@octokit/types": "^9.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/graphql": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.6.tgz", + "integrity": "sha512-Fxyxdy/JH0MnIB5h+UQ3yCoh1FG4kWXfFKkpWqjZHw/p+Kc8Y44Hu/kCgNBT6nU1shNumEchmW/sUO1JuQnPcw==", + "requires": { + "@octokit/request": "^6.0.0", + "@octokit/types": "^9.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/plugin-retry": { + "version": "4.1.6", + "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-4.1.6.tgz", + "integrity": "sha512-obkYzIgEC75r8+9Pnfiiqy3y/x1bc3QLE5B7qvv9wi9Kj0R5tGQFC6QMBg1154WQ9lAVypuQDGyp3hNpp15gQQ==", + "requires": { + "@octokit/types": "^9.0.0", + "bottleneck": "^2.15.3" + } + }, + "@octokit/plugin-throttling": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-5.2.3.tgz", + "integrity": "sha512-C9CFg9mrf6cugneKiaI841iG8DOv6P5XXkjmiNNut+swePxQ7RWEdAZRp5rJoE1hjsIqiYcKa/ZkOQ+ujPI39Q==", + "requires": { + "@octokit/types": "^9.0.0", + "bottleneck": "^2.15.3" + } + } } }, "on-finished": { diff --git a/package.json b/package.json index 983b03f..d88c7d9 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "devDependencies": { "@types/express": "^4.17.17", "@types/jest": "~29.5", - "@types/node": "~18", + "@types/node": "^18.16.19", "@types/supertest": "^2.0.12", "@typescript-eslint/eslint-plugin": "~5.59", "@typescript-eslint/parser": "~5.59", @@ -44,8 +44,12 @@ "author": "Ishaan Shah ", "license": "Apache-2.0", "dependencies": { + "@octokit/plugin-retry": "^6.0.0", + "@octokit/plugin-throttling": "^7.0.0", "dotenv": "^16.1.4", + "elasticsearch": "^16.7.3", "express": "^4.18.2", + "fs": "^0.0.1-security", "oas-normalize": "^8.4.1", "octokit": "^2.0.19", "tslib": "~2.5" diff --git a/src/DB/dbutils.ts b/src/DB/dbutils.ts new file mode 100644 index 0000000..da9171b --- /dev/null +++ b/src/DB/dbutils.ts @@ -0,0 +1,23 @@ +export async function checkClusterHealth(esClient: any) : Promise { + try { + const response = await esClient.cat.health(); + console.log('Cluster health:', response); + return response; + } catch (error) { + console.error('Error checking cluster health:', error); + return ""; + } + } + +export async function BulkStoreToDB(validFiles: any,esClient: any) : Promise{ + try { + if(validFiles.length == 0){ + return; + } + const response = await esClient.bulk({ body: validFiles }); + // console.log('Bulk response:', response); + return response; + } catch (error) { + console.error('Error bulk indexing:', error); + } +} \ No newline at end of file diff --git a/src/app.ts b/src/app.ts index e4c53b7..b35073d 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,36 +1,78 @@ import express from 'express'; import { Octokit } from 'octokit'; -import { activeSearch } from './searchtools/search.js'; +import { activeSearch, passiveSearch } from './searchtools/search.js'; import dotenv from 'dotenv'; +import es from 'elasticsearch'; +import { checkClusterHealth } from './DB/dbutils.js'; // import { router as userRoutes } from "./routes/user.routes.js"; +import { throttling } from '@octokit/plugin-throttling' +import { retry } from '@octokit/plugin-retry' + + +const CustomOctokit = Octokit.plugin(throttling as any, retry as any); dotenv.config(); -const app = express(); -const octokit = new Octokit({ +const octokit = new CustomOctokit({ userAgent: 'github-openapi-search/v0.0.1', auth: process.env.GITHUB_API_KEY, + throttle: { + onRateLimit: (retryAfter, options) => { + octokit.log.warn( + `Request quota exhausted for request ${options.method} ${options.url}`, + ); + console.log(`Retrying after ${retryAfter} seconds!`); + return true; + }, + onSecondaryRateLimit: (retryAfter, options, octokit) => { + // does not retry, only logs a warning + octokit.log.warn( + `Secondary quota detected for request ${options.method} ${options.url}`, + ); + }, + }, +}); + +const app = express(); + + + +const esClient = new es.Client({ + host: 'http://localhost:9200', + log: 'trace', }); // Should not even be an API endpoint for passive search // Should just fetch repositories and go through them (with ETAG to make sure no repeats) // Check for openapi.json in the contents of the repository // If it exists, then store in database with important content + +app.use('/passive', async (_req, _res) => { + const query = _req.query.q as string; + const results = await passiveSearch(query, esClient); + _res.send(results); +}) + app.use('/search', async (_req, _res) => { const Repository = _req.query.repo as string; const Organisation = _req.query.org as string; const User = _req.query.user as string; const Prompt = _req.query.prompt as string; - const results = await activeSearch( Prompt as string, Repository as string, Organisation as string, User as string, + esClient as any, ); _res.send(results); }); +app.use('/ping', async (_req, _res) => { + const response = await checkClusterHealth(esClient); + _res.send(response); +}) + app.get('/', (_req, _res) => { _res.send('TypeScript With Express'); }); diff --git a/src/routes/user.routes.ts b/src/routes/user.routes.ts deleted file mode 100644 index 885a39b..0000000 --- a/src/routes/user.routes.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Router, Response } from 'express'; - -const router = Router(); - -router.get('/', (_req, _res): Response => { - const users = ['Goon', 'Tsuki', 'Joe']; - return _res.send(users); -}); - -export { router }; diff --git a/src/searchtools/search.ts b/src/searchtools/search.ts index 9084447..c05eb37 100644 --- a/src/searchtools/search.ts +++ b/src/searchtools/search.ts @@ -1,39 +1,90 @@ -import { getFileContents, handleCodeSearch } from './searchutils.js'; -import OASNormalize from 'oas-normalize'; +import { queryBuilder, ValidateandStoreFiles} from './searchutils.js'; +import { octokit } from '../app.js'; + + +let processCount = 1; +let finishedCount = 1; export async function activeSearch( prompt: string, repo: string, organisation: string, username: string, + esClient: any, ): Promise { - //3 types of searches: - // If repository is specified, then search through that repository: /search/code with repo name - // If organisation is specified then search through that organisation: /search/code with org name - // If user is specified then search through that user: /search/code with user name - // If none is specified then return error - const validFiles = []; - const files = await handleCodeSearch(prompt, repo, organisation, username, 1); - for (const file of files) { - const base64content = await getFileContents( - file.repository.owner.login, - file.repository.name, - file.path, - ); - const content = Buffer.from(base64content, 'base64').toString(); - const oas = new OASNormalize.default(content); - oas - .validate() - .then((definition) => { - console.log('File ' + file.name + ' is valid'); - validFiles.push(definition); - console.log(validFiles); - }) - .catch((error) => { - // Error will be an array of validation errors. - console.log('File ' + file.name + ' is not valid'); + const query = await queryBuilder(prompt, repo, organisation, username); + let files = []; + let validFiles = []; + await octokit.paginate(octokit.rest.search.code, { + q: query, + per_page: 100 + }, + (response : any) => { + files = files.concat(response.data) + if(files.length >= 500){ + console.log("Validating and storing files since rate limit reached") + processCount++; + ValidateandStoreFiles(files, esClient).then((validatedFiles) => { + validFiles = validFiles.concat(validatedFiles); + finishedCount++; }); + files = [] + } + } + ); + //this ending before the above one + ValidateandStoreFiles(files, esClient).then((validatedFiles) => { + validFiles = validFiles.concat(validatedFiles); + }); + while(processCount > finishedCount){ + await new Promise(r => setTimeout(r, 3000)); + console.log("Total Processes: "+processCount+"\nFinished Processes: "+finishedCount) + console.log("Waiting for all files to be processed") } - return validFiles; } + +export async function passiveSearch( + query: string, + esClient: any, +): Promise { + try { + if (esClient === undefined) { + throw new Error('Invalid Elasticsearch client'); + } + const result = await esClient.search({ + index: 'openapi', + body: { + query: { + simple_query_string: { + query: query, + fields: ["servers^2","paths^1.5","data^1"], + default_operator: "and" + } + } + } + }); + + if (result.hits.hits) { + if (result.hits.hits.length === 0) { + console.log('No results found in the database'); + // activeSearch(query, "", "", "", esClient); + } + return result.hits.hits; + } + } catch (error) { + if (error.message.includes('No Living connections')) { + console.log('Elasticsearch connection error:', error); + return error + } else { + console.log('Error occurred during passive search:', error); + return error; + } + } + + return 'Database not found'; +} + + + + diff --git a/src/searchtools/searchutils.ts b/src/searchtools/searchutils.ts index aa3b4ed..c5c419c 100644 --- a/src/searchtools/searchutils.ts +++ b/src/searchtools/searchutils.ts @@ -1,5 +1,25 @@ import { octokit } from '../app.js'; // import { CodeSearchResponse } from "./searchstructs.js"; +import crypto from "crypto"; +import OASNormalize from 'oas-normalize'; +import { BulkStoreToDB } from '../DB/dbutils.js'; + + + + +export function generateUUID() : string { + // Generate a random buffer of 16 bytes + const buffer = crypto.randomBytes(16); + + // Set the version (4) and variant (2) bits + buffer[6] = (buffer[6] & 0x0f) | 0x40; + buffer[8] = (buffer[8] & 0x3f) | 0x80; + + // Convert the buffer to a string representation of the UUID + const uuid = buffer.toString('hex').match(/(.{8})(.{4})(.{4})(.{4})(.{12})/); + uuid.shift(); + return uuid.join('-'); +} export async function getFileContents( repoowner: string, @@ -17,30 +37,55 @@ export async function getFileContents( return response.data['content']; } -export async function handleCodeSearch( - prompt: string, - repo: string, - organisation: string, - username: string, - page: number, -): Promise { - //Why does /(openai|swagger)/ not work :/ - //Even Parenthesis doesn't work +export async function queryBuilder(prompt: string, repo: string, organisation: string, username: string): Promise { + if(prompt == undefined){ + prompt = "" + } let query = prompt + ' AND "openapi: 3"'; - // query+= prompt + ' AND "swagger: \\"2"' + // query+= prompt + ' AND "swagger: \\"2"' if (repo != undefined) { query += '+repo:' + repo; } else if (organisation != undefined) { query += '+org:' + organisation; } else if (username != undefined) { query += '+user:' + username; + } else { + return query; + } + return query; +} + +export async function ValidateandStoreFiles(files: any[], esClient: any): Promise { + if(files.length == 0){ + return; } - console.log('Query: ' + query); - //api call handling - const results = await octokit.paginate(octokit.rest.search.code, { - q: query, - per_page: 100, - }); - console.log(results.length) - return results; + console.log("Validating and storing files") + let validFiles = []; + for (const file of files) { + const base64content = await getFileContents( + file.repository.owner.login, + file.repository.name, + file.path, + ); + const content = Buffer.from(base64content, 'base64').toString(); + const oas = new OASNormalize.default(content); + oas + .validate() + .then((definition) => { + console.log('File ' + file.name + ' is valid'); + console.log(definition?.info?.title) + validFiles.push({index: { _index: 'openapi', _id: generateUUID()}}); + validFiles.push({title: definition?.info?.title, description: definition?.info?.description, version: definition?.info?.version, servers: JSON.stringify(definition?.servers), paths: JSON.stringify(definition?.paths) , path: file.path, repository: file?.repository?.name, owner: file?.repository?.owner?.login, data: content}); + if(validFiles.length >= 50){ + console.log("Storing some of the valid files") + BulkStoreToDB(validFiles as any[],esClient as any); + validFiles = [] + } + }) + .catch((error) => { + console.log('File ' + file.name + ' is not valid'); + }); + } + BulkStoreToDB(validFiles as any[],esClient as any); + return validFiles; } From aa559b90f95a6d4a7fd9ab1add155f093bb0bd1b Mon Sep 17 00:00:00 2001 From: Ishaan Shah <70190533+ishaan812@users.noreply.github.com> Date: Sun, 5 Nov 2023 14:14:37 +0530 Subject: [PATCH 5/5] FEATURE: Added seed script (#3) * FEATURE: Added PassiveSearch v1 - Added Connection to ElasticSearch - Added 2 API's - /ping and /passive - Keyword Search using /passive - Added Storing to DB capability when using /search * Fix: Added error handling -added error handling for passivesearch api endpoint * FEATURE: Added Simple Query String - Changed ElasticSearch Search type to Simple Query String * REFACTOR: Improved Active Search -Made Active Search make better use of downtimes (Rate Limits or Validation Downtime) * FIX: Issues with Active Search -Fixed errors with parrallel processing * FEATURE: Added SeedScript - Added RootQuery as an optional query param for activesearch which basically takes out the prompt builder and takes a raw query to look for in Github API. - Added Python SeedScript --- scripts/seed_script.py | 31 +++++++++++++++++++++++++++++++ src/app.ts | 4 +++- src/searchtools/search.ts | 13 ++++++++++--- src/searchtools/searchutils.ts | 9 +++++++-- 4 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 scripts/seed_script.py diff --git a/scripts/seed_script.py b/scripts/seed_script.py new file mode 100644 index 0000000..21f45d2 --- /dev/null +++ b/scripts/seed_script.py @@ -0,0 +1,31 @@ +# Description: This script is used to seed the database with data from the data folder +# !pip install requests +# python scripts/seed_script.py to run from main folder + +import requests + +def call_local_endpoint(prompt): + #TODO: Change this to the correct URL when activesearch endpoint is changed + url = f'http://localhost:8080/search?rootquery="{prompt}"' + + try: + response = requests.get(url) + + # Check if the response was successful (status code 200) + if response.status_code == 200: + print("Request to localhost:8080/search was successful!") + print("Response content:") + print(response.text) + else: + print(f"Request to localhost:8080/active failed with status code: {response.status_code}") + + except requests.exceptions.RequestException as e: + print(f"An error occurred: {e}") + +if __name__ == "__main__": + #Get Open API files + call_local_endpoint("openapi: 3") + #Get Swagger files + call_local_endpoint("swagger: 2") + +#PS: Takes a long time to run \ No newline at end of file diff --git a/src/app.ts b/src/app.ts index b35073d..623778e 100644 --- a/src/app.ts +++ b/src/app.ts @@ -4,7 +4,6 @@ import { activeSearch, passiveSearch } from './searchtools/search.js'; import dotenv from 'dotenv'; import es from 'elasticsearch'; import { checkClusterHealth } from './DB/dbutils.js'; -// import { router as userRoutes } from "./routes/user.routes.js"; import { throttling } from '@octokit/plugin-throttling' import { retry } from '@octokit/plugin-retry' @@ -47,6 +46,7 @@ const esClient = new es.Client({ // Check for openapi.json in the contents of the repository // If it exists, then store in database with important content + app.use('/passive', async (_req, _res) => { const query = _req.query.q as string; const results = await passiveSearch(query, esClient); @@ -58,11 +58,13 @@ app.use('/search', async (_req, _res) => { const Organisation = _req.query.org as string; const User = _req.query.user as string; const Prompt = _req.query.prompt as string; + const RootQuery = _req.query.rootquery as string; const results = await activeSearch( Prompt as string, Repository as string, Organisation as string, User as string, + RootQuery as string, esClient as any, ); _res.send(results); diff --git a/src/searchtools/search.ts b/src/searchtools/search.ts index c05eb37..cdf1e2d 100644 --- a/src/searchtools/search.ts +++ b/src/searchtools/search.ts @@ -10,31 +10,38 @@ export async function activeSearch( repo: string, organisation: string, username: string, + rootquery: string, esClient: any, ): Promise { - const query = await queryBuilder(prompt, repo, organisation, username); + const query = await queryBuilder(prompt, repo, organisation, username, rootquery); let files = []; let validFiles = []; + console.log("Query: "+query) await octokit.paginate(octokit.rest.search.code, { q: query, per_page: 100 }, (response : any) => { files = files.concat(response.data) - if(files.length >= 500){ - console.log("Validating and storing files since rate limit reached") + if(files.length >= 200){ processCount++; + console.log("ValidateandStoreFiles Process Number "+processCount+" Started") ValidateandStoreFiles(files, esClient).then((validatedFiles) => { validFiles = validFiles.concat(validatedFiles); finishedCount++; + console.log("ValidateandStoreFiles Process Number "+finishedCount+" Started") }); files = [] } } ); //this ending before the above one + processCount++; + console.log("ValidateandStoreFiles Process Number "+processCount+" Started") ValidateandStoreFiles(files, esClient).then((validatedFiles) => { validFiles = validFiles.concat(validatedFiles); + console.log("ValidateandStoreFiles Process Number "+finishedCount+" Started") + finishedCount++; }); while(processCount > finishedCount){ await new Promise(r => setTimeout(r, 3000)); diff --git a/src/searchtools/searchutils.ts b/src/searchtools/searchutils.ts index c5c419c..bbe42ad 100644 --- a/src/searchtools/searchutils.ts +++ b/src/searchtools/searchutils.ts @@ -37,11 +37,16 @@ export async function getFileContents( return response.data['content']; } -export async function queryBuilder(prompt: string, repo: string, organisation: string, username: string): Promise { +export async function queryBuilder(prompt: string, repo: string, organisation: string, username: string, rootquery: string): Promise { if(prompt == undefined){ prompt = "" } - let query = prompt + ' AND "openapi: 3"'; + let query + if(rootquery != undefined){ + query = rootquery + return query + } + query = prompt + ' AND "openapi: 3"'; // query+= prompt + ' AND "swagger: \\"2"' if (repo != undefined) { query += '+repo:' + repo;