From 0553512d391148e7439e768de73bb39d43a4e08f Mon Sep 17 00:00:00 2001 From: Ishaan Shah <70190533+ishaan812@users.noreply.github.com> Date: Tue, 22 Aug 2023 21:50:42 +0530 Subject: [PATCH 1/7] Update README.md (#1) * Update README.md * FEATURE: Added Elastic Search Runbook to README * Update Readme.md --- README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/README.md b/README.md index 366cdae..7d74505 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ + # GitHub OpenAPI Search The goal of this project is to provide a robust yet easy way to search Github for Swagger and OpenAPI definitions. Understanding that there is a lot of noise available, that we only care about OpenAPIs that validate, and that the Github API has rate limits that require you to automate the crawling over time. Providing a robust open-source solution that will crawl public Github repositories for machine-readable API definitions. @@ -10,4 +11,32 @@ The project will consist of developing an open-source API that allows you to pas - Octokit.JS - Jest +## Dev Runbook +Dependancies: NodeJS 18, npm, GithubAPIKey +How to get a Github API Key: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens + + 1. Pull the repository to your local setup + 2. Run `npm i` + 3. Make a `.env` file in the directory and add the variables: + **PORT**= **(port number you want to host the api)** + **GITHUB_API_KEY**= **(Github API key)** +4. Run `npm run build:watch` on one terminal. +5. On another terminal, run `npm run start` to start the server on the port specified on. +6. Now the nodejs server should be running! To test it just go to `localhost:{{PORT}}` and the text `TypeScript With Express` should be returned. + +## Setting up ElasticSearch Runbook (Dev/Local) +-**docker pull docker.elastic.co/elasticsearch/elasticsearch:8.8.2** +-**docker network create elastic** +-**docker run \ + -p 9200:9200 \ + -p 9300:9300 \ + -e "discovery.type=single-node" \ + -e "xpack.security.enabled=false" \ + docker.elastic.co/elasticsearch/elasticsearch:8.8.2** + +## API Endpoints +[![Run in Postman](https://run.pstmn.io/button.svg)](https://app.getpostman.com/run-collection/19841716-f1801bb7-b189-429b-a875-91b115d349a2?action=collection%2Ffork&source=rip_markdown&collection-url=entityId%3D19841716-f1801bb7-b189-429b-a875-91b115d349a2%26entityType%3Dcollection%26workspaceId%3D5ebe19fb-61d4-47a7-9cae-de3834853f6b) + - **/search?prompt={{prompt}}** : The active search endpoint. Add the search prompt in the prompt variable in query params. + - **/passive?q={}** : The passive search endpoint. Add the search prompt in the query param (q) to query the database. + 🚧Under Construction From 561555ae992a0bffec4dd09dc11de40affc4f994 Mon Sep 17 00:00:00 2001 From: Ishaan Shah <70190533+ishaan812@users.noreply.github.com> Date: Sat, 26 Aug 2023 11:59:23 +0530 Subject: [PATCH 2/7] Update README.md (#5) --- README.md | 59 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 7d74505..931d6e3 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,59 @@ + # GitHub OpenAPI Search -The goal of this project is to provide a robust yet easy way to search Github for Swagger and OpenAPI definitions. Understanding that there is a lot of noise available, that we only care about OpenAPIs that validate, and that the Github API has rate limits that require you to automate the crawling over time. Providing a robust open-source solution that will crawl public Github repositories for machine-readable API definitions. -The project will consist of developing an open-source API that allows you to pass in search parameters and then utilize the GitHub API to perform the search, helping simplify the search interface, make rate limits visible as part of the response, and handle conducting a search in an asynchronous way, allowing the user to make a call to initiate, but then separate calls to receive results over time as results come in, helping show outcomes over time. +The goal of this project is to provide a robust yet easy way to search Github for OpenAPI and Swagger definitions. Understanding that there is a lot of noise available, that we only care about OpenAPIs that validate, and that the Github API has rate limits that require you to automate the crawling over time. Providing a robust open-source solution that will crawl public Github repositories for machine-readable API definitions. +The project will consist of developing an open-source API that allows you to pass in search parameters and then utilize the GitHub API to perform the search, helping simplify the search interface, and handle conducting a search in an asynchronous way, allowing the user to make a call to initiate, but then separate calls to receive results over time as results come in, helping show outcomes over time. ## Tech Stack - - Node JS/Express JS - Typescript - Octokit.JS -- Jest +- Jest (For testing) +- Docker +- Python (Scripting) -## Dev Runbook -Dependancies: NodeJS 18, npm, GithubAPIKey +## Dev Runbook +Dependancies: NodeJS 19, npm, Github APIKey How to get a Github API Key: https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens - 1. Pull the repository to your local setup +## Setting up OpenAPI Search with Docker Compose + +1. Clone the repository to your local setup +2. Make sure you have Docker installed locally. +3. Run `docker compose up` +4. Two Containers - Elasticsearch (The database container) and an instance of the server should have started. + +## Setting up the server manually + + 1. Clone the repository to your local setup 2. Run `npm i` 3. Make a `.env` file in the directory and add the variables: **PORT**= **(port number you want to host the api)** - **GITHUB_API_KEY**= **(Github API key)** + **GITHUB_API_KEY**= **(github API key)** + **ES_HOST**= **(determines location of elasticsearch db)** 4. Run `npm run build:watch` on one terminal. 5. On another terminal, run `npm run start` to start the server on the port specified on. -6. Now the nodejs server should be running! To test it just go to `localhost:{{PORT}}` and the text `TypeScript With Express` should be returned. - -## Setting up ElasticSearch Runbook (Dev/Local) --**docker pull docker.elastic.co/elasticsearch/elasticsearch:8.8.2** --**docker network create elastic** --**docker run \ - -p 9200:9200 \ - -p 9300:9300 \ - -e "discovery.type=single-node" \ - -e "xpack.security.enabled=false" \ - docker.elastic.co/elasticsearch/elasticsearch:8.8.2** +6. Now the nodejs server should be running! To test it just go to `localhost:{{PORT}}` and then you will be able to see the admin panel through which you can inference with some of the API's +7. Now to load the database with OpenAPI Files, run +`python scripts/seed_script.py` from the root of the folder. (Takes around 2-3hrs) + +## Setting up ElasticSearch locally (Manually) + 1. docker pull docker.elastic.co/elasticsearch/elasticsearch:8.8.2 + 2. docker network create elastic + 3. docker run \ + -p 9200:9200 \ + -p 9300:9300 \ + -e "discovery.type=single-node" \ + -e "xpack.security.enabled=false" \ + docker.elastic.co/elasticsearch/elasticsearch:8.8.2 + +## Loading Details +Currently, we are only indexing OpenAPI Files from the top 1000 most popular organisations from Github (Based on stars). Although more organisations can be indexed by adding them to the `scripts/assets/organisations.txt` file. + ## API Endpoints [![Run in Postman](https://run.pstmn.io/button.svg)](https://app.getpostman.com/run-collection/19841716-f1801bb7-b189-429b-a875-91b115d349a2?action=collection%2Ffork&source=rip_markdown&collection-url=entityId%3D19841716-f1801bb7-b189-429b-a875-91b115d349a2%26entityType%3Dcollection%26workspaceId%3D5ebe19fb-61d4-47a7-9cae-de3834853f6b) - - **/search?prompt={{prompt}}** : The active search endpoint. Add the search prompt in the prompt variable in query params. - - **/passive?q={}** : The passive search endpoint. Add the search prompt in the query param (q) to query the database. + 🚧Under Construction From f5e400aeb445a7e059de03b81d2eb08877cc86e2 Mon Sep 17 00:00:00 2001 From: Ishaan Shah <70190533+ishaan812@users.noreply.github.com> Date: Mon, 28 Aug 2023 18:29:10 +0530 Subject: [PATCH 3/7] Update README.md --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 931d6e3..1e13cf1 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ The project will consist of developing an open-source API that allows you to pas - Jest (For testing) - Docker - Python (Scripting) +- ElasticSearch ## Dev Runbook Dependancies: NodeJS 19, npm, Github APIKey @@ -23,6 +24,9 @@ How to get a Github API Key: https://docs.github.com/en/authentication/keeping-y 2. Make sure you have Docker installed locally. 3. Run `docker compose up` 4. Two Containers - Elasticsearch (The database container) and an instance of the server should have started. +5. Now to load the database with OpenAPI Files, run +`python scripts/seed_script.py` from the root of the folder. (Takes around 2-3hrs) +(More configuration of organisation list you can edit the scripts/assets/organisations1.txt, scripts/assets/organisations2.txt is for the next 1000 organisations) ## Setting up the server manually From 986f3a4efd3af31d389656d095e118a7557c84b8 Mon Sep 17 00:00:00 2001 From: Ishaan Shah <70190533+ishaan812@users.noreply.github.com> Date: Sun, 5 Nov 2023 14:11:57 +0530 Subject: [PATCH 4/7] FEATURE: Added PassiveSearch v1 (#2) * FEATURE: Added PassiveSearch v1 - Added Connection to ElasticSearch - Added 2 API's - /ping and /passive - Keyword Search using /passive - Added Storing to DB capability when using /search * Fix: Added error handling -added error handling for passivesearch api endpoint * FEATURE: Added Simple Query String - Changed ElasticSearch Search type to Simple Query String * REFACTOR: Improved Active Search -Made Active Search make better use of downtimes (Rate Limits or Validation Downtime) * FIX: Issues with Active Search -Fixed errors with parrallel processing --- package-lock.json | 901 ++++++++++++++++++++++++++++++--- package.json | 6 +- src/DB/dbutils.ts | 23 + src/app.ts | 50 +- src/routes/user.routes.ts | 10 - src/searchtools/search.ts | 107 +++- src/searchtools/searchutils.ts | 81 ++- 7 files changed, 1048 insertions(+), 130 deletions(-) create mode 100644 src/DB/dbutils.ts delete mode 100644 src/routes/user.routes.ts diff --git a/package-lock.json b/package-lock.json index 99963a1..d67d26b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,8 +9,12 @@ "version": "0.0.0", "license": "Apache-2.0", "dependencies": { + "@octokit/plugin-retry": "^6.0.0", + "@octokit/plugin-throttling": "^7.0.0", "dotenv": "^16.1.4", + "elasticsearch": "^16.7.3", "express": "^4.18.2", + "fs": "^0.0.1-security", "oas-normalize": "^8.4.1", "octokit": "^2.0.19", "tslib": "~2.5" @@ -18,7 +22,7 @@ "devDependencies": { "@types/express": "^4.17.17", "@types/jest": "~29.5", - "@types/node": "~18", + "@types/node": "^18.16.19", "@types/supertest": "^2.0.12", "@typescript-eslint/eslint-plugin": "~5.59", "@typescript-eslint/parser": "~5.59", @@ -1281,6 +1285,44 @@ "node": ">= 14" } }, + "node_modules/@octokit/app/node_modules/@octokit/auth-token": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.4.tgz", + "integrity": "sha512-TWFX7cZF2LXoCvdmJWY7XVPi74aSY0+FfBZNSXEXFkMpjcqsQwDSYVv5FhRFaI0V1ECnwbz4j59T/G+rXNWaIQ==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/@octokit/app/node_modules/@octokit/core": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.4.tgz", + "integrity": "sha512-rYKilwgzQ7/imScn3M9/pFfUf4I1AZEH3KhyJmtPdE2zfaXAn2mFfUy4FbKewzc2We5y/LlKLj36fWJLKC2SIQ==", + "dependencies": { + "@octokit/auth-token": "^3.0.0", + "@octokit/graphql": "^5.0.0", + "@octokit/request": "^6.0.0", + "@octokit/request-error": "^3.0.0", + "@octokit/types": "^9.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@octokit/app/node_modules/@octokit/graphql": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.6.tgz", + "integrity": "sha512-Fxyxdy/JH0MnIB5h+UQ3yCoh1FG4kWXfFKkpWqjZHw/p+Kc8Y44Hu/kCgNBT6nU1shNumEchmW/sUO1JuQnPcw==", + "dependencies": { + "@octokit/request": "^6.0.0", + "@octokit/types": "^9.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/@octokit/auth-app": { "version": "4.0.13", "resolved": "https://registry.npmjs.org/@octokit/auth-app/-/auth-app-4.0.13.tgz", @@ -1356,11 +1398,12 @@ } }, "node_modules/@octokit/auth-token": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.4.tgz", - "integrity": "sha512-TWFX7cZF2LXoCvdmJWY7XVPi74aSY0+FfBZNSXEXFkMpjcqsQwDSYVv5FhRFaI0V1ECnwbz4j59T/G+rXNWaIQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", + "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==", + "peer": true, "engines": { - "node": ">= 14" + "node": ">= 18" } }, "node_modules/@octokit/auth-unauthenticated": { @@ -1376,20 +1419,89 @@ } }, "node_modules/@octokit/core": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.1.tgz", - "integrity": "sha512-tEDxFx8E38zF3gT7sSMDrT1tGumDgsw5yPG6BBh/X+5ClIQfMH/Yqocxz1PnHx6CHyF6pxmovUTOfZAUvQ0Lvw==", - "dependencies": { - "@octokit/auth-token": "^3.0.0", - "@octokit/graphql": "^5.0.0", - "@octokit/request": "^6.0.0", - "@octokit/request-error": "^3.0.0", - "@octokit/types": "^9.0.0", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.0.0.tgz", + "integrity": "sha512-YbAtMWIrbZ9FCXbLwT9wWB8TyLjq9mxpKdgB3dUNxQcIVTf9hJ70gRPwAcqGZdY6WdJPZ0I7jLaaNDCiloGN2A==", + "peer": true, + "dependencies": { + "@octokit/auth-token": "^4.0.0", + "@octokit/graphql": "^7.0.0", + "@octokit/request": "^8.0.2", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.0.0", "before-after-hook": "^2.2.0", "universal-user-agent": "^6.0.0" }, "engines": { - "node": ">= 14" + "node": ">= 18" + } + }, + "node_modules/@octokit/core/node_modules/@octokit/endpoint": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.0.tgz", + "integrity": "sha512-szrQhiqJ88gghWY2Htt8MqUDO6++E/EIXqJ2ZEp5ma3uGS46o7LZAzSLt49myB7rT+Hfw5Y6gO3LmOxGzHijAQ==", + "peer": true, + "dependencies": { + "@octokit/types": "^11.0.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/core/node_modules/@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==", + "peer": true + }, + "node_modules/@octokit/core/node_modules/@octokit/request": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.1.0.tgz", + "integrity": "sha512-0gg/NwewU0iXctYBale0VVcCPqOtoW5lsog8cNBJgzV/CyTHa2gicUBOlNnzOk6pJkuwXI34qkq+uRm40PmD4A==", + "peer": true, + "dependencies": { + "@octokit/endpoint": "^9.0.0", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.1.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/core/node_modules/@octokit/request-error": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.0.tgz", + "integrity": "sha512-1ue0DH0Lif5iEqT52+Rf/hf0RmGO9NWFjrzmrkArpG9trFfDM/efx00BJHdLGuro4BR/gECxCU2Twf5OKrRFsQ==", + "peer": true, + "dependencies": { + "@octokit/types": "^11.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/core/node_modules/@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "peer": true, + "dependencies": { + "@octokit/openapi-types": "^18.0.0" + } + }, + "node_modules/@octokit/core/node_modules/is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "peer": true, + "engines": { + "node": ">=0.10.0" } }, "node_modules/@octokit/endpoint": { @@ -1414,16 +1526,85 @@ } }, "node_modules/@octokit/graphql": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.6.tgz", - "integrity": "sha512-Fxyxdy/JH0MnIB5h+UQ3yCoh1FG4kWXfFKkpWqjZHw/p+Kc8Y44Hu/kCgNBT6nU1shNumEchmW/sUO1JuQnPcw==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.0.1.tgz", + "integrity": "sha512-T5S3oZ1JOE58gom6MIcrgwZXzTaxRnxBso58xhozxHpOqSTgDS6YNeEUvZ/kRvXgPrRz/KHnZhtb7jUMRi9E6w==", + "peer": true, "dependencies": { - "@octokit/request": "^6.0.0", - "@octokit/types": "^9.0.0", + "@octokit/request": "^8.0.1", + "@octokit/types": "^11.0.0", "universal-user-agent": "^6.0.0" }, "engines": { - "node": ">= 14" + "node": ">= 18" + } + }, + "node_modules/@octokit/graphql/node_modules/@octokit/endpoint": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.0.tgz", + "integrity": "sha512-szrQhiqJ88gghWY2Htt8MqUDO6++E/EIXqJ2ZEp5ma3uGS46o7LZAzSLt49myB7rT+Hfw5Y6gO3LmOxGzHijAQ==", + "peer": true, + "dependencies": { + "@octokit/types": "^11.0.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/graphql/node_modules/@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==", + "peer": true + }, + "node_modules/@octokit/graphql/node_modules/@octokit/request": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.1.0.tgz", + "integrity": "sha512-0gg/NwewU0iXctYBale0VVcCPqOtoW5lsog8cNBJgzV/CyTHa2gicUBOlNnzOk6pJkuwXI34qkq+uRm40PmD4A==", + "peer": true, + "dependencies": { + "@octokit/endpoint": "^9.0.0", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.1.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/graphql/node_modules/@octokit/request-error": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.0.tgz", + "integrity": "sha512-1ue0DH0Lif5iEqT52+Rf/hf0RmGO9NWFjrzmrkArpG9trFfDM/efx00BJHdLGuro4BR/gECxCU2Twf5OKrRFsQ==", + "peer": true, + "dependencies": { + "@octokit/types": "^11.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/graphql/node_modules/@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "peer": true, + "dependencies": { + "@octokit/openapi-types": "^18.0.0" + } + }, + "node_modules/@octokit/graphql/node_modules/is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "peer": true, + "engines": { + "node": ">=0.10.0" } }, "node_modules/@octokit/oauth-app": { @@ -1445,6 +1626,44 @@ "node": ">= 14" } }, + "node_modules/@octokit/oauth-app/node_modules/@octokit/auth-token": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.4.tgz", + "integrity": "sha512-TWFX7cZF2LXoCvdmJWY7XVPi74aSY0+FfBZNSXEXFkMpjcqsQwDSYVv5FhRFaI0V1ECnwbz4j59T/G+rXNWaIQ==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/@octokit/oauth-app/node_modules/@octokit/core": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.4.tgz", + "integrity": "sha512-rYKilwgzQ7/imScn3M9/pFfUf4I1AZEH3KhyJmtPdE2zfaXAn2mFfUy4FbKewzc2We5y/LlKLj36fWJLKC2SIQ==", + "dependencies": { + "@octokit/auth-token": "^3.0.0", + "@octokit/graphql": "^5.0.0", + "@octokit/request": "^6.0.0", + "@octokit/request-error": "^3.0.0", + "@octokit/types": "^9.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@octokit/oauth-app/node_modules/@octokit/graphql": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.6.tgz", + "integrity": "sha512-Fxyxdy/JH0MnIB5h+UQ3yCoh1FG4kWXfFKkpWqjZHw/p+Kc8Y44Hu/kCgNBT6nU1shNumEchmW/sUO1JuQnPcw==", + "dependencies": { + "@octokit/request": "^6.0.0", + "@octokit/types": "^9.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/@octokit/oauth-authorization-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/@octokit/oauth-authorization-url/-/oauth-authorization-url-5.0.0.tgz", @@ -1503,33 +1722,73 @@ } }, "node_modules/@octokit/plugin-retry": { - "version": "4.1.6", - "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-4.1.6.tgz", - "integrity": "sha512-obkYzIgEC75r8+9Pnfiiqy3y/x1bc3QLE5B7qvv9wi9Kj0R5tGQFC6QMBg1154WQ9lAVypuQDGyp3hNpp15gQQ==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-6.0.0.tgz", + "integrity": "sha512-a1/A4A+PB1QoAHQfLJxGHhLfSAT03bR1jJz3GgQJZvty2ozawFWs93MiBQXO7SL2YbO7CIq0Goj4qLOBj8JeMQ==", "dependencies": { - "@octokit/types": "^9.0.0", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.0.0", "bottleneck": "^2.15.3" }, "engines": { - "node": ">= 14" + "node": ">= 18" }, "peerDependencies": { - "@octokit/core": ">=3" + "@octokit/core": ">=5" + } + }, + "node_modules/@octokit/plugin-retry/node_modules/@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==" + }, + "node_modules/@octokit/plugin-retry/node_modules/@octokit/request-error": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.0.tgz", + "integrity": "sha512-1ue0DH0Lif5iEqT52+Rf/hf0RmGO9NWFjrzmrkArpG9trFfDM/efx00BJHdLGuro4BR/gECxCU2Twf5OKrRFsQ==", + "dependencies": { + "@octokit/types": "^11.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/@octokit/plugin-retry/node_modules/@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "dependencies": { + "@octokit/openapi-types": "^18.0.0" } }, "node_modules/@octokit/plugin-throttling": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-5.2.3.tgz", - "integrity": "sha512-C9CFg9mrf6cugneKiaI841iG8DOv6P5XXkjmiNNut+swePxQ7RWEdAZRp5rJoE1hjsIqiYcKa/ZkOQ+ujPI39Q==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-7.0.0.tgz", + "integrity": "sha512-KL2k/d0uANc8XqP5S64YcNFCudR3F5AaKO39XWdUtlJIjT9Ni79ekWJ6Kj5xvAw87udkOMEPcVf9xEge2+ahew==", "dependencies": { - "@octokit/types": "^9.0.0", + "@octokit/types": "^11.0.0", "bottleneck": "^2.15.3" }, "engines": { - "node": ">= 14" + "node": ">= 18" }, "peerDependencies": { - "@octokit/core": "^4.0.0" + "@octokit/core": "^5.0.0" + } + }, + "node_modules/@octokit/plugin-throttling/node_modules/@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==" + }, + "node_modules/@octokit/plugin-throttling/node_modules/@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "dependencies": { + "@octokit/openapi-types": "^18.0.0" } }, "node_modules/@octokit/request": { @@ -1936,9 +2195,9 @@ "dev": true }, "node_modules/@types/node": { - "version": "18.15.13", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.13.tgz", - "integrity": "sha512-N+0kuo9KgrUQ1Sn/ifDXsvg0TTleP7rIy4zOBGECxAljqvqfqpTfzx0Q1NUedOixRMBfe2Whhb056a42cWs26Q==" + "version": "18.16.19", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.19.tgz", + "integrity": "sha512-IXl7o+R9iti9eBW4Wg2hx1xQDig183jj7YLn8F7udNceyfkbn1ZxmzZXuak20gR40D7pIkIY1kYGx5VIGbaHKA==" }, "node_modules/@types/prettier": { "version": "2.7.2", @@ -2260,6 +2519,17 @@ "node": ">=0.4.0" } }, + "node_modules/agentkeepalive": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz", + "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 4.0.0" + } + }, "node_modules/aggregate-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", @@ -4805,6 +5075,78 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, + "node_modules/elasticsearch": { + "version": "16.7.3", + "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.3.tgz", + "integrity": "sha512-e9kUNhwnIlu47fGAr4W6yZJbkpsgQJB0TqNK8rCANe1J4P65B1sGnbCFTgcKY3/dRgCWnuP1AJ4obvzW604xEQ==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "dependencies": { + "agentkeepalive": "^3.4.1", + "chalk": "^1.0.0", + "lodash": "^4.17.10" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/elasticsearch/node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/elasticsearch/node_modules/ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/elasticsearch/node_modules/chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "dependencies": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/elasticsearch/node_modules/escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/elasticsearch/node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/elasticsearch/node_modules/supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", + "engines": { + "node": ">=0.8.0" + } + }, "node_modules/electron-to-chromium": { "version": "1.4.368", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.368.tgz", @@ -5640,6 +5982,11 @@ } ] }, + "node_modules/fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==" + }, "node_modules/fs-readdir-recursive": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fs-readdir-recursive/-/fs-readdir-recursive-1.1.0.tgz", @@ -5873,7 +6220,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", - "dev": true, "dependencies": { "ansi-regex": "^2.0.0" }, @@ -5885,7 +6231,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -6055,6 +6400,14 @@ "node": ">=10.17.0" } }, + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "dependencies": { + "ms": "^2.0.0" + } + }, "node_modules/iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -8098,6 +8451,74 @@ "node": ">= 14" } }, + "node_modules/octokit/node_modules/@octokit/auth-token": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.4.tgz", + "integrity": "sha512-TWFX7cZF2LXoCvdmJWY7XVPi74aSY0+FfBZNSXEXFkMpjcqsQwDSYVv5FhRFaI0V1ECnwbz4j59T/G+rXNWaIQ==", + "engines": { + "node": ">= 14" + } + }, + "node_modules/octokit/node_modules/@octokit/core": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.4.tgz", + "integrity": "sha512-rYKilwgzQ7/imScn3M9/pFfUf4I1AZEH3KhyJmtPdE2zfaXAn2mFfUy4FbKewzc2We5y/LlKLj36fWJLKC2SIQ==", + "dependencies": { + "@octokit/auth-token": "^3.0.0", + "@octokit/graphql": "^5.0.0", + "@octokit/request": "^6.0.0", + "@octokit/request-error": "^3.0.0", + "@octokit/types": "^9.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/octokit/node_modules/@octokit/graphql": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.6.tgz", + "integrity": "sha512-Fxyxdy/JH0MnIB5h+UQ3yCoh1FG4kWXfFKkpWqjZHw/p+Kc8Y44Hu/kCgNBT6nU1shNumEchmW/sUO1JuQnPcw==", + "dependencies": { + "@octokit/request": "^6.0.0", + "@octokit/types": "^9.0.0", + "universal-user-agent": "^6.0.0" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/octokit/node_modules/@octokit/plugin-retry": { + "version": "4.1.6", + "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-4.1.6.tgz", + "integrity": "sha512-obkYzIgEC75r8+9Pnfiiqy3y/x1bc3QLE5B7qvv9wi9Kj0R5tGQFC6QMBg1154WQ9lAVypuQDGyp3hNpp15gQQ==", + "dependencies": { + "@octokit/types": "^9.0.0", + "bottleneck": "^2.15.3" + }, + "engines": { + "node": ">= 14" + }, + "peerDependencies": { + "@octokit/core": ">=3" + } + }, + "node_modules/octokit/node_modules/@octokit/plugin-throttling": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-5.2.3.tgz", + "integrity": "sha512-C9CFg9mrf6cugneKiaI841iG8DOv6P5XXkjmiNNut+swePxQ7RWEdAZRp5rJoE1hjsIqiYcKa/ZkOQ+ujPI39Q==", + "dependencies": { + "@octokit/types": "^9.0.0", + "bottleneck": "^2.15.3" + }, + "engines": { + "node": ">= 14" + }, + "peerDependencies": { + "@octokit/core": "^4.0.0" + } + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -11498,6 +11919,37 @@ "@octokit/plugin-paginate-rest": "^6.0.0", "@octokit/types": "^9.0.0", "@octokit/webhooks": "^10.0.0" + }, + "dependencies": { + "@octokit/auth-token": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.4.tgz", + "integrity": "sha512-TWFX7cZF2LXoCvdmJWY7XVPi74aSY0+FfBZNSXEXFkMpjcqsQwDSYVv5FhRFaI0V1ECnwbz4j59T/G+rXNWaIQ==" + }, + "@octokit/core": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.4.tgz", + "integrity": "sha512-rYKilwgzQ7/imScn3M9/pFfUf4I1AZEH3KhyJmtPdE2zfaXAn2mFfUy4FbKewzc2We5y/LlKLj36fWJLKC2SIQ==", + "requires": { + "@octokit/auth-token": "^3.0.0", + "@octokit/graphql": "^5.0.0", + "@octokit/request": "^6.0.0", + "@octokit/request-error": "^3.0.0", + "@octokit/types": "^9.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/graphql": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.6.tgz", + "integrity": "sha512-Fxyxdy/JH0MnIB5h+UQ3yCoh1FG4kWXfFKkpWqjZHw/p+Kc8Y44Hu/kCgNBT6nU1shNumEchmW/sUO1JuQnPcw==", + "requires": { + "@octokit/request": "^6.0.0", + "@octokit/types": "^9.0.0", + "universal-user-agent": "^6.0.0" + } + } } }, "@octokit/auth-app": { @@ -11562,9 +12014,10 @@ } }, "@octokit/auth-token": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.4.tgz", - "integrity": "sha512-TWFX7cZF2LXoCvdmJWY7XVPi74aSY0+FfBZNSXEXFkMpjcqsQwDSYVv5FhRFaI0V1ECnwbz4j59T/G+rXNWaIQ==" + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", + "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==", + "peer": true }, "@octokit/auth-unauthenticated": { "version": "3.0.5", @@ -11576,17 +12029,76 @@ } }, "@octokit/core": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.1.tgz", - "integrity": "sha512-tEDxFx8E38zF3gT7sSMDrT1tGumDgsw5yPG6BBh/X+5ClIQfMH/Yqocxz1PnHx6CHyF6pxmovUTOfZAUvQ0Lvw==", - "requires": { - "@octokit/auth-token": "^3.0.0", - "@octokit/graphql": "^5.0.0", - "@octokit/request": "^6.0.0", - "@octokit/request-error": "^3.0.0", - "@octokit/types": "^9.0.0", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.0.0.tgz", + "integrity": "sha512-YbAtMWIrbZ9FCXbLwT9wWB8TyLjq9mxpKdgB3dUNxQcIVTf9hJ70gRPwAcqGZdY6WdJPZ0I7jLaaNDCiloGN2A==", + "peer": true, + "requires": { + "@octokit/auth-token": "^4.0.0", + "@octokit/graphql": "^7.0.0", + "@octokit/request": "^8.0.2", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.0.0", "before-after-hook": "^2.2.0", "universal-user-agent": "^6.0.0" + }, + "dependencies": { + "@octokit/endpoint": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.0.tgz", + "integrity": "sha512-szrQhiqJ88gghWY2Htt8MqUDO6++E/EIXqJ2ZEp5ma3uGS46o7LZAzSLt49myB7rT+Hfw5Y6gO3LmOxGzHijAQ==", + "peer": true, + "requires": { + "@octokit/types": "^11.0.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==", + "peer": true + }, + "@octokit/request": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.1.0.tgz", + "integrity": "sha512-0gg/NwewU0iXctYBale0VVcCPqOtoW5lsog8cNBJgzV/CyTHa2gicUBOlNnzOk6pJkuwXI34qkq+uRm40PmD4A==", + "peer": true, + "requires": { + "@octokit/endpoint": "^9.0.0", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.1.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/request-error": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.0.tgz", + "integrity": "sha512-1ue0DH0Lif5iEqT52+Rf/hf0RmGO9NWFjrzmrkArpG9trFfDM/efx00BJHdLGuro4BR/gECxCU2Twf5OKrRFsQ==", + "peer": true, + "requires": { + "@octokit/types": "^11.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "peer": true, + "requires": { + "@octokit/openapi-types": "^18.0.0" + } + }, + "is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "peer": true + } } }, "@octokit/endpoint": { @@ -11607,13 +12119,72 @@ } }, "@octokit/graphql": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.6.tgz", - "integrity": "sha512-Fxyxdy/JH0MnIB5h+UQ3yCoh1FG4kWXfFKkpWqjZHw/p+Kc8Y44Hu/kCgNBT6nU1shNumEchmW/sUO1JuQnPcw==", + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.0.1.tgz", + "integrity": "sha512-T5S3oZ1JOE58gom6MIcrgwZXzTaxRnxBso58xhozxHpOqSTgDS6YNeEUvZ/kRvXgPrRz/KHnZhtb7jUMRi9E6w==", + "peer": true, "requires": { - "@octokit/request": "^6.0.0", - "@octokit/types": "^9.0.0", + "@octokit/request": "^8.0.1", + "@octokit/types": "^11.0.0", "universal-user-agent": "^6.0.0" + }, + "dependencies": { + "@octokit/endpoint": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.0.tgz", + "integrity": "sha512-szrQhiqJ88gghWY2Htt8MqUDO6++E/EIXqJ2ZEp5ma3uGS46o7LZAzSLt49myB7rT+Hfw5Y6gO3LmOxGzHijAQ==", + "peer": true, + "requires": { + "@octokit/types": "^11.0.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==", + "peer": true + }, + "@octokit/request": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.1.0.tgz", + "integrity": "sha512-0gg/NwewU0iXctYBale0VVcCPqOtoW5lsog8cNBJgzV/CyTHa2gicUBOlNnzOk6pJkuwXI34qkq+uRm40PmD4A==", + "peer": true, + "requires": { + "@octokit/endpoint": "^9.0.0", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.1.0", + "is-plain-object": "^5.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/request-error": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.0.tgz", + "integrity": "sha512-1ue0DH0Lif5iEqT52+Rf/hf0RmGO9NWFjrzmrkArpG9trFfDM/efx00BJHdLGuro4BR/gECxCU2Twf5OKrRFsQ==", + "peer": true, + "requires": { + "@octokit/types": "^11.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "peer": true, + "requires": { + "@octokit/openapi-types": "^18.0.0" + } + }, + "is-plain-object": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", + "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", + "peer": true + } } }, "@octokit/oauth-app": { @@ -11630,6 +12201,37 @@ "@types/aws-lambda": "^8.10.83", "fromentries": "^1.3.1", "universal-user-agent": "^6.0.0" + }, + "dependencies": { + "@octokit/auth-token": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.4.tgz", + "integrity": "sha512-TWFX7cZF2LXoCvdmJWY7XVPi74aSY0+FfBZNSXEXFkMpjcqsQwDSYVv5FhRFaI0V1ECnwbz4j59T/G+rXNWaIQ==" + }, + "@octokit/core": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.4.tgz", + "integrity": "sha512-rYKilwgzQ7/imScn3M9/pFfUf4I1AZEH3KhyJmtPdE2zfaXAn2mFfUy4FbKewzc2We5y/LlKLj36fWJLKC2SIQ==", + "requires": { + "@octokit/auth-token": "^3.0.0", + "@octokit/graphql": "^5.0.0", + "@octokit/request": "^6.0.0", + "@octokit/request-error": "^3.0.0", + "@octokit/types": "^9.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/graphql": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.6.tgz", + "integrity": "sha512-Fxyxdy/JH0MnIB5h+UQ3yCoh1FG4kWXfFKkpWqjZHw/p+Kc8Y44Hu/kCgNBT6nU1shNumEchmW/sUO1JuQnPcw==", + "requires": { + "@octokit/request": "^6.0.0", + "@octokit/types": "^9.0.0", + "universal-user-agent": "^6.0.0" + } + } } }, "@octokit/oauth-authorization-url": { @@ -11672,21 +12274,62 @@ } }, "@octokit/plugin-retry": { - "version": "4.1.6", - "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-4.1.6.tgz", - "integrity": "sha512-obkYzIgEC75r8+9Pnfiiqy3y/x1bc3QLE5B7qvv9wi9Kj0R5tGQFC6QMBg1154WQ9lAVypuQDGyp3hNpp15gQQ==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-6.0.0.tgz", + "integrity": "sha512-a1/A4A+PB1QoAHQfLJxGHhLfSAT03bR1jJz3GgQJZvty2ozawFWs93MiBQXO7SL2YbO7CIq0Goj4qLOBj8JeMQ==", "requires": { - "@octokit/types": "^9.0.0", + "@octokit/request-error": "^5.0.0", + "@octokit/types": "^11.0.0", "bottleneck": "^2.15.3" + }, + "dependencies": { + "@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==" + }, + "@octokit/request-error": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.0.tgz", + "integrity": "sha512-1ue0DH0Lif5iEqT52+Rf/hf0RmGO9NWFjrzmrkArpG9trFfDM/efx00BJHdLGuro4BR/gECxCU2Twf5OKrRFsQ==", + "requires": { + "@octokit/types": "^11.0.0", + "deprecation": "^2.0.0", + "once": "^1.4.0" + } + }, + "@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "requires": { + "@octokit/openapi-types": "^18.0.0" + } + } } }, "@octokit/plugin-throttling": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-5.2.3.tgz", - "integrity": "sha512-C9CFg9mrf6cugneKiaI841iG8DOv6P5XXkjmiNNut+swePxQ7RWEdAZRp5rJoE1hjsIqiYcKa/ZkOQ+ujPI39Q==", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-7.0.0.tgz", + "integrity": "sha512-KL2k/d0uANc8XqP5S64YcNFCudR3F5AaKO39XWdUtlJIjT9Ni79ekWJ6Kj5xvAw87udkOMEPcVf9xEge2+ahew==", "requires": { - "@octokit/types": "^9.0.0", + "@octokit/types": "^11.0.0", "bottleneck": "^2.15.3" + }, + "dependencies": { + "@octokit/openapi-types": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-18.0.0.tgz", + "integrity": "sha512-V8GImKs3TeQRxRtXFpG2wl19V7444NIOTDF24AWuIbmNaNYOQMWRbjcGDXV5B+0n887fgDcuMNOmlul+k+oJtw==" + }, + "@octokit/types": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-11.1.0.tgz", + "integrity": "sha512-Fz0+7GyLm/bHt8fwEqgvRBWwIV1S6wRRyq+V6exRKLVWaKGsuy6H9QFYeBVDV7rK6fO3XwHgQOPxv+cLj2zpXQ==", + "requires": { + "@octokit/openapi-types": "^18.0.0" + } + } } }, "@octokit/request": { @@ -12053,9 +12696,9 @@ "dev": true }, "@types/node": { - "version": "18.15.13", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.15.13.tgz", - "integrity": "sha512-N+0kuo9KgrUQ1Sn/ifDXsvg0TTleP7rIy4zOBGECxAljqvqfqpTfzx0Q1NUedOixRMBfe2Whhb056a42cWs26Q==" + "version": "18.16.19", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.16.19.tgz", + "integrity": "sha512-IXl7o+R9iti9eBW4Wg2hx1xQDig183jj7YLn8F7udNceyfkbn1ZxmzZXuak20gR40D7pIkIY1kYGx5VIGbaHKA==" }, "@types/prettier": { "version": "2.7.2", @@ -12274,6 +12917,14 @@ "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", "dev": true }, + "agentkeepalive": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-3.5.2.tgz", + "integrity": "sha512-e0L/HNe6qkQ7H19kTlRRqUibEAwDK5AFk6y3PtMsuut2VAH6+Q4xZml1tNDJD7kSAyqmbG/K08K5WEJYtUrSlQ==", + "requires": { + "humanize-ms": "^1.2.1" + } + }, "aggregate-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", @@ -14402,6 +15053,58 @@ "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" }, + "elasticsearch": { + "version": "16.7.3", + "resolved": "https://registry.npmjs.org/elasticsearch/-/elasticsearch-16.7.3.tgz", + "integrity": "sha512-e9kUNhwnIlu47fGAr4W6yZJbkpsgQJB0TqNK8rCANe1J4P65B1sGnbCFTgcKY3/dRgCWnuP1AJ4obvzW604xEQ==", + "requires": { + "agentkeepalive": "^3.4.1", + "chalk": "^1.0.0", + "lodash": "^4.17.10" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==" + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==" + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + }, + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==" + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==" + } + } + }, "electron-to-chromium": { "version": "1.4.368", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.368.tgz", @@ -15041,6 +15744,11 @@ "resolved": "https://registry.npmjs.org/fromentries/-/fromentries-1.3.2.tgz", "integrity": "sha512-cHEpEQHUg0f8XdtZCc2ZAhrHzKzT0MrFUTcvx+hfxYu7rGMDc5SKoXFh+n4YigxsHXRzc6OrCshdR1bWH6HHyg==" }, + "fs": { + "version": "0.0.1-security", + "resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz", + "integrity": "sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w==" + }, "fs-readdir-recursive": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fs-readdir-recursive/-/fs-readdir-recursive-1.1.0.tgz", @@ -15215,7 +15923,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", - "dev": true, "requires": { "ansi-regex": "^2.0.0" }, @@ -15223,8 +15930,7 @@ "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "dev": true + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==" } } }, @@ -15354,6 +16060,14 @@ "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", "dev": true }, + "humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "requires": { + "ms": "^2.0.0" + } + }, "iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -16918,6 +17632,55 @@ "@octokit/plugin-retry": "^4.1.3", "@octokit/plugin-throttling": "^5.2.2", "@octokit/types": "^9.2.2" + }, + "dependencies": { + "@octokit/auth-token": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-3.0.4.tgz", + "integrity": "sha512-TWFX7cZF2LXoCvdmJWY7XVPi74aSY0+FfBZNSXEXFkMpjcqsQwDSYVv5FhRFaI0V1ECnwbz4j59T/G+rXNWaIQ==" + }, + "@octokit/core": { + "version": "4.2.4", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-4.2.4.tgz", + "integrity": "sha512-rYKilwgzQ7/imScn3M9/pFfUf4I1AZEH3KhyJmtPdE2zfaXAn2mFfUy4FbKewzc2We5y/LlKLj36fWJLKC2SIQ==", + "requires": { + "@octokit/auth-token": "^3.0.0", + "@octokit/graphql": "^5.0.0", + "@octokit/request": "^6.0.0", + "@octokit/request-error": "^3.0.0", + "@octokit/types": "^9.0.0", + "before-after-hook": "^2.2.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/graphql": { + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-5.0.6.tgz", + "integrity": "sha512-Fxyxdy/JH0MnIB5h+UQ3yCoh1FG4kWXfFKkpWqjZHw/p+Kc8Y44Hu/kCgNBT6nU1shNumEchmW/sUO1JuQnPcw==", + "requires": { + "@octokit/request": "^6.0.0", + "@octokit/types": "^9.0.0", + "universal-user-agent": "^6.0.0" + } + }, + "@octokit/plugin-retry": { + "version": "4.1.6", + "resolved": "https://registry.npmjs.org/@octokit/plugin-retry/-/plugin-retry-4.1.6.tgz", + "integrity": "sha512-obkYzIgEC75r8+9Pnfiiqy3y/x1bc3QLE5B7qvv9wi9Kj0R5tGQFC6QMBg1154WQ9lAVypuQDGyp3hNpp15gQQ==", + "requires": { + "@octokit/types": "^9.0.0", + "bottleneck": "^2.15.3" + } + }, + "@octokit/plugin-throttling": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@octokit/plugin-throttling/-/plugin-throttling-5.2.3.tgz", + "integrity": "sha512-C9CFg9mrf6cugneKiaI841iG8DOv6P5XXkjmiNNut+swePxQ7RWEdAZRp5rJoE1hjsIqiYcKa/ZkOQ+ujPI39Q==", + "requires": { + "@octokit/types": "^9.0.0", + "bottleneck": "^2.15.3" + } + } } }, "on-finished": { diff --git a/package.json b/package.json index 983b03f..d88c7d9 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "devDependencies": { "@types/express": "^4.17.17", "@types/jest": "~29.5", - "@types/node": "~18", + "@types/node": "^18.16.19", "@types/supertest": "^2.0.12", "@typescript-eslint/eslint-plugin": "~5.59", "@typescript-eslint/parser": "~5.59", @@ -44,8 +44,12 @@ "author": "Ishaan Shah ", "license": "Apache-2.0", "dependencies": { + "@octokit/plugin-retry": "^6.0.0", + "@octokit/plugin-throttling": "^7.0.0", "dotenv": "^16.1.4", + "elasticsearch": "^16.7.3", "express": "^4.18.2", + "fs": "^0.0.1-security", "oas-normalize": "^8.4.1", "octokit": "^2.0.19", "tslib": "~2.5" diff --git a/src/DB/dbutils.ts b/src/DB/dbutils.ts new file mode 100644 index 0000000..da9171b --- /dev/null +++ b/src/DB/dbutils.ts @@ -0,0 +1,23 @@ +export async function checkClusterHealth(esClient: any) : Promise { + try { + const response = await esClient.cat.health(); + console.log('Cluster health:', response); + return response; + } catch (error) { + console.error('Error checking cluster health:', error); + return ""; + } + } + +export async function BulkStoreToDB(validFiles: any,esClient: any) : Promise{ + try { + if(validFiles.length == 0){ + return; + } + const response = await esClient.bulk({ body: validFiles }); + // console.log('Bulk response:', response); + return response; + } catch (error) { + console.error('Error bulk indexing:', error); + } +} \ No newline at end of file diff --git a/src/app.ts b/src/app.ts index e4c53b7..b35073d 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,36 +1,78 @@ import express from 'express'; import { Octokit } from 'octokit'; -import { activeSearch } from './searchtools/search.js'; +import { activeSearch, passiveSearch } from './searchtools/search.js'; import dotenv from 'dotenv'; +import es from 'elasticsearch'; +import { checkClusterHealth } from './DB/dbutils.js'; // import { router as userRoutes } from "./routes/user.routes.js"; +import { throttling } from '@octokit/plugin-throttling' +import { retry } from '@octokit/plugin-retry' + + +const CustomOctokit = Octokit.plugin(throttling as any, retry as any); dotenv.config(); -const app = express(); -const octokit = new Octokit({ +const octokit = new CustomOctokit({ userAgent: 'github-openapi-search/v0.0.1', auth: process.env.GITHUB_API_KEY, + throttle: { + onRateLimit: (retryAfter, options) => { + octokit.log.warn( + `Request quota exhausted for request ${options.method} ${options.url}`, + ); + console.log(`Retrying after ${retryAfter} seconds!`); + return true; + }, + onSecondaryRateLimit: (retryAfter, options, octokit) => { + // does not retry, only logs a warning + octokit.log.warn( + `Secondary quota detected for request ${options.method} ${options.url}`, + ); + }, + }, +}); + +const app = express(); + + + +const esClient = new es.Client({ + host: 'http://localhost:9200', + log: 'trace', }); // Should not even be an API endpoint for passive search // Should just fetch repositories and go through them (with ETAG to make sure no repeats) // Check for openapi.json in the contents of the repository // If it exists, then store in database with important content + +app.use('/passive', async (_req, _res) => { + const query = _req.query.q as string; + const results = await passiveSearch(query, esClient); + _res.send(results); +}) + app.use('/search', async (_req, _res) => { const Repository = _req.query.repo as string; const Organisation = _req.query.org as string; const User = _req.query.user as string; const Prompt = _req.query.prompt as string; - const results = await activeSearch( Prompt as string, Repository as string, Organisation as string, User as string, + esClient as any, ); _res.send(results); }); +app.use('/ping', async (_req, _res) => { + const response = await checkClusterHealth(esClient); + _res.send(response); +}) + app.get('/', (_req, _res) => { _res.send('TypeScript With Express'); }); diff --git a/src/routes/user.routes.ts b/src/routes/user.routes.ts deleted file mode 100644 index 885a39b..0000000 --- a/src/routes/user.routes.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Router, Response } from 'express'; - -const router = Router(); - -router.get('/', (_req, _res): Response => { - const users = ['Goon', 'Tsuki', 'Joe']; - return _res.send(users); -}); - -export { router }; diff --git a/src/searchtools/search.ts b/src/searchtools/search.ts index 9084447..c05eb37 100644 --- a/src/searchtools/search.ts +++ b/src/searchtools/search.ts @@ -1,39 +1,90 @@ -import { getFileContents, handleCodeSearch } from './searchutils.js'; -import OASNormalize from 'oas-normalize'; +import { queryBuilder, ValidateandStoreFiles} from './searchutils.js'; +import { octokit } from '../app.js'; + + +let processCount = 1; +let finishedCount = 1; export async function activeSearch( prompt: string, repo: string, organisation: string, username: string, + esClient: any, ): Promise { - //3 types of searches: - // If repository is specified, then search through that repository: /search/code with repo name - // If organisation is specified then search through that organisation: /search/code with org name - // If user is specified then search through that user: /search/code with user name - // If none is specified then return error - const validFiles = []; - const files = await handleCodeSearch(prompt, repo, organisation, username, 1); - for (const file of files) { - const base64content = await getFileContents( - file.repository.owner.login, - file.repository.name, - file.path, - ); - const content = Buffer.from(base64content, 'base64').toString(); - const oas = new OASNormalize.default(content); - oas - .validate() - .then((definition) => { - console.log('File ' + file.name + ' is valid'); - validFiles.push(definition); - console.log(validFiles); - }) - .catch((error) => { - // Error will be an array of validation errors. - console.log('File ' + file.name + ' is not valid'); + const query = await queryBuilder(prompt, repo, organisation, username); + let files = []; + let validFiles = []; + await octokit.paginate(octokit.rest.search.code, { + q: query, + per_page: 100 + }, + (response : any) => { + files = files.concat(response.data) + if(files.length >= 500){ + console.log("Validating and storing files since rate limit reached") + processCount++; + ValidateandStoreFiles(files, esClient).then((validatedFiles) => { + validFiles = validFiles.concat(validatedFiles); + finishedCount++; }); + files = [] + } + } + ); + //this ending before the above one + ValidateandStoreFiles(files, esClient).then((validatedFiles) => { + validFiles = validFiles.concat(validatedFiles); + }); + while(processCount > finishedCount){ + await new Promise(r => setTimeout(r, 3000)); + console.log("Total Processes: "+processCount+"\nFinished Processes: "+finishedCount) + console.log("Waiting for all files to be processed") } - return validFiles; } + +export async function passiveSearch( + query: string, + esClient: any, +): Promise { + try { + if (esClient === undefined) { + throw new Error('Invalid Elasticsearch client'); + } + const result = await esClient.search({ + index: 'openapi', + body: { + query: { + simple_query_string: { + query: query, + fields: ["servers^2","paths^1.5","data^1"], + default_operator: "and" + } + } + } + }); + + if (result.hits.hits) { + if (result.hits.hits.length === 0) { + console.log('No results found in the database'); + // activeSearch(query, "", "", "", esClient); + } + return result.hits.hits; + } + } catch (error) { + if (error.message.includes('No Living connections')) { + console.log('Elasticsearch connection error:', error); + return error + } else { + console.log('Error occurred during passive search:', error); + return error; + } + } + + return 'Database not found'; +} + + + + diff --git a/src/searchtools/searchutils.ts b/src/searchtools/searchutils.ts index aa3b4ed..c5c419c 100644 --- a/src/searchtools/searchutils.ts +++ b/src/searchtools/searchutils.ts @@ -1,5 +1,25 @@ import { octokit } from '../app.js'; // import { CodeSearchResponse } from "./searchstructs.js"; +import crypto from "crypto"; +import OASNormalize from 'oas-normalize'; +import { BulkStoreToDB } from '../DB/dbutils.js'; + + + + +export function generateUUID() : string { + // Generate a random buffer of 16 bytes + const buffer = crypto.randomBytes(16); + + // Set the version (4) and variant (2) bits + buffer[6] = (buffer[6] & 0x0f) | 0x40; + buffer[8] = (buffer[8] & 0x3f) | 0x80; + + // Convert the buffer to a string representation of the UUID + const uuid = buffer.toString('hex').match(/(.{8})(.{4})(.{4})(.{4})(.{12})/); + uuid.shift(); + return uuid.join('-'); +} export async function getFileContents( repoowner: string, @@ -17,30 +37,55 @@ export async function getFileContents( return response.data['content']; } -export async function handleCodeSearch( - prompt: string, - repo: string, - organisation: string, - username: string, - page: number, -): Promise { - //Why does /(openai|swagger)/ not work :/ - //Even Parenthesis doesn't work +export async function queryBuilder(prompt: string, repo: string, organisation: string, username: string): Promise { + if(prompt == undefined){ + prompt = "" + } let query = prompt + ' AND "openapi: 3"'; - // query+= prompt + ' AND "swagger: \\"2"' + // query+= prompt + ' AND "swagger: \\"2"' if (repo != undefined) { query += '+repo:' + repo; } else if (organisation != undefined) { query += '+org:' + organisation; } else if (username != undefined) { query += '+user:' + username; + } else { + return query; + } + return query; +} + +export async function ValidateandStoreFiles(files: any[], esClient: any): Promise { + if(files.length == 0){ + return; } - console.log('Query: ' + query); - //api call handling - const results = await octokit.paginate(octokit.rest.search.code, { - q: query, - per_page: 100, - }); - console.log(results.length) - return results; + console.log("Validating and storing files") + let validFiles = []; + for (const file of files) { + const base64content = await getFileContents( + file.repository.owner.login, + file.repository.name, + file.path, + ); + const content = Buffer.from(base64content, 'base64').toString(); + const oas = new OASNormalize.default(content); + oas + .validate() + .then((definition) => { + console.log('File ' + file.name + ' is valid'); + console.log(definition?.info?.title) + validFiles.push({index: { _index: 'openapi', _id: generateUUID()}}); + validFiles.push({title: definition?.info?.title, description: definition?.info?.description, version: definition?.info?.version, servers: JSON.stringify(definition?.servers), paths: JSON.stringify(definition?.paths) , path: file.path, repository: file?.repository?.name, owner: file?.repository?.owner?.login, data: content}); + if(validFiles.length >= 50){ + console.log("Storing some of the valid files") + BulkStoreToDB(validFiles as any[],esClient as any); + validFiles = [] + } + }) + .catch((error) => { + console.log('File ' + file.name + ' is not valid'); + }); + } + BulkStoreToDB(validFiles as any[],esClient as any); + return validFiles; } From aa559b90f95a6d4a7fd9ab1add155f093bb0bd1b Mon Sep 17 00:00:00 2001 From: Ishaan Shah <70190533+ishaan812@users.noreply.github.com> Date: Sun, 5 Nov 2023 14:14:37 +0530 Subject: [PATCH 5/7] FEATURE: Added seed script (#3) * FEATURE: Added PassiveSearch v1 - Added Connection to ElasticSearch - Added 2 API's - /ping and /passive - Keyword Search using /passive - Added Storing to DB capability when using /search * Fix: Added error handling -added error handling for passivesearch api endpoint * FEATURE: Added Simple Query String - Changed ElasticSearch Search type to Simple Query String * REFACTOR: Improved Active Search -Made Active Search make better use of downtimes (Rate Limits or Validation Downtime) * FIX: Issues with Active Search -Fixed errors with parrallel processing * FEATURE: Added SeedScript - Added RootQuery as an optional query param for activesearch which basically takes out the prompt builder and takes a raw query to look for in Github API. - Added Python SeedScript --- scripts/seed_script.py | 31 +++++++++++++++++++++++++++++++ src/app.ts | 4 +++- src/searchtools/search.ts | 13 ++++++++++--- src/searchtools/searchutils.ts | 9 +++++++-- 4 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 scripts/seed_script.py diff --git a/scripts/seed_script.py b/scripts/seed_script.py new file mode 100644 index 0000000..21f45d2 --- /dev/null +++ b/scripts/seed_script.py @@ -0,0 +1,31 @@ +# Description: This script is used to seed the database with data from the data folder +# !pip install requests +# python scripts/seed_script.py to run from main folder + +import requests + +def call_local_endpoint(prompt): + #TODO: Change this to the correct URL when activesearch endpoint is changed + url = f'http://localhost:8080/search?rootquery="{prompt}"' + + try: + response = requests.get(url) + + # Check if the response was successful (status code 200) + if response.status_code == 200: + print("Request to localhost:8080/search was successful!") + print("Response content:") + print(response.text) + else: + print(f"Request to localhost:8080/active failed with status code: {response.status_code}") + + except requests.exceptions.RequestException as e: + print(f"An error occurred: {e}") + +if __name__ == "__main__": + #Get Open API files + call_local_endpoint("openapi: 3") + #Get Swagger files + call_local_endpoint("swagger: 2") + +#PS: Takes a long time to run \ No newline at end of file diff --git a/src/app.ts b/src/app.ts index b35073d..623778e 100644 --- a/src/app.ts +++ b/src/app.ts @@ -4,7 +4,6 @@ import { activeSearch, passiveSearch } from './searchtools/search.js'; import dotenv from 'dotenv'; import es from 'elasticsearch'; import { checkClusterHealth } from './DB/dbutils.js'; -// import { router as userRoutes } from "./routes/user.routes.js"; import { throttling } from '@octokit/plugin-throttling' import { retry } from '@octokit/plugin-retry' @@ -47,6 +46,7 @@ const esClient = new es.Client({ // Check for openapi.json in the contents of the repository // If it exists, then store in database with important content + app.use('/passive', async (_req, _res) => { const query = _req.query.q as string; const results = await passiveSearch(query, esClient); @@ -58,11 +58,13 @@ app.use('/search', async (_req, _res) => { const Organisation = _req.query.org as string; const User = _req.query.user as string; const Prompt = _req.query.prompt as string; + const RootQuery = _req.query.rootquery as string; const results = await activeSearch( Prompt as string, Repository as string, Organisation as string, User as string, + RootQuery as string, esClient as any, ); _res.send(results); diff --git a/src/searchtools/search.ts b/src/searchtools/search.ts index c05eb37..cdf1e2d 100644 --- a/src/searchtools/search.ts +++ b/src/searchtools/search.ts @@ -10,31 +10,38 @@ export async function activeSearch( repo: string, organisation: string, username: string, + rootquery: string, esClient: any, ): Promise { - const query = await queryBuilder(prompt, repo, organisation, username); + const query = await queryBuilder(prompt, repo, organisation, username, rootquery); let files = []; let validFiles = []; + console.log("Query: "+query) await octokit.paginate(octokit.rest.search.code, { q: query, per_page: 100 }, (response : any) => { files = files.concat(response.data) - if(files.length >= 500){ - console.log("Validating and storing files since rate limit reached") + if(files.length >= 200){ processCount++; + console.log("ValidateandStoreFiles Process Number "+processCount+" Started") ValidateandStoreFiles(files, esClient).then((validatedFiles) => { validFiles = validFiles.concat(validatedFiles); finishedCount++; + console.log("ValidateandStoreFiles Process Number "+finishedCount+" Started") }); files = [] } } ); //this ending before the above one + processCount++; + console.log("ValidateandStoreFiles Process Number "+processCount+" Started") ValidateandStoreFiles(files, esClient).then((validatedFiles) => { validFiles = validFiles.concat(validatedFiles); + console.log("ValidateandStoreFiles Process Number "+finishedCount+" Started") + finishedCount++; }); while(processCount > finishedCount){ await new Promise(r => setTimeout(r, 3000)); diff --git a/src/searchtools/searchutils.ts b/src/searchtools/searchutils.ts index c5c419c..bbe42ad 100644 --- a/src/searchtools/searchutils.ts +++ b/src/searchtools/searchutils.ts @@ -37,11 +37,16 @@ export async function getFileContents( return response.data['content']; } -export async function queryBuilder(prompt: string, repo: string, organisation: string, username: string): Promise { +export async function queryBuilder(prompt: string, repo: string, organisation: string, username: string, rootquery: string): Promise { if(prompt == undefined){ prompt = "" } - let query = prompt + ' AND "openapi: 3"'; + let query + if(rootquery != undefined){ + query = rootquery + return query + } + query = prompt + ' AND "openapi: 3"'; // query+= prompt + ' AND "swagger: \\"2"' if (repo != undefined) { query += '+repo:' + repo; From 8a2db0ce5a7db5f753193031a3dd0b98c70fd2b1 Mon Sep 17 00:00:00 2001 From: Ishaan Shah <70190533+ishaan812@users.noreply.github.com> Date: Sun, 5 Nov 2023 14:18:06 +0530 Subject: [PATCH 6/7] REFACTOR: Changed Data Model for storing data in Database (#4) * FEATURE: Added PassiveSearch v1 - Added Connection to ElasticSearch - Added 2 API's - /ping and /passive - Keyword Search using /passive - Added Storing to DB capability when using /search * Fix: Added error handling -added error handling for passivesearch api endpoint * FEATURE: Added Simple Query String - Changed ElasticSearch Search type to Simple Query String * REFACTOR: Improved Active Search -Made Active Search make better use of downtimes (Rate Limits or Validation Downtime) * FIX: Issues with Active Search -Fixed errors with parrallel processing * FEATURE: Added SeedScript - Added RootQuery as an optional query param for activesearch which basically takes out the prompt builder and takes a raw query to look for in Github API. - Added Python SeedScript * REFACTOR: Changed Data Model for Storing in Database - Added other fields like LastUpdated, ETAG, URL, LastModified to Schema. - Added SHA hash as the unique parameter to not allow duplicates in the database * REFACTOR: Minor Log Refactor - Changed console.log to console.info / console.error * FIX: Storing only the reqd part of URL * FEATURE: Update API -Added PUT /database endpoint for updating the database - Changed name of the endpoints - Commented Database Trace Logs * FIX: Update API Fix + Global ElasticSearch and Octokit Clients - Fixed GetFileContents - Added Global variables for EsClient and Octokit for easy access and clean code - Better logs * FIX: Minor Comment * REFACTOR: UpdateAPI refactor - Update API Refactored * REFACTOR: Minor Fixes - Better API Naming - Fixed Update API --- .eslintrc.json | 4 +- scripts/seed_script.py | 8 ++- src/DB/dbutils.ts | 100 ++++++++++++++++++++++------- src/app.ts | 40 +++++++++--- src/main.ts | 2 +- src/searchtools/search.ts | 113 +++++++++++++++++++-------------- src/searchtools/searchutils.ts | 93 ++++++++++++++++++--------- src/updatetools/update.ts | 19 ++++++ src/updatetools/updateutils.ts | 102 +++++++++++++++++++++++++++++ 9 files changed, 369 insertions(+), 112 deletions(-) create mode 100644 src/updatetools/update.ts create mode 100644 src/updatetools/updateutils.ts diff --git a/.eslintrc.json b/.eslintrc.json index a7a3f42..8ceabc0 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -15,7 +15,9 @@ "eslint:recommended", "plugin:@typescript-eslint/recommended", "plugin:jest/recommended", - "prettier" + "prettier", + "modular/best-practices", + "modular/style" ], "rules": { // The following rule is enabled only to supplement the inline suppression diff --git a/scripts/seed_script.py b/scripts/seed_script.py index 21f45d2..e3a79a8 100644 --- a/scripts/seed_script.py +++ b/scripts/seed_script.py @@ -6,7 +6,8 @@ def call_local_endpoint(prompt): #TODO: Change this to the correct URL when activesearch endpoint is changed - url = f'http://localhost:8080/search?rootquery="{prompt}"' + url = f'http://localhost:8080/database?rootquery="{prompt}"' + try: response = requests.get(url) @@ -24,8 +25,9 @@ def call_local_endpoint(prompt): if __name__ == "__main__": #Get Open API files - call_local_endpoint("openapi: 3") + call_local_endpoint('openapi: 3') #Get Swagger files - call_local_endpoint("swagger: 2") + # call_local_endpoint('"swagger: \"2"') + #PS: Takes a long time to run \ No newline at end of file diff --git a/src/DB/dbutils.ts b/src/DB/dbutils.ts index da9171b..4abe2e5 100644 --- a/src/DB/dbutils.ts +++ b/src/DB/dbutils.ts @@ -1,23 +1,77 @@ -export async function checkClusterHealth(esClient: any) : Promise { - try { - const response = await esClient.cat.health(); - console.log('Cluster health:', response); - return response; - } catch (error) { - console.error('Error checking cluster health:', error); - return ""; - } - } - -export async function BulkStoreToDB(validFiles: any,esClient: any) : Promise{ - try { - if(validFiles.length == 0){ - return; - } - const response = await esClient.bulk({ body: validFiles }); - // console.log('Bulk response:', response); - return response; - } catch (error) { - console.error('Error bulk indexing:', error); - } -} \ No newline at end of file +import { esClient } from "../app.js"; + +export async function checkClusterHealth(): Promise { + try { + const response = await esClient.cat.health(); + console.info('Cluster health:', response); + return response; + } catch (error) { + console.error('Error checking cluster health:', error); + return ''; + } +} + +export async function BulkStoreToDB( + validFiles: any, +): Promise { + try { + if (validFiles.length == 0) { + return; + } + const response = await esClient.bulk({ body: validFiles }); + return response; + } catch (error) { + console.error('Error bulk indexing:', error); + } +} + +export async function DeleteDocumentWithId(Id : string): Promise { + try { + const index = 'openapi'; + const updatedDocument = { + isDeleted: true, + }; + await esClient.update({ + index, + id: Id, + body: { + doc: updatedDocument, + }, + }); + console.info(`Document with ID ${Id} soft deleted.`); + } catch (error) { + console.error('Error deleting document from the database:', error); + } +} + +export async function CreateDocument(Id:string, document: any): Promise { + try { + const index = 'openapi'; + await esClient.index({ + index, + id: Id, + body: { + doc: document, + }, + }); + console.info(`New Document Added with ID ${Id}`); + } catch (error) { + //TODO: Add error handling for 400 over here + console.error('Error creating the document:', error); + } +} + +export async function GetDocumentWithId(Id:string): Promise { + try { + const index = 'openapi'; + const document = await esClient.get({ + index, + id: Id, + }); + return document; + } catch (error) { + console.error('Error getting document from database:', error); + } +} + + diff --git a/src/app.ts b/src/app.ts index 623778e..ddf60e3 100644 --- a/src/app.ts +++ b/src/app.ts @@ -3,11 +3,10 @@ import { Octokit } from 'octokit'; import { activeSearch, passiveSearch } from './searchtools/search.js'; import dotenv from 'dotenv'; import es from 'elasticsearch'; -import { checkClusterHealth } from './DB/dbutils.js'; -import { throttling } from '@octokit/plugin-throttling' -import { retry } from '@octokit/plugin-retry' - - +import { checkClusterHealth} from './DB/dbutils.js'; +import { throttling } from '@octokit/plugin-throttling'; +import { retry } from '@octokit/plugin-retry'; +import { UpdateOpenAPIFiles } from './updatetools/update.js'; const CustomOctokit = Octokit.plugin(throttling as any, retry as any); dotenv.config(); @@ -20,7 +19,8 @@ const octokit = new CustomOctokit({ octokit.log.warn( `Request quota exhausted for request ${options.method} ${options.url}`, ); - console.log(`Retrying after ${retryAfter} seconds!`); + console.info(`Retrying after ${retryAfter} seconds!`); + return true; }, onSecondaryRateLimit: (retryAfter, options, octokit) => { @@ -34,6 +34,21 @@ const octokit = new CustomOctokit({ const app = express(); +export const esClient = new es.Client({ + host: 'http://localhost:9200', + // log: 'trace', +}); + +//TODO: Iterate on api endpoints +app.get('/search', async (_req, _res) => { + const query = _req.query.q as string; + const results = await passiveSearch(query); + _res.send(results); +}); + +//openapi2db +app.post('/openapi', async (_req, _res) => { +======= const esClient = new es.Client({ @@ -70,14 +85,23 @@ app.use('/search', async (_req, _res) => { _res.send(results); }); + +app.put('/openapi', async (_req, _res) => { + const results = await UpdateOpenAPIFiles(); + _res.send(results); +}); + app.use('/ping', async (_req, _res) => { - const response = await checkClusterHealth(esClient); + const response = await checkClusterHealth(); _res.send(response); -}) +}); + app.get('/', (_req, _res) => { _res.send('TypeScript With Express'); }); + + export default app; export { octokit }; diff --git a/src/main.ts b/src/main.ts index b2226e7..ae02113 100644 --- a/src/main.ts +++ b/src/main.ts @@ -5,5 +5,5 @@ dotenv.config(); const port = process.env.PORT; app.listen(port, () => { - console.log(`⚡️[server]: Server is running at http://localhost:${port}`); + console.info(`⚡️[server]: Server is running at http://localhost:${port}`); }); diff --git a/src/searchtools/search.ts b/src/searchtools/search.ts index cdf1e2d..1a8b1ce 100644 --- a/src/searchtools/search.ts +++ b/src/searchtools/search.ts @@ -1,9 +1,9 @@ -import { queryBuilder, ValidateandStoreFiles} from './searchutils.js'; -import { octokit } from '../app.js'; +import { queryBuilder, ValidateandStoreFiles } from './searchutils.js'; +import { octokit, esClient } from '../app.js'; +let processCount = 0; +let finishedCount = 0; -let processCount = 1; -let finishedCount = 1; export async function activeSearch( prompt: string, @@ -11,49 +11,71 @@ export async function activeSearch( organisation: string, username: string, rootquery: string, - esClient: any, ): Promise { - const query = await queryBuilder(prompt, repo, organisation, username, rootquery); + const query = await queryBuilder( + prompt, + repo, + organisation, + username, + rootquery, + ); let files = []; let validFiles = []; - console.log("Query: "+query) - await octokit.paginate(octokit.rest.search.code, { - q: query, - per_page: 100 - }, - (response : any) => { - files = files.concat(response.data) - if(files.length >= 200){ - processCount++; - console.log("ValidateandStoreFiles Process Number "+processCount+" Started") - ValidateandStoreFiles(files, esClient).then((validatedFiles) => { - validFiles = validFiles.concat(validatedFiles); - finishedCount++; - console.log("ValidateandStoreFiles Process Number "+finishedCount+" Started") - }); - files = [] - } - } + console.info('Query: ' + query); + await octokit.paginate( + octokit.rest.search.code, + { + q: query, + per_page: 100, + }, + (response: any) => { + files = files.concat(response.data); + if (files.length >= 200) { + processCount++; + console.info( + 'ValidateandStoreFiles Process Number ' + processCount + ' Started', + ); + ValidateandStoreFiles(files).then((validatedFiles) => { + validFiles = validFiles.concat(validatedFiles); + finishedCount++; + console.info( + 'ValidateandStoreFiles Process Number ' + + finishedCount + + ' Finished', + ); + }); + files = []; + } + }, ); //this ending before the above one processCount++; - console.log("ValidateandStoreFiles Process Number "+processCount+" Started") - ValidateandStoreFiles(files, esClient).then((validatedFiles) => { + console.info( + 'ValidateandStoreFiles Process Number ' + processCount + ' Started', + ); + ValidateandStoreFiles(files).then((validatedFiles) => { validFiles = validFiles.concat(validatedFiles); - console.log("ValidateandStoreFiles Process Number "+finishedCount+" Started") + console.info( + 'ValidateandStoreFiles Process Number ' + finishedCount + ' Finished', + ); finishedCount++; }); - while(processCount > finishedCount){ - await new Promise(r => setTimeout(r, 3000)); - console.log("Total Processes: "+processCount+"\nFinished Processes: "+finishedCount) - console.log("Waiting for all files to be processed") + while (processCount > finishedCount) { + await new Promise((r) => setTimeout(r, 5000)); + console.info( + 'Total Processes: ' + + processCount + + '\nFinished Processes: ' + + finishedCount, + ); + console.info('Waiting for all files to be processed'); + } return validFiles; } export async function passiveSearch( query: string, - esClient: any, ): Promise { try { if (esClient === undefined) { @@ -65,33 +87,32 @@ export async function passiveSearch( query: { simple_query_string: { query: query, - fields: ["servers^2","paths^1.5","data^1"], - default_operator: "and" - } - } - } - }); + fields: ['title^3', 'servers^2', 'paths^1.5', 'data^1'], + default_operator: 'and', + }, + }, + }, + }); if (result.hits.hits) { if (result.hits.hits.length === 0) { - console.log('No results found in the database'); + console.error('No results found in the database'); // activeSearch(query, "", "", "", esClient); } return result.hits.hits; } } catch (error) { if (error.message.includes('No Living connections')) { - console.log('Elasticsearch connection error:', error); - return error + + console.error('Elasticsearch connection error:', error); + return error; } else { - console.log('Error occurred during passive search:', error); - return error; + console.error('Error occurred during passive search:', error); + return error; + } } return 'Database not found'; } - - - diff --git a/src/searchtools/searchutils.ts b/src/searchtools/searchutils.ts index bbe42ad..22cd1b2 100644 --- a/src/searchtools/searchutils.ts +++ b/src/searchtools/searchutils.ts @@ -1,13 +1,11 @@ import { octokit } from '../app.js'; // import { CodeSearchResponse } from "./searchstructs.js"; -import crypto from "crypto"; +import crypto from 'crypto'; import OASNormalize from 'oas-normalize'; import { BulkStoreToDB } from '../DB/dbutils.js'; +export function generateUUID(): string { - - -export function generateUUID() : string { // Generate a random buffer of 16 bytes const buffer = crypto.randomBytes(16); @@ -25,7 +23,7 @@ export async function getFileContents( repoowner: string, reponame: string, filepath: string, -): Promise { +): Promise { const response = await octokit.request( 'GET /repos/{owner}/{repo}/contents/{path}', { @@ -34,20 +32,30 @@ export async function getFileContents( path: filepath, }, ); - return response.data['content']; + return response; } -export async function queryBuilder(prompt: string, repo: string, organisation: string, username: string, rootquery: string): Promise { - if(prompt == undefined){ - prompt = "" +export async function queryBuilder( + prompt: string, + repo: string, + organisation: string, + username: string, + rootquery: string, +): Promise { + if (prompt == undefined) { + prompt = ''; } - let query - if(rootquery != undefined){ - query = rootquery - return query + let query; + if (rootquery != undefined) { + if (rootquery === 'openapi') { + query = 'openapi: 3' + } else if (rootquery === 'swagger') { + query = '"swagger: \\"2"' + } + return query; } query = prompt + ' AND "openapi: 3"'; - // query+= prompt + ' AND "swagger: \\"2"' + // query+= prompt + ' AND "swagger: \\"2"' if (repo != undefined) { query += '+repo:' + repo; } else if (organisation != undefined) { @@ -60,37 +68,62 @@ export async function queryBuilder(prompt: string, repo: string, organisation: s return query; } -export async function ValidateandStoreFiles(files: any[], esClient: any): Promise { - if(files.length == 0){ + +export async function ValidateandStoreFiles( + files: any[], +): Promise { + if (files.length == 0) { return; } - console.log("Validating and storing files") + console.info('Validating and storing files'); let validFiles = []; for (const file of files) { - const base64content = await getFileContents( + const response = await getFileContents( file.repository.owner.login, file.repository.name, file.path, ); - const content = Buffer.from(base64content, 'base64').toString(); + const content = Buffer.from( + response['data']['content'], + 'base64', + ).toString(); const oas = new OASNormalize.default(content); oas .validate() .then((definition) => { - console.log('File ' + file.name + ' is valid'); - console.log(definition?.info?.title) - validFiles.push({index: { _index: 'openapi', _id: generateUUID()}}); - validFiles.push({title: definition?.info?.title, description: definition?.info?.description, version: definition?.info?.version, servers: JSON.stringify(definition?.servers), paths: JSON.stringify(definition?.paths) , path: file.path, repository: file?.repository?.name, owner: file?.repository?.owner?.login, data: content}); - if(validFiles.length >= 50){ - console.log("Storing some of the valid files") - BulkStoreToDB(validFiles as any[],esClient as any); - validFiles = [] + console.info('File ' + file.name + ' is valid'); + console.info(definition?.info?.title); + console.info(validFiles.length); + validFiles.push({ + index: { _index: 'openapi', _id: response['data']['sha'] }, + }); + validFiles.push({ + URL: response['url'].split('api.github.com/repos/')[1], + ETAG: response['headers']['etag'], + title: definition?.info?.title, + description: definition?.info?.description, + version: definition?.info?.version, + servers: JSON.stringify(definition?.servers), + paths: JSON.stringify(definition?.paths), + filepath: file.path, + repository: file?.repository?.name, + owner: file?.repository?.owner?.login, + data: content, + LastModified: response['headers']['last-modified'], + LastUpdated: new Date().toISOString(), + isDeleted: false, + }); + if (validFiles.length >= 50) { + console.info('Storing some of the valid files'); + BulkStoreToDB(validFiles as any[]); + validFiles = []; } }) .catch((error) => { - console.log('File ' + file.name + ' is not valid'); + console.info('File ' + file.name + ' is not valid'); }); - } - BulkStoreToDB(validFiles as any[],esClient as any); + } + BulkStoreToDB(validFiles as any[]); + return validFiles; } diff --git a/src/updatetools/update.ts b/src/updatetools/update.ts new file mode 100644 index 0000000..4a993d5 --- /dev/null +++ b/src/updatetools/update.ts @@ -0,0 +1,19 @@ +import { UpdateDocument, scrollSearch } from './updateutils.js'; + +export async function UpdateOpenAPIFiles(): Promise { + const params = { + index: 'openapi', + scroll: '30s', + size: 1, + _source: ['owner', 'repository', 'filepath', 'ETAG', 'isDeleted'], + body:{ + query: { + match_all: {} + } + } + } + for await (const hit of scrollSearch(params)) { + await UpdateDocument(hit); + } + return "Updated All OpenAPI Files"; +} diff --git a/src/updatetools/updateutils.ts b/src/updatetools/updateutils.ts new file mode 100644 index 0000000..1c16324 --- /dev/null +++ b/src/updatetools/updateutils.ts @@ -0,0 +1,102 @@ +import OASNormalize from "oas-normalize" +import { octokit, esClient } from "../app.js" +import { DeleteDocumentWithId, CreateDocument } from "../DB/dbutils.js" +import { url } from "inspector"; + + +async function ETAGRequestBuilder(document: any): Promise { + const requestConfig = { + owner: document._source.owner, + repo: document._source.repository, + path: document._source.filepath, + }; + requestConfig['headers'] = { + 'If-None-Match': document._source.ETAG, + }; + const request = { + method: 'GET', + url: "/repos/"+requestConfig.owner+"/"+requestConfig.repo+"/contents/"+requestConfig.path, + headers: { + 'If-None-Match': document._source.ETAG, + } + }; + return request; +} + +export async function * scrollSearch (params) { + let response = await esClient.search(params) + while (true) { + const sourceHits = response.hits.hits + if (sourceHits.length === 0) { + break + } + for (const hit of sourceHits) { + yield hit + } + if (!response._scroll_id) { + break + } + response = await esClient.scroll({ + scroll_id: response._scroll_id, + scroll: params.scroll + }) + } +} + +export async function UpdateDocument(document: any): Promise { + if(document._source.isDeleted === true){ + return + } + const request = await ETAGRequestBuilder(document) + await octokit.request(request).then(async(response) => { + console.info("File "+document._id+" to be updated") + // make isDeleted true for current Document and update + DeleteDocumentWithId(document._id) + // create new Document with new content + const content = Buffer.from( + response['data']['content'], + 'base64', + ).toString(); + const oas = new OASNormalize.default(content); + oas + .validate() + .then(async(definition) => { + console.info('Updated file is valid'); + console.info(definition?.info?.title); + const newData = { + URL: response['url'].split('api.github.com/repos/')[1], + ETAG: response['headers']['etag'], + title: definition?.info?.title, + description: definition?.info?.description, + version: definition?.info?.version, + servers: JSON.stringify(definition?.servers), + paths: JSON.stringify(definition?.paths), + filepath: document._source.filepath, + repository: document._source.repository, + owner: document._source.owner, + data: content, + LastModified: response['headers']['last-modified'], + LastUpdated: new Date().toISOString(), + isDeleted: false, + } + await CreateDocument(response['data']['sha'], newData) + console.info('Updated File '+response['data']['sha']+' Added To Database'); + }) + .catch(() => { + console.info('Updated File is not valid'); + }); + }).catch((error) => { + if(error.status == 304){ + console.info("File "+document._id+" has not changed") + return + } else if (error.status == 404){ + console.info("File "+document._id+" does not exist anymore and has been deleted") + DeleteDocumentWithId(document._id) + } else { + console.error(error) + } + }); +} + + + From 0db51e460514b00ce218db0fa3d17c500357b0ba Mon Sep 17 00:00:00 2001 From: Ishaan Shah <70190533+ishaan812@users.noreply.github.com> Date: Sun, 5 Nov 2023 14:26:27 +0530 Subject: [PATCH 7/7] Small Merge Fix --- src/app.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/app.ts b/src/app.ts index ddf60e3..be79e6e 100644 --- a/src/app.ts +++ b/src/app.ts @@ -48,12 +48,10 @@ app.get('/search', async (_req, _res) => { //openapi2db app.post('/openapi', async (_req, _res) => { -======= - - const esClient = new es.Client({ host: 'http://localhost:9200', log: 'trace', +}) }); // Should not even be an API endpoint for passive search @@ -64,7 +62,7 @@ const esClient = new es.Client({ app.use('/passive', async (_req, _res) => { const query = _req.query.q as string; - const results = await passiveSearch(query, esClient); + const results = await passiveSearch(query); _res.send(results); }) @@ -80,7 +78,6 @@ app.use('/search', async (_req, _res) => { Organisation as string, User as string, RootQuery as string, - esClient as any, ); _res.send(results); });