merge code deployed to PROD from dev to main

ChangeLog.md

@@ -1,7 +1,12 @@
 # Change Log
 
-## 2.0.0-rc14
-  - initial version of `cve-core` as a peer project to other `cve-projects`.  Can be used as part of a monorepo
+### 2.1.0
+  - wildcard search using "*" and "?"
+  - AppConfig to manage hierarchical environment variables and all default values for environment variables
+  - code refactoring: reorganizing code, general cleaning up for migration to github
+
+### 2.0.0-rc14 - deployed: 2025-06-05
+  - initial version of `cve-core` as a peer project to other `cve-projects`.  Can be used as part of a monorepo (e.g., https://github.com/CVEProject/CVE-Search-API/tree/dev)
   - search using `axios`, NodeJS-native `fetch` and `@opensearch-project/opensearch` libraries
     - CVE-, CWE-, and CAPAC- IDs
     - CVE YEAR
@@ -12,7 +17,6 @@
     - hyphenated words (e.g., "man-in-the-middle")
     - software names (e.g., "Node.JS", ".NET")
     - file extension (e.g., "matvar_struct.c")
-    - repeating non-language characters (e.g., "aaaaa" is ok, but "?????" is replaced by "")
     - can run as AWS Lambda Layer
   - new adapters
     - CVE Services reader
@@ -26,18 +30,47 @@
 
 ## Older Milestones from the older `cveUtils`/`cvelist-bulk-download` repositories
 
-Note that the following milestones were in other repositories, which contained a superset of the source code in this npm library.  The milestones below are meant only for historic reference, in case a full history of an implementation is needed.
+Note that the following milestones were in multiple repositories, and together contained a superset of the source code in this npm library.  The milestones below are meant only for historic reference, in case a full history of an implementation is needed.
+
+### 1.3.0 - deployed only on AWS in 2024-12 for initial search capability (tag `2024-12-06`)
+  - search using `axios` and `@opensearch-project/opensearch` libraries
+    - general search for tokenized strings in all fields
+    - CVE-ID
+
+### 2.0.0-rc14
+  - initial version of `cve-core` as a peer project to other `cve-projects`.  Can be used as part of a monorepo (e.g., https://github.com/CVEProject/CVE-Search-API/tree/dev)
+  - search using `axios`, NodeJS-native `fetch` and `@opensearch-project/opensearch` libraries
+    - CVE-, CWE-, and CAPAC- IDs
+    - CVE YEAR
+    - basic version strings (e.g., "v3.2.5", "v3.2.5-RC1")
+    - basic IPv4 and IPv6
+    - URLs
+    - compound words (e.g., "docker-compose", "microsoft word")
+    - hyphenated words (e.g., "man-in-the-middle")
+    - software names (e.g., "Node.JS", ".NET")
+    - file extension (e.g., "matvar_struct.c")
+    - repeating non-language characters (e.g., "aaaaa" is ok, but "?????" is replaced by "")
+    - can run as AWS Lambda Layer
+  - new adapters
+    - CVE Services reader
+    - CVE Search reader
+    - CVE file reader
+    - file reader/writer
+    - console input for interacting with a user in a CLI
+  - CveResult class with standardized errors and messages (this version is aimed at the search service)
+  - object (JSON) comparer using `json-difference` library
+  - JSON replacer that alphabetizes keys when serializing using JSON.stringify()
 
 ### 1.2.0 - deployed 2024-07-18 (tag `2024-07-18_v1.2.0`)
   - baseline for the `cve-core` npm library
   - changes for cisa adp, reference ingest
-  - axios-retry for network retry
-  - optimized update.yml to use fetch-depth: 1
-  - CVES_MAX_ALLOWABLE_CVE_YEAR environment variable set to 2025
-  - GIT_MAX_FILESIZE_MB environment variable set to 100
+  - `axios-retry` library for network retry
+  - optimized `update.yml` to use `fetch-depth: 1`
+  - `CVES_MAX_ALLOWABLE_CVE_YEAR` environment variable set to 2025
+  - `GIT_MAX_FILESIZE_MB` environment variable set to 100
   - initial refactoring of core classes to separate I/O functions from business logic classes (work in progress)
-  - minimized 3rd party dependency in IsoDateString class to minimize footprint for AWS Lambda
-  - import specific lodash functions instead of the full lodash to minimize footprint for AWS Lambda
+  - minimized 3rd party dependency in IsoDateString class to minimize AWS Lambda footprint
+  - import specific lodash functions instead of the full lodash to minimize AWS Lambda footprint
   - dependabot PRs defaults to develop branch
   - cveUtils/GitLab PR 32
 
@@ -46,31 +79,30 @@ Note that the following milestones were in other repositories, which contained a
   - tested but not used on cvelistV5
 
 ### 1.1.0 - 2023-09-26 (tag `2023-09-26_v1.1.0`)
-  - Delta files in /cves (delta.json and deltaLog.json), replacing recent_activities.json
+  - Official support for delta files in /cves (delta.json and deltaLog.json), replacing recent_activities.json
 
 ### 1.0.0 - 2023-05-26 (tag `2023-04-25_v1.0.0`)
   - Official version using public domain code in https://github.com/CVEProject/cvelist-bulk-download
 
-
 ### `Sprint-0` - 2023-04-20 (tag `2023-04-20_initial_cveUtils_on_github`)
-  - initial version selectively copied from internal MITRE gitlab to https://github.com/hkong-mitre/cvelist-bulk-download
-  - https://github.com/hkong-mitre/cvelist-bulk-download/commit/207b9f2b82908afbd8d9d2270969f6781f9d39e4
-  - (note date is different):  https://gitlab.mitre.org/hkong/cve_utils/-/tags/2023-04-25_to_github_hkong-mitre_cvelist_bulk_download
+  - initial version selectively copied from internal MITRE gitlab to https://github.com/CVEProject/cvelist-bulk-download
+  - https://github.com/CVEProject/cvelist-bulk-download/commit/207b9f2b82908afbd8d9d2270969f6781f9d39e4
+  - note date is slightly different in GitLab tag in cve_utils, but the code is functionly the same: `2023-04-25_to_github_hkong-mitre_cvelist_bulk_download`
 
 
 ### 2023-03-29
-  - official version used in GitHub actions that updated /cves when cvelistV5 was announced at CNA Summit 2023
-  - https://gitlab.mitre.org/hkong/cve_utils/-/tags/2023-03-29-cveproject_cvelistV5_dist_(similar)
+  - official version used in GitHub actions that updated `/cves` when cvelistV5 was announced at CNA Summit 2023
+  - GitLab tag in cve_utils: `2023-03-29-cveproject_cvelistV5_dist_(similar)`
 
 
 ### 2023-03-10
   - code during team code walkthru
-  - https://gitlab.mitre.org/hkong/cve_utils/-/tags/2023_03_10_code_walkthrough_with_team
+  - GitLab tag in cve_utils: `2023_03_10_code_walkthrough_with_team`
 
 
 ### 2023-03-06
   - first version deployed to cvelistV5 for testing (using `preview_cves` instead of `cves`)
-  - https://gitlab.mitre.org/hkong/cve_utils/-/tags/2023_03_06_deployed_to_cveproject_cvelistv5
+  - GitLab tag in cve_utils: `2023_03_06_deployed_to_cveproject_cvelistv5`
 
 
 ## Additional Information

PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,20 @@
+Note: Please format the pull request title like:
+"Resolves issue ###, High level description of pull request."
+
+Closes Issue ###
+
+# Summary
+Write out a concise summary of this MR.
+
+# Important Changes
+`example_file.js`
+- Cleaned code and added comments.
+- Added method to handle adding items. 
+
+# Testing
+
+Steps to manually test updated functionality, if possible
+- [ ] 1) Example test step.
+
+# Notes
+- Some additional notes about this MR.

README.md

@@ -6,7 +6,7 @@ This CVE project implements the `cve-core` common library containing the general
 
 ## Versioning
 
-The first version of this library is **version 2.0.0**.  This is because the capabilities of this library have already been in use in [cvelistV5](xxxxx), and to preserve the versioning of capabilities, we decided to start this library at 2.0.0.  See [the ChangeLog](./ChangeLog.md) for specific details.
+The first version of this library is **version 2.0.0**.  This is because the capabilities of this library have already been in use in [cvelistV5](https://github.com/CVEProject/cvelistV5), and to preserve the versioning of capabilities, we decided to start this library at 2.0.0.  See [the ChangeLog](./ChangeLog.md) for specific details.
 
 ## Usage
 

config/custom-environment-variables.jsonc

@@ -0,0 +1,24 @@
+{
+  // this file is used by node-config to map a node-config (AppConfig)
+  //  hierarchy of constants to an environment variable
+  // Note that much of the environment variables mapped here existed for some time
+  //  without AppConfig, this file bridges the historical uses of those with the new
+  //  as we transition to AppConfig
+  "appConfig": {
+    // constants for search capability
+    "search": {
+      "providerEndpoint": "OpenSearchDomainEndpoint",
+      "index": "OpenSearchCveIndex",
+      // allows local development using containers that do not have SSL certs
+      "allowUnknownSslCerts": "OpenSearchAllowUnknownSslCerts"
+    },
+    // constants for testing node-config
+    // these values are only used to test node-config in AppConfig.test.int.ts
+    // DO NOT USE THIS FOR ANYTHING ELSE
+    "test": {
+      "appConfigTest": {
+        "test": "JEST_env_config_test"
+      }
+    }
+  }
+}

config/default.jsonc

@@ -0,0 +1,22 @@
+{
+  // Default configuration
+  //  These values are overridable using other *.jsonc (e.g., prod.jsonc)
+  //    as well as using environment variables (e.g., in `.env`)
+  //    - Each configuration is mapped using custom-environment-variables.jsonc to enable environment varialbe overrides.
+  //    - For more information, see cve-core/src/adapters/config/AppConfig.ts
+  //  NOTE for consistency, all values need to be strings for proper type when using .env overrides
+  "appConfig": {
+    // constants for search capability
+    "search": {
+      // minimum versions for servers that are compatible with current code
+      "minServer": [
+        "elasticsearch:7.10.2",
+        "opensearch:2.10.0"
+      ],
+      // setting this to FALSE (recommended) requires an SSL cert to access the search server
+      // The only time this should be allowed to be true is when developing or testing
+      //   using containers that do not have SSL certs
+      "allowUnknownSslCerts": "FALSE"
+    }
+  }
+}

config/devel.jsonc

@@ -0,0 +1,47 @@
+{
+  // development configurations
+  // overrides values specified in default.jsonc, read additional comments there
+  //   and in cve-core/src/adapters/config/AppConfig.ts
+  // NOTE for consistency, all values need to be strings for proper type when using .env overrides
+  "appConfig": {
+    // constants for search capability
+    "search": {
+      // minimum versions for servers that are compatible with current code
+      "minServer": [
+        "opensearch:2.10.0"
+      ],
+      // URL to reach search server
+      "providerEndpoint": "https://admin:admin@localhost:9200",
+      // index on search server related to searching CVEs
+      "index": "e2e-cve-test-index-1109",
+      // setting this to FALSE (recommended) requires an SSL cert to access the search server
+      // The only time this should be allowed to be true is when developing or testing
+      //   using containers that do not have SSL certs
+      // DO NOT USE THIS IN ANY PUBLIC OR PRODUCTION ENVIRONMENTS
+      "allowUnknownSslCerts": "TRUE"
+    },
+    // constants for unit, int, e2e testing
+    "test": {
+      // constants for testing search capability
+      "searchTest": {
+        // many tests for search uses snapshots, which requires CVEs to remain unchanged
+        // since the live server is updated all the time, a fixture containing fixed CVEs
+        // is required to keep the test consistent.  "fixtures" provides the link
+        // to the cve-fixtures repository
+        "fixtures": {
+          // @todo these constants needs to be in sync in cve-fixtures 
+          // so that testing snapshots are consistent and valid
+          "name": "fixtures-search-baseline-1086", // release tag
+          "numCves": "1086" // possible identifier assuming we always add cves to a new release
+        }
+      },
+      // constants for testing node-config
+      "appConfigTest": {
+        // these values are only used to test node-config in AppConfig.test.int.ts
+        // DO NOT USE THIS FOR ANYTHING ELSE
+        "two": "2",
+        "five": "5"
+      }
+    }
+  }
+}

config/prod.jsonc

@@ -0,0 +1,17 @@
+{
+  // production (example) constants
+  // overrides values specified in default.jsonc, read additional comments there
+  //   and in cve-core/src/adapters/config/AppConfig.ts
+  // NOTE for consistency, all values need to be strings for proper type when using .env overrides
+  "appConfig": {
+    "search": {
+      "minServer": [
+        "elasticsearch:7.10.2",
+        "opensearch:2.10.0"
+      ],
+      "providerEndpoint": "", // preference is to specify this in a (secret) environment variable on production platforms
+      "index": "", // preference is to specify this in a (secret) environment variable on production platforms
+      "allowUnknownSslCerts": "FALSE"
+    }
+  }
+}

docs/BasicSearchManager.md

@@ -1,23 +1,23 @@
 # BasicSearchManager
 
 `BasicSearchManager` provides basic facilities for working with an ElasticSearch/OpenSearch instance.  It provides the following:
-- `search` providing a standardized way to do a search.  This method hides the details of how a search is done depending on the user's search text and other input parameters (e.g., when doing faceted search or when all matches are requested, requiring paging).  When using this asynchronous method, the returned `CveResult` will contain results from the search, plus possibly notes and errors that were found during data validation and searching.
-- `validateSearchText` is a synchronous method that will return a `CveResult` object potentially containing notes and errors.
+- `search()` providing a standardized way to do a search.  This method hides the details of the different ways a search is carried out in ElasticSearch/OpenSearch using the user's search text and other input parameters (e.g., when doing faceted search or when all matches are requested, requiring paging).  When using this asynchronous method, the returned `CveResult` will contain results from the search, plus possibly notes and errors that were found during data validation and searching.
 
 In addition, the following associated classes and types are also defined:
-- `SearchProviderInfo` --- an object to fully represent a specific index in an ElasticSearch/OpenSearch instance
-- `SearchOptions` --- options when searching
+- `SearchProviderSpec` --- an object to fully represent a specific index in an ElasticSearch/OpenSearch instance
+- `SearchOptions` --- options when searching (e.g., `default_operator`)
 - `SearchResultData` --- a strongly typed type to facilitate working with search results
 
-For an example of how to use the BasicSearchManager and its associated classes and types, see [BasicSearchManager Examples](#basicsearchmanager-examples).
+For examples of how to use the BasicSearchManager see [Simple Search Example](#simple-search-example).
 
 ## BasicSearchManager Examples
 
 ### Simple Search Example
 
 ```typescript
 import { CveResult } from "cve-core/CveResult.js"
-import { SearchResultData, BasicSearchManager } from "cve-core/BasicSearchManager.js";
+import { BasicSearchManager } from "cve-core/BasicSearchManager.js";
+import { SearchResultData } from "cve-core/SearchResultData.js";
 const simpleSearch = async () => {
   const searchManager = new BasicSearchManager({
     index: "cve-index-local",

index.ts

@@ -1,52 +1,61 @@
 /**
- * This is intended to be the main export file for cve-core.
+ * This is the main export file for cve-core when used as library
+ * Files that makes up this library should not use this, however,
+ *   and should use relative paths
  */
 
 // adapters
-export * from "./src/adapters/fs/CveFsReader.js"
-export * from "./src/adapters/fs/FsReader.js"
-export * from "./src/adapters/fs/FsWriter.js"
-export * from './src/adapters/search/SearchAdapter.js'
-export * from './src/adapters/search/SearchReader.js'
+export * from "./src/adapters/config/AppConfig.js";
+export * from './src/adapters/console/ConsoleInputReader.js';
+export * from './src/adapters/cveservice/CveService.js';
+export * from './src/adapters/cveservice/CveServiceBaseUrl.js';
+export * from './src/adapters/cveservice/CveServiceCreds.js';
+export * from './src/adapters/cveservice/cve/CveServiceCveReader.js';
+export * from './src/adapters/cveservice/healthCheck/CveServiceHealthReader.js';
+export * from "./src/adapters/fs/CveFsReader.js";
+export * from "./src/adapters/fs/DirectoryWalker.js";
+export * from "./src/adapters/fs/FsReader.js";
+export * from "./src/adapters/fs/FsWriter.js";
+export * from './src/adapters/search/SearchAdapter.js';
+export * from './src/adapters/search/SearchReader.js';
+
 
 // commands
-export * from "./src/commands/DateCommand.js"
-export * from "./src/commands/GenericCommand.js"
-export * from "./src/commands/MainCommands.js"
+export * from "./src/commands/DateCommand.js";
+export * from "./src/commands/GenericCommand.js";
+export * from "./src/commands/MainCommands.js";
 
 // common
-export * from "./src/common/IsoDate/IsoDateString.js"
-export * from "./src/common/Json/Json.js"
-export * from "./src/common/comparer/ObjectComparer.js"
+export * from "./src/common/IsoDate/IsoDateString.js";
+export * from "./src/common/Json/Json.js";
+export * from "./src/common/comparer/ObjectComparer.js";
 
 // core
-export * from "./src/core/result/CveResult.js"
-export * from "./src/core/CveId.js";
+export * from "./src/cveId/CveId.js";
 export * from "./src/core/Activity.js";
 export * from "./src/core/ActivityLog.js";
-export * from "./src/core/CveComparer.js";
-export * from "./src/core/CveCore.js";
-export * from "./src/core/CveCorePlus.js";
-export * from "./src/core/CveDate.js";
-export * from "./src/core/CveId.js";
-export * from "./src/core/CveListDir.js";
-export * from "./src/core/CveRecord.js";
+export * from "./src/common/comparer/CveComparer.js";
+export * from "./src/cve/CveCore.js";
+export * from "./src/cve/CveCorePlus.js";
+export * from "./src/date/CveDate.js";
+export * from "./src/deprecated/CveListDir.js";
+export * from "./src/cve/CveRecord.js";
 export * from "./src/core/Delta.js";
 export * from "./src/core/DeltaFs.js";
 export * from "./src/core/DeltaLog.js";
-export * from "./src/core/fsUtils.js";
+export * from "./src/deprecated/fsUtils.js";
 export * from "./src/core/git.js";
+export * from "./src/date/CveDate.js";
 
-//search
-export * from './src/core/search/BasicSearchManager.js'
+// cve result
+export * from "./src/result/CveResult.js";
 
-// generated
-export * from "./src/generated/quicktools/CveRecordV5.js";
+// search
+export * from './src/search/BasicSearchManager.js';
+export * from './src/search/SearchRequest.js';
 
-// net
-export * from "./src/net/ApiBaseService.js";
-export * from "./src/net/CveService.js";
-export * from "./src/net/CveUpdater.js";
+// generated
+export * from "./src/cve/record/generated/CveRecordV5.js";
 
 // package info
 import * as packageJSON from './package.json';