add migrations for ML generated fields

lunatrace/bsl/hasura/migrations/lunatrace/1679066158985_add_ml_tables/down.sql

@@ -0,0 +1,8 @@
+
+DROP TABLE vulnerability.code_snippet;
+
+-- Add a generated summary to the reference to make it easier for the LLM to choose what to read
+ALTER TABLE vulnerability.reference_content DROP COLUMN summary;
+
+ALTER TABLE package.package DROP COLUMN readme_text;
+ALTER TABLE package.package DROP COLUMN use_case_summary;

lunatrace/bsl/hasura/migrations/lunatrace/1679066158985_add_ml_tables/up.sql

@@ -0,0 +1,22 @@
+
+CREATE TABLE vulnerability.code_snippet
+(
+    id          uuid                        DEFAULT public.gen_random_uuid() NOT NULL PRIMARY KEY,
+    created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP NOT NULL,
+    -- Reference may be null because we may have pulled code from a non-web source such as vuln-db
+    reference_id uuid NULL references vulnerability.reference,
+    -- Include url since reference might be null but its still nice to be able to point a source like a vuln-db link for non-scraped content
+    source_url text NOT NULL,
+    vulnerability uuid NOT NULL references vulnerability.vulnerability,
+    code text NOT NULL,
+    score integer NOT NULL,
+    summary text NOT NULL,
+    type text NOT NULL,
+    language text NOT NULL
+);
+
+-- Add a generated summary to the reference to make it easier for the LLM to choose what to read
+ALTER TABLE vulnerability.reference_content ADD COLUMN summary text NULL;
+
+ALTER TABLE package.package ADD COLUMN readme_text text NULL;
+ALTER TABLE package.package ADD COLUMN use_case_summary text NULL;

lunatrace/bsl/ml/python/scrape_utils/summarize_package_readme.py

@@ -18,6 +18,7 @@
 An explanation of the Fastify framework, for instance, 
 would be identical, because the use case of the two libraries is the same. Don't mention the license, the creators name, or anything that isn't relevant to what the library is used for. 
 
+If you can't tell because the readme is useless and you're not familiar with the library from prior knowledge, return nothing at all, just empty, no words or explanation.
 ---- BEGIN README ----
 {text}
 ---- END README ----

lunatrace/bsl/ml/python/scrape_utils/summarize_scraped.py

@@ -88,7 +88,7 @@ def main(args):
 		print(results)
 
 def add_subparser(subparsers):
-	subparser = subparsers.add_parser('summarize-scraped', help="takes any page content and a command to extract some information from it, as desired. Useful when you have a specific question to ask of an advisory")
+	subparser = subparsers.add_parser('summarize-scraped', help="takes any page content and a command to extract some information from it, as desired. Useful when you have a specific question to ask of an advisory. Not used for advisory ingestion, instead used by the chat-bot in real time.")
 
 	subparser.add_argument("contents", nargs = 1, type = str, help = "a string of page contents")
 	subparser.add_argument("query", nargs = 1, type = str, help = "query string that the scraper will try to focus on. can be phraised as a question or command, both are fine")