Update various tables fields and constraints

reworked various tables to better fit our criteria

src/pe_reports/data/data_schema.sql

@@ -1,4 +1,3 @@
---
 -- PostgreSQL database dump
 --
 
@@ -16,6 +15,7 @@ CREATE TABLE IF NOT EXISTS public.organizations
     organizations_uid uuid default uuid_generate_v1() NOT NULL,
     name text NOT NULL,
     cyhy_db_name text,
+    UNIQUE(name),
     PRIMARY KEY (organizations_uid)
 );
 
@@ -24,9 +24,10 @@ CREATE TABLE IF NOT EXISTS public.root_domains
 (
     root_domain_uid uuid default uuid_generate_v1() NOT NULL,
     organizations_uid uuid NOT NULL,
-    organization_name text, NOT NULL,
+    organization_name text NOT NULL,
     root_domain text NOT NULL,
     ip_address text,
+    UNIQUE(root_domain, organizations_uid),
     PRIMARY KEY (root_domain_uid)
 );
 
@@ -37,18 +38,31 @@ CREATE TABLE IF NOT EXISTS public.sub_domains
     sub_domain text NOT NULL,
     root_domain_uid uuid NOT NULL,
     root_domain text NOT NULL,
+    UNIQUE(sub_domain, root_domain_uid),
     PRIMARY KEY (sub_domain_uid)
 );
 
+-- Organization's Sub Domains IPs Link Table
+CREATE TABLE IF NOT EXISTS Sub_domains_Ips
+(
+    sub_domain_uid uuid NOT NULL,
+    asset_uid uuid NOT NULL,
+    PRIMARY KEY (sub_domain_uid, asset_uid)
+);
+
 -- Organization's IPs Table
-CREATE TABLE IF NOT EXISTS public.ip_addresses
+CREATE TABLE IF NOT EXISTS public.web_assets
 (
-    ip_address_uid uuid default uuid_generate_v1() NOT NULL,
-    ip_address text NOT NULL,
+    asset_uid uuid default uuid_generate_v1() NOT NULL,
+    asset_type text Not NULL,
+    asset text NOT NULL,
     ip_type text,
-    sub_domain_uid uuid NOT NULL,
-    sub_domain text NOT NULL,
-    PRIMARY KEY (ip_address_uid)
+    verified boolean,
+    organizations_uid uuid NOT NULL,
+    report_on boolean DEFAULT TRUE,
+    last_scanned timestamp,
+    UNIQUE(asset),
+    PRIMARY KEY (asset_uid)
 );
 
 -- Organization's Aliases Table
@@ -57,6 +71,7 @@ CREATE TABLE IF NOT EXISTS public.alias
     alias_uid uuid default uuid_generate_v1() NOT NULL,
     organizations_uid uuid NOT NULL,
     alias text NOT NULL,
+    UNIQUE (alias),
     PRIMARY KEY (alias_uid)
 );
 
@@ -72,20 +87,23 @@ CREATE TABLE IF NOT EXISTS public.executives
 
 -- Reporting Tables ----
 -- Domain Masquerading Table
-CREATE TABLE IF NOT EXISTS public."DNSTwist"
+CREATE TABLE IF NOT EXISTS public."dnstwist_domain_masq"
 (
-    dnstwist_uid uuid default uuid_generate_v1() NOT NULL,
-    "discoveredBy" uuid NOT NULL,
-    "domain-name" text,
-    "dns-a" text,
-    "dns-aaaa" text,
-    "dns-mx" text,
-    "dns-ns" text,
-    fuzzer text,
-    "date-observed" text,
-    "ssdeep-score" text,
+    suspected_domain_uid uuid default uuid_generate_v1() NOT NULL,
     organizations_uid uuid NOT NULL,
-    PRIMARY KEY (dnstwist_uid)
+    "domain_permutation" text,
+    "ipv4" text,
+    "ipv6" text,
+    "mail_server" text,
+    "name_server" text,
+    fuzzer text,
+    "date_observed" date,
+    "ssdeep_score" text,
+    "malicious" boolean,
+    "blocklist_attack_count" integer,
+    "blocklist_report_count" integer,
+    UNIQUE ("domain_permutation"),
+    PRIMARY KEY (suspected_domain_uid)
 );
 
 -- Dark Web Alerts Table
@@ -94,7 +112,7 @@ CREATE TABLE IF NOT EXISTS public.alerts
     alerts_uid uuid default uuid_generate_v1() NOT NULL,
     alert_name text,
     content text,
-    date text,
+    date date,
     sixgill_id text,
     read text,
     severity text,
@@ -103,6 +121,9 @@ CREATE TABLE IF NOT EXISTS public.alerts
     threats text,
     title text,
     user_id text,
+    category text,
+    lang text,
+    UNIQUE (sixgill_id),
     organizations_uid uuid NOT NULL,
     PRIMARY KEY (alerts_uid)
 );
@@ -115,18 +136,20 @@ CREATE TABLE IF NOT EXISTS public.mentions
     collection_date text,
     content text,
     creator text,
-    date text,
+    date date,
+    sixgill_mention_id text,
     post_id text,
+    lang text,
     rep_grade text,
     site text,
     site_grade text,
     title text,
     type text,
     url text,
-    tags text,
     comments_count text,
     sub_category text,
-    query text,
+    tags text,
+    UNIQUE (sixgill_mention_id),
     organizations_uid uuid NOT NULL,
     PRIMARY KEY (mentions_uid)
 );
@@ -155,7 +178,6 @@ CREATE TABLE IF NOT EXISTS public.shodan_insecure_protocols_unverified_vulns
     UNIQUE (organizations_uid, ip, port, protocol, timestamp),
     PRIMARY KEY (insecure_product_uid)
 );
-
 --Shodan Veriried Vulnerabilities table
 CREATE TABLE IF NOT EXISTS public.shodan_verified_vulns
 (
@@ -189,7 +211,6 @@ CREATE TABLE IF NOT EXISTS public.shodan_verified_vulns
     UNIQUE (organizations_uid, ip, port, protocol, timestamp),
     PRIMARY KEY (verified_vuln_uid)
 );
-
 --Shodan Assets and IPs table
 CREATE TABLE IF NOT EXISTS public.shodan_assets
 (
@@ -228,7 +249,7 @@ CREATE TABLE IF NOT EXISTS public.hibp_breaches
     is_sensitive boolean,
     is_retired boolean,
     is_spam_list boolean,
-    UNIQUE (breach_name)
+    UNIQUE (breach_name),
     PRIMARY KEY (hibp_breaches_uid)
 );
 
@@ -240,9 +261,9 @@ CREATE TABLE IF NOT EXISTS public.hibp_exposed_credentials
     organizations_uid uuid NOT NULL,
     root_domain text,
     sub_domain text,
-    modified_date timestamp without time zone,
     breach_name text,
-	breach_id uuid NOT NULL,
+    modified_date timestamp without time zone,
+	  breach_id uuid NOT NULL,
     UNIQUE (email, breach_name),
     PRIMARY KEY (hibp_exposed_credentials_uid)
 );
@@ -255,7 +276,7 @@ CREATE TABLE IF NOT EXISTS public.cybersix_exposed_credentials
     breach_date date,
     breach_id integer,
     breach_name text NOT NULL,
-    create_time timestamp without time zone[],
+    create_time timestamp without time zone,
     description text,
     domain text,
     email text NOT NULL,
@@ -264,20 +285,23 @@ CREATE TABLE IF NOT EXISTS public.cybersix_exposed_credentials
     login_id text,
     name text,
     phone text,
+    UNIQUE (email, breach_id),
     PRIMARY KEY (csg_exposed_credentials_uid)
 );
 
 -- Top CVEs
 CREATE TABLE IF NOT EXISTS public.top_cves
 (
-   top_cves_uid uuid default uuid_generate_v1() NOT NULL,
-    type text,
-    cve text,
-    description text,
+    top_cves_uid uuid default uuid_generate_v1() NOT NULL,
+    cve_id text,
+    dynamic_rating text,
+    nvd_base_score text,
+    date date,
+    summary text,
+    UNIQUE (cve_id, date),
     PRIMARY KEY (top_cves_uid)
 );
 
-
 -- Table Relationships --
 -- One to many relation between Organization and Root Domains
 ALTER TABLE public.root_domains
@@ -291,23 +315,25 @@ ALTER TABLE public.sub_domains
  REFERENCES public.root_domains (root_domain_uid)
  NOT VALID;
 
- -- One to many relation between sub domains and IPs
-ALTER TABLE public.ip_addresses
- ADD FOREIGN KEY (sub_domain_uid)
+ -- many to many relation between sub domains and IPs
+ALTER TABLE public.Sub_domains_Ips
+ADD FOREIGN KEY (sub_domain_uid)
  REFERENCES public.sub_domains (sub_domain_uid)
+ NOT VALID,
+ADD FOREIGN KEY (asset_uid)
+ REFERENCES public.web_assets (asset_uid)
  NOT VALID;
 
-
--- One to many relation between Organization and DNSTwist results
-ALTER TABLE public."DNSTwist"
+-- One to many relation between orgs and web_assets
+ALTER TABLE public.web_assets
  ADD FOREIGN KEY (organizations_uid)
  REFERENCES public.organizations (organizations_uid)
  NOT VALID;
 
--- One to many relation between Domains and DNSTwist results
-ALTER TABLE public."DNSTwist"
- ADD FOREIGN KEY ("discoveredBy")
- REFERENCES public.sub_domains ("sub_domain_uid")
+-- One to many relation between Organization and DNSTwist results
+ALTER TABLE public."dnstwist_domain_masq"
+ ADD FOREIGN KEY (organizations_uid)
+ REFERENCES public.organizations (organizations_uid)
  NOT VALID;
 
 -- One to many relation between Organization and Shodan Assets
@@ -364,11 +390,9 @@ ALTER TABLE public.alerts
     REFERENCES public.organizations (organizations_uid)
     NOT VALID;
 
-
 -- One to Many Relationship for Mentions
 -- Represented in complex SixGill "query": API.
 
-
 -- Views --
 -- HIBP complete breach view
 Create View vw_breach_complete