Merge branch 'develop' into ss-wip-cmd-line-update

src/pe_reports/data/data_schema.sql

@@ -6,13 +6,6 @@
 -- Includes Domain Masquerading, Credentals Exposed, Inffered Vulns, and Dark Web data
 
 
-
-
-# TODO commit these changes
-
-
-
-
 BEGIN;
 -- Enable uuid extension in Postgres
 CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
@@ -23,33 +16,34 @@ CREATE TABLE IF NOT EXISTS public.organizations
     organizations_uid uuid default uuid_generate_v1() NOT NULL,
     name text NOT NULL,
     root_domains text[],
+    cyhy_db_name text,
     PRIMARY KEY (organizations_uid)
 );
 
 -- Organization's Domains Table
 CREATE TABLE IF NOT EXISTS public.domains
 (
     domain_uid uuid default uuid_generate_v1() NOT NULL,
-    organization_id text NOT NULL,
+    organizations_uid uuid NOT NULL,
     root_domain text NOT NULL,
     ip_address text,
     PRIMARY KEY (domain_uid)
 );
 
 -- Organization's Aliases Table
-CREATE TABLE public.alias
+CREATE TABLE IF NOT EXISTS public.alias
 (
     alias_uid uuid default uuid_generate_v1() NOT NULL,
-    organization_id text NOT NULL,
+    organizations_uid uuid NOT NULL,
     alias text NOT NULL,
     PRIMARY KEY (alias_uid)
 );
 
 -- Organization's Evecutives Table
-CREATE TABLE public.executives
+CREATE TABLE IF NOT EXISTS public.executives
 (
     executives_uid uuid default uuid_generate_v1() NOT NULL,
-    organization_id text NOT NULL,
+    organizations_uid uuid NOT NULL,
     executives text NOT NULL,
     PRIMARY KEY (executives_uid)
 );
@@ -60,7 +54,7 @@ CREATE TABLE public.executives
 CREATE TABLE IF NOT EXISTS public."DNSTwist"
 (
     dnstwist_uid uuid default uuid_generate_v1() NOT NULL,
-    "discoveredBy" text NOT NULL,
+    "discoveredBy" uuid NOT NULL,
     "domain-name" text,
     "dns-a" text,
     "dns-aaaa" text,
@@ -69,12 +63,12 @@ CREATE TABLE IF NOT EXISTS public."DNSTwist"
     fuzzer text,
     "date-observed" text,
     "ssdeep-score" text,
-    organization_id text NOT NULL,
+    organizations_uid uuid NOT NULL,
     PRIMARY KEY (dnstwist_uid)
 );
 
 -- Dark Web Alerts Table
-CREATE TABLE public.alerts
+CREATE TABLE IF NOT EXISTS public.alerts
 (
     alerts_uid uuid default uuid_generate_v1() NOT NULL,
     alert_name text,
@@ -88,12 +82,12 @@ CREATE TABLE public.alerts
     threats text,
     title text,
     user_id text,
-    organization_id text NOT NULL,
+    organizations_uid uuid NOT NULL,
     PRIMARY KEY (alerts_uid)
 );
 
 -- Dark Web Mentions Table
-CREATE TABLE public.mentions
+CREATE TABLE IF NOT EXISTS public.mentions
 (
     mentions_uid uuid default uuid_generate_v1() NOT NULL,
     category text,
@@ -112,16 +106,98 @@ CREATE TABLE public.mentions
     comments_count text,
     sub_category text,
     query text,
-    organization_id text NOT NULL,
+    organizations_uid uuid NOT NULL,
     PRIMARY KEY (mentions_uid)
 );
 
+-- Shodan Insecure protocols and unverified vulnerabilities table
+CREATE TABLE IF NOT EXISTS public.shodan_insecure_protocols_unverified_vulns
+(
+    insecure_product_uid uuid default uuid_generate_v1() NOT NULL,
+    organizations_uid uuid NOT NULL,
+    organization text,
+    ip text,
+    port integer,
+    protocol text,
+    type text,
+    name text,
+    potential_vulns text[],
+    mitigation text,
+    timestamp timestamp,
+    product text,
+    server text,
+    tags text[],
+    domains text[],
+    hostnames text[],
+    isn text,
+    asn integer,
+    UNIQUE (organizations_uid, ip, port, protocol, timestamp),
+    PRIMARY KEY (insecure_product_uid)
+);
+
+--Shodan Veriried Vulnerabilities table
+CREATE TABLE IF NOT EXISTS public.shodan_verified_vulns
+(
+    verified_vuln_uid uuid default uuid_generate_v1() NOT NULL,
+    organizations_uid uuid NOT NULL,
+    organization text,
+    ip text,
+    port text,
+    protocol text,
+    timestamp timestamp,
+    cve text,
+    severity text,
+    cvss numeric,
+    summary text,
+    product text,
+    attack_vector text,
+    av_description text,
+    attack_complexity text,
+    ac_description text,
+    confidentiality_impact text,
+    ci_description text,
+    integrity_impact text,
+    ii_description text,
+    availability_impact text,
+    ai_description text,
+    tags text[],
+    domains text[],
+    hostnames text[],
+    isn text,
+    asn integer,
+    UNIQUE (organizations_uid, ip, port, protocol, timestamp),
+    PRIMARY KEY (verified_vuln_uid)
+);
+
+--Shodan Assets and IPs table
+CREATE TABLE IF NOT EXISTS public.shodan_assets
+(
+    shodan_asset_uid uuid default uuid_generate_v1() NOT NULL,
+    organizations_uid uuid NOT NULL,
+    organization text,
+    ip text,
+    port integer,
+    protocol text,
+    timestamp timestamp,
+    product text,
+    server text,
+    tags text[],
+    domains text[],
+    hostnames text[],
+    isn text,
+    asn integer,
+    UNIQUE (organizations_uid, ip, port, protocol, timestamp),
+    PRIMARY KEY (shodan_asset_uid)
+);
+
 -- HIBP breaches Table
 CREATE TABLE IF NOT EXISTS public.hibp_breaches
 (
     hibp_breaches_uid uuid default uuid_generate_v1() NOT NULL,
+    breach_id uuid NOT NULL,
     breach_name text NOT NULL,
     description text,
+    exposed_cred_count bigint,
     breach_date date,
     added_date timestamp without time zone,
     modified_date timestamp without time zone,
@@ -140,10 +216,11 @@ CREATE TABLE IF NOT EXISTS public.hibp_exposed_credentials
 (
     hibp_exposed_credentials_uid uuid default uuid_generate_v1() NOT NULL,
     email text NOT NULL,
-    organization text,
+    organizations_uid uuid NOT NULL,
     root_domain text,
     sub_domain text,
     breach_name text,
+	breach_id uuid NOT NULL,
     UNIQUE (email, breach_name),
     PRIMARY KEY (hibp_exposed_credentials_uid)
 );
@@ -152,8 +229,9 @@ CREATE TABLE IF NOT EXISTS public.hibp_exposed_credentials
 CREATE TABLE IF NOT EXISTS public.cybersix_exposed_credentials
 (
     csg_exposed_credentials_uid uuid default uuid_generate_v1() NOT NULL,
+    organizations_uid uuid NOT NULL,
     breach_date date,
-    "breach_id " integer,
+    breach_id integer,
     breach_name text NOT NULL,
     create_time timestamp without time zone[],
     description text,
@@ -168,7 +246,7 @@ CREATE TABLE IF NOT EXISTS public.cybersix_exposed_credentials
 );
 
 -- Top CVEs
-CREATE TABLE public.top_cves
+CREATE TABLE IF NOT EXISTS public.top_cves
 (
    top_cves_uid uuid default uuid_generate_v1() NOT NULL,
     type text,
@@ -187,8 +265,8 @@ ALTER TABLE public.domains
 
 -- One to many relation between Organization and DNSTwist results
 ALTER TABLE public."DNSTwist"
- ADD FOREIGN KEY (organization_uid)
- REFERENCES public.organizations (organization_uid)
+ ADD FOREIGN KEY (organizations_uid)
+ REFERENCES public.organizations (organizations_uid)
  NOT VALID;
 
 -- One to many relation between Domains and DNSTwist results
@@ -197,12 +275,42 @@ ALTER TABLE public."DNSTwist"
  REFERENCES public.domains ("domain_uid")
  NOT VALID;
 
--- One to many relation between Organization and Domains
+-- One to many relation between Organization and Shodan Assets
+ALTER TABLE public.shodan_assets
+    ADD FOREIGN KEY (organizations_uid)
+    REFERENCES public.organizations (organizations_uid)
+    NOT VALID;
+
+-- One to many relation between Organization and Shodan Unverified Vulns
+ALTER TABLE public.shodan_insecure_protocols_unverified_vulns
+    ADD FOREIGN KEY (organizations_uid)
+    REFERENCES public.organizations (organizations_uid)
+    NOT VALID;
+
+-- One to many relation between Organization and Shodan Verified Vulns
+ALTER TABLE public.shodan_verified_vulns
+    ADD FOREIGN KEY (organizations_uid)
+    REFERENCES public.organizations (organizations_uid)
+    NOT VALID;
+
+-- One to many relation between Breaches and HIBP Exposed Credentials
 ALTER TABLE public.hibp_exposed_credentials
     ADD FOREIGN KEY (breach_id)
     REFERENCES public.hibp_breaches (hibp_breaches_uid)
     NOT VALID;
 
+-- One to many relation between Organization and HIBP Exposed Credentials
+ALTER TABLE public.hibp_exposed_credentials
+    ADD FOREIGN KEY (organizations_uid)
+    REFERENCES public.organizations (organizations_uid)
+    NOT VALID;
+
+-- One to many relation between Organization and SixGill Exposed Credentials
+ALTER TABLE public.cybersix_exposed_credentials
+    ADD FOREIGN KEY (organizations_uid)
+    REFERENCES public.organizations (organizations_uid)
+    NOT VALID;
+
 -- One to many relation between Organization and Aliases
 ALTER TABLE public.alias
     ADD FOREIGN KEY (organizations_uid)
@@ -216,11 +324,12 @@ ALTER TABLE public.executives
     NOT VALID;
 
 -- One to many relation between Organization and SixGill Alert API
-ALTER TABLE public.organizations
+ALTER TABLE public.alerts
     ADD FOREIGN KEY (organizations_uid)
-    REFERENCES public.alerts (organizations_uid)
+    REFERENCES public.organizations (organizations_uid)
     NOT VALID;
 
+
 -- One to Many Relationship for Mentions
 -- Represented in complex SixGill "query": API.
 
@@ -229,14 +338,13 @@ ALTER TABLE public.organizations
 -- HIBP complete breach view
 Create View vw_breach_complete
 AS
-SELECT creds.hibp_exposed_credentials_uid,creds.email, creds.breach_name, creds.organization, creds.root_domain, creds.sub_domain,
+SELECT creds.hibp_exposed_credentials_uid,creds.email, creds.breach_name, creds.organizations_uid, creds.root_domain, creds.sub_domain,
     b.description, b.breach_date, b.added_date, b.modified_date,  b.data_classes,
     b.password_included, b.is_verified, b.is_fabricated, b.is_sensitive, b.is_retired, b.is_spam_list
 
     FROM hibp_exposed_credentials as creds
 
     JOIN hibp_breaches as b
-    ON creds.breach_name = b.breach_name;
-
+    ON creds.breach_id = b.breach_id;
 
 END;