Merge branch 'dev' into servicemanagement-getconfig

.github/PULL_REQUEST_TEMPLATE.md

@@ -7,5 +7,6 @@ Here's a handy checklist to ensure your PR goes smoothly.
 - [ ] My PR at a minimum doesn't decrease unit-test coverage (if applicable).
 - [ ] My PR has been functionally tested.
 - [ ] All of the [tests](https://forsetisecurity.org/docs/latest/develop/dev/testing.html) pass
+- [ ] I have submitted a corresponding PR in the [Forseti Terraform module](https://github.com/forseti-security/terraform-google-forseti) (if applicable).
 
 These guidelines and more can be found in our [contributing guidelines](https://github.com/forseti-security/forseti-security/blob/dev/.github/CONTRIBUTING.md).

.github/stale.yml

@@ -0,0 +1,45 @@
+# Forseti-Security repo probot-stale configuration
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
+exemptLabels:
+  - security
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: true
+
+# Label to use when marking as stale
+staleLabel: wontfix
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 5
+
+# Configuration settings specific to pull requests
+# Details: pull requests older than 30 days will become stale, after 37 days of no activity the issue is closed.
+# Stale notice message will be posted to the pull request at 30 days.
+pulls:
+    daysUntilStale: 30
+    daysUntilClose: 7
+    markComment: >
+    This pull request has been automatically marked as stale because it has not had
+    recent activity. It will be closed if no further activity occurs. Thank you
+    for your contributions.
+
+# Configuration settings specific to issues
+# Details: Issues older than 5 months will become stale at 152 days. After 182 days of no activity the issue is closed.
+# Stale notice message will be posted to the issue at 152 days.
+issues:
+    daysUntilStale: 152
+    daysUntilClose: 30
+    markComment: >
+    This issue has been automatically marked as stale because it has not had
+    recent activity. It will be closed if no further activity occurs. Thank you
+    for your contributions.

Dockerfile

@@ -13,23 +13,20 @@
 # limitations under the License.
 
 ##### BEGIN BASE IMAGE #####
-FROM python:3.6-slim as base
+FROM python:3.6.9-slim-buster as base
+
+ARG UID=1000
+ARG GID=1000
 
 ENV HOME=/home/forseti \
     WORK_DIR=/home/forseti/forseti-security \
     PATH=/home/forseti/.local/bin:$PATH
 
-RUN groupadd -g 1000 forseti && \
-    useradd -d ${HOME} -u 1000 -g forseti forseti && \
+RUN groupadd -g $GID forseti && \
+    useradd -d ${HOME} -u $UID -g forseti forseti && \
     mkdir -p ${HOME}/forseti-security && \
     chown -R forseti:forseti ${HOME}
 
-# Install host dependencies.
-RUN apt-get update  && \
-    apt-get install --no-install-recommends -y libmariadbclient18 && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
 WORKDIR ${WORK_DIR}
 
 USER forseti

configs/server/forseti_conf_server.yaml.in

@@ -16,6 +16,15 @@ inventory:
     # project number instead of project id, e.g. "projects/<project number>".
     root_resource_id: {ROOT_RESOURCE_ID}
 
+    # Resources to be excluded during the inventory process.
+    # Only organizations/<ORG_NUMBER>, folders/<FOLDER_NUMBER>,
+    # projects/<PROJECT_ID> or projects/<PROJECT_NUMBER> are accepted.
+    # The child resources under the excluded resources will also be excluded.
+    #
+    # Example:
+    # excluded_resources: ['folders/1234', 'projects/my-project-123', 'projects/4321']
+    excluded_resources: []
+
     # Composite root resources: combines multiple resource roots into a single
     # inventory, for use across all Forseti modules. Can contain one or more
     # resources from the GCP Resource Hierarchy in any combination.
@@ -131,19 +140,24 @@ inventory:
         #    - appengine.googleapis.com/Version
         #    - bigquery.googleapis.com/Dataset
         #    - bigquery.googleapis.com/Table
+        #    - bigtableadmin.googleapis.com/Cluster
+        #    - bigtableadmin.googleapis.com/Instance
+        #    - bigtableadmin.googleapis.com/Table
         #    - cloudbilling.googleapis.com/BillingAccount
         #    - cloudkms.googleapis.com/CryptoKey
         #    - cloudkms.googleapis.com/CryptoKeyVersion
         #    - cloudkms.googleapis.com/KeyRing
         #    - cloudresourcemanager.googleapis.com/Folder
         #    - cloudresourcemanager.googleapis.com/Organization
         #    - cloudresourcemanager.googleapis.com/Project
+        #    - compute.googleapis.com/Address
         #    - compute.googleapis.com/Autoscaler
         #    - compute.googleapis.com/BackendBucket
         #    - compute.googleapis.com/BackendService
         #    - compute.googleapis.com/Disk
         #    - compute.googleapis.com/Firewall
         #    - compute.googleapis.com/ForwardingRule
+        #    - compute.googleapis.com/GlobalAddress
         #    - compute.googleapis.com/GlobalForwardingRule
         #    - compute.googleapis.com/HealthCheck
         #    - compute.googleapis.com/HttpHealthCheck
@@ -153,12 +167,15 @@ inventory:
         #    - compute.googleapis.com/InstanceGroup
         #    - compute.googleapis.com/InstanceGroupManager
         #    - compute.googleapis.com/InstanceTemplate
+        #    - compute.googleapis.com/Interconnect
+        #    - compute.googleapis.com/InterconnectAttachment
         #    - compute.googleapis.com/License
         #    - compute.googleapis.com/Network
         #    - compute.googleapis.com/Project
         #    - compute.googleapis.com/RegionBackendService
         #    - compute.googleapis.com/Route
         #    - compute.googleapis.com/Router
+        #    - compute.googleapis.com/SecurityPolicy
         #    - compute.googleapis.com/Snapshot
         #    - compute.googleapis.com/SslCertificate
         #    - compute.googleapis.com/Subnetwork

configs/server/forseti_conf_server.yaml.sample

@@ -22,6 +22,7 @@ inventory:
     # service account before they can be included in the inventory.
     #
     # Forseti Explain is not supported with a composite root at this time.
+    #
     # Resources can exist in multiple organizations.
     #
     #composite_root_resources:
@@ -127,19 +128,24 @@ inventory:
         #    - appengine.googleapis.com/Version
         #    - bigquery.googleapis.com/Dataset
         #    - bigquery.googleapis.com/Table
+        #    - bigtableadmin.googleapis.com/Cluster
+        #    - bigtableadmin.googleapis.com/Instance
+        #    - bigtableadmin.googleapis.com/Table
         #    - cloudbilling.googleapis.com/BillingAccount
         #    - cloudkms.googleapis.com/CryptoKey
         #    - cloudkms.googleapis.com/CryptoKeyVersion
         #    - cloudkms.googleapis.com/KeyRing
         #    - cloudresourcemanager.googleapis.com/Folder
         #    - cloudresourcemanager.googleapis.com/Organization
         #    - cloudresourcemanager.googleapis.com/Project
+        #    - compute.googleapis.com/Address
         #    - compute.googleapis.com/Autoscaler
         #    - compute.googleapis.com/BackendBucket
         #    - compute.googleapis.com/BackendService
         #    - compute.googleapis.com/Disk
         #    - compute.googleapis.com/Firewall
         #    - compute.googleapis.com/ForwardingRule
+        #    - compute.googleapis.com/GlobalAddress
         #    - compute.googleapis.com/GlobalForwardingRule
         #    - compute.googleapis.com/HealthCheck
         #    - compute.googleapis.com/HttpHealthCheck
@@ -149,12 +155,15 @@ inventory:
         #    - compute.googleapis.com/InstanceGroup
         #    - compute.googleapis.com/InstanceGroupManager
         #    - compute.googleapis.com/InstanceTemplate
+        #    - compute.googleapis.com/Interconnect
+        #    - compute.googleapis.com/InterconnectAttachment
         #    - compute.googleapis.com/License
         #    - compute.googleapis.com/Network
         #    - compute.googleapis.com/Project
         #    - compute.googleapis.com/RegionBackendService
         #    - compute.googleapis.com/Route
         #    - compute.googleapis.com/Router
+        #    - compute.googleapis.com/SecurityPolicy
         #    - compute.googleapis.com/Snapshot
         #    - compute.googleapis.com/SslCertificate
         #    - compute.googleapis.com/Subnetwork
@@ -211,7 +220,7 @@ scanner:
 
     # Enable the scanners as default to true when integrated for Forseti 2.0.
 
-     scanners:
+    scanners:
         - name: audit_logging
           enabled: false
         - name: bigquery
@@ -274,7 +283,7 @@ notifier:
                 data_format: csv
                 # gcs_path should begin with "gs://"
                 gcs_path: ''
-            # Slack webhook pipeline
+            # Slack webhook pipeline.
             # Create an incoming webhook in your organization's Slack setting, located at:
             # https://[your_org].slack.com/apps/manage/custom-integrations
             # Add the provided URL in the configuration below in `webhook_url`.
@@ -298,7 +307,7 @@ notifier:
     violation:
       cscc:
         enabled: true
-        # Cloud SCC API uses a new source_id. It is unique per
+        # Cloud SCC uses a source_id. It is unique per
         # organization and must be generated via a self-registration process.
         # The format is: organizations/ORG_ID/sources/SOURCE_ID
         source_id:
@@ -312,4 +321,3 @@ notifier:
         gcs_path: gs://MY_BUCKET/inventory_summary
       email_summary:
         enabled: true
-

contrib/incident-response/infrastructure/main.tf

@@ -37,9 +37,27 @@ provider "google" {
 # Timesketch #
 #------------#
 module "timesketch" {
-  source                      = "modules/timesketch"
+  source                      = "./modules/timesketch"
   gcp_project                 = "${var.gcp_project}"
   gcp_region                  = "${var.gcp_region}"
   gcp_zone                    = "${var.gcp_zone}"
   gcp_ubuntu_1804_image       = "${var.gcp_ubuntu_1804_image}"
+  infrastructure_id           = "${coalesce(var.infrastructure_id, random_id.infrastructure-random-id.hex)}"
+}
+
+#------------#
+# Turbinia   #
+#------------#
+module "turbinia" {
+  source                      = "./modules/turbinia"
+  gcp_project                 = "${var.gcp_project}"
+  gcp_region                  = "${var.gcp_region}"
+  gcp_zone                    = "${var.gcp_zone}"
+  gcp_ubuntu_1804_image       = "${var.gcp_ubuntu_1804_image}"
+  infrastructure_id           = "${coalesce(var.infrastructure_id, random_id.infrastructure-random-id.hex)}"
+}
+
+# Random ID for creating unique resource names.
+resource "random_id" "infrastructure-random-id" {
+  byte_length = 8
 }

contrib/incident-response/infrastructure/modules/timesketch/main.tf

@@ -14,18 +14,13 @@
  * limitations under the License.
  */
 
-# Random ID for creating unique resource names.
-resource "random_id" "infrastructure-random-id" {
-  byte_length = 8
-}
-
 #-----------------------#
 # Elasticsearch cluster #
 #-----------------------#
 data "template_file" "elasticsearch-startup-script" {
   template = "${file("${path.module}/templates/scripts/install-elasticsearch.sh.tpl")}"
 
-  vars {
+  vars = {
     cluster_name  = "${var.elasticsearch_cluster_name}"
     project       = "${var.gcp_project}"
     zone          = "${var.gcp_zone}"
@@ -34,7 +29,7 @@ data "template_file" "elasticsearch-startup-script" {
 
 resource "google_compute_instance" "elasticsearch" {
   count        = "${var.elasticsearch_node_count}"
-  name         = "elasticsearch-node-${count.index}"
+  name         = "elasticsearch-node-${var.infrastructure_id}-${count.index}"
   machine_type = "${var.elasticsearch_machine_type}"
   zone         = "${var.gcp_zone}"
 
@@ -52,7 +47,7 @@ resource "google_compute_instance" "elasticsearch" {
   # Assign a generated public IP address. Needed for SSH access.
   network_interface {
     network       = "default"
-    access_config = {}
+    access_config {}
   }
 
   # Tag for service enumeration.
@@ -94,7 +89,7 @@ resource "random_string" "timesketch-db-password" {
 }
 
 resource "google_sql_database_instance" "timesketch-db-instance" {
-  name              = "timesketch-db-instance-${random_id.infrastructure-random-id.hex}"
+  name              = "timesketch-db-instance-${var.infrastructure_id}"
   region            = "${var.gcp_region}"
   database_version  = "POSTGRES_9_6"
   depends_on        = ["google_project_service.sql-admin-service-api"]
@@ -105,7 +100,7 @@ resource "google_sql_database_instance" "timesketch-db-instance" {
     ip_configuration {
       ipv4_enabled = true
       require_ssl  = false
-      authorized_networks = {
+      authorized_networks {
         name  = "timesketch-server"
         value = "${google_compute_address.timesketch-server-address.address}"
       }
@@ -136,7 +131,7 @@ resource "google_project_service" "redis-service-api" {
 
 # Redis is used as the task queue backend for importing data into Timesketch.
 resource "google_redis_instance" "redis" {
-  name           = "redis"
+  name           = "redis-${var.infrastructure_id}"
   memory_size_gb = 1
   depends_on     = ["google_project_service.redis-service-api"]
 }
@@ -146,7 +141,7 @@ resource "google_redis_instance" "redis" {
 #-------------------#
 data "template_file" "timesketch-server-startup-script" {
   template = "${file("${path.module}/templates/scripts/install-timesketch.sh.tpl")}"
-  vars {
+  vars = {
     timesketch_admin_username = "${var.timesketch_admin_username}"
     timesketch_admin_password = "${random_string.timesketch-admin-password.result}"
     elasticsearch_node        = "${google_compute_instance.elasticsearch.*.name[0]}"
@@ -180,7 +175,7 @@ resource "google_compute_firewall" "allow-external-timesketch-server" {
 }
 
 resource "google_compute_instance" "timesketch-server" {
-  name          = "timesketch-server"
+  name          = "timesketch-server-${var.infrastructure_id}"
   machine_type  = "${var.timesketch_machine_type}"
   zone          = "${var.gcp_zone}"
 
@@ -204,6 +199,10 @@ resource "google_compute_instance" "timesketch-server" {
     }
   }
 
+  service_account {
+    scopes = ["storage-ro", "pubsub"]
+  }
+
   # Allow HTTPS traffic
   tags = ["timesketch-https-server", "https-server"]
 

contrib/incident-response/infrastructure/modules/timesketch/variables.tf

@@ -33,6 +33,10 @@ variable "gcp_ubuntu_1804_image" {
   default     = "ubuntu-os-cloud/ubuntu-1804-lts"
 }
 
+variable "infrastructure_id" {
+  description = "Unique indentifier for the deployment"
+}
+
 variable "timesketch_machine_type" {
   description = "Machine type for Timesketch server"
   default     = "n1-standard-2"

contrib/incident-response/infrastructure/modules/turbinia/README.md

@@ -0,0 +1 @@
+# Turbinia deployment