Fix api and workers network settings (#617)

observatory-platform/observatory/platform/docker/docker-compose.observatory.yml.jinja2

@@ -54,7 +54,6 @@ x-environment: &environment
 
   # Variables
   AIRFLOW_VAR_DATA_PATH: "/opt/observatory/data"
-  AIRFLOW_VAR_DAGS_MODULE_NAMES: {{ config.dags_module_names  }}
   {%- for var in config.make_airflow_variables() %}
   {%- if config.backend.type.value == 'local' %}
   {{ var.env_var_name }}: '{{ var.value }}'
@@ -245,6 +244,8 @@ services:
       - OBSERVATORY_API_PORT=5002
     volumes: *volumes
     restart: always
+    ports:
+      - ${HOST_API_SERVER_PORT}:5002
     networks:
       - {{ config.observatory.docker_network_name }}
     {% if config.backend.type.value == 'local' %} 
@@ -257,8 +258,6 @@ services:
       interval: 30s
       retries: 20
     command: apiserver
-    ports:
-      - ${HOST_API_SERVER_PORT}:5002
     entrypoint: /entrypoint-api.sh
 
 {% if config.backend.type.value == 'local' %} 

observatory-platform/observatory/platform/observatory_config.py

@@ -712,7 +712,7 @@ def __init__(
             self.airflow_variables = []
 
         self.airflow_connections = airflow_connections
-        if airflow_variables is None:
+        if airflow_connections is None:
             self.airflow_connections = []
 
         self.workflows_projects = workflows_projects
@@ -817,6 +817,13 @@ def make_airflow_variables(self) -> List[AirflowVariable]:
             if self.terraform.organization is not None:
                 variables.append(AirflowVariable(AirflowVars.TERRAFORM_ORGANIZATION, self.terraform.organization))
 
+        # Add dags module names
+        variables.append(
+            AirflowVariable(
+                AirflowVars.DAGS_MODULE_NAMES, json.dumps([proj.dags_module for proj in self.workflows_projects])
+            )
+        )
+
         # Add user defined variables to list
         variables += self.airflow_variables
 

observatory-platform/observatory/platform/terraform/main.tf

@@ -333,25 +333,26 @@ data "google_compute_subnetwork" "observatory_subnetwork" {
   depends_on = [google_compute_network.observatory_network] # necessary to force reading of data
 }
 
-resource "google_compute_firewall" "allow_internal" {
-  name = "allow-internal"
-  description = "Allow internal connections"
+resource "google_compute_firewall" "allow_internal_airflow" {
+  name = "allow-internal-airflow"
+  description = "Allow internal Airflow connections"
   network = google_compute_network.observatory_network.name
   source_ranges = ["10.128.0.0/9"]
+  target_tags   = ["allow-internal-airflow"]
+
   allow {
     protocol = "tcp"
-    ports = ["6379"] # Open redis port to the internal network which is used by Airflow
+    ports = ["5002", "6379", "8793"] # Open apiserver, redis and Airflow worker ports to the internal network
   }
-  source_tags = [google_compute_network.observatory_network.name]
   priority = 65534
 }
 
 resource "google_compute_firewall" "allow_ssh" {
   name = "allow-ssh"
   description = "Allow SSH from anywhere"
   network = google_compute_network.observatory_network.name
-  target_tags   = ["allow-ssh"]
   source_ranges = ["0.0.0.0/0"]
+  target_tags   = ["allow-ssh"]
 
   allow {
     protocol = "tcp"
@@ -532,7 +533,7 @@ locals {
   metadata_variables = {
     project_id = var.google_cloud.project_id
     postgres_hostname = google_sql_database_instance.observatory_db_instance.private_ip_address
-    redis_hostname = module.airflow_main_vm.private_ip_address
+    redis_hostname = local.main_vm_name # this becomes the hostname of the main vm
     data_location = var.google_cloud.data_location
     download_bucket = google_storage_bucket.observatory_download_bucket.name
     transform_bucket = google_storage_bucket.observatory_transform_bucket.name
@@ -542,10 +543,8 @@ locals {
     airflow_variables = local.airflow_variables
   }
 
-  worker_vm_static_external_ip_address = try(google_compute_address.airflow_worker_vm_static_external_ip[0].address,
-  null)
+  worker_vm_static_external_ip_address = try(google_compute_address.airflow_worker_vm_static_external_ip[0].address, null)
   main_vm_static_external_ip_address = try(google_compute_address.airflow_main_vm_static_external_ip[0].address, null)
-
 }
 
 ########################################################################################################################

observatory-platform/observatory/platform/terraform/startup.tpl.jinja2

@@ -37,7 +37,7 @@ export AIRFLOW_UI_USER_PASSWORD="sm://${project_id}/airflow_ui_user_password"
 export AIRFLOW_UI_USER_EMAIL="sm://${project_id}/airflow_ui_user_email"
 {% set docker_containers="redis flower webserver scheduler worker_local airflow_init apiserver"%}
 {%- else %}
-{% set docker_containers="worker_remote apiserver"%}
+{% set docker_containers="worker_remote"%}
 {%- endif %}
 
 # Export environment variables for all Airflow variables

observatory-platform/observatory/platform/terraform/vm/main.tf

@@ -9,7 +9,7 @@ resource "google_compute_address" "vm_private_ip" {
 resource "google_compute_instance" "vm_instance" {
   name = var.name
   machine_type = var.machine_type
-  tags = ["allow-ssh"]
+  tags = ["allow-ssh", "allow-internal-airflow"]
   allow_stopping_for_update = true
 
   boot_disk {