Merge pull request #97 from vbalalian/refactor-1

refactor: group ELT services in one folder
vbalalian · Mar 3, 2024 · 28e61f8 · 28e61f8
2 parents bf7bc1b + da8ef10
commit 28e61f8
Show file tree

Hide file tree

Showing 30 changed files with 65 additions and 11 deletions.
diff --git a/compose.test.yaml b/compose.test.yaml
@@ -12,6 +12,7 @@ services:
         condition: service_started
       db:
         condition: service_healthy
+
   web_scraper:
     build: ./web_scraping
     environment:
@@ -23,8 +24,9 @@ services:
     depends_on:
       db:
         condition: service_healthy
+
   db:
-    image: postgres:latest
+    image: postgres
     restart: unless-stopped
     volumes:
       - db-data:/var/lib/postgresql/data
@@ -39,8 +41,9 @@ services:
       interval: 10s
       timeout: 5s
       retries: 5
+
   minio:
-    image: minio/minio:latest
+    image: minio/minio
     volumes:
       - minio-data:/data
     environment:
@@ -50,8 +53,9 @@ services:
     ports:
       - 9000:9000
       - 9090:9090
+
   minio-setup:
-    image: minio/mc:latest
+    image: minio/mc
     depends_on:
       - minio
     entrypoint: >
@@ -64,14 +68,16 @@ services:
       mc admin user add myminio ${MINIO_NEW_USER} ${MINIO_NEW_USER_PASSWORD};
       mc admin policy attach myminio readwrite --user=${MINIO_NEW_USER};
       "
+
   custom-airbyte-connector:
     environment:
-      - HOST=${HOST}
+      - AIRBYTE_HOST=http://${HOST}
     build:
-      context: ./custom-airbyte-connector
+      context: ./extract-load-transform/custom-airbyte-connector
       dockerfile: Dockerfile
     image: airbyte/source-roman-coins-api:latest
     pull_policy: build
+
   airbyte-configurator:
     depends_on:
       - api
@@ -80,14 +86,62 @@ services:
       - MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME}
       - MINIO_NEW_USER=${MINIO_NEW_USER}
       - MINIO_NEW_USER_PASSWORD=${MINIO_NEW_USER_PASSWORD}
-      - AIRBYTE_HOST=${HOST}
+      - AIRBYTE_HOST=http://${HOST}
       - AIRBYTE_USERNAME=${AIRBYTE_USERNAME}
       - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD}
       - CONNECTOR_START_DATE=${CONNECTOR_START_DATE}
     build: 
-      context: ./airbyte-api-minio-connection
+      context: ./extract-load-transform/airbyte-api-minio-connection
       dockerfile: Dockerfile
     pull_policy: build
+
+  dagster-webserver:
+    depends_on:
+      - db
+      - airbyte-configurator
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=${POSTGRES_DB}
+      - DB_HOST=db
+      - AIRBYTE_USERNAME=${AIRBYTE_USERNAME}
+      - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD}
+      - HOST=${HOST}
+    build: 
+      context: ./extract-load-transform/orchestration
+      dockerfile: Dockerfile
+    entrypoint:
+      - dagster-webserver
+      - -h
+      - "0.0.0.0"
+      - -p
+      - "3000"
+      - -w
+      - workspace.yaml
+    expose:
+      - 3000
+    ports:
+      - 3000:3000
+
+  dagster-daemon:
+    depends_on:
+      - db
+      - airbyte-configurator
+    environment:
+      - POSTGRES_USER=${POSTGRES_USER}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - POSTGRES_DB=${POSTGRES_DB}
+      - DB_HOST=db
+      - AIRBYTE_USERNAME=${AIRBYTE_USERNAME}
+      - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD}
+      - HOST=${HOST}
+    build: 
+      context: ./extract-load-transform/orchestration
+      dockerfile: Dockerfile
+    entrypoint:
+      - dagster-daemon
+      - run
+    restart: on-failure
   test_db:
     image: postgres:latest
     restart: unless-stopped

diff --git a/compose.yaml b/compose.yaml
@@ -73,7 +73,7 @@ services:
     environment:
       - AIRBYTE_HOST=http://${HOST}
     build:
-      context: ./custom-airbyte-connector
+      context: ./extract-load-transform/custom-airbyte-connector
       dockerfile: Dockerfile
     image: airbyte/source-roman-coins-api:latest
     pull_policy: build
@@ -91,7 +91,7 @@ services:
       - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD}
       - CONNECTOR_START_DATE=${CONNECTOR_START_DATE}
     build: 
-      context: ./airbyte-api-minio-connection
+      context: ./extract-load-transform/airbyte-api-minio-connection
       dockerfile: Dockerfile
     pull_policy: build
 
@@ -108,7 +108,7 @@ services:
       - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD}
       - HOST=${HOST}
     build: 
-      context: ./orchestration
+      context: ./extract-load-transform/orchestration
       dockerfile: Dockerfile
     entrypoint:
       - dagster-webserver
@@ -136,7 +136,7 @@ services:
       - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD}
       - HOST=${HOST}
     build: 
-      context: ./orchestration
+      context: ./extract-load-transform/orchestration
       dockerfile: Dockerfile
     entrypoint:
       - dagster-daemon

diff --git a/airbyte-api-minio-connection/.dockerignore → ...irbyte-api-minio-connection/.dockerignore b/airbyte-api-minio-connection/.dockerignore → ...irbyte-api-minio-connection/.dockerignore
diff --git a/airbyte-api-minio-connection/Dockerfile → ...m/airbyte-api-minio-connection/Dockerfile b/airbyte-api-minio-connection/Dockerfile → ...m/airbyte-api-minio-connection/Dockerfile
diff --git a/...o-connection/airbyte_connection_config.py → ...o-connection/airbyte_connection_config.py b/...o-connection/airbyte_connection_config.py → ...o-connection/airbyte_connection_config.py
diff --git a/...yte-api-minio-connection/requirements.txt → ...yte-api-minio-connection/requirements.txt b/...yte-api-minio-connection/requirements.txt → ...yte-api-minio-connection/requirements.txt
diff --git a/custom-airbyte-connector/Dockerfile → ...sform/custom-airbyte-connector/Dockerfile b/custom-airbyte-connector/Dockerfile → ...sform/custom-airbyte-connector/Dockerfile
diff --git a/...ild/lib/source_roman_coin_api/__init__.py → ...ild/lib/source_roman_coin_api/__init__.py b/...ild/lib/source_roman_coin_api/__init__.py → ...ild/lib/source_roman_coin_api/__init__.py
diff --git a/...in_api/schemas/roman_coin_api_stream.json → ...in_api/schemas/roman_coin_api_stream.json b/...in_api/schemas/roman_coin_api_stream.json → ...in_api/schemas/roman_coin_api_stream.json
diff --git a/...build/lib/source_roman_coin_api/source.py → ...build/lib/source_roman_coin_api/source.py b/...build/lib/source_roman_coin_api/source.py → ...build/lib/source_roman_coin_api/source.py
diff --git a/...build/lib/source_roman_coin_api/spec.yaml → ...build/lib/source_roman_coin_api/spec.yaml b/...build/lib/source_roman_coin_api/spec.yaml → ...build/lib/source_roman_coin_api/spec.yaml
diff --git a/custom-airbyte-connector/main.py → ...ransform/custom-airbyte-connector/main.py b/custom-airbyte-connector/main.py → ...ransform/custom-airbyte-connector/main.py
diff --git a/custom-airbyte-connector/requirements.txt → ...custom-airbyte-connector/requirements.txt b/custom-airbyte-connector/requirements.txt → ...custom-airbyte-connector/requirements.txt
diff --git a/...ctor/sample_files/configured_catalog.json → ...ctor/sample_files/configured_catalog.json b/...ctor/sample_files/configured_catalog.json → ...ctor/sample_files/configured_catalog.json
diff --git a/custom-airbyte-connector/secrets/config.json → ...tom-airbyte-connector/secrets/config.json b/custom-airbyte-connector/secrets/config.json → ...tom-airbyte-connector/secrets/config.json
diff --git a/custom-airbyte-connector/setup.py → ...ansform/custom-airbyte-connector/setup.py b/custom-airbyte-connector/setup.py → ...ansform/custom-airbyte-connector/setup.py
diff --git a/...nnector/source_roman_coin_api/__init__.py → ...nnector/source_roman_coin_api/__init__.py b/...nnector/source_roman_coin_api/__init__.py → ...nnector/source_roman_coin_api/__init__.py
diff --git a/...in_api/schemas/roman_coin_api_stream.json → ...in_api/schemas/roman_coin_api_stream.json b/...in_api/schemas/roman_coin_api_stream.json → ...in_api/schemas/roman_coin_api_stream.json
diff --git a/...connector/source_roman_coin_api/source.py → ...connector/source_roman_coin_api/source.py b/...connector/source_roman_coin_api/source.py → ...connector/source_roman_coin_api/source.py
diff --git a/...connector/source_roman_coin_api/spec.yaml → ...connector/source_roman_coin_api/spec.yaml b/...connector/source_roman_coin_api/spec.yaml → ...connector/source_roman_coin_api/spec.yaml
diff --git a/orchestration/Dockerfile → ...t-load-transform/orchestration/Dockerfile b/orchestration/Dockerfile → ...t-load-transform/orchestration/Dockerfile
diff --git a/orchestration/dagster.yaml → ...load-transform/orchestration/dagster.yaml b/orchestration/dagster.yaml → ...load-transform/orchestration/dagster.yaml
diff --git a/orchestration/orchestration/__init__.py → ...m/orchestration/orchestration/__init__.py b/orchestration/orchestration/__init__.py → ...m/orchestration/orchestration/__init__.py
diff --git a/orchestration/orchestration/jobs/__init__.py → ...hestration/orchestration/jobs/__init__.py b/orchestration/orchestration/jobs/__init__.py → ...hestration/orchestration/jobs/__init__.py
diff --git a/...ation/orchestration/resources/__init__.py → ...ation/orchestration/resources/__init__.py b/...ation/orchestration/resources/__init__.py → ...ation/orchestration/resources/__init__.py
diff --git a/...ation/orchestration/schedules/__init__.py → ...ation/orchestration/schedules/__init__.py b/...ation/orchestration/schedules/__init__.py → ...ation/orchestration/schedules/__init__.py
diff --git a/orchestration/pyproject.toml → ...ad-transform/orchestration/pyproject.toml b/orchestration/pyproject.toml → ...ad-transform/orchestration/pyproject.toml
diff --git a/orchestration/setup.cfg → ...ct-load-transform/orchestration/setup.cfg b/orchestration/setup.cfg → ...ct-load-transform/orchestration/setup.cfg
diff --git a/orchestration/setup.py → ...act-load-transform/orchestration/setup.py b/orchestration/setup.py → ...act-load-transform/orchestration/setup.py
diff --git a/orchestration/workspace.yaml → ...ad-transform/orchestration/workspace.yaml b/orchestration/workspace.yaml → ...ad-transform/orchestration/workspace.yaml