diff --git a/compose.test.yaml b/compose.test.yaml index 44da0e8..f6bb6fb 100644 --- a/compose.test.yaml +++ b/compose.test.yaml @@ -12,6 +12,7 @@ services: condition: service_started db: condition: service_healthy + web_scraper: build: ./web_scraping environment: @@ -23,8 +24,9 @@ services: depends_on: db: condition: service_healthy + db: - image: postgres:latest + image: postgres restart: unless-stopped volumes: - db-data:/var/lib/postgresql/data @@ -39,8 +41,9 @@ services: interval: 10s timeout: 5s retries: 5 + minio: - image: minio/minio:latest + image: minio/minio volumes: - minio-data:/data environment: @@ -50,8 +53,9 @@ services: ports: - 9000:9000 - 9090:9090 + minio-setup: - image: minio/mc:latest + image: minio/mc depends_on: - minio entrypoint: > @@ -64,14 +68,16 @@ services: mc admin user add myminio ${MINIO_NEW_USER} ${MINIO_NEW_USER_PASSWORD}; mc admin policy attach myminio readwrite --user=${MINIO_NEW_USER}; " + custom-airbyte-connector: environment: - - HOST=${HOST} + - AIRBYTE_HOST=http://${HOST} build: - context: ./custom-airbyte-connector + context: ./extract-load-transform/custom-airbyte-connector dockerfile: Dockerfile image: airbyte/source-roman-coins-api:latest pull_policy: build + airbyte-configurator: depends_on: - api @@ -80,14 +86,62 @@ services: - MINIO_BUCKET_NAME=${MINIO_BUCKET_NAME} - MINIO_NEW_USER=${MINIO_NEW_USER} - MINIO_NEW_USER_PASSWORD=${MINIO_NEW_USER_PASSWORD} - - AIRBYTE_HOST=${HOST} + - AIRBYTE_HOST=http://${HOST} - AIRBYTE_USERNAME=${AIRBYTE_USERNAME} - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD} - CONNECTOR_START_DATE=${CONNECTOR_START_DATE} build: - context: ./airbyte-api-minio-connection + context: ./extract-load-transform/airbyte-api-minio-connection dockerfile: Dockerfile pull_policy: build + + dagster-webserver: + depends_on: + - db + - airbyte-configurator + environment: + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_DB=${POSTGRES_DB} + - DB_HOST=db + - AIRBYTE_USERNAME=${AIRBYTE_USERNAME} + - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD} + - HOST=${HOST} + build: + context: ./extract-load-transform/orchestration + dockerfile: Dockerfile + entrypoint: + - dagster-webserver + - -h + - "0.0.0.0" + - -p + - "3000" + - -w + - workspace.yaml + expose: + - 3000 + ports: + - 3000:3000 + + dagster-daemon: + depends_on: + - db + - airbyte-configurator + environment: + - POSTGRES_USER=${POSTGRES_USER} + - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} + - POSTGRES_DB=${POSTGRES_DB} + - DB_HOST=db + - AIRBYTE_USERNAME=${AIRBYTE_USERNAME} + - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD} + - HOST=${HOST} + build: + context: ./extract-load-transform/orchestration + dockerfile: Dockerfile + entrypoint: + - dagster-daemon + - run + restart: on-failure test_db: image: postgres:latest restart: unless-stopped diff --git a/compose.yaml b/compose.yaml index 3e4fc9d..8189140 100644 --- a/compose.yaml +++ b/compose.yaml @@ -73,7 +73,7 @@ services: environment: - AIRBYTE_HOST=http://${HOST} build: - context: ./custom-airbyte-connector + context: ./extract-load-transform/custom-airbyte-connector dockerfile: Dockerfile image: airbyte/source-roman-coins-api:latest pull_policy: build @@ -91,7 +91,7 @@ services: - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD} - CONNECTOR_START_DATE=${CONNECTOR_START_DATE} build: - context: ./airbyte-api-minio-connection + context: ./extract-load-transform/airbyte-api-minio-connection dockerfile: Dockerfile pull_policy: build @@ -108,7 +108,7 @@ services: - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD} - HOST=${HOST} build: - context: ./orchestration + context: ./extract-load-transform/orchestration dockerfile: Dockerfile entrypoint: - dagster-webserver @@ -136,7 +136,7 @@ services: - AIRBYTE_PASSWORD=${AIRBYTE_PASSWORD} - HOST=${HOST} build: - context: ./orchestration + context: ./extract-load-transform/orchestration dockerfile: Dockerfile entrypoint: - dagster-daemon diff --git a/airbyte-api-minio-connection/.dockerignore b/extract-load-transform/airbyte-api-minio-connection/.dockerignore similarity index 100% rename from airbyte-api-minio-connection/.dockerignore rename to extract-load-transform/airbyte-api-minio-connection/.dockerignore diff --git a/airbyte-api-minio-connection/Dockerfile b/extract-load-transform/airbyte-api-minio-connection/Dockerfile similarity index 100% rename from airbyte-api-minio-connection/Dockerfile rename to extract-load-transform/airbyte-api-minio-connection/Dockerfile diff --git a/airbyte-api-minio-connection/airbyte_connection_config.py b/extract-load-transform/airbyte-api-minio-connection/airbyte_connection_config.py similarity index 100% rename from airbyte-api-minio-connection/airbyte_connection_config.py rename to extract-load-transform/airbyte-api-minio-connection/airbyte_connection_config.py diff --git a/airbyte-api-minio-connection/requirements.txt b/extract-load-transform/airbyte-api-minio-connection/requirements.txt similarity index 100% rename from airbyte-api-minio-connection/requirements.txt rename to extract-load-transform/airbyte-api-minio-connection/requirements.txt diff --git a/custom-airbyte-connector/Dockerfile b/extract-load-transform/custom-airbyte-connector/Dockerfile similarity index 100% rename from custom-airbyte-connector/Dockerfile rename to extract-load-transform/custom-airbyte-connector/Dockerfile diff --git a/custom-airbyte-connector/build/lib/source_roman_coin_api/__init__.py b/extract-load-transform/custom-airbyte-connector/build/lib/source_roman_coin_api/__init__.py similarity index 100% rename from custom-airbyte-connector/build/lib/source_roman_coin_api/__init__.py rename to extract-load-transform/custom-airbyte-connector/build/lib/source_roman_coin_api/__init__.py diff --git a/custom-airbyte-connector/build/lib/source_roman_coin_api/schemas/roman_coin_api_stream.json b/extract-load-transform/custom-airbyte-connector/build/lib/source_roman_coin_api/schemas/roman_coin_api_stream.json similarity index 100% rename from custom-airbyte-connector/build/lib/source_roman_coin_api/schemas/roman_coin_api_stream.json rename to extract-load-transform/custom-airbyte-connector/build/lib/source_roman_coin_api/schemas/roman_coin_api_stream.json diff --git a/custom-airbyte-connector/build/lib/source_roman_coin_api/source.py b/extract-load-transform/custom-airbyte-connector/build/lib/source_roman_coin_api/source.py similarity index 100% rename from custom-airbyte-connector/build/lib/source_roman_coin_api/source.py rename to extract-load-transform/custom-airbyte-connector/build/lib/source_roman_coin_api/source.py diff --git a/custom-airbyte-connector/build/lib/source_roman_coin_api/spec.yaml b/extract-load-transform/custom-airbyte-connector/build/lib/source_roman_coin_api/spec.yaml similarity index 100% rename from custom-airbyte-connector/build/lib/source_roman_coin_api/spec.yaml rename to extract-load-transform/custom-airbyte-connector/build/lib/source_roman_coin_api/spec.yaml diff --git a/custom-airbyte-connector/main.py b/extract-load-transform/custom-airbyte-connector/main.py similarity index 100% rename from custom-airbyte-connector/main.py rename to extract-load-transform/custom-airbyte-connector/main.py diff --git a/custom-airbyte-connector/requirements.txt b/extract-load-transform/custom-airbyte-connector/requirements.txt similarity index 100% rename from custom-airbyte-connector/requirements.txt rename to extract-load-transform/custom-airbyte-connector/requirements.txt diff --git a/custom-airbyte-connector/sample_files/configured_catalog.json b/extract-load-transform/custom-airbyte-connector/sample_files/configured_catalog.json similarity index 100% rename from custom-airbyte-connector/sample_files/configured_catalog.json rename to extract-load-transform/custom-airbyte-connector/sample_files/configured_catalog.json diff --git a/custom-airbyte-connector/secrets/config.json b/extract-load-transform/custom-airbyte-connector/secrets/config.json similarity index 100% rename from custom-airbyte-connector/secrets/config.json rename to extract-load-transform/custom-airbyte-connector/secrets/config.json diff --git a/custom-airbyte-connector/setup.py b/extract-load-transform/custom-airbyte-connector/setup.py similarity index 100% rename from custom-airbyte-connector/setup.py rename to extract-load-transform/custom-airbyte-connector/setup.py diff --git a/custom-airbyte-connector/source_roman_coin_api/__init__.py b/extract-load-transform/custom-airbyte-connector/source_roman_coin_api/__init__.py similarity index 100% rename from custom-airbyte-connector/source_roman_coin_api/__init__.py rename to extract-load-transform/custom-airbyte-connector/source_roman_coin_api/__init__.py diff --git a/custom-airbyte-connector/source_roman_coin_api/schemas/roman_coin_api_stream.json b/extract-load-transform/custom-airbyte-connector/source_roman_coin_api/schemas/roman_coin_api_stream.json similarity index 100% rename from custom-airbyte-connector/source_roman_coin_api/schemas/roman_coin_api_stream.json rename to extract-load-transform/custom-airbyte-connector/source_roman_coin_api/schemas/roman_coin_api_stream.json diff --git a/custom-airbyte-connector/source_roman_coin_api/source.py b/extract-load-transform/custom-airbyte-connector/source_roman_coin_api/source.py similarity index 100% rename from custom-airbyte-connector/source_roman_coin_api/source.py rename to extract-load-transform/custom-airbyte-connector/source_roman_coin_api/source.py diff --git a/custom-airbyte-connector/source_roman_coin_api/spec.yaml b/extract-load-transform/custom-airbyte-connector/source_roman_coin_api/spec.yaml similarity index 100% rename from custom-airbyte-connector/source_roman_coin_api/spec.yaml rename to extract-load-transform/custom-airbyte-connector/source_roman_coin_api/spec.yaml diff --git a/orchestration/Dockerfile b/extract-load-transform/orchestration/Dockerfile similarity index 100% rename from orchestration/Dockerfile rename to extract-load-transform/orchestration/Dockerfile diff --git a/orchestration/dagster.yaml b/extract-load-transform/orchestration/dagster.yaml similarity index 100% rename from orchestration/dagster.yaml rename to extract-load-transform/orchestration/dagster.yaml diff --git a/orchestration/orchestration/__init__.py b/extract-load-transform/orchestration/orchestration/__init__.py similarity index 100% rename from orchestration/orchestration/__init__.py rename to extract-load-transform/orchestration/orchestration/__init__.py diff --git a/orchestration/orchestration/jobs/__init__.py b/extract-load-transform/orchestration/orchestration/jobs/__init__.py similarity index 100% rename from orchestration/orchestration/jobs/__init__.py rename to extract-load-transform/orchestration/orchestration/jobs/__init__.py diff --git a/orchestration/orchestration/resources/__init__.py b/extract-load-transform/orchestration/orchestration/resources/__init__.py similarity index 100% rename from orchestration/orchestration/resources/__init__.py rename to extract-load-transform/orchestration/orchestration/resources/__init__.py diff --git a/orchestration/orchestration/schedules/__init__.py b/extract-load-transform/orchestration/orchestration/schedules/__init__.py similarity index 100% rename from orchestration/orchestration/schedules/__init__.py rename to extract-load-transform/orchestration/orchestration/schedules/__init__.py diff --git a/orchestration/pyproject.toml b/extract-load-transform/orchestration/pyproject.toml similarity index 100% rename from orchestration/pyproject.toml rename to extract-load-transform/orchestration/pyproject.toml diff --git a/orchestration/setup.cfg b/extract-load-transform/orchestration/setup.cfg similarity index 100% rename from orchestration/setup.cfg rename to extract-load-transform/orchestration/setup.cfg diff --git a/orchestration/setup.py b/extract-load-transform/orchestration/setup.py similarity index 100% rename from orchestration/setup.py rename to extract-load-transform/orchestration/setup.py diff --git a/orchestration/workspace.yaml b/extract-load-transform/orchestration/workspace.yaml similarity index 100% rename from orchestration/workspace.yaml rename to extract-load-transform/orchestration/workspace.yaml