Merge pull request #39 from ks6088ts-labs/feature/issue-38_refactor-envs

refactor codes

.github/workflows/docker-release.yaml

@@ -28,7 +28,7 @@ jobs:
             core.setOutput('tag', tag)
             core.setOutput('no-v', no_v)
             core.setOutput('no-dash', no_dash)
-      - name: Build and push tag ${{steps.set_version.outputs.no-dash}}
+      - name: Build and push tag ${{steps.set_version.outputs.no-dash}} for backend
         uses: docker/build-push-action@v5
         with:
           context: .
@@ -38,5 +38,17 @@ jobs:
             GIT_REVISION=${{ github.sha }}
             GIT_TAG=${{steps.set_version.outputs.no-dash}}
           tags: |
-            ${{ secrets.DOCKERHUB_USERNAME }}/azure-ai-services-solutions:${{steps.set_version.outputs.no-dash}}
-            ${{ secrets.DOCKERHUB_USERNAME }}/azure-ai-services-solutions:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/azure-ai-services-solutions:backend-${{steps.set_version.outputs.no-dash}}
+            ${{ secrets.DOCKERHUB_USERNAME }}/azure-ai-services-solutions:backend-latest
+      - name: Build and push tag ${{steps.set_version.outputs.no-dash}} for frontend
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./dockerfiles/frontend.Dockerfile
+          push: true
+          build-args: |
+            GIT_REVISION=${{ github.sha }}
+            GIT_TAG=${{steps.set_version.outputs.no-dash}}
+          tags: |
+            ${{ secrets.DOCKERHUB_USERNAME }}/azure-ai-services-solutions:frontend-${{steps.set_version.outputs.no-dash}}
+            ${{ secrets.DOCKERHUB_USERNAME }}/azure-ai-services-solutions:frontend-latest

Makefile

@@ -51,9 +51,10 @@ ci-test: install-deps-dev format-check lint test ## run CI tests
 # ---
 DOCKER_REPO_NAME ?= ks6088ts
 DOCKER_IMAGE_NAME ?= azure-ai-services-solutions
-DOCKER_COMMAND ?= python main.py --help
-DOCKER_TAG ?= local
-DOCKERFILE ?= dockerfiles/backend.Dockerfile
+DOCKER_IMAGE_COMPONENT ?= backend
+DOCKER_COMMAND ?=
+DOCKER_TAG ?= $(DOCKER_IMAGE_COMPONENT)-$(GIT_TAG)
+DOCKER_FILE ?= ./dockerfiles/$(DOCKER_IMAGE_COMPONENT).Dockerfile
 
 # Tools
 TOOLS_DIR ?= $(HOME)/.local/bin
@@ -63,50 +64,48 @@ TRIVY_VERSION ?= 0.49.1
 docker-build: ## build Docker image
 	docker build \
 		--tag $(DOCKER_REPO_NAME)/$(DOCKER_IMAGE_NAME):$(DOCKER_TAG) \
-		--file $(DOCKERFILE) \
+		--file $(DOCKER_FILE) \
 		--build-arg GIT_REVISION=$(GIT_REVISION) \
 		--build-arg GIT_TAG=$(GIT_TAG) \
 		.
 
 .PHONY: docker-run
 docker-run: ## run Docker container
-	docker run --rm \
-		--publish 8888:8888 \
-		--volume $(PWD)/azure_ai_speech.env.sample:/app/azure_ai_speech.env \
-		--volume $(PWD)/azure_ai_vision.env.sample:/app/azure_ai_vision.env \
-		--volume $(PWD)/azure_event_grid.env.sample:/app/azure_event_grid.env \
-		--volume $(PWD)/azure_openai.env.sample:/app/azure_openai.env \
-		--volume $(PWD)/azure_storage.env.sample:/app/azure_storage.env \
-		--volume $(PWD)/azure_storage_queue.env.sample:/app/azure_storage_queue.env \
-		--volume $(PWD)/document_intelligence.env.sample:/app/document_intelligence.env \
-		$(DOCKER_REPO_NAME)/$(DOCKER_IMAGE_NAME):$(DOCKER_TAG) \
-		$(DOCKER_COMMAND)
+	docker run --rm $(DOCKER_REPO_NAME)/$(DOCKER_IMAGE_NAME):$(DOCKER_TAG) $(DOCKER_COMMAND)
 
 .PHONY: docker-lint
 docker-lint: ## lint Dockerfile
-	docker run --rm -i hadolint/hadolint < $(DOCKERFILE)
+	docker run --rm -i hadolint/hadolint < $(DOCKER_FILE)
 
 .PHONY: docker-scan
 docker-scan: ## scan Docker image
 	@# https://aquasecurity.github.io/trivy/v0.18.3/installation/#install-script
 	@which trivy || curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b $(TOOLS_DIR) v$(TRIVY_VERSION)
 	trivy image $(DOCKER_REPO_NAME)/$(DOCKER_IMAGE_NAME):$(DOCKER_TAG)
 
+.PHONY: _ci-test-docker
+_ci-test-docker: docker-lint docker-build docker-scan docker-run
+
 .PHONY: ci-test-docker
-ci-test-docker: docker-lint docker-build docker-scan docker-run ## run CI test for Docker
+ci-test-docker: ## run CI test for Docker
+	$(MAKE) _ci-test-docker DOCKER_IMAGE_COMPONENT=backend
+	$(MAKE) _ci-test-docker DOCKER_IMAGE_COMPONENT=frontend
 
 # ---
 # Application
 # ---
 SOLUTION_NAME ?= "SANDBOX"
+BACKEND_URL ?= "http://localhost:8000"
 
 .PHONY: backend
 backend: ## run backend
 	poetry run python main.py backend --reload
 
 .PHONY: frontend
 frontend: ## run frontend
-	poetry run streamlit run main.py -- frontend -- --solution-name=$(SOLUTION_NAME)
+	poetry run streamlit run main.py -- frontend -- \
+		--solution-name=$(SOLUTION_NAME) \
+		--backend-url=$(BACKEND_URL)
 
 # ---
 # Azure Functions
@@ -149,3 +148,6 @@ generate-openapi-client: ## generate OpenAPI client
 		--output ./client
 	@echo "Get the list of dependencies"
 	@kiota info -d ./specs/openapi.json --language Python
+
+.PHONY: generate-openapi
+generate-openapi: generate-openapi-spec generate-openapi-client ## generate OpenAPI artifacts

README.md

@@ -45,9 +45,20 @@ make docker-run
 make ci-test-docker
 ```
 
-To publish the docker image to Docker Hub, you need to set the following secrets in the repository settings.
+## Deployment instructions
+
+### Docker Hub
+
+To publish the docker image to Docker Hub via GitHub Actions, you need to set the following secrets in the repository.
 
 ```shell
 gh secret set DOCKERHUB_USERNAME --body $DOCKERHUB_USERNAME
 gh secret set DOCKERHUB_TOKEN --body $DOCKERHUB_TOKEN
 ```
+
+### Azure Functions
+
+To deploy the Azure Functions, you can refer to the following scripts.
+
+- [scripts/deploy-azure-functions.sh](./scripts/deploy-azure-functions.sh): Deploy the Azure Functions using Azure CLI.
+- [scripts/destroy-azure-functions.sh](./scripts/destroy-azure-functions.sh): Destroy the Azure Functions using Azure CLI.

azure_ai_document_intelligence.env.sample

@@ -0,0 +1,2 @@
+AZURE_AI_DOCUMENT_INTELLIGENCE_ENDPOINT="https://<your-document-intelligence-name>.cognitiveservices.azure.com"
+AZURE_AI_DOCUMENT_INTELLIGENCE_API_KEY="<your-api-key>"

azure_storage_blob.env.sample

@@ -0,0 +1,3 @@
+AZURE_STORAGE_BLOB_ACCOUNT_NAME = "<account-name>"
+AZURE_STORAGE_BLOB_SAS_TOKEN = "<sas-token>"
+AZURE_STORAGE_BLOB_CONTAINER_NAME = "<blob-container-name>"

backend/fastapi.py

@@ -1,23 +1,26 @@
 from fastapi import FastAPI
 from fastapi.openapi.utils import get_openapi
 
+from backend.routers import azure_ai_document_intelligence as azure_ai_document_intelligence_router
 from backend.routers import azure_ai_vision as azure_ai_vision_router
 from backend.routers import azure_event_grid as azure_event_grid_router
 from backend.routers import azure_openai as azure_openai_router
-from backend.routers import azure_storage as azure_storage_router
+from backend.routers import azure_storage_blob as azure_storage_blob_router
 from backend.routers import azure_storage_queue as azure_storage_queue_router
-from backend.routers import document_intelligence as document_intelligence_router
 
 app = FastAPI(
     docs_url="/",
 )
 
-app.include_router(azure_openai_router.router)
-app.include_router(document_intelligence_router.router)
-app.include_router(azure_storage_router.router)
-app.include_router(azure_ai_vision_router.router)
-app.include_router(azure_event_grid_router.router)
-app.include_router(azure_storage_queue_router.router)
+for router in [
+    azure_ai_document_intelligence_router.router,
+    azure_ai_vision_router.router,
+    azure_event_grid_router.router,
+    azure_openai_router.router,
+    azure_storage_blob_router.router,
+    azure_storage_queue_router.router,
+]:
+    app.include_router(router)
 
 
 def custom_openapi():

backend/internals/azure_ai_document_intelligence.py

@@ -0,0 +1,36 @@
+from logging import getLogger
+
+from azure.ai.documentintelligence import DocumentIntelligenceClient
+from azure.ai.documentintelligence.models import AnalyzeDocumentRequest, AnalyzeResult, ContentFormat
+from azure.core.credentials import AzureKeyCredential
+
+from backend.settings.azure_ai_document_intelligence import Settings
+
+logger = getLogger(__name__)
+
+
+class Client:
+    def __init__(self, settings: Settings) -> None:
+        self.settings = settings
+
+    def get_document_intelligence_client(self) -> DocumentIntelligenceClient:
+        return DocumentIntelligenceClient(
+            endpoint=self.settings.azure_ai_document_intelligence_endpoint,
+            credential=AzureKeyCredential(self.settings.azure_ai_document_intelligence_api_key),
+        )
+
+    def analyze_document(
+        self,
+        bytes_source: bytes,
+    ) -> AnalyzeResult:
+        client = self.get_document_intelligence_client()
+        poller = client.begin_analyze_document(
+            model_id="prebuilt-read",
+            analyze_request=AnalyzeDocumentRequest(
+                bytes_source=bytes_source,
+            ),
+            output_content_format=ContentFormat.MARKDOWN,
+        )
+        result = poller.result()
+        logger.info(result)
+        return result

backend/internals/azure_ai_vision.py

@@ -9,7 +9,7 @@
 logger = getLogger(__name__)
 
 
-class AzureAiVisionClient:
+class Client:
     def __init__(self, settings: Settings) -> None:
         self.settings = settings
 

backend/internals/azure_openai.py

@@ -4,15 +4,13 @@
 from openai import AzureOpenAI
 from openai.types.chat import ChatCompletion
 
-from backend.settings import azure_openai as azure_openai_settings
+from backend.settings.azure_openai import Settings
 
 logger = getLogger(__name__)
 
-settings = azure_openai_settings.Settings()
-
 
 class Client:
-    def __init__(self, settings: azure_openai_settings.Settings) -> None:
+    def __init__(self, settings: Settings) -> None:
         self.settings = settings
 
     def get_client(self) -> AzureOpenAI:

backend/internals/azure_storage.py → backend/internals/azure_storage_blob.py

@@ -2,19 +2,19 @@
 
 from azure.storage.blob import BlobServiceClient
 
-from backend.settings import azure_storage as azure_storage_settings
+from backend.settings.azure_storage_blob import Settings
 
 logger = getLogger(__name__)
 
 
-class BlobStorageClient:
-    def __init__(self, settings: azure_storage_settings.Settings):
+class Client:
+    def __init__(self, settings: Settings):
         self.settings = settings
 
     def get_blob_service_client(self) -> BlobServiceClient:
         return BlobServiceClient(
-            account_url=f"https://{self.settings.azure_storage_account_name}.blob.core.windows.net",
-            credential=self.settings.azure_storage_sas_token,
+            account_url=f"https://{self.settings.azure_storage_blob_account_name}.blob.core.windows.net",
+            credential=self.settings.azure_storage_blob_sas_token,
         )
 
     def upload_blob_stream(

backend/internals/azure_storage_queue.py

@@ -3,13 +3,13 @@
 from azure.core.paging import ItemPaged
 from azure.storage.queue import QueueMessage, QueueServiceClient
 
-from backend.settings.azure_storage_queue import AzureStorageQueueSettings
+from backend.settings.azure_storage_queue import Settings
 
 logger = getLogger(__name__)
 
 
-class AzureStorageQueueClient:
-    def __init__(self, settings: AzureStorageQueueSettings):
+class Client:
+    def __init__(self, settings: Settings):
         self.client = QueueServiceClient.from_connection_string(settings.azure_storage_queue_connection_string)
 
     def create_queue(

backend/routers/azure_ai_document_intelligence.py

@@ -0,0 +1,38 @@
+from logging import getLogger
+
+from fastapi import APIRouter, UploadFile
+
+from backend.internals.azure_ai_document_intelligence import Client
+from backend.schemas import azure_ai_document_intelligence as azure_ai_document_intelligence_schemas
+from backend.settings.azure_ai_document_intelligence import Settings
+
+logger = getLogger(__name__)
+
+client = Client(
+    settings=Settings(),
+)
+
+router = APIRouter(
+    prefix="/azure_ai_document_intelligence",
+    tags=["azure_ai_document_intelligence"],
+    responses={404: {"description": "Not found"}},
+)
+
+
+@router.post(
+    "/analyze_document/",
+    response_model=azure_ai_document_intelligence_schemas.AnalyzeDocumentResponse,
+    status_code=200,
+)
+async def analyze_document(file: UploadFile):
+    try:
+        content = await file.read()
+        result = client.analyze_document(
+            bytes_source=content,
+        )
+    except Exception as e:
+        logger.error(f"Failed to read file: {e}")
+        raise
+    return azure_ai_document_intelligence_schemas.AnalyzeDocumentResponse(
+        content=result.content,
+    )

backend/routers/azure_ai_vision.py

@@ -2,12 +2,13 @@
 
 from fastapi import APIRouter, UploadFile
 
-from backend.internals import azure_ai_vision
+from backend.internals.azure_ai_vision import Client
 from backend.schemas import azure_ai_vision as azure_ai_vision_schemas
 from backend.settings.azure_ai_vision import Settings
 
 logger = getLogger(__name__)
-client = azure_ai_vision.AzureAiVisionClient(
+
+client = Client(
     settings=Settings(),
 )
 

backend/routers/azure_event_grid.py

@@ -2,11 +2,12 @@
 
 from fastapi import APIRouter
 
-from backend.internals import azure_event_grid
+from backend.internals.azure_event_grid import Client
 from backend.settings.azure_event_grid import Settings
 
 logger = getLogger(__name__)
-client = azure_event_grid.Client(
+
+client = Client(
     settings=Settings(),
 )