Merge pull request #11 from wednesday-solutions/fix/ci-lint #26
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Data Engineering - Deploy to AWS Glue | |
| on: | |
| push: | |
| branches: | |
| - main | |
| jobs: | |
| deploy: | |
| runs-on: ubuntu-latest | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }} | |
| S3_SCRIPTS_PATH: ${{ secrets.S3_SCRIPTS_PATH }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_GLUE_ROLE: ${{ secrets.AWS_GLUE_ROLE}} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.9 | |
| - name: Build App Wheel | |
| run: | | |
| pip install setuptools wheel | |
| python3 setup.py bdist_wheel | |
| - name: Setup AWS cli & upload App Wheel to S3 | |
| uses: jakejarvis/s3-sync-action@v0.5.0 | |
| with: | |
| args: --follow-symlinks | |
| env: | |
| SOURCE_DIR: "dist" | |
| DEST_DIR: $S3_SCRIPTS_PATH | |
| AWS_S3_BUCKET: $S3_BUCKET_NAME | |
| - name: Upload Scripts to S3 | |
| run: aws s3 cp jobs "s3://$S3_BUCKET_NAME/$S3_SCRIPTS_PATH/" --recursive --region ap-south-1 | |
| - name: Deploy Jobs on Glue | |
| run: | | |
| automation/deploy_glue_job.sh $S3_BUCKET_NAME $AWS_GLUE_ROLE $KAGGLE_TOKEN $KAGGLE_USERNAME |