ci: Migrate to new Runners (#74)

huggingface · Nov 14, 2023 · ddaf53e · ddaf53e
1 parent 2b4b5d2
commit ddaf53e
Show file tree

Hide file tree

Showing 5 changed files with 5 additions and 304 deletions.
diff --git a/.github/workflows/build_75.yaml b/.github/workflows/build_75.yaml
@@ -21,47 +21,11 @@
        - 'main'
 
  jobs:
-   start-runner:
-     name: Start self-hosted EC2 runner
-     runs-on: ubuntu-latest
-     env:
-       AWS_REGION: us-east-1
-       EC2_AMI_ID: ami-03cfed9ea28f4b002
-       EC2_INSTANCE_TYPE: r5.8xlarge
-       EC2_SUBNET_ID: subnet-931b34f5,subnet-ecb993cd,subnet-943dc2d8,subnet-45371f1a,subnet-ee93e0df,subnet-fddc3dfc
-       EC2_SECURITY_GROUP: sg-030175c435ac141d6
-     outputs:
-       label: ${{ steps.start-ec2-runner.outputs.label }}
-       ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
-     steps:
-       - name: Configure AWS credentials
-         uses: aws-actions/configure-aws-credentials@v1
-         with:
-           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-           aws-region: ${{ env.AWS_REGION }}
-       - name: Start EC2 runner
-         id: start-ec2-runner
-         uses: philschmid/philschmid-ec2-github-runner@main
-         with:
-           mode: start
-           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-           ec2-image-id: ${{ env.EC2_AMI_ID }}
-           ec2-instance-type: ${{ env.EC2_INSTANCE_TYPE }}
-           subnet-id: ${{ env.EC2_SUBNET_ID }}
-           security-group-id: ${{ env.EC2_SECURITY_GROUP }}
-           aws-resource-tags: > # optional, requires additional permissions
-             [
-               {"Key": "Name", "Value": "ec2-tei-github-runner"},
-               {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}
-             ]
-
    build-and-push-image:
      concurrency:
        group: ${{ github.workflow }}-${{ github.job }}-75-${{ github.head_ref || github.run_id }}
        cancel-in-progress: true
-     needs: start-runner # required to start the main job when the runner is ready
-     runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+     runs-on: [self-hosted, intel-cpu, 32-cpu, tgi-ci]
      permissions:
        contents: write
        packages: write
@@ -135,27 +99,3 @@
            labels: ${{ steps.meta-75.outputs.labels }}
            cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/text-embeddings-inference:cache-75,mode=max
            cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/text-embeddings-inference:cache-75,mode=max
-
-   stop-runner:
-     name: Stop self-hosted EC2 runner
-     needs:
-       - start-runner
-       - build-and-push-image
-     runs-on: ubuntu-latest
-     env:
-       AWS_REGION: us-east-1
-     if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
-     steps:
-       - name: Configure AWS credentials
-         uses: aws-actions/configure-aws-credentials@v1
-         with:
-           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-           aws-region: ${{ env.AWS_REGION }}
-       - name: Stop EC2 runner
-         uses: philschmid/philschmid-ec2-github-runner@main
-         with:
-           mode: stop
-           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-           label: ${{ needs.start-runner.outputs.label }}
-           ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}
diff --git a/.github/workflows/build_80.yaml b/.github/workflows/build_80.yaml
@@ -21,47 +21,11 @@
        - 'main'
 
  jobs:
-   start-runner:
-     name: Start self-hosted EC2 runner
-     runs-on: ubuntu-latest
-     env:
-       AWS_REGION: us-east-1
-       EC2_AMI_ID: ami-03cfed9ea28f4b002
-       EC2_INSTANCE_TYPE: r5.8xlarge
-       EC2_SUBNET_ID: subnet-931b34f5,subnet-ecb993cd,subnet-943dc2d8,subnet-45371f1a,subnet-ee93e0df,subnet-fddc3dfc
-       EC2_SECURITY_GROUP: sg-030175c435ac141d6
-     outputs:
-       label: ${{ steps.start-ec2-runner.outputs.label }}
-       ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
-     steps:
-       - name: Configure AWS credentials
-         uses: aws-actions/configure-aws-credentials@v1
-         with:
-           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-           aws-region: ${{ env.AWS_REGION }}
-       - name: Start EC2 runner
-         id: start-ec2-runner
-         uses: philschmid/philschmid-ec2-github-runner@main
-         with:
-           mode: start
-           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-           ec2-image-id: ${{ env.EC2_AMI_ID }}
-           ec2-instance-type: ${{ env.EC2_INSTANCE_TYPE }}
-           subnet-id: ${{ env.EC2_SUBNET_ID }}
-           security-group-id: ${{ env.EC2_SECURITY_GROUP }}
-           aws-resource-tags: > # optional, requires additional permissions
-             [
-               {"Key": "Name", "Value": "ec2-tei-github-runner"},
-               {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}
-             ]
-
    build-and-push-image:
      concurrency:
        group: ${{ github.workflow }}-${{ github.job }}-80-${{ github.head_ref || github.run_id }}
        cancel-in-progress: true
-     needs: start-runner # required to start the main job when the runner is ready
-     runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+     runs-on: [self-hosted, intel-cpu, 32-cpu, tgi-ci]
      permissions:
        contents: write
        packages: write
@@ -134,27 +98,3 @@
            labels: ${{ steps.meta-80.outputs.labels }}
            cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/text-embeddings-inference:cache-80,mode=max
            cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/text-embeddings-inference:cache-80,mode=max
-
-   stop-runner:
-     name: Stop self-hosted EC2 runner
-     needs:
-       - start-runner
-       - build-and-push-image
-     runs-on: ubuntu-latest
-     env:
-       AWS_REGION: us-east-1
-     if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
-     steps:
-       - name: Configure AWS credentials
-         uses: aws-actions/configure-aws-credentials@v1
-         with:
-           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-           aws-region: ${{ env.AWS_REGION }}
-       - name: Stop EC2 runner
-         uses: philschmid/philschmid-ec2-github-runner@main
-         with:
-           mode: stop
-           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-           label: ${{ needs.start-runner.outputs.label }}
-           ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}
diff --git a/.github/workflows/build_86.yaml b/.github/workflows/build_86.yaml
@@ -21,47 +21,11 @@
        - 'main'
 
  jobs:
-   start-runner:
-     name: Start self-hosted EC2 runner
-     runs-on: ubuntu-latest
-     env:
-       AWS_REGION: us-east-1
-       EC2_AMI_ID: ami-03cfed9ea28f4b002
-       EC2_INSTANCE_TYPE: r5.8xlarge
-       EC2_SUBNET_ID: subnet-931b34f5,subnet-ecb993cd,subnet-943dc2d8,subnet-45371f1a,subnet-ee93e0df,subnet-fddc3dfc
-       EC2_SECURITY_GROUP: sg-030175c435ac141d6
-     outputs:
-       label: ${{ steps.start-ec2-runner.outputs.label }}
-       ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
-     steps:
-       - name: Configure AWS credentials
-         uses: aws-actions/configure-aws-credentials@v1
-         with:
-           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-           aws-region: ${{ env.AWS_REGION }}
-       - name: Start EC2 runner
-         id: start-ec2-runner
-         uses: philschmid/philschmid-ec2-github-runner@main
-         with:
-           mode: start
-           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-           ec2-image-id: ${{ env.EC2_AMI_ID }}
-           ec2-instance-type: ${{ env.EC2_INSTANCE_TYPE }}
-           subnet-id: ${{ env.EC2_SUBNET_ID }}
-           security-group-id: ${{ env.EC2_SECURITY_GROUP }}
-           aws-resource-tags: > # optional, requires additional permissions
-             [
-               {"Key": "Name", "Value": "ec2-tei-github-runner"},
-               {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}
-             ]
-
    build-and-push-image:
      concurrency:
        group: ${{ github.workflow }}-${{ github.job }}-86-${{ github.head_ref || github.run_id }}
        cancel-in-progress: true
-     needs: start-runner # required to start the main job when the runner is ready
-     runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+     runs-on: [self-hosted, intel-cpu, 32-cpu, tgi-ci]
      permissions:
        contents: write
        packages: write
@@ -134,27 +98,3 @@
            labels: ${{ steps.meta-86.outputs.labels }}
            cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/text-embeddings-inference:cache-86,mode=max
            cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/text-embeddings-inference:cache-86,mode=max
-
-   stop-runner:
-     name: Stop self-hosted EC2 runner
-     needs:
-       - start-runner
-       - build-and-push-image
-     runs-on: ubuntu-latest
-     env:
-       AWS_REGION: us-east-1
-     if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
-     steps:
-       - name: Configure AWS credentials
-         uses: aws-actions/configure-aws-credentials@v1
-         with:
-           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-           aws-region: ${{ env.AWS_REGION }}
-       - name: Stop EC2 runner
-         uses: philschmid/philschmid-ec2-github-runner@main
-         with:
-           mode: stop
-           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-           label: ${{ needs.start-runner.outputs.label }}
-           ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}
diff --git a/.github/workflows/build_89.yaml b/.github/workflows/build_89.yaml
@@ -21,47 +21,11 @@
        - 'main'
 
  jobs:
-   start-runner:
-     name: Start self-hosted EC2 runner
-     runs-on: ubuntu-latest
-     env:
-       AWS_REGION: us-east-1
-       EC2_AMI_ID: ami-03cfed9ea28f4b002
-       EC2_INSTANCE_TYPE: r5.8xlarge
-       EC2_SUBNET_ID: subnet-931b34f5,subnet-ecb993cd,subnet-943dc2d8,subnet-45371f1a,subnet-ee93e0df,subnet-fddc3dfc
-       EC2_SECURITY_GROUP: sg-030175c435ac141d6
-     outputs:
-       label: ${{ steps.start-ec2-runner.outputs.label }}
-       ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
-     steps:
-       - name: Configure AWS credentials
-         uses: aws-actions/configure-aws-credentials@v1
-         with:
-           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-           aws-region: ${{ env.AWS_REGION }}
-       - name: Start EC2 runner
-         id: start-ec2-runner
-         uses: philschmid/philschmid-ec2-github-runner@main
-         with:
-           mode: start
-           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-           ec2-image-id: ${{ env.EC2_AMI_ID }}
-           ec2-instance-type: ${{ env.EC2_INSTANCE_TYPE }}
-           subnet-id: ${{ env.EC2_SUBNET_ID }}
-           security-group-id: ${{ env.EC2_SECURITY_GROUP }}
-           aws-resource-tags: > # optional, requires additional permissions
-             [
-               {"Key": "Name", "Value": "ec2-tei-github-runner"},
-               {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}
-             ]
-
    build-and-push-image:
      concurrency:
        group: ${{ github.workflow }}-${{ github.job }}-89-${{ github.head_ref || github.run_id }}
        cancel-in-progress: true
-     needs: start-runner # required to start the main job when the runner is ready
-     runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+     runs-on: [self-hosted, intel-cpu, 32-cpu, tgi-ci]
      permissions:
        contents: write
        packages: write
@@ -134,27 +98,3 @@
            labels: ${{ steps.meta-89.outputs.labels }}
            cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/text-embeddings-inference:cache-89,mode=max
            cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/text-embeddings-inference:cache-89,mode=max
-
-   stop-runner:
-     name: Stop self-hosted EC2 runner
-     needs:
-       - start-runner
-       - build-and-push-image
-     runs-on: ubuntu-latest
-     env:
-       AWS_REGION: us-east-1
-     if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
-     steps:
-       - name: Configure AWS credentials
-         uses: aws-actions/configure-aws-credentials@v1
-         with:
-           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-           aws-region: ${{ env.AWS_REGION }}
-       - name: Stop EC2 runner
-         uses: philschmid/philschmid-ec2-github-runner@main
-         with:
-           mode: stop
-           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-           label: ${{ needs.start-runner.outputs.label }}
-           ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}
diff --git a/.github/workflows/build_90.yaml b/.github/workflows/build_90.yaml
@@ -21,47 +21,11 @@
        - 'main'
 
  jobs:
-   start-runner:
-     name: Start self-hosted EC2 runner
-     runs-on: ubuntu-latest
-     env:
-       AWS_REGION: us-east-1
-       EC2_AMI_ID: ami-03cfed9ea28f4b002
-       EC2_INSTANCE_TYPE: r5.8xlarge
-       EC2_SUBNET_ID: subnet-931b34f5,subnet-ecb993cd,subnet-943dc2d8,subnet-45371f1a,subnet-ee93e0df,subnet-fddc3dfc
-       EC2_SECURITY_GROUP: sg-030175c435ac141d6
-     outputs:
-       label: ${{ steps.start-ec2-runner.outputs.label }}
-       ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
-     steps:
-       - name: Configure AWS credentials
-         uses: aws-actions/configure-aws-credentials@v1
-         with:
-           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-           aws-region: ${{ env.AWS_REGION }}
-       - name: Start EC2 runner
-         id: start-ec2-runner
-         uses: philschmid/philschmid-ec2-github-runner@main
-         with:
-           mode: start
-           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-           ec2-image-id: ${{ env.EC2_AMI_ID }}
-           ec2-instance-type: ${{ env.EC2_INSTANCE_TYPE }}
-           subnet-id: ${{ env.EC2_SUBNET_ID }}
-           security-group-id: ${{ env.EC2_SECURITY_GROUP }}
-           aws-resource-tags: > # optional, requires additional permissions
-             [
-               {"Key": "Name", "Value": "ec2-tei-github-runner"},
-               {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}
-             ]
-
    build-and-push-image:
      concurrency:
        group: ${{ github.workflow }}-${{ github.job }}-90-${{ github.head_ref || github.run_id }}
        cancel-in-progress: true
-     needs: start-runner # required to start the main job when the runner is ready
-     runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+     runs-on: [self-hosted, intel-cpu, 32-cpu, tgi-ci]
      permissions:
        contents: write
        packages: write
@@ -135,26 +99,3 @@
            cache-from: type=registry,ref=registry.internal.huggingface.tech/api-inference/text-embeddings-inference:cache-90,mode=max
            cache-to: type=registry,ref=registry.internal.huggingface.tech/api-inference/text-embeddings-inference:cache-90,mode=max
 
-   stop-runner:
-     name: Stop self-hosted EC2 runner
-     needs:
-       - start-runner
-       - build-and-push-image
-     runs-on: ubuntu-latest
-     env:
-       AWS_REGION: us-east-1
-     if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
-     steps:
-       - name: Configure AWS credentials
-         uses: aws-actions/configure-aws-credentials@v1
-         with:
-           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-           aws-region: ${{ env.AWS_REGION }}
-       - name: Stop EC2 runner
-         uses: philschmid/philschmid-ec2-github-runner@main
-         with:
-           mode: stop
-           github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
-           label: ${{ needs.start-runner.outputs.label }}
-           ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}