diff --git a/.github/actions/trigger-nightly/action.yml b/.github/actions/trigger-nightly/action.yml
new file mode 100644
index 0000000000..7714697c03
--- /dev/null
+++ b/.github/actions/trigger-nightly/action.yml
@@ -0,0 +1,34 @@
+name: Trigger nightly builds
+
+inputs:
+  github-secret:
+    description: GitHub token
+    required: true
+  repository:
+    required: true
+    description: 'Which repo to checkout'
+    default: false
+  branch:
+    description: Branch to cut from
+    required: true
+
+jobs:
+  nightly:
+    name: Trigger nightly build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.branch }}
+          repository: ${{ inputs.repository }}
+          token: ${{ inputs.github-secret }}
+      - run: |
+          git config --global user.email "pytorchbot@pytorch.com"
+          git config --global user.name "pytorchbot"
+          git fetch origin nightly
+          HEAD_COMMIT_HASH=$(git rev-parse HEAD)
+          NIGHTLY_DATE=$(date +"%Y-%m-%d")
+          # shellcheck disable=SC1083
+          NIGHTLY_RELEASE_COMMIT=$(git commit-tree -p FETCH_HEAD HEAD^{tree} -m "${NIGHTLY_DATE} nightly release (${HEAD_COMMIT_HASH})")
+          # shellcheck disable=SC1083
+          git push -f origin "${NIGHTLY_RELEASE_COMMIT}:nightly"
diff --git a/.github/workflows/trigger_nightly.yml b/.github/workflows/trigger_nightly.yml
index 97ec2eacfd..e701e4f1b9 100644
--- a/.github/workflows/trigger_nightly.yml
+++ b/.github/workflows/trigger_nightly.yml
@@ -21,21 +21,24 @@ jobs:
   tag_nightly_text:
     if: ${{ github.event_name == 'schedule' ||  inputs.domain == 'text' || inputs.domain == 'all' }}
     name: Trigger nightly text build
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          ref: main
-          repository: pytorch/text
-          token: ${{ secrets.GH_PYTORCHBOT_TOKEN }}
-      - run: |
-          git config --global user.email "pytorchbot@pytorch.com"
-          git config --global user.name "pytorchbot"
-          git checkout origin/main
-          git fetch origin nightly
-          HEAD_COMMIT_HASH=$(git rev-parse HEAD)
-          NIGHTLY_DATE=$(date +"%Y-%m-%d")
-          # shellcheck disable=SC1083
-          NIGHTLY_RELEASE_COMMIT=$(git commit-tree -p FETCH_HEAD HEAD^{tree} -m "${NIGHTLY_DATE} nightly release (${HEAD_COMMIT_HASH})")
-          # shellcheck disable=SC1083
-          git push -f origin "${NIGHTLY_RELEASE_COMMIT}:nightly"
+    uses: ./test-infra/.github/actions/trigger-nightly
+    with:
+      ref: main
+      repository: pytorch/text
+      token: ${{ secrets.GH_PYTORCHBOT_TOKEN }}
+  tag_nightly_vision:
+    if: ${{ github.event_name == 'schedule' ||  inputs.domain == 'vision' || inputs.domain == 'all' }}
+    name: Trigger nightly vision build
+    uses: ./test-infra/.github/actions/trigger-nightly
+    with:
+      ref: main
+      repository: pytorch/vision
+      token: ${{ secrets.GH_PYTORCHBOT_TOKEN }}
+  tag_nightly_audio:
+    if: ${{ github.event_name == 'schedule' ||  inputs.domain == 'audio' || inputs.domain == 'all' }}
+    name: Trigger nightly audio build
+    uses: ./test-infra/.github/actions/trigger-nightly
+    with:
+      ref: main
+      repository: pytorch/audio
+      token: ${{ secrets.GH_PYTORCHBOT_TOKEN }}