From 26dc645a2872b17b3a1341d5377c25b8d0ce2dbb Mon Sep 17 00:00:00 2001
From: Thanh Ha <thanh.ha@linuxfoundation.org>
Date: Fri, 29 Aug 2025 12:26:36 -0400
Subject: [PATCH] Add check for Intel runners

Also add `.idc.` to the Meta runners excludes list.

Signed-off-by: Thanh Ha <thanh.ha@linuxfoundation.org>
---
 datadog-synthetics_tests.tf       | 29 +++++++++++++++++++++++++++++
 scripts/check-long-queue-intel.js | 19 +++++++++++++++++++
 scripts/check-long-queue-meta.js  |  2 +-
 3 files changed, 49 insertions(+), 1 deletion(-)
 create mode 100644 scripts/check-long-queue-intel.js

diff --git a/datadog-synthetics_tests.tf b/datadog-synthetics_tests.tf
index 08fde64..53af083 100644
--- a/datadog-synthetics_tests.tf
+++ b/datadog-synthetics_tests.tf
@@ -399,6 +399,35 @@ EOT
   }
 }
 
+resource "datadog_synthetics_test" "pytorch-gha-runners-queue-check-intel" {
+  type      = "api"
+  name      = "GHA Runner Queue Check - Intel Runners"
+  message   = <<EOT
+Detected GitHub Runner Queue - Intel Runners has jobs waiting
+unusually long for runners.
+
+{{{synthetics.attributes.result.failure.message}}}
+
+Check https://hud.pytorch.org/metrics for more details.
+
+@slack-pytorch-infra-alerts
+EOT
+  status    = "live"
+  tags      = ["env:project", "project:pytorch", "service:gha-runners"]
+  locations = ["aws:us-west-2"]
+  options_list {
+    tick_every = 900
+  }
+  request_definition {
+    method = "GET"
+    url    = "https://hud.pytorch.org/api/clickhouse/queued_jobs_by_label?parameters=%7B%7D"
+  }
+  assertion {
+    type = "javascript"
+    code = file("scripts/check-long-queue-intel.js")
+  }
+}
+
 resource "datadog_synthetics_test" "pytorch-gha-runners-queue-check-meta" {
   type      = "api"
   name      = "GHA Runner Queue Check - Meta Runners"
diff --git a/scripts/check-long-queue-intel.js b/scripts/check-long-queue-intel.js
new file mode 100644
index 0000000..40cc5a6
--- /dev/null
+++ b/scripts/check-long-queue-intel.js
@@ -0,0 +1,19 @@
+dd.expect(dd.response.statusCode).to.equal(200);
+
+const MACHINE_TYPE_FILTER = '.idc.';
+const jsonData = dd.response.body;
+const parsedData = JSON.parse(jsonData);
+
+const highQueueItems = parsedData
+  .filter(item => item.machine_type.includes(MACHINE_TYPE_FILTER) && item.avg_queue_s > 10800)
+  .map(item => ({ machine_type: item.machine_type, avg_queue_s: item.avg_queue_s }));
+
+if (highQueueItems.length > 0) {
+  const machineDetails = highQueueItems
+    .map(item => `${item.machine_type} (${item.avg_queue_s}s)`)
+    .join(', ');
+  const message = `High queue detected for machine types containing ${MACHINE_TYPE_FILTER}: ${machineDetails}`;
+  console.error(message);
+}
+
+dd.expect(highQueueItems.length > 0).to.be.false;
diff --git a/scripts/check-long-queue-meta.js b/scripts/check-long-queue-meta.js
index c62c8f5..6ee37a6 100644
--- a/scripts/check-long-queue-meta.js
+++ b/scripts/check-long-queue-meta.js
@@ -1,5 +1,5 @@
 dd.expect(dd.response.statusCode).to.equal(200);
-const EXCLUDED_MACHINE_PATTERNS = ['.dgx.', '.rocm.', '.s390x', '^lf\\.', '^linux.aws.h100'];
+const EXCLUDED_MACHINE_PATTERNS = ['.dgx.', '.idc.', '.rocm.', '.s390x', '^lf\\.', '^linux.aws.h100'];
 const jsonData = dd.response.body;
 const parsedData = JSON.parse(jsonData);
 const highQueueItems = parsedData