diff --git a/.circleci/config.yml b/.circleci/config.yml
index f8d6503f6e4b..a7bb13a60a1b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -25,6 +25,14 @@ one_gpu: &one_gpu
   # https://circleci.com/product/features/resource-classes/#linux-vm
   resource_class: gpu.small
 
+
+one_gpu_windows: &one_gpu_windows
+  machine:
+    resource_class: windows.gpu.nvidia.medium
+    image: windows-server-2019-nvidia:stable
+    shell: bash.exe
+
+
 two_gpus: &two_gpus
   machine:
     # https://circleci.com/docs/2.0/configuration-reference/#available-linux-gpu-images
@@ -100,7 +108,7 @@ jobs:
       - <<: *run_pytorch_container
       - <<: *install_dependencies
       - run:
-          name: Run GPU Unit Tests
+          name: Run GPU Unit Tests and Examples
           command: |
 
             # pytest on cuda
@@ -146,6 +154,32 @@ jobs:
             bash <(curl -s https://codecov.io/bash) -Z -F gpu
 
 
+  one_gpu_windows_tests:
+    <<: *one_gpu_windows
+
+    working_directory: << pipeline.parameters.workingdir >>
+
+    steps:
+      - checkout
+      - run:
+          name: Trigger job if modified
+          command: |
+            bash .circleci/trigger_if_modified.sh "^(ignite|tests|examples|\.circleci).*"
+
+      - run:
+          name: Install dependencies
+          command: |
+            conda --version
+            conda install -y pytorch torchvision cudatoolkit=10.1 -c pytorch
+            pip install -r requirements-dev.txt
+            pip install .
+
+      - run:
+          name: Run GPU Unit Tests
+          command: |
+            # pytest on cuda
+            SKIP_DISTRIB_TESTS=1 bash tests/run_gpu_tests.sh
+
   two_gpus_tests:
     <<: *two_gpus
 
@@ -302,6 +336,7 @@ workflows:
 
     jobs:
       - one_gpu_tests
+      - one_gpu_windows_tests
       - two_gpus_tests
       - two_gpus_check_dist_cifar10_example
       - two_gpus_hvd_tests
diff --git a/tests/run_gpu_tests.sh b/tests/run_gpu_tests.sh
index e773f8d5965d..ce06bbe06c8a 100644
--- a/tests/run_gpu_tests.sh
+++ b/tests/run_gpu_tests.sh
@@ -10,6 +10,11 @@ set -xeu
 
 py.test --cov ignite --cov-report term-missing --cov-report xml -vvv tests/ -k 'on_cuda'
 
+# https://pubs.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_06_02
+if [ "${SKIP_DISTRIB_TESTS:-0}" -eq "1" ]; then
+    exit 0
+fi
+
 py.test --cov ignite --cov-append --cov-report term-missing --cov-report xml -vvv tests/ -m distributed