From 001474cb5debf0b0472b35335b43e4596a87af70 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 11:32:37 +0800
Subject: [PATCH 01/33] first

---
 .github/workflows/examples-ci.yml             | 28 +++++++++++++++++++
 .../tests/quick_quick_dataset_analysis.py     | 16 +++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 .github/workflows/examples-ci.yml
 create mode 100644 .github/workflows/tests/quick_quick_dataset_analysis.py

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
new file mode 100644
index 00000000..c1268a1b
--- /dev/null
+++ b/.github/workflows/examples-ci.yml
@@ -0,0 +1,28 @@
+name: Test Run Examples
+
+on: 
+  push:
+
+jobs:
+  test-quick-dataset-analysis:
+  runs-on: ubuntu-latest
+  steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+      with:
+           fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.9'
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install fastdup
+
+    - name: Run example
+      run: |
+        export SENTRY_OPT_OUT=True
+        python .github/workflows/tests/test_quick_dataset_analysis.py
diff --git a/.github/workflows/tests/quick_quick_dataset_analysis.py b/.github/workflows/tests/quick_quick_dataset_analysis.py
new file mode 100644
index 00000000..9e120d0a
--- /dev/null
+++ b/.github/workflows/tests/quick_quick_dataset_analysis.py
@@ -0,0 +1,16 @@
+import os
+import subprocess
+
+def callsh(command):
+  status = subprocess.run(command)
+  status.check_returncode()
+  print(status.stdout)
+
+callsh(['wget', 'https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz', '-O', 'images.tar.gz'])
+callsh(['tar', 'xf', 'images.tar.gz'])
+
+import fastdup
+print(f'fastdup version: {fastdup.__version__}')
+
+fd = fastdup.create(work_dir="fastdup_work_dir/", input_dir="images/")
+fd.run(num_images=1000)
\ No newline at end of file

From 6da3799880d975d0437e954fa944d3b989a8767c Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 11:34:42 +0800
Subject: [PATCH 02/33] update

---
 .github/workflows/examples-ci.yml | 36 +++++++++++++++----------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index c1268a1b..ac30376c 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -6,23 +6,23 @@ on:
 jobs:
   test-quick-dataset-analysis:
   runs-on: ubuntu-latest
-  steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-      with:
-           fetch-depth: 0
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+            fetch-depth: 0
 
-    - name: Set up Python
-      uses: actions/setup-python@v3
-      with:
-        python-version: '3.9'
-    
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install fastdup
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: '3.9'
+      
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install fastdup
 
-    - name: Run example
-      run: |
-        export SENTRY_OPT_OUT=True
-        python .github/workflows/tests/test_quick_dataset_analysis.py
+      - name: Run example
+        run: |
+          export SENTRY_OPT_OUT=True
+          python .github/workflows/tests/test_quick_dataset_analysis.py

From 692df7bcdce85bf2cc5ff64d255e78e01369f28e Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 11:36:12 +0800
Subject: [PATCH 03/33] opt

---
 .github/workflows/examples-ci.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index ac30376c..9dac015c 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -5,7 +5,9 @@ on:
 
 jobs:
   test-quick-dataset-analysis:
-  runs-on: ubuntu-latest
+    runs-on: ubuntu-latest
+    env:
+        SENTRY_OPT_OUT: True
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -24,5 +26,4 @@ jobs:
 
       - name: Run example
         run: |
-          export SENTRY_OPT_OUT=True
           python .github/workflows/tests/test_quick_dataset_analysis.py

From d9e3926f547df02aefbce64b56ce94f7d079a591 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 11:38:49 +0800
Subject: [PATCH 04/33] update filename

---
 .github/workflows/examples-ci.yml                               | 2 +-
 ...uick_quick_dataset_analysis.py => quick_dataset_analysis.py} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename .github/workflows/tests/{quick_quick_dataset_analysis.py => quick_dataset_analysis.py} (100%)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 9dac015c..7a5a6417 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -26,4 +26,4 @@ jobs:
 
       - name: Run example
         run: |
-          python .github/workflows/tests/test_quick_dataset_analysis.py
+          python .github/workflows/tests/quick_dataset_analysis.py
diff --git a/.github/workflows/tests/quick_quick_dataset_analysis.py b/.github/workflows/tests/quick_dataset_analysis.py
similarity index 100%
rename from .github/workflows/tests/quick_quick_dataset_analysis.py
rename to .github/workflows/tests/quick_dataset_analysis.py

From bf27550cc48314a1ce92616b88770766e3fecfae Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 11:42:17 +0800
Subject: [PATCH 05/33] download datasets

---
 .github/workflows/examples-ci.yml                |  5 +++++
 .../workflows/tests/quick_dataset_analysis.py    | 16 ++++++++--------
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 7a5a6417..214c0151 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -24,6 +24,11 @@ jobs:
           python -m pip install --upgrade pip
           pip install fastdup
 
+      - name: Download dataset
+        run: |
+          wget https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz -O images.tar.gz
+          tar xf images.tar.gz
+
       - name: Run example
         run: |
           python .github/workflows/tests/quick_dataset_analysis.py
diff --git a/.github/workflows/tests/quick_dataset_analysis.py b/.github/workflows/tests/quick_dataset_analysis.py
index 9e120d0a..1aaa4e45 100644
--- a/.github/workflows/tests/quick_dataset_analysis.py
+++ b/.github/workflows/tests/quick_dataset_analysis.py
@@ -1,13 +1,13 @@
-import os
-import subprocess
+# import os
+# import subprocess
 
-def callsh(command):
-  status = subprocess.run(command)
-  status.check_returncode()
-  print(status.stdout)
+# def callsh(command):
+#   status = subprocess.run(command)
+#   status.check_returncode()
+#   print(status.stdout)
 
-callsh(['wget', 'https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz', '-O', 'images.tar.gz'])
-callsh(['tar', 'xf', 'images.tar.gz'])
+# callsh(['wget', 'https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz', '-O', 'images.tar.gz'])
+# callsh(['tar', 'xf', 'images.tar.gz'])
 
 import fastdup
 print(f'fastdup version: {fastdup.__version__}')

From cbd86d20d903db09b151d7be3cde0b06b7c9996f Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 11:47:11 +0800
Subject: [PATCH 06/33] save artifact

---
 .github/workflows/examples-ci.yml             |  8 +++++++-
 .../workflows/tests/quick_dataset_analysis.py | 19 +++++++------------
 2 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 214c0151..137bca74 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -22,7 +22,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install fastdup
+          pip install fastdup matplotlib
 
       - name: Download dataset
         run: |
@@ -32,3 +32,9 @@ jobs:
       - name: Run example
         run: |
           python .github/workflows/tests/quick_dataset_analysis.py
+      
+      - name: Save artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: fastdup_work_dir
+          path: fastdup_work_dir/
diff --git a/.github/workflows/tests/quick_dataset_analysis.py b/.github/workflows/tests/quick_dataset_analysis.py
index 1aaa4e45..92e3fdf8 100644
--- a/.github/workflows/tests/quick_dataset_analysis.py
+++ b/.github/workflows/tests/quick_dataset_analysis.py
@@ -1,16 +1,11 @@
-# import os
-# import subprocess
-
-# def callsh(command):
-#   status = subprocess.run(command)
-#   status.check_returncode()
-#   print(status.stdout)
-
-# callsh(['wget', 'https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz', '-O', 'images.tar.gz'])
-# callsh(['tar', 'xf', 'images.tar.gz'])
-
 import fastdup
 print(f'fastdup version: {fastdup.__version__}')
 
 fd = fastdup.create(work_dir="fastdup_work_dir/", input_dir="images/")
-fd.run(num_images=1000)
\ No newline at end of file
+fd.run(num_images=10000)
+
+fd.vis.duplicates_gallery()
+fd.vis.outliers_gallery() 
+fd.vis.stats_gallery(metric='dark')
+fd.vis.component_gallery()
+fd.vis.similarity_gallery()
\ No newline at end of file

From fafc633b91e38166614d6beda4088479b534c3d1 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 11:53:23 +0800
Subject: [PATCH 07/33] add matrix

---
 .github/workflows/examples-ci.yml | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 137bca74..7bdcd195 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -2,12 +2,19 @@ name: Test Run Examples
 
 on: 
   push:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 8 * * *' # Trigger at 8 AM every day
 
 jobs:
   test-quick-dataset-analysis:
-    runs-on: ubuntu-latest
+    runs-on: runs-on: ${{ matrix.os }}
     env:
         SENTRY_OPT_OUT: True
+    strategy:
+        matrix:
+            os: [ubuntu-latest, macos-latest, windows-latest]
+            python-version: ['3.9', '3.10']
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -17,7 +24,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v3
         with:
-          python-version: '3.9'
+          python-version: ${{ matrix.python-version }}
       
       - name: Install dependencies
         run: |

From bf1bf92de6bc18314386582d79506e26a0a101a3 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 11:53:49 +0800
Subject: [PATCH 08/33] run on

---
 .github/workflows/examples-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 7bdcd195..65d25bc3 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -8,7 +8,7 @@ on:
 
 jobs:
   test-quick-dataset-analysis:
-    runs-on: runs-on: ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
     env:
         SENTRY_OPT_OUT: True
     strategy:

From 48cd51ab09ad55b41d71cf58f85cd96b17091057 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 11:56:03 +0800
Subject: [PATCH 09/33] remove python 310

---
 .github/workflows/examples-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 65d25bc3..61542545 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -14,7 +14,7 @@ jobs:
     strategy:
         matrix:
             os: [ubuntu-latest, macos-latest, windows-latest]
-            python-version: ['3.9', '3.10']
+            python-version: ['3.9']
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3

From 2a6ada7dd8fd460e688d5022647887ec0e24e46c Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 12:05:25 +0800
Subject: [PATCH 10/33] specify matplotlib

---
 .github/workflows/examples-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 61542545..16c1fa8e 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -29,7 +29,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install fastdup matplotlib
+          pip install fastdup matplotlib==3.7.1
 
       - name: Download dataset
         run: |

From af0386c2548bedbcb8b9bd1690e64d07ca1316e6 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 12:10:19 +0800
Subject: [PATCH 11/33] windows wget

---
 .github/workflows/examples-ci.yml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 16c1fa8e..8a501013 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -29,12 +29,17 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install fastdup matplotlib==3.7.1
+          pip install fastdup matplotlib==3.6.3
 
       - name: Download dataset
         run: |
-          wget https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz -O images.tar.gz
-          tar xf images.tar.gz
+          if ($env:RUNNER_OS -eq "Windows") {
+            Invoke-WebRequest -Uri "https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz" -OutFile "images.tar.gz"
+            Expand-Archive -Path "images.tar.gz" -DestinationPath "path/to/dataset"
+          } else {
+            wget "https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz" -O "images.tar.gz"
+            tar xf "images.tar.gz" -C "path/to/dataset"
+          }
 
       - name: Run example
         run: |

From fd57ae51590b3495efd3ae0a1df300bb6d0cd6e6 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 12:14:17 +0800
Subject: [PATCH 12/33] matrix

---
 .github/workflows/examples-ci.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 8a501013..42edfc75 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -13,7 +13,7 @@ jobs:
         SENTRY_OPT_OUT: True
     strategy:
         matrix:
-            os: [ubuntu-latest, macos-latest, windows-latest]
+            os: [ubuntu-latest, windows-latest]
             python-version: ['3.9']
     steps:
       - name: Checkout repository
@@ -33,13 +33,13 @@ jobs:
 
       - name: Download dataset
         run: |
-          if ($env:RUNNER_OS -eq "Windows") {
+          if [ "${{ matrix.os }}" == "windows-latest" ]; then
             Invoke-WebRequest -Uri "https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz" -OutFile "images.tar.gz"
             Expand-Archive -Path "images.tar.gz" -DestinationPath "path/to/dataset"
-          } else {
+          else
             wget "https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz" -O "images.tar.gz"
             tar xf "images.tar.gz" -C "path/to/dataset"
-          }
+          fi
 
       - name: Run example
         run: |

From e2d7bcb5833ff9b643e1ae9b6eea9b27bb6e01f5 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 12:16:56 +0800
Subject: [PATCH 13/33] ubuntu

---
 .github/workflows/examples-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 42edfc75..929811ed 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -13,7 +13,7 @@ jobs:
         SENTRY_OPT_OUT: True
     strategy:
         matrix:
-            os: [ubuntu-latest, windows-latest]
+            os: [ubuntu-latest]
             python-version: ['3.9']
     steps:
       - name: Checkout repository

From 382100f47f1c249c961bc078e4092cf5c66efe2b Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 12:19:11 +0800
Subject: [PATCH 14/33] update

---
 .github/workflows/examples-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 929811ed..b372b702 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -35,10 +35,10 @@ jobs:
         run: |
           if [ "${{ matrix.os }}" == "windows-latest" ]; then
             Invoke-WebRequest -Uri "https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz" -OutFile "images.tar.gz"
-            Expand-Archive -Path "images.tar.gz" -DestinationPath "path/to/dataset"
+            Expand-Archive -Path "images.tar.gz"
           else
             wget "https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz" -O "images.tar.gz"
-            tar xf "images.tar.gz" -C "path/to/dataset"
+            tar xf "images.tar.gz""
           fi
 
       - name: Run example

From c065e0d4d8b9238e8969a3c0b00f0f60399096e4 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 12:20:53 +0800
Subject: [PATCH 15/33] ubuntu

---
 .github/workflows/examples-ci.yml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index b372b702..a26c826b 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -33,13 +33,9 @@ jobs:
 
       - name: Download dataset
         run: |
-          if [ "${{ matrix.os }}" == "windows-latest" ]; then
-            Invoke-WebRequest -Uri "https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz" -OutFile "images.tar.gz"
-            Expand-Archive -Path "images.tar.gz"
-          else
             wget "https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz" -O "images.tar.gz"
             tar xf "images.tar.gz""
-          fi
+
 
       - name: Run example
         run: |

From 44e1600d9ebd3cd43cb8a6c5e4ddf6cfdc1874d3 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 12:22:56 +0800
Subject: [PATCH 16/33] error

---
 .github/workflows/examples-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index a26c826b..88802263 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -34,7 +34,7 @@ jobs:
       - name: Download dataset
         run: |
             wget "https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz" -O "images.tar.gz"
-            tar xf "images.tar.gz""
+            tar xf "images.tar.gz"
 
 
       - name: Run example

From e93b3c31359facea33e7eee845f71037feed2415 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 12:30:34 +0800
Subject: [PATCH 17/33] cleaning image

---
 .github/workflows/examples-ci.yml             | 42 ++++++++++++++++++-
 .../workflows/tests/cleaning_image_dataset.py | 12 ++++++
 2 files changed, 52 insertions(+), 2 deletions(-)
 create mode 100644 .github/workflows/tests/cleaning_image_dataset.py

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 88802263..4f695701 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -29,14 +29,13 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install fastdup matplotlib==3.6.3
+          pip install fastdup matplotlib
 
       - name: Download dataset
         run: |
             wget "https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz" -O "images.tar.gz"
             tar xf "images.tar.gz"
 
-
       - name: Run example
         run: |
           python .github/workflows/tests/quick_dataset_analysis.py
@@ -46,3 +45,42 @@ jobs:
         with:
           name: fastdup_work_dir
           path: fastdup_work_dir/
+
+  test-cleaning-image-dtaset:
+    runs-on: ${{ matrix.os }}
+    env:
+        SENTRY_OPT_OUT: True
+    strategy:
+        matrix:
+            os: [ubuntu-latest]
+            python-version: ['3.9']
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+            fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install fastdup matplotlib
+
+      - name: Download dataset
+        run: |
+            wget http://data.vision.ee.ethz.ch/cvl/food-101.tar.gz
+            tar -xf food-101.tar.gz
+
+      - name: Run example
+        run: |
+          python .github/workflows/tests/cleaning_image_dataset.py
+      
+      - name: Save artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: fastdup_work_dir
+          path: fastdup_work_dir/
diff --git a/.github/workflows/tests/cleaning_image_dataset.py b/.github/workflows/tests/cleaning_image_dataset.py
new file mode 100644
index 00000000..00854d18
--- /dev/null
+++ b/.github/workflows/tests/cleaning_image_dataset.py
@@ -0,0 +1,12 @@
+import fastdup
+print(f'fastdup version: {fastdup.__version__}')
+
+fd = fastdup.create(work_dir="fastdup_work_dir/", input_dir="food-101/images/")
+fd.run(num_images=1000)
+
+fd.vis.duplicates_gallery(num_images=5)
+fd.vis.component_gallery(num_images=5)
+fd.vis.outliers_gallery(num_images=5)
+fd.vis.stats_gallery(metric='dark', num_images=5)
+fd.vis.stats_gallery(metric='bright', num_images=5)
+fd.vis.stats_gallery(metric='blur', num_images=5)
\ No newline at end of file

From 6cbe16384cf1f07d6d7c476f5becdaadc87f16e6 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 12:40:30 +0800
Subject: [PATCH 18/33] common

---
 .github/workflows/examples-ci.yml | 131 ++++++++++++++++++++++++------
 1 file changed, 105 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 4f695701..a25c17ee 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -6,31 +6,120 @@ on:
   schedule:
     - cron: '0 8 * * *' # Trigger at 8 AM every day
 
+# jobs:
+#   test-quick-dataset-analysis:
+#     runs-on: ${{ matrix.os }}
+#     env:
+#         SENTRY_OPT_OUT: True
+#     strategy:
+#         matrix:
+#             os: [ubuntu-latest]
+#             python-version: ['3.9']
+#     steps:
+#       - name: Checkout repository
+#         uses: actions/checkout@v3
+#         with:
+#             fetch-depth: 0
+
+#       - name: Set up Python
+#         uses: actions/setup-python@v3
+#         with:
+#           python-version: ${{ matrix.python-version }}
+      
+#       - name: Install dependencies
+#         run: |
+#           python -m pip install --upgrade pip
+#           pip install fastdup matplotlib
+
+#       - name: Download dataset
+#         run: |
+#             wget "https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz" -O "images.tar.gz"
+#             tar xf "images.tar.gz"
+
+#       - name: Run example
+#         run: |
+#           python .github/workflows/tests/quick_dataset_analysis.py
+      
+#       - name: Save artifacts
+#         uses: actions/upload-artifact@v3
+#         with:
+#           name: fastdup_work_dir
+#           path: fastdup_work_dir/
+
+#   test-cleaning-image-dtaset:
+#     runs-on: ${{ matrix.os }}
+#     env:
+#         SENTRY_OPT_OUT: True
+#     strategy:
+#         matrix:
+#             os: [ubuntu-latest]
+#             python-version: ['3.9']
+#     steps:
+#       - name: Checkout repository
+#         uses: actions/checkout@v3
+#         with:
+#             fetch-depth: 0
+
+#       - name: Set up Python
+#         uses: actions/setup-python@v3
+#         with:
+#           python-version: ${{ matrix.python-version }}
+      
+#       - name: Install dependencies
+#         run: |
+#           python -m pip install --upgrade pip
+#           pip install fastdup matplotlib
+
+#       - name: Download dataset
+#         run: |
+#             wget http://data.vision.ee.ethz.ch/cvl/food-101.tar.gz
+#             tar -xf food-101.tar.gz
+
+#       - name: Run example
+#         run: |
+#           python .github/workflows/tests/cleaning_image_dataset.py
+      
+#       - name: Save artifacts
+#         uses: actions/upload-artifact@v3
+#         with:
+#           name: fastdup_work_dir
+#           path: fastdup_work_dir/
+
+
+
+# reusable actions
 jobs:
-  test-quick-dataset-analysis:
+  common-steps:
     runs-on: ${{ matrix.os }}
-    env:
-        SENTRY_OPT_OUT: True
-    strategy:
-        matrix:
-            os: [ubuntu-latest]
-            python-version: ['3.9']
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
         with:
-            fetch-depth: 0
+          fetch-depth: 0
 
       - name: Set up Python
         uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
-      
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install fastdup matplotlib
 
+# jobs using common steps
+jobs:
+  test-quick-dataset-analysis:
+    runs-on: ${{ matrix.os }}
+    env:
+        SENTRY_OPT_OUT: True
+    strategy:
+        matrix:
+            os: [ubuntu-latest]
+            python-version: ['3.9']
+    steps:
+      - uses: common-steps
+
       - name: Download dataset
         run: |
             wget "https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz" -O "images.tar.gz"
@@ -43,10 +132,10 @@ jobs:
       - name: Save artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: fastdup_work_dir
+          name: fastdup_work_dir_quick
           path: fastdup_work_dir/
 
-  test-cleaning-image-dtaset:
+  test-cleaning-image-dataset:
     runs-on: ${{ matrix.os }}
     env:
         SENTRY_OPT_OUT: True
@@ -55,20 +144,7 @@ jobs:
             os: [ubuntu-latest]
             python-version: ['3.9']
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-        with:
-            fetch-depth: 0
-
-      - name: Set up Python
-        uses: actions/setup-python@v3
-        with:
-          python-version: ${{ matrix.python-version }}
-      
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install fastdup matplotlib
+      - uses: common-steps
 
       - name: Download dataset
         run: |
@@ -82,5 +158,8 @@ jobs:
       - name: Save artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: fastdup_work_dir
+          name: fastdup_work_dir_cleaning
           path: fastdup_work_dir/
+
+
+      
\ No newline at end of file

From d0e7a0e218583ad24ef889d1d7b5e6d68e47d250 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 12:43:25 +0800
Subject: [PATCH 19/33] common

---
 .github/workflows/examples-ci.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index a25c17ee..826b950b 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -107,8 +107,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install fastdup matplotlib
 
-# jobs using common steps
-jobs:
+
   test-quick-dataset-analysis:
     runs-on: ${{ matrix.os }}
     env:

From 590047b5de976c79a1d6c3f966aba88dec396c63 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 12:46:46 +0800
Subject: [PATCH 20/33] needs

---
 .github/workflows/examples-ci.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 826b950b..6ebed8fc 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -109,6 +109,7 @@ jobs:
 
 
   test-quick-dataset-analysis:
+    needs: common-steps
     runs-on: ${{ matrix.os }}
     env:
         SENTRY_OPT_OUT: True
@@ -117,8 +118,6 @@ jobs:
             os: [ubuntu-latest]
             python-version: ['3.9']
     steps:
-      - uses: common-steps
-
       - name: Download dataset
         run: |
             wget "https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz" -O "images.tar.gz"
@@ -135,6 +134,7 @@ jobs:
           path: fastdup_work_dir/
 
   test-cleaning-image-dataset:
+    needs: common-steps
     runs-on: ${{ matrix.os }}
     env:
         SENTRY_OPT_OUT: True
@@ -143,8 +143,6 @@ jobs:
             os: [ubuntu-latest]
             python-version: ['3.9']
     steps:
-      - uses: common-steps
-
       - name: Download dataset
         run: |
             wget http://data.vision.ee.ethz.ch/cvl/food-101.tar.gz

From a6c16f49dcf2f4923769ca300ae957bdfde1357d Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 12:48:47 +0800
Subject: [PATCH 21/33] matrix

---
 .github/workflows/examples-ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 6ebed8fc..94228ca3 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -91,6 +91,9 @@ on:
 jobs:
   common-steps:
     runs-on: ${{ matrix.os }}
+    strategy:
+        matrix:
+            os: [ubuntu-latest]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3

From e3dcac71829a0f44756687767ef38808745d8b67 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 13:16:41 +0800
Subject: [PATCH 22/33] revert

---
 .github/workflows/examples-ci.yml | 127 ++++++------------------------
 1 file changed, 24 insertions(+), 103 deletions(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 94228ca3..26658f4d 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -6,121 +6,31 @@ on:
   schedule:
     - cron: '0 8 * * *' # Trigger at 8 AM every day
 
-# jobs:
-#   test-quick-dataset-analysis:
-#     runs-on: ${{ matrix.os }}
-#     env:
-#         SENTRY_OPT_OUT: True
-#     strategy:
-#         matrix:
-#             os: [ubuntu-latest]
-#             python-version: ['3.9']
-#     steps:
-#       - name: Checkout repository
-#         uses: actions/checkout@v3
-#         with:
-#             fetch-depth: 0
-
-#       - name: Set up Python
-#         uses: actions/setup-python@v3
-#         with:
-#           python-version: ${{ matrix.python-version }}
-      
-#       - name: Install dependencies
-#         run: |
-#           python -m pip install --upgrade pip
-#           pip install fastdup matplotlib
-
-#       - name: Download dataset
-#         run: |
-#             wget "https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz" -O "images.tar.gz"
-#             tar xf "images.tar.gz"
-
-#       - name: Run example
-#         run: |
-#           python .github/workflows/tests/quick_dataset_analysis.py
-      
-#       - name: Save artifacts
-#         uses: actions/upload-artifact@v3
-#         with:
-#           name: fastdup_work_dir
-#           path: fastdup_work_dir/
-
-#   test-cleaning-image-dtaset:
-#     runs-on: ${{ matrix.os }}
-#     env:
-#         SENTRY_OPT_OUT: True
-#     strategy:
-#         matrix:
-#             os: [ubuntu-latest]
-#             python-version: ['3.9']
-#     steps:
-#       - name: Checkout repository
-#         uses: actions/checkout@v3
-#         with:
-#             fetch-depth: 0
-
-#       - name: Set up Python
-#         uses: actions/setup-python@v3
-#         with:
-#           python-version: ${{ matrix.python-version }}
-      
-#       - name: Install dependencies
-#         run: |
-#           python -m pip install --upgrade pip
-#           pip install fastdup matplotlib
-
-#       - name: Download dataset
-#         run: |
-#             wget http://data.vision.ee.ethz.ch/cvl/food-101.tar.gz
-#             tar -xf food-101.tar.gz
-
-#       - name: Run example
-#         run: |
-#           python .github/workflows/tests/cleaning_image_dataset.py
-      
-#       - name: Save artifacts
-#         uses: actions/upload-artifact@v3
-#         with:
-#           name: fastdup_work_dir
-#           path: fastdup_work_dir/
-
-
-
-# reusable actions
 jobs:
-  common-steps:
+  test-quick-dataset-analysis:
     runs-on: ${{ matrix.os }}
+    env:
+        SENTRY_OPT_OUT: True
     strategy:
         matrix:
             os: [ubuntu-latest]
+            python-version: ['3.9']
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+            fetch-depth: 0
 
       - name: Set up Python
         uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
-
+      
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install fastdup matplotlib
 
-
-  test-quick-dataset-analysis:
-    needs: common-steps
-    runs-on: ${{ matrix.os }}
-    env:
-        SENTRY_OPT_OUT: True
-    strategy:
-        matrix:
-            os: [ubuntu-latest]
-            python-version: ['3.9']
-    steps:
       - name: Download dataset
         run: |
             wget "https://thor.robots.ox.ac.uk/~vgg/data/pets/images.tar.gz" -O "images.tar.gz"
@@ -133,11 +43,10 @@ jobs:
       - name: Save artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: fastdup_work_dir_quick
+          name: fastdup_work_dir
           path: fastdup_work_dir/
 
   test-cleaning-image-dataset:
-    needs: common-steps
     runs-on: ${{ matrix.os }}
     env:
         SENTRY_OPT_OUT: True
@@ -146,6 +55,21 @@ jobs:
             os: [ubuntu-latest]
             python-version: ['3.9']
     steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+            fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install fastdup matplotlib
+
       - name: Download dataset
         run: |
             wget http://data.vision.ee.ethz.ch/cvl/food-101.tar.gz
@@ -158,8 +82,5 @@ jobs:
       - name: Save artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: fastdup_work_dir_cleaning
-          path: fastdup_work_dir/
-
-
-      
\ No newline at end of file
+          name: fastdup_work_dir
+          path: fastdup_work_dir/
\ No newline at end of file

From 2efa2b3da4541eed7d8a88c25b4687924eca7daf Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 13:17:26 +0800
Subject: [PATCH 23/33] rename artifact

---
 .github/workflows/examples-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 26658f4d..e2d610f0 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -43,7 +43,7 @@ jobs:
       - name: Save artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: fastdup_work_dir
+          name: fastdup_work_dir_quick_dataset_analysis
           path: fastdup_work_dir/
 
   test-cleaning-image-dataset:
@@ -82,5 +82,5 @@ jobs:
       - name: Save artifacts
         uses: actions/upload-artifact@v3
         with:
-          name: fastdup_work_dir
+          name: fastdup_work_dir_cleaning_image_dataset
           path: fastdup_work_dir/
\ No newline at end of file

From 0122b1e09b20fe09398d21e418ac1fc1ea20bf16 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 13:22:47 +0800
Subject: [PATCH 24/33] add classification and object det

---
 .github/workflows/examples-ci.yml             | 79 +++++++++++++++++++
 .../tests/labeled_image_classification.py     | 50 ++++++++++++
 .../tests/labeled_object_detection.py         | 21 +++++
 3 files changed, 150 insertions(+)
 create mode 100644 .github/workflows/tests/labeled_image_classification.py
 create mode 100644 .github/workflows/tests/labeled_object_detection.py

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index e2d610f0..628d1458 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -83,4 +83,83 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: fastdup_work_dir_cleaning_image_dataset
+          path: fastdup_work_dir/
+
+  test-labeled-image-classification:
+    runs-on: ${{ matrix.os }}
+    env:
+        SENTRY_OPT_OUT: True
+    strategy:
+        matrix:
+            os: [ubuntu-latest]
+            python-version: ['3.9']
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+            fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install fastdup matplotlib
+
+      - name: Download dataset
+        run: |
+            wget https://s3.amazonaws.com/fast-ai-imageclas/imagenette2-160.tgz
+            tar -xf imagenette2-160.tgz
+
+      - name: Run example
+        run: |
+          python .github/workflows/tests/labeled_image_classification.py
+      
+      - name: Save artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: fastdup_work_dir_labeled_image_classification
+          path: fastdup_work_dir/
+
+  test-labeled-object-detection:
+    runs-on: ${{ matrix.os }}
+    env:
+        SENTRY_OPT_OUT: True
+    strategy:
+        matrix:
+            os: [ubuntu-latest]
+            python-version: ['3.9']
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+            fetch-depth: 0
+
+      - name: Set up Python
+        uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+      
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install fastdup fastdup plotly gdown
+
+      - name: Download dataset
+        run: |
+            gdown --fuzzy https://drive.google.com/file/d/1iSXVTlkV1_DhdYpVDqsjlT4NJFQ7OkyK/view
+            unzip -qq coco_minitrain_25k.zip
+            cd coco_minitrain_25k/annotations && gdown --fuzzy https://drive.google.com/file/d/1i12p23cXlqp1QrXjAD_vu467r4q67Mq9/view
+
+      - name: Run example
+        run: |
+          python .github/workflows/tests/labeled_object_detection.py
+      
+      - name: Save artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: fastdup_work_dir_labeled_object_detection
           path: fastdup_work_dir/
\ No newline at end of file
diff --git a/.github/workflows/tests/labeled_image_classification.py b/.github/workflows/tests/labeled_image_classification.py
new file mode 100644
index 00000000..af9c4083
--- /dev/null
+++ b/.github/workflows/tests/labeled_image_classification.py
@@ -0,0 +1,50 @@
+import pandas as pd
+data_dir = 'imagenette2-160/'
+csv_path = 'imagenette2-160/noisy_imagenette.csv'
+
+label_map = {
+    'n02979186': 'cassette_player', 
+    'n03417042': 'garbage_truck', 
+    'n01440764': 'tench', 
+    'n02102040': 'English_springer', 
+    'n03028079': 'church',
+    'n03888257': 'parachute', 
+    'n03394916': 'French_horn', 
+    'n03000684': 'chain_saw', 
+    'n03445777': 'golf_ball', 
+    'n03425413': 'gas_pump'
+}
+
+df_annot = pd.read_csv(csv_path)
+# take relevant columns
+df_annot = df_annot[['path', 'noisy_labels_0']]
+
+# rename columns to fastdup's column names
+df_annot = df_annot.rename({'noisy_labels_0': 'label', 'path': 'filename'}, axis='columns')
+
+# append datadir
+df_annot['filename'] = df_annot['filename'].apply(lambda x: data_dir + x)
+
+# create split column
+df_annot['split'] = df_annot['filename'].apply(lambda x: x.split("/")[1])
+
+# map label ids to regular labels
+df_annot['label'] = df_annot['label'].map(label_map)
+
+
+import fastdup
+print(f'fastdup version: {fastdup.__version__}')
+
+work_dir = 'fastdup_imagenette'
+fd = fastdup.create(work_dir=work_dir, input_dir=data_dir) 
+fd.run(annotations=df_annot, ccthreshold=0.9, threshold=0.8)
+
+fd.vis.duplicates_gallery(num_images=5)
+fd.vis.component_gallery(num_images=5)
+fd.vis.component_gallery(slice='chain_saw')
+fd.vis.outliers_gallery(num_images=5)
+fd.vis.similarity_gallery() 
+
+fd.vis.stats_gallery(metric='dark', num_images=5)
+fd.vis.stats_gallery(metric='bright', num_images=5)
+fd.vis.stats_gallery(metric='blur', num_images=5)
\ No newline at end of file
diff --git a/.github/workflows/tests/labeled_object_detection.py b/.github/workflows/tests/labeled_object_detection.py
new file mode 100644
index 00000000..75631116
--- /dev/null
+++ b/.github/workflows/tests/labeled_object_detection.py
@@ -0,0 +1,21 @@
+import fastdup
+print(f'fastdup version: {fastdup.__version__}')
+
+import pandas as pd
+coco_csv = 'coco_minitrain_25k/annotations/coco_minitrain2017.csv'
+coco_annotations = pd.read_csv(coco_csv, header=None, names=['filename', 'col_x', 'row_y',
+                                                             'width', 'height', 'label', 'ext'])
+
+coco_annotations['split'] = 'train'  # Only train files were loaded
+coco_annotations['filename'] = coco_annotations['filename'].apply(lambda x: 'coco_minitrain_25k/images/train2017/'+x)
+coco_annotations = coco_annotations.drop_duplicates()
+
+input_dir = '.'
+work_dir = 'fastdup_minicoco'
+
+fd = fastdup.create(work_dir=work_dir, input_dir=input_dir)
+fd.run(annotations=coco_annotations, overwrite=True, num_images=10000)
+
+fd.vis.component_gallery(metric='size', max_width=900)
+fd.vis.outliers_gallery()
+fd.vis.component_gallery(num_images=25, slice='diff')
\ No newline at end of file

From 4abb44876d1f0c72810879cc810b502a1f77efc2 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 13:34:28 +0800
Subject: [PATCH 25/33] add explicit os

---
 .github/workflows/examples-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 628d1458..e9fc75f5 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -13,7 +13,7 @@ jobs:
         SENTRY_OPT_OUT: True
     strategy:
         matrix:
-            os: [ubuntu-latest]
+            os: [ubuntu-22.04, macos-13]
             python-version: ['3.9']
     steps:
       - name: Checkout repository

From ef148abd2e0fc42b3cb54873bef21b2a02c76e9c Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 13:37:56 +0800
Subject: [PATCH 26/33] update workdir

---
 .github/workflows/tests/labeled_image_classification.py | 2 +-
 .github/workflows/tests/labeled_object_detection.py     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests/labeled_image_classification.py b/.github/workflows/tests/labeled_image_classification.py
index af9c4083..2d39e64d 100644
--- a/.github/workflows/tests/labeled_image_classification.py
+++ b/.github/workflows/tests/labeled_image_classification.py
@@ -35,7 +35,7 @@
 import fastdup
 print(f'fastdup version: {fastdup.__version__}')
 
-work_dir = 'fastdup_imagenette'
+work_dir = 'fastdup_work_dir'
 fd = fastdup.create(work_dir=work_dir, input_dir=data_dir) 
 fd.run(annotations=df_annot, ccthreshold=0.9, threshold=0.8)
 
diff --git a/.github/workflows/tests/labeled_object_detection.py b/.github/workflows/tests/labeled_object_detection.py
index 75631116..894338fb 100644
--- a/.github/workflows/tests/labeled_object_detection.py
+++ b/.github/workflows/tests/labeled_object_detection.py
@@ -11,7 +11,7 @@
 coco_annotations = coco_annotations.drop_duplicates()
 
 input_dir = '.'
-work_dir = 'fastdup_minicoco'
+work_dir = 'fastdup_work_dir'
 
 fd = fastdup.create(work_dir=work_dir, input_dir=input_dir)
 fd.run(annotations=coco_annotations, overwrite=True, num_images=10000)

From 5ac3e49914211a1162e6eeb9eeb9bf5ce533f5a1 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 13:57:46 +0800
Subject: [PATCH 27/33] remove macos

---
 .github/workflows/examples-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index e9fc75f5..df5b4677 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -13,7 +13,7 @@ jobs:
         SENTRY_OPT_OUT: True
     strategy:
         matrix:
-            os: [ubuntu-22.04, macos-13]
+            os: [ubuntu-22.04]
             python-version: ['3.9']
     steps:
       - name: Checkout repository

From 390c15bcc23a6648dd0a2cd160ffd95089a15ab7 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 14:18:12 +0800
Subject: [PATCH 28/33] trigget at 4pm

---
 .github/workflows/examples-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index df5b4677..f4a31f4f 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -4,7 +4,7 @@ on:
   push:
   workflow_dispatch:
   schedule:
-    - cron: '0 8 * * *' # Trigger at 8 AM every day
+    - cron: '0 16 * * *' # Trigger at 8 AM every day
 
 jobs:
   test-quick-dataset-analysis:

From 82625d3390b0fd201b789900c15e656cc2486d9c Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 21:30:42 +0800
Subject: [PATCH 29/33] macos-10

---
 .github/workflows/examples-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index f4a31f4f..587ea69a 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -13,7 +13,7 @@ jobs:
         SENTRY_OPT_OUT: True
     strategy:
         matrix:
-            os: [ubuntu-22.04]
+            os: [ubuntu-22.04, macos-10]
             python-version: ['3.9']
     steps:
       - name: Checkout repository
@@ -162,4 +162,4 @@ jobs:
         uses: actions/upload-artifact@v3
         with:
           name: fastdup_work_dir_labeled_object_detection
-          path: fastdup_work_dir/
\ No newline at end of file
+          path: fastdup_work_dir/

From 843a415704e425223a9a283a14d1a7c9cb6492d6 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 21:35:23 +0800
Subject: [PATCH 30/33] macos-11

---
 .github/workflows/examples-ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 587ea69a..7a1ece87 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -4,7 +4,7 @@ on:
   push:
   workflow_dispatch:
   schedule:
-    - cron: '0 16 * * *' # Trigger at 8 AM every day
+    - cron: '0 14 * * *' # Trigger at 2PM every day
 
 jobs:
   test-quick-dataset-analysis:
@@ -13,7 +13,7 @@ jobs:
         SENTRY_OPT_OUT: True
     strategy:
         matrix:
-            os: [ubuntu-22.04, macos-10]
+            os: [ubuntu-22.04, macos-11]
             python-version: ['3.9']
     steps:
       - name: Checkout repository

From ac993eb79a79f6d5410c9e80128c41b3838fbc00 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 21:38:17 +0800
Subject: [PATCH 31/33] Update examples-ci.yml

---
 .github/workflows/examples-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 7a1ece87..9757b358 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -13,7 +13,7 @@ jobs:
         SENTRY_OPT_OUT: True
     strategy:
         matrix:
-            os: [ubuntu-22.04, macos-11]
+            os: [ubuntu-22.04]
             python-version: ['3.9']
     steps:
       - name: Checkout repository

From bb21621b7167c9c6b7986b5da345aaf799d9b673 Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Sat, 3 Jun 2023 22:43:03 +0800
Subject: [PATCH 32/33] Update examples-ci.yml

---
 .github/workflows/examples-ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index 9757b358..df255a4b 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -4,7 +4,7 @@ on:
   push:
   workflow_dispatch:
   schedule:
-    - cron: '0 14 * * *' # Trigger at 2PM every day
+    - cron: '0 16 * * *' # Trigger at 4PM every day
 
 jobs:
   test-quick-dataset-analysis:

From 41e3f22639e1b15a8737acee5a20d37178e0a8ef Mon Sep 17 00:00:00 2001
From: Dickson Neoh <dickson.neoh@gmail.com>
Date: Tue, 6 Jun 2023 09:20:46 +0800
Subject: [PATCH 33/33] Remove on push trigger

---
 .github/workflows/examples-ci.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/examples-ci.yml b/.github/workflows/examples-ci.yml
index df255a4b..da852268 100644
--- a/.github/workflows/examples-ci.yml
+++ b/.github/workflows/examples-ci.yml
@@ -1,7 +1,6 @@
 name: Test Run Examples
 
 on: 
-  push:
   workflow_dispatch:
   schedule:
     - cron: '0 16 * * *' # Trigger at 4PM every day