diff --git a/autotest/cpp/test_cpl.cpp b/autotest/cpp/test_cpl.cpp
index 30b729214a36..b3749fa8b8de 100644
--- a/autotest/cpp/test_cpl.cpp
+++ b/autotest/cpp/test_cpl.cpp
@@ -4197,6 +4197,16 @@ TEST_F(test_cpl, VSI_plugin_minimal_testing)
 
     VSIFCloseL(fp);
     EXPECT_TRUE(VSIFOpenL("/vsimyplugin/i_dont_exist", "rb") == nullptr);
+
+    // Check that we can remove the handler
+    VSIRemovePluginHandler("/vsimyplugin/");
+
+    EXPECT_TRUE(VSIFOpenL("/vsimyplugin/test", "rb") == nullptr);
+    EXPECT_TRUE(VSIFOpenL("/vsimyplugin/i_dont_exist", "rb") == nullptr);
+
+    // Removing a non-existing handler is a no-op
+    VSIRemovePluginHandler("/vsimyplugin/");
+    VSIRemovePluginHandler("/vsifoobar/");
 }
 
 TEST_F(test_cpl, VSI_plugin_advise_read)
diff --git a/port/cpl_vsi.h b/port/cpl_vsi.h
index 80c66795a3bf..e14b3cca37e0 100644
--- a/port/cpl_vsi.h
+++ b/port/cpl_vsi.h
@@ -719,6 +719,17 @@ void CPL_DLL VSIFreeFilesystemPluginCallbacksStruct(
 int CPL_DLL VSIInstallPluginHandler(
     const char *pszPrefix, const VSIFilesystemPluginCallbacksStruct *poCb);
 
+/**
+ * Unregister a handler previously installed with VSIInstallPluginHandler() on
+ * the given prefix.
+ * Note: it is generally unsafe to remove a handler while there are still file
+ * handles opened that are managed by that handler. It is the responsibility of
+ * the caller to ensure that it calls this function in a situation where it is
+ * safe to do so.
+ * @since GDAL 3.9
+ */
+int CPL_DLL VSIRemovePluginHandler(const char *pszPrefix);
+
 /* ==================================================================== */
 /*      Time querying.                                                  */
 /* ==================================================================== */
diff --git a/port/cpl_vsi_virtual.h b/port/cpl_vsi_virtual.h
index 7c9b9ca30483..68bbee978342 100644
--- a/port/cpl_vsi_virtual.h
+++ b/port/cpl_vsi_virtual.h
@@ -327,8 +327,7 @@ class CPL_DLL VSIFileManager
     static VSIFilesystemHandler *GetHandler(const char *);
     static void InstallHandler(const std::string &osPrefix,
                                VSIFilesystemHandler *);
-    /* RemoveHandler is never defined. */
-    /* static void RemoveHandler( const std::string& osPrefix ); */
+    static void RemoveHandler(const std::string &osPrefix);
 
     static char **GetPrefixes();
 };
diff --git a/port/cpl_vsil.cpp b/port/cpl_vsil.cpp
index 3a649d50bf2d..3752005da80f 100644
--- a/port/cpl_vsil.cpp
+++ b/port/cpl_vsil.cpp
@@ -3253,6 +3253,18 @@ void VSIFileManager::InstallHandler(const std::string &osPrefix,
         Get()->oHandlers[osPrefix] = poHandler;
 }
 
+/************************************************************************/
+/*                          RemoveHandler()                             */
+/************************************************************************/
+
+void VSIFileManager::RemoveHandler(const std::string &osPrefix)
+{
+    if (osPrefix == "")
+        Get()->poDefaultHandler = nullptr;
+    else
+        Get()->oHandlers.erase(osPrefix);
+}
+
 /************************************************************************/
 /*                       VSICleanupFileManager()                        */
 /************************************************************************/
diff --git a/port/cpl_vsil_plugin.cpp b/port/cpl_vsil_plugin.cpp
index 945f878f55d8..f35d0aaf74b4 100644
--- a/port/cpl_vsil_plugin.cpp
+++ b/port/cpl_vsil_plugin.cpp
@@ -470,6 +470,12 @@ int VSIInstallPluginHandler(const char *pszPrefix,
     return 0;
 }
 
+int VSIRemovePluginHandler(const char *pszPrefix)
+{
+    VSIFileManager::RemoveHandler(pszPrefix);
+    return 0;
+}
+
 VSIFilesystemPluginCallbacksStruct *
 VSIAllocFilesystemPluginCallbacksStruct(void)
 {