[GEOS-8976] Add a purge=metadata|all parameter to the structured cove…

…rage granules removal REST endpoint

doc/en/api/1.0.0/structuredcoverages.yaml

@@ -385,18 +385,17 @@ paths:
           required: false
           description: A CQL filter to reduce the returned granules
           type: string
-        - name: offset
+        - name: purge
           in: query
-          description: Used for paging, the start of the current page
+          description: The purge parameter specifies if and how the underlying raster data source is deleted.
+                      Allowable values for this parameter are "none", "metadata" and "all".
+                      When set to "none" data and auxiliary files are preserved, only the registration in the mosaic is removed
+                      When set to "metadata" delete only auxiliary files and metadata (e.g. NetCDF sidecar indexes). 
+                      It’s recommended when data files (such as granules) should not be deleted from disk.
+                      Finally, when set to "all" both data and auxiliary files are removed.
           required: false
-          type: integer
+          type: string
           minimum: 0
-        - name: limit
-          in: query
-          description: Used for paging, the number of items to be removed
-          required: false
-          type: integer
-          minimum: 1
       responses:
         200:
           description: OK
@@ -574,6 +573,17 @@ paths:
           in: path
           required: true
           description: The granule ID
+        - name: purge
+          in: query
+          description: The purge parameter specifies if and how the underlying raster data source is deleted.
+            Allowable values for this parameter are "none", "metadata" and "all".
+            When set to "none" data and auxiliary files are preserved, only the registration in the mosaic is removed
+            When set to "metadata" delete only auxiliary files and metadata (e.g. NetCDF sidecar indexes).
+            It’s recommended when data files (such as granules) should not be deleted from disk.
+            Finally, when set to "all" both data and auxiliary files are removed.
+          required: false
+          type: string
+          minimum: 0         
       responses:
         200:
           description: OK

src/main/src/main/java/org/geoserver/catalog/StructuredCoverageViewReader.java

@@ -195,8 +195,12 @@ public void addGranules(SimpleFeatureCollection granules) {
             throw new UnsupportedOperationException();
         }
 
-        @Override
         public int removeGranules(Filter filter) {
+            return removeGranules(filter, new Hints());
+        }
+
+        @Override
+        public int removeGranules(Filter filter, Hints hints) {
             // unmap the feature identifiers
             Filter unmapped =
                     GranuleStoreViewFilterVisitor.unmapIdentifiers(filter, coverageView.getName());
@@ -211,7 +215,7 @@ public int removeGranules(Filter filter) {
                     // TODO: We may revisit the #removed granules computation to take into
                     // account cases where we remove different number of records across different
                     // input coverages
-                    removed = granuleStore.removeGranules(unmapped);
+                    removed = granuleStore.removeGranules(unmapped, hints);
                 } catch (UnsupportedOperationException e) {
                     LOGGER.log(Level.FINER, e.getMessage(), e);
                 } catch (IOException e) {

src/restconfig/src/main/java/org/geoserver/rest/catalog/StructuredCoverageController.java

@@ -11,6 +11,7 @@
 import java.io.IOException;
 import java.lang.reflect.Type;
 import java.net.URI;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
 import java.util.logging.Level;
@@ -30,12 +31,14 @@
 import org.geoserver.rest.converters.XStreamMessageConverter;
 import org.geoserver.rest.util.MediaTypeExtensions;
 import org.geoserver.rest.wrapper.RestWrapper;
+import org.geotools.coverage.grid.io.GranuleRemovalPolicy;
 import org.geotools.coverage.grid.io.GranuleSource;
 import org.geotools.coverage.grid.io.GranuleStore;
 import org.geotools.coverage.grid.io.StructuredGridCoverage2DReader;
 import org.geotools.data.Query;
 import org.geotools.data.simple.SimpleFeatureCollection;
 import org.geotools.factory.CommonFactoryFinder;
+import org.geotools.factory.Hints;
 import org.geotools.feature.FeatureTypes;
 import org.geotools.filter.text.cql2.CQLException;
 import org.geotools.filter.text.ecql.ECQL;
@@ -147,16 +150,10 @@ public void granulesDelete(
             @PathVariable String storeName,
             @PathVariable String coverageName,
             @RequestParam(name = "filter", required = false) String filter,
-            @RequestParam(name = "offset", required = false) Integer offset,
-            @RequestParam(name = "limit", required = false) Integer limit)
+            @RequestParam(name = "purge", required = false, defaultValue = "none") String purge)
             throws IOException {
-
-        GranuleStore store = getGranuleStore(workspaceName, storeName, coverageName);
-        Query q = toQuery(filter, offset, limit);
-
-        LOGGER.log(Level.SEVERE, "Still need to parse the filters");
-
-        store.removeGranules(q.getFilter());
+        Query q = toQuery(filter, 0, 1);
+        granulesDeleteInternal(workspaceName, storeName, coverageName, purge, q.getFilter());
     }
 
     /*
@@ -221,14 +218,44 @@ public void granuleDelete(
             @PathVariable(name = "workspaceName") String workspaceName,
             @PathVariable String storeName,
             @PathVariable String coverageName,
-            @PathVariable String granuleId)
+            @PathVariable String granuleId,
+            @RequestParam(name = "purge", required = false, defaultValue = "none") String purge)
             throws IOException {
 
-        // gsConfigForma allows for weird calls gsconfig does, like granules/granule.id/.json
-        GranuleStore store = getGranuleStore(workspaceName, storeName, coverageName);
+        // gsConfig allows for weird calls, like granules/granule.id/.json
         Filter filter = getGranuleIdFilter(granuleId);
 
-        store.removeGranules(filter);
+        granulesDeleteInternal(workspaceName, storeName, coverageName, purge, filter);
+    }
+
+    private void granulesDeleteInternal(
+            String workspaceName,
+            String storeName,
+            String coverageName,
+            String purge,
+            Filter filter)
+            throws IOException {
+        GranuleStore store = getGranuleStore(workspaceName, storeName, coverageName);
+        if (purge != null) {
+            GranuleRemovalPolicy policy = mapRemovalPolicy(purge);
+            Hints hints = new Hints(Hints.GRANULE_REMOVAL_POLICY, policy);
+            store.removeGranules(filter, hints);
+        } else {
+            store.removeGranules(filter);
+        }
+    }
+
+    private GranuleRemovalPolicy mapRemovalPolicy(String key) {
+        try {
+            return GranuleRemovalPolicy.valueOf(key.toUpperCase());
+        } catch (Exception e) {
+            throw new RestException(
+                    "Invalid purge value "
+                            + key
+                            + ", allowed values are "
+                            + Arrays.toString(GranuleRemovalPolicy.values()),
+                    HttpStatus.BAD_REQUEST);
+        }
     }
 
     @Override

src/restconfig/src/test/java/org/geoserver/rest/catalog/StructuredCoverageStoresTest.java

@@ -8,6 +8,9 @@
 import static org.custommonkey.xmlunit.XMLAssert.assertXpathEvaluatesTo;
 import static org.custommonkey.xmlunit.XMLAssert.assertXpathExists;
 import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.Matchers.allOf;
+import static org.hamcrest.Matchers.hasItem;
+import static org.hamcrest.Matchers.not;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertThat;
@@ -21,6 +24,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 import javax.xml.namespace.QName;
 import net.sf.json.JSONArray;
 import net.sf.json.JSONObject;
@@ -39,6 +43,7 @@
 import org.geoserver.catalog.CoverageView.InputCoverageBand;
 import org.geoserver.data.test.MockData;
 import org.geoserver.data.test.SystemTestData;
+import org.geoserver.platform.resource.Resource;
 import org.geoserver.rest.RestBaseController;
 import org.geotools.util.URLs;
 import org.junit.AfterClass;
@@ -53,6 +58,7 @@ public class StructuredCoverageStoresTest extends CatalogRESTTestSupport {
     private static final String WATER_VIEW = "waterView";
     protected static QName WATTEMP =
             new QName(MockData.WCS_PREFIX, "watertemp", MockData.WCS_PREFIX);
+    protected static QName S2_OVR = new QName(MockData.WCS_PREFIX, "s2_ovr", MockData.WCS_PREFIX);
     protected static QName IR_RGB = new QName(MockData.SF_URI, "ir-rgb", MockData.SF_PREFIX);
     private static final String RGB_IR_VIEW = "RgbIrView";
 
@@ -108,7 +114,6 @@ public void setupWaterTemp() throws Exception {
         }
 
         // setup a view on watertemp
-
         final CoverageStoreInfo storeInfo = cat.getCoverageStoreByName("watertemp");
 
         final InputCoverageBand band = new InputCoverageBand("watertemp", "0");
@@ -127,6 +132,17 @@ public void setupWaterTemp() throws Exception {
                 coverageView.createCoverageInfo(WATER_VIEW, storeInfo, builder);
         coverageInfo.getParameters().put("USE_JAI_IMAGEREAD", "false");
         cat.add(coverageInfo);
+
+        // setup the hetero_s2_ovr mosaic
+        getDataDirectory().getRoot("s2_ovr").delete();
+        getTestData()
+                .addRasterLayer(
+                        S2_OVR,
+                        "hetero_s2_ovr.zip",
+                        null,
+                        null,
+                        StructuredCoverageStoresTest.class,
+                        getCatalog());
     }
 
     @Before
@@ -802,4 +818,53 @@ public void testDeleteGranuleInSingleCoverageView() throws Exception {
                                 + "/workspaces/wcs/coveragestores/watertemp/coverages/waterView/index/granules/waterView.watertemp.1");
         assertEquals(404, response.getStatus());
     }
+
+    @Test
+    public void testDeletePurgeNoneFilter() throws Exception {
+        List<String> filesAfter = removeWithPurge("&purge=none");
+
+        // list the files, they must still contain g3.tif
+        assertThat(filesAfter, allOf(hasItem("g3.tif"), hasItem("g3.tif.ovr")));
+    }
+
+    @Test
+    public void testDeletePurgeAll() throws Exception {
+        List<String> filesAfter = removeWithPurge("&purge=all");
+
+        // list the files, g3 must be gone
+        assertThat(filesAfter, not(hasItem(containsString("g3.tif"))));
+    }
+
+    private List<String> removeWithPurge(String purgeSpec) throws Exception {
+        Document dom =
+                getAsDOM(
+                        RestBaseController.ROOT_PATH
+                                + "/workspaces/wcs/coveragestores/s2_ovr/coverages/s2_ovr/index/granules.xml");
+        assertXpathEvaluatesTo("4", "count(//gf:s2_ovr)", dom);
+        // print(dom);
+
+        MockHttpServletResponse response =
+                deleteAsServletResponse(
+                        RestBaseController.ROOT_PATH
+                                + "/workspaces/wcs/coveragestores/s2_ovr/coverages"
+                                + "/s2_ovr/index/granules?filter=location LIKE 'g3%25'"
+                                + purgeSpec);
+        assertEquals(200, response.getStatus());
+
+        // check it's gone from the index
+        dom =
+                getAsDOM(
+                        RestBaseController.ROOT_PATH
+                                + "/workspaces/wcs/coveragestores/s2_ovr/coverages/s2_ovr/index/granules.xml");
+        // print(dom);
+        assertXpathEvaluatesTo("3", "count(//gf:s2_ovr)", dom);
+        assertXpathExists("//gf:s2_ovr[gf:location = 'g1.tif']", dom);
+        assertXpathExists("//gf:s2_ovr[gf:location = 'g2.tif']", dom);
+        assertXpathExists("//gf:s2_ovr[gf:location = 'g4.tif']", dom);
+
+        Resource mosaicContents = getDataDirectory().getRoot("s2_ovr");
+        List<String> files =
+                mosaicContents.list().stream().map(r -> r.name()).collect(Collectors.toList());
+        return files;
+    }
 }