[To rel/0.13][IOTDB-3969]Fix timeseries statistic bug

docs/UserGuide/Maintenance-Tools/Metric-Tool.md

@@ -111,7 +111,7 @@ Next, we will choose Prometheus format data as samples to describe each kind of
 
 | Metric   | Tag                                   | level     | Description                                                   | Sample                           |
 | -------- | ------------------------------------- | --------- | ------------------------------------------------------------- | -------------------------------- |
-| quantity | name="timeSeries/storageGroup/device" | important | The current count of timeSeries/storageGroup/devices in IoTDB | quantity{name="timeSeries",} 1.0 |
+| quantity | name="storageGroup/device";<br/>name="timeSeries", type="normal/template" | important | The current count of timeSeries/storageGroup/devices in IoTDB | quantity{name="timeSeries",type="normal"} 1.0 |
 
 #### 4.3.6. Cluster
 

docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md

@@ -110,7 +110,7 @@ IoTDB对外提供JMX和Prometheus格式的监控指标，对于JMX，可以通
 
 | Metric   | Tag                                   | level     | 说明                                         | 示例                             |
 | -------- | ------------------------------------- | --------- | -------------------------------------------- | -------------------------------- |
-| quantity | name="timeSeries/storageGroup/device" | important | 当前时间timeSeries/storageGroup/device的数量 | quantity{name="timeSeries",} 1.0 |
+| quantity | name="timeSeries/storageGroup/device";<br/>name="timeSeries", type="normal/template"  | important | 当前时间timeSeries/storageGroup/device的数量 | quantity{name="timeSeries",type="normal"} 1.0 |
 
 #### 4.3.6. 集群
 

server/src/main/java/org/apache/iotdb/db/exception/metadata/MeasurementInsideTemplateException.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.exception.metadata;
+
+public class MeasurementInsideTemplateException extends MetadataException {
+
+  private String path;
+
+  public MeasurementInsideTemplateException(String path) {
+    super(String.format("Cannot delete a timeseries inside a template: %s", path));
+    this.path = path;
+  }
+
+  public String getPath() {
+    return path;
+  }
+}

server/src/main/java/org/apache/iotdb/db/metadata/MManager.java

@@ -30,6 +30,7 @@
 import org.apache.iotdb.db.exception.metadata.DeleteFailedException;
 import org.apache.iotdb.db.exception.metadata.DifferentTemplateException;
 import org.apache.iotdb.db.exception.metadata.MNodeTypeMismatchException;
+import org.apache.iotdb.db.exception.metadata.MeasurementInsideTemplateException;
 import org.apache.iotdb.db.exception.metadata.MetadataException;
 import org.apache.iotdb.db.exception.metadata.NoTemplateOnMNodeException;
 import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException;
@@ -185,7 +186,8 @@ public class MManager {
   private boolean initialized;
   private boolean allowToCreateNewSeries = true;
 
-  private AtomicLong totalSeriesNumber = new AtomicLong();
+  private AtomicLong totalNormalSeriesNumber = new AtomicLong();
+  private AtomicLong totalTemplateSeriesNumber = new AtomicLong();
 
   private final int mtreeSnapshotInterval;
   private final long mtreeSnapshotThresholdTime;
@@ -319,10 +321,24 @@ private void startStatisticCounts() {
         .getOrCreateAutoGauge(
             Metric.QUANTITY.toString(),
             MetricLevel.IMPORTANT,
-            totalSeriesNumber,
+            totalNormalSeriesNumber,
             AtomicLong::get,
             Tag.NAME.toString(),
-            "timeSeries");
+            "timeSeries",
+            Tag.TYPE.toString(),
+            "normal");
+
+    MetricsService.getInstance()
+        .getMetricManager()
+        .getOrCreateAutoGauge(
+            Metric.QUANTITY.toString(),
+            MetricLevel.IMPORTANT,
+            totalTemplateSeriesNumber,
+            AtomicLong::get,
+            Tag.NAME.toString(),
+            "timeSeries",
+            Tag.TYPE.toString(),
+            "template");
 
     MetricsService.getInstance()
         .getMetricManager()
@@ -445,7 +461,8 @@ public synchronized void clear() {
       if (this.mNodeCache != null) {
         this.mNodeCache.invalidateAll();
       }
-      this.totalSeriesNumber.set(0);
+      this.totalNormalSeriesNumber.set(0);
+      this.totalTemplateSeriesNumber.set(0);
       this.templateManager.clear();
       if (logWriter != null) {
         logWriter.close();
@@ -596,9 +613,9 @@ public void createTimeseries(CreateTimeSeriesPlan plan, long offset) throws Meta
       }
 
       // update statistics and schemaDataTypeNumMap
-      totalSeriesNumber.addAndGet(1);
-      if (totalSeriesNumber.get() * ESTIMATED_SERIES_SIZE >= MTREE_SIZE_THRESHOLD) {
-        logger.warn("Current series number {} is too large...", totalSeriesNumber);
+      totalNormalSeriesNumber.addAndGet(1);
+      if (totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE >= MTREE_SIZE_THRESHOLD) {
+        logger.warn("Current series number {} is too large...", totalNormalSeriesNumber);
         allowToCreateNewSeries = false;
       }
 
@@ -700,9 +717,9 @@ public void createAlignedTimeSeries(CreateAlignedTimeSeriesPlan plan) throws Met
       mNodeCache.invalidate(prefixPath);
 
       // update statistics and schemaDataTypeNumMap
-      totalSeriesNumber.addAndGet(measurements.size());
-      if (totalSeriesNumber.get() * ESTIMATED_SERIES_SIZE >= MTREE_SIZE_THRESHOLD) {
-        logger.warn("Current series number {} is too large...", totalSeriesNumber);
+      totalNormalSeriesNumber.addAndGet(measurements.size());
+      if (totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE >= MTREE_SIZE_THRESHOLD) {
+        logger.warn("Current series number {} is too large...", totalNormalSeriesNumber);
         allowToCreateNewSeries = false;
       }
       // write log
@@ -793,6 +810,8 @@ private void deleteSingleTimeseriesInternal(PartialPath p, Set<String> failedNam
         deleteTimeSeriesPlan.setDeletePathList(Collections.singletonList(p));
         logWriter.deleteTimeseries(deleteTimeSeriesPlan);
       }
+    } catch (MeasurementInsideTemplateException e) {
+      failedNames.add(e.getMessage());
     } catch (DeleteFailedException e) {
       failedNames.add(e.getName());
     }
@@ -825,10 +844,10 @@ private PartialPath deleteOneTimeseriesUpdateStatisticsAndDropTrigger(PartialPat
       mNodeCache.invalidate(node.getPartialPath());
       node = node.getParent();
     }
-    totalSeriesNumber.addAndGet(-1);
+    totalNormalSeriesNumber.addAndGet(-1);
     if (!allowToCreateNewSeries
-        && totalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) {
-      logger.info("Current series number {} come back to normal level", totalSeriesNumber);
+        && totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) {
+      logger.info("Current series number {} come back to normal level", totalNormalSeriesNumber);
       allowToCreateNewSeries = true;
     }
     return storageGroupPath;
@@ -865,12 +884,14 @@ public void setStorageGroup(PartialPath storageGroup) throws MetadataException {
   public void deleteStorageGroups(List<PartialPath> storageGroups) throws MetadataException {
     try {
       for (PartialPath storageGroup : storageGroups) {
-        totalSeriesNumber.addAndGet(
-            -mtree.getAllTimeseriesCount(storageGroup.concatNode(MULTI_LEVEL_PATH_WILDCARD)));
+        totalNormalSeriesNumber.addAndGet(
+            -mtree.getAllTimeseriesCount(
+                storageGroup.concatNode(MULTI_LEVEL_PATH_WILDCARD), false, false));
         // clear cached MNode
         if (!allowToCreateNewSeries
-            && totalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) {
-          logger.info("Current series number {} come back to normal level", totalSeriesNumber);
+            && totalNormalSeriesNumber.get() * ESTIMATED_SERIES_SIZE < MTREE_SIZE_THRESHOLD) {
+          logger.info(
+              "Current series number {} come back to normal level", totalNormalSeriesNumber);
           allowToCreateNewSeries = true;
         }
         mNodeCache.invalidateAll();
@@ -998,8 +1019,12 @@ public String getMetadataInString() {
 
   // region Interfaces for metadata count
 
-  public long getTotalSeriesNumber() {
-    return totalSeriesNumber.get();
+  public long getTotalNormalSeriesNumber() {
+    return totalNormalSeriesNumber.get();
+  }
+
+  public long getTotalTemplateSeriesNumber() {
+    return totalTemplateSeriesNumber.get();
   }
 
   /**
@@ -1009,7 +1034,7 @@ public long getTotalSeriesNumber() {
    */
   public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch)
       throws MetadataException {
-    return mtree.getAllTimeseriesCount(pathPattern, isPrefixMatch);
+    return mtree.getAllTimeseriesCount(pathPattern, isPrefixMatch, true);
   }
 
   /**
@@ -2452,6 +2477,8 @@ public void deactivateSchemaTemplate(DeactivateTemplatePlan plan) throws Metadat
       }
 
       node.setUseTemplate(false);
+      totalTemplateSeriesNumber.addAndGet(-node.getUpperTemplate().getMeasurementsCount());
+
       // clear caches within MManger
       mNodeCache.invalidate(node);
       if (node.isEntity()) {
@@ -2470,7 +2497,8 @@ public void deactivateSchemaTemplate(DeactivateTemplatePlan plan) throws Metadat
 
   IMNode setUsingSchemaTemplate(IMNode node) throws MetadataException {
     // check whether any template has been set on designated path
-    if (node.getUpperTemplate() == null) {
+    Template template = node.getUpperTemplate();
+    if (template == null) {
       throw new MetadataException(
           String.format("Path [%s] has not been set any template.", node.getFullPath()));
     }
@@ -2480,8 +2508,7 @@ IMNode setUsingSchemaTemplate(IMNode node) throws MetadataException {
 
     // check alignment of template and mounted node
     // if direct measurement exists, node will be replaced
-    IMNode mountedMNode =
-        mtree.checkTemplateAlignmentWithMountedNode(node, node.getUpperTemplate());
+    IMNode mountedMNode = mtree.checkTemplateAlignmentWithMountedNode(node, template);
 
     // if has direct measurement (be a EntityNode), to ensure alignment adapt with former node or
     // template
@@ -2494,6 +2521,7 @@ IMNode setUsingSchemaTemplate(IMNode node) throws MetadataException {
                   : node.getUpperTemplate().isDirectAligned());
     }
     mountedMNode.setUseTemplate(true);
+    totalTemplateSeriesNumber.addAndGet(template.getMeasurementsCount());
 
     if (node != mountedMNode) {
       mNodeCache.invalidate(mountedMNode.getPartialPath());

server/src/main/java/org/apache/iotdb/db/metadata/mtree/MTree.java

@@ -25,6 +25,7 @@
 import org.apache.iotdb.db.exception.metadata.AlignedTimeseriesException;
 import org.apache.iotdb.db.exception.metadata.IllegalPathException;
 import org.apache.iotdb.db.exception.metadata.MNodeTypeMismatchException;
+import org.apache.iotdb.db.exception.metadata.MeasurementInsideTemplateException;
 import org.apache.iotdb.db.exception.metadata.MetadataException;
 import org.apache.iotdb.db.exception.metadata.PathAlreadyExistException;
 import org.apache.iotdb.db.exception.metadata.PathNotExistException;
@@ -540,8 +541,7 @@ public Pair<PartialPath, IMeasurementMNode> deleteTimeseriesAndReturnEmptyStorag
     }
 
     if (isPathExistsWithinTemplate(path)) {
-      throw new MetadataException(
-          "Cannot delete a timeseries inside a template: " + path.toString());
+      throw new MeasurementInsideTemplateException(path.getFullPath());
     }
 
     IMeasurementMNode deletedNode = getMeasurementMNode(path);
@@ -1272,9 +1272,11 @@ protected void transferToResult(IMNode node) {
    *
    * @param pathPattern a path pattern or a full path, may contain wildcard
    */
-  public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch)
+  public int getAllTimeseriesCount(
+      PartialPath pathPattern, boolean isPrefixMatch, boolean traverseTemplate)
       throws MetadataException {
     CounterTraverser counter = new MeasurementCounter(root, pathPattern);
+    counter.setShouldTraverseTemplate(traverseTemplate);
     counter.setPrefixMatch(isPrefixMatch);
     counter.traverse();
     return counter.getCount();
@@ -1286,7 +1288,7 @@ public int getAllTimeseriesCount(PartialPath pathPattern, boolean isPrefixMatch)
    * @param pathPattern a path pattern or a full path, may contain wildcard
    */
   public int getAllTimeseriesCount(PartialPath pathPattern) throws MetadataException {
-    return getAllTimeseriesCount(pathPattern, false);
+    return getAllTimeseriesCount(pathPattern, false, true);
   }
 
   /**

server/src/main/java/org/apache/iotdb/db/metadata/mtree/traverser/Traverser.java

@@ -279,6 +279,10 @@ public void setPrefixMatch(boolean isPrefixMatch) {
     this.isPrefixMatch = isPrefixMatch;
   }
 
+  public void setShouldTraverseTemplate(boolean shouldTraverseTemplate) {
+    this.shouldTraverseTemplate = shouldTraverseTemplate;
+  }
+
   /**
    * @param currentNode the node need to get the full path of
    * @return full path from traverse start node to the current node

server/src/test/java/org/apache/iotdb/db/metadata/MManagerBasicTest.java

@@ -35,6 +35,7 @@
 import org.apache.iotdb.db.qp.physical.sys.AppendTemplatePlan;
 import org.apache.iotdb.db.qp.physical.sys.CreateTemplatePlan;
 import org.apache.iotdb.db.qp.physical.sys.CreateTimeSeriesPlan;
+import org.apache.iotdb.db.qp.physical.sys.DeactivateTemplatePlan;
 import org.apache.iotdb.db.qp.physical.sys.SetTemplatePlan;
 import org.apache.iotdb.db.qp.physical.sys.ShowTimeSeriesPlan;
 import org.apache.iotdb.db.qp.physical.sys.UnsetTemplatePlan;
@@ -1918,13 +1919,13 @@ public void testTotalSeriesNumber() throws Exception {
           CompressionType.GZIP,
           null);
 
-      assertEquals(6, manager.getTotalSeriesNumber());
+      assertEquals(6, manager.getTotalNormalSeriesNumber());
       EnvironmentUtils.restartDaemon();
-      assertEquals(6, manager.getTotalSeriesNumber());
+      assertEquals(6, manager.getTotalNormalSeriesNumber());
       manager.deleteTimeseries(new PartialPath("root.laptop.d2.s1"));
-      assertEquals(5, manager.getTotalSeriesNumber());
+      assertEquals(5, manager.getTotalNormalSeriesNumber());
       manager.deleteStorageGroups(Collections.singletonList(new PartialPath("root.laptop")));
-      assertEquals(0, manager.getTotalSeriesNumber());
+      assertEquals(0, manager.getTotalNormalSeriesNumber());
     } catch (MetadataException e) {
       e.printStackTrace();
       fail(e.getMessage());
@@ -2582,4 +2583,85 @@ public void testShowChildWithLimitAndOffset() throws Exception {
     Assert.assertEquals(1, result.size());
     Assert.assertTrue(result.contains("d1"));
   }
+
+  @Test
+  public void testTimeseriesNumberStatistic() throws Exception {
+    MManager manager = IoTDB.metaManager;
+
+    try {
+      manager.setStorageGroup(new PartialPath("root.laptop"));
+      manager.createTimeseries(
+          new PartialPath("root.laptop.d0"),
+          TSDataType.INT32,
+          TSEncoding.PLAIN,
+          CompressionType.GZIP,
+          null);
+      manager.createTimeseries(
+          new PartialPath("root.laptop.d1.s1"),
+          TSDataType.INT32,
+          TSEncoding.PLAIN,
+          CompressionType.GZIP,
+          null);
+      manager.createTimeseries(
+          new PartialPath("root.laptop.d1.s2.t1"),
+          TSDataType.INT32,
+          TSEncoding.PLAIN,
+          CompressionType.GZIP,
+          null);
+      manager.createTimeseries(
+          new PartialPath("root.laptop.d1.s3"),
+          TSDataType.INT32,
+          TSEncoding.PLAIN,
+          CompressionType.GZIP,
+          null);
+      manager.createTimeseries(
+          new PartialPath("root.laptop.d2.s1"),
+          TSDataType.INT32,
+          TSEncoding.PLAIN,
+          CompressionType.GZIP,
+          null);
+      manager.createTimeseries(
+          new PartialPath("root.laptop.d2.s2"),
+          TSDataType.INT32,
+          TSEncoding.PLAIN,
+          CompressionType.GZIP,
+          null);
+
+      assertEquals(6, manager.getTotalNormalSeriesNumber());
+
+      manager.setStorageGroup(new PartialPath("root.sg"));
+
+      CreateTemplatePlan plan = getCreateTemplatePlan("s0");
+      manager.createSchemaTemplate(plan);
+      SetTemplatePlan setPlan = new SetTemplatePlan("template1", "root.sg.d1");
+      manager.setSchemaTemplate(setPlan);
+
+      manager.setUsingSchemaTemplate(manager.getDeviceNode(new PartialPath("root.sg.d1")));
+
+      assertEquals(6, manager.getTotalNormalSeriesNumber());
+      assertEquals(1, manager.getTotalTemplateSeriesNumber());
+
+      manager.deleteTimeseries(new PartialPath("root.laptop.d2.s1"));
+      assertEquals(5, manager.getTotalNormalSeriesNumber());
+      assertEquals(1, manager.getTotalTemplateSeriesNumber());
+
+      manager.deleteStorageGroups(Collections.singletonList(new PartialPath("root.laptop")));
+      assertEquals(0, manager.getTotalNormalSeriesNumber());
+      assertEquals(1, manager.getTotalTemplateSeriesNumber());
+
+      String failedName = manager.deleteTimeseries(new PartialPath("root.sg.d1.s0"));
+      assertTrue(failedName.contains("root.sg.d1.s0"));
+      assertEquals(1, manager.getTotalTemplateSeriesNumber());
+
+      DeactivateTemplatePlan deactivateTemplatePlan =
+          new DeactivateTemplatePlan("template1", "root.sg.d1");
+      deactivateTemplatePlan.setPaths(Collections.singletonList(new PartialPath("root.sg.d1")));
+      manager.deactivateSchemaTemplate(deactivateTemplatePlan);
+      assertEquals(0, manager.getTotalTemplateSeriesNumber());
+
+    } catch (MetadataException e) {
+      e.printStackTrace();
+      fail(e.getMessage());
+    }
+  }
 }