Add @Feature annotation with categories

.github/workflows/testing.yml

@@ -116,11 +116,6 @@ jobs:
         run: |
           bin/test_sequencer $GCP_TARGET_PROJECT
           more /tmp/sequencer.out /tmp/generated.md
-          diff -u /tmp/sequencer.out etc/sequencer.out
-          diff -u /tmp/generated.md docs/specs/sequences/generated.md
-          ls -1 sites/udmi_site_model/out/devices/AHU-1/tests/ | xargs -I% diff -u \
-              sites/udmi_site_model/out/devices/AHU-1/tests/%/sequence.md \
-              validator/sequences/%/sequence.md
       - name: itemized sequencer tests
         env:
           GCP_TARGET_PROJECT: ${{ secrets.GCP_TARGET_PROJECT }}
@@ -129,7 +124,6 @@ jobs:
         run: |
           bin/test_itemized $GCP_TARGET_PROJECT
           more out/test_itemized.out
-          diff -u out/test_itemized.out etc/test_itemized.out
       - name: output logs
         if: ${{ always() }}
         run: |

bin/test_itemized

@@ -65,9 +65,12 @@ while read -u 7 action test_name remainder; do
 
     kill $(ps ax | fgrep pubber | fgrep java | awk '{print $1}')
 
-    result=$(grep -E -m 1 "RESULT [a-z]+ $test_name " $SEQUENCER_OUT)
+    result=$(grep -E -m 1 "RESULT [a-z]+ [a-z.]+ $test_name " $SEQUENCER_OUT)
     echo ${result/*sequencer RESULT/RESULT} >> $RESULTS_OUT
 
     sleep 2
 
 done 7< $GOLDEN_FILE
+
+echo Comparing diff $RESULTS_OUT $GOLDEN_FILE
+diff -u out/test_itemized.out etc/test_itemized.out

bin/test_sequencer

@@ -33,6 +33,10 @@ else
     done
 fi
 
+test_out_base=sites/udmi_site_model/out/devices/AHU-1/tests
+mkdir -p $test_out_base
+all_tests=$(ls -1 $test_out_base)
+
 serial_no=sequencer-$RANDOM
 echo Using pubber with serial $serial_no
 
@@ -88,4 +92,15 @@ generated=docs/specs/sequences/generated.md
 cp $generated /tmp/  # Save for test/comparison later
 bin/gencode_seq
 
+echo Comparing diff /tmp/sequencer.out etc/sequencer.out
+diff -u /tmp/sequencer.out etc/sequencer.out
+
+echo Comparing diff /tmp/generated.md docs/specs/sequences/generated.md
+diff -u /tmp/generated.md docs/specs/sequences/generated.md
+
+generated_sequences=$test_out_base/%/sequence.md
+expected_sequences=validator/sequences/%/sequence.md
+echo Comparing diff $generated_sequences $expected_sequences
+echo $all_tests | xargs -I% diff -u $generated_sequences $expected_sequences
+
 echo Done with base test_sequencer run.

etc/sequencer.out

@@ -1,27 +1,27 @@
-RESULT pass broken_config Sequence complete
-RESULT pass device_config_acked Sequence complete
-RESULT pass empty_enumeration ALPHA Sequence complete
-RESULT pass endpoint_connection_bad_hash Sequence complete
-RESULT pass endpoint_connection_error Sequence complete
-RESULT pass endpoint_connection_retry Sequence complete
-RESULT pass endpoint_connection_success_alternate Sequence complete
-RESULT pass endpoint_connection_success_reconnect Sequence complete
-RESULT pass extra_config Sequence complete
-RESULT pass family_enumeration ALPHA Sequence complete
-RESULT pass feature_enumeration ALPHA Sequence complete
-RESULT pass multi_enumeration ALPHA Sequence complete
-RESULT pass periodic_scan Sequence complete
-RESULT pass pointset_enumeration ALPHA Sequence complete
-RESULT pass pointset_publish_interval ALPHA Sequence complete
-RESULT pass pointset_sample_rate ALPHA Sequence complete
-RESULT pass single_scan Sequence complete
-RESULT pass system_last_update Sequence complete
-RESULT pass system_min_loglevel BETA Sequence complete
-RESULT pass system_mode_restart PREVIEW Sequence complete
-RESULT pass valid_serial_no Sequence complete
-RESULT pass valid_serial_no Sequence complete
-RESULT pass valid_serial_no Sequence complete
-RESULT pass valid_serial_no Sequence complete
-RESULT pass writeback_failure Sequence complete
-RESULT pass writeback_invalid Sequence complete
-RESULT pass writeback_success Sequence complete
+RESULT pass discovery.scan periodic_scan REQUIRED 5 Sequence complete
+RESULT pass enumeration empty_enumeration ALPHA 5 Sequence complete
+RESULT pass enumeration.families family_enumeration ALPHA 5 Sequence complete
+RESULT pass enumeration.features feature_enumeration ALPHA 5 Sequence complete
+RESULT pass enumeration.pointset pointset_enumeration ALPHA 5 Sequence complete
+RESULT pass unknown broken_config REQUIRED 5 Sequence complete
+RESULT pass unknown device_config_acked REQUIRED 5 Sequence complete
+RESULT pass unknown endpoint_connection_bad_hash REQUIRED 5 Sequence complete
+RESULT pass unknown endpoint_connection_error REQUIRED 5 Sequence complete
+RESULT pass unknown endpoint_connection_retry REQUIRED 5 Sequence complete
+RESULT pass unknown endpoint_connection_success_alternate REQUIRED 5 Sequence complete
+RESULT pass unknown endpoint_connection_success_reconnect REQUIRED 5 Sequence complete
+RESULT pass unknown extra_config REQUIRED 5 Sequence complete
+RESULT pass unknown multi_enumeration ALPHA 5 Sequence complete
+RESULT pass unknown pointset_publish_interval ALPHA 5 Sequence complete
+RESULT pass unknown pointset_sample_rate ALPHA 5 Sequence complete
+RESULT pass unknown single_scan REQUIRED 5 Sequence complete
+RESULT pass unknown system_last_update REQUIRED 5 Sequence complete
+RESULT pass unknown system_min_loglevel BETA 5 Sequence complete
+RESULT pass unknown system_mode_restart PREVIEW 5 Sequence complete
+RESULT pass unknown valid_serial_no REQUIRED 5 Sequence complete
+RESULT pass unknown valid_serial_no REQUIRED 5 Sequence complete
+RESULT pass unknown valid_serial_no REQUIRED 5 Sequence complete
+RESULT pass unknown valid_serial_no REQUIRED 5 Sequence complete
+RESULT pass unknown writeback_failure REQUIRED 5 Sequence complete
+RESULT pass unknown writeback_invalid REQUIRED 5 Sequence complete
+RESULT pass unknown writeback_success REQUIRED 5 Sequence complete

etc/test_itemized.out

@@ -1,3 +1,3 @@
-TEST writeback_success noWriteback RESULT fail writeback_success timeout waiting for point filter_differential_pressure_setpoint to have value_state applied
-TEST writeback_success noPointState RESULT fail writeback_success timeout waiting for point filter_differential_pressure_setpoint to have value_state applied
-TEST pointset_sample_rate fixedSampleRate=10 RESULT fail pointset_sample_rate ALPHA Failed check that time period between successive pointset events is between 1 and 5 seconds
+TEST writeback_success noWriteback RESULT fail unknown writeback_success REQUIRED 5 timeout waiting for point filter_differential_pressure_setpoint to have value_state applied
+TEST writeback_success noPointState RESULT fail unknown writeback_success REQUIRED 5 timeout waiting for point filter_differential_pressure_setpoint to have value_state applied
+TEST pointset_sample_rate fixedSampleRate=10 RESULT fail unknown pointset_sample_rate ALPHA 5 Failed check that time period between successive pointset events is between 1 and 5 seconds

validator/src/main/java/com/google/daq/mqtt/sequencer/Feature.java

@@ -0,0 +1,54 @@
+package com.google.daq.mqtt.sequencer;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Feature designation for line-item tests.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.METHOD})
+public @interface
+Feature {
+  String IMPLICIT_CATEGORY = "";
+  Stage DEFAULT_STAGE = Stage.REQUIRED;
+  int DEFAULT_SCORE = 5;
+
+  /**
+   * Defines the category for this feature, as defined by a named attribute.
+   *
+   * @return feature category using named attribute
+   */
+  String category() default IMPLICIT_CATEGORY;
+
+  /**
+   * Defines the category for this feature, as defined by an implicit attribute.
+   *
+   * @return feature category using implicit argument
+   */
+  String value() default IMPLICIT_CATEGORY;
+
+  /**
+   * Default value is REQUIRED which should be the end-state for all tests.
+   *
+   * @return annotated release stage of this test
+   */
+  Stage stage() default Stage.REQUIRED;
+
+  /**
+   * Defines the feature score for this test, in AU.
+   *
+   * @return feature score value
+   */
+  int score() default DEFAULT_SCORE;
+
+  /**
+   * Enum of allowed stages.
+   */
+  enum Stage {
+    ALPHA,
+    REQUIRED
+  }
+}

validator/src/main/java/com/google/daq/mqtt/sequencer/SequenceBase.java

@@ -2,7 +2,7 @@
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
-import static com.google.daq.mqtt.sequencer.FeatureStage.Stage.REQUIRED;
+import static com.google.daq.mqtt.sequencer.Feature.Stage.REQUIRED;
 import static com.google.daq.mqtt.sequencer.semantic.SemanticValue.actualize;
 import static com.google.daq.mqtt.util.Common.EXCEPTION_KEY;
 import static com.google.daq.mqtt.util.Common.TIMESTAMP_PROPERTY_KEY;
@@ -25,7 +25,6 @@
 import com.google.daq.mqtt.util.ConfigDiffEngine;
 import com.google.daq.mqtt.util.ConfigUtil;
 import com.google.daq.mqtt.util.MessagePublisher;
-import com.google.daq.mqtt.util.TimePeriodConstants;
 import com.google.daq.mqtt.validator.AugmentedState;
 import com.google.daq.mqtt.validator.AugmentedSystemConfig;
 import com.google.daq.mqtt.validator.Validator.MessageBundle;
@@ -93,7 +92,7 @@ public class SequenceBase {
   public static final String RESULT_FAIL = "fail";
   public static final String RESULT_PASS = "pass";
   public static final String RESULT_SKIP = "skip";
-  public static final String RESULT_FORMAT = "RESULT %s %s%s %s";
+  public static final String RESULT_FORMAT = "RESULT %s %s %s %s %s %s";
   public static final String TESTS_OUT_DIR = "tests";
   public static final String SERIAL_NO_MISSING = "//";
   public static final String SEQUENCER_CATEGORY = "sequencer";
@@ -135,6 +134,7 @@ public class SequenceBase {
   private static final int EXIT_CODE_PRESERVE = -9;
   private static final String SYSTEM_TESTING_MARKER = " `system.testing";
   private static final Map<SubFolder, String> sentConfig = new HashMap<>();
+  private static final String UNKNOWN_CATEGORY = "unknown";
   protected static Metadata deviceMetadata;
   protected static String projectId;
   protected static String cloudRegion;
@@ -468,7 +468,7 @@ private void waitForConfigSync(Instant configUpdateStart) {
   }
 
   @Test
-  @FeatureStage(REQUIRED)
+  @Feature(stage = REQUIRED)
   public void valid_serial_no() {
     if (serialNo == null) {
       throw new SkipTest("No test serial number provided");
@@ -479,9 +479,12 @@ public void valid_serial_no() {
   private void recordResult(String result, org.junit.runner.Description description,
       String message) {
     String methodName = description.getMethodName();
-    FeatureStage stage = description.getAnnotation(FeatureStage.class);
-    String suffix = (stage == null || stage.value() == REQUIRED) ? "" : (" " + stage.value());
-    String resultString = String.format(RESULT_FORMAT, result, methodName, suffix, message);
+    Feature feature = description.getAnnotation(Feature.class);
+    String category = getCategory(feature);
+    String stage = (feature == null ? Feature.DEFAULT_STAGE : feature.stage()).name();
+    int score = (feature == null ? Feature.DEFAULT_SCORE : feature.score());
+    String resultString = String.format(RESULT_FORMAT, result, category, methodName, stage, score,
+        message);
     notice(resultString);
     try (PrintWriter log = new PrintWriter(new FileOutputStream(resultSummary, true))) {
       log.print(resultString);
@@ -491,6 +494,21 @@ private void recordResult(String result, org.junit.runner.Description descriptio
     }
   }
 
+  private String getCategory(Feature feature) {
+    if (feature == null) {
+      return UNKNOWN_CATEGORY;
+    }
+    String value = feature.value();
+    String category = feature.category();
+    if (!Strings.isNullOrEmpty(value) && !Strings.isNullOrEmpty(category)) {
+      throw new RuntimeException("Both value and category defined for feature");
+    }
+    if (Strings.isNullOrEmpty(value) && Strings.isNullOrEmpty(category)) {
+      return UNKNOWN_CATEGORY;
+    }
+    return Strings.isNullOrEmpty(value) ? category : value;
+  }
+
   private void recordRawMessage(Map<String, Object> message, Map<String, String> attributes) {
     if (testName == null) {
       return;

validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/BlobsetSequences.java

@@ -1,13 +1,13 @@
 package com.google.daq.mqtt.sequencer.sequences;
 
-import static com.google.daq.mqtt.sequencer.FeatureStage.Stage.PREVIEW;
+import static com.google.daq.mqtt.sequencer.Feature.Stage.REQUIRED;
 import static com.google.udmi.util.GeneralUtils.encodeBase64;
 import static com.google.udmi.util.GeneralUtils.sha256;
 import static com.google.udmi.util.JsonUtil.stringify;
 import static org.junit.Assert.assertNotEquals;
 import static udmi.schema.Category.BLOBSET_BLOB_APPLY;
 
-import com.google.daq.mqtt.sequencer.FeatureStage;
+import com.google.daq.mqtt.sequencer.Feature;
 import com.google.daq.mqtt.sequencer.SequenceBase;
 import com.google.daq.mqtt.sequencer.SkipTest;
 import com.google.daq.mqtt.sequencer.semantic.SemanticDate;
@@ -198,7 +198,7 @@ public void endpoint_connection_success_alternate() {
 
   @Test
   @Description("Restart and connect to same endpoint and expect it returns.")
-  @FeatureStage(PREVIEW)
+  @Feature(stage = REQUIRED)
   public void system_mode_restart() {
     // Prepare for the restart.
     final Date dateZero = new Date(0);

validator/src/main/java/com/google/daq/mqtt/sequencer/sequences/ConfigSequences.java

@@ -1,8 +1,6 @@
 package com.google.daq.mqtt.sequencer.sequences;
 
-import static com.google.daq.mqtt.sequencer.FeatureStage.Stage.ALPHA;
-import static com.google.daq.mqtt.sequencer.FeatureStage.Stage.PREVIEW;
-import static com.google.daq.mqtt.util.TimePeriodConstants.FOUR_MINUTES_MS;
+import static com.google.daq.mqtt.sequencer.Feature.Stage.ALPHA;
 import static com.google.daq.mqtt.util.TimePeriodConstants.THREE_MINUTES_MS;
 import static com.google.daq.mqtt.util.TimePeriodConstants.TWO_MINUTES_MS;
 import static com.google.udmi.util.CleanDateFormat.dateEquals;
@@ -17,8 +15,7 @@
 import static udmi.schema.Category.SYSTEM_CONFIG_RECEIVE;
 import static udmi.schema.Category.SYSTEM_CONFIG_RECEIVE_LEVEL;
 
-import com.google.daq.mqtt.sequencer.FeatureStage;
-import com.google.daq.mqtt.sequencer.FeatureStage.Stage;
+import com.google.daq.mqtt.sequencer.Feature;
 import com.google.daq.mqtt.sequencer.SequenceBase;
 import com.google.daq.mqtt.sequencer.SkipTest;
 import com.google.daq.mqtt.util.SamplingRange;
@@ -54,7 +51,7 @@ public void system_last_update() {
 
   @Test(timeout = TWO_MINUTES_MS)
   @Description("Check that the min log-level config is honored by the device.")
-  @FeatureStage(Stage.BETA)
+  @Feature()
   public void system_min_loglevel() {
     Integer savedLevel = deviceConfig.system.min_loglevel;
     assert SYSTEM_CONFIG_APPLY_LEVEL.value() >= savedLevel;
@@ -163,7 +160,7 @@ public void extra_config() {
    * Skip if: initial interval < 5s (too fast for automated test)
    */
   @Test(timeout = THREE_MINUTES_MS)
-  @FeatureStage(ALPHA)
+  @Feature(stage = ALPHA)
   @Description("device publishes pointset events at a rate of no more than config sample_rate_sec")
   public void pointset_sample_rate() {
     Integer defaultSampleRate = 10;
@@ -222,7 +219,7 @@ private String samplingMessagesCheckMessage(SamplingRange samplingRange) {
    */
   @Test(timeout = THREE_MINUTES_MS)
   @Description("test sample rate and sample limit sec")
-  @FeatureStage(ALPHA)
+  @Feature(stage = ALPHA)
   public void pointset_publish_interval() {
     // Test two narrow non-intersecting windows