diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index d3ba43a743..9f74fa68ae 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -89,7 +89,7 @@ jobs: fgrep 'Done with pubber run, exit code 193' pubber.out.2 # last_start auto-kill check udmi: - name: Sequence tests + name: Integration Tests runs-on: ubuntu-latest timeout-minutes: 20 needs: redirect # Access to UDMI-REFLECTOR is mutually exclusive diff --git a/bin/test_registrar b/bin/test_registrar index 24c0cecedf..18fe0033c6 100755 --- a/bin/test_registrar +++ b/bin/test_registrar @@ -19,30 +19,29 @@ exit_status=0 echo Found ${devices} clean devices. [ "${devices}" == 4 ] || exit_status=1 -if [[ -f ${TEST_SITE}/site_metadata.json ]]; then +# Test site_metadata settings for system.location.site. +site=$(jq -r .system.location.site < ${TEST_SITE}/site_metadata.json) - # Test site_metadata settings for system.location.site. - site=$(jq -r .system.location.site < ${TEST_SITE}/site_metadata.json) - - sm_devices=0 - for name in ${TEST_SITE}/devices/* ; do +sm_devices=0 +for name in ${TEST_SITE}/devices/* ; do if [[ -f ${name}/out/metadata_norm.json ]]; then - supplied_site=$(jq -r ".system.location.site" < ${name}/metadata.json) - # If no site value is supplied in per-device metadata, expect default. - if [[ "${supplied_site}" == "null" ]]; then - jq -e ".system.location.site == \"${site}\"" \ - ${name}/out/metadata_norm.json > /dev/null \ - && sm_devices=$[sm_devices+1] - else - jq -e ".system.location.site == \"${supplied_site}\"" \ - ${name}/out/metadata_norm.json > /dev/null \ - && sm_devices=$[sm_devices+1] - fi + supplied_site=$(jq -r ".system.location.site" < ${name}/metadata.json) + # If no site value is supplied in per-device metadata, expect default. + if [[ "${supplied_site}" == "null" ]]; then + jq -e ".system.location.site == \"${site}\"" \ + ${name}/out/metadata_norm.json > /dev/null \ + && sm_devices=$[sm_devices+1] + else + jq -e ".system.location.site == \"${supplied_site}\"" \ + ${name}/out/metadata_norm.json > /dev/null \ + && sm_devices=$[sm_devices+1] + fi fi - done +done - echo Found ${sm_devices} devices with correct site_metadata values. - [ "${sm_devices}" == "${devices}" ] || exit_status=1 -fi +echo Found ${sm_devices} devices with correct site_metadata values. +[ "${sm_devices}" == "${devices}" ] || exit_status=1 +echo Done with registrar test, exit code $exit_status exit $exit_status + diff --git a/tests/downgrade.site/registration_summary.json b/tests/downgrade.site/registration_summary.json index 07fe360d0c..8b1a18c219 100644 --- a/tests/downgrade.site/registration_summary.json +++ b/tests/downgrade.site/registration_summary.json @@ -8,6 +8,6 @@ "DWN-2" : "devices/DWN-2" }, "Version" : { - "main" : "1.3.13-87-g8566aca1" + "main" : "unknown" } } \ No newline at end of file diff --git a/validator/bin/build b/validator/bin/build index 52e87fcfbf..00b42fdcb9 100755 --- a/validator/bin/build +++ b/validator/bin/build @@ -22,7 +22,7 @@ export JAVA_HOME=$JAVA_HOME_11_X64 echo Building validataor in $PWD rm -rf build -./gradlew shadow $check +./gradlew shadow $check $* ls -l $jarfile diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java index 4be75b1c9d..97046ddade 100644 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java +++ b/validator/src/main/java/com/google/daq/mqtt/registrar/Registrar.java @@ -30,6 +30,7 @@ import java.net.URI; import java.time.Duration; import java.time.Instant; +import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; import java.util.Base64; import java.util.Date; @@ -77,7 +78,6 @@ public class Registrar { private final Map schemas = new HashMap<>(); private final String generation = getGenerationString(); private CloudIotManager cloudIotManager; - private File siteDir; private File schemaBase; private PubSubPusher updatePusher; private PubSubPusher feedPusher; @@ -92,6 +92,8 @@ public class Registrar { private Map> lastErrorSummary; private boolean validateMetadata = false; private List deviceList; + private boolean blockUnknown; + private File siteDir; /** * Main entry point for registrar. @@ -131,7 +133,8 @@ static void processArgs(List argList, Registrar registrar) { registrar.setValidateMetadata(true); break; case "--": - break; + registrar.setDeviceList(argList); + return; default: if (option.startsWith("-")) { throw new RuntimeException("Unknown cmdline option " + option); @@ -177,6 +180,8 @@ private void setValidateMetadata(boolean validateMetadata) { private void setDeviceList(List deviceList) { this.deviceList = deviceList; + Preconditions.checkNotNull(cloudIotManager, "cloudIotManager not yet defined"); + blockUnknown = false; } private void setFeedTopic(String feedTopic) { @@ -251,6 +256,7 @@ private void initializeCloudProject() { if (cloudIotManager.getUpdateTopic() != null) { updatePusher = new PubSubPusher(projectId, cloudIotManager.getUpdateTopic()); } + blockUnknown = cloudIotManager.cloudIotConfig.block_unknown; } private String getGenerationString() { @@ -264,11 +270,7 @@ private String getGenerationString() { } private void processDevices() { - processDevices(this.deviceList); - } - - private void processDevices(List devices) { - Set deviceSet = calculateDevices(devices); + Set deviceSet = calculateDevices(); AtomicInteger updatedCount = new AtomicInteger(); AtomicInteger processedCount = new AtomicInteger(); try { @@ -414,11 +416,11 @@ private void updateCloudIoT(LocalDevice localDevice) { } } - private Set calculateDevices(List devices) { - if (devices == null) { + private Set calculateDevices() { + if (deviceList == null) { return null; } - return devices.stream().map(this::deviceNameFromPath).collect(Collectors.toSet()); + return deviceList.stream().map(this::deviceNameFromPath).collect(Collectors.toSet()); } private String deviceNameFromPath(String device) { @@ -434,7 +436,7 @@ private String deviceNameFromPath(String device) { private ExceptionMap blockExtraDevices(Set extraDevices) { ExceptionMap exceptionMap = new ExceptionMap("Block devices errors"); - if (!cloudIotManager.cloudIotConfig.block_unknown) { + if (!blockUnknown) { return exceptionMap; } for (String extraName : extraDevices) { @@ -534,9 +536,9 @@ private void shutdown() { private Set fetchCloudDevices() { boolean requiresCloud = updateCloudIoT || (idleLimit != null); if (requiresCloud) { - Set devices = cloudIotManager.fetchDeviceList(); - System.err.printf("Fetched %d devices from cloud registry %s%n", - devices.size(), cloudIotManager.getRegistryPath()); + Set devices = cloudIotManager.fetchDeviceIds(); + System.err.printf("Fetched %d devices from cloud registry %s%n", devices.size(), + cloudIotManager.getRegistryId()); return devices; } else { System.err.println("Skipping remote registry fetch"); @@ -732,6 +734,10 @@ protected Map getSchemas() { return schemas; } + public List getMockActions() { + return cloudIotManager.getMockActions(); + } + class RelativeDownloader implements URIDownloader { @Override diff --git a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java b/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java deleted file mode 100644 index a329cf0c8f..0000000000 --- a/validator/src/main/java/com/google/daq/mqtt/registrar/RegistrarTest.java +++ /dev/null @@ -1,80 +0,0 @@ -package com.google.daq.mqtt.registrar; - -import static org.junit.Assert.fail; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.github.fge.jsonschema.main.JsonSchema; -import java.util.ArrayList; -import java.util.Map; -import org.junit.Test; - -/** - * Test suite for basic registrar functionality. - */ -public class RegistrarTest { - - private static final String SCHEMA_BASE_PATH = "schema"; - private static final String METADATA_JSON = "metadata.json"; - private static final String PROJECT_ID = "unit-testing"; - private static final String SITE_PATH = "../sites/udmi_site_model"; - private static final String TOOL_ROOT = "../"; - - private static final String SYSTEM_LOCATION_SITE = "ZZ-TRI-FECTA"; - private static final String DEVICE_NAME = "AHU-1"; - private ObjectMapper mapper = new ObjectMapper(); - - private static class RegistrarUnderTest extends Registrar { - protected JsonSchema getJsonSchema(String schemaName) { - return getSchemas().get(schemaName); - } - } - - private void assertErrorSummaryValidateSuccess(Map> summary) { - if ((summary == null) || (summary.get("Validating") == null) - || (summary.get("Validating").size() == 0)) { - return; - } - fail(summary.get("Validating").toString()); - } - - private void assertErrorSummaryValidateFailure(Map> summary) { - if ((summary == null) || (summary.get("Validating") == null)) { - fail("Error summary for Validating key is null"); - } - if (summary.get("Validating").size() == 0) { - fail("Error summary for Validating key is size 0"); - } - } - - private RegistrarUnderTest getRegistrarUnderTest() { - RegistrarUnderTest registrar = new RegistrarUnderTest(); - registrar.setSitePath(SITE_PATH); - registrar.setProjectId(PROJECT_ID); - registrar.setToolRoot(TOOL_ROOT); - return registrar; - } - - @Test public void metadataValidateSuccessTest() { - final RegistrarUnderTest registrar = getRegistrarUnderTest(); - - ArrayList argList = new ArrayList(); - argList.add("-s"); - argList.add(SITE_PATH); - Registrar.processArgs(argList, registrar); - registrar.execute(); - assertErrorSummaryValidateSuccess(registrar.getLastErrorSummary()); - } - - @Test public void metadataValidateFailureTest() { - final RegistrarUnderTest registrar = getRegistrarUnderTest(); - - ArrayList argList = new ArrayList(); - argList.add("-t"); - argList.add("-s"); - argList.add(SITE_PATH); - Registrar.processArgs(argList, registrar); - registrar.execute(); - assertErrorSummaryValidateFailure(registrar.getLastErrorSummary()); - } - -} diff --git a/validator/src/main/java/com/google/daq/mqtt/util/CloudIotManager.java b/validator/src/main/java/com/google/daq/mqtt/util/CloudIotManager.java index 482035e17f..92405895e1 100644 --- a/validator/src/main/java/com/google/daq/mqtt/util/CloudIotManager.java +++ b/validator/src/main/java/com/google/daq/mqtt/util/CloudIotManager.java @@ -2,36 +2,21 @@ import static com.google.daq.mqtt.util.ConfigUtil.readCloudIotConfig; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.googleapis.json.GoogleJsonResponseException; -import com.google.api.client.http.HttpRequestInitializer; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.services.cloudiot.v1.CloudIot; -import com.google.api.services.cloudiot.v1.CloudIotScopes; -import com.google.api.services.cloudiot.v1.model.BindDeviceToGatewayRequest; import com.google.api.services.cloudiot.v1.model.Device; -import com.google.api.services.cloudiot.v1.model.DeviceConfig; import com.google.api.services.cloudiot.v1.model.DeviceCredential; import com.google.api.services.cloudiot.v1.model.GatewayConfig; -import com.google.api.services.cloudiot.v1.model.ListDevicesResponse; -import com.google.api.services.cloudiot.v1.model.ModifyCloudToDeviceConfigRequest; import com.google.api.services.cloudiot.v1.model.PublicKeyCredential; -import com.google.auth.http.HttpCredentialsAdapter; -import com.google.auth.oauth2.GoogleCredentials; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; +import java.util.AbstractMap.SimpleEntry; import java.util.Base64; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; /** * Encapsulation of all Cloud IoT interaction functions. @@ -40,13 +25,12 @@ public class CloudIotManager { public static final String UDMI_METADATA = "udmi_metadata"; public static final String CLOUD_IOT_CONFIG_JSON = "cloud_iot_config.json"; - private static final String DEVICE_UPDATE_MASK = "blocked,credentials,metadata"; private static final String UDMI_CONFIG = "udmi_config"; private static final String UDMI_GENERATION = "udmi_generation"; private static final String UDMI_UPDATED = "udmi_updated"; private static final String KEY_BYTES_KEY = "key_bytes"; private static final String KEY_ALGORITHM_KEY = "key_algorithm"; - private static final int LIST_PAGE_SIZE = 1000; + private static final String MOCK_PROJECT = "unit-testing"; public final CloudIotConfig cloudIotConfig; @@ -54,9 +38,7 @@ public class CloudIotManager { private final String projectId; private final String cloudRegion; private final Map deviceMap = new ConcurrentHashMap<>(); - private CloudIot cloudIotService; - private String projectPath; - private CloudIot.Projects.Locations.Registries cloudIotRegistries; + private IotProvider iotProvider; /** * Create a new CloudIoTManager. @@ -73,11 +55,11 @@ public CloudIotManager(String projectId, File siteDir) { cloudIotConfig = validate(readCloudIotConfig(cloudConfig), projectId); registryId = cloudIotConfig.registry_id; cloudRegion = cloudIotConfig.cloud_region; - initializeCloudIoT(); + initializeIotProvider(); } catch (Exception e) { throw new RuntimeException( String.format("While initializing project %s from file %s", projectId, - cloudConfig.getAbsolutePath())); + cloudConfig.getAbsolutePath()), e); } } @@ -119,28 +101,14 @@ public static DeviceCredential makeCredentials(String keyFormat, String keyData) return deviceCredential; } - public String getRegistryPath() { - return projectPath + "/registries/" + registryId; - } - - private String getDevicePath(String deviceId) { - return getRegistryPath() + "/devices/" + deviceId; - } - - private void initializeCloudIoT() { - projectPath = "projects/" + projectId + "/locations/" + cloudRegion; + private void initializeIotProvider() { try { - System.err.println("Initializing with default credentials..."); - GoogleCredentials credential = GoogleCredentials.getApplicationDefault() - .createScoped(CloudIotScopes.all()); - JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); - HttpRequestInitializer init = new HttpCredentialsAdapter(credential); - cloudIotService = new CloudIot.Builder(GoogleNetHttpTransport.newTrustedTransport(), - jsonFactory, init).setApplicationName("com.google.iot.bos").build(); - cloudIotRegistries = cloudIotService.projects().locations().registries(); - System.err.println("Created service for project " + projectPath); + iotProvider = projectId.equals(MOCK_PROJECT) + ? new IotMockProvider(projectId, registryId, cloudRegion) + : new IotCoreProvider(projectId, registryId, cloudRegion); + System.err.println("Created service for project " + projectId); } catch (Exception e) { - throw new RuntimeException("While initializing Cloud IoT project " + projectPath, e); + throw new RuntimeException("While initializing Cloud IoT project " + projectId, e); } } @@ -153,7 +121,6 @@ private void initializeCloudIoT() { */ public boolean registerDevice(String deviceId, CloudDeviceSettings settings) { try { - Preconditions.checkNotNull(cloudIotService, "CloudIoT service not initialized"); Preconditions.checkNotNull(deviceMap, "deviceMap not initialized"); Device device = deviceMap.get(deviceId); boolean isNewDevice = device == null; @@ -170,13 +137,7 @@ public boolean registerDevice(String deviceId, CloudDeviceSettings settings) { } private void writeDeviceConfig(String deviceId, String config) { - try { - cloudIotRegistries.devices().modifyCloudToDeviceConfig(getDevicePath(deviceId), - new ModifyCloudToDeviceConfigRequest().setBinaryData( - Base64.getEncoder().encodeToString(config.getBytes()))).execute(); - } catch (Exception e) { - throw new RuntimeException("While modifying device config", e); - } + iotProvider.updateConfig(deviceId, config); } /** @@ -186,15 +147,7 @@ private void writeDeviceConfig(String deviceId, String config) { * @param blocked should this device be blocked? */ public void blockDevice(String deviceId, boolean blocked) { - try { - Device device = new Device(); - device.setBlocked(blocked); - String path = getDevicePath(deviceId); - cloudIotRegistries.devices().patch(path, device).setUpdateMask("blocked").execute(); - } catch (Exception e) { - throw new RuntimeException( - String.format("While (un)blocking device %s/%s=%s", registryId, deviceId, blocked), e); - } + iotProvider.setBlocked(deviceId, blocked); } private Device makeDevice(String deviceId, CloudDeviceSettings settings, Device oldDevice) { @@ -231,22 +184,12 @@ private GatewayConfig getGatewayConfig(CloudDeviceSettings settings) { } private void createDevice(String deviceId, CloudDeviceSettings settings) throws IOException { - try { - cloudIotRegistries.devices().create(getRegistryPath(), makeDevice(deviceId, settings, null)) - .execute(); - } catch (GoogleJsonResponseException e) { - throw new RuntimeException("Remote error creating device " + deviceId, e); - } + iotProvider.createDevice(makeDevice(deviceId, settings, null)); } private void updateDevice(String deviceId, CloudDeviceSettings settings, Device oldDevice) { - try { - Device device = makeDevice(deviceId, settings, oldDevice).setId(null).setNumId(null); - cloudIotRegistries.devices().patch(getDevicePath(deviceId), device) - .setUpdateMask(DEVICE_UPDATE_MASK).execute(); - } catch (Exception e) { - throw new RuntimeException("Remote error patching device " + deviceId, e); - } + Device device = makeDevice(deviceId, settings, oldDevice).setId(null).setNumId(null); + iotProvider.updateDevice(deviceId, device); } /** @@ -254,23 +197,8 @@ private void updateDevice(String deviceId, CloudDeviceSettings settings, Device * * @return registered device list */ - public Set fetchDeviceList() { - Preconditions.checkNotNull(cloudIotService, "CloudIoT service not initialized"); - Set allDevices = new HashSet<>(); - String nextPageToken = null; - try { - do { - ListDevicesResponse response = cloudIotRegistries.devices().list(getRegistryPath()) - .setPageToken(nextPageToken).setPageSize(LIST_PAGE_SIZE).execute(); - List devices = response.getDevices(); - allDevices.addAll(devices == null ? ImmutableList.of() : devices); - System.err.printf("Retrieved %d devices from registry...%n", allDevices.size()); - nextPageToken = response.getNextPageToken(); - } while (nextPageToken != null); - return allDevices.stream().map(Device::getId).collect(Collectors.toSet()); - } catch (Exception e) { - throw new RuntimeException("While listing devices for registry " + registryId, e); - } + public Set fetchDeviceIds() { + return iotProvider.fetchDeviceIds(); } public Device fetchDevice(String deviceId) { @@ -278,15 +206,7 @@ public Device fetchDevice(String deviceId) { } private Device fetchDeviceRaw(String deviceId) { - try { - return cloudIotRegistries.devices().get(getDevicePath(deviceId)).execute(); - } catch (Exception e) { - if (e instanceof GoogleJsonResponseException - && ((GoogleJsonResponseException) e).getDetails().getCode() == 404) { - return null; - } - throw new RuntimeException("While fetching " + deviceId, e); - } + return iotProvider.fetchDevice(deviceId); } /** @@ -334,13 +254,8 @@ public Object getCloudRegion() { return cloudRegion; } - public void bindDevice(String proxyDeviceId, String gatewayDeviceId) throws IOException { - cloudIotRegistries.bindDeviceToGateway(getRegistryPath(), - getBindRequest(proxyDeviceId, gatewayDeviceId)).execute(); - } - - private BindDeviceToGatewayRequest getBindRequest(String deviceId, String gatewayId) { - return new BindDeviceToGatewayRequest().setDeviceId(deviceId).setGatewayId(gatewayId); + public void bindDevice(String proxyDeviceId, String gatewayDeviceId) { + iotProvider.bindDeviceToGateway(proxyDeviceId, gatewayDeviceId); } /** @@ -350,36 +265,10 @@ private BindDeviceToGatewayRequest getBindRequest(String deviceId, String gatewa * @return device configuration */ public String getDeviceConfig(String deviceId) { - try { - List deviceConfigs = cloudIotRegistries.devices().configVersions() - .list(getDevicePath(deviceId)).execute().getDeviceConfigs(); - if (deviceConfigs.size() > 0) { - return new String(Base64.getDecoder().decode(deviceConfigs.get(0).getBinaryData())); - } - return null; - } catch (Exception e) { - throw new RuntimeException("While fetching device configurations for " + deviceId); - } + return iotProvider.getDeviceConfig(deviceId); } - /** - * Set the device configuration. - * - * @param deviceId target device - * @param data configuration to set - */ - public void setDeviceConfig(String deviceId, String data) { - try { - ModifyCloudToDeviceConfigRequest req = new ModifyCloudToDeviceConfigRequest(); - - String encPayload = Base64.getEncoder() - .encodeToString(data.getBytes(StandardCharsets.UTF_8.name())); - req.setBinaryData(encPayload); - - cloudIotRegistries.devices().modifyCloudToDeviceConfig(getDevicePath(deviceId), req) - .execute(); - } catch (Exception e) { - throw new RuntimeException("While setting device config for " + deviceId); - } + public List getMockActions() { + return iotProvider.getMockActions(); } } diff --git a/validator/src/main/java/com/google/daq/mqtt/util/IotCoreProvider.java b/validator/src/main/java/com/google/daq/mqtt/util/IotCoreProvider.java new file mode 100644 index 0000000000..ca44c1ae23 --- /dev/null +++ b/validator/src/main/java/com/google/daq/mqtt/util/IotCoreProvider.java @@ -0,0 +1,173 @@ +package com.google.daq.mqtt.util; + +import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; +import com.google.api.client.googleapis.json.GoogleJsonResponseException; +import com.google.api.client.http.HttpRequestInitializer; +import com.google.api.client.json.JsonFactory; +import com.google.api.client.json.jackson2.JacksonFactory; +import com.google.api.services.cloudiot.v1.CloudIot; +import com.google.api.services.cloudiot.v1.CloudIot.Builder; +import com.google.api.services.cloudiot.v1.CloudIotScopes; +import com.google.api.services.cloudiot.v1.model.BindDeviceToGatewayRequest; +import com.google.api.services.cloudiot.v1.model.Device; +import com.google.api.services.cloudiot.v1.model.DeviceConfig; +import com.google.api.services.cloudiot.v1.model.ListDevicesResponse; +import com.google.api.services.cloudiot.v1.model.ModifyCloudToDeviceConfigRequest; +import com.google.auth.http.HttpCredentialsAdapter; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.common.collect.ImmutableList; +import java.util.Base64; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +class IotCoreProvider implements IotProvider { + + private static final String DEVICE_UPDATE_MASK = "blocked,credentials,metadata"; + private static final int LIST_PAGE_SIZE = 1000; + private final CloudIot.Projects.Locations.Registries registries; + private final String projectId; + private final String cloudRegion; + private final String registryId; + + IotCoreProvider(String projectId, String registryId, String cloudRegion) { + try { + this.projectId = projectId; + this.registryId = registryId; + this.cloudRegion = cloudRegion; + System.err.println("Initializing with default credentials..."); + GoogleCredentials credential = GoogleCredentials.getApplicationDefault() + .createScoped(CloudIotScopes.all()); + JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); + HttpRequestInitializer init = new HttpCredentialsAdapter(credential); + CloudIot cloudIotService = new Builder(GoogleNetHttpTransport.newTrustedTransport(), + jsonFactory, init).setApplicationName("com.google.iot.bos").build(); + registries = cloudIotService.projects().locations().registries(); + } catch (Exception e) { + throw new RuntimeException("While creating IoTCoreProvider", e); + } + } + + @Override + public void updateConfig(String deviceId, String config) { + try { + registries.devices().modifyCloudToDeviceConfig( + getDevicePath(deviceId), + new ModifyCloudToDeviceConfigRequest().setBinaryData( + Base64.getEncoder().encodeToString(config.getBytes()))).execute(); + } catch (Exception e) { + throw new RuntimeException("While modifying device config", e); + } + } + + @Override + public void setBlocked(String deviceId, boolean blocked) { + try { + Device device = new Device(); + device.setBlocked(blocked); + registries.devices().patch(getDevicePath(deviceId), device).setUpdateMask("blocked") + .execute(); + } catch (Exception e) { + throw new RuntimeException( + String.format("While (un)blocking device %s/%s=%s", registryId, deviceId, blocked), e); + } + } + + @Override + public void updateDevice(String deviceId, Device device) { + try { + registries.devices().patch(getDevicePath(deviceId), device).setUpdateMask(DEVICE_UPDATE_MASK) + .execute(); + } catch (Exception e) { + throw new RuntimeException("Remote error patching device " + deviceId, e); + } + } + + @Override + public void createDevice(Device makeDevice) { + String deviceId = makeDevice.getId(); + try { + registries.devices().create(getRegistryPath(), makeDevice).execute(); + } catch (Exception e) { + throw new RuntimeException("Remote error creating device " + deviceId, e); + } + } + + @Override + public Device fetchDevice(String deviceId) { + try { + return registries.devices().get(getDevicePath(deviceId)).execute(); + } catch (Exception e) { + if (e instanceof GoogleJsonResponseException + && ((GoogleJsonResponseException) e).getDetails().getCode() == 404) { + return null; + } + throw new RuntimeException("While fetching " + deviceId, e); + } + } + + @Override + public void bindDeviceToGateway(String proxyDeviceId, String gatewayDeviceId) { + try { + registries.bindDeviceToGateway(getRegistryPath(), + new BindDeviceToGatewayRequest() + .setDeviceId(proxyDeviceId) + .setGatewayId(gatewayDeviceId)) + .execute(); + } catch (Exception e) { + throw new RuntimeException( + String.format("While binding device %s to %s", proxyDeviceId, gatewayDeviceId), e); + } + } + + @Override + public Set fetchDeviceIds() { + Set allDevices = new HashSet<>(); + String nextPageToken = null; + try { + do { + ListDevicesResponse response = registries.devices().list(getRegistryPath()) + .setPageToken(nextPageToken).setPageSize(LIST_PAGE_SIZE).execute(); + java.util.List devices = response.getDevices(); + allDevices.addAll(devices == null ? ImmutableList.of() : devices); + System.err.printf("Retrieved %d devices from registry...%n", allDevices.size()); + nextPageToken = response.getNextPageToken(); + } while (nextPageToken != null); + return allDevices.stream().map(Device::getId).collect(Collectors.toSet()); + } catch (Exception e) { + throw new RuntimeException("While listing devices for registry " + registryId, e); + } + } + + @Override + public String getDeviceConfig(String deviceId) { + try { + List deviceConfigs = registries.devices().configVersions() + .list(getDevicePath(deviceId)).execute().getDeviceConfigs(); + if (deviceConfigs.size() > 0) { + return new String(Base64.getDecoder().decode(deviceConfigs.get(0).getBinaryData())); + } + return null; + } catch (Exception e) { + throw new RuntimeException("While fetching device configurations for " + deviceId, e); + } + } + + private String getProjectPath() { + return "projects/" + projectId + "/locations/" + cloudRegion; + } + + private String getRegistryPath() { + return getProjectPath() + "/registries/" + registryId; + } + + private String getDevicePath(String deviceId) { + return getRegistryPath() + "/devices/" + deviceId; + } + + @Override + public List getMockActions() { + throw new RuntimeException("This is not a mock provider!"); + } +} diff --git a/validator/src/main/java/com/google/daq/mqtt/util/IotMockProvider.java b/validator/src/main/java/com/google/daq/mqtt/util/IotMockProvider.java new file mode 100644 index 0000000000..a18cbd0b7c --- /dev/null +++ b/validator/src/main/java/com/google/daq/mqtt/util/IotMockProvider.java @@ -0,0 +1,101 @@ +package com.google.daq.mqtt.util; + +import com.google.api.services.cloudiot.v1.model.Device; +import com.google.udmi.util.SiteModel; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Mocked IoT provider for unit testing. + */ +public class IotMockProvider implements IotProvider { + + public static final String MOCK_DEVICE_ID = "MOCK-1"; + private final SiteModel siteModel; + private List mockActions = new ArrayList<>(); + public static final String BLOCK_DEVICE_ACTION = "block"; + public static final String UPDATE_DEVICE_ACTION = "update"; + public static final String BIND_DEVICE_ACTION = "bind"; + public static final String CONFIG_DEVICE_ACTION = "config"; + + IotMockProvider(String projectId, String registryId, String cloudRegion) { + siteModel = new SiteModel("../sites/udmi_site_model"); + siteModel.initialize(); + } + + private void mockAction(String action, String deviceId, Object paramater) { + MockAction mockAction = new MockAction(); + mockAction.action = action; + mockAction.deviceId = deviceId; + mockAction.data = paramater; + mockActions.add(mockAction); + } + + @Override + public void updateConfig(String deviceId, String config) { + mockAction(CONFIG_DEVICE_ACTION, deviceId, config); + } + + @Override + public void setBlocked(String deviceId, boolean blocked) { + mockAction(BLOCK_DEVICE_ACTION, deviceId, blocked); + } + + @Override + public void updateDevice(String deviceId, Device device) { + mockAction(UPDATE_DEVICE_ACTION, deviceId, device); + } + + @Override + public void createDevice(Device makeDevice) { + throw new RuntimeException("Not yet implemented"); + } + + @Override + public Device fetchDevice(String deviceId) { + Device device = new Device(); + SiteModel.Device modelDevice = siteModel.getDevice(deviceId); + device.setId(deviceId); + device.setNumId(new BigInteger("" + Objects.hash(deviceId), 10)); + return device; + } + + @Override + public void bindDeviceToGateway(String proxyDeviceId, String gatewayDeviceId) { + mockAction(BIND_DEVICE_ACTION, proxyDeviceId, gatewayDeviceId); + } + + @Override + public Set fetchDeviceIds() { + HashSet deviceIds = new HashSet<>(siteModel.allDeviceIds()); + deviceIds.add(MOCK_DEVICE_ID); + return deviceIds; + } + + @Override + public String getDeviceConfig(String deviceId) { + throw new RuntimeException("Not yet implemented"); + } + + @Override + public List getMockActions() { + List savedActions = mockActions.stream().map(a -> (Object) a) + .collect(Collectors.toList()); + mockActions = new ArrayList<>(); + return savedActions; + } + + /** + * Holder class for mocked actions. + */ + public static class MockAction { + public String action; + public String deviceId; + public Object data; + } +} diff --git a/validator/src/main/java/com/google/daq/mqtt/util/IotProvider.java b/validator/src/main/java/com/google/daq/mqtt/util/IotProvider.java new file mode 100644 index 0000000000..d97b421efb --- /dev/null +++ b/validator/src/main/java/com/google/daq/mqtt/util/IotProvider.java @@ -0,0 +1,82 @@ +package com.google.daq.mqtt.util; + +import com.google.api.services.cloudiot.v1.model.Device; +import java.util.List; +import java.util.Set; + +/** + * Abstraction for a cloud-based IoT provider. Provides methods for all the different operations + * that UDMI tools need to do on the target registry. Nominally for GCP IoT Core, but can also be + * mocked or backed by different providers. + */ +public interface IotProvider { + + /** + * Update the device config with the supplied block (usually JSON). + * + * @param deviceId device to update + * @param config config data block + */ + void updateConfig(String deviceId, String config); + + /** + * Set the blocked (not receiving) status for a given device. + * + * @param deviceId device to (un)block + * @param blocked block or not + */ + void setBlocked(String deviceId, boolean blocked); + + /** + * Update a device entry with { blocked, credentials, metadata } fields from the provided Device. + * + * @param deviceId device to update + * @param device data to update with + */ + void updateDevice(String deviceId, Device device); + + /** + * Create a new device entry. + * + * @param makeDevice device specification to create + */ + void createDevice(Device makeDevice); + + /** + * Fetch a Device object for the given device. + * + * @param deviceId device id to fetch + * @return Device object + */ + Device fetchDevice(String deviceId); + + /** + * Make the given proxy device bound to the given gateway. + * + * @param proxyDeviceId device to bind + * @param gatewayDeviceId thing to bind to + */ + void bindDeviceToGateway(String proxyDeviceId, String gatewayDeviceId); + + /** + * Return all the device ids currently registered. + * + * @return set of registered device ids + */ + Set fetchDeviceIds(); + + /** + * Get the device config blob for the indicated device. + * + * @param deviceId device to query + * @return config blob + */ + String getDeviceConfig(String deviceId); + + /** + * Get a list of mocked device objects. Used for unit testing only with mocked implementation. + * + * @return list of mocked objects + */ + List getMockActions(); +} diff --git a/validator/src/test/java/com/google/daq/mqtt/registrar/RegistrarTest.java b/validator/src/test/java/com/google/daq/mqtt/registrar/RegistrarTest.java new file mode 100644 index 0000000000..ecaacb119e --- /dev/null +++ b/validator/src/test/java/com/google/daq/mqtt/registrar/RegistrarTest.java @@ -0,0 +1,124 @@ +package com.google.daq.mqtt.registrar; + +import static com.google.daq.mqtt.util.IotMockProvider.BIND_DEVICE_ACTION; +import static com.google.daq.mqtt.util.IotMockProvider.BLOCK_DEVICE_ACTION; +import static com.google.daq.mqtt.util.IotMockProvider.MOCK_DEVICE_ID; +import static com.google.daq.mqtt.util.IotMockProvider.UPDATE_DEVICE_ACTION; +import static java.lang.Boolean.TRUE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import com.google.api.services.cloudiot.v1.model.Device; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.daq.mqtt.util.IotMockProvider.MockAction; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.junit.Test; + +/** + * Test suite for basic registrar functionality. + */ +public class RegistrarTest { + + private static final String PROJECT_ID = "unit-testing"; + private static final String SITE_PATH = "../sites/udmi_site_model"; + private static final String TOOL_ROOT = "../"; + + private void assertErrorSummaryValidateSuccess(Map> summary) { + if ((summary == null) || (summary.get("Validating") == null) + || (summary.get("Validating").size() == 0)) { + return; + } + fail(summary.get("Validating").toString()); + } + + private void assertErrorSummaryValidateFailure(Map> summary) { + if ((summary == null) || (summary.get("Validating") == null)) { + fail("Error summary for Validating key is null"); + } + if (summary.get("Validating").size() == 0) { + fail("Error summary for Validating key is size 0"); + } + } + + private Registrar getRegistrar(List args) { + try { + Registrar registrar = new Registrar(); + registrar.setSitePath(SITE_PATH); + registrar.setProjectId(PROJECT_ID); + registrar.setToolRoot(TOOL_ROOT); + if (args != null) { + Registrar.processArgs(new ArrayList<>(args), registrar); + } + return registrar; + } catch (Exception e) { + throw new RuntimeException("While getting test registrar", e); + } + } + + @Test + public void metadataValidateSuccessTest() { + Registrar registrar = getRegistrar(null); + registrar.execute(); + assertErrorSummaryValidateSuccess(registrar.getLastErrorSummary()); + } + + @Test + public void metadataValidateFailureTest() { + Registrar registrar = getRegistrar(ImmutableList.of("-t")); + registrar.execute(); + assertErrorSummaryValidateFailure(registrar.getLastErrorSummary()); + } + + @Test + public void noBlockDevicesTest() { + List mockActions = getMockedActions(ImmutableList.of("-u", "--", "AHU-1")); + mockActions.forEach(action -> assertEquals("Mocked device " + action.action, "AHU-1", + action.deviceId)); + assertTrue("Device is not blocked", filterActions(mockActions, BLOCK_DEVICE_ACTION).stream() + .allMatch(action -> action.data.equals(TRUE))); + } + + @Test + public void basicUpdates() { + List mockActions = getMockedActions(ImmutableList.of("-u")); + List blockActions = filterActions(mockActions, BLOCK_DEVICE_ACTION); + assertEquals("block action count", 1, blockActions.size()); + assertEquals("block action distinct devices", blockActions.size(), + blockActions.stream().map(action -> action.deviceId).collect( + Collectors.toSet()).size()); + blockActions.forEach(action -> assertEquals("device blocked " + action.deviceId, + action.deviceId.equals(MOCK_DEVICE_ID), action.data)); + List updateActions = filterActions(mockActions, UPDATE_DEVICE_ACTION); + assertEquals("Devices updated", 4, updateActions.size()); + assertTrue("all devices not blocked", updateActions.stream().allMatch(this::isNotBlocking)); + List bindActions = filterActions(mockActions, BIND_DEVICE_ACTION); + assertEquals("bind actions", 2, bindActions.size()); + assertTrue("bind gateway", + bindActions.stream().allMatch(action -> action.data.equals("GAT-123"))); + assertEquals("bind devices", ImmutableSet.of("SNS-4", "AHU-22"), + bindActions.stream().map(action -> action.deviceId).collect( + Collectors.toSet())); + } + + private Boolean isNotBlocking(MockAction action) { + return !TRUE.equals(((Device) action.data).getBlocked()); + } + + private List filterActions(List mockActions, String actionKey) { + return mockActions.stream() + .filter(action -> action.action.equals(actionKey)) + .collect(Collectors.toList()); + } + + private List getMockedActions(ImmutableList optArgs) { + Registrar registrar = getRegistrar(optArgs); + registrar.execute(); + return registrar.getMockActions().stream().map(a -> (MockAction) a) + .collect(Collectors.toList()); + } +}