Changes in the mapping agent to map the extra unknown discovered devices to devices in the site model (#1128)

jainrocks · web-flow · commit 26cffc5b173a · 2025-06-14T12:33:09.000+05:30
* Handling spurious log for last start updates when the last start is not changed * updating comment style * changes from master * Revert "changes from master" This reverts commit d53c53b. * changes in the mapping agent to map the extra unknown discovered devices to devices in the site model * changing the map command only and not the filename used earlier * changing the file name for map command * changing the file name for map command * file name to be used for the new device discovered * changes to address review comments * changes to address review comments * changes to address review comments * changes to address review comments * changes to address review comments * update in test/automapper to call the mapping command * fixing unit tests failure * using automapper to start mapping and test the process * using automapper to start mapping and test the process * addressing review comments
diff --git a/bin/test_automapper b/bin/test_automapper
@@ -59,6 +59,36 @@ echo Checking discovered device metadata
 addr=$(jq -r .addr $site_path/extras/discovered_vendor-20231/cloud_metadata/udmi_discovered_with.json)
 [[ $addr == 20231 ]] || fail Vendor device addr should be 20231, was $addr
 
+echo "Changing the address for AHU-22 device to 0x68"
+device_metadata=$site_path/devices/AHU-22/metadata.json
+jq '.localnet.families.vendor.addr = "0x68"' $device_metadata | sponge $device_metadata
+
+echo "Starting the mapping process for the extra devices"
+bin/mapper GAT-123 map
+
+[[ -f $site_path/devices/UNK-1/metadata.json ]] || fail "Terminating because of extra device which does not exist didn't map correctly."
+
+echo Checking address of new mapped device
+addr=$(jq -r .localnet.families.vendor.addr $site_path/devices/UNK-1/metadata.json )
+[[ $addr == 0x65 ]] || fail Vendor device addr should be 0x65, was $addr
+
+echo "Reverting the address for AHU-22 device back"
+device_metadata=$site_path/devices/AHU-22/metadata.json
+jq '.localnet.families.vendor.addr = "0x65"' $device_metadata | sponge $device_metadata
+
+
+if [[ -d $site_path/devices/UNK-1 ]]; then
+  echo "Directory '$site_path/devices/UNK-1' exists. Removing it..."
+  if rm -r "$site_path/devices/UNK-1"; then
+    echo "Directory '$site_path/devices/UNK-1' removed successfully."
+  else
+    echo "Error: Failed to remove '$site_path/devices/UNK-1'."
+    exit 1
+  fi
+else
+  echo "Directory '$site_path/devices/UNK-1' does not exist. Nothing to remove."
+fi
+
 first_generation=`jq -r .generation $site_path/extras/discovered_vendor-20231/cloud_metadata/udmi_discovered_with.json`
 echo First discovered generation is $first_generation
 
diff --git a/common/src/main/java/com/google/udmi/util/Common.java b/common/src/main/java/com/google/udmi/util/Common.java
@@ -13,6 +13,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.MissingFormatArgumentException;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.TreeSet;
@@ -71,6 +72,10 @@ public abstract class Common {
   public static final int EXIT_CODE_ERROR = 1;
   public static final String UNKNOWN_UDMI_VERSION = "unknown";
 
+  public static final String UNKNOWN_DEVICE_ID_PREFIX = "UNK-";
+
+  public static final String DOUBLE_COLON_SEPARATOR = "::";
+
   /**
    * Remove the next item from the list in an exception-safe way.
    */
@@ -208,4 +213,10 @@ public static void forcedDelayedShutdown() {
     safeSleep(2000);
     System.exit(0);
   }
+
+  public static String generateColonKey(String field1, String field2) {
+    Objects.requireNonNull(field1, "field1 cannot be null");
+    Objects.requireNonNull(field2, "field2 cannot be null");
+    return field1 + DOUBLE_COLON_SEPARATOR + field2;
+  }
 }
diff --git a/common/src/main/java/com/google/udmi/util/SiteModel.java b/common/src/main/java/com/google/udmi/util/SiteModel.java
@@ -629,6 +629,13 @@ public boolean getStrictWarnings() {
     return isTrue(siteMetadata.strict_warnings);
   }
 
+  public void createNewDevice(String deviceId, Metadata metadata) {
+    File metadataFile = getDeviceFile(deviceId, METADATA_JSON);
+    System.err.println("Writing device metadata file " + metadataFile);
+    metadataFile.getParentFile().mkdirs();
+    JsonUtil.writeFile(metadata, metadataFile);
+  }
+
   public static class MetadataException extends Metadata {
 
     public final File file;
diff --git a/validator/.idea/runConfigurations/MappingAgent_Map.xml b/validator/.idea/runConfigurations/MappingAgent_Map.xml
diff --git a/validator/src/main/java/com/google/daq/mqtt/mapping/MappingAgent.java b/validator/src/main/java/com/google/daq/mqtt/mapping/MappingAgent.java
@@ -3,6 +3,8 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.daq.mqtt.util.ConfigUtil.UDMI_VERSION;
+import static com.google.udmi.util.Common.UNKNOWN_DEVICE_ID_PREFIX;
+import static com.google.udmi.util.Common.generateColonKey;
 import static com.google.udmi.util.Common.removeNextArg;
 import static com.google.udmi.util.GeneralUtils.catchToNull;
 import static com.google.udmi.util.JsonUtil.isoConvert;
@@ -21,7 +23,6 @@
 import com.google.daq.mqtt.util.CloudIotManager;
 import com.google.daq.mqtt.util.ConfigUtil;
 import com.google.udmi.util.Common;
-import com.google.udmi.util.JsonUtil;
 import com.google.udmi.util.SiteModel;
 import java.io.File;
 import java.util.ArrayList;
@@ -33,6 +34,7 @@
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
 import udmi.schema.CloudModel;
 import udmi.schema.CloudModel.ModelOperation;
 import udmi.schema.DiscoveryConfig;
@@ -41,7 +43,9 @@
 import udmi.schema.Envelope.SubFolder;
 import udmi.schema.ExecutionConfiguration;
 import udmi.schema.FamilyDiscoveryConfig;
+import udmi.schema.FamilyLocalnetModel;
 import udmi.schema.GatewayModel;
+import udmi.schema.LocalnetModel;
 import udmi.schema.Metadata;
 import udmi.schema.SystemModel;
 
@@ -60,6 +64,8 @@ public class MappingAgent {
   private SiteModel siteModel;
   private Date generationDate;
 
+  private AtomicInteger suffixToStart = new AtomicInteger(1);
+
   /**
    * Create an agent given the configuration.
    */
@@ -98,7 +104,7 @@ void process(List<String> argsList) {
       switch (mappingCommand) {
         case "provision" -> setupProvision();
         case "discover" -> initiateDiscover();
-        case "reconcile" -> reconcileDiscovery();
+        case "map" -> mapDiscoveredDevices();
         default -> throw new RuntimeException("Unknown mapping command " + mappingCommand);
       }
       System.err.printf("Completed mapper %s command%n", mappingCommand);
@@ -141,26 +147,83 @@ private FamilyDiscoveryConfig getFamilyDiscoveryConfig(String family) {
     return familyDiscoveryConfig;
   }
 
-  private void reconcileDiscovery() {
+  private void mapDiscoveredDevices() {
+    List<Entry<String, Metadata>> mappedDiscoveredEntries = getMappedDiscoveredEntries();
+    Map<String, Metadata> devicesEntriesMap = getDevicesEntries();
+    Map<String, String> devicesFamilyAddressMap = getDeviceFamilyAddressMap();
+
+    Set<String> devicesPresent = new HashSet<>();
+    mappedDiscoveredEntries.forEach(entry -> {
+
+      if (devicesEntriesMap.containsKey(entry.getKey())) {
+        System.err.println("Skipping existing device file for family::address = " + entry.getKey());
+        devicesPresent.add(devicesFamilyAddressMap.get(entry.getKey()));
+        //TODO: update the existing device
+      } else {
+        String newDeviceId = getNextDeviceId();
+        while (devicesPresent.contains(newDeviceId)) {
+          newDeviceId = getNextDeviceId();
+        }
+        devicesPresent.add(newDeviceId);
+        siteModel.createNewDevice(newDeviceId, entry.getValue());
+      }
+    });
+
+    updateProxyIdsForDiscoveryNode(devicesPresent);
+  }
+
+  private Map<String, String> getDeviceFamilyAddressMap() {
+    Map<String, String> devicesFamilyAddressMap = new HashMap<>();
+
+    for (String device : siteModel.allMetadata().keySet()) {
+      Metadata deviceMetadata = siteModel.allMetadata().get(device);
+      if (deviceMetadata.localnet == null || deviceMetadata.localnet.families == null) {
+        continue;
+      }
+      Map<String, FamilyLocalnetModel> deviceFamilies = deviceMetadata.localnet.families;
+      for (String familyName : deviceFamilies.keySet()) {
+        devicesFamilyAddressMap.put(generateColonKey(familyName,
+            deviceFamilies.get(familyName).addr), device);
+      }
+    }
+
+    return devicesFamilyAddressMap;
+  }
+
+  private String getNextDeviceId() {
+    return UNKNOWN_DEVICE_ID_PREFIX + suffixToStart.getAndIncrement();
+  }
+
+  private List<Entry<String, Metadata>> getMappedDiscoveredEntries() {
     File extrasDir = siteModel.getExtrasDir();
     File[] extras = extrasDir.listFiles();
     if (extras == null || extras.length == 0) {
       throw new RuntimeException("No extras found to reconcile");
     }
-    List<Entry<String, Metadata>> entries = Arrays.stream(extras).map(this::convertExtra)
+    List<Entry<String, Metadata>> mappedDiscoveredEntries = Arrays.stream(extras)
+        .map(this::convertExtraDiscoveredEvents)
         .filter(entry -> !entry.getKey().equals(NO_DISCOVERY)).toList();
-    entries.forEach(entry -> {
-      File metadataFile = siteModel.getDeviceFile(entry.getKey(), METADATA_JSON);
-      if (metadataFile.exists()) {
-        System.err.println("Skipping existing device file " + metadataFile);
-      } else {
-        System.err.println("Writing device metadata file " + metadataFile);
-        metadataFile.getParentFile().mkdirs();
-        JsonUtil.writeFile(entry.getValue(), metadataFile);
+    return mappedDiscoveredEntries;
+  }
+
+  private Map<String, Metadata> getDevicesEntries() {
+    Map<String, Metadata> devicesEntriesMap = new HashMap<>();
+
+    for (Metadata deviceMetadata : siteModel.allMetadata().values()) {
+      if (deviceMetadata.localnet == null || deviceMetadata.localnet.families == null) {
+        continue;
       }
-    });
+      Map<String, FamilyLocalnetModel> deviceFamilies = deviceMetadata.localnet.families;
+      for (String familyName : deviceFamilies.keySet()) {
+        devicesEntriesMap.put(generateColonKey(familyName,
+            deviceFamilies.get(familyName).addr), deviceMetadata);
+      }
+    }
+
+    return devicesEntriesMap;
+  }
 
-    List<String> proxyIds = entries.stream().map(Entry::getKey).toList();
+  private void updateProxyIdsForDiscoveryNode(Set<String> proxyIds) {
     File gatewayMetadata = siteModel.getDeviceFile(deviceId, METADATA_JSON);
     Metadata metadata = loadFileStrictRequired(Metadata.class, gatewayMetadata);
     List<String> idList = ofNullable(metadata.gateway.proxy_ids).orElse(ImmutableList.of());
@@ -173,20 +236,34 @@ private void reconcileDiscovery() {
     writeFile(metadata, gatewayMetadata);
   }
 
-  private Entry<String, Metadata> convertExtra(File file) {
+  private Entry<String, Metadata> convertExtraDiscoveredEvents(File file) {
     DiscoveryEvents discoveryEvents = loadFileStrict(DiscoveryEvents.class,
         new File(file, "cloud_metadata/udmi_discovered_with.json"));
-    if (discoveryEvents == null) {
+    boolean isInvalidDiscoveryEvent = false;
+    if (discoveryEvents.family == null || discoveryEvents.addr == null) {
+      System.err.println("Invalid discovery event, family or address not present");
+      isInvalidDiscoveryEvent = true;
+    }
+    if (discoveryEvents == null || isInvalidDiscoveryEvent) {
       return Map.entry(NO_DISCOVERY, new Metadata());
     }
     Metadata metadata = new Metadata();
     metadata.version = UDMI_VERSION;
     metadata.timestamp = new Date();
     metadata.system = new SystemModel();
     metadata.gateway = new GatewayModel();
+    populateMetadataLocalnet(discoveryEvents, metadata);
     metadata.gateway.gateway_id = deviceId;
-    String deviceName = (String) discoveryEvents.system.ancillary.get("device-name");
-    return Map.entry(deviceName, metadata);
+    return Map.entry(generateColonKey(discoveryEvents.family,
+        discoveryEvents.addr), metadata);
+  }
+
+  private static void populateMetadataLocalnet(DiscoveryEvents discoveryEvents, Metadata metadata) {
+    metadata.localnet = new LocalnetModel();
+    metadata.localnet.families = new HashMap<>();
+    FamilyLocalnetModel familyLocalnetModel = new FamilyLocalnetModel();
+    familyLocalnetModel.addr = discoveryEvents.addr;
+    metadata.localnet.families.put(discoveryEvents.family, familyLocalnetModel);
   }
 
   private void initialize() {
