[projmgr] Update list layers behaviour: load device/board specific packs

libs/rtemodel/test/src/RteModelTest.cpp

@@ -323,9 +323,9 @@ TEST_F(RteModelPrjTest, LoadCprj) {
   EXPECT_EQ(boardName, "RteTest Test board");
   // get layers
   auto& allLayerDescriptors = rteKernel.GetGlobalModel()->GetLayerDescriptors();
-  EXPECT_EQ(allLayerDescriptors.size(), 8);
+  EXPECT_EQ(allLayerDescriptors.size(), 9);
   auto& filteredLayerDescriptors = activeTarget->GetFilteredModel()->GetLayerDescriptors();
-  EXPECT_EQ(filteredLayerDescriptors.size(), 6);
+  EXPECT_EQ(filteredLayerDescriptors.size(), 7);
 
   const string rteDir = RteUtils::ExtractFilePath(RteTestM3_cprj, true) + "RTE/";
   const string CompConfig_0_Base_Version = rteDir + "RteTest/" + "[email protected]";
@@ -804,9 +804,9 @@ TEST_F(RteModelPrjTest, LoadCprjM4) {
   EXPECT_TRUE(boardName.empty());
   // get layers
   auto& allLayerDescriptors = rteKernel.GetGlobalModel()->GetLayerDescriptors();
-  EXPECT_EQ(allLayerDescriptors.size(), 8);
+  EXPECT_EQ(allLayerDescriptors.size(), 9);
   auto& filteredLayerDescriptors = activeTarget->GetFilteredModel()->GetLayerDescriptors();
-  EXPECT_EQ(filteredLayerDescriptors.size(), 8);
+  EXPECT_EQ(filteredLayerDescriptors.size(), 9);
 
   const string projDir = RteUtils::ExtractFilePath(RteTestM4_cprj, true);
   const string rteDir = projDir + "RTE/";
@@ -872,9 +872,9 @@ TEST_F(RteModelPrjTest, LoadCprjM4_Board) {
   EXPECT_EQ(boardName, "RteTest CM4 board");
   // get layers
   auto& allLayerDescriptors = rteKernel.GetGlobalModel()->GetLayerDescriptors();
-  EXPECT_EQ(allLayerDescriptors.size(), 8);
+  EXPECT_EQ(allLayerDescriptors.size(), 9);
   auto& filteredLayerDescriptors = activeTarget->GetFilteredModel()->GetLayerDescriptors();
-  EXPECT_EQ(filteredLayerDescriptors.size(), 5);
+  EXPECT_EQ(filteredLayerDescriptors.size(), 6);
 
   const string projDir = RteUtils::ExtractFilePath(RteTestM4_Board_cprj, true);
   const string rteDir = projDir + "RTE_BOARD/";

test/packs/ARM/RteTest_DFP/0.2.0/ARM.RteTest_DFP.pdsc

@@ -518,6 +518,10 @@
     <clayer name="Config2" type="Config2" file="Layers/config2.clayer.yml">
       <description>"Test board layer with shield"</description>
     </clayer>
+    <clayer name="BoardSpecific" type="BoardSpecific" file="Layers/board-specific.clayer.yml">
+      <description>"Test board layer with board specific pack"</description>
+    </clayer>
+
   </clayers>
 
 </package>

test/packs/ARM/RteTest_DFP/0.2.0/Layers/board-specific.clayer.yml

@@ -0,0 +1,22 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/Open-CMSIS-Pack/devtools/main/tools/projmgr/schemas/clayer.schema.json
+
+layer:
+
+  packs:
+    - pack: ARM::[email protected]
+
+  type: BoardSpecific
+
+  for-board: RteTest board listing
+
+  components:
+    - component: Board:Test:Rev1
+
+  connections:
+    - connect: Board1 Connections
+      consumes:
+        - ExactMatch: 42                   # both key and value exact match
+        - EmptyConsumedValue               # key exact match, consumed value is empty
+        - EmptyValues                      # key exact match, both values empty
+        - AddedValueLessThanProvided: +49  # added consumed values are less than provided
+        - AddedValueEqualToProvided: +499  # added consumed values are equal to provided

tools/projmgr/include/ProjMgrUtils.h

@@ -102,6 +102,11 @@ typedef std::map<std::string, StrVec> StrVecMap;
 */
 typedef std::map<std::string, int> IntMap;
 
+/**
+ * @brief map of bool
+*/
+typedef std::map<std::string, bool> BoolMap;
+
 /**
  * @brief map of string
 */
@@ -201,6 +206,14 @@ class ProjMgrUtils {
   static void PushBackUniquely(std::list<std::string>& lst, const std::string& value);
   static void PushBackUniquely(StrPairVec& vec, const StrPair& value);
 
+  /**
+   * @brief merge two string vector maps
+   * @param map1 first string vector map
+   * @param map2 second string vector map
+   * @return StrVecMap merged map
+  */
+ static StrVecMap MergeStrVecMap(const StrVecMap& map1, const StrVecMap& map2);
+
   /**
    * @brief convert string to int, return 0 if it's empty or not convertible
    * @param string

tools/projmgr/include/ProjMgrWorker.h

@@ -29,6 +29,24 @@ struct ConnectionsValidationResult {
   StrPairPtrVec provides;
 };
 
+/**
+ * @brief layers discovering variables
+ *        required layer types,
+ *        missed required types,
+ *        optional type flags,
+ *        generic clayers from search path,
+ *        generic clayers from packs,
+ *        candidate layers,
+*/
+struct LayersDiscovering {
+  StrVec requiredLayerTypes;
+  StrVec missedRequiredTypes;
+  BoolMap optionalTypeFlags;
+  StrVecMap genericClayersFromSearchPath;
+  StrVecMap genericClayersFromPacks;
+  StrVecMap candidateClayers;
+};
+
 /**
  * @brief connections lists
  *        list of consumes
@@ -646,10 +664,14 @@ class ProjMgrWorker {
   void PrintConnectionsValidation(ConnectionsValidationResult result, std::string& msg);
   bool CollectLayersFromPacks(ContextItem& context, StrVecMap& clayers);
   bool CollectLayersFromSearchPath(const std::string& clayerSearchPath, StrVecMap& clayers);
-  bool DiscoverMatchingLayers(ContextItem& context, const std::string& clayerSearchPath);
+  void GetRequiredLayerTypes(ContextItem& context, LayersDiscovering& discover);
+  bool GetCandidateLayers(LayersDiscovering& discover);
+  bool ProcessCandidateLayers(ContextItem& context, LayersDiscovering& discover);
+  bool ProcessLayerCombinations(ContextItem& context, LayersDiscovering& discover);
+  bool DiscoverMatchingLayers(ContextItem& context, std::string clayerSearchPath);
   void CollectConnections(ContextItem& context, ConnectionsCollectionVec& connections);
   void GetConsumesProvides(const ConnectionsCollectionVec& collection, ConnectionsList& connections);
-  ConnectionsCollectionMap ClassifyConnections(const ConnectionsCollectionVec& connections, std::map<std::string, bool> optionalTypeFlags);
+  ConnectionsCollectionMap ClassifyConnections(const ConnectionsCollectionVec& connections, BoolMap optionalTypeFlags);
   ConnectionsValidationResult ValidateConnections(ConnectionsCollectionVec combination);
   void GetAllCombinations(const ConnectionsCollectionMap& src, const ConnectionsCollectionMap::iterator& it,
     std::vector<ConnectionsCollectionVec>& combinations, const ConnectionsCollectionVec& previous = ConnectionsCollectionVec());
@@ -673,6 +695,7 @@ class ProjMgrWorker {
   void ExpandAccessSequence(const ContextItem& context, const ContextItem& refContext, const std::string& sequence, std::string& item, bool withHeadingDot);
   bool GetGeneratorDir(const RteGenerator* generator, ContextItem& context, const std::string& layer, std::string& genDir);
   bool ParseContextLayers(ContextItem& context);
+  bool AddPackRequirements(ContextItem& context, const std::vector<PackItem> packRequirements);
 };
 
 #endif  // PROJMGRWORKER_H

tools/projmgr/src/ProjMgrUtils.cpp

@@ -105,6 +105,12 @@ void ProjMgrUtils::PushBackUniquely(StrPairVec& vec, const StrPair& value) {
   vec.push_back(value);
 }
 
+StrVecMap ProjMgrUtils::MergeStrVecMap(const StrVecMap& map1, const StrVecMap& map2) {
+  StrVecMap mergedMap(map1);
+  mergedMap.insert(map2.begin(), map2.end());
+  return mergedMap;
+}
+
 int ProjMgrUtils::StringToInt(const string& value) {
   int intValue = 0;
   smatch sm;

tools/projmgr/src/ProjMgrWorker.cpp

@@ -587,53 +587,118 @@ bool ProjMgrWorker::CollectLayersFromSearchPath(const string& clayerSearchPath,
   return true;
 }
 
-bool ProjMgrWorker::DiscoverMatchingLayers(ContextItem& context, const string& clayerSearchPath) {
-  // required layer types
-  StrVec requiredLayerTypes;
-  map<string, bool> optionalTypeFlags;
+void ProjMgrWorker::GetRequiredLayerTypes(ContextItem& context, LayersDiscovering& discover) {
   for (const auto& clayer : context.cproject->clayers) {
     if (clayer.type.empty() || !CheckContextFilters(clayer.typeFilter, context) ||
       (ExpandString(clayer.layer, context.variables) != clayer.layer)) {
       continue;
     }
-    requiredLayerTypes.push_back(clayer.type);
-    optionalTypeFlags[clayer.type] = clayer.optional;
+    discover.requiredLayerTypes.push_back(clayer.type);
+    discover.optionalTypeFlags[clayer.type] = clayer.optional;
   }
+}
+
+bool ProjMgrWorker::ProcessCandidateLayers(ContextItem& context, LayersDiscovering& discover) {
+  // get all candidate layers
+  if (!GetCandidateLayers(discover)) {
+    return false;
+  }
+  // load device/board specific packs specified in candidate layers
+  vector<PackItem> packRequirements;
+  for (const auto& [type, clayers] : discover.candidateClayers) {
+    for (const auto& clayer : clayers) {
+      const ClayerItem& clayerItem = m_parser->GetGenericClayers()[clayer];
+      if (!clayerItem.forBoard.empty() || !clayerItem.forDevice.empty()) {
+        packRequirements.insert(packRequirements.end(), clayerItem.packs.begin(), clayerItem.packs.end());
+      }
+    }
+  }
+  if (packRequirements.size() > 0) {
+    AddPackRequirements(context, packRequirements);
+    if (!LoadAllRelevantPacks() || !LoadPacks(context)) {
+      return false;
+    }
+  }
+  // process board/device filtering
+  if (!ProcessDevice(context)) {
+    return false;
+  }
+  if (!SetTargetAttributes(context, context.targetAttributes)) {
+    return false;
+  }
+  // recollect layers from packs after filtering
+  discover.genericClayersFromPacks.clear();
+  if (!CollectLayersFromPacks(context, discover.genericClayersFromPacks)) {
+    return false;
+  }
+  discover.candidateClayers.clear();
+  if (!GetCandidateLayers(discover)) {
+    return false;
+  }
+  return true;
+}
+
+bool ProjMgrWorker::GetCandidateLayers(LayersDiscovering& discover) {
+  // clayers matching required types
+  StrVecMap genericClayers = ProjMgrUtils::MergeStrVecMap(discover.genericClayersFromSearchPath, discover.genericClayersFromPacks);
+  for (const auto& requiredType : discover.requiredLayerTypes) {
+    if (genericClayers.find(requiredType) != genericClayers.end()) {
+      for (const auto& clayer : genericClayers.at(requiredType)) {
+        discover.candidateClayers[requiredType].push_back(clayer);
+      }
+    } else {
+      ProjMgrUtils::PushBackUniquely(discover.missedRequiredTypes, requiredType);
+    }
+  }
+  // parse matched type layers
+  for (const auto& [type, clayers] : discover.candidateClayers) {
+    for (const auto& clayer : clayers) {
+      if (!m_parser->ParseGenericClayer(clayer, m_checkSchema)) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+bool ProjMgrWorker::DiscoverMatchingLayers(ContextItem& context, string clayerSearchPath) {
+  // get all layers from packs and from search path
+  LayersDiscovering discover;
+  if (!CollectLayersFromPacks(context, discover.genericClayersFromPacks)) {
+    return false;
+  }
+  if (!CollectLayersFromSearchPath(clayerSearchPath, discover.genericClayersFromSearchPath)) {
+    return false;
+  }
+  // get required layer types
+  GetRequiredLayerTypes(context, discover);
+  // process candidate layers
+  if (!ProcessCandidateLayers(context, discover)) {
+    return false;
+  }
+  // process layer combinations
+  if (!ProcessLayerCombinations(context, discover)) {
+    return false;
+  }
+  return true;
+}
 
+bool ProjMgrWorker::ProcessLayerCombinations(ContextItem& context, LayersDiscovering& discover) {
   // debug message
   string debugMsg;
   if (m_debug) {
     debugMsg = "check for context '" + context.name + "'\n";
+
+    for (const auto& missedRequiredType : discover.missedRequiredTypes) {
+      debugMsg += "no clayer matches type '" + missedRequiredType + "'\n";
+    }
   }
 
+  // collect connections from candidate layers
   ConnectionsCollectionVec allConnections;
-  StrVecMap matchedTypeClayers;
-
-  if (!requiredLayerTypes.empty()) {
-    // collect generic clayers from loaded packs and from search path
-    StrVecMap genericClayers;
-    if (!CollectLayersFromPacks(context, genericClayers) ||
-        !CollectLayersFromSearchPath(clayerSearchPath, genericClayers)) {
-      return false;
-    }
-    // clayers matching required types
-    for (const auto& requiredType : requiredLayerTypes) {
-      if (genericClayers.find(requiredType) != genericClayers.end()) {
-        for (const auto& clayer : genericClayers.at(requiredType)) {
-          matchedTypeClayers[requiredType].push_back(clayer);
-        }
-      } else {
-        if (m_debug) {
-          debugMsg += "no clayer matches type '" + requiredType + "'\n";
-        }
-      }
-    }
-    // parse matched type layers and collect connections
-    for (const auto& [type, clayers] : matchedTypeClayers) {
+  if (!discover.requiredLayerTypes.empty()) {
+    for (const auto& [type, clayers] : discover.candidateClayers) {
       for (const auto& clayer : clayers) {
-        if (!m_parser->ParseGenericClayer(clayer, m_checkSchema)) {
-          return false;
-        }
         const ClayerItem& clayerItem = m_parser->GetGenericClayers()[clayer];
         if (type != clayerItem.type) {
           if (m_debug) {
@@ -657,7 +722,7 @@ bool ProjMgrWorker::DiscoverMatchingLayers(ContextItem& context, const string& c
   CollectConnections(context, allConnections);
 
   // classify connections according to layer types and set config-ids
-  ConnectionsCollectionMap classifiedConnections = ClassifyConnections(allConnections, optionalTypeFlags);
+  ConnectionsCollectionMap classifiedConnections = ClassifyConnections(allConnections, discover.optionalTypeFlags);
 
   // cross classified connections to get all combinations to be validated
   vector<ConnectionsCollectionVec> combinations;
@@ -686,7 +751,7 @@ bool ProjMgrWorker::DiscoverMatchingLayers(ContextItem& context, const string& c
     // update list of compatible layers
     if (result.valid) {
       context.validConnections.push_back(combination);
-      for (const auto& [type, _] : matchedTypeClayers) {
+      for (const auto& [type, _] : discover.candidateClayers) {
         for (const auto& collection : combination) {
           if (collection.type == type) {
             ProjMgrUtils::PushBackUniquely(context.compatibleLayers[type], collection.filename);
@@ -703,9 +768,9 @@ bool ProjMgrWorker::DiscoverMatchingLayers(ContextItem& context, const string& c
   }
 
   // assess generic layers validation results
-  if (!matchedTypeClayers.empty()) {
+  if (!discover.candidateClayers.empty()) {
     if (!context.compatibleLayers.empty()) {
-      for (const auto& [type, _] : matchedTypeClayers) {
+      for (const auto& [type, _] : discover.candidateClayers) {
         if (context.compatibleLayers[type].size() == 1) {
           // unique match
           const auto& clayer = context.compatibleLayers[type].front();
@@ -735,7 +800,7 @@ bool ProjMgrWorker::DiscoverMatchingLayers(ContextItem& context, const string& c
     ProjMgrLogger::Debug(debugMsg);
   }
 
-  if (!matchedTypeClayers.empty() && context.compatibleLayers.empty()) {
+  if (!discover.candidateClayers.empty() && context.compatibleLayers.empty()) {
     return false;
   }
 
@@ -1229,7 +1294,11 @@ bool ProjMgrWorker::ProcessPackages(ContextItem& context) {
   for (const auto& [_, clayer] : context.clayers) {
     packRequirements.insert(packRequirements.end(), clayer->packs.begin(), clayer->packs.end());
   }
+  AddPackRequirements(context, packRequirements);
+  return true;
+}
 
+bool ProjMgrWorker::AddPackRequirements(ContextItem& context, const vector<PackItem> packRequirements) {
   // Filter context specific package requirements
   vector<PackItem> packages;
   for (const auto& packItem : packRequirements) {
@@ -3049,16 +3118,10 @@ bool ProjMgrWorker::ListLayers(vector<string>& layers, const string& clayerSearc
         }
       }
     } else {
-      // process board/device filtering
+      // process precedences
       if (!ProcessPrecedences(context)) {
         return false;
       }
-      if (!ProcessDevice(context)) {
-        return false;
-      }
-      if (!SetTargetAttributes(context, context.targetAttributes)) {
-        return false;
-      }
       // get matching layers for selected context
       if (!DiscoverMatchingLayers(context, clayerSearchPath)) {
         return false;

tools/projmgr/test/data/TestLayers/genericlayers.cproject.yml

@@ -22,6 +22,9 @@ project:
     - type: Board
       optional: true
       for-context: .OptionalLayerType
+    - type: BoardSpecific
+      for-context: +BoardSpecific
+
 
   connections:
     - connect: Project Connections

tools/projmgr/test/data/TestLayers/genericlayers.csolution.yml

@@ -13,6 +13,8 @@ solution:
       device: :cm0_core0
     - type: AnyBoard
       device: RteTest_ARMCM3
+    - type: BoardSpecific
+      board: RteTest board listing
 
   projects:
     - project: genericlayers.cproject.yml