[projmgr] Update list layers behaviour: consider layer type optiona…

…l by default (#1095)

tools/projmgr/include/ProjMgrParser.h

@@ -188,11 +188,13 @@ struct GeneratorsItem {
  * @brief layer item containing
  *        layer name,
  *        layer type,
+ *        optional flag,
  *        type inclusion
 */
 struct LayerItem {
   std::string layer;
   std::string type;
+  bool optional;
   TypeFilter typeFilter;
 };
 

tools/projmgr/include/ProjMgrWorker.h

@@ -649,7 +649,7 @@ class ProjMgrWorker {
   bool DiscoverMatchingLayers(ContextItem& context, const std::string& clayerSearchPath);
   void CollectConnections(ContextItem& context, ConnectionsCollectionVec& connections);
   void GetConsumesProvides(const ConnectionsCollectionVec& collection, ConnectionsList& connections);
-  ConnectionsCollectionMap ClassifyConnections(const ConnectionsCollectionVec& connections);
+  ConnectionsCollectionMap ClassifyConnections(const ConnectionsCollectionVec& connections, std::map<std::string, bool> optionalTypeFlags);
   ConnectionsValidationResult ValidateConnections(ConnectionsCollectionVec combination);
   void GetAllCombinations(const ConnectionsCollectionMap& src, const ConnectionsCollectionMap::iterator& it,
     std::vector<ConnectionsCollectionVec>& combinations, const ConnectionsCollectionVec& previous = ConnectionsCollectionVec());

tools/projmgr/include/ProjMgrYamlParser.h

@@ -92,6 +92,7 @@ static constexpr const char* YAML_MISC_LINK_C = "Link-C";
 static constexpr const char* YAML_MISC_LINK_CPP = "Link-CPP";
 static constexpr const char* YAML_NOTFORCONTEXT = "not-for-context";
 static constexpr const char* YAML_OPTIMIZE = "optimize";
+static constexpr const char* YAML_OPTIONAL = "optional";
 static constexpr const char* YAML_OPTIONS = "options";
 static constexpr const char* YAML_OUTPUT = "output";
 static constexpr const char* YAML_OUTPUTDIRS = "output-dirs";
@@ -181,6 +182,7 @@ class ProjMgrYamlParser {
   void ParseDefine(const YAML::Node& parent, std::vector<std::string>& define);
   void ParsePacks(const YAML::Node& parent, const std::string& file, std::vector<PackItem>& packs);
   void ParseProcessor(const YAML::Node& parent, ProcessorItem& processor);
+  void ParseBoolean(const YAML::Node& parent, const std::string& key, bool& value, bool def);
   void ParseString(const YAML::Node& parent, const std::string& key, std::string& value);
   void ParseVector(const YAML::Node& parent, const std::string& key, std::vector<std::string>& value);
   void ParseVectorOfStringPairs(const YAML::Node& parent, const std::string& key, std::vector<std::pair<std::string, std::string>>& value);

tools/projmgr/schemas/common.schema.json

@@ -633,13 +633,17 @@
           "type": "string",
           "description": "Layer type for compatible layers matching"
         },
+        "optional": {
+          "type": "boolean",
+          "description": "Set optional to 'false' to require the layer type when searching compatible layers; default: 'true'"
+        },
         "for-context":     { "$ref": "#/definitions/ForContext" },
         "not-for-context": { "$ref": "#/definitions/NotForContext" }
       },
       "allOf": [
         { "$ref": "#/definitions/TypeListMutualExclusion" },
         { "anyOf": [
-            { "required": ["layer"] },
+            { "required": ["layer"], "not": {"required": ["optional"]} },
             { "required": ["type"]  }
           ]
         }

tools/projmgr/src/ProjMgrWorker.cpp

@@ -521,7 +521,9 @@ void ProjMgrWorker::GetAllCombinations(const ConnectionsCollectionMap& src, cons
   // iterate over the input columns
   for (const auto& item : it->second) {
     ConnectionsCollectionVec combination = previous;
-    combination.push_back(item);
+    if (!item.filename.empty()) {
+      combination.push_back(item);
+    }
     if (nextIt != src.end()) {
       // run recursively over the next item
       GetAllCombinations(src, nextIt, combinations, combination);
@@ -533,9 +535,9 @@ void ProjMgrWorker::GetAllCombinations(const ConnectionsCollectionMap& src, cons
 }
 
 void ProjMgrWorker::GetAllSelectCombinations(const ConnectPtrVec& src, const ConnectPtrVec::iterator& it,
+  std::vector<ConnectPtrVec>& combinations) {
   // combine items from a vector of 'select' nodes
   // see an example in the test case ProjMgrWorkerUnitTests.GetAllSelectCombinations
-  std::vector<ConnectPtrVec>& combinations) {
   // for every past combination add a new combination containing additionally the current item
   for (auto combination : vector<ConnectPtrVec>(combinations)) {
     combination.push_back(*it);
@@ -588,12 +590,14 @@ bool ProjMgrWorker::CollectLayersFromSearchPath(const string& clayerSearchPath,
 bool ProjMgrWorker::DiscoverMatchingLayers(ContextItem& context, const string& clayerSearchPath) {
   // required layer types
   StrVec requiredLayerTypes;
+  map<string, bool> optionalTypeFlags;
   for (const auto& clayer : context.cproject->clayers) {
     if (clayer.type.empty() || !CheckContextFilters(clayer.typeFilter, context) ||
       (ExpandString(clayer.layer, context.variables) != clayer.layer)) {
       continue;
     }
     requiredLayerTypes.push_back(clayer.type);
+    optionalTypeFlags[clayer.type] = clayer.optional;
   }
 
   // debug message
@@ -653,7 +657,7 @@ bool ProjMgrWorker::DiscoverMatchingLayers(ContextItem& context, const string& c
   CollectConnections(context, allConnections);
 
   // classify connections according to layer types and set config-ids
-  ConnectionsCollectionMap classifiedConnections = ClassifyConnections(allConnections);
+  ConnectionsCollectionMap classifiedConnections = ClassifyConnections(allConnections, optionalTypeFlags);
 
   // cross classified connections to get all combinations to be validated
   vector<ConnectionsCollectionVec> combinations;
@@ -828,7 +832,7 @@ void ProjMgrWorker::CollectConnections(ContextItem& context, ConnectionsCollecti
   }
 }
 
-ConnectionsCollectionMap ProjMgrWorker::ClassifyConnections(const ConnectionsCollectionVec& connections) {
+ConnectionsCollectionMap ProjMgrWorker::ClassifyConnections(const ConnectionsCollectionVec& connections, map<string, bool> optionalTypeFlags) {
   // classify connections according to layer types and set config-ids
   ConnectionsCollectionMap classifiedConnections;
   for (const auto& collectionEntry : connections) {
@@ -879,6 +883,13 @@ ConnectionsCollectionMap ProjMgrWorker::ClassifyConnections(const ConnectionsCol
       classifiedConnections[classifiedType].push_back(collection);
     }
   }
+  // add empty connection for optional handling in combinatory flow, unless differently specified
+  for (auto& [type, collectionVec] : classifiedConnections) {
+    if (optionalTypeFlags[type]) {
+      collectionVec.push_back({ RteUtils::EMPTY_STRING, RteUtils::EMPTY_STRING });
+    }
+  }
+
   return classifiedConnections;
 }
 

tools/projmgr/src/ProjMgrYamlParser.cpp

@@ -266,6 +266,15 @@ void ProjMgrYamlParser::ParsePortablePaths(const YAML::Node& parent, const strin
   }
 }
 
+void ProjMgrYamlParser::ParseBoolean(const YAML::Node& parent, const string& key, bool& value, bool def) {
+  if (parent[key].IsDefined()) {
+    value = parent[key].as<bool>();
+    if (parent[key].Type() == YAML::NodeType::Null) {
+      value = def;
+    }
+  }
+}
+
 void ProjMgrYamlParser::ParseString(const YAML::Node& parent, const string& key, string& value) {
   if (parent[key].IsDefined()) {
     value = parent[key].as<string>();
@@ -533,6 +542,7 @@ bool ProjMgrYamlParser::ParseLayers(const YAML::Node& parent, const string& file
       }
       ParsePortablePath(layerEntry, file, YAML_LAYER, layerItem.layer);
       ParseString(layerEntry, YAML_TYPE, layerItem.type);
+      ParseBoolean(layerEntry, YAML_OPTIONAL, layerItem.optional, true);
       layers.push_back(layerItem);
     }
   }
@@ -976,6 +986,7 @@ const set<string> linkerKeys = {
 const set<string> layersKeys = {
   YAML_LAYER,
   YAML_TYPE,
+  YAML_OPTIONAL,
   YAML_FORCONTEXT,
   YAML_NOTFORCONTEXT,
 };

tools/projmgr/test/data/TestLayers/genericlayers.cproject.yml

@@ -5,14 +5,23 @@ project:
 
   layers:
     - type: Board
+      optional: false
+      not-for-context: .OptionalLayerType
     - type: TestVariant
+      optional: false
       for-context: .CompatibleLayers
     - type: Incompatible
+      optional: false
       for-context: .IncompatibleLayers
     - type: UnknownType
+      optional: false
       for-context: .IncompatibleLayers
     - type: PdscType
+      optional: false
       for-context: .IncompatibleLayers
+    - type: Board
+      optional: true
+      for-context: .OptionalLayerType
 
   connections:
     - connect: Project Connections

tools/projmgr/test/data/TestLayers/genericlayers.csolution.yml

@@ -5,7 +5,7 @@ solution:
   build-types:
     - type: CompatibleLayers
     - type: IncompatibleLayers
-    - type: PdscTypeMismatch
+    - type: OptionalLayerType
 
   target-types:
     - type: Board3

tools/projmgr/test/src/ProjMgrUnitTests.cpp

@@ -1097,6 +1097,44 @@ no valid combination of clayers was found\n\
   EXPECT_TRUE(regex_match(outStr, regex(expectedOutStr)));
 }
 
+TEST_F(ProjMgrUnitTests, ListLayersOptionalLayerType) {
+  StdStreamRedirect streamRedirect;
+  char* argv[8];
+  const string& csolution = testinput_folder + "/TestLayers/genericlayers.csolution.yml";
+  const string& context = "genericlayers.OptionalLayerType+AnyBoard";
+  argv[1] = (char*)"list";
+  argv[2] = (char*)"layers";
+  argv[3] = (char*)"--solution";
+  argv[4] = (char*)csolution.c_str();
+  argv[5] = (char*)"-c";
+  argv[6] = (char*)context.c_str();
+  argv[7] = (char*)"-d";
+  EXPECT_EQ(0, RunProjMgr(8, argv, 0));
+
+  const string& expected ="\
+check combined connections:\n\
+  .*/TestLayers/genericlayers.cproject.yml\n\
+    \\(Project Connections\\)\n\
+provided combined connections not consumed:\n\
+  .*/TestLayers/genericlayers.cproject.yml\n\
+    ExactMatch\n\
+    EmptyConsumedValue\n\
+    EmptyValues\n\
+    AddedValueLessThanProvided\n\
+    AddedValueEqualToProvided\n\
+    MultipleProvided\n\
+    MultipleProvidedNonIdentical0\n\
+    MultipleProvidedNonIdentical1\n\
+    ProvidedDontMatch\n\
+    ProvidedEmpty\n\
+    AddedValueHigherThanProvided\n\
+connections are invalid\n\
+";
+
+  const string& errStr = streamRedirect.GetErrorString();
+  EXPECT_TRUE(regex_search(errStr, regex(expected)));
+}
+
 TEST_F(ProjMgrUnitTests, ListLayersInvalidContext) {
   char* argv[7];
   const string& csolution = testinput_folder + "/TestLayers/genericlayers.csolution.yml";