fixes python generation with comment opening/closing tags

CHANGELOG.md

@@ -13,6 +13,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Fixed python generation in scenarios with opening/closing tags for code comments. [#5636](https://github.com/microsoft/kiota/issues/5636)
+
 ## [1.20.0] - 2024-11-07
 
 ### Added

it/config.json

@@ -40,10 +40,6 @@
       {
         "Language": "ruby",
         "Rationale": "https://github.com/microsoft/kiota/issues/1816"
-      },
-      {
-        "Language": "python",
-        "Rationale": "https://github.com/microsoft/kiota/issues/5636"
       }
     ],
     "ExcludePatterns": [
@@ -271,4 +267,4 @@
     "Suppressions": [],
     "IdempotencySuppressions": []
   }
-}
+}

it/python/requirements-dev.txt

@@ -98,19 +98,19 @@ httpx[http2]==0.27.2
 
 hyperframe==6.0.1 ; python_full_version >= '3.6.1'
 
-microsoft-kiota-abstractions==1.6.0
+microsoft-kiota-abstractions==1.6.1
 
-microsoft-kiota-authentication-azure==1.6.0
+microsoft-kiota-authentication-azure==1.6.1
 
-microsoft-kiota-http==1.6.0
+microsoft-kiota-http==1.6.1
 
-microsoft-kiota-serialization-json==1.6.0
+microsoft-kiota-serialization-json==1.6.1
 
-microsoft-kiota-serialization-text==1.6.0
+microsoft-kiota-serialization-text==1.6.1
 
-microsoft-kiota-serialization-form==1.6.0
+microsoft-kiota-serialization-form==1.6.1
 
-microsoft-kiota-serialization-multipart==1.6.0
+microsoft-kiota-serialization-multipart==1.6.1
 
 msal==1.31.0
 

src/Kiota.Builder/Refiners/CommonLanguageRefiner.cs

@@ -649,9 +649,9 @@
         if (currentElement is CodeIndexer currentIndexer &&
             currentElement.Parent is CodeClass indexerParentClass)
         {
             if (indexerParentClass.ContainsMember(currentElement.Name)) // TODO remove condition for v2 necessary because of the second case of Go block
                 indexerParentClass.RemoveChildElement(currentElement);
             //TODO remove whole block except for last else if body for v2
             if (language == GenerationLanguage.Go)
             {
                 if (currentIndexer.IsLegacyIndexer)
@@ -1475,15 +1475,28 @@
 
         CrawlTree(currentElement, x => RemoveRequestConfigurationClassesCommonProperties(x, baseTypeUsing));
     }
-    protected static void RemoveUntypedNodePropertyValues(CodeElement currentElement)
+    protected static void RemoveUntypedNodeTypeValues(CodeElement currentElement)
     {
-        if (currentElement is CodeProperty currentProperty
-            && currentElement.Parent is CodeClass parentClass
-            && currentProperty.Type.Name.Equals(KiotaBuilder.UntypedNodeName, StringComparison.OrdinalIgnoreCase))
+        switch (currentElement)
         {
-            parentClass.RemoveChildElement(currentProperty);
+            case CodeProperty currentProperty when currentElement.Parent is CodeClass parentClass && currentProperty.Type.Name.Equals(KiotaBuilder.UntypedNodeName, StringComparison.OrdinalIgnoreCase):
+                parentClass.RemoveChildElement(currentProperty);
+                break;
+            case CodeMethod currentMethod when currentMethod.IsOfKind(CodeMethodKind.RequestExecutor):
+                if (currentMethod.ReturnType.Name.Equals(KiotaBuilder.UntypedNodeName, StringComparison.OrdinalIgnoreCase))
+                {
+                    currentMethod.ReturnType = new CodeType { Name = "binary", IsExternal = true };
+                }
+                if (currentMethod.Parameters.Where(x => x.Kind is CodeParameterKind.RequestBody && x.Type.Name.Equals(KiotaBuilder.UntypedNodeName, StringComparison.OrdinalIgnoreCase)).ToList() is { Count: > 0 } parameters)
+                {
+                    foreach (var parameter in parameters)
+                    {
+                        parameter.Type = new CodeType { Name = "binary", IsExternal = true };
+                    }
+                }
+                break;
         }
-        CrawlTree(currentElement, RemoveUntypedNodePropertyValues);
+        CrawlTree(currentElement, RemoveUntypedNodeTypeValues);
     }
     protected static void RemoveRequestConfigurationClasses(CodeElement currentElement, CodeUsing? configurationParameterTypeUsing = null, CodeType? defaultValueForGenericTypeParam = null, bool keepRequestConfigurationClass = false, bool addDeprecation = false, CodeUsing? usingForDefaultGenericParameter = null)
     {

src/Kiota.Builder/Refiners/PhpRefiner.cs

@@ -57,7 +57,7 @@
             AddParsableImplementsForModelClasses(generatedCode, "Parsable");
             AddRequestConfigurationConstructors(generatedCode);
             AddDefaultImports(generatedCode, defaultUsingEvaluators);
-            RemoveUntypedNodePropertyValues(generatedCode);
+            RemoveUntypedNodeTypeValues(generatedCode);
             AddCollectionValidationUtilImportToModels(generatedCode);
             cancellationToken.ThrowIfCancellationRequested();
             AddGetterAndSetterMethods(generatedCode,
@@ -175,7 +175,7 @@
             })
         },
         {
             "DateTimeOffset", ("DateTime", new CodeUsing
             {
                 Name = "DateTime",
                 Declaration = new CodeType
@@ -186,7 +186,7 @@
             })
         }
     };
     private static readonly AdditionalUsingEvaluator[] defaultUsingEvaluators = {
         new (static x => x is CodeProperty prop && prop.IsOfKind(CodePropertyKind.RequestAdapter),
             AbstractionsNamespaceName, "RequestAdapter"),
         new (static x => x is CodeMethod method && method.IsOfKind(CodeMethodKind.RequestGenerator),
@@ -231,7 +231,7 @@
             currentProperty.Type.Name = currentProperty.Type.Name[1..];
         else if (currentProperty.IsOfKind(CodePropertyKind.AdditionalData))
         {
             currentProperty.Type.Name = "array";
             currentProperty.Type.IsNullable = true;
             currentProperty.DefaultValue = "[]";
             currentProperty.Type.CollectionKind = CodeTypeBase.CodeTypeCollectionKind.Complex;

src/Kiota.Builder/Refiners/PythonRefiner.cs

@@ -26,7 +26,7 @@
             );
             CorrectCommonNames(generatedCode);
             RemoveMethodByKind(generatedCode, CodeMethodKind.RawUrlConstructor);
-            RemoveUntypedNodePropertyValues(generatedCode);
+            RemoveUntypedNodeTypeValues(generatedCode);
             DisableActionOf(generatedCode,
             CodeParameterKind.RequestConfiguration);
             MoveRequestBuilderPropertiesToBaseType(generatedCode,
@@ -160,7 +160,7 @@
     private const string AbstractionsPackageName = "kiota_abstractions";
     private const string SerializationModuleName = $"{AbstractionsPackageName}.serialization";
     private const string StoreModuleName = $"{AbstractionsPackageName}.store";
     private static readonly AdditionalUsingEvaluator[] defaultUsingEvaluators = {
         new (static x => x is CodeClass, "__future__", "annotations"),
         new (static x => x is CodeClass, "typing", "Any, Callable, Dict, List, Optional, TYPE_CHECKING, Union"),
         new (static x => x is CodeProperty prop && prop.IsOfKind(CodePropertyKind.RequestAdapter),

src/Kiota.Builder/Refiners/RubyRefiner.cs

@@ -63,7 +63,7 @@ public override Task RefineAsync(CodeNamespace generatedCode, CancellationToken
             cancellationToken.ThrowIfCancellationRequested();
             AddParsableImplementsForModelClasses(generatedCode, "MicrosoftKiotaAbstractions::Parsable");
             AddDefaultImports(generatedCode, defaultUsingEvaluators);
-            RemoveUntypedNodePropertyValues(generatedCode);
+            RemoveUntypedNodeTypeValues(generatedCode);
             CorrectCoreType(generatedCode, CorrectMethodType, CorrectPropertyType, CorrectImplements);
             cancellationToken.ThrowIfCancellationRequested();
             ReplacePropertyNames(generatedCode,

src/Kiota.Builder/Refiners/SwiftRefiner.cs

@@ -41,10 +41,10 @@
                 true,
                 false,
                 true);
+            RemoveUntypedNodeTypeValues(generatedCode);
             AddDefaultImports(
                 generatedCode,
                 defaultUsingEvaluators);
-            RemoveUntypedNodePropertyValues(generatedCode);
             cancellationToken.ThrowIfCancellationRequested();
             CorrectCoreType(
                 generatedCode,
@@ -70,7 +70,7 @@
                                             (@class.Properties.Any(x => x.IsOfKind(CodePropertyKind.AdditionalData)) ||
                                             @class.StartBlock.Implements.Any(x => KiotaBuilder.AdditionalHolderInterface.Equals(x.Name, StringComparison.OrdinalIgnoreCase))),
             "MicrosoftKiotaAbstractions", "AdditionalDataHolder"),
     };//TODO add backing store types once we have them defined
     private static void CorrectImplements(ProprietableBlockDeclaration block)
     {
         block.ReplaceImplementByName(KiotaBuilder.AdditionalHolderInterface, "AdditionalDataHolder");
@@ -124,14 +124,14 @@
     }
     private static readonly Dictionary<string, (string, CodeUsing?)> DateTypesReplacements = new(StringComparer.OrdinalIgnoreCase) {
         {"DateTimeOffset", ("Date", new CodeUsing {
                                         Name = "Date",//TODO
                                         Declaration = new CodeType {
                                             Name = "Foundation",
                                             IsExternal = true,
                                         },
                                     })},
         {"TimeSpan", ("Date", new CodeUsing {
                                         Name = "Date",//TODO
                                         Declaration = new CodeType {
                                             Name = "Foundation",
                                             IsExternal = true,
@@ -145,7 +145,7 @@
                                 },
                             })},
         {"TimeOnly", ("Date", new CodeUsing {
                                 Name = "Date",//TODO
                                 Declaration = new CodeType {
                                     Name = "Foundation",
                                     IsExternal = true,

src/Kiota.Builder/Writers/Python/CodeMethodWriter.cs

@@ -384,10 +384,15 @@ private void WriteDirectAccessProperties(CodeClass parentClass, LanguageWriter w
         {
             var returnType = conventions.GetTypeString(propWithDefault.Type, propWithDefault, true, writer);
             var defaultValue = propWithDefault.DefaultValue;
-            if (propWithDefault.Type is CodeType propertyType && propertyType.TypeDefinition is CodeEnum enumDefinition)
+            switch (propWithDefault.Type)
             {
-                _codeUsingWriter.WriteDeferredImport(parentClass, enumDefinition.Name, writer);
-                defaultValue = $"{enumDefinition.Name}({defaultValue})";
+                case CodeType { TypeDefinition: CodeEnum enumDefinition }:
+                    _codeUsingWriter.WriteDeferredImport(parentClass, enumDefinition.Name, writer);
+                    defaultValue = $"{enumDefinition.Name}({defaultValue})";
+                    break;
+                case CodeType propType when propType.Name.Equals("boolean", StringComparison.OrdinalIgnoreCase):
+                    defaultValue = defaultValue.TrimQuotes().ToFirstCharacterUpperCase();// python booleans start in uppercase
+                    break;
             }
             conventions.WriteInLineDescription(propWithDefault, writer);
             if (parentClass.IsOfKind(CodeClassKind.Model))
@@ -412,10 +417,15 @@ private void WriteSetterAccessProperties(CodeClass parentClass, LanguageWriter w
                                         .ThenBy(static x => x.Name))
         {
             var defaultValue = propWithDefault.DefaultValue;
-            if (propWithDefault.Type is CodeType propertyType && propertyType.TypeDefinition is CodeEnum enumDefinition)
+            switch (propWithDefault.Type)
             {
-                _codeUsingWriter.WriteDeferredImport(parentClass, enumDefinition.Name, writer);
-                defaultValue = $"{enumDefinition.Name}({defaultValue})";
+                case CodeType { TypeDefinition: CodeEnum enumDefinition }:
+                    _codeUsingWriter.WriteDeferredImport(parentClass, enumDefinition.Name, writer);
+                    defaultValue = $"{enumDefinition.Name}({defaultValue})";
+                    break;
+                case CodeType propType when propType.Name.Equals("boolean", StringComparison.OrdinalIgnoreCase):
+                    defaultValue = defaultValue.TrimQuotes().ToFirstCharacterUpperCase();// python booleans start in uppercase
+                    break;
             }
             var returnType = conventions.GetTypeString(propWithDefault.Type, propWithDefault, true, writer);
             conventions.WriteInLineDescription(propWithDefault, writer);
@@ -583,7 +593,7 @@ private void WriteRequestExecutorBody(CodeMethod codeElement, RequestParams requ
         var isStream = conventions.StreamTypeName.Equals(returnType, StringComparison.OrdinalIgnoreCase);
         var returnTypeWithoutCollectionSymbol = GetReturnTypeWithoutCollectionSymbol(codeElement, returnType);
         var genericTypeForSendMethod = GetSendRequestMethodName(isVoid, isStream, codeElement.ReturnType.IsCollection, returnTypeWithoutCollectionSymbol, isEnum);
-        var newFactoryParameter = GetTypeFactory(isVoid, isStream, isEnum, returnTypeWithoutCollectionSymbol);
+        var newFactoryParameter = GetTypeFactory(isVoid, isStream, isEnum, returnTypeWithoutCollectionSymbol, codeElement.ReturnType.IsCollection);
         var errorMappingVarName = NoneKeyword;
         if (codeElement.ErrorMappings.Any())
         {
@@ -809,11 +819,11 @@ private string GetSerializationMethodName(CodeTypeBase propType)
             _ => "write_object_value",
         };
     }
-    internal string GetTypeFactory(bool isVoid, bool isStream, bool isEnum, string returnType)
+    internal string GetTypeFactory(bool isVoid, bool isStream, bool isEnum, string returnType, bool isCollection)
     {
         if (isVoid) return string.Empty;
         if (isStream || isEnum) return $" \"{returnType}\",";
-        if (conventions.IsPrimitiveType(returnType)) return $" {returnType},";
+        if (conventions.IsPrimitiveType(returnType)) return isCollection ? $" {returnType}," : $" \"{returnType}\",";
         return $" {returnType},";
     }
     private string GetSendRequestMethodName(bool isVoid, bool isStream, bool isCollection, string returnType,

src/Kiota.Builder/Writers/Python/PythonConventionService.cs

@@ -95,7 +95,7 @@ public override string GetTypeString(CodeTypeBase code, CodeElement targetElemen
         throw new InvalidOperationException($"type of type {code.GetType()} is unknown");
     }
 #pragma warning restore CA1822 // Method should be static
-    internal static string RemoveInvalidDescriptionCharacters(string originalDescription) => originalDescription.Replace("\\", "/", StringComparison.OrdinalIgnoreCase);
+    internal static string RemoveInvalidDescriptionCharacters(string originalDescription) => originalDescription.Replace("\\", "/", StringComparison.OrdinalIgnoreCase).Replace("\"\"\"", "\\\"\\\"\\\"", StringComparison.OrdinalIgnoreCase);
     public override string TranslateType(CodeType type)
     {
         ArgumentNullException.ThrowIfNull(type);
@@ -130,7 +130,7 @@ public bool IsPrimitiveType(string typeName)
     {
         return typeName switch
         {
-            "int" or "float" or "str" or "bool" or "None" => true,
+            "int" or "float" or "str" or "bool" or "None" or "datetime.datetime" or "datetime.timedelta" or "datetime.date" or "datetime.time" => true,
             _ => false,
         };
     }

tests/Kiota.Builder.Tests/Refiners/PythonLanguageRefinerTests.cs

@@ -584,5 +584,34 @@ public async Task AddsPropertiesAndMethodTypesImportsPythonAsync()
         Assert.Single(requestBuilder.Methods, x => x.IsOfKind(CodeMethodKind.RequestExecutor));
         Assert.DoesNotContain("QueryParameters", declaration.Usings.Select(x => x.Name));
     }
+    [Fact]
+    public async Task ReplacesUntypedNodeInMethodParameterAndReturnTypeAsync()
+    {
+        var requestBuilderClass = root.AddClass(new CodeClass() { Name = "NodeRequestBuilder" }).First();
+        var method = new CodeMethod
+        {
+            Name = "getAsync",
+            ReturnType = new CodeType()
+            {
+                Name = KiotaBuilder.UntypedNodeName,//Returns untyped node
+                IsExternal = true
+            },
+            Kind = CodeMethodKind.RequestExecutor
+        };
+        method.AddParameter(new CodeParameter()
+        {
+            Name = "jsonData",
+            Type = new CodeType()
+            {
+                Name = KiotaBuilder.UntypedNodeName, //Has untyped node parameter
+                IsExternal = true
+            },
+            Kind = CodeParameterKind.RequestBody
+        });
+        requestBuilderClass.AddMethod(method);
+        await ILanguageRefiner.RefineAsync(new GenerationConfiguration { Language = GenerationLanguage.Python }, root);
+        Assert.Equal("bytes", method.Parameters.First().Type.Name);// type is renamed to use the stream type
+        Assert.Equal("bytes", method.ReturnType.Name);// return type is renamed to use the stream type
+    }
     #endregion
 }

tests/Kiota.Builder.Tests/Writers/Python/CodeMethodWriterTests.cs

@@ -820,12 +820,13 @@ public void WritesUnionDeSerializerBody()
         Assert.Contains("return {}", result);
     }
     [Theory]
-    [InlineData(true, false, false, "string", "")]
-    [InlineData(false, true, false, "Stream", " \"Stream\",")]
-    [InlineData(false, false, true, "SomeEnum", " \"SomeEnum\",")]
-    [InlineData(false, false, false, "int", " int,")]
-    [InlineData(false, false, false, "CustomType", " CustomType,")]
-    public void GetTypeFactory_ReturnsCorrectString(bool isVoid, bool isStream, bool isEnum, string returnType, string expected)
+    [InlineData(true, false, false, false, "string", "")]
+    [InlineData(false, true, false, false, "Stream", " \"Stream\",")]
+    [InlineData(false, false, true, false, "SomeEnum", " \"SomeEnum\",")]
+    [InlineData(false, false, false, true, "int", " int,")]
+    [InlineData(false, false, false, false, "int", " \"int\",")]
+    [InlineData(false, false, false, false, "CustomType", " CustomType,")]
+    public void GetTypeFactory_ReturnsCorrectString(bool isVoid, bool isStream, bool isEnum, bool isCollection, string returnType, string expected)
     {
         var mockConventionService = new Mock<PythonConventionService>();
 
@@ -835,9 +836,7 @@ public void GetTypeFactory_ReturnsCorrectString(bool isVoid, bool isStream, bool
             false // usesBackingStore
         );
 
-        var result = codeMethodWriter.GetTypeFactory(isVoid, isStream, isEnum, returnType);
-
-
+        var result = codeMethodWriter.GetTypeFactory(isVoid, isStream, isEnum, returnType, isCollection);
         Assert.Equal(expected, result);
     }
     [Fact]
@@ -1592,6 +1591,30 @@ public void WritesConstructor()
         Assert.DoesNotContain("get_path_parameters(", result);
     }
     [Fact]
+    public void EscapesCommentCharactersInDescription()
+    {
+        setup();
+        method.Kind = CodeMethodKind.Constructor;
+        method.IsAsync = false;
+        parentClass.Kind = CodeClassKind.Custom;
+        parentClass.AddProperty(new CodeProperty
+        {
+            Name = "prop_without_default_value",
+            Kind = CodePropertyKind.Custom,
+            Documentation = new()
+            {
+                DescriptionTemplate = "This property has a description with comments \"\"\".",
+            },
+            Type = new CodeType
+            {
+                Name = "string"
+            }
+        });
+        writer.Write(method);
+        var result = tw.ToString();
+        Assert.Contains("This property has a description with comments \\\"\\\"\\\".", result);
+    }
+    [Fact]
     public void WritesWithUrl()
     {
         setup();