experimental cache azure pipelines

src/Sdk/AzurePipelines/AzureDevops.cs

@@ -3,6 +3,8 @@
 using System.IO;
 using System.Linq;
 using System.Reflection;
+using System.Security.Cryptography;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using GitHub.DistributedTask.Expressions2;
@@ -638,26 +640,56 @@ public static async Task<MappingToken> ReadTemplate(Runner.Server.Azure.Devops.C
             {
                 throw new Exception($"Couldn't read template {filenameAndRef} resolved to {finalFileName} ({finalRepository ?? "self"})");
             }
-            context.TraceWriter?.Info("{0}", $"Parsing template {filenameAndRef} resolved to {finalFileName} ({finalRepository ?? "self"}) using Schema {schemaName ?? "pipeline-root"}");
-            context.TraceWriter?.Verbose("{0}", fileContent);
-
+            var key = SHA1.HashData(Encoding.Default.GetBytes(fileContent));
+
             var errorTemplateFileName = $"({finalRepository ?? "self"})/{finalFileName}";
             context.FileTable ??= new List<string>();
             context.FileTable.Add(errorTemplateFileName);
             var templateContext = AzureDevops.CreateTemplateContext(context.TraceWriter ?? new EmptyTraceWriter(), context.FileTable, context.Flags);
             var fileId = templateContext.GetFileId(errorTemplateFileName);
 
-            TemplateToken token;
-            using (var stringReader = new StringReader(fileContent))
-            {
-                // preserveString is needed for azure pipelines compatability of the templateContext property all boolean and number token are casted to string without loosing it's exact string value
-                var yamlObjectReader = new YamlObjectReader(fileId, stringReader, preserveString: true, forceAzurePipelines: true);
-                token = TemplateReader.Read(templateContext, schemaName ?? "pipeline-root", yamlObjectReader, fileId, out _);
-            }
+            MappingToken pipelineroot;
+            context.TraceWriter?.Info("{0}", $"Parsing template {filenameAndRef} resolved to {finalFileName} ({finalRepository ?? "self"}) using Schema {schemaName ?? "pipeline-root"}");
+            if(!context.YamlCache.TryGetValue(key, out pipelineroot)) {
+                context.TraceWriter?.Verbose("{0}", fileContent);
 
-            templateContext.Errors.Check();
+                TemplateToken token;
+                using (var stringReader = new StringReader(fileContent))
+                {
+                    // preserveString is needed for azure pipelines compatability of the templateContext property all boolean and number token are casted to string without loosing it's exact string value
+                    var yamlObjectReader = new YamlObjectReader(fileId, stringReader, preserveString: true, forceAzurePipelines: true);
+                    token = TemplateReader.Read(templateContext, schemaName ?? "pipeline-root", yamlObjectReader, fileId, out _);
+                }
+
+                templateContext.Errors.Check();
 
-            var pipelineroot = token.AssertMapping("root");
+                pipelineroot = token.AssertMapping("root");
+                context.YamlCache.TryAdd(key, pipelineroot.Clone() as MappingToken);
+            } else {
+                Console.WriteLine("Cache Hit Yaml!");
+                pipelineroot = pipelineroot.Clone() as MappingToken;
+            }
+            var h = SHA1.Create();
+            h.TransformBlock(key, 0, key.Length, null, 0);
+            if(cparameters != null) {
+                foreach (var p in cparameters)
+                {
+                    Console.WriteLine($"{p.Key}: {p.Value?.ToContextData()?.ToJToken()}");
+                    var k = Encoding.Default.GetBytes(p.Key.ToString());
+                    h.TransformBlock(k, 0, k.Length, null, 0);
+                    foreach (var item in p.Value.Traverse())
+                    {
+                        var inp = Encoding.Default.GetBytes(item.ToString());
+                        h.TransformBlock(inp, 0, inp.Length, null, 0);
+                    }
+                }
+            }
+            h.TransformFinalBlock(new byte[0], 0, 0);
+            var ha = h.Hash;
+            if(context.YamlCache.TryGetValue(ha, out var x)) {
+                Console.WriteLine("Cache Hit Eval!");
+                return x.Clone() as MappingToken;
+            }
 
             var childContext = context.ChildContext(pipelineroot, filenameAndRef);
 
@@ -818,7 +850,7 @@ public static async Task<MappingToken> ReadTemplate(Runner.Server.Azure.Devops.C
             var evaluatedResult = TemplateEvaluator.Evaluate(templateContext, schemaName ?? "pipeline-root", pipelineroot, 0, fileId);
             templateContext.Errors.Check();
             context.TraceWriter?.Verbose("{0}", evaluatedResult.ToContextData().ToJToken().ToString());
-            return evaluatedResult.AssertMapping("root");
+            return context.YamlCache[ha] = evaluatedResult.AssertMapping("root");
 
             async Task<TemplateContext> evalJobsWithExtraVars(Context context, TemplateContext templateContext, int fileId, DictionaryContextData contextData, DictionaryContextData variablesData, Dictionary<string, object> dict, GitHub.DistributedTask.Expressions2.Sdk.IReadOnlyObject stage, DictionaryContextData vardata)
             {

src/Sdk/AzurePipelines/Context.cs

@@ -1,10 +1,27 @@
 using GitHub.DistributedTask.Expressions2;
 using GitHub.DistributedTask.ObjectTemplating.Tokens;
+using GitHub.DistributedTask.Pipelines;
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
 
 namespace Runner.Server.Azure.Devops
 {
+    public class HashComparer : IEqualityComparer<byte[]>
+    {
+        public bool Equals(byte[] x, byte[] y)
+        {
+            return x.SequenceEqual(y);
+        }
+
+        public int GetHashCode([DisallowNull] byte[] obj)
+        {
+            return obj[0] << 24 | obj[1] << 16 | obj[2] << 8 | obj[3];
+        }
+
+    }
+
     public class Context {
         public ExpressionFlags Flags { get; set; }
         public IFileProvider FileProvider { get; set; }
@@ -18,6 +35,11 @@ public class Context {
         public IRequiredParametersProvider RequiredParametersProvider { get; set; }
         public List<string> FileTable { get; set; } = new List<string>();
 
+        public IDictionary<byte[], MappingToken> YamlCache  { get; set; } = new Dictionary<byte[], MappingToken>(new HashComparer());
+        public IDictionary<byte[], Stage> StagesCache  { get; set; } = new Dictionary<byte[], Stage>(new HashComparer());
+        public IDictionary<byte[], Job> JobsCache  { get; set; } = new Dictionary<byte[], Job>(new HashComparer());
+        public IDictionary<byte[], Step> StepsCache  { get; set; } = new Dictionary<byte[], Step>(new HashComparer());
+
         public Context Clone() {
             return MemberwiseClone() as Context;
         }

src/azure-pipelines-vscode-ext/ext-core/Program.cs

@@ -78,14 +78,17 @@ public class ErrorWrapper {
         public List<string> Errors { get; set; }
     }
 
+    private static readonly Context CacheCtx = new();
+
     [MethodImpl(MethodImplOptions.NoInlining)]
     public static async Task<string> ExpandCurrentPipeline(JSObject handle, string currentFileName, string variables, string parameters, bool returnErrorContent) {
         var context = new Runner.Server.Azure.Devops.Context {
             FileProvider = new MyFileProvider(handle),
-            TraceWriter = new TraceWriter(handle),
-            Flags = GitHub.DistributedTask.Expressions2.ExpressionFlags.DTExpressionsV1 | GitHub.DistributedTask.Expressions2.ExpressionFlags.ExtendedDirectives,
+            //TraceWriter = new TraceWriter(handle),
+            Flags = GitHub.DistributedTask.Expressions2.ExpressionFlags.DTExpressionsV1 | GitHub.DistributedTask.Expressions2.ExpressionFlags.ExtendedDirectives | GitHub.DistributedTask.Expressions2.ExpressionFlags.AllowAnyForInsert,
             RequiredParametersProvider = new RequiredParametersProvider(handle),
-            VariablesProvider = new VariablesProvider { Variables = JsonConvert.DeserializeObject<Dictionary<string, string>>(variables) }
+            VariablesProvider = new VariablesProvider { Variables = JsonConvert.DeserializeObject<Dictionary<string, string>>(variables) },
+            YamlCache = CacheCtx.YamlCache
         };
         string yaml = null;
         try {