TestMu Conference is Back! Future of Testing + AI LambdaTest G2 Announcement

How to use Storage class of Microsoft.Coyote.Benchmarking package

Best Coyote code snippet using Microsoft.Coyote.Benchmarking.Storage

Program.cs

Source:Program.cs

...149            foreach (var file in Directory.GetFiles(this.OutputDir))150            {151                File.Delete(file);152            }153            Storage storage = new Storage();154            foreach (var b in Benchmarks)155            {156                var metadata = new TestMetadata(b.Test);157                object target = metadata.InstantiateTest();158                List<string> rowKeys = new List<string>();159                foreach (var comboList in metadata.EnumerateParamCombinations(0, new Stack<ParamInfo>()))160                {161                    foreach (var test in metadata.TestMethods)162                    {163                        string name = test.ApplyParams(target, comboList);164                        rowKeys.Add(this.CommitId + "." + b.Test.Name + "." + name);165                    }166                }167                Console.WriteLine("Downloading results for test {0}...", b.Name);168                string summaryFile = Path.Combine(this.OutputDir, "summary.csv");169                bool writeHeaders = !File.Exists(summaryFile);170                using (var file = new StreamWriter(summaryFile, true, Encoding.UTF8))171                {172                    if (writeHeaders)173                    {174                        PerfSummary.WriteHeaders(file);175                    }176                    foreach (var summary in await storage.DownloadAsync(this.DownloadPartition, rowKeys))177                    {178                        if (summary is null)179                        {180                            Console.WriteLine("Summary missing for {0}", b.Name);181                        }182                        else183                        {184                            summary.CommitId = this.CommitId;185                            summary.SetPartitionKey(this.DownloadPartition);186                            summary.WriteCsv(file);187                        }188                    }189                }190            }191        }192        private async Task<int> Run()193        {194            if (!string.IsNullOrEmpty(this.DownloadPartition))195            {196                await this.DowwnloadResults();197                return 0;198            }199            if (this.UploadCommits)200            {201                return await UploadCommitHistory();202            }203            if (string.IsNullOrEmpty(this.CommitId))204            {205                this.CommitId = Guid.NewGuid().ToString().Replace("-", string.Empty);206            }207            Storage storage = new Storage();208            List<PerfSummary> results = new List<PerfSummary>();209            int matching = 0;210            foreach (var b in Benchmarks)211            {212                if (FilterMatches(b.Name, this.Filters))213                {214                    matching++;215                    var config = DefaultConfig.Instance.WithArtifactsPath(this.OutputDir)216                        .WithOption(ConfigOptions.DisableOptimizationsValidator, true);217                    config.AddDiagnoser(new CpuDiagnoser());218                    config.AddDiagnoser(new TotalMemoryDiagnoser());219                    var summary = BenchmarkRunner.Run(b.Test, config);220                    foreach (var report in summary.Reports)221                    {222                        var data = this.GetEntities(report);223                        if (data.Count > 0)224                        {225                            results.Add(new PerfSummary(data));226                        }227                    }228                }229            }230            if (matching is 0)231            {232                Console.ForegroundColor = ConsoleColor.Red;233                Console.WriteLine("No benchmarks matching filter: {0}", string.Join(",", this.Filters));234                Console.ResetColor();235                PrintUsage();236                return 1;237            }238            else if (this.Cosmos)239            {240                await storage.UploadAsync(results);241                await UploadCommitHistory(1); // upload this commit id and it's commit date.242            }243            return 0;244        }245        private List<PerfEntity> GetEntities(BenchmarkReport report)246        {247            List<PerfEntity> results = new List<PerfEntity>();248            string testName = report.BenchmarkCase.Descriptor.DisplayInfo;249            foreach (var p in report.BenchmarkCase.Parameters.Items)250            {251                testName += string.Format(" {0}={1}", p.Name, p.Value);252            }253            // Right now we are choosing NOT to return each test result as254            // a separate entity, as this is too much information, so for now255            // we return the "Min" time from this run, based on the idea that the256            // minimum time has the least OS noise in it so it should be more stable.257            List<double> times = new List<double>();258            foreach (var row in report.GetResultRuns())259            {260                double msPerTest = row.Nanoseconds / 1000000.0 / row.Operations;261                times.Add(msPerTest);262            }263            var e = new PerfEntity(this.MachineName, this.RuntimeVersion, this.CommitId, testName, 0)264            {265                Time = times.Min(),266                TimeStdDev = MathHelpers.StandardDeviation(times),267            };268            if (report.Metrics.ContainsKey("CPU"))269            {270                e.Cpu = report.Metrics["CPU"].Value;271            }272            if (report.Metrics.ContainsKey("TotalMemory"))273            {274                e.Memory = report.Metrics["TotalMemory"].Value;275            }276            results.Add(e);277            return results;278        }279        private void ExportToCsv(List<PerfSummary> results)280        {281            this.SaveSummary(results);282            foreach (var item in results)283            {284                this.SaveReport(item.Data);285            }286        }287        private void SaveSummary(List<PerfSummary> report)288        {289            string filename = Path.Combine(this.OutputDir, "summary.csv");290            bool writeHeaders = !File.Exists(filename);291            using (StreamWriter writer = new StreamWriter(filename, true, Encoding.UTF8))292            {293                if (writeHeaders)294                {295                    PerfSummary.WriteHeaders(writer);296                }297                foreach (var item in report)298                {299                    item.WriteCsv(writer);300                }301            }302        }303        private void SaveReport(List<PerfEntity> data)304        {305            var testName = data[0].TestName.Split(' ')[0];306            string filename = Path.Combine(this.OutputDir, testName + ".csv");307            bool writeHeaders = !File.Exists(filename);308            using (StreamWriter writer = new StreamWriter(filename, true, Encoding.UTF8))309            {310                if (writeHeaders)311                {312                    PerfEntity.WriteHeaders(writer);313                }314                foreach (var item in data)315                {316                    item.WriteCsv(writer);317                }318            }319        }320        private static void PrintUsage()321        {322            Console.WriteLine("Usage: BenchmarkRunner [-outdir name] [-commit id] [-cosmos] [filter] [-upload_commit_log");323            Console.WriteLine("Runs all benchmarks matching the optional filter");324            Console.WriteLine("Writing output csv files to the specified outdir folder");325            Console.WriteLine("Benchmark names are:");326            foreach (var item in Benchmarks)327            {328                Console.WriteLine("  {0}", item.Name);329            }330        }331        private static bool FilterMatches(string name, List<string> filters)332        {333            if (filters.Count is 0)334            {335                return true;336            }337            return (from f in filters where name.IndexOf(f, StringComparison.OrdinalIgnoreCase) >= 0 select f).Any();338        }339        public static string GetRuntimeVersion()340        {341#if NET5_0342            return "net5.0";343#elif NET48344            return "net48";345#elif NETSTANDARD2_1346            return "netstandard2.1";347#elif NETSTANDARD2_0348            return "netstandard2.0";349#elif NETSTANDARD350            return "netstandard";351#elif NETCOREAPP3_1352            return "netcoreapp3.1";353#elif NETCOREAPP354            return "netcoreapp";355#elif NETFRAMEWORK356            return "net";357#endif358        }359        private static async Task<string> RunCommandAsync(string cmd, string args)360        {361            StringBuilder sb = new StringBuilder();362            string fullPath = FindProgram(cmd);363            if (fullPath.Contains(' '))364            {365                fullPath = "\"" + fullPath + "\"";366            }367            ProcessStartInfo info = new ProcessStartInfo(fullPath, args);368            info.RedirectStandardOutput = true;369            info.RedirectStandardError = true;370            info.UseShellExecute = false;371            Process p = new Process();372            p.StartInfo = info;373            var outputEnded = new TaskCompletionSource<bool>();374            var errorEnded = new TaskCompletionSource<bool>();375            p.OutputDataReceived += (s, e) =>376            {377                if (!string.IsNullOrEmpty(e.Data))378                {379                    sb.AppendLine(e.Data);380                }381                else382                {383                    outputEnded.TrySetResult(true);384                }385            };386            p.ErrorDataReceived += (s, e) =>387            {388                if (!string.IsNullOrEmpty(e.Data))389                {390                    sb.AppendLine(e.Data);391                }392                else393                {394                    errorEnded.TrySetResult(true);395                }396            };397            if (!p.Start())398            {399                Console.WriteLine("Error running '{0} {1}'", fullPath, args);400                return null;401            }402            p.BeginErrorReadLine();403            p.BeginOutputReadLine();404            if (!p.HasExited)405            {406                p.WaitForExit();407            }408            await Task.WhenAll(outputEnded.Task, errorEnded.Task);409            return sb.ToString();410        }411        private static string FindProgram(string name)412        {413            string path = Environment.GetEnvironmentVariable("PATH");414            foreach (var part in path.Split(Path.PathSeparator))415            {416                string fullPath = Path.Combine(part, name);417                if (File.Exists(fullPath))418                {419                    return fullPath;420                }421                if (File.Exists(fullPath + ".exe"))422                {423                    return fullPath + ".exe";424                }425            }426            return null;427        }428        private static async Task<int> UploadCommitHistory(int max = 0)429        {430            string args = "log --pretty=medium";431            if (max != 0)432            {433                args += string.Format(" -n {0}", max);434            }435            var gitLog = await RunCommandAsync("git", args);436            if (string.IsNullOrEmpty(gitLog))437            {438                return 1;439            }440            var log = ParseLog(gitLog);441            Storage storage = new Storage();442            await storage.UploadLogAsync(log);443            return 0;444        }445        private static List<CommitHistoryEntity> ParseLog(string log)446        {447            List<CommitHistoryEntity> result = new List<CommitHistoryEntity>();448            string commit = null;449            string author = null;450            foreach (string line in log.Split('\n'))451            {452                string trimmed = line.Trim('\r');453                if (trimmed.StartsWith("commit "))454                {455                    commit = trimmed.Substring(7).Trim();...

Storage.cs

Source:Storage.cs

...13{14    /// <summary>15    /// Wrapper on the Cosmos DB table used to store the perf results.16    /// </summary>17    public class Storage18    {19        private const string CosmosDatabaseId = "actorperfdb";20        private const string SummaryTableName = "actorperfsummary";21        private const string CommitLogTableName = "commitlog";22        // private const string DetailsTableName = "actorperfdetails";23        // Maximum number of operations in a transactional batch is 10024        // See https://docs.microsoft.com/en-us/azure/cosmos-db/concepts-limits#per-request-limits25        // private const int BatchMaximum = 100;26        private CosmosClient CosmosClient;27        private Database CosmosDatabase;28        private Container SummaryContainer;29        private async Task Connect()30        {31            var endpointUri = Environment.GetEnvironmentVariable("AZURE_COSMOSDB_ENDPOINT");...

Storage

Using AI Code Generation

1using Microsoft.Coyote.Benchmarking;2Storage.Write("1.cs", "Hello World!");3using Microsoft.Coyote.Benchmarking;4Storage.Write("2.cs", "Hello World!");5using Microsoft.Coyote.Benchmarking;6Storage.Write("3.cs", "Hello World!");7using Microsoft.Coyote.Benchmarking;8Storage.Write("4.cs", "Hello World!");9using Microsoft.Coyote.Benchmarking;10Storage.Write("5.cs", "Hello World!");11using Microsoft.Coyote.Benchmarking;12Storage.Write("6.cs", "Hello World!");13using Microsoft.Coyote.Benchmarking;14Storage.Write("7.cs", "Hello World!");15using Microsoft.Coyote.Benchmarking;16Storage.Write("8.cs", "Hello World!");17using Microsoft.Coyote.Benchmarking;18Storage.Write("9.cs", "Hello World!");19using Microsoft.Coyote.Benchmarking;20Storage.Write("10.cs", "Hello World!");21using Microsoft.Coyote.Benchmarking;22Storage.Write("11.cs", "Hello World!");23using Microsoft.Coyote.Benchmarking;24Storage.Write("12.cs", "Hello World!");

Storage

Using AI Code Generation

1using Microsoft.Coyote.Benchmarking;2using System;3{4    {5        public static void Main(string[] args)6        {7            Storage storage = new Storage();8            storage.Write("Hello");9            Console.WriteLine(storage.Read());10        }11    }12}13using Microsoft.Coyote.Benchmarking;14using System;15{16    {17        public static void Main(string[] args)18        {19            Storage storage = new Storage();20            storage.Write("Hello");21            Console.WriteLine(storage.Read());22        }23    }24}

Storage

Using AI Code Generation

1{2    {3        public static void Write(string key, string value)4        {5        }6    }7}8{9    {10        public static void Write(string key, string value)11        {12        }13    }14}15{16    {17        public static void Write(string key, string value)18        {19        }20    }21}22{23    {24        public static void Write(string key, string value)25        {26        }27    }28}29{30    {31        public static void Write(string key, string value)32        {33        }34    }35}36{37    {38        public static void Write(string key, string value)39        {40        }41    }42}43{44    {45        public static void Write(string key, string value)46        {47        }48    }49}

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.