Unlock 30% off on Manual Testing Annual Plans this Holiday Season.

Copied to Clipboard!

How to use Mean method of Microsoft.Coyote.Benchmarking.MathHelpers class

Best Coyote code snippet using Microsoft.Coyote.Benchmarking.MathHelpers.Mean

Storage.cs

Source:Storage.cs

...66                try67                {68                    var response = await this.SummaryContainer.ReadItemAsync<PerfSummary>(s.Id, new PartitionKey(s.PartitionKey));69                    PerfSummary old = response.Resource;70                    if (old.TimeMean < s.TimeMean)71                    {72                        better = false;73                    }74                }75                catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound)76                {77                }78                if (better)79                {80                    Console.WriteLine("===> Uploading summary for {0}...", s.TestName);81                    await this.SummaryContainer.UpsertItemAsync(s, new PartitionKey(s.PartitionKey));82                    count++;83                }84                else85                {86                    Console.WriteLine("===> Existing record is better for {0}...", s.TestName);87                }88            }89            return count;90        }91        internal async Task<List<PerfSummary>> DownloadAsync(string partitionKey, List<string> rowKeys)92        {93            List<PerfSummary> results = new List<PerfSummary>();94            if (this.CosmosDatabase is null)95            {96                await this.Connect();97            }98            if (this.CosmosDatabase is null)99            {100                return results;101            }102            if (this.SummaryContainer is null)103            {104                var response = await this.CosmosDatabase.CreateContainerIfNotExistsAsync(SummaryTableName, "/PartitionKey");105                this.SummaryContainer = response.Container;106            }107            foreach (var rowKey in rowKeys)108            {109                try110                {111                    var response = await this.SummaryContainer.ReadItemAsync<PerfSummary>(rowKey, new PartitionKey(partitionKey));112                    results.Add(response.Resource);113                }114                catch (CosmosException ex) when (ex.StatusCode == HttpStatusCode.NotFound)115                {116                }117            }118            return results;119        }120        internal async Task UploadLogAsync(List<CommitHistoryEntity> log)121        {122            if (this.CosmosDatabase is null)123            {124                await this.Connect();125            }126            if (this.CosmosDatabase is null)127            {128                return;129            }130            var response = await this.CosmosDatabase.CreateContainerIfNotExistsAsync(CommitLogTableName, "/PartitionKey");131            var container = response.Container;132            foreach (var item in log)133            {134                Console.WriteLine("===> Uploading commit info {0}...", item.Id);135                await container.UpsertItemAsync(item, new PartitionKey(item.PartitionKey));136            }137        }138    }139    /// <summary>140    /// Entity representing a test result.141    /// </summary>142    public class PerfEntity143    {144        /// <summary>145        /// The row id.146        /// </summary>147        [JsonProperty(PropertyName = "id")]148        public string Id { get; set; }149        /// <summary>150        /// The parition key for the data.151        /// </summary>152        public string PartitionKey { get; set; }153        /// <summary>154        /// The computer name where the test was run.155        /// </summary>156        public string MachineName { get; set; }157        /// <summary>158        /// The .net runtime version used.159        /// </summary>160        public string RuntimeVersion { get; set; }161        /// <summary>162        /// The git commit id of code being tested.163        /// </summary>164        public string CommitId { get; set; }165        /// <summary>166        /// UTC date and time the test was run.167        /// </summary>168        public DateTime Date { get; set; }169        /// <summary>170        /// The unique name of the test.171        /// </summary>172        public string TestName { get; set; }173        /// <summary>174        /// The test iteration.175        /// </summary>176        public int Iteration { get; set; }177        /// <summary>178        /// Time to complete the test in milliseconds.179        /// </summary>180        public double Time { get; set; }181        /// <summary>182        /// Standard deviation in the times.183        /// </summary>184        public double TimeStdDev { get; set; }185        /// <summary>186        /// Process working set during the test.187        /// </summary>188        public double Memory { get; set; }189        /// <summary>190        /// Standard deviation in the memory numbers.191        /// </summary>192        public double MemoryStdDev { get; set; }193        /// <summary>194        /// Process total CPU usage during the test as a % of total number of cores.195        /// </summary>196        public double Cpu { get; set; }197        /// <summary>198        /// Standard deviation in the CPU numbers.199        /// </summary>200        public double CpuStdDev { get; set; }201        /// <summary>202        /// Additional notes about the test.203        /// </summary>204        public string Comments { get; set; }205        /// <summary>206        /// Initializes a new instance of the <see cref="PerfEntity"/> class.207        /// </summary>208        public PerfEntity(string machine, string runtime, string commit, string testName, int iteration)209        {210            this.MachineName = machine;211            this.RuntimeVersion = runtime;212            this.CommitId = commit;213            this.TestName = testName;214            this.Iteration = iteration;215            this.Date = DateTime.Now.ToUniversalTime();216            this.PartitionKey = string.Format("{0}.{1}", machine, runtime);217            this.Id = string.Format("{0}.{1}.{2}", commit, testName, iteration);218        }219        /// <summary>220        /// Initializes a new instance of the <see cref="PerfEntity"/> class.221        /// Needed for retreival.222        /// </summary>223        public PerfEntity()224        {225        }226        internal static void WriteHeaders(TextWriter outFile)227        {228            outFile.WriteLine("Name,Iteration,MinTime,StdDevTime,Memory,StdDevMemory,Cpu,StdDevCpu");229        }230        internal void WriteCsv(TextWriter outFile)231        {232            outFile.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7}", this.TestName, this.Iteration, this.Time, this.TimeStdDev, this.Memory, this.MemoryStdDev, this.Cpu, this.CpuStdDev);233        }234    }235    /// <summary>236    /// An entity representing the summary of all test iterations on a given test.237    /// </summary>238    public class PerfSummary239    {240        /// <summary>241        /// The row id.242        /// </summary>243        [JsonProperty(PropertyName = "id")]244        public string Id { get; set; }245        /// <summary>246        /// The parition key for the data.247        /// </summary>248        public string PartitionKey { get; set; }249        /// <summary>250        /// The computer name where the test was run.251        /// </summary>252        public string MachineName { get; set; }253        /// <summary>254        /// The .net runtime version used.255        /// </summary>256        public string RuntimeVersion { get; set; }257        /// <summary>258        /// The git commit id of code being tested.259        /// </summary>260        public string CommitId { get; set; }261        /// <summary>262        /// UTC date and time the test was run.263        /// </summary>264        public DateTime Date { get; set; }265        /// <summary>266        /// The unique name of the test.267        /// </summary>268        public string TestName { get; set; }269        /// <summary>270        /// The mean time in milliseconds.271        /// </summary>272        public double TimeMean { get; set; }273        /// <summary>274        /// The standard deviation of the test times.275        /// </summary>276        public double TimeStdDev { get; set; }277        /// <summary>278        /// The slope of the linear regression of the test times.279        /// </summary>280        public double TimeSlope { get; set; }281        /// <summary>282        /// The mean time in milliseconds.283        /// </summary>284        public double MemoryMean { get; set; }285        /// <summary>286        /// The standard deviation of the memory usage.287        /// </summary>288        public double MemoryStdDev { get; set; }289        /// <summary>290        /// The slope of the linear regression of the memory usage.291        /// </summary>292        public double MemorySlope { get; set; }293        /// <summary>294        /// The process cpu utilization as a percentage of total cores.295        /// </summary>296        public double CpuMean { get; set; }297        /// <summary>298        /// The standard deviation of the cpu times.299        /// </summary>300        public double CpuStdDev { get; set; }301        /// <summary>302        /// The slope of the linear regression of the cpu utilization.303        /// </summary>304        public double CpuSlope { get; set; }305        /// <summary>306        /// Additional notes about the test.307        /// </summary>308        public string Comments { get; set; }309        /// <summary>310        /// The raw test iterations.311        /// </summary>312        [IgnoreDataMember]313        internal readonly List<PerfEntity> Data;314        /// <summary>315        /// Initializes a new instance of the <see cref="PerfSummary"/> class.316        /// </summary>317        public PerfSummary(List<PerfEntity> data)318        {319            PerfEntity e = data[0];320            this.MachineName = e.MachineName;321            this.RuntimeVersion = e.RuntimeVersion;322            this.CommitId = e.CommitId;323            this.Data = data;324            this.TestName = e.TestName;325            this.Date = DateTime.Now.ToUniversalTime();326            this.PartitionKey = string.Format("{0}.{1}", e.MachineName, e.RuntimeVersion);327            this.Id = string.Format("{0}.{1}", e.CommitId, e.TestName);328            // summaryize the data.329            double meanTime = MathHelpers.Mean(from i in data select i.Time);330            double meanMemory = MathHelpers.Mean(from i in data select i.Memory);331            double meanCpu = MathHelpers.Mean(from i in data select i.Cpu);332            double meanStdDevTime = MathHelpers.Mean(from i in data select i.TimeStdDev);333            double meanStdDevMemory = MathHelpers.Mean(from i in data select i.MemoryStdDev);334            double meanStdDevCpu = MathHelpers.Mean(from i in data select i.CpuStdDev);335            if (meanStdDevTime is 0)336            {337                meanStdDevTime = MathHelpers.StandardDeviation(from i in data select i.Time);338                meanStdDevMemory = MathHelpers.StandardDeviation(from i in data select i.Memory);339                meanStdDevCpu = MathHelpers.StandardDeviation(from i in data select i.Cpu);340            }341            double timeSlope = MathHelpers.LinearRegression(MathHelpers.ToDataPoints(from i in data select i.Time)).Slope / meanTime;342            double memSlope = MathHelpers.LinearRegression(MathHelpers.ToDataPoints(from i in data select i.Memory)).Slope / meanMemory;343            double cpuSlope = MathHelpers.LinearRegression(MathHelpers.ToDataPoints(from i in data select i.Cpu)).Slope / meanCpu;344            // more than 10% slope we have a problem!345            if (timeSlope > 0.1)346            {347                this.Comments = "Slow down?";348            }349            else if (memSlope > 0.1)350            {351                this.Comments = "Memory leak?";352            }353            else if (cpuSlope > 0.1)354            {355                this.Comments = "Thread leak?";356            }357            this.TimeMean = meanTime;358            this.TimeStdDev = meanStdDevTime;359            this.TimeSlope = timeSlope;360            this.MemoryMean = meanMemory;361            this.MemoryStdDev = meanStdDevMemory;362            this.MemorySlope = memSlope;363            this.CpuMean = meanCpu;364            this.CpuStdDev = meanStdDevCpu;365            this.CpuSlope = cpuSlope;366        }367        /// <summary>368        /// Initializes a new instance of the <see cref="PerfSummary"/> class.369        /// Needed for retreival.370        /// </summary>371        public PerfSummary()372        {373        }374        internal static void WriteHeaders(TextWriter outFile)375        {376            outFile.WriteLine("Machine,Runtime,Commit,Date,Test,TimeMean,TimeStdDev,TimeSlope,MemoryMean,MemoryStdDev,MemorySlope,CpuMean,CpuStdDev,CpuSlope");377        }378        internal void WriteCsv(TextWriter outFile)379        {380            outFile.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11},{12},{13}", this.MachineName,381                this.RuntimeVersion, this.CommitId,  this.Date.ToLocalTime(), this.TestName, this.TimeMean,382                this.TimeStdDev, this.TimeSlope, this.MemoryMean, this.MemoryStdDev, this.MemorySlope,383                this.CpuMean, this.CpuStdDev, this.CpuSlope);384        }385        internal void SetPartitionKey(string partitionKey)386        {387            this.PartitionKey = partitionKey;388            int pos = partitionKey.IndexOf(".");389            if (pos > 0)390            {391                this.MachineName = partitionKey.Substring(0, pos);392                this.RuntimeVersion = partitionKey.Substring(pos + 1);393            }394        }395    }396    /// <summary>397    /// Entity representing a commit id....

MathHelpers.cs

Source:MathHelpers.cs

...54    /// </summary>55    public static class MathHelpers56    {57        /// <summary>58        /// Return the Mean of the given numbers.59        /// </summary>60        public static double Mean(IEnumerable<double> values)61        {62            double sum = 0;63            double count = 0;64            foreach (double d in values)65            {66                sum += d;67                count++;68            }69            if (count is 0)70            {71                return 0;72            }73            return sum / count;74        }75        /// <summary>76        /// Return the standard deviation of the given values.77        /// </summary>78        public static double StandardDeviation(IEnumerable<double> values)79        {80            double mean = Mean(values);81            double totalSquares = 0;82            int count = 0;83            foreach (double v in values)84            {85                count++;86                double diff = mean - v;87                totalSquares += diff * diff;88            }89            if (count is 0)90            {91                return 0;92            }93            return Math.Sqrt(totalSquares / count);94        }95        /// <summary>96        /// Trim values outside of +/- the given range from the mean.97        /// </summary>98        internal static IEnumerable<double> Trim(IEnumerable<double> times, double range)99        {100            double mean = Mean(times);101            foreach (var item in times)102            {103                if (item <= mean + range && item >= mean - range)104                {105                    yield return item;106                }107            }108        }109        /// <summary>110        /// Trim values outside mean + range.111        /// </summary>112        internal static IEnumerable<double> TrimHigh(IEnumerable<double> times, double range)113        {114            double mean = Mean(times);115            foreach (var item in times)116            {117                if (item <= mean + range)118                {119                    yield return item;120                }121            }122        }123        /// <summary>124        /// Return the variance, sum of the difference between each value and the mean, squared.125        /// </summary>126        public static double Variance(IEnumerable<double> values)127        {128            double mean = Mean(values);129            double variance = 0;130            foreach (double d in values)131            {132                double diff = d - mean;133                variance += diff * diff;134            }135            return variance;136        }137        /// <summary>138        /// Convert the list of doubles into a list of DataPoints.139        /// </summary>140        public static List<DataPoint> ToDataPoints(IEnumerable<double> values)141        {142            int index = 0;143            return new List<DataPoint>(from v in values select new DataPoint(index++, v));144        }145        /// <summary>146        /// Return the covariance in the given x,y values.147        /// The sum of the difference between x and its mean times the difference between y and its mean.148        /// </summary>149        public static double Covariance(IEnumerable<DataPoint> pts)150        {151            double xsum = 0;152            double ysum = 0;153            double count = 0;154            foreach (var d in pts)155            {156                xsum += d.X;157                ysum += d.Y;158                count++;159            }160            if (count is 0)161            {162                return 0;163            }164            double xMean = xsum / count;165            double yMean = ysum / count;166            double covariance = 0;167            foreach (var d in pts)168            {169                covariance += (d.X - xMean) * (d.Y - yMean);170            }171            return covariance;172        }173        /// <summary>174        /// Compute the trend line through the given points, and return the line in the form:175        /// y = a + b.x.176        /// </summary>177        /// <param name="pts">The data to analyze.</param>178        public static Line LinearRegression(IEnumerable<DataPoint> pts)179        {180            double xMean = Mean(from p in pts select p.X);181            double yMean = Mean(from p in pts select p.Y);182            double xVariance = Variance(from p in pts select p.X);183            double yVariance = Variance(from p in pts select p.Y);184            double covariance = Covariance(pts);185            double a = 0;186            double b = 0;187            if (xVariance is 0)188            {189                a = yMean;190                b = 1;191            }192            else193            {194                b = covariance / xVariance;195                a = yMean - (b * xMean);196            }197            return new Line(a, b);198        }199    }200}...

Mean

Using AI Code Generation

1using Microsoft.Coyote.Benchmarking;2using System;3using System.Collections.Generic;4using System.Linq;5using System.Text;6using System.Threading.Tasks;7{8    {9        static void Main(string[] args)10        {11            double[] array = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };12            double mean = MathHelpers.Mean(array);13            Console.WriteLine("mean is " + mean);14        }15    }16}17using Microsoft.Coyote.Benchmarking;18using Microsoft.Coyote.Benchmarking;19using System;20using System.Collections.Generic;21using System.Linq;22using System.Text;23using System.Threading.Tasks;24{25    {26        static void Main(string[] args)27        {28            double[] array = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };29            double median = MathHelpers.Median(array);30            Console.WriteLine("median is " + median);31        }32    }33}

Mean

Using AI Code Generation

1using Microsoft.Coyote.Benchmarking;2var mean = MathHelpers.Mean(values);3using Microsoft.Coyote.Benchmarking;4var mean = MathHelpers.Mean(values);5using Microsoft.Coyote.Benchmarking;6var mean = MathHelpers.Mean(values);7using Microsoft.Coyote.Benchmarking;8var mean = MathHelpers.Mean(values);9using Microsoft.Coyote.Benchmarking;10var mean = MathHelpers.Mean(values);11using Microsoft.Coyote.Benchmarking;12var mean = MathHelpers.Mean(values);13using Microsoft.Coyote.Benchmarking;14var mean = MathHelpers.Mean(values);15using Microsoft.Coyote.Benchmarking;16var mean = MathHelpers.Mean(values);17using Microsoft.Coyote.Benchmarking;18var mean = MathHelpers.Mean(values);19using Microsoft.Coyote.Benchmarking;20var mean = MathHelpers.Mean(values);21using Microsoft.Coyote.Benchmarking;22var mean = MathHelpers.Mean(values);23using Microsoft.Coyote.Benchmarking;

Mean

Using AI Code Generation

1using Microsoft.Coyote.Benchmarking;2using System;3{4    {5        public static double Mean(double[] values)6        {7            double sum = 0;8            for (int i = 0; i < values.Length; i++)9            {10                sum += values[i];11            }12            return sum / values.Length;13        }14    }15}16using Microsoft.Coyote.Benchmarking;17using System;18{19    {20        public static double Mean(double[] values)21        {22            double sum = 0;23            for (int i = 0; i < values.Length; i++)24            {25                sum += values[i];26            }27            return sum / values.Length;28        }29    }30}31using Microsoft.Coyote.Benchmarking;32using System;33{34    {35        public static double Mean(double[] values)36        {37            double sum = 0;38            for (int i = 0; i < values.Length; i++)39            {40                sum += values[i];41            }42            return sum / values.Length;43        }44    }45}46using Microsoft.Coyote.Benchmarking;47using System;48{49    {50        public static double Mean(double[] values)51        {52            double sum = 0;53            for (int i = 0; i < values.Length; i++)54            {55                sum += values[i];56            }57            return sum / values.Length;58        }59    }60}61using Microsoft.Coyote.Benchmarking;62using System;63{64    {65        public static double Mean(double

Mean

Using AI Code Generation

1using Microsoft.Coyote.Benchmarking;2double result = MathHelpers.Mean(1, 2, 3);3result = MathHelpers.Median(1, 2, 3);4using Microsoft.Coyote.Benchmarking;5double result = MathHelpers.Mean(new List<int> { 1, 2, 3 });6result = MathHelpers.Median(new List<int> { 1, 2, 3 });7using Microsoft.Coyote.Benchmarking;8double result = MathHelpers.Mean(new List<double> { 1.1, 2.2, 3.3 });9result = MathHelpers.Median(new List<double> { 1.1, 2.2, 3.3 });10using Microsoft.Coyote.Benchmarking;11double result = MathHelpers.Mean(new List<int> { 1, 2, 3 }, new ReverseComparer());12result = MathHelpers.Median(new List<int> { 1, 2, 3 }, new ReverseComparer());

Mean

Using AI Code Generation

1using Microsoft.Coyote.Benchmarking;2using System;3using System.Collections.Generic;4using System.Linq;5{6    {7        static void Main(string[] args)8        {9            List<int> list = new List<int> { 1, 2, 3, 4, 5 };10            var mean = MathHelpers.Mean(list);11            Console.WriteLine(mean);12        }13    }14}15using Microsoft.Coyote.Benchmarking;16using System;17using System.Collections.Generic;18using System.Linq;19{20    {21        static void Main(string[] args)22        {23            List<int> list = new List<int> { 1, 2, 3, 4, 5 };24            var mean = MathHelpers.Mean(list);25            Console.WriteLine(mean);26        }27    }28}29using Microsoft.Coyote.Benchmarking;30using System;31using System.Collections.Generic;32using System.Linq;33{34    {35        static void Main(string[] args)36        {37            List<int> list = new List<int> { 1, 2, 3, 4, 5 };38            var mean = MathHelpers.Mean(list);39            Console.WriteLine(mean);40        }41    }42}43using Microsoft.Coyote.Benchmarking;44using System;45using System.Collections.Generic;46using System.Linq;47{48    {49        static void Main(string[] args)50        {51            List<int> list = new List<int> { 1, 2, 3, 4, 5 };52            var mean = MathHelpers.Mean(list);53            Console.WriteLine(mean);54        }55    }56}57using Microsoft.Coyote.Benchmarking;58using System;59using System.Collections.Generic;60using System.Linq;61{62    {63        static void Main(string[] args)

Mean

Using AI Code Generation

1using Microsoft.Coyote.Benchmarking;2using System;3{4    {5        static void Main(string[] args)6        {7            var data = new double[] { 1.0, 2.0, 3.0, 4.0, 5.0 };8            var mean = MathHelpers.Mean(data);9            Console.WriteLine($"Mean: {mean}");10        }11    }12}

Mean

Using AI Code Generation

1using Microsoft.Coyote.Benchmarking;2{3    {4        public static void Main()5        {6            double[] data = new double[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };7            double mean = MathHelpers.Mean(data);8            System.Console.WriteLine(mean);9        }10    }11}12{13    {14        public static double Mean(double[] values)15        {16            double sum = 0;17            for (int i = 0; i < values.Length; i++)18            {19                sum += values[i];20            }21            return sum / values.Length;22        }23    }24}25{26    {27        public static double Mean(double[] values)28        {29            double sum = 0;30            Parallel.For(0, values.Length, i => sum += values[i]);

Mean

Using AI Code Generation

1using System.Collections.Generic;2using Microsoft.Coyote.Benchmarking;3{4    {5        static void Main(string[] args)6        {7            List<double> list = new List<double> { 1.0, 2.0, 3.0, 4.0, 5.0 };8            double mean = MathHelpers.Mean(list);9        }10    }11}12using System.Collections.Generic;13using Microsoft.Coyote.Benchmarking;14{15    {16        static void Main(string[] args)17        {18            List<double> list = new List<double> { 1.0, 2.0, 3.0, 4.0, 5.0 };19            double median = MathHelpers.Median(list);20        }21    }22}23using System.Collections.Generic;24using Microsoft.Coyote.Benchmarking;25{26    {27        static void Main(string[] args)28        {29            List<double> list = new List<double> { 1.0, 2.0, 3.0, 4.0, 5.0 };30            double standardDeviation = MathHelpers.StandardDeviation(list);31        }32    }33}34using System.Collections.Generic;35using Microsoft.Coyote.Benchmarking;36{37    {38        static void Main(string[] args)39        {40            List<double> list = new List<double> { 1.0, 2.0, 3.0, 4.0, 5.0 };41            double standardError = MathHelpers.StandardError(list);42        }43    }44}

Mean

Using AI Code Generation

1public static void Main(string[] args)2{3  var mean = MathHelpers.Mean(new double[] { 1, 2, 3, 4, 5 });4  Console.WriteLine($"Mean is {mean}");5}6public static void Main(string[] args)7{8  var mean = MathHelpers.Mean(new double[] { 1, 2, 3, 4, 5 });9  Console.WriteLine($"Mean is {mean}");10}11public static void Main(string[] args)12{13  var mean = MathHelpers.Mean(new double[] { 1, 2, 3, 4, 5 });14  Console.WriteLine($"Mean is {mean}");15}16public static void Main(string[] args)17{18  var mean = MathHelpers.Mean(new double[] { 1, 2, 3, 4, 5 });19  Console.WriteLine($"Mean is {mean}");20}21public static void Main(string[] args)22{23  var mean = MathHelpers.Mean(new double[] { 1, 2, 3, 4, 5 });24  Console.WriteLine($"Mean is {mean}");25}26public static void Main(string[] args)27{28  var mean = MathHelpers.Mean(new double[] { 1, 2, 3, 4, 5 });29  Console.WriteLine($"Mean is {mean}");30}31public static void Main(string[] args)32{33  var mean = MathHelpers.Mean(new double[] { 1, 2, 3, 4, 5 });34  Console.WriteLine($"Mean is {mean}");35}

Automation Testing Tutorials

Learn to execute automation testing from scratch with LambdaTest Learning Hub. Right from setting up the prerequisites to run your first automation test, to following best practices and diving deeper into advanced test scenarios. LambdaTest Learning Hubs compile a list of step-by-step guides to help you be proficient with different test automation frameworks i.e. Selenium, Cypress, TestNG etc.