diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e96287f --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +################################################################################ +# This .gitignore file was automatically created by Microsoft(R) Visual Studio. +################################################################################ + +/.vs +/BenchmarkDotNetBigQuery/bin +/BenchmarkDotNetBigQuery/obj +*.user +/BenchmarkDotNetBigQuery/project.lock.json diff --git a/BenchmarkDotNetBigQuery.sln b/BenchmarkDotNetBigQuery.sln new file mode 100644 index 0000000..6fc9d2b --- /dev/null +++ b/BenchmarkDotNetBigQuery.sln @@ -0,0 +1,27 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 14 +VisualStudioVersion = 14.0.25420.1 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "BenchmarkDotNetBigQuery", "BenchmarkDotNetBigQuery\BenchmarkDotNetBigQuery.xproj", "{0A4F8F8F-E568-4B5C-B8C2-1B36BDB2BC94}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{EA215BB9-D476-40C3-9CE8-4A62B325A10F}" + ProjectSection(SolutionItems) = preProject + README.md = README.md + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {0A4F8F8F-E568-4B5C-B8C2-1B36BDB2BC94}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0A4F8F8F-E568-4B5C-B8C2-1B36BDB2BC94}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0A4F8F8F-E568-4B5C-B8C2-1B36BDB2BC94}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0A4F8F8F-E568-4B5C-B8C2-1B36BDB2BC94}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/BenchmarkDotNetBigQuery/BenchmarkDotNetBigQuery.xproj b/BenchmarkDotNetBigQuery/BenchmarkDotNetBigQuery.xproj new file mode 100644 index 0000000..2b9e0e0 --- /dev/null +++ b/BenchmarkDotNetBigQuery/BenchmarkDotNetBigQuery.xproj @@ -0,0 +1,21 @@ + + + + 14.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + + + 0a4f8f8f-e568-4b5c-b8c2-1b36bdb2bc94 + BenchmarkDotNetBigQuery + .\obj + .\bin\ + v4.5.2 + + + + 2.0 + + + diff --git a/BenchmarkDotNetBigQuery/BigQueryExporter.cs b/BenchmarkDotNetBigQuery/BigQueryExporter.cs new file mode 100644 index 0000000..8960113 --- /dev/null +++ b/BenchmarkDotNetBigQuery/BigQueryExporter.cs @@ -0,0 +1,256 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using Google.Apis.Auth.OAuth2; +using Google.Apis.Bigquery.v2.Data; +using Google.Cloud.BigQuery.V2; + +namespace BenchmarkDotNetBigQuery +{ + /// + /// A BenchmarkDotNet Exporter that saves benchmark data to Google BigQuery Tables. + /// + public class BigQueryExporter: IExporter + { + private string CommitId { get; } + private bool OneSummary { get; } + private BigQueryTable SummaryTable { get; } + private BigQueryTable ReportTable { get; } + + private TableSchema SummaryTableSchema { get; } = new TableSchemaBuilder + { + {"Id", BigQueryDbType.String}, + {"Commit", BigQueryDbType.String}, + {"Timestamp", BigQueryDbType.Timestamp}, + {"HostName", BigQueryDbType.String}, + {"OsVersion", BigQueryDbType.String}, + {"ProcessorName", BigQueryDbType.String}, + {"ProcessorCount", BigQueryDbType.Int64}, + {"RuntimeVersion", BigQueryDbType.String}, + {"Architecture", BigQueryDbType.String}, + {"JitModules", BigQueryDbType.String}, + {"DotNetCoreVersion", BigQueryDbType.String}, + {"BenchmarkDotNetVersion", BigQueryDbType.String}, + {"ChronometerFrequency", BigQueryDbType.Int64}, + {"HardwareTimerKind", BigQueryDbType.String} + }.Build(); + + private TableSchema ReportTableSchema { get; } = new TableSchemaBuilder + { + {"Id", BigQueryDbType.String }, + {"SummaryId", BigQueryDbType.String}, + {"Namespace", BigQueryDbType.String}, + {"Type", BigQueryDbType.String}, + {"FullType", BigQueryDbType.String}, + {"MethodName", BigQueryDbType.String}, + {"FullMethodName", BigQueryDbType.String}, + {"Parameters", BigQueryDbType.String}, + {"MethodSignature", BigQueryDbType.String}, + {"Min", BigQueryDbType.Float64}, + {"Max", BigQueryDbType.Float64}, + {"Mean", BigQueryDbType.Float64}, + {"Median", BigQueryDbType.Float64}, + {"StandardDeviation", BigQueryDbType.Float64}, + {"StandardError", BigQueryDbType.Float64}, + {"Variance", BigQueryDbType.Float64}, + {"Percentile67", BigQueryDbType.Float64}, + {"Percentile85", BigQueryDbType.Float64}, + {"Percentile95", BigQueryDbType.Float64}, + {"Percentile100", BigQueryDbType.Float64} + }.Build(); + + private string _summaryId; + + /// + /// During construction, the BigQueryExporter will create the dataset and tables if needed. + /// If the dataset and tables already exist, it will validate that the tables contain the necessary fields. + /// + /// Id of the commit e.g. git hash. + /// The id of the google project to upload to. + /// + /// The id of the Google BigQuery dataset that contains the target tables. + /// + /// + /// BenchmarkDotNet provides a separate summary for every class. If this parameter is false, the exporter + /// creates a new summary row for every summary. If true, creates a single summary row for the entire + /// lifetime of the exporter. Defaults to true. + /// + /// The id of table to put summary information in. + /// The id of table to put report information in. + /// Defaults to application default credentials if unspecified. + public BigQueryExporter( + string commitId, + string googleProjectId, + string datasetId, + bool oneSummary = true, + string summaryTableId = "BenchmarkSummary", + string reportTableId = "BenchmarkReport", + GoogleCredential googleCredential = null) + { + CommitId = commitId; + OneSummary = oneSummary; + Task bqClientTask = BigQueryClient.CreateAsync(googleProjectId, googleCredential); + Tuple tables = + GetValidTablesFromDataset(datasetId, summaryTableId, reportTableId, bqClientTask).Result; + SummaryTable = tables.Item1; + ReportTable = tables.Item2; + } + + /// + /// BigQueryExporter does not write to a logger. It sends an error message to the logger. + /// + public void ExportToLog(Summary summary, ILogger logger) + { + logger.WriteLine(LogKind.Error, $"{nameof(BigQueryExporter)} does not output to a logger."); + } + + /// + /// This is where BigQueryExporter writes benchmark data to the BigQuery tables. + /// + /// The summary to upload to Google BigQuery + /// Unused + /// A string containing the summary guid and the names of the tables they are stoed in. + public IEnumerable ExportToFiles(Summary summary, ILogger logger) + { + Task insertSummaryTask = null; + if (!OneSummary || _summaryId == null) + { + _summaryId = Guid.NewGuid().ToString(); + var summaryRow = BuildSummaryRow(summary, _summaryId); + insertSummaryTask = SummaryTable.InsertAsync(summaryRow); + } + IEnumerable reportRows = summary.Reports.Select(BuildReportRowCurried(_summaryId)); + Task insertReportTask = ReportTable.InsertAsync(reportRows); + insertReportTask.Wait(); + if (insertSummaryTask != null) + { + insertSummaryTask.Wait(); + yield return $"{_summaryId} in {SummaryTable.FullyQualifiedId} and {ReportTable.FullyQualifiedId}"; + } + } + + private BigQueryInsertRow BuildSummaryRow(Summary summary, string summaryId) + { + return new BigQueryInsertRow + { + {"Id", summaryId}, + {"Commit", CommitId}, + {"Timestamp", DateTimeOffset.UtcNow}, + {"HostName", Environment.MachineName}, + {"OsVersion", summary.HostEnvironmentInfo.OsVersion.Value}, + {"ProcessorName", summary.HostEnvironmentInfo.ProcessorName.Value}, + {"ProcessorCount", summary.HostEnvironmentInfo.ProcessorCount}, + {"RuntimeVersion", summary.HostEnvironmentInfo.RuntimeVersion}, + {"Architecture", summary.HostEnvironmentInfo.Architecture}, + {"JitModules", summary.HostEnvironmentInfo.JitModules}, + {"DotNetCoreVersion", summary.HostEnvironmentInfo.DotNetCliVersion.Value}, + {"BenchmarkDotNetVersion", summary.HostEnvironmentInfo.BenchmarkDotNetVersion}, + {"ChronometerFrequency", summary.HostEnvironmentInfo.ChronometerFrequency.Hertz}, + {"HardwareTimerKind", summary.HostEnvironmentInfo.HardwareTimerKind.ToString()} + }; + } + + private static Func BuildReportRowCurried(string summaryId) + { + return (report) => BuildReportRow(summaryId, report); + } + + private static BigQueryInsertRow BuildReportRow(string summaryId, BenchmarkReport report) + { + var fullMethodName = $"{report.Benchmark.Target.Type.FullName}.{report.Benchmark.Target.Method.Name}"; + return new BigQueryInsertRow + { + {"Id", Guid.NewGuid().ToString()}, + {"SummaryId", summaryId}, + {"Namespace", report.Benchmark.Target.Type.Namespace}, + {"Type", report.Benchmark.Target.Type.Name}, + {"FullType", report.Benchmark.Target.Type.FullName}, + {"MethodName", report.Benchmark.Target.Method.Name}, + {"FullMethodName", fullMethodName}, + {"Parameters", report.Benchmark.Parameters.PrintInfo}, + {"MethodSignature", report.Benchmark.Target.Method.ToString()}, + {"Min", report.ResultStatistics.Min}, + {"Max", report.ResultStatistics.Max}, + {"Median", report.ResultStatistics.Median}, + {"Mean", report.ResultStatistics.Mean }, + {"StandardDeviation", report.ResultStatistics.StandardDeviation}, + {"StandardError", report.ResultStatistics.StandardError}, + {"Variance", report.ResultStatistics.Variance}, + {"Percentile67", report.ResultStatistics.Percentiles.P67}, + {"Percentile85", report.ResultStatistics.Percentiles.P85}, + {"Percentile95", report.ResultStatistics.Percentiles.P95}, + {"Percentile100", report.ResultStatistics.Percentiles.P100} + }; + } + + private async Task> GetValidTablesFromDataset( + string datasetId, string summaryTableId, string reportTableId, Task bigQueryClientTask) + { + BigQueryClient bigQueryClient = await bigQueryClientTask; + BigQueryDataset dataset = await bigQueryClient.GetOrCreateDatasetAsync(datasetId); + + Task summaryTableTask = dataset.GetOrCreateTableAsync(summaryTableId, SummaryTableSchema); + Task reportTableTask = dataset.GetOrCreateTableAsync(reportTableId, ReportTableSchema); + + Task validateSummaryTableTask = ValidateTableSchemaAsync(SummaryTableSchema, summaryTableTask); + Task validateReportTableTask = ValidateTableSchemaAsync(ReportTableSchema, reportTableTask); + + await Task.WhenAll(validateSummaryTableTask, validateReportTableTask); + return Tuple.Create(await summaryTableTask, await reportTableTask); + } + + private async Task ValidateTableSchemaAsync(TableSchema testSchema, Task tableTask) + { + BigQueryTable actualTable = await tableTask; + ValidateSchema(testSchema.Fields, actualTable.Schema.Fields, actualTable.Reference.TableId); + } + + private void ValidateSchema( + ICollection testSchema, ICollection actualSchema, string schemaId) + { + int actualCount = actualSchema?.Count ?? 0; + int testCount = testSchema?.Count ?? 0; + if (actualCount < testCount) + { + throw new InvalidOperationException($"Schema for {schemaId} has too few fields."); + } + if (testSchema != null && actualSchema != null) + { + Func fieldNameSelector = schema => schema.Name; + var fields = + testSchema.GroupJoin(actualSchema, fieldNameSelector, fieldNameSelector, Tuple.Create); + foreach (Tuple> fieldTuple in fields) + { + TableFieldSchema testField = fieldTuple.Item1; + IEnumerable actualFields = fieldTuple.Item2; + ValidateFieldSchema(testField, actualFields, $"{schemaId}.{testField.Name}"); + } + } + } + + private void ValidateFieldSchema( + TableFieldSchema testField, IEnumerable actualFields, string fieldSchemaId) + { + TableFieldSchema actualField; + try + { + actualField = actualFields.Single(); + } + catch (Exception e) + { + throw new InvalidOperationException( + $"Field {fieldSchemaId} does not exist exactly once on actual table.", e); + } + ValidateSchema(testField.Fields, actualField.Fields, fieldSchemaId); + if (!testField.Type.Equals(actualField.Type)) + { + throw new InvalidOperationException( + $"Field {fieldSchemaId} had Type {actualField.Type} but should have {testField.Type}"); + } + } + } +} diff --git a/BenchmarkDotNetBigQuery/DatastoreExporter.cs b/BenchmarkDotNetBigQuery/DatastoreExporter.cs new file mode 100644 index 0000000..c4c1431 --- /dev/null +++ b/BenchmarkDotNetBigQuery/DatastoreExporter.cs @@ -0,0 +1,143 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Reports; +using Google.Cloud.Datastore.V1; +using MoreLinq; + +namespace BenchmarkDotNetBigQuery +{ + /// + /// A BenchmarkDotNet exporter that exports to Google Cloud Datastore. + /// + public class DatastoreExporter : IExporter + { + private Key _summaryKey; + private bool OneSummary { get; } + private string CommitId { get; } + private string SummaryEntityKind { get; } + private string ReportEntityKind { get; } + private DatastoreDb DatastoreDb { get; } + + /// + /// During construction, the BigQueryExporter will create the dataset and tables if needed. + /// If the dataset and tables already exist, it will validate that the tables contain the necessary fields. + /// + /// Id of the commit e.g. git hash. + /// The id of the google project to upload to. + /// + /// BenchmarkDotNet provides a separate summary for every class. If this parameter is + /// false, the exporter creates all summary entities. If true, creates a single summary entity for the entire + /// lifetime of the exporter. + /// + /// + /// The name of the Google Cloud Datastore Namespace to place the entities in. + /// + /// The kind of entity to put summary information in. + /// + /// The kind of entity to put report information in. + /// Report entities will always be children of a summary entity + /// + public DatastoreExporter( + string commitId, + string googleProjectId, + bool oneSummary = true, + string gcdNamespace = "", + string summaryEntityKind = "BenchmarkSummary", + string reportEntityKind = "BenchmarkReport") + { + CommitId = commitId; + OneSummary = oneSummary; + SummaryEntityKind = summaryEntityKind; + ReportEntityKind = reportEntityKind; + DatastoreDb = DatastoreDb.Create(googleProjectId, gcdNamespace); + } + + + /// + /// DatastoreExporter does not write to a logger. It will send an error message to the logger. + /// + public void ExportToLog(Summary summary, ILogger logger) + { + logger.WriteLine(LogKind.Error, $"{nameof(DatastoreExporter)} does not output to a logger."); + } + + /// + /// This is where DatastoreExporter writes benchmark data to Datastore entities. + /// + /// The summary to export to Google Cloud Datastore. + /// Unused + /// A string specifiying the key of the summary entity. + public IEnumerable ExportToFiles(Summary summary, ILogger consoleLogger) + { + KeyFactory summaryKeyFactory = DatastoreDb.CreateKeyFactory(SummaryEntityKind); + if (!OneSummary || _summaryKey == null) + { + Entity summaryEntity = BuildSummaryEntity(summary, summaryKeyFactory); + _summaryKey = DatastoreDb.Insert(summaryEntity); + yield return $"Datastore summary entity key: {_summaryKey}"; + } + KeyFactory reportKeyFactory = new KeyFactory(_summaryKey, ReportEntityKind); + var reportBatches = summary.Reports.Select(BuildReportEntityCurry(reportKeyFactory)).Batch(500); + foreach (IEnumerable reportBatch in reportBatches) + { + DatastoreDb.Insert(reportBatch); + } + } + + private Func BuildReportEntityCurry(KeyFactory reportKeyFactory) + { + return (report) => BuildReportEntity(report, reportKeyFactory); + } + + private Entity BuildReportEntity(BenchmarkReport report, KeyFactory reportKeyFactory) + { + var fullMethodName = $"{report.Benchmark.Target.Type.FullName}.{report.Benchmark.Target.Method.Name}"; + return new Entity + { + Key = reportKeyFactory.CreateIncompleteKey(), + ["Namespace"] = report.Benchmark.Target.Type.Namespace, + ["Type"] = report.Benchmark.Target.Type.Name, + ["FullType"] = report.Benchmark.Target.Type.FullName, + ["MethodName"] = report.Benchmark.Target.Method.Name, + ["FullMethodName"] = fullMethodName, + ["Parameters"] = report.Benchmark.Parameters.PrintInfo, + ["MethodSignature"] = report.Benchmark.Target.Method.ToString(), + ["Min"] = report.ResultStatistics.Min, + ["Max"] = report.ResultStatistics.Max, + ["Median"] = report.ResultStatistics.Median, + ["Mean"] = report.ResultStatistics.Mean, + ["StandardDeviation"] = report.ResultStatistics.StandardDeviation, + ["StandardError"] = report.ResultStatistics.StandardError, + ["Variance"] = report.ResultStatistics.Variance, + ["Percentile67"] = report.ResultStatistics.Percentiles.P67, + ["Percentile85"] = report.ResultStatistics.Percentiles.P85, + ["Percentile95"] = report.ResultStatistics.Percentiles.P95, + ["Percentile100"] = report.ResultStatistics.Percentiles.P100 + }; + } + + private Entity BuildSummaryEntity(Summary summary, KeyFactory summaryKeyFactory) + { + return new Entity + { + Key = summaryKeyFactory.CreateIncompleteKey(), + ["Commit"] = CommitId, + ["Timestamp"] = DateTimeOffset.UtcNow, + ["HostName"] = Environment.MachineName, + ["OsVersion"] = summary.HostEnvironmentInfo.OsVersion.Value, + ["ProcessorName"] = summary.HostEnvironmentInfo.ProcessorName.Value, + ["ProcessorCount"] = summary.HostEnvironmentInfo.ProcessorCount, + ["RuntimeVersion"] = summary.HostEnvironmentInfo.RuntimeVersion, + ["Architecture"] = summary.HostEnvironmentInfo.Architecture, + ["JitModules"] = summary.HostEnvironmentInfo.JitModules, + ["DotNetCoreVersion"] = summary.HostEnvironmentInfo.DotNetCliVersion.Value, + ["BenchmarkDotNetVersion"] = summary.HostEnvironmentInfo.BenchmarkDotNetVersion, + ["ChronometerFrequency"] = summary.HostEnvironmentInfo.ChronometerFrequency.Hertz, + ["HardwareTimerKind"] = summary.HostEnvironmentInfo.HardwareTimerKind.ToString() + }; + } + } +} diff --git a/BenchmarkDotNetBigQuery/Properties/AssemblyInfo.cs b/BenchmarkDotNetBigQuery/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..068dc84 --- /dev/null +++ b/BenchmarkDotNetBigQuery/Properties/AssemblyInfo.cs @@ -0,0 +1,20 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("BenchmarkDotNetBigQuery")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyKeyFile("key.snk")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("0a4f8f8f-e568-4b5c-b8c2-1b36bdb2bc94")] diff --git a/BenchmarkDotNetBigQuery/key.snk b/BenchmarkDotNetBigQuery/key.snk new file mode 100644 index 0000000..0f42099 Binary files /dev/null and b/BenchmarkDotNetBigQuery/key.snk differ diff --git a/BenchmarkDotNetBigQuery/project.json b/BenchmarkDotNetBigQuery/project.json new file mode 100644 index 0000000..ea1c08c --- /dev/null +++ b/BenchmarkDotNetBigQuery/project.json @@ -0,0 +1,22 @@ +{ + "title": "BenchmarkDotNetBigQuery", + "description": "BenchmarkDotNet exporter to Google BigQuery", + "version": "1.0.0-alpha-*", + "dependencies": { + "BenchmarkDotNet": "0.10.3", + "Google.Cloud.BigQuery.V2": "1.0.0-beta09", + "Google.Cloud.Datastore.V1": "1.0.0-beta08", + "morelinq": "2.2.0", + "NETStandard.Library": "1.6.1" + }, + "frameworks": { + "net45": { + }, + "netcoreapp1.1": { + } + }, + "buildOptions": { + "define": [ "DEBUG" ], + "xmlDoc": true + } +}