Skip to content

test: group related assertions with Assert.Multiple #25

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
samtrion merged 1 commit into from
Jul 31, 2026
Merged

test: group related assertions with Assert.Multiple #25

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
163 changes: 98 additions & 65 deletions tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/CultureMutationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,10 +355,13 @@ public async Task Analyze_UntestedOrdinalComparison_ReportsAGapPerRemainingCompa

var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false);

_ = await Assert.That(Errors(compilation)).IsEmpty();
_ = await Assert.That(Gaps(diagnostics)).IsEqualTo(ExpectAt(ComparisonLine, _ordinalMutants));
_ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics);
_ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty();
using (Assert.Multiple())
{
_ = await Assert.That(Errors(compilation)).IsEmpty();
_ = await Assert.That(Gaps(diagnostics)).IsEqualTo(ExpectAt(ComparisonLine, _ordinalMutants));
_ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics);
_ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty();
}
}

/// <summary>
Expand All @@ -373,8 +376,13 @@ public async Task Analyze_CoveredOrdinalComparison_ReportsNothing()

var diagnostics = await RunAsync(compilation, [CreateManifest(ComparisonMemberId)]).ConfigureAwait(false);

_ = await Assert.That(Errors(compilation)).IsEmpty();
_ = await Assert.That(DiagnosticAssertions.Describe(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics);
using (Assert.Multiple())
{
_ = await Assert.That(Errors(compilation)).IsEmpty();
_ = await Assert
.That(DiagnosticAssertions.Describe(diagnostics))
.IsEqualTo(DiagnosticAssertions.NoDiagnostics);
}
}

/// <summary>
Expand All @@ -388,16 +396,19 @@ public async Task Analyze_UntestedInvariantCultureFormatting_ReportsTheCultureSw

var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false);

_ = await Assert.That(Errors(compilation)).IsEmpty();
_ = await Assert
.That(Gaps(diagnostics))
.IsEqualTo(
Expect(
(FormatLine, "InvariantCulture => CurrentCulture"),
(FormatLine, "CultureInfo.InvariantCulture => (removed)")
)
);
_ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics);
using (Assert.Multiple())
{
_ = await Assert.That(Errors(compilation)).IsEmpty();
_ = await Assert
.That(Gaps(diagnostics))
.IsEqualTo(
Expect(
(FormatLine, "InvariantCulture => CurrentCulture"),
(FormatLine, "CultureInfo.InvariantCulture => (removed)")
)
);
_ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics);
}
}

/// <summary>
Expand All @@ -414,17 +425,20 @@ public async Task Analyze_ProviderRemovalWithoutAMatchingOverload_IsDroppedAsANo

var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false);

_ = await Assert.That(Errors(compilation)).IsEmpty();
_ = await Assert
.That(Gaps(diagnostics))
.IsEqualTo(
Expect(
(RemovableProviderLine, "CultureInfo.InvariantCulture => (removed)"),
(RemovableProviderLine, "InvariantCulture => CurrentCulture"),
(RequiredProviderLine, "InvariantCulture => CurrentCulture")
)
);
_ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics);
using (Assert.Multiple())
{
_ = await Assert.That(Errors(compilation)).IsEmpty();
_ = await Assert
.That(Gaps(diagnostics))
.IsEqualTo(
Expect(
(RemovableProviderLine, "CultureInfo.InvariantCulture => (removed)"),
(RemovableProviderLine, "InvariantCulture => CurrentCulture"),
(RequiredProviderLine, "InvariantCulture => CurrentCulture")
)
);
_ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics);
}
}

/// <summary>
Expand Down Expand Up @@ -468,17 +482,20 @@ public async Task Analyze_UntestedInvariantCaseConversion_ReportsOnlyTheConversi
.Summarise(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.UnreachableMutationPoint))
.Select(summary => summary.Line);

_ = await Assert.That(Errors(compilation)).IsEmpty();
_ = await Assert
.That(Gaps(diagnostics))
.IsEqualTo(
Expect(
(CaseConversionLine, "ToUpperInvariant => ToUpper"),
(CaseConversionLine, "ToUpperInvariant => ToLowerInvariant")
)
);
_ = await Assert.That(lines.Distinct()).IsEquivalentTo(new[] { CaseConversionLine });
_ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics);
using (Assert.Multiple())
{
_ = await Assert.That(Errors(compilation)).IsEmpty();
_ = await Assert
.That(Gaps(diagnostics))
.IsEqualTo(
Expect(
(CaseConversionLine, "ToUpperInvariant => ToUpper"),
(CaseConversionLine, "ToUpperInvariant => ToLowerInvariant")
)
);
_ = await Assert.That(lines.Distinct()).IsEquivalentTo(new[] { CaseConversionLine });
_ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics);
}
}

/// <summary>
Expand All @@ -492,24 +509,27 @@ public async Task Analyze_UntestedRegexOptions_ReportsAGapPerOfferedFlag()

var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false);

_ = await Assert.That(Errors(compilation)).IsEmpty();
_ = await Assert
.That(Gaps(diagnostics))
.IsEqualTo(
ExpectAt(
RegexOptionsLine,
[
"RegexOptions - IgnoreCase",
"RegexOptions + CultureInvariant",
"RegexOptions + Multiline",
"RegexOptions + Singleline",
"RegexOptions + ExplicitCapture",
"RegexOptions + IgnorePatternWhitespace",
"RegexOptions + RightToLeft",
]
)
);
_ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics);
using (Assert.Multiple())
{
_ = await Assert.That(Errors(compilation)).IsEmpty();
_ = await Assert
.That(Gaps(diagnostics))
.IsEqualTo(
ExpectAt(
RegexOptionsLine,
[
"RegexOptions - IgnoreCase",
"RegexOptions + CultureInvariant",
"RegexOptions + Multiline",
"RegexOptions + Singleline",
"RegexOptions + ExplicitCapture",
"RegexOptions + IgnorePatternWhitespace",
"RegexOptions + RightToLeft",
]
)
);
_ = await Assert.That(Trivial(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics);
}
}

/// <summary>
Expand All @@ -524,8 +544,13 @@ public async Task Analyze_CultureValueInAConstantContext_ReportsNothing()

var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false);

_ = await Assert.That(Errors(compilation)).IsEmpty();
_ = await Assert.That(DiagnosticAssertions.Describe(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics);
using (Assert.Multiple())
{
_ = await Assert.That(Errors(compilation)).IsEmpty();
_ = await Assert
.That(DiagnosticAssertions.Describe(diagnostics))
.IsEqualTo(DiagnosticAssertions.NoDiagnostics);
}
}

/// <summary>
Expand All @@ -551,10 +576,15 @@ public async Task Analyze_MemberDenseWithCultureMutations_IsBoundedByTheMutantBu
var four = await RunAsync(compilation, manifest, CreateBudget(4)).ConfigureAwait(false);
var one = await RunAsync(compilation, manifest, CreateBudget(1)).ConfigureAwait(false);

_ = await Assert.That(Errors(compilation)).IsEmpty();
_ = await Assert.That(Gaps(unlimited)).IsEqualTo(ExpectAt(DenseLine, [.. caseConversions, .. _ordinalMutants]));
_ = await Assert.That(Gaps(four)).IsEqualTo(ExpectAt(DenseLine, caseConversions));
_ = await Assert.That(Gaps(one)).IsEqualTo(ExpectAt(DenseLine, [caseConversions[0]]));
using (Assert.Multiple())
{
_ = await Assert.That(Errors(compilation)).IsEmpty();
_ = await Assert
.That(Gaps(unlimited))
.IsEqualTo(ExpectAt(DenseLine, [.. caseConversions, .. _ordinalMutants]));
_ = await Assert.That(Gaps(four)).IsEqualTo(ExpectAt(DenseLine, caseConversions));
_ = await Assert.That(Gaps(one)).IsEqualTo(ExpectAt(DenseLine, [caseConversions[0]]));
}
}

/// <summary>
Expand All @@ -572,9 +602,12 @@ public async Task Analyze_EveryFixture_CompilesAndReportsNoAnalyzerFailure(strin

var diagnostics = await RunAsync(compilation, [CreateManifest(AnchorMemberId)]).ConfigureAwait(false);

_ = await Assert.That(string.Join("; ", Errors(compilation))).IsEqualTo(string.Empty);
_ = await Assert.That(AnalyzerRunner.OfId(diagnostics, AnalyzerRunner.AnalyzerFailureId)).IsEmpty();
_ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty();
using (Assert.Multiple())
{
_ = await Assert.That(string.Join("; ", Errors(compilation))).IsEqualTo(string.Empty);
_ = await Assert.That(AnalyzerRunner.OfId(diagnostics, AnalyzerRunner.AnalyzerFailureId)).IsEmpty();
_ = await Assert.That(AnalyzerRunner.OfId(diagnostics, DiagnosticIds.InvalidTestSurfaceManifest)).IsEmpty();
}
}

private static Task<ImmutableArray<Diagnostic>> RunAsync(
Expand Down
54 changes: 35 additions & 19 deletions tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/MSTestSurfaceAnalyzerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,12 +108,15 @@ public async Task Fixtures_BothAssemblies_CompileWithoutErrors()
var production = CreateProduction();
var test = CreateTest(production);

_ = await Assert
.That(DiagnosticAssertions.Describe(CompilationFactory.GetCompileErrors(production)))
.IsEqualTo(DiagnosticAssertions.NoDiagnostics);
_ = await Assert
.That(DiagnosticAssertions.Describe(CompilationFactory.GetCompileErrors(test)))
.IsEqualTo(DiagnosticAssertions.NoDiagnostics);
using (Assert.Multiple())
{
_ = await Assert
.That(DiagnosticAssertions.Describe(CompilationFactory.GetCompileErrors(production)))
.IsEqualTo(DiagnosticAssertions.NoDiagnostics);
_ = await Assert
.That(DiagnosticAssertions.Describe(CompilationFactory.GetCompileErrors(test)))
.IsEqualTo(DiagnosticAssertions.NoDiagnostics);
}
}

/// <summary>
Expand Down Expand Up @@ -144,10 +147,15 @@ public async Task Analyzer_FrameworkWithoutAnyTest_ReportsNothing()

var diagnostics = await RunAllAsync(compilation, MalformedManifest).ConfigureAwait(false);

_ = await Assert.That(DiagnosticAssertions.Describe(diagnostics)).IsEqualTo(DiagnosticAssertions.NoDiagnostics);
_ = await Assert
.That(DiagnosticAssertions.Describe(CompilationFactory.GetCompileErrors(compilation)))
.IsEqualTo(DiagnosticAssertions.NoDiagnostics);
using (Assert.Multiple())
{
_ = await Assert
.That(DiagnosticAssertions.Describe(diagnostics))
.IsEqualTo(DiagnosticAssertions.NoDiagnostics);
_ = await Assert
.That(DiagnosticAssertions.Describe(CompilationFactory.GetCompileErrors(compilation)))
.IsEqualTo(DiagnosticAssertions.NoDiagnostics);
}
}

[Test]
Expand All @@ -171,11 +179,14 @@ public async Task Analyzer_TestWithoutProductionReference_IsReportedOnceAtItsIde

var diagnostics = await RunAsync(test, DiagnosticIds.TestWithoutProductionReference).ConfigureAwait(false);

_ = await Assert.That(diagnostics.Length).IsEqualTo(1);
_ = await Assert.That(diagnostics[0].Location.SourceSpan).IsEqualTo(identifier.Span);
_ = await Assert
.That(GetMessage(diagnostics[0]).Contains(LocalOnlyTestName, StringComparison.Ordinal))
.IsTrue();
using (Assert.Multiple())
{
_ = await Assert.That(diagnostics.Length).IsEqualTo(1);
_ = await Assert.That(diagnostics[0].Location.SourceSpan).IsEqualTo(identifier.Span);
_ = await Assert
.That(GetMessage(diagnostics[0]).Contains(LocalOnlyTestName, StringComparison.Ordinal))
.IsTrue();
}
Comment on lines +182 to +189

@coderabbitai coderabbitai Bot Jul 31, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟠 Major | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Locate the referenced test files and inspect asserted relevant snippets.
for f in \
  tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/MSTestSurfaceAnalyzerTests.cs \
  tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/MixedFrameworkTestSurfaceTests.cs \
  tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/NUnitTestSurfaceAnalyzerTests.cs \
  tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/TUnitTestSurfaceAnalyzerTests.cs \
  tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/XunitV2TestSurfaceAnalyzerTests.cs
do
  echo "== $f =="
  if [ -f "$f" ]; then
    rg -n --only-matching 'Assert\.Multiple\(\)|diagnostics\[[^]]+\]\.Location|otherDiagnostics\[[^]]+\]\.Location|\.Count\(\)\.IsEqualTo\(1\)|Contains\([^)]+\|StringComparison\.Ordinal\)' "$f"
    echo "--- relevant sections ---"
    rg -n -C 4 'Assert\.Multiple\(\)' "$f" | sed -n '1,220p'
  else
    echo "MISSING"
  fi
done

echo "== TUnit Assert.Multiple docs references in repository/config?"
rg -n 'Assert\.Multiple|Multiple\(\)|tunit\.dev|Combining assertions' . -g '*.cs' -g '*.md' -g '*.txt' -g '*.yml' -g '*.yaml' | sed -n '1,180p'

Repository: dailydevops/frameshift

Length of output: 35883

🌐 Web query:

TUnit Assert.Multiple continue after assertion failed documentation

💡 Result:

In TUnit, Assert.Multiple is specifically designed to allow multiple assertions to execute within a test without stopping at the first failure [1]. When assertions are wrapped in a using (Assert.Multiple) block, any failures encountered are accumulated and reported together at the end of the block, rather than throwing an exception immediately upon the first failure [2][1]. Key behaviors and documentation points: - Purpose: It allows you to group related assertions so that all failures are captured and reported, which is useful for diagnosing multiple issues in a single test run [1]. - Implementation: You use it with a using statement. All assertions inside this block that are awaited will be tracked by the scope [1]. - Assertion Execution: It is critical that all assertions within the scope are awaited (using await Assert.That(...)) [3][1]. Unawaited assertions will not execute, and the test may pass incorrectly [1]. - Behavior with Assert.Fail: Historically, there have been nuances regarding whether all methods behave consistently within these blocks. While standard Assert.That calls accumulate correctly [2][1], you should ensure that the assertion methods you use are designed to participate in the TUnit assertion pipeline [2]. If you find that your test is still stopping prematurely, ensure that you are using standard TUnit assertions and that every assertion within the using block is properly awaited [1].

Citations:

Keep dependent diagnostic assertions behind a count gate.

Assert.Multiple() aggregates failures instead of stopping on the first failed assertion. If the diagnostics count assertion fails, the following diagnostics[0] assertions still execute and can throw with a less useful exception. Skip index-based assertions when the count is not the expected single value.

Proposed pattern 
 using (Assert.Multiple())
 {
     _ = await Assert.That(diagnostics).Count().IsEqualTo(1);
-    _ = await Assert.That(diagnostics[0].Location.SourceSpan).IsEqualTo(identifier.Span);
-    _ = await Assert.That(GetMessage(diagnostics[0]).Contains(name, StringComparison.Ordinal)).IsTrue();
+    if (diagnostics.Length == 1)
+    {
+        _ = await Assert.That(diagnostics[0].Location.SourceSpan).IsEqualTo(identifier.Span);
+        _ = await Assert.That(GetMessage(diagnostics[0]).Contains(name, StringComparison.Ordinal)).IsTrue();
+    }
 }
📍 Affects 5 files
  • tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/MSTestSurfaceAnalyzerTests.cs#L182-L189 (this comment)
  • tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/MixedFrameworkTestSurfaceTests.cs#L580-L584
  • tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/MixedFrameworkTestSurfaceTests.cs#L715-L719
  • tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/NUnitTestSurfaceAnalyzerTests.cs#L240-L247
  • tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/TUnitTestSurfaceAnalyzerTests.cs#L368-L375
  • tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/TUnitTestSurfaceAnalyzerTests.cs#L427-L437
  • tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/TUnitTestSurfaceAnalyzerTests.cs#L803-L807
  • tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/XunitV2TestSurfaceAnalyzerTests.cs#L225-L232
  • tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/XunitV2TestSurfaceAnalyzerTests.cs#L371-L380
🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In
`@tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/MSTestSurfaceAnalyzerTests.cs`
around lines 182 - 189, Keep the dependent diagnostics[0] assertions behind a
diagnostics.Length == 1 gate so Assert.Multiple() cannot evaluate them after the
count assertion fails. Apply this pattern at
tests/NetEvolve.FrameShift.Tests.Integration/Analyzers/MSTestSurfaceAnalyzerTests.cs:182-189,
MixedFrameworkTestSurfaceTests.cs:580-584 and :715-719,
NUnitTestSurfaceAnalyzerTests.cs:240-247,
TUnitTestSurfaceAnalyzerTests.cs:368-375, :427-437, and :803-807, and
XunitV2TestSurfaceAnalyzerTests.cs:225-232 and :371-380; retain the count
assertion and only run index-based location/message checks when exactly one
diagnostic exists.

}

[Test]
Expand Down Expand Up @@ -287,10 +298,15 @@ public async Task Analyzer_EveryManifestShape_NeverCrashes()
reported.AddRange(DiagnosticAssertions.Ids(await RunAllAsync(test, shape).ConfigureAwait(false)));
}

_ = await Assert.That(reported.Contains(AnalyzerRunner.AnalyzerFailureId, StringComparer.Ordinal)).IsFalse();
_ = await Assert
.That(reported.Contains(DiagnosticIds.TestWithoutProductionReference, StringComparer.Ordinal))
.IsTrue();
using (Assert.Multiple())
{
_ = await Assert
.That(reported.Contains(AnalyzerRunner.AnalyzerFailureId, StringComparer.Ordinal))
.IsFalse();
_ = await Assert
.That(reported.Contains(DiagnosticIds.TestWithoutProductionReference, StringComparer.Ordinal))
.IsTrue();
}
}

[Test]
Expand Down
Loading