Add --show-all-tests option to GenerateTestSummary tool

.github/workflows/specialized-test-runner.yml

@@ -28,6 +28,12 @@ on:
         required: false
         type: boolean
         default: false
+      # When true (default), only failed tests are shown in the summary. When false, all tests are shown.
+      showOnlyFailedTestsInSummary:
+        required: false
+        type: boolean
+        default: true
+        description: 'When true, only failed tests are shown in the summary. When false, all tests are shown.'
 
 jobs:
 
@@ -180,6 +186,7 @@ jobs:
           --
           ${{ github.workspace }}/testresults
           --combined
+          ${{ inputs.showOnlyFailedTestsInSummary == false && '--show-all-tests' || '' }}
 
       - name: Fail if any dependency failed
         # 'skipped' can be when a transitive dependency fails and the dependent job gets 'skipped'.

.github/workflows/tests-outerloop.yml

@@ -39,3 +39,4 @@ jobs:
       extraRunSheetBuilderArgs: "-p:RunOuterloopTests=true"
       extraTestArgs: "--filter-trait outerloop=true"
       enablePlaywrightInstall: true
+      showOnlyFailedTestsInSummary: false

.github/workflows/tests-quarantine.yml

@@ -41,3 +41,4 @@ jobs:
       extraTestArgs: "--filter-trait quarantined=true"
       enablePlaywrightInstall: true
       ignoreTestFailures: true
+      showOnlyFailedTestsInSummary: false

tools/GenerateTestSummary/Program.cs

@@ -5,21 +5,23 @@
 using System.CommandLine;
 using Aspire.TestTools;
 
-// Usage: dotnet tools run GenerateTestSummary --dirPathOrTrxFilePath <path> [--output <output>] [--combined]
+// Usage: dotnet tools run GenerateTestSummary --dirPathOrTrxFilePath <path> [--output <output>] [--combined] [--show-all-tests]
 // Generate a summary report from trx files.
 // And write to $GITHUB_STEP_SUMMARY if running in GitHub Actions.
 
 var dirPathOrTrxFilePathArgument = new Argument<string>("dirPathOrTrxFilePath");
 var outputOption = new Option<string>("--output", "-o") { Description = "Output file path" };
 var combinedSummaryOption = new Option<bool>("--combined", "-c") { Description = "Generate combined summary report" };
 var urlOption = new Option<string>("--url", "-u") { Description = "URL for test links" };
+var showAllTestsOption = new Option<bool>("--show-all-tests", "-a") { Description = "Show all tests (passing, failing, skipped) in a single list ordered by name" };
 
 var rootCommand = new RootCommand
 {
     dirPathOrTrxFilePathArgument,
     outputOption,
     combinedSummaryOption,
-    urlOption
+    urlOption,
+    showAllTestsOption
 };
 
 rootCommand.SetAction(result =>
@@ -33,6 +35,7 @@
 
     var combinedSummary = result.GetValue<bool>(combinedSummaryOption);
     var url = result.GetValue<string>(urlOption);
+    var showAllTests = result.GetValue<bool>(showAllTestsOption);
 
     if (combinedSummary && !string.IsNullOrEmpty(url))
     {
@@ -43,7 +46,7 @@
     string report;
     if (combinedSummary)
     {
-        report = TestSummaryGenerator.CreateCombinedTestSummaryReport(dirPathOrTrxFilePath);
+        report = TestSummaryGenerator.CreateCombinedTestSummaryReport(dirPathOrTrxFilePath, showAllTests);
     }
     else
     {
@@ -59,13 +62,13 @@
             {
                 foreach (var trxFile in trxFiles)
                 {
-                    TestSummaryGenerator.CreateSingleTestSummaryReport(trxFile, reportBuilder, url);
+                    TestSummaryGenerator.CreateSingleTestSummaryReport(trxFile, reportBuilder, url, showAllTests);
                 }
             }
         }
         else
         {
-            TestSummaryGenerator.CreateSingleTestSummaryReport(dirPathOrTrxFilePath, reportBuilder, url);
+            TestSummaryGenerator.CreateSingleTestSummaryReport(dirPathOrTrxFilePath, reportBuilder, url, showAllTests);
         }
 
         report = reportBuilder.ToString();

tools/GenerateTestSummary/TestSummaryGenerator.cs

@@ -9,7 +9,7 @@ namespace Aspire.TestTools;
 
 sealed partial class TestSummaryGenerator
 {
-    public static string CreateCombinedTestSummaryReport(string basePath)
+    public static string CreateCombinedTestSummaryReport(string basePath, bool showAllTests = false)
     {
         var resolved = Path.GetFullPath(basePath);
         if (!Directory.Exists(resolved))
@@ -196,10 +196,73 @@ public static string CreateCombinedTestSummaryReport(string basePath)
         overallTableBuilder.AppendLine("## Duration Statistics");
         overallTableBuilder.Append(GenerateDurationStatistics(basePath));
 
+        // Add all individual test results if showAllTests is true
+        if (showAllTests)
+        {
+            overallTableBuilder.AppendLine();
+            overallTableBuilder.AppendLine("## All Test Results");
+            overallTableBuilder.AppendLine();
+            overallTableBuilder.Append(GenerateAllTestsSection(basePath));
+        }
+
         return overallTableBuilder.ToString();
     }
 
-    public static void CreateSingleTestSummaryReport(string trxFilePath, StringBuilder reportBuilder, string? url)
+    private static string GenerateAllTestsSection(string basePath)
+    {
+        var resultBuilder = new StringBuilder();
+        var allTests = new List<(string TestName, string Outcome, string? ErrorInfo, string? StdOut)>();
+
+        var trxFiles = Directory.EnumerateFiles(basePath, "*.trx", SearchOption.AllDirectories);
+        foreach (var filePath in trxFiles)
+        {
+            TestRun? testRun;
+            try
+            {
+                testRun = TrxReader.DeserializeTrxFile(filePath);
+                if (testRun?.Results?.UnitTestResults is null)
+                {
+                    continue;
+                }
+            }
+            catch
+            {
+                continue;
+            }
+
+            foreach (var test in testRun.Results.UnitTestResults)
+            {
+                if (!string.IsNullOrEmpty(test.TestName))
+                {
+                    allTests.Add((test.TestName, test.Outcome ?? "Unknown", test.Output?.ErrorInfo?.InnerText, test.Output?.StdOut));
+                }
+            }
+        }
+
+        // Sort by test name
+        allTests.Sort((a, b) => string.Compare(a.TestName, b.TestName, StringComparison.OrdinalIgnoreCase));
+
+        foreach (var test in allTests)
+        {
+            if (test.Outcome == "Failed")
+            {
+                AppendFailedTestDetails(resultBuilder, test.TestName, test.ErrorInfo, test.StdOut);
+            }
+            else if (test.Outcome == "Passed")
+            {
+                resultBuilder.AppendLine(CultureInfo.InvariantCulture, $"- ✅ {test.TestName}");
+            }
+            else
+            {
+                // Skipped/NotExecuted tests
+                resultBuilder.AppendLine(CultureInfo.InvariantCulture, $"- ⏭️ {test.TestName}");
+            }
+        }
+
+        return resultBuilder.ToString();
+    }
+
+    public static void CreateSingleTestSummaryReport(string trxFilePath, StringBuilder reportBuilder, string? url, bool showAllTests = false)
     {
         if (!File.Exists(trxFilePath))
         {
@@ -222,7 +285,9 @@ public static void CreateSingleTestSummaryReport(string trxFilePath, StringBuild
 
         var counters = testRun.ResultSummary.Counters;
         var failed = counters.Failed;
-        if (failed == 0)
+
+        // Skip if no failed tests and we're only showing failed tests
+        if (failed == 0 && !showAllTests)
         {
             Console.WriteLine($"No failed tests in {trxFilePath}");
             return;
@@ -248,45 +313,29 @@ public static void CreateSingleTestSummaryReport(string trxFilePath, StringBuild
             return;
         }
 
-        var failedTests = testRun.Results.UnitTestResults.Where(r => r.Outcome == "Failed");
-        if (failedTests.Any())
+        // Get tests to display - all tests if showAllTests, otherwise only failed
+        var testsToDisplay = showAllTests
+            ? testRun.Results.UnitTestResults.OrderBy(r => r.TestName, StringComparer.OrdinalIgnoreCase)
+            : testRun.Results.UnitTestResults.Where(r => r.Outcome == "Failed");
+
+        foreach (var test in testsToDisplay)
         {
-            foreach (var test in failedTests)
+            if (test.Outcome == "Failed")
             {
-                reportBuilder.AppendLine("<div>");
-                reportBuilder.AppendLine(CultureInfo.InvariantCulture, $"""
-                    <details><summary>🔴 <b>{test.TestName}</b></summary>
-
-                """);
-
-                reportBuilder.AppendLine();
-                reportBuilder.AppendLine("```yml");
-
-                reportBuilder.AppendLine(test.Output?.ErrorInfo?.InnerText);
-                if (test.Output?.StdOut is not null)
+                AppendFailedTestDetails(reportBuilder, test.TestName ?? "Unknown", test.Output?.ErrorInfo?.InnerText, test.Output?.StdOut);
+            }
+            else if (showAllTests)
+            {
+                // Only show passing/skipped tests when showAllTests is true
+                if (test.Outcome == "Passed")
                 {
-                    const int halfLength = 25_000;
-                    var stdOutSpan = test.Output.StdOut.AsSpan();
-
-                    reportBuilder.AppendLine();
-                    reportBuilder.AppendLine("### StdOut");
-
-                    var startSpan = stdOutSpan[..Math.Min(halfLength, stdOutSpan.Length)];
-                    reportBuilder.AppendLine(startSpan.ToString());
-
-                    if (stdOutSpan.Length > halfLength)
-                    {
-                        reportBuilder.AppendLine(CultureInfo.InvariantCulture, $"{Environment.NewLine}... (snip) ...{Environment.NewLine}");
-                        var endSpan = stdOutSpan[^halfLength..];
-                        // `endSpan` might not begin at the true beginning of the original line
-                        reportBuilder.Append("... ");
-                        reportBuilder.Append(endSpan);
-                    }
+                    reportBuilder.AppendLine(CultureInfo.InvariantCulture, $"- ✅ {test.TestName}");
+                }
+                else
+                {
+                    // Skipped/NotExecuted tests
+                    reportBuilder.AppendLine(CultureInfo.InvariantCulture, $"- ⏭️ {test.TestName}");
                 }
-
-                reportBuilder.AppendLine("```");
-                reportBuilder.AppendLine();
-                reportBuilder.AppendLine("</div>");
             }
         }
         reportBuilder.AppendLine();
@@ -517,6 +566,46 @@ private static string GenerateTopTestsPerRun(string basePath)
         return resultBuilder.ToString();
     }
 
+    private static void AppendFailedTestDetails(StringBuilder builder, string testName, string? errorInfo, string? stdOut)
+    {
+        builder.AppendLine("<div>");
+        builder.AppendLine(CultureInfo.InvariantCulture, $"""
+            <details><summary>❌ <b>{testName}</b></summary>
+
+        """);
+
+        builder.AppendLine();
+        builder.AppendLine("```yml");
+
+        builder.AppendLine(errorInfo);
+        if (stdOut is not null)
+        {
+            const int halfLength = 25_000;
+            var stdOutSpan = stdOut.AsSpan();
+
+            builder.AppendLine();
+            builder.AppendLine("### StdOut");
+
+            var startSpan = stdOutSpan[..Math.Min(halfLength, stdOutSpan.Length)];
+            builder.AppendLine(startSpan.ToString());
+
+            if (stdOutSpan.Length > halfLength)
+            {
+                builder.AppendLine(CultureInfo.InvariantCulture, $"{Environment.NewLine}... (snip) ...{Environment.NewLine}");
+                var endSpan = stdOutSpan[^halfLength..];
+                // `endSpan` might not begin at the true beginning of the original line
+                builder.Append("... ");
+                builder.Append(endSpan);
+            }
+        }
+
+        builder.AppendLine("```");
+        builder.AppendLine();
+        builder.AppendLine("</details>");
+        builder.AppendLine("</div>");
+        builder.AppendLine();
+    }
+
     public static string GetTestTitle(string trxFileName)
     {
         var filename = Path.GetFileNameWithoutExtension(trxFileName);