Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 62 additions & 2 deletions src/Build/BackEnd/BuildManager/BuildManager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1038,9 +1038,22 @@ public void EndBuild()
}
}

_noActiveSubmissionsEvent!.WaitOne();
{
Stopwatch hangWatch = Stopwatch.StartNew();
while (!_noActiveSubmissionsEvent!.WaitOne(CrashTelemetryRecorder.EndBuildHangDiagnosticsIntervalMs))
{
EmitEndBuildHangDiagnostics("WaitingForSubmissions", hangWatch);
}
}

ShutdownConnectedNodes(false /* normal termination */);
_noNodesActiveEvent!.WaitOne();
{
Stopwatch hangWatch = Stopwatch.StartNew();
while (!_noNodesActiveEvent!.WaitOne(CrashTelemetryRecorder.EndBuildHangDiagnosticsIntervalMs))
{
EmitEndBuildHangDiagnostics("WaitingForNodes", hangWatch);
}
}

// Wait for all of the actions in the work queue to drain.
// _workQueue.Completion.Wait() could throw here if there was an unhandled exception in the work queue,
Expand Down Expand Up @@ -1230,6 +1243,53 @@ private void RecordCrashTelemetry(Exception exception, bool isUnhandled)
host);
}

/// <summary>
/// Extracts build state under lock and delegates to <see cref="CrashTelemetryRecorder"/>
/// for EndBuild hang diagnostic telemetry emission. Also writes diagnostics to disk
/// via <see cref="ExceptionHandling.DumpHangDiagnosticsToFile"/>.
/// </summary>
private void EmitEndBuildHangDiagnostics(string waitPhase, Stopwatch hangWatch)
{
int pendingSubmissionCount;
int submissionsWithResultNoLogging = 0;
bool threadExceptionRecorded;
int unmatchedProjectStartedCount;
string? host;

lock (_syncLock)
{
foreach (BuildSubmissionBase submission in _buildSubmissions.Values)
{
if (submission.BuildResultBase is not null && !submission.LoggingCompleted)
{
submissionsWithResultNoLogging++;
}
}

pendingSubmissionCount = _buildSubmissions.Count;
threadExceptionRecorded = _threadException is not null;
unmatchedProjectStartedCount = _projectStartedEvents.Count;
host = _buildTelemetry?.BuildEngineHost ?? BuildEnvironmentState.GetHostName();
}

string diagnostics = $"Phase={waitPhase}, Duration={hangWatch.ElapsedMilliseconds}ms, " +
$"PendingSubmissions={pendingSubmissionCount}, WithResultNoLogging={submissionsWithResultNoLogging}, " +
$"ThreadException={threadExceptionRecorded}, UnmatchedProjectStarted={unmatchedProjectStartedCount}";

ExceptionHandling.DumpHangDiagnosticsToFile(diagnostics);

CrashTelemetryRecorder.CollectAndEmitEndBuildHangDiagnostics(
waitPhase,
hangWatch.ElapsedMilliseconds,
pendingSubmissionCount,
submissionsWithResultNoLogging,
threadExceptionRecorded,
unmatchedProjectStartedCount,
ProjectCollection.Version?.ToString(),
NativeMethodsShared.FrameworkName,
host);
}


/// <summary>
/// Convenience method. Submits a lone build request and blocks until results are available.
Expand Down
Loading
Loading