fix(threading): Fix in-flight counter and improve stuck task detection
1. ThreadPool counter fix:
- The outer catch block in WorkerThread::Run() was only updating
the per-worker counter, not the pool's totalCompleted
- This caused GetInFlightTasks() to return permanently inflated
values when exceptions occurred (explaining "1 in-flight, 0 active
workers" with all tasks done)
- Now properly updates pool counter in outer catch
2. Improved stuck task detection:
- LogStuckTasks now always outputs a summary showing how many tasks
are tracked and how many are stuck
- This helps diagnose whether tasks are being registered properly
Expected fix: The "1 in-flight, 0 active workers" issue should no longer
occur if the root cause was exception handling leaving the counter in
an inconsistent state.
Co-Authored-By: Claude Opus 4.5 <[email protected]>
Signed-off-by: luis <[email protected]>
This commit is contained in:
committed by
luis
co-authored by
Claude Opus 4.5
parent
9b830878a9
commit
cd2f95ff2a
@@ -270,7 +270,14 @@ void WorkerThread::Run()
|
||||
// NOTE: Cannot use TC_LOG here as it might not be initialized
|
||||
// Error will be recorded in metrics instead
|
||||
_metrics.tasksCompleted.fetch_add(1, ::std::memory_order_relaxed); // Count as completed but failed
|
||||
if (_diagnostics)
|
||||
|
||||
// CRITICAL FIX: Also update POOL counter to maintain in-flight balance!
|
||||
// If a task was popped but an exception occurred before RecordTaskCompletion,
|
||||
// the pool's totalCompleted would never be updated, causing GetInFlightTasks()
|
||||
// to return a permanently inflated value (leading to "1 in-flight, 0 active workers")
|
||||
_pool->_metrics.totalCompleted.fetch_add(1, ::std::memory_order_relaxed);
|
||||
|
||||
if (_diagnostics)
|
||||
{
|
||||
_diagnostics->tasksFailed.fetch_add(1, ::std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
@@ -64,16 +64,25 @@ namespace {
|
||||
::std::lock_guard lock(_executingTasksMutex);
|
||||
auto now = ::std::chrono::steady_clock::now();
|
||||
|
||||
size_t totalTracked = _executingTasks.size();
|
||||
size_t stuckCount = 0;
|
||||
|
||||
for (auto const& [guid, task] : _executingTasks)
|
||||
{
|
||||
auto elapsed = ::std::chrono::duration_cast<::std::chrono::milliseconds>(now - task.startTime).count();
|
||||
if (elapsed > thresholdMs)
|
||||
{
|
||||
++stuckCount;
|
||||
TC_LOG_ERROR("module.playerbot.session",
|
||||
"STUCK TASK DETECTED: Bot {} (GUID: {}) has been executing for {}ms!",
|
||||
task.botName, guid.ToString(), elapsed);
|
||||
}
|
||||
}
|
||||
|
||||
// Always log summary for diagnostics
|
||||
TC_LOG_ERROR("module.playerbot.session",
|
||||
"LogStuckTasks: {} tasks tracked, {} stuck (>{} ms)",
|
||||
totalTracked, stuckCount, thresholdMs);
|
||||
}
|
||||
} // anonymous namespace
|
||||
|
||||
|
||||
Reference in New Issue
Block a user