LANG-134: Finish original inbound monitoring row on pipeline error (#2729)

* fix(inbound-error-state): Finish original inbound monitoring row on pipeline error (LANG-134)

* fix(pipeline): finalize monitoring on early exits and cancellation

Co-authored-by: multica-agent <[email protected]>

---------

Co-authored-by: multica-agent <[email protected]>
This commit is contained in:
RockChinQ
2026-10-20 20:39:10 +08:00
committed by GitHub
co-authored by multica-agent
parent 819f65bd8a
commit f8e32ef14f
4 changed files with 206 additions and 1 deletions
@@ -6,6 +6,7 @@ without cluttering the main pipeline code.
from __future__ import annotations
import asyncio
import traceback
import typing
import time
@@ -118,6 +119,25 @@ async def prepare_monitoring_identity(
class MonitoringHelper:
"""Helper class for monitoring operations"""
@staticmethod
async def finish_query(ap, query, message_id: str, status: str) -> None:
"""Bound terminal writes and retry transient failures without blocking cleanup."""
if not message_id:
return
for attempt in range(3):
try:
async with asyncio.timeout(2):
await ap.monitoring_service.update_message_status(
get_query_execution_context(query),
message_id=message_id,
status=status,
level='error' if status == 'error' else 'info',
)
return
except Exception as exc:
ap.logger.warning(f'Terminal monitoring write failed (attempt {attempt + 1}): {exc}')
ap.logger.error(f'Unable to finish monitoring message {message_id}: {status}')
@staticmethod
async def record_query_start(
ap: app.Application,
@@ -283,6 +303,9 @@ class MonitoringHelper:
) -> str:
"""Record query processing error, returns message_id"""
try:
original_message_id = (query.variables or {}).get('_monitoring_message_id')
if original_message_id:
await MonitoringHelper.finish_query(ap, query, original_message_id, 'error')
session_id = f'{query.launcher_type.value if hasattr(query.launcher_type, "value") else query.launcher_type}_{query.launcher_id}'
# Get sender name from message event
+24 -1
View File
@@ -497,6 +497,28 @@ class RuntimePipeline:
processing_mode.reset(token)
async def _process_query(self, query: pipeline_query.Query):
"""Close monitoring on every exit, including cancellation and plugin veto."""
from .monitoring_helper import MonitoringHelper
status = 'error'
try:
await self._process_query_body(query)
if not query.variables.get('_monitoring_has_error'):
status = 'success'
finally:
try:
# Never bypass the placement fence to repair a stale execution.
async with asyncio.timeout(2):
await self._assert_execution_active(query)
await MonitoringHelper.finish_query(
self.ap, query, query.variables.get('_monitoring_message_id', ''), status
)
except Exception as exc:
self.ap.logger.error(f'Failed to finalize query {query.query_id}: {exc}')
finally:
await self.ap.query_pool.remove_query(query)
async def _process_query_body(self, query: pipeline_query.Query):
"""处理请求"""
await self._assert_execution_active(query)
# Get monitoring metadata
@@ -602,8 +624,10 @@ class RuntimePipeline:
self.ap.logger.error(f'Failed to record query response: {e}')
except WorkspaceError as e:
query.variables['_monitoring_has_error'] = True
self.ap.logger.info(f'Dropped query {query.query_id} because its Workspace execution binding is stale: {e}')
except Exception as e:
query.variables['_monitoring_has_error'] = True
inst_name = query.current_stage_name if query.current_stage_name else 'unknown'
self.ap.logger.error(f'Error processing query {query.query_id} stage={inst_name} : {e}')
self.ap.logger.error(f'Traceback: {traceback.format_exc()}')
@@ -631,7 +655,6 @@ class RuntimePipeline:
finally:
self.ap.logger.debug(f'Query {query.query_id} processed')
await self.ap.query_pool.remove_query(query)
def _record_lane_failure(self, query: pipeline_query.Query, reason: str) -> None:
"""Land one failed ``runner/execute`` stage on the owning execution trace."""