From 2b6dcfe9c7c42df7283ff7d14cd2edb893a3b618 Mon Sep 17 00:00:00 2001 From: RockChinQ Date: Fri, 12 Jun 2026 09:40:07 -0400 Subject: [PATCH] feat(survey): add bot_response_success_100 milestone trigger event Counts successful non-WebSocket bot responses (persisted in the metadata table as survey_bot_response_count, survives restarts) and fires the bot_response_success_100 survey event once the instance reaches 100 responses. Counting stops after the milestone has been triggered. Existing first_bot_response_success behavior unchanged. 6 new unit tests. --- .../pkg/pipeline/process/handlers/chat.py | 4 +- src/langbot/pkg/survey/manager.py | 55 +++++++ .../unit_tests/survey/test_survey_manager.py | 135 ++++++++++++++++-- 3 files changed, 178 insertions(+), 16 deletions(-) diff --git a/src/langbot/pkg/pipeline/process/handlers/chat.py b/src/langbot/pkg/pipeline/process/handlers/chat.py index 2e2e7f28..488e1921 100644 --- a/src/langbot/pkg/pipeline/process/handlers/chat.py +++ b/src/langbot/pkg/pipeline/process/handlers/chat.py @@ -226,10 +226,12 @@ class ChatMessageHandler(handler.MessageHandler): # Send telemetry asynchronously and do not block pipeline via app's telemetry manager await self.ap.telemetry.start_send_task(payload) - # Trigger survey event on first successful non-WebSocket response + # Trigger survey events on successful non-WebSocket responses if not locals().get('error_info') and adapter_name and 'WebSocket' not in adapter_name: if self.ap.survey: await self.ap.survey.trigger_event('first_bot_response_success') + # Counts toward the bot_response_success_100 milestone event + await self.ap.survey.record_bot_response_success() except Exception as ex: # Ensure telemetry issues do not affect normal flow self.ap.logger.warning(f'Failed to send telemetry: {ex}') diff --git a/src/langbot/pkg/survey/manager.py b/src/langbot/pkg/survey/manager.py index 0e554050..bf28404b 100644 --- a/src/langbot/pkg/survey/manager.py +++ b/src/langbot/pkg/survey/manager.py @@ -13,6 +13,11 @@ from ..entity.persistence.metadata import Metadata from ..utils import constants SURVEY_TRIGGERED_KEY = 'survey_triggered_events' +BOT_RESPONSE_COUNT_KEY = 'survey_bot_response_count' + +# Milestone event fired when an instance accumulates this many successful bot responses +BOT_RESPONSE_MILESTONE = 100 +BOT_RESPONSE_MILESTONE_EVENT = f'bot_response_success_{BOT_RESPONSE_MILESTONE}' class SurveyManager: @@ -23,11 +28,13 @@ class SurveyManager: self._triggered_events: set[str] = set() self._pending_survey: typing.Optional[dict] = None self._space_url: str = '' + self._bot_response_count: int = 0 async def initialize(self): space_config = self.ap.instance_config.data.get('space', {}) self._space_url = space_config.get('url', '').rstrip('/') await self._load_triggered_events() + await self._load_bot_response_count() async def _load_triggered_events(self): """Load previously triggered events from metadata table.""" @@ -65,6 +72,54 @@ class SurveyManager: return False return bool(self._space_url) + async def _load_bot_response_count(self): + """Load the persisted successful bot response count from metadata table.""" + try: + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(Metadata).where(Metadata.key == BOT_RESPONSE_COUNT_KEY) + ) + row = result.first() + if row: + self._bot_response_count = int(row[0].value) + except Exception: + self._bot_response_count = 0 + + async def _save_bot_response_count(self): + """Persist the successful bot response count to metadata table.""" + try: + value = str(self._bot_response_count) + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(Metadata).where(Metadata.key == BOT_RESPONSE_COUNT_KEY) + ) + if result.first(): + await self.ap.persistence_mgr.execute_async( + sqlalchemy.update(Metadata).where(Metadata.key == BOT_RESPONSE_COUNT_KEY).values(value=value) + ) + else: + await self.ap.persistence_mgr.execute_async( + sqlalchemy.insert(Metadata).values(key=BOT_RESPONSE_COUNT_KEY, value=value) + ) + except Exception as e: + self.ap.logger.debug(f'Failed to save survey bot response count: {e}') + + async def record_bot_response_success(self): + """Count a successful bot response; fires the milestone event at the threshold. + + Called by the chat handler after each successful (non-WebSocket) response. + The count is persisted so it survives restarts. Once the milestone event + has been triggered, counting stops (no further writes needed). + """ + if BOT_RESPONSE_MILESTONE_EVENT in self._triggered_events: + return + if not self._is_space_configured(): + return + + self._bot_response_count += 1 + await self._save_bot_response_count() + + if self._bot_response_count >= BOT_RESPONSE_MILESTONE: + await self.trigger_event(BOT_RESPONSE_MILESTONE_EVENT) + async def trigger_event(self, event: str): """Called when an event occurs. Checks Space for a pending survey.""" if event in self._triggered_events: diff --git a/tests/unit_tests/survey/test_survey_manager.py b/tests/unit_tests/survey/test_survey_manager.py index ae6017e1..5bf219ad 100644 --- a/tests/unit_tests/survey/test_survey_manager.py +++ b/tests/unit_tests/survey/test_survey_manager.py @@ -7,6 +7,7 @@ Tests cover: - Survey response submission - Survey dismissal """ + from __future__ import annotations import pytest @@ -127,9 +128,7 @@ class TestLoadTriggeredEvents: """Test that empty set is used when no events stored.""" survey_module = get_survey_module() mock_app = create_mock_app() - mock_app.persistence_mgr.execute_async = AsyncMock( - return_value=Mock(first=Mock(return_value=None)) - ) + mock_app.persistence_mgr.execute_async = AsyncMock(return_value=Mock(first=Mock(return_value=None))) manager = survey_module.SurveyManager(mock_app) await manager._load_triggered_events() @@ -219,9 +218,7 @@ class TestTriggerEvent: """Test that new event is added and saved.""" survey_module = get_survey_module() mock_app = create_mock_app() - mock_app.persistence_mgr.execute_async = AsyncMock( - return_value=Mock(first=Mock(return_value=None)) - ) + mock_app.persistence_mgr.execute_async = AsyncMock(return_value=Mock(first=Mock(return_value=None))) manager = survey_module.SurveyManager(mock_app) manager._space_url = 'https://space.example.com' @@ -231,6 +228,104 @@ class TestTriggerEvent: assert 'new_event' in manager._triggered_events +class TestRecordBotResponseSuccess: + """Tests for the bot_response_success_100 milestone counter.""" + + def _make_manager(self, survey_module, mock_app): + manager = survey_module.SurveyManager(mock_app) + manager._space_url = 'https://space.example.com' + # No existing metadata rows: select returns no row + mock_app.persistence_mgr.execute_async = AsyncMock(return_value=Mock(first=Mock(return_value=None))) + return manager + + @pytest.mark.asyncio + async def test_increments_and_persists_count(self): + survey_module = get_survey_module() + mock_app = create_mock_app() + manager = self._make_manager(survey_module, mock_app) + + await manager.record_bot_response_success() + + assert manager._bot_response_count == 1 + # select + insert for the count key + assert mock_app.persistence_mgr.execute_async.call_count >= 2 + + @pytest.mark.asyncio + async def test_fires_milestone_event_at_threshold(self): + survey_module = get_survey_module() + mock_app = create_mock_app() + manager = self._make_manager(survey_module, mock_app) + manager._bot_response_count = survey_module.BOT_RESPONSE_MILESTONE - 1 + + await manager.record_bot_response_success() + + assert manager._bot_response_count == survey_module.BOT_RESPONSE_MILESTONE + assert survey_module.BOT_RESPONSE_MILESTONE_EVENT in manager._triggered_events + + @pytest.mark.asyncio + async def test_does_not_fire_below_threshold(self): + survey_module = get_survey_module() + mock_app = create_mock_app() + manager = self._make_manager(survey_module, mock_app) + manager._bot_response_count = 5 + + await manager.record_bot_response_success() + + assert survey_module.BOT_RESPONSE_MILESTONE_EVENT not in manager._triggered_events + + @pytest.mark.asyncio + async def test_stops_counting_after_milestone_triggered(self): + survey_module = get_survey_module() + mock_app = create_mock_app() + manager = self._make_manager(survey_module, mock_app) + manager._triggered_events.add(survey_module.BOT_RESPONSE_MILESTONE_EVENT) + manager._bot_response_count = survey_module.BOT_RESPONSE_MILESTONE + + await manager.record_bot_response_success() + + # No persistence write, count unchanged + mock_app.persistence_mgr.execute_async.assert_not_called() + assert manager._bot_response_count == survey_module.BOT_RESPONSE_MILESTONE + + @pytest.mark.asyncio + async def test_skips_when_space_not_configured(self): + survey_module = get_survey_module() + mock_app = create_mock_app() + manager = self._make_manager(survey_module, mock_app) + manager._space_url = '' + + await manager.record_bot_response_success() + + assert manager._bot_response_count == 0 + mock_app.persistence_mgr.execute_async.assert_not_called() + + @pytest.mark.asyncio + async def test_count_loaded_on_initialize(self): + survey_module = get_survey_module() + mock_app = create_mock_app() + + count_row = Mock() + count_row.value = '42' + + def execute_side_effect(stmt): + result = Mock() + # Both _load_triggered_events and _load_bot_response_count select + # from Metadata; return the count row only for the count key. + stmt_str = str(stmt.compile(compile_kwargs={'literal_binds': True})) + if survey_module.BOT_RESPONSE_COUNT_KEY in stmt_str: + result.first.return_value = (count_row,) + else: + result.first.return_value = None + return result + + mock_app.persistence_mgr.execute_async = AsyncMock(side_effect=execute_side_effect) + + manager = survey_module.SurveyManager(mock_app) + await manager.initialize() + + assert manager._bot_response_count == 42 + + class TestPendingSurvey: """Tests for get_pending_survey and clear_pending_survey.""" @@ -296,14 +391,19 @@ class TestSubmitResponse: # Mock successful HTTP response import httpx + mock_response = Mock() mock_response.status_code = 200 with pytest.MonkeyPatch().context() as m: - m.setattr(httpx, 'AsyncClient', lambda **kwargs: MagicMock( - __aenter__=AsyncMock(return_value=Mock(post=AsyncMock(return_value=mock_response))), - __aexit__=AsyncMock(return_value=None) - )) + m.setattr( + httpx, + 'AsyncClient', + lambda **kwargs: MagicMock( + __aenter__=AsyncMock(return_value=Mock(post=AsyncMock(return_value=mock_response))), + __aexit__=AsyncMock(return_value=None), + ), + ) result = await manager.submit_response('survey123', {'q1': 'answer1'}) assert result is True @@ -338,15 +438,20 @@ class TestDismissSurvey: # Mock successful HTTP response import httpx + mock_response = Mock() mock_response.status_code = 200 with pytest.MonkeyPatch().context() as m: - m.setattr(httpx, 'AsyncClient', lambda **kwargs: MagicMock( - __aenter__=AsyncMock(return_value=Mock(post=AsyncMock(return_value=mock_response))), - __aexit__=AsyncMock(return_value=None) - )) + m.setattr( + httpx, + 'AsyncClient', + lambda **kwargs: MagicMock( + __aenter__=AsyncMock(return_value=Mock(post=AsyncMock(return_value=mock_response))), + __aexit__=AsyncMock(return_value=None), + ), + ) result = await manager.dismiss_survey('survey123') assert result is True - assert manager._pending_survey is None \ No newline at end of file + assert manager._pending_survey is None