fix(plugin): report when an installation failure happened, not when it was sampled

The connector stored only uuid/code/message, so the ops reporter stamped every entry with its own sampling time and a stale failure looked freshly broken on each report. Failures now carry failed_at and the payload publishes it.
This commit is contained in:
Hyu
2026-10-02 01:34:46 +08:00
parent 9186c8e43a
commit 38d5750785
4 changed files with 78 additions and 6 deletions
@@ -735,11 +735,13 @@ async def test_apply_dependency_failure_raises_stable_observable_error():
assert error.installation_uuid == setting.installation_uuid
assert error.error_code == 'dependency_prepare_failed'
assert '[dependency_prepare_failed]' in str(error)
assert connector._installation_failures[setting.installation_uuid] == {
recorded = connector._installation_failures[setting.installation_uuid]
assert {key: value for key, value in recorded.items() if key != 'failed_at'} == {
'installation_uuid': setting.installation_uuid,
'error_code': 'dependency_prepare_failed',
'message': 'Plugin dependency installer exited with code 1',
}
assert str(recorded['failed_at']).endswith('Z')
connector.ap.logger.error.assert_called_once()
@@ -767,9 +769,12 @@ async def test_shared_reconcile_records_one_failure_without_blocking_other_state
setting_a.installation_uuid,
setting_b.installation_uuid,
}
assert connector._installation_failures == {
setting_a.installation_uuid: failure,
}
recorded_failures = connector._installation_failures
assert {
uuid: {key: value for key, value in record.items() if key != 'failed_at'}
for uuid, record in recorded_failures.items()
} == {setting_a.installation_uuid: failure}
assert str(recorded_failures[setting_a.installation_uuid]['failed_at']).endswith('Z')
assert set(connector._known_desired_states) == {
setting_a.installation_uuid,
setting_b.installation_uuid,
@@ -282,6 +282,39 @@ class TestUpload:
await reporter.stop()
class TestFailureTimestamps:
def test_publishes_when_the_failure_happened_not_when_it_was_sampled(self):
failures = {
'inst-1': {
'installation_uuid': 'inst-1',
'error_code': 'dependency_prepare_failed',
'message': 'prepare failed',
'failed_at': '2026-10-01T07:11:00Z',
}
}
reporter = RuntimeOpsReporter(make_app(connector=make_connector(failures=failures)))
payload = reporter.build_payload({'live': True})
entry = payload['failures'][0]
assert entry['observed_at'] == '2026-10-01T07:11:00Z'
assert entry['observed_at'] != payload['generated_at']
def test_legacy_records_without_a_timestamp_fall_back_to_the_sample_time(self):
failures = {
'inst-2': {
'installation_uuid': 'inst-2',
'error_code': 'worker_launch_failed',
'message': 'boom',
}
}
reporter = RuntimeOpsReporter(make_app(connector=make_connector(failures=failures)))
payload = reporter.build_payload({'live': True})
assert payload['failures'][0]['observed_at'] == payload['generated_at']
def test_payload_never_contains_a_control_token(monkeypatch):
monkeypatch.setenv(PLUGIN_RUNTIME_CONTROL_TOKEN_ENV, 'runtime-control-secret')
monkeypatch.setenv(_CONTROL_PLANE_TOKEN_ENV, 'space-control-plane-secret')