From 34ac1a8ee5f215ed9466c783612eaa5eb8cd4c33 Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Mon, 2 Nov 2026 20:44:35 +0800 Subject: [PATCH] fix(LongTermMemory): preserve temporal precision and complete evidence spans --- .../KnowledgeEngine/LongTermMemory/README.md | 4 +-- .../store/candidate_extraction.py | 2 +- .../store/episode_provenance.py | 10 +++---- .../tests/store/test_episode_provenance.py | 30 +++++++++++++++++++ .../store/test_model_candidate_extraction.py | 26 ++++++++++++++++ 5 files changed, 63 insertions(+), 9 deletions(-) diff --git a/plugins/KnowledgeEngine/LongTermMemory/README.md b/plugins/KnowledgeEngine/LongTermMemory/README.md index 387abfd62..7b75acacc 100644 --- a/plugins/KnowledgeEngine/LongTermMemory/README.md +++ b/plugins/KnowledgeEngine/LongTermMemory/README.md @@ -620,7 +620,7 @@ Star the repo if you find it useful! | `source_ref` | Declared reference to a source message or document; not proof of Host verification. | | `claim_type` | `unknown` (default), `user_statement`, or `inference`. | -Time fields require ISO-8601 values with an explicit timezone and are normalized to UTC with second precision. When both validity bounds are present, `valid_from` must precede `valid_until`. Leave unknown values empty; the plugin does not turn “tomorrow” into an invented date or assume a timezone for these fields. Imports reject malformed temporal/provenance metadata before writing the batch. Old records missing these fields remain readable with unknown evidence and no invented event time. +Time fields require ISO-8601 values with an explicit timezone and are normalized to UTC while preserving subsecond precision. When both validity bounds are present, `valid_from` must precede `valid_until`. Leave unknown values empty; the plugin does not turn “tomorrow” into an invented date or assume a timezone for these fields. Imports reject malformed temporal/provenance metadata before writing the batch. Old records missing these fields remain readable with unknown evidence and no invented event time. `remember` accepts these optional fields. A caller can declare `user_statement` only for a direct user statement; this declaration is not a verification or authorization mechanism. Model deductions belong under `inference`. JSON document ingestion supplies a `document:` reference when no explicit source reference was supplied. Exported references remain declarations when imported elsewhere. @@ -630,7 +630,7 @@ Automatic candidate extraction uses actual `user_message_text` only. An assistan Set `candidate_model_uuid` on the memory knowledge base to select a model for candidate extraction. Leave it empty to keep the existing rule-based path and avoid an extra model call. Extraction still requires `candidate_extraction_enabled=true`. -The optional path selects and classifies complete original sentences or the full current user text; it does not rewrite statements into new facts. `evidence_quote` must match an original span exactly, and the quote itself becomes the episodic content or profile value. This preserves negations and qualifiers rather than accepting a generated fact that could contradict its evidence. Only the current actual user text and the Host's UTC processing timestamp are sent; assistant replies, older conversations, and model-invented source IDs are not used. No automatic relative-date conversion or cross-turn reference resolution is performed. +The optional path selects and classifies complete original sentences or the full current user text; it does not rewrite statements into new facts. `evidence_quote` must match an original span exactly, and the quote itself becomes the episodic content or profile value. This preserves negations and qualifiers rather than accepting a generated fact that could contradict its evidence. Only the current actual user text and the plugin Runtime's UTC processing timestamp are sent; assistant replies, older conversations, and model-invented source IDs are not used. No automatic relative-date conversion or cross-turn reference resolution is performed. Model-selected candidates are labeled `inference` because selection/classification remains a model judgment. They always remain pending for manual review, even when `candidate_auto_apply=true`; the original quote and the declared query reference stay inspectable in the candidate record. This is evidence selection, not proof that a statement is true or that a quotation is safe to treat as a current preference. diff --git a/plugins/KnowledgeEngine/LongTermMemory/store/candidate_extraction.py b/plugins/KnowledgeEngine/LongTermMemory/store/candidate_extraction.py index c9a15c73c..2bc3735b9 100644 --- a/plugins/KnowledgeEngine/LongTermMemory/store/candidate_extraction.py +++ b/plugins/KnowledgeEngine/LongTermMemory/store/candidate_extraction.py @@ -14,7 +14,7 @@ async def extract_candidates( return [] spans = { part.strip() - for part in re.split(r"(?<=[.!?。！？])\s*|\n+", text) + for part in re.split(r"(?<=[.!?])\s+|(?<=[。！？])\s*|\n+", text) if part.strip() } spans.add(text.strip()) diff --git a/plugins/KnowledgeEngine/LongTermMemory/store/episode_provenance.py b/plugins/KnowledgeEngine/LongTermMemory/store/episode_provenance.py index d14752c45..fb1cea35b 100644 --- a/plugins/KnowledgeEngine/LongTermMemory/store/episode_provenance.py +++ b/plugins/KnowledgeEngine/LongTermMemory/store/episode_provenance.py @@ -15,6 +15,7 @@ CLAIM_TYPES = {"unknown", "user_statement", "inference"} def normalize_provenance(values: dict[str, Any]) -> dict[str, str]: result = {} + parsed_times = {} for field in ("event_at", "valid_from", "valid_until"): value = values.get(field, "") if not isinstance(value, str): @@ -33,11 +34,8 @@ def normalize_provenance(values: dict[str, Any]) -> dict[str, str]: if parsed.tzinfo is None: raise ValueError(f"{field} must include an explicit timezone") try: - result[field] = ( - parsed.astimezone(timezone.utc) - .isoformat(timespec="seconds") - .replace("+00:00", "Z") - ) + parsed_times[field] = parsed.astimezone(timezone.utc) + result[field] = parsed_times[field].isoformat().replace("+00:00", "Z") except (ValueError, OverflowError) as exc: raise ValueError( f"{field} is outside the supported UTC date range" @@ -45,7 +43,7 @@ def normalize_provenance(values: dict[str, Any]) -> dict[str, str]: if ( result["valid_from"] and result["valid_until"] - and result["valid_from"] >= result["valid_until"] + and parsed_times["valid_from"] >= parsed_times["valid_until"] ): raise ValueError( "valid_from must be earlier than valid_until; validity uses [from, until)" diff --git a/plugins/KnowledgeEngine/LongTermMemory/tests/store/test_episode_provenance.py b/plugins/KnowledgeEngine/LongTermMemory/tests/store/test_episode_provenance.py index dfa87d62c..ef3daf495 100644 --- a/plugins/KnowledgeEngine/LongTermMemory/tests/store/test_episode_provenance.py +++ b/plugins/KnowledgeEngine/LongTermMemory/tests/store/test_episode_provenance.py @@ -292,3 +292,33 @@ async def test_document_ingestion_validates_and_preserves_provenance(invalid): assert meta["event_at"] == "2026-11-03T04:00:00Z" assert meta["claim_type"] == "user_statement" assert meta["source_ref"] == FACT["source_ref"] + + +def test_subsecond_validity_and_offset_conversion_preserve_precision(): + value = normalize_provenance( + { + "event_at": "2026-11-03T12:00:00.123456+08:00", + "valid_from": "2026-11-03T12:00:00.1+08:00", + "valid_until": "2026-11-03T12:00:00.2+08:00", + } + ) + assert value["event_at"] == "2026-11-03T04:00:00.123456Z" + assert value["valid_from"] == "2026-11-03T04:00:00.100000Z" + assert value["valid_until"] == "2026-11-03T04:00:00.200000Z" + + +def test_validity_compares_instants_instead_of_variable_precision_strings(): + value = normalize_provenance( + { + "valid_from": "2026-11-03T04:00:00Z", + "valid_until": "2026-11-03T04:00:00.1Z", + } + ) + assert value["valid_from"] == "2026-11-03T04:00:00Z" + with pytest.raises(ValueError, match="valid_from"): + normalize_provenance( + { + "valid_from": "2026-11-03T04:00:00.1Z", + "valid_until": "2026-11-03T04:00:00Z", + } + ) diff --git a/plugins/KnowledgeEngine/LongTermMemory/tests/store/test_model_candidate_extraction.py b/plugins/KnowledgeEngine/LongTermMemory/tests/store/test_model_candidate_extraction.py index 05e7ff430..02cc681c1 100644 --- a/plugins/KnowledgeEngine/LongTermMemory/tests/store/test_model_candidate_extraction.py +++ b/plugins/KnowledgeEngine/LongTermMemory/tests/store/test_model_candidate_extraction.py @@ -185,3 +185,29 @@ async def test_sensitive_user_text_is_not_sent_to_candidate_model(monkeypatch): "bot:person_user", include_statuses=["accepted", "pending"] ) assert all("do-not-store-this" not in json.dumps(item) for item in entries) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("quote", ["We do not support v1.", "5."]) +async def test_model_cannot_select_a_span_cut_inside_a_version_number(quote): + text = "We do not support v1.5. Use v2.0." + plugin = ModelPlugin( + json.dumps([{"candidate_type": "l2_episode", "evidence_quote": quote}]) + ) + with pytest.raises(ValueError, match="original span"): + await extract_candidates( + plugin, "model", text, "2026-11-02T00:00:00Z", "alice", 3 + ) + + +@pytest.mark.asyncio +async def test_full_version_statement_keeps_its_negation_and_decimal(): + text = "We do not support v1.5. Use v2.0." + quote = "We do not support v1.5." + plugin = ModelPlugin( + json.dumps([{"candidate_type": "l2_episode", "evidence_quote": quote}]) + ) + result = await extract_candidates( + plugin, "model", text, "2026-11-02T00:00:00Z", "alice", 3 + ) + assert result[0]["payload"]["content"] == quote