From 9373725a65b32b9ed08189b17354e83aa56e8b45 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Thu, 9 Jul 2026 21:00:33 +0000 Subject: [PATCH] feat(memory): operator alert when embedding/reranker model provisioning fails When Memory.Embeddings.Enabled=true and either ONNX model (the embedder or the ms-marco-minilm-l-6-v2 relevance/reranker model) fails to provision or load, the daemon previously only logged memory_embedding_unavailable / memory_relevance_gate_unavailable and left the failure to be discovered via netclaw doctor or the health endpoint -- both pull-based. Operators had no push notification that memory was running degraded. Reuses the existing IOperationalNotificationSink/OperationalAlert seam (Netclaw.Configuration) -- the same push-to-operator mechanism McpReconnectionService, ReminderManagerActor, and RoutingChatClient already use for MCP/reminder/provider degradation, wired to Slack/webhook targets by WebhookNotificationService. Two new AlertType values (MemoryEmbeddingModelUnavailable, MemoryRelevanceModelUnavailable). Each alert carries the model id, the failure reason, the concrete consequence (lexical-only recall/dedup, or an unfiltered relevance gate), and a remediation hint (check network/disk, netclaw doctor, netclaw memory backfill-embeddings where applicable) -- content mirrors the existing MemoryEmbeddingDoctorCheck/MemoryRelevanceGateDoctorCheck wording. Latched per model (Interlocked-guarded) so each model alerts at most once per daemon run, not per retry. No alert when Embeddings.Enabled=false (an intentional, not degraded, state). Deliberately did NOT wire the keep-warm loop's mid-run failure (memory_embedding_keep_warm_failed) into the same alert path -- a single keep-warm miss is a transient probe result the method's own doc comment already calls out as not user-visible degradation, and alerting on the first miss would false-positive on exactly that. Doing it properly needs a consecutive-failure threshold (mirroring ReminderManagerActor's auto-disable pattern), which is a design decision, not just plumbing -- left as a follow-up. Also fixes a pre-existing bug surfaced by testing the two-model-failure case: WarmUpAsync returned early from the embedder's catch block, so WarmUpRelevanceGateAsync was unreachable whenever the embedder itself failed -- contradicting the method's own 'runs regardless' contract for the relevance gate and silently suppressing the relevance-model alert in the worst-case (both models down) scenario. Tests: embedder-only failure, relevance-only failure, both-fail (two distinct alerts), success path (no alerts), disabled config (no alerts), and a latch test proving repeated WarmUpAsync calls don't refire. 25 tests in EmbeddingWarmupHostedServiceTests, all green. Full Netclaw.Daemon.Tests (854), Netclaw.Actors.Tests (2671), Netclaw.Embeddings.Tests (48), and Netclaw.Configuration.Tests (467) green. Full solution build clean. Updates netclaw-operations (2.25.0 -> 2.26.0, diagnostics reference) and netclaw-memory (1.13.0 -> 1.14.0, Embeddings section) skills per the constitution's skill-sync rule. --- .../.system/files/netclaw-memory/SKILL.md | 11 +- .../.system/files/netclaw-operations/SKILL.md | 2 +- .../references/diagnostics.md | 7 +- src/Netclaw.Configuration/OperationalAlert.cs | 2 + .../EmbeddingWarmupHostedServiceTests.cs | 150 +++++++++++++++++- .../Services/EmbeddingWarmupHostedService.cs | 148 ++++++++++++++--- 6 files changed, 293 insertions(+), 27 deletions(-) diff --git a/feeds/skills/.system/files/netclaw-memory/SKILL.md b/feeds/skills/.system/files/netclaw-memory/SKILL.md index ddb5ddb80..123ccc33f 100644 --- a/feeds/skills/.system/files/netclaw-memory/SKILL.md +++ b/feeds/skills/.system/files/netclaw-memory/SKILL.md @@ -3,7 +3,7 @@ name: netclaw-memory description: "REQUIRED when the user asks what you remember, recall, or know from past conversations, previous sessions, cross-session memory, memory classes, or memory types. Also before using memory tools: find_memories, get_memories, store_memory, update_memory." metadata: author: netclaw - version: "1.13.0" + version: "1.14.0" --- # Netclaw Memory @@ -288,9 +288,16 @@ Useful log events: Embeddings are provisioned at daemon start when `Memory.Embeddings.Enabled` is `true` (default `false` for now). When unavailable: -- Log: `memory_embedding_unavailable` +- Log: `memory_embedding_unavailable` (embedder) or `memory_relevance_gate_unavailable` + (relevance/cross-encoder model) - Daemon status shows: `embeddings: degraded` - Lexical recall continues to work normally +- An operator alert (`memory.embedding_model.unavailable` / + `memory.relevance_model.unavailable`, pushed via the same notification sink as + `provider.unreachable`/`reminder.execution.failed`) fires once per model per + daemon run, naming the model, the failure reason, and the consequence (lexical-only + recall/dedup, or an unfiltered relevance gate) — this is the push-based signal; + `netclaw doctor`/`netclaw status` remain the pull-based ones `netclaw doctor`'s Memory Embeddings check reports whether the active model has a query prefix (`queryPrefix=True/False`) and the effective retrieval diff --git a/feeds/skills/.system/files/netclaw-operations/SKILL.md b/feeds/skills/.system/files/netclaw-operations/SKILL.md index 863c76be2..55d305e7d 100644 --- a/feeds/skills/.system/files/netclaw-operations/SKILL.md +++ b/feeds/skills/.system/files/netclaw-operations/SKILL.md @@ -3,7 +3,7 @@ name: netclaw-operations description: "REQUIRED when the user asks about scheduling, reminders, cron jobs, timers, background jobs, diagnostics, troubleshooting, MCP tools, daemon health, identity updates, or Netclaw capabilities and self-maintenance." metadata: author: netclaw - version: "2.25.0" + version: "2.26.0" --- # Netclaw Operations diff --git a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md index ec9ffd12e..440856bb3 100644 --- a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md +++ b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md @@ -28,8 +28,10 @@ Log split — one stream, partitioned locally by session: id — nothing is duplicated locally. - `daemon.log` holds only sessionless, daemon-wide lines: startup/config, session start/stop, and operational **alerts** (e.g. the `provider.unreachable` / - `provider.failover` alert raised when an inference provider goes down — surfaced - here, and to webhooks, by the notification sink). Note the *per-call* failover/retry + `provider.failover` alert raised when an inference provider goes down, or + `memory.embedding_model.unavailable` / `memory.relevance_model.unavailable` when a + memory ONNX model fails to provision or load — surfaced here, and to webhooks, by + the notification sink). Note the *per-call* failover/retry log lines emitted while serving a specific session carry that session's id, so they partition into its `session.log`; the daemon-wide outage signal is the alert in `daemon.log`. Rolled daily, capped at 10 MB per file. @@ -75,6 +77,7 @@ debugging a daemon-wide problem → read `daemon.log`. | No LLM responses | `netclaw doctor`; verify provider credentials | | Missing tools | `netclaw mcp list`; check MCP connection state | | Memory recall degraded | `netclaw status` memory section | +| Memory embedding/relevance model unavailable | Fires a `memory.embedding_model.unavailable` / `memory.relevance_model.unavailable` operational alert (once per model per daemon run) naming the model, failure reason, and consequence when `Memory.Embeddings.Enabled=true` and either ONNX model fails to provision or load; see `netclaw-memory`'s Embeddings section and `netclaw doctor`'s Memory Embeddings / Memory Relevance Gate checks | | Daemon won't start | crash logs at `~/.netclaw/logs/crash-*.log` | | Docker daemon cannot create `/home/netclaw/.netclaw/*` | Official image entrypoint repairs writable bind mounts to UID/GID `1654:1654`; if bypassed or read-only, run `sudo chown -R 1654:1654 ` or use a Docker named volume | | Discord/Slack channel offline | `netclaw status` shows the channel `disconnected` with a reason. Discord may also report `degraded` when Discord.Net says the socket is connected but the gateway is not ready, such as after a resumed session that Netclaw is replacing with a clean reconnect. A misconfigured channel (bad token, missing Discord Message Content intent) degrades only that channel — the daemon keeps running and other channels are unaffected. A transient network failure retries automatically; a config/permission failure stays offline until the operator fixes the config and restarts the daemon. | diff --git a/src/Netclaw.Configuration/OperationalAlert.cs b/src/Netclaw.Configuration/OperationalAlert.cs index 7495c3f25..d2bb7762a 100644 --- a/src/Netclaw.Configuration/OperationalAlert.cs +++ b/src/Netclaw.Configuration/OperationalAlert.cs @@ -37,6 +37,8 @@ public enum AlertType DaemonStopping, DaemonCrashed, UpdateAvailable, + MemoryEmbeddingModelUnavailable, + MemoryRelevanceModelUnavailable, } /// diff --git a/src/Netclaw.Daemon.Tests/Services/EmbeddingWarmupHostedServiceTests.cs b/src/Netclaw.Daemon.Tests/Services/EmbeddingWarmupHostedServiceTests.cs index f180a0711..4a95b526b 100644 --- a/src/Netclaw.Daemon.Tests/Services/EmbeddingWarmupHostedServiceTests.cs +++ b/src/Netclaw.Daemon.Tests/Services/EmbeddingWarmupHostedServiceTests.cs @@ -286,6 +286,137 @@ public async Task Relevance_gate_disabled_config_leaves_the_relevance_holder_at_ Assert.Same(initialRelevance, relevanceHolder.Current); } + // ── Operator alerting (memory embedding/reranker provisioning-failure alert) ── + + [Fact] + public async Task Embedder_provisioning_failure_emits_exactly_one_operator_alert_naming_the_model_and_reason() + { + // No PrePlaceValidModelFiles() call -- the embedder fails. The relevance model succeeds so + // only the embedder's alert is under test here. + PrePlaceValidRelevanceModelFiles(); + + var holder = new MemoryEmbedderHolder(new UnavailableMemoryEmbedder(ModelId, "warmup not yet run"), initialQueryPrefix: "", initialCalibratedMinCosineSimilarity: null); + var relevanceHolder = CreateRelevanceScorerHolder(); + var memoryConfig = new MemoryConfig { Embeddings = { Enabled = true, ModelId = ModelId, AutoDownload = false } }; + var sink = new FakeNotificationSink(); + var service = CreateService(holder, memoryConfig, relevanceHolder, RelevanceFixtureAllowlist(), notificationSink: sink); + + await service.WarmUpAsync(TestContext.Current.CancellationToken); + + Assert.True(relevanceHolder.Current.IsAvailable); + var alert = Assert.Single(sink.Alerts); + Assert.Equal(AlertType.MemoryEmbeddingModelUnavailable, alert.Category); + Assert.Equal(ModelId, alert.Source); + Assert.Contains(ModelId, alert.Summary); + Assert.Equal(ModelId, alert.Context?["modelId"]); + Assert.False(string.IsNullOrWhiteSpace(alert.Context?["reason"])); + Assert.Contains("lexical-only", alert.Context?["consequence"]); + Assert.Contains("netclaw doctor", alert.Context?["remediation"]); + } + + [Fact] + public async Task Relevance_model_provisioning_failure_emits_exactly_one_operator_alert_naming_the_model_and_reason() + { + // Embedder succeeds; the relevance model fails (no PrePlaceValidRelevanceModelFiles call). + PrePlaceValidModelFiles(); + + var holder = new MemoryEmbedderHolder(new UnavailableMemoryEmbedder(ModelId, "warmup not yet run"), initialQueryPrefix: "", initialCalibratedMinCosineSimilarity: null); + var relevanceHolder = CreateRelevanceScorerHolder(); + var memoryConfig = new MemoryConfig { Embeddings = { Enabled = true, ModelId = ModelId, AutoDownload = false } }; + var sink = new FakeNotificationSink(); + var service = CreateService(holder, memoryConfig, relevanceHolder, RelevanceFixtureAllowlist(), notificationSink: sink); + + await service.WarmUpAsync(TestContext.Current.CancellationToken); + + Assert.True(holder.Current.IsAvailable); + var alert = Assert.Single(sink.Alerts); + Assert.Equal(AlertType.MemoryRelevanceModelUnavailable, alert.Category); + Assert.Equal(RelevanceModelId, alert.Source); + Assert.Contains(RelevanceModelId, alert.Summary); + Assert.Equal(RelevanceModelId, alert.Context?["modelId"]); + Assert.False(string.IsNullOrWhiteSpace(alert.Context?["reason"])); + Assert.Contains("relevance gate is disabled", alert.Context?["consequence"]); + // The relevance model has no backfill-embeddings analogue -- its remediation must not + // suggest that command (mirrors MemoryRelevanceGateDoctorCheck's own wording). + Assert.DoesNotContain("backfill-embeddings", alert.Context?["remediation"]); + } + + [Fact] + public async Task Both_models_failing_emits_two_distinct_operator_alerts() + { + // Neither PrePlaceValidModelFiles() nor PrePlaceValidRelevanceModelFiles() is called -- + // both models fail to provision independently. + var holder = new MemoryEmbedderHolder(new UnavailableMemoryEmbedder(ModelId, "warmup not yet run"), initialQueryPrefix: "", initialCalibratedMinCosineSimilarity: null); + var relevanceHolder = CreateRelevanceScorerHolder(); + var memoryConfig = new MemoryConfig { Embeddings = { Enabled = true, ModelId = ModelId, AutoDownload = false } }; + var sink = new FakeNotificationSink(); + var service = CreateService(holder, memoryConfig, relevanceHolder, RelevanceFixtureAllowlist(), notificationSink: sink); + + await service.WarmUpAsync(TestContext.Current.CancellationToken); + + Assert.False(holder.Current.IsAvailable); + Assert.False(relevanceHolder.Current.IsAvailable); + Assert.Equal(2, sink.Alerts.Count); + Assert.Contains(sink.Alerts, a => a.Category == AlertType.MemoryEmbeddingModelUnavailable); + Assert.Contains(sink.Alerts, a => a.Category == AlertType.MemoryRelevanceModelUnavailable); + // Distinct alert ids -- these are two independent events, not one duplicated. + Assert.NotEqual(sink.Alerts[0].AlertId, sink.Alerts[1].AlertId); + } + + [Fact] + public async Task Success_path_emits_no_operator_alerts() + { + PrePlaceValidModelFiles(); + PrePlaceValidRelevanceModelFiles(); + + var holder = new MemoryEmbedderHolder(new UnavailableMemoryEmbedder(ModelId, "warmup not yet run"), initialQueryPrefix: "", initialCalibratedMinCosineSimilarity: null); + var relevanceHolder = CreateRelevanceScorerHolder(); + var memoryConfig = new MemoryConfig { Embeddings = { Enabled = true, ModelId = ModelId, AutoDownload = true } }; + var sink = new FakeNotificationSink(); + var service = CreateService(holder, memoryConfig, relevanceHolder, RelevanceFixtureAllowlist(), notificationSink: sink); + + await service.WarmUpAsync(TestContext.Current.CancellationToken); + + Assert.True(holder.Current.IsAvailable); + Assert.True(relevanceHolder.Current.IsAvailable); + Assert.Empty(sink.Alerts); + } + + [Fact] + public async Task Disabled_config_emits_no_operator_alerts() + { + var holder = new MemoryEmbedderHolder(new UnavailableMemoryEmbedder(ModelId, "embeddings disabled"), initialQueryPrefix: "", initialCalibratedMinCosineSimilarity: null); + var relevanceHolder = CreateRelevanceScorerHolder(); + var memoryConfig = new MemoryConfig { Embeddings = { Enabled = false, ModelId = ModelId } }; + var sink = new FakeNotificationSink(); + var service = CreateService(holder, memoryConfig, relevanceHolder, RelevanceFixtureAllowlist(), notificationSink: sink); + + await service.WarmUpAsync(TestContext.Current.CancellationToken); + + // Embeddings disabled is an intentional, not degraded, state -- no alert should fire. + Assert.Empty(sink.Alerts); + } + + [Fact] + public async Task Provisioning_failure_alert_is_latched_and_does_not_refire_across_repeated_warmup_runs() + { + // Neither model's fixture files are placed -- both fail every time WarmUpAsync runs. + var holder = new MemoryEmbedderHolder(new UnavailableMemoryEmbedder(ModelId, "warmup not yet run"), initialQueryPrefix: "", initialCalibratedMinCosineSimilarity: null); + var relevanceHolder = CreateRelevanceScorerHolder(); + var memoryConfig = new MemoryConfig { Embeddings = { Enabled = true, ModelId = ModelId, AutoDownload = false } }; + var sink = new FakeNotificationSink(); + var service = CreateService(holder, memoryConfig, relevanceHolder, RelevanceFixtureAllowlist(), notificationSink: sink); + + await service.WarmUpAsync(TestContext.Current.CancellationToken); + await service.WarmUpAsync(TestContext.Current.CancellationToken); + + // Exactly one alert per model in total across both runs -- the latch, not the retry count, + // governs how many alerts an operator sees. + Assert.Equal(2, sink.Alerts.Count); + Assert.Single(sink.Alerts, a => a.Category == AlertType.MemoryEmbeddingModelUnavailable); + Assert.Single(sink.Alerts, a => a.Category == AlertType.MemoryRelevanceModelUnavailable); + } + // ── Keep-warm ticks (memory-relevance-gate 2026-07 canary fix) ── // // These tests exercise KeepWarmTickAsync/KeepWarmLoopAsync directly against simple signaling @@ -415,9 +546,11 @@ private EmbeddingWarmupHostedService CreateService( MemoryConfig memoryConfig, RelevanceScorerHolder relevanceScorerHolder, IReadOnlyDictionary relevanceAllowlist, - TimeProvider? timeProvider = null) + TimeProvider? timeProvider = null, + IOperationalNotificationSink? notificationSink = null) => new(_provisioner, _store, holder, relevanceScorerHolder, _allowlist, relevanceAllowlist, memoryConfig, _paths, - timeProvider ?? TimeProvider.System, NullLogger.Instance); + timeProvider ?? TimeProvider.System, notificationSink ?? NullNotificationSink.Instance, + NullLogger.Instance); private static RelevanceScorerHolder CreateRelevanceScorerHolder() => new(new UnavailableRelevanceScorer(RelevanceModelId, "warmup not yet run"), initialCalibratedThreshold: 0.0); @@ -550,4 +683,17 @@ public ValueTask> ScoreAsync(string query, IReadOnlyList>(candidates.Select(_ => 1.0).ToArray()); } } + + /// + /// Captures every emitted during a test — mirrors + /// McpReconnectionServiceTests.FakeNotificationSink's shape. Tests below only ever + /// await WarmUpAsync to completion before inspecting , so no + /// additional synchronization is needed. + /// + private sealed class FakeNotificationSink : IOperationalNotificationSink + { + public List Alerts { get; } = []; + + public void Emit(OperationalAlert alert) => Alerts.Add(alert); + } } diff --git a/src/Netclaw.Daemon/Services/EmbeddingWarmupHostedService.cs b/src/Netclaw.Daemon/Services/EmbeddingWarmupHostedService.cs index b2f983f29..2ba5128b5 100644 --- a/src/Netclaw.Daemon/Services/EmbeddingWarmupHostedService.cs +++ b/src/Netclaw.Daemon/Services/EmbeddingWarmupHostedService.cs @@ -48,6 +48,23 @@ namespace Netclaw.Daemon.Services; /// relevance-gate sub-budget remarks for the other half of this fix (the envelope-derived /// sub-budget clamp). /// +/// +/// +/// Operator alerting: the log line alone is not operator-facing — nobody watches daemon +/// logs in steady state, and the health endpoint/doctor check are pull-based (someone has to go +/// look). Each provision-or-degrade failure above additionally fires an +/// through the injected +/// (the same push-to-operator seam McpReconnectionService, ReminderManagerActor, and +/// RoutingChatClient already use for MCP/reminder/provider degradation) carrying the model +/// id, the failure reason, the concrete consequence (lexical-only recall/dedup, or an unfiltered +/// relevance gate), and a remediation hint. Latched per model (, +/// ) so a given model fires at most once per daemon run — this +/// method only ever runs once per host lifetime in production (see ), but +/// the latch is cheap insurance against a future caller awaiting it more than once, and is the +/// seam a mid-run keep-warm failure would also latch through if that path is ever wired up (see +/// 's remarks for why it currently is not). No alert fires when +/// Memory.Embeddings.Enabled is false — that is an intentional, not degraded, state. +/// /// internal sealed class EmbeddingWarmupHostedService( EmbeddingModelProvisioner provisioner, @@ -59,6 +76,7 @@ internal sealed class EmbeddingWarmupHostedService( MemoryConfig memoryConfig, NetclawPaths paths, TimeProvider timeProvider, + IOperationalNotificationSink notificationSink, ILogger logger) : IHostedService, IDisposable { /// @@ -92,6 +110,12 @@ internal sealed class EmbeddingWarmupHostedService( // logs, and there is no risk of the subtraction below overflowing. private long _lastKeepWarmFailureLogMs; + // Operator-alert latches (0/1 via Interlocked.CompareExchange): guarantee each model fires at + // most one OperationalAlert per daemon run even though this is currently only ever reachable + // from one call site each (see the class remarks' "Operator alerting" paragraph). + private int _embedderAlertFired; + private int _relevanceAlertFired; + public Task StartAsync(CancellationToken cancellationToken) { _ = Task.Run(() => WarmUpAsync(CancellationToken.None), CancellationToken.None); @@ -173,6 +197,18 @@ internal async Task KeepWarmTickAsync(CancellationToken ct) /// SQLiteMemoryRecallCoordinator's degradation logs use, so a persistently failing /// keep-warm tick (e.g. a model that failed to load) does not spam the log every 5 minutes /// forever. + /// + /// + /// Deliberately not wired to the operator-alert latches: a single keep-warm tick + /// failure is a transient probe result (a slow/hung ONNX call under load, a momentary holder + /// swap mid-tick), not proof a model "went bad" — the very next tick, 5 minutes later, may + /// well succeed. Promoting the first miss to an operator page would be a false-positive + /// alert on exactly the condition this method's own doc comment already calls out as not + /// user-visible degradation. Doing this properly needs a consecutive-failure threshold + /// (mirroring ReminderManagerActor's auto-disable threshold pattern) before treating a + /// keep-warm miss as equivalent-severity to a provisioning failure — a real design decision, + /// not just plumbing, so it is left as a follow-up rather than bolted on here. + /// /// private void LogKeepWarmFailed(Exception ex) { @@ -205,37 +241,45 @@ internal async Task WarmUpAsync(CancellationToken ct) var queryPrefix = manifestEntry?.QueryPrefix ?? string.Empty; var calibratedMinCosineSimilarity = manifestEntry?.CalibratedMinCosineSimilarity; - IMemoryEmbedder embedder; + // Nullable and only ever assigned on the success path below -- deliberately NOT an early + // return out of the catch block (a pre-existing bug this PR fixes: the relevance gate's + // provisioning attempt below was unreachable whenever the embedder itself failed, + // contradicting this method's own "runs regardless" contract for the relevance gate, and + // silently suppressing the relevance-model alert in exactly the both-models-degraded case + // an operator most needs to hear about). + IMemoryEmbedder? embedder = null; try { embedder = await LoadEmbedderAsync(modelId, queryPrefix, ct).ConfigureAwait(false); + holder.Set(embedder, queryPrefix, calibratedMinCosineSimilarity); + logger.LogInformation( + "memory_embedding_ready model={ModelId} dims={Dimensions} hasQueryPrefix={HasQueryPrefix} calibratedMinCosineSimilarity={CalibratedMinCosineSimilarity}", + embedder.ModelId, + embedder.Dimensions, + queryPrefix.Length > 0, + calibratedMinCosineSimilarity); } catch (Exception ex) { logger.LogError(ex, "memory_embedding_unavailable model={ModelId} reason={Reason}", modelId, ex.Message); holder.Set(new UnavailableMemoryEmbedder(modelId, ex.Message), queryPrefix, calibratedMinCosineSimilarity); - return; + EmitEmbedderUnavailableAlert(modelId, ex.Message); } - holder.Set(embedder, queryPrefix, calibratedMinCosineSimilarity); - logger.LogInformation( - "memory_embedding_ready model={ModelId} dims={Dimensions} hasQueryPrefix={HasQueryPrefix} calibratedMinCosineSimilarity={CalibratedMinCosineSimilarity}", - embedder.ModelId, - embedder.Dimensions, - queryPrefix.Length > 0, - calibratedMinCosineSimilarity); - - try + if (embedder is not null) { - await GapRepairAsync(embedder, ct).ConfigureAwait(false); - } - catch (Exception ex) - { - // The embedder itself is already loaded and the holder is already populated — a - // gap-repair failure (e.g. a transient store error) must not undo that or leave an - // unobserved exception on this fire-and-forget warmup task. The doctor check and - // the next daemon restart's sweep both retry whatever remains unembedded. - logger.LogWarning(ex, "memory_embedding_gap_repair_failed model={ModelId}", embedder.ModelId); + try + { + await GapRepairAsync(embedder, ct).ConfigureAwait(false); + } + catch (Exception ex) + { + // The embedder itself is already loaded and the holder is already populated — a + // gap-repair failure (e.g. a transient store error) must not undo that or leave an + // unobserved exception on this fire-and-forget warmup task. The doctor check and + // the next daemon restart's sweep both retry whatever remains unembedded. + logger.LogWarning(ex, "memory_embedding_gap_repair_failed model={ModelId}", embedder.ModelId); + } } // Relevance gate (memory-relevance-gate, design D4, task 1.4): a second, independent @@ -272,9 +316,73 @@ private async Task WarmUpRelevanceGateAsync(CancellationToken ct) { logger.LogError(ex, "memory_relevance_gate_unavailable model={ModelId} reason={Reason}", modelId, ex.Message); relevanceScorerHolder.Set(new UnavailableRelevanceScorer(modelId, ex.Message), calibratedThreshold); + EmitRelevanceModelUnavailableAlert(modelId, ex.Message); } } + /// + /// Fires at most once per daemon run + /// (see ). Content mirrors the doctor check's own remediation + /// wording (MemoryEmbeddingDoctorCheck) so an operator sees the same guidance whether + /// they are pulling netclaw doctor or reacting to a pushed alert. + /// + private void EmitEmbedderUnavailableAlert(string modelId, string reason) + { + if (Interlocked.CompareExchange(ref _embedderAlertFired, 1, 0) != 0) + return; + + const string consequence = "Memory recall/dedup is running lexical-only — semantic features are degraded."; + const string remediation = "Check network access and disk space, run `netclaw doctor`, or run " + + "`netclaw memory backfill-embeddings` — the daemon re-provisions the model on its next start."; + + notificationSink.Emit(OperationalAlert.Create( + timeProvider, + "memory.embedding_model.unavailable", + AlertType.MemoryEmbeddingModelUnavailable, + $"Memory embedding model '{modelId}' could not be provisioned or loaded: {reason} {consequence}", + AlertSeverity.Warning, + source: modelId, + context: new Dictionary + { + ["modelId"] = modelId, + ["reason"] = reason, + ["consequence"] = consequence, + ["remediation"] = remediation, + })); + } + + /// + /// Fires at most once per daemon run + /// (see ). Unlike , + /// the remediation does not mention netclaw memory backfill-embeddings — that command + /// only re-embeds the document corpus, it has no relevance-model analogue (mirrors + /// MemoryRelevanceGateDoctorCheck's own remediation wording). + /// + private void EmitRelevanceModelUnavailableAlert(string modelId, string reason) + { + if (Interlocked.CompareExchange(ref _relevanceAlertFired, 1, 0) != 0) + return; + + const string consequence = "The relevance gate is disabled — recall is unfiltered by the cross-encoder."; + const string remediation = "Check network access and disk space, then run `netclaw doctor` or restart the " + + "daemon to re-provision the model."; + + notificationSink.Emit(OperationalAlert.Create( + timeProvider, + "memory.relevance_model.unavailable", + AlertType.MemoryRelevanceModelUnavailable, + $"Memory relevance (cross-encoder) model '{modelId}' could not be provisioned or loaded: {reason} {consequence}", + AlertSeverity.Warning, + source: modelId, + context: new Dictionary + { + ["modelId"] = modelId, + ["reason"] = reason, + ["consequence"] = consequence, + ["remediation"] = remediation, + })); + } + private async Task LoadRelevanceScorerAsync(string modelId, CancellationToken ct) { // Keyed under the same ModelsDirectory root as embedding models (NetclawPaths.