diff --git a/feeds/skills/.system/files/netclaw-memory/SKILL.md b/feeds/skills/.system/files/netclaw-memory/SKILL.md index ddb5ddb80..123ccc33f 100644 --- a/feeds/skills/.system/files/netclaw-memory/SKILL.md +++ b/feeds/skills/.system/files/netclaw-memory/SKILL.md @@ -3,7 +3,7 @@ name: netclaw-memory description: "REQUIRED when the user asks what you remember, recall, or know from past conversations, previous sessions, cross-session memory, memory classes, or memory types. Also before using memory tools: find_memories, get_memories, store_memory, update_memory." metadata: author: netclaw - version: "1.13.0" + version: "1.14.0" --- # Netclaw Memory @@ -288,9 +288,16 @@ Useful log events: Embeddings are provisioned at daemon start when `Memory.Embeddings.Enabled` is `true` (default `false` for now). When unavailable: -- Log: `memory_embedding_unavailable` +- Log: `memory_embedding_unavailable` (embedder) or `memory_relevance_gate_unavailable` + (relevance/cross-encoder model) - Daemon status shows: `embeddings: degraded` - Lexical recall continues to work normally +- An operator alert (`memory.embedding_model.unavailable` / + `memory.relevance_model.unavailable`, pushed via the same notification sink as + `provider.unreachable`/`reminder.execution.failed`) fires once per model per + daemon run, naming the model, the failure reason, and the consequence (lexical-only + recall/dedup, or an unfiltered relevance gate) — this is the push-based signal; + `netclaw doctor`/`netclaw status` remain the pull-based ones `netclaw doctor`'s Memory Embeddings check reports whether the active model has a query prefix (`queryPrefix=True/False`) and the effective retrieval diff --git a/feeds/skills/.system/files/netclaw-operations/SKILL.md b/feeds/skills/.system/files/netclaw-operations/SKILL.md index 863c76be2..55d305e7d 100644 --- a/feeds/skills/.system/files/netclaw-operations/SKILL.md +++ b/feeds/skills/.system/files/netclaw-operations/SKILL.md @@ -3,7 +3,7 @@ name: netclaw-operations description: "REQUIRED when the user asks about scheduling, reminders, cron jobs, timers, background jobs, diagnostics, troubleshooting, MCP tools, daemon health, identity updates, or Netclaw capabilities and self-maintenance." metadata: author: netclaw - version: "2.25.0" + version: "2.26.0" --- # Netclaw Operations diff --git a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md index ec9ffd12e..440856bb3 100644 --- a/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md +++ b/feeds/skills/.system/files/netclaw-operations/references/diagnostics.md @@ -28,8 +28,10 @@ Log split — one stream, partitioned locally by session: id — nothing is duplicated locally. - `daemon.log` holds only sessionless, daemon-wide lines: startup/config, session start/stop, and operational **alerts** (e.g. the `provider.unreachable` / - `provider.failover` alert raised when an inference provider goes down — surfaced - here, and to webhooks, by the notification sink). Note the *per-call* failover/retry + `provider.failover` alert raised when an inference provider goes down, or + `memory.embedding_model.unavailable` / `memory.relevance_model.unavailable` when a + memory ONNX model fails to provision or load — surfaced here, and to webhooks, by + the notification sink). Note the *per-call* failover/retry log lines emitted while serving a specific session carry that session's id, so they partition into its `session.log`; the daemon-wide outage signal is the alert in `daemon.log`. Rolled daily, capped at 10 MB per file. @@ -75,6 +77,7 @@ debugging a daemon-wide problem → read `daemon.log`. | No LLM responses | `netclaw doctor`; verify provider credentials | | Missing tools | `netclaw mcp list`; check MCP connection state | | Memory recall degraded | `netclaw status` memory section | +| Memory embedding/relevance model unavailable | Fires a `memory.embedding_model.unavailable` / `memory.relevance_model.unavailable` operational alert (once per model per daemon run) naming the model, failure reason, and consequence when `Memory.Embeddings.Enabled=true` and either ONNX model fails to provision or load; see `netclaw-memory`'s Embeddings section and `netclaw doctor`'s Memory Embeddings / Memory Relevance Gate checks | | Daemon won't start | crash logs at `~/.netclaw/logs/crash-*.log` | | Docker daemon cannot create `/home/netclaw/.netclaw/*` | Official image entrypoint repairs writable bind mounts to UID/GID `1654:1654`; if bypassed or read-only, run `sudo chown -R 1654:1654 ` or use a Docker named volume | | Discord/Slack channel offline | `netclaw status` shows the channel `disconnected` with a reason. Discord may also report `degraded` when Discord.Net says the socket is connected but the gateway is not ready, such as after a resumed session that Netclaw is replacing with a clean reconnect. A misconfigured channel (bad token, missing Discord Message Content intent) degrades only that channel — the daemon keeps running and other channels are unaffected. A transient network failure retries automatically; a config/permission failure stays offline until the operator fixes the config and restarts the daemon. | diff --git a/src/Netclaw.Configuration/OperationalAlert.cs b/src/Netclaw.Configuration/OperationalAlert.cs index 7495c3f25..d2bb7762a 100644 --- a/src/Netclaw.Configuration/OperationalAlert.cs +++ b/src/Netclaw.Configuration/OperationalAlert.cs @@ -37,6 +37,8 @@ public enum AlertType DaemonStopping, DaemonCrashed, UpdateAvailable, + MemoryEmbeddingModelUnavailable, + MemoryRelevanceModelUnavailable, } /// diff --git a/src/Netclaw.Daemon.Tests/Services/EmbeddingWarmupHostedServiceTests.cs b/src/Netclaw.Daemon.Tests/Services/EmbeddingWarmupHostedServiceTests.cs index f180a0711..4a95b526b 100644 --- a/src/Netclaw.Daemon.Tests/Services/EmbeddingWarmupHostedServiceTests.cs +++ b/src/Netclaw.Daemon.Tests/Services/EmbeddingWarmupHostedServiceTests.cs @@ -286,6 +286,137 @@ public async Task Relevance_gate_disabled_config_leaves_the_relevance_holder_at_ Assert.Same(initialRelevance, relevanceHolder.Current); } + // ── Operator alerting (memory embedding/reranker provisioning-failure alert) ── + + [Fact] + public async Task Embedder_provisioning_failure_emits_exactly_one_operator_alert_naming_the_model_and_reason() + { + // No PrePlaceValidModelFiles() call -- the embedder fails. The relevance model succeeds so + // only the embedder's alert is under test here. + PrePlaceValidRelevanceModelFiles(); + + var holder = new MemoryEmbedderHolder(new UnavailableMemoryEmbedder(ModelId, "warmup not yet run"), initialQueryPrefix: "", initialCalibratedMinCosineSimilarity: null); + var relevanceHolder = CreateRelevanceScorerHolder(); + var memoryConfig = new MemoryConfig { Embeddings = { Enabled = true, ModelId = ModelId, AutoDownload = false } }; + var sink = new FakeNotificationSink(); + var service = CreateService(holder, memoryConfig, relevanceHolder, RelevanceFixtureAllowlist(), notificationSink: sink); + + await service.WarmUpAsync(TestContext.Current.CancellationToken); + + Assert.True(relevanceHolder.Current.IsAvailable); + var alert = Assert.Single(sink.Alerts); + Assert.Equal(AlertType.MemoryEmbeddingModelUnavailable, alert.Category); + Assert.Equal(ModelId, alert.Source); + Assert.Contains(ModelId, alert.Summary); + Assert.Equal(ModelId, alert.Context?["modelId"]); + Assert.False(string.IsNullOrWhiteSpace(alert.Context?["reason"])); + Assert.Contains("lexical-only", alert.Context?["consequence"]); + Assert.Contains("netclaw doctor", alert.Context?["remediation"]); + } + + [Fact] + public async Task Relevance_model_provisioning_failure_emits_exactly_one_operator_alert_naming_the_model_and_reason() + { + // Embedder succeeds; the relevance model fails (no PrePlaceValidRelevanceModelFiles call). + PrePlaceValidModelFiles(); + + var holder = new MemoryEmbedderHolder(new UnavailableMemoryEmbedder(ModelId, "warmup not yet run"), initialQueryPrefix: "", initialCalibratedMinCosineSimilarity: null); + var relevanceHolder = CreateRelevanceScorerHolder(); + var memoryConfig = new MemoryConfig { Embeddings = { Enabled = true, ModelId = ModelId, AutoDownload = false } }; + var sink = new FakeNotificationSink(); + var service = CreateService(holder, memoryConfig, relevanceHolder, RelevanceFixtureAllowlist(), notificationSink: sink); + + await service.WarmUpAsync(TestContext.Current.CancellationToken); + + Assert.True(holder.Current.IsAvailable); + var alert = Assert.Single(sink.Alerts); + Assert.Equal(AlertType.MemoryRelevanceModelUnavailable, alert.Category); + Assert.Equal(RelevanceModelId, alert.Source); + Assert.Contains(RelevanceModelId, alert.Summary); + Assert.Equal(RelevanceModelId, alert.Context?["modelId"]); + Assert.False(string.IsNullOrWhiteSpace(alert.Context?["reason"])); + Assert.Contains("relevance gate is disabled", alert.Context?["consequence"]); + // The relevance model has no backfill-embeddings analogue -- its remediation must not + // suggest that command (mirrors MemoryRelevanceGateDoctorCheck's own wording). + Assert.DoesNotContain("backfill-embeddings", alert.Context?["remediation"]); + } + + [Fact] + public async Task Both_models_failing_emits_two_distinct_operator_alerts() + { + // Neither PrePlaceValidModelFiles() nor PrePlaceValidRelevanceModelFiles() is called -- + // both models fail to provision independently. + var holder = new MemoryEmbedderHolder(new UnavailableMemoryEmbedder(ModelId, "warmup not yet run"), initialQueryPrefix: "", initialCalibratedMinCosineSimilarity: null); + var relevanceHolder = CreateRelevanceScorerHolder(); + var memoryConfig = new MemoryConfig { Embeddings = { Enabled = true, ModelId = ModelId, AutoDownload = false } }; + var sink = new FakeNotificationSink(); + var service = CreateService(holder, memoryConfig, relevanceHolder, RelevanceFixtureAllowlist(), notificationSink: sink); + + await service.WarmUpAsync(TestContext.Current.CancellationToken); + + Assert.False(holder.Current.IsAvailable); + Assert.False(relevanceHolder.Current.IsAvailable); + Assert.Equal(2, sink.Alerts.Count); + Assert.Contains(sink.Alerts, a => a.Category == AlertType.MemoryEmbeddingModelUnavailable); + Assert.Contains(sink.Alerts, a => a.Category == AlertType.MemoryRelevanceModelUnavailable); + // Distinct alert ids -- these are two independent events, not one duplicated. + Assert.NotEqual(sink.Alerts[0].AlertId, sink.Alerts[1].AlertId); + } + + [Fact] + public async Task Success_path_emits_no_operator_alerts() + { + PrePlaceValidModelFiles(); + PrePlaceValidRelevanceModelFiles(); + + var holder = new MemoryEmbedderHolder(new UnavailableMemoryEmbedder(ModelId, "warmup not yet run"), initialQueryPrefix: "", initialCalibratedMinCosineSimilarity: null); + var relevanceHolder = CreateRelevanceScorerHolder(); + var memoryConfig = new MemoryConfig { Embeddings = { Enabled = true, ModelId = ModelId, AutoDownload = true } }; + var sink = new FakeNotificationSink(); + var service = CreateService(holder, memoryConfig, relevanceHolder, RelevanceFixtureAllowlist(), notificationSink: sink); + + await service.WarmUpAsync(TestContext.Current.CancellationToken); + + Assert.True(holder.Current.IsAvailable); + Assert.True(relevanceHolder.Current.IsAvailable); + Assert.Empty(sink.Alerts); + } + + [Fact] + public async Task Disabled_config_emits_no_operator_alerts() + { + var holder = new MemoryEmbedderHolder(new UnavailableMemoryEmbedder(ModelId, "embeddings disabled"), initialQueryPrefix: "", initialCalibratedMinCosineSimilarity: null); + var relevanceHolder = CreateRelevanceScorerHolder(); + var memoryConfig = new MemoryConfig { Embeddings = { Enabled = false, ModelId = ModelId } }; + var sink = new FakeNotificationSink(); + var service = CreateService(holder, memoryConfig, relevanceHolder, RelevanceFixtureAllowlist(), notificationSink: sink); + + await service.WarmUpAsync(TestContext.Current.CancellationToken); + + // Embeddings disabled is an intentional, not degraded, state -- no alert should fire. + Assert.Empty(sink.Alerts); + } + + [Fact] + public async Task Provisioning_failure_alert_is_latched_and_does_not_refire_across_repeated_warmup_runs() + { + // Neither model's fixture files are placed -- both fail every time WarmUpAsync runs. + var holder = new MemoryEmbedderHolder(new UnavailableMemoryEmbedder(ModelId, "warmup not yet run"), initialQueryPrefix: "", initialCalibratedMinCosineSimilarity: null); + var relevanceHolder = CreateRelevanceScorerHolder(); + var memoryConfig = new MemoryConfig { Embeddings = { Enabled = true, ModelId = ModelId, AutoDownload = false } }; + var sink = new FakeNotificationSink(); + var service = CreateService(holder, memoryConfig, relevanceHolder, RelevanceFixtureAllowlist(), notificationSink: sink); + + await service.WarmUpAsync(TestContext.Current.CancellationToken); + await service.WarmUpAsync(TestContext.Current.CancellationToken); + + // Exactly one alert per model in total across both runs -- the latch, not the retry count, + // governs how many alerts an operator sees. + Assert.Equal(2, sink.Alerts.Count); + Assert.Single(sink.Alerts, a => a.Category == AlertType.MemoryEmbeddingModelUnavailable); + Assert.Single(sink.Alerts, a => a.Category == AlertType.MemoryRelevanceModelUnavailable); + } + // ── Keep-warm ticks (memory-relevance-gate 2026-07 canary fix) ── // // These tests exercise KeepWarmTickAsync/KeepWarmLoopAsync directly against simple signaling @@ -415,9 +546,11 @@ private EmbeddingWarmupHostedService CreateService( MemoryConfig memoryConfig, RelevanceScorerHolder relevanceScorerHolder, IReadOnlyDictionary relevanceAllowlist, - TimeProvider? timeProvider = null) + TimeProvider? timeProvider = null, + IOperationalNotificationSink? notificationSink = null) => new(_provisioner, _store, holder, relevanceScorerHolder, _allowlist, relevanceAllowlist, memoryConfig, _paths, - timeProvider ?? TimeProvider.System, NullLogger.Instance); + timeProvider ?? TimeProvider.System, notificationSink ?? NullNotificationSink.Instance, + NullLogger.Instance); private static RelevanceScorerHolder CreateRelevanceScorerHolder() => new(new UnavailableRelevanceScorer(RelevanceModelId, "warmup not yet run"), initialCalibratedThreshold: 0.0); @@ -550,4 +683,17 @@ public ValueTask> ScoreAsync(string query, IReadOnlyList>(candidates.Select(_ => 1.0).ToArray()); } } + + /// + /// Captures every emitted during a test — mirrors + /// McpReconnectionServiceTests.FakeNotificationSink's shape. Tests below only ever + /// await WarmUpAsync to completion before inspecting , so no + /// additional synchronization is needed. + /// + private sealed class FakeNotificationSink : IOperationalNotificationSink + { + public List Alerts { get; } = []; + + public void Emit(OperationalAlert alert) => Alerts.Add(alert); + } } diff --git a/src/Netclaw.Daemon/Services/EmbeddingWarmupHostedService.cs b/src/Netclaw.Daemon/Services/EmbeddingWarmupHostedService.cs index b2f983f29..2ba5128b5 100644 --- a/src/Netclaw.Daemon/Services/EmbeddingWarmupHostedService.cs +++ b/src/Netclaw.Daemon/Services/EmbeddingWarmupHostedService.cs @@ -48,6 +48,23 @@ namespace Netclaw.Daemon.Services; /// relevance-gate sub-budget remarks for the other half of this fix (the envelope-derived /// sub-budget clamp). /// +/// +/// +/// Operator alerting: the log line alone is not operator-facing — nobody watches daemon +/// logs in steady state, and the health endpoint/doctor check are pull-based (someone has to go +/// look). Each provision-or-degrade failure above additionally fires an +/// through the injected +/// (the same push-to-operator seam McpReconnectionService, ReminderManagerActor, and +/// RoutingChatClient already use for MCP/reminder/provider degradation) carrying the model +/// id, the failure reason, the concrete consequence (lexical-only recall/dedup, or an unfiltered +/// relevance gate), and a remediation hint. Latched per model (, +/// ) so a given model fires at most once per daemon run — this +/// method only ever runs once per host lifetime in production (see ), but +/// the latch is cheap insurance against a future caller awaiting it more than once, and is the +/// seam a mid-run keep-warm failure would also latch through if that path is ever wired up (see +/// 's remarks for why it currently is not). No alert fires when +/// Memory.Embeddings.Enabled is false — that is an intentional, not degraded, state. +/// /// internal sealed class EmbeddingWarmupHostedService( EmbeddingModelProvisioner provisioner, @@ -59,6 +76,7 @@ internal sealed class EmbeddingWarmupHostedService( MemoryConfig memoryConfig, NetclawPaths paths, TimeProvider timeProvider, + IOperationalNotificationSink notificationSink, ILogger logger) : IHostedService, IDisposable { /// @@ -92,6 +110,12 @@ internal sealed class EmbeddingWarmupHostedService( // logs, and there is no risk of the subtraction below overflowing. private long _lastKeepWarmFailureLogMs; + // Operator-alert latches (0/1 via Interlocked.CompareExchange): guarantee each model fires at + // most one OperationalAlert per daemon run even though this is currently only ever reachable + // from one call site each (see the class remarks' "Operator alerting" paragraph). + private int _embedderAlertFired; + private int _relevanceAlertFired; + public Task StartAsync(CancellationToken cancellationToken) { _ = Task.Run(() => WarmUpAsync(CancellationToken.None), CancellationToken.None); @@ -173,6 +197,18 @@ internal async Task KeepWarmTickAsync(CancellationToken ct) /// SQLiteMemoryRecallCoordinator's degradation logs use, so a persistently failing /// keep-warm tick (e.g. a model that failed to load) does not spam the log every 5 minutes /// forever. + /// + /// + /// Deliberately not wired to the operator-alert latches: a single keep-warm tick + /// failure is a transient probe result (a slow/hung ONNX call under load, a momentary holder + /// swap mid-tick), not proof a model "went bad" — the very next tick, 5 minutes later, may + /// well succeed. Promoting the first miss to an operator page would be a false-positive + /// alert on exactly the condition this method's own doc comment already calls out as not + /// user-visible degradation. Doing this properly needs a consecutive-failure threshold + /// (mirroring ReminderManagerActor's auto-disable threshold pattern) before treating a + /// keep-warm miss as equivalent-severity to a provisioning failure — a real design decision, + /// not just plumbing, so it is left as a follow-up rather than bolted on here. + /// /// private void LogKeepWarmFailed(Exception ex) { @@ -205,37 +241,45 @@ internal async Task WarmUpAsync(CancellationToken ct) var queryPrefix = manifestEntry?.QueryPrefix ?? string.Empty; var calibratedMinCosineSimilarity = manifestEntry?.CalibratedMinCosineSimilarity; - IMemoryEmbedder embedder; + // Nullable and only ever assigned on the success path below -- deliberately NOT an early + // return out of the catch block (a pre-existing bug this PR fixes: the relevance gate's + // provisioning attempt below was unreachable whenever the embedder itself failed, + // contradicting this method's own "runs regardless" contract for the relevance gate, and + // silently suppressing the relevance-model alert in exactly the both-models-degraded case + // an operator most needs to hear about). + IMemoryEmbedder? embedder = null; try { embedder = await LoadEmbedderAsync(modelId, queryPrefix, ct).ConfigureAwait(false); + holder.Set(embedder, queryPrefix, calibratedMinCosineSimilarity); + logger.LogInformation( + "memory_embedding_ready model={ModelId} dims={Dimensions} hasQueryPrefix={HasQueryPrefix} calibratedMinCosineSimilarity={CalibratedMinCosineSimilarity}", + embedder.ModelId, + embedder.Dimensions, + queryPrefix.Length > 0, + calibratedMinCosineSimilarity); } catch (Exception ex) { logger.LogError(ex, "memory_embedding_unavailable model={ModelId} reason={Reason}", modelId, ex.Message); holder.Set(new UnavailableMemoryEmbedder(modelId, ex.Message), queryPrefix, calibratedMinCosineSimilarity); - return; + EmitEmbedderUnavailableAlert(modelId, ex.Message); } - holder.Set(embedder, queryPrefix, calibratedMinCosineSimilarity); - logger.LogInformation( - "memory_embedding_ready model={ModelId} dims={Dimensions} hasQueryPrefix={HasQueryPrefix} calibratedMinCosineSimilarity={CalibratedMinCosineSimilarity}", - embedder.ModelId, - embedder.Dimensions, - queryPrefix.Length > 0, - calibratedMinCosineSimilarity); - - try + if (embedder is not null) { - await GapRepairAsync(embedder, ct).ConfigureAwait(false); - } - catch (Exception ex) - { - // The embedder itself is already loaded and the holder is already populated — a - // gap-repair failure (e.g. a transient store error) must not undo that or leave an - // unobserved exception on this fire-and-forget warmup task. The doctor check and - // the next daemon restart's sweep both retry whatever remains unembedded. - logger.LogWarning(ex, "memory_embedding_gap_repair_failed model={ModelId}", embedder.ModelId); + try + { + await GapRepairAsync(embedder, ct).ConfigureAwait(false); + } + catch (Exception ex) + { + // The embedder itself is already loaded and the holder is already populated — a + // gap-repair failure (e.g. a transient store error) must not undo that or leave an + // unobserved exception on this fire-and-forget warmup task. The doctor check and + // the next daemon restart's sweep both retry whatever remains unembedded. + logger.LogWarning(ex, "memory_embedding_gap_repair_failed model={ModelId}", embedder.ModelId); + } } // Relevance gate (memory-relevance-gate, design D4, task 1.4): a second, independent @@ -272,9 +316,73 @@ private async Task WarmUpRelevanceGateAsync(CancellationToken ct) { logger.LogError(ex, "memory_relevance_gate_unavailable model={ModelId} reason={Reason}", modelId, ex.Message); relevanceScorerHolder.Set(new UnavailableRelevanceScorer(modelId, ex.Message), calibratedThreshold); + EmitRelevanceModelUnavailableAlert(modelId, ex.Message); } } + /// + /// Fires at most once per daemon run + /// (see ). Content mirrors the doctor check's own remediation + /// wording (MemoryEmbeddingDoctorCheck) so an operator sees the same guidance whether + /// they are pulling netclaw doctor or reacting to a pushed alert. + /// + private void EmitEmbedderUnavailableAlert(string modelId, string reason) + { + if (Interlocked.CompareExchange(ref _embedderAlertFired, 1, 0) != 0) + return; + + const string consequence = "Memory recall/dedup is running lexical-only — semantic features are degraded."; + const string remediation = "Check network access and disk space, run `netclaw doctor`, or run " + + "`netclaw memory backfill-embeddings` — the daemon re-provisions the model on its next start."; + + notificationSink.Emit(OperationalAlert.Create( + timeProvider, + "memory.embedding_model.unavailable", + AlertType.MemoryEmbeddingModelUnavailable, + $"Memory embedding model '{modelId}' could not be provisioned or loaded: {reason} {consequence}", + AlertSeverity.Warning, + source: modelId, + context: new Dictionary + { + ["modelId"] = modelId, + ["reason"] = reason, + ["consequence"] = consequence, + ["remediation"] = remediation, + })); + } + + /// + /// Fires at most once per daemon run + /// (see ). Unlike , + /// the remediation does not mention netclaw memory backfill-embeddings — that command + /// only re-embeds the document corpus, it has no relevance-model analogue (mirrors + /// MemoryRelevanceGateDoctorCheck's own remediation wording). + /// + private void EmitRelevanceModelUnavailableAlert(string modelId, string reason) + { + if (Interlocked.CompareExchange(ref _relevanceAlertFired, 1, 0) != 0) + return; + + const string consequence = "The relevance gate is disabled — recall is unfiltered by the cross-encoder."; + const string remediation = "Check network access and disk space, then run `netclaw doctor` or restart the " + + "daemon to re-provision the model."; + + notificationSink.Emit(OperationalAlert.Create( + timeProvider, + "memory.relevance_model.unavailable", + AlertType.MemoryRelevanceModelUnavailable, + $"Memory relevance (cross-encoder) model '{modelId}' could not be provisioned or loaded: {reason} {consequence}", + AlertSeverity.Warning, + source: modelId, + context: new Dictionary + { + ["modelId"] = modelId, + ["reason"] = reason, + ["consequence"] = consequence, + ["remediation"] = remediation, + })); + } + private async Task LoadRelevanceScorerAsync(string modelId, CancellationToken ct) { // Keyed under the same ModelsDirectory root as embedding models (NetclawPaths.