Expose per-endpoint dead-letter destination as a queryable contract (#3104)

docs/guide/durability/dead-letter-storage.md

@@ -23,6 +23,31 @@ To replay dead lettered messages back to the incoming table, you also have a com
 dotnet run -- storage replay
 ```
 
+## Introspecting an Endpoint's Dead Letter Destination <Badge type="tip" text="6.9" />
+
+Where an endpoint's dead letters actually go varies by transport and configuration: some endpoints
+move failures to Wolverine's durable `wolverine_dead_letters` storage, while others use a **native
+broker dead letter queue** (RabbitMQ DLX, an SQS dead letter queue, the Azure Service Bus
+`$DeadLetterQueue`, etc.) that a tool managing the durable store can't see.
+
+Every endpoint declares its effective destination through a single transport-agnostic enum,
+`DeadLetterStorageMode`, so monitoring tools can introspect it without transport-specific knowledge:
+
+| Value | Meaning |
+|-------|---------|
+| `Durable` | Dead letters go to Wolverine's durable store (`wolverine_dead_letters`) — queryable and replayable through `IDeadLetters`. |
+| `Native` | Dead letters go to a native broker dead letter queue and are **not** bridged into durable storage. |
+| `NativeWithRecovery` | Dead letters go to a native broker dead letter queue **and** are bridged back into durable storage via [`EnableDeadLetterQueueRecovery()`](/guide/messaging/transports/rabbitmq/deadletterqueues.html#recovering-native-dead-letters-to-durable-storage). |
+
+It is exposed two ways:
+
+- `Endpoint.DeadLetterStorage` on the endpoint model.
+- `EndpointDescriptor.DeadLetterStorage` on the diagnostic descriptor surface that monitoring tools
+  (for example [CritterWatch](https://github.com/JasperFx/CritterWatch)) read.
+
+This lets a monitor detect endpoints that dead-letter **natively without recovery** (`Native`) and
+recommend enabling recovery so those dead letters become visible and replayable in the durable store.
+
 ## Dead Letter Expiration <Badge type="tip" text="3.9" />
 
 ::: tip

src/Testing/CoreTests/Configuration/endpoint_descriptor_dead_letter_storage_tests.cs

@@ -0,0 +1,47 @@
+using Shouldly;
+using Wolverine.Configuration;
+using Wolverine.Configuration.Capabilities;
+using Wolverine.Transports.Local;
+using Xunit;
+
+namespace CoreTests.Configuration;
+
+/// <summary>
+/// Locks down GH-3104: every endpoint declares a transport-agnostic
+/// <see cref="Endpoint.DeadLetterStorage"/>, lifted onto
+/// <see cref="EndpointDescriptor.DeadLetterStorage"/> as a first-class typed field so monitoring
+/// tools can introspect where an endpoint's dead letters go without transport-specific knowledge.
+/// Endpoints with no native dead letter queue (the core/local default) report
+/// <see cref="DeadLetterStorageMode.Durable"/>.
+/// </summary>
+public class endpoint_descriptor_dead_letter_storage_tests
+{
+    [Fact]
+    public void base_endpoint_default_is_durable()
+    {
+        var queue = new LocalQueue("dlq-default");
+
+        queue.DeadLetterStorage.ShouldBe(DeadLetterStorageMode.Durable);
+    }
+
+    [Fact]
+    public void descriptor_surfaces_dead_letter_storage()
+    {
+        var queue = new LocalQueue("dlq-descriptor");
+
+        var descriptor = new EndpointDescriptor(queue);
+
+        descriptor.DeadLetterStorage.ShouldBe(DeadLetterStorageMode.Durable);
+    }
+
+    [Fact]
+    public void does_not_duplicate_dead_letter_storage_in_properties()
+    {
+        var queue = new LocalQueue("dlq-no-dupes");
+
+        var descriptor = new EndpointDescriptor(queue);
+
+        // Promoted to a typed field — the generic Properties row must be gone so we don't double-ship.
+        descriptor.Properties.ShouldNotContain(x => x.Name == nameof(Endpoint.DeadLetterStorage));
+    }
+}

src/Transports/AWS/Wolverine.AmazonSqs.Tests/dead_letter_storage_contract.cs

@@ -0,0 +1,38 @@
+using Shouldly;
+using Wolverine.AmazonSqs.Internal;
+using Wolverine.Configuration;
+using Xunit;
+
+namespace Wolverine.AmazonSqs.Tests;
+
+/// <summary>
+/// Coverage for the transport-agnostic dead-letter-destination contract (GH-3104) as reported by SQS
+/// queues: a configured native dead letter queue reports <see cref="DeadLetterStorageMode.Native"/>,
+/// while disabling it (per-queue or globally) falls back to <see cref="DeadLetterStorageMode.Durable"/>.
+/// </summary>
+public class dead_letter_storage_contract
+{
+    [Fact]
+    public void native_by_default()
+    {
+        var transport = new AmazonSqsTransport();
+        transport.Queues["orders"].DeadLetterStorage.ShouldBe(DeadLetterStorageMode.Native);
+    }
+
+    [Fact]
+    public void durable_when_dead_lettering_disabled_for_the_queue()
+    {
+        var transport = new AmazonSqsTransport();
+        var queue = transport.Queues["orders"];
+        queue.DeadLetterQueueName = null;
+
+        queue.DeadLetterStorage.ShouldBe(DeadLetterStorageMode.Durable);
+    }
+
+    [Fact]
+    public void durable_when_all_native_dead_lettering_disabled()
+    {
+        var transport = new AmazonSqsTransport { DisableDeadLetterQueues = true };
+        transport.Queues["orders"].DeadLetterStorage.ShouldBe(DeadLetterStorageMode.Durable);
+    }
+}

src/Transports/AWS/Wolverine.AmazonSqs/Internal/AmazonSqsQueue.cs

@@ -467,4 +467,9 @@ public override bool TryBuildDeadLetterSender(IWolverineRuntime runtime, out ISe
         deadLetterSender = default;
         return false;
     }
+
+    public override DeadLetterStorageMode DeadLetterStorage =>
+        DeadLetterQueueName.IsNotEmpty() && !_parent.DisableDeadLetterQueues
+            ? DeadLetterStorageMode.Native
+            : DeadLetterStorageMode.Durable;
 }

src/Transports/Azure/Wolverine.AzureServiceBus.Tests/dead_letter_storage_contract.cs

@@ -0,0 +1,43 @@
+using Shouldly;
+using Wolverine.AzureServiceBus.Internal;
+using Wolverine.Configuration;
+using Xunit;
+
+namespace Wolverine.AzureServiceBus.Tests;
+
+/// <summary>
+/// Coverage for the transport-agnostic dead-letter-destination contract (GH-3104) as reported by
+/// Azure Service Bus endpoints: queues and subscriptions report
+/// <see cref="DeadLetterStorageMode.Native"/> by default (managed dead letter queue / native
+/// sub-queue), while disabling dead lettering on a queue falls back to
+/// <see cref="DeadLetterStorageMode.Durable"/>.
+/// </summary>
+public class dead_letter_storage_contract
+{
+    [Fact]
+    public void queue_is_native_by_default()
+    {
+        var transport = new AzureServiceBusTransport();
+        transport.Queues["orders"].DeadLetterStorage.ShouldBe(DeadLetterStorageMode.Native);
+    }
+
+    [Fact]
+    public void queue_is_durable_when_dead_lettering_disabled()
+    {
+        var transport = new AzureServiceBusTransport();
+        var queue = transport.Queues["orders"];
+        queue.DeadLetterQueueName = null;
+
+        queue.DeadLetterStorage.ShouldBe(DeadLetterStorageMode.Durable);
+    }
+
+    [Fact]
+    public void subscription_is_native()
+    {
+        var transport = new AzureServiceBusTransport();
+        var topic = transport.Topics["events"];
+        var subscription = new AzureServiceBusSubscription(transport, topic, "orders");
+
+        subscription.DeadLetterStorage.ShouldBe(DeadLetterStorageMode.Native);
+    }
+}

src/Transports/Azure/Wolverine.AzureServiceBus/Internal/AzureServiceBusQueue.cs

@@ -219,6 +219,14 @@ public override bool TryBuildDeadLetterSender(IWolverineRuntime runtime, out ISe
         deadLetterSender = default;
         return false;
     }
+
+    // Buffered/durable queues move failures to the managed dead letter queue; inline queues use the
+    // native $DeadLetterQueue sub-queue. Either way it's a native broker destination unless dead
+    // lettering was explicitly disabled (DeadLetterQueueName set to null), which falls back to
+    // Wolverine's durable storage.
+    public override DeadLetterStorageMode DeadLetterStorage => DeadLetterQueueName.IsNotEmpty()
+        ? DeadLetterStorageMode.Native
+        : DeadLetterStorageMode.Durable;
 
     // NServiceBus interop: NSB writes the .NET assembly-qualified type name
     // to the message header; we resolve it via Type.GetType(string). Type

src/Transports/Azure/Wolverine.AzureServiceBus/Internal/AzureServiceBusSubscription.cs

@@ -56,6 +56,9 @@ public AzureServiceBusSubscription(AzureServiceBusTransport parent, AzureService
     // so this property renders as the topic name string (per audit decision).
     public AzureServiceBusTopic Topic { get; }
 
+    // Subscriptions dead-letter to their native $DeadLetterQueue sub-queue.
+    public override DeadLetterStorageMode DeadLetterStorage => DeadLetterStorageMode.Native;
+
     public override ValueTask<IListener> BuildListenerAsync(IWolverineRuntime runtime, IReceiver receiver)
     {
         return Parent.BuildListenerForSubscription(runtime, receiver, this);

src/Transports/GCP/Wolverine.Pubsub/PubsubEndpoint.cs

@@ -320,6 +320,10 @@ public override ValueTask<IListener> BuildListenerAsync(IWolverineRuntime runtim
         ));
     }
 
+    public override DeadLetterStorageMode DeadLetterStorage => DeadLetterName.IsNotEmpty()
+        ? DeadLetterStorageMode.Native
+        : DeadLetterStorageMode.Durable;
+
     public override bool TryBuildDeadLetterSender(IWolverineRuntime runtime, out ISender? deadLetterSender)
     {
         EnvelopeMapper ??= BuildMapper(runtime);

src/Transports/Kafka/Wolverine.Kafka/KafkaTopic.cs

@@ -64,6 +64,11 @@ public override bool AutoStartSendingAgent()
     /// </summary>
     public bool NativeDeadLetterQueueEnabled { get; set; }
 
+    // Inherited by KafkaTopicGroup, which shares the same NativeDeadLetterQueueEnabled flag.
+    public override DeadLetterStorageMode DeadLetterStorage => NativeDeadLetterQueueEnabled
+        ? DeadLetterStorageMode.Native
+        : DeadLetterStorageMode.Durable;
+
     /// <summary>
     /// When true, the Kafka consumer group ID will be stamped onto the incoming
     /// envelope's GroupId property. Useful when you want the consumer group name

src/Transports/RabbitMQ/Wolverine.RabbitMQ.Tests/dead_letter_storage_contract.cs

@@ -0,0 +1,40 @@
+using Shouldly;
+using Wolverine.Configuration;
+using Wolverine.RabbitMQ.Internal;
+using Xunit;
+
+namespace Wolverine.RabbitMQ.Tests;
+
+/// <summary>
+/// Coverage for the transport-agnostic dead-letter-destination contract (GH-3104) as reported by
+/// RabbitMQ queues: native dead lettering reports <see cref="DeadLetterStorageMode.Native"/>,
+/// <c>EnableDeadLetterQueueRecovery()</c> promotes it to
+/// <see cref="DeadLetterStorageMode.NativeWithRecovery"/>, and <c>WolverineStorage</c> mode reports
+/// <see cref="DeadLetterStorageMode.Durable"/>.
+/// </summary>
+public class dead_letter_storage_contract
+{
+    [Fact]
+    public void native_by_default()
+    {
+        var transport = new RabbitMqTransport();
+        transport.Queues["orders"].DeadLetterStorage.ShouldBe(DeadLetterStorageMode.Native);
+    }
+
+    [Fact]
+    public void native_with_recovery_when_recovery_enabled()
+    {
+        var transport = new RabbitMqTransport { EnableDeadLetterQueueRecovery = true };
+        transport.Queues["orders"].DeadLetterStorage.ShouldBe(DeadLetterStorageMode.NativeWithRecovery);
+    }
+
+    [Fact]
+    public void durable_when_using_wolverine_storage_mode()
+    {
+        var transport = new RabbitMqTransport();
+        var queue = transport.Queues["orders"];
+        queue.DeadLetterQueue!.Mode = DeadLetterQueueMode.WolverineStorage;
+
+        queue.DeadLetterStorage.ShouldBe(DeadLetterStorageMode.Durable);
+    }
+}

src/Transports/RabbitMQ/Wolverine.RabbitMQ/Internal/RabbitMqQueue.cs

@@ -391,4 +391,22 @@ public override bool TryBuildDeadLetterSender(IWolverineRuntime runtime, out ISe
         deadLetterSender = default;
         return false;
     }
+
+    // Native and InteropFriendly modes route to a native RabbitMQ dead letter queue; WolverineStorage
+    // (and no DLQ) uses Wolverine's durable storage. EnableDeadLetterQueueRecovery() bridges the
+    // native queue back into durable storage.
+    public override DeadLetterStorageMode DeadLetterStorage
+    {
+        get
+        {
+            if (DeadLetterQueue is null || DeadLetterQueue.Mode == DeadLetterQueueMode.WolverineStorage)
+            {
+                return DeadLetterStorageMode.Durable;
+            }
+
+            return _parent.EnableDeadLetterQueueRecovery
+                ? DeadLetterStorageMode.NativeWithRecovery
+                : DeadLetterStorageMode.Native;
+        }
+    }
 }

src/Transports/Redis/Wolverine.Redis/Internal/RedisStreamEndpoint.cs

@@ -54,7 +54,11 @@ internal RedisStreamEndpoint(Uri uri, RedisTransport transport, EndpointRole rol
     /// Enable native dead letter queue support for this endpoint
     /// </summary>
     public bool NativeDeadLetterQueueEnabled { get; set; } = true;
-
+
+    public override DeadLetterStorageMode DeadLetterStorage => NativeDeadLetterQueueEnabled
+        ? DeadLetterStorageMode.Native
+        : DeadLetterStorageMode.Durable;
+
     /// <summary>
     /// The consumer group name for this endpoint (if listening)
     /// </summary>

src/Wolverine/Configuration/Capabilities/EndpointDescriptor.cs

@@ -42,12 +42,14 @@ public EndpointDescriptor(Endpoint endpoint) : base(endpoint)
         BrokerRole = endpoint.BrokerRole;
         Mode = endpoint.Mode;
         IsListener = endpoint.IsListener;
-
-        // Mode and IsListener are now first-class typed fields (GH-3009). Drop the generic
-        // OptionsDescription rows the base ctor reflected off the Endpoint so we don't ship
-        // them twice — CritterWatch reads the typed fields at the service-overview level, and
-        // Properties is becoming lazy-fetchable downstream.
-        Properties.RemoveAll(x => x.Name is nameof(Endpoint.Mode) or nameof(Endpoint.IsListener));
+        DeadLetterStorage = endpoint.DeadLetterStorage;
+
+        // Mode and IsListener are now first-class typed fields (GH-3009); DeadLetterStorage is
+        // likewise a typed field (GH-3104). Drop the generic OptionsDescription rows the base ctor
+        // reflected off the Endpoint so we don't ship them twice — CritterWatch reads the typed
+        // fields at the service-overview level, and Properties is becoming lazy-fetchable downstream.
+        Properties.RemoveAll(x =>
+            x.Name is nameof(Endpoint.Mode) or nameof(Endpoint.IsListener) or nameof(Endpoint.DeadLetterStorage));
     }
 
     public Uri Uri { get; set; } = null!;
@@ -120,6 +122,17 @@ public EndpointDescriptor(Endpoint endpoint) : base(endpoint)
     /// </summary>
     public bool IsListener { get; init; }
 
+    /// <summary>
+    /// Where this endpoint's dead letters effectively go — <see cref="DeadLetterStorageMode.Durable"/>
+    /// (Wolverine's durable, queryable store), <see cref="DeadLetterStorageMode.Native"/> (a native
+    /// broker dead letter queue, un-bridged), or <see cref="DeadLetterStorageMode.NativeWithRecovery"/>
+    /// (native, but bridged back into durable storage). A transport-agnostic contract (GH-3104) so
+    /// monitoring tools like CritterWatch can detect endpoints that dead-letter natively without
+    /// recovery — and recommend enabling it — without transport-specific knowledge. Lifted from
+    /// <see cref="Endpoint.DeadLetterStorage"/>.
+    /// </summary>
+    public DeadLetterStorageMode DeadLetterStorage { get; init; }
+
     internal static string? ResolveInteropMode(Endpoint endpoint)
     {
         // Mapper signal wins over serializer signal (last-usage-wins). If the user

src/Wolverine/Configuration/DeadLetterStorageMode.cs

@@ -0,0 +1,33 @@
+namespace Wolverine.Configuration;
+
+/// <summary>
+/// A transport-agnostic declaration of where an endpoint's dead letters effectively go. Surfaced on
+/// <see cref="Capabilities.EndpointDescriptor.DeadLetterStorage"/> so monitoring tools (for example
+/// CritterWatch) can introspect each endpoint's dead-letter destination without transport-specific
+/// knowledge — most importantly, to detect endpoints that dead-letter natively without being bridged
+/// into Wolverine's durable, queryable storage.
+/// </summary>
+public enum DeadLetterStorageMode
+{
+    /// <summary>
+    /// Dead letters are written to Wolverine's durable message store (the <c>wolverine_dead_letters</c>
+    /// table), where they are queryable and replayable through <c>IDeadLetters</c>. This is the
+    /// default for endpoints with no native broker dead letter queue.
+    /// </summary>
+    Durable,
+
+    /// <summary>
+    /// Dead letters are moved to a native broker dead letter queue and are <em>not</em> bridged into
+    /// Wolverine's durable storage. These dead letters are invisible to tools that manage the durable
+    /// dead letter queue — enabling native→durable recovery (see <c>EnableDeadLetterQueueRecovery()</c>)
+    /// promotes this to <see cref="NativeWithRecovery"/>.
+    /// </summary>
+    Native,
+
+    /// <summary>
+    /// Dead letters are moved to a native broker dead letter queue <em>and</em> a background recovery
+    /// listener copies them into Wolverine's durable storage, so they are both natively dead-lettered
+    /// and queryable/replayable through <c>IDeadLetters</c>.
+    /// </summary>
+    NativeWithRecovery
+}

src/Wolverine/Configuration/Endpoint.cs

@@ -679,6 +679,18 @@ public virtual bool TryBuildDeadLetterSender(IWolverineRuntime runtime, out ISen
         return false;
     }
 
+    /// <summary>
+    /// A transport-agnostic declaration of where this endpoint's dead letters effectively go —
+    /// Wolverine's durable store (<see cref="DeadLetterStorageMode.Durable"/>), a native broker dead
+    /// letter queue (<see cref="DeadLetterStorageMode.Native"/>), or a native queue bridged back into
+    /// durable storage (<see cref="DeadLetterStorageMode.NativeWithRecovery"/>). Monitoring tools read
+    /// this through <see cref="Capabilities.EndpointDescriptor.DeadLetterStorage"/> to detect
+    /// endpoints whose dead letters are native and un-bridged. The default is
+    /// <see cref="DeadLetterStorageMode.Durable"/>; transports with a native dead letter queue
+    /// override this.
+    /// </summary>
+    public virtual DeadLetterStorageMode DeadLetterStorage => DeadLetterStorageMode.Durable;
+
     internal bool ShouldAutoStartAsListener(DurabilitySettings durability)
     {
         if (!IsListener) return false;