diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 4cd70e0a7..09ac62e08 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -57,6 +57,16 @@ jobs: chmod 644 /usr/share/keyrings/redis-archive-keyring.gpg echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/redis.list apt-get update + # Don't let dpkg start the packaged redis-server service: we start our own + # instances below, and the packaged unit would otherwise race them for + # 127.0.0.1:6379 on any run where WSL came up with systemd. + # This also sidesteps an install hang we hit with the 8.10.0 package, whose + # redis.conf lost "supervised auto" - redis then never sd_notify()s, so the + # Type=notify unit never leaves "activating" and the postinst's + # "systemctl start" blocks. That one's an upstream packaging slip being + # fixed there, but the guard above is worth keeping either way. + printf '#!/bin/sh\nexit 101\n' > /usr/sbin/policy-rc.d + chmod +x /usr/sbin/policy-rc.d apt-get install -y redis mkdir redis - name: Run redis-server diff --git a/Directory.Build.props b/Directory.Build.props index d504cd70d..4c83282d3 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -11,7 +11,7 @@ $(MSBuildThisFileDirectory)Shared.ruleset NETSDK1069 - $(NoWarn);NU5105;NU1507;SER001;SER004;SER005;SER008 + $(NoWarn);NU5105;NU1507;SER001;SER004;SER005;SER007;SER008 https://stackexchange.github.io/StackExchange.Redis/ReleaseNotes https://stackexchange.github.io/StackExchange.Redis/ MIT diff --git a/Directory.Packages.props b/Directory.Packages.props index 88bb94c5e..c5261d0e1 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -18,7 +18,7 @@ - + diff --git a/docs/Configuration.md b/docs/Configuration.md index ce3ab93f6..c79082fa9 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -128,6 +128,20 @@ Additional code-only options: - **Note: heartbeats are not free and that's why the default is 1 second. There is additional overhead to running this more often simply because it does some work each time it fires.** - LibraryName - Default: `SE.Redis` (unless a `DefaultOptionsProvider` specifies otherwise) - The library name to use with `CLIENT SETINFO` when setting the library name/version on the connection +- IncludeDetailInExceptions - Default: `true` + - Whether exceptions include identifiable details (key names, and additional `.Data` annotations) +- Defaults (`DefaultOptionsProvider`) - Default: resolved from the configured endpoints + - The provider that supplies default values for options that have not been set explicitly (for example, recognized cloud endpoints can apply tuned defaults) +- IncludePerformanceCountersInExceptions - Default: `false` + - Whether exceptions include performance counter details (CPU usage, etc); note that this can be problematic on some platforms +- RequestBufferPool (`MemoryPool`) - Default: `null` + - The buffer pool to use when buffering requests; when `null`, a shared default pool is used +- ResponseBufferPool (`MemoryPool`) - Default: `null` + - The buffer pool to use when buffering responses (and for allocating `Lease` results); when `null`, a shared default pool is used +- CircuitBreaker (`CircuitBreaker`) - Default: `null` + - **[Experimental](exp/SER007)** (client-side geographic failover). A per-connection circuit breaker that *passively* observes the outcome of normal traffic and tears the connection down when it becomes unstable. When the connection is a member of a connection group, this flows in from `MultiGroupOptions.CircuitBreaker` if not set explicitly. See [Client-side geographic failover](Failover) +- HealthCheck (`HealthCheck`) - Default: `null` + - **[Experimental](exp/SER007)** (client-side geographic failover). An *active* health check used when the connection is a member of a connection group; when `null`, the group-level `MultiGroupOptions.HealthCheck` is used. See [Client-side geographic failover](Failover) Tokens in the configuration string are comma-separated; any without an `=` sign are assumed to be redis server endpoints. Endpoints without an explicit port will use 6379 if ssl is not enabled, and 6380 if ssl is enabled. Tokens starting with `$` are taken to represent command maps, for example: `$config=cfg`. diff --git a/docs/Failover.md b/docs/Failover.md new file mode 100644 index 000000000..d3b742555 --- /dev/null +++ b/docs/Failover.md @@ -0,0 +1,1117 @@ +# Client-side geographic failover + +> **What this is, and what it is called elsewhere.** This feature does one thing: it moves traffic between several independent endpoints that can each serve +> your workload, using health checks, circuit breakers and retries to decide when to move. Nothing in it is tied to a particular server topology, which is why +> the naming here is about the *behaviour* (failover between redundant endpoints) rather than about any specific deployment. +> +> The deployment it is **designed** for is **Active-Active** (geo-distributed, bidirectionally-replicated) Redis - Redis Software / Redis Cloud Active-Active +> databases, or any other topology where the same logical dataset is reachable through more than one independent endpoint. That is where failover is closest to +> transparent: because the members replicate to each other, an operation that lands on the new endpoint still sees what you wrote to the old one. +> +> It works just as well across endpoints that are *not* replicated to each other - separate OSS servers or clusters, instances in different regions with no link +> between them - provided you accept what that means for your data. With no replication between members, each endpoint holds only what was written to it: a +> failover moves you to a *different* dataset, writes made before the failover are not visible after it, and the two do not converge afterwards. That is +> perfectly reasonable for a cache you are content to re-populate, or where the data is reconstructible from a system of record; it is not reasonable if you are +> treating the group as a single durable store. Only you can make that call - the library has no way to know which situation it is in. +> +> The shape will be familiar if you have used the **"multi-DB client"** concept in other Redis client libraries: Jedis' +> [`MultiDbClient`](https://redis.io/docs/latest/develop/clients/jedis/failover/) and redis-py's +> [`MultiDBClient`](https://redis.readthedocs.io/en/stable/multi_database.html) (see also Redis' overview of +> [client-side geographic failover for Active-Active](https://redis.io/blog/client-side-geographic-failover-for-redis-active-active/)) - a weighted set of +> endpoints, one active at a time, with health checks, circuit breakers and retries deciding when to move. + +## Overview + +Client-side geographic failover provides automatic failover, failback, and intelligent routing across multiple Redis deployments. It is built from several cooperating pieces: + +1. **Connecting to multiple servers or regions at once**, giving a deployment redundant endpoints to fall back on. +2. **Health checks** that *actively* probe each endpoint on a timer to monitor its availability. +3. **Circuit breakers** that *passively* monitor availability from the observed success and failure of the traffic already flowing. +4. **Automatic retries** that let straightforward operations ride out a possibly-unstable connection — including silently transitioning to another endpoint during a failover, with no change to your calling code. + +These features are available in the `Availability` sub-namespace: + +``` csharp +using StackExchange.Redis; +using StackExchange.Redis.Availability; +``` + +### How the configuration types work + +Every configurable piece in this namespace follows the same three-part shape, so once you have learned one you have learned all of them: + +1. The **policy type** (`HealthCheck`, `CircuitBreaker`, `RetryPolicy`) is **immutable** and safe to share between members and connections. It exposes a static `Default` and a static `None`. +2. Each policy has a nested **`Builder`** carrying the mutable knobs. A new `Builder` already starts from the default values, so you only set what you want to change; `Create()` validates the values and returns the policy, and a `Builder` also converts *implicitly* to its policy, so it can be assigned or passed inline. +3. **`MultiGroupOptions`** (itself immutable, with its own `Builder`) holds the group-wide defaults; the matching nullable property on `ConnectionGroupMember` overrides them per member. + +```csharp +// the same pattern, three times; each builder only mentions what differs from the default +HealthCheck healthCheck = new HealthCheck.Builder { ProbeCount = 5 }; +CircuitBreaker breaker = new CircuitBreaker.Builder { FailureRateThreshold = 25 }; +RetryPolicy retry = new RetryPolicy.Builder { MaxAttempts = 5 }; + +MultiGroupOptions options = new MultiGroupOptions.Builder +{ + HealthCheck = healthCheck, + CircuitBreaker = breaker, + RetryPolicy = retry, + HealthCheckInterval = TimeSpan.FromSeconds(2), +}; +``` + +Anything you leave out keeps its default, so a group that only wants a longer failback is just: + +```csharp +MultiGroupOptions options = new MultiGroupOptions.Builder { FailbackDelay = TimeSpan.FromMinutes(2) }; +``` + +Because the policies are immutable, there is no question of whether a change "takes effect" after connecting: to change something, build a new instance. Values are validated once, in `Create()`, which throws `ArgumentOutOfRangeException`/`ArgumentException` naming the offending builder property - so a bad `ProbeCount` or `MaxAttempts` fails at the point you configure it, not later. + +### Where each setting lives + +| Setting | Group-wide default | Per-member override | +|---------|--------------------|---------------------| +| `HealthCheck` | `MultiGroupOptions.HealthCheck` | `ConnectionGroupMember.HealthCheck` | +| `CircuitBreaker` | `MultiGroupOptions.CircuitBreaker` | `ConnectionGroupMember.CircuitBreaker`, else that member's `ConfigurationOptions.CircuitBreaker` | +| `RetryPolicy` | `MultiGroupOptions.RetryPolicy` | *(none; retry is applied per-database via `WithRetry`)* | +| `HealthCheckInterval` | `MultiGroupOptions.HealthCheckInterval` | *(none; it is the group's re-evaluation cadence)* | +| `FailbackDelay` | `MultiGroupOptions.FailbackDelay` | `ConnectionGroupMember.FailbackDelay` | +| `Weight` | *(none)* | `ConnectionGroupMember.Weight` | +| `SkipInitialHealthCheck` | *(none)* | `ConnectionGroupMember.SkipInitialHealthCheck` | + +Resolution is always "member override, else group default". Resolving a group default never writes back into your `ConfigurationOptions` - you can safely reuse one `ConfigurationOptions` for a group member and for an unrelated `Connect` without the group's policies leaking across. + +`ConfigurationOptions` carries only the two availability settings that are meaningful for a **single** connection - `CircuitBreaker` and `RetryPolicy`. `HealthCheck` is a group-only concept and lives on `ConnectionGroupMember`. +The library automatically selects the best available endpoint based on: + +1. **Availability** - Connected endpoints are always preferred over disconnected ones +2. **Weight** - User-defined preference values (higher is better) +3. **Latency** - Measured response times (lower is better) + +This enables scenarios such as: +- Multi-datacenter deployments with automatic failover +- Geographic routing to the nearest Redis instance +- Graceful degradation during maintenance or outages +- Load distribution across multiple Redis clusters + +## Basic Usage + +### Connecting to Multiple Groups + +To create a failover-capable connection, use `ConnectionMultiplexer.ConnectGroupAsync()` with an array of `ConnectionGroupMember` instances: + +```csharp +// Define your Redis endpoints +ConnectionGroupMember[] members = [ + new("us-east.redis.example.com:6379", name: "US East"), + new("us-west.redis.example.com:6379", name: "US West"), + new("eu-central.redis.example.com:6379", name: "EU Central") +]; + +// Connect to all members +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); + +// Use the connection normally +var db = conn.GetDatabase(); +await db.StringSetAsync("mykey", "myvalue"); +var value = await db.StringGetAsync("mykey"); +``` + +### Using ConfigurationOptions + +You can also use `ConfigurationOptions` for more advanced configuration: + +```csharp +var eastConfig = new ConfigurationOptions +{ + EndPoints = { "us-east-1.redis.example.com:6379", "us-east-2.redis.example.com:6379" }, + Password = "your-password", + Ssl = true, +}; + +var westConfig = new ConfigurationOptions +{ + EndPoints = { "us-west-1.redis.example.com:6379", "us-west-2.redis.example.com:6379" }, + Password = "another-different-password", + Ssl = true, +}; + +ConnectionGroupMember[] members = [ + new(eastConfig, name: "US East"), + new(westConfig, name: "US West") +]; + +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); +``` + +## Configuring Weights + +Weights allow you to express preference for specific endpoints. Higher weights are preferred when multiple endpoints are available: + +```csharp +ConnectionGroupMember[] members = [ + new("local-dc.redis.example.com:6379") { Weight = 10 }, // Strongly preferred + new("nearby-dc.redis.example.com:6379") { Weight = 5 }, // Moderately preferred + new("remote-dc.redis.example.com:6379") { Weight = 1 } // Fallback option +]; + +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); +``` + +Weights can be adjusted dynamically: + +```csharp +// Adjust weight based on runtime conditions +members[0].Weight = 1; // Reduce preference for local DC +members[2].Weight = 10; // Increase preference for remote DC +``` + +## Working with IDatabase + +The `IDatabase` interface works transparently with connection groups. All operations are automatically routed to the currently selected endpoint: + +```csharp +var db = conn.GetDatabase(); + +// String operations +await db.StringSetAsync("user:1:name", "Alice"); +var name = await db.StringGetAsync("user:1:name"); + +// Hash operations +await db.HashSetAsync("user:1", new HashEntry[] { + new("name", "Alice"), + new("email", "alice@example.com") +}); + +// List operations +await db.ListRightPushAsync("queue:tasks", "task1"); +var task = await db.ListLeftPopAsync("queue:tasks"); + +// Set operations +await db.SetAddAsync("tags", new RedisValue[] { "redis", "cache", "database" }); +var members = await db.SetMembersAsync("tags"); + +// Sorted set operations +await db.SortedSetAddAsync("leaderboard", "player1", 100); +var rank = await db.SortedSetRankAsync("leaderboard", "player1"); + +// Transactions +var tran = db.CreateTransaction(); +var t1 = tran.StringSetAsync("key1", "value1"); +var t2 = tran.StringSetAsync("key2", "value2"); +if (await tran.ExecuteAsync()) +{ + await t1; + await t2; +} + +// Batches +var batch = db.CreateBatch(); +var b1 = batch.StringSetAsync("key1", "value1"); +var b2 = batch.StringSetAsync("key2", "value2"); +batch.Execute(); +await Task.WhenAll(b1, b2); +``` + +## Working with ISubscriber + +Pub/Sub operations work across all connected endpoints. When you subscribe to a channel, the subscription is established against *all* endpoints (for immediate pickup +during failover events), and received messages are filtered in the library so only the messages for the *active* endpoint are observed. Message publishing +occurs only to the *active* endpoint. The effect of this is that pub/sub works transparently as though +you were only talking to the *active* endpoint: + +```csharp +var subscriber = conn.GetSubscriber(); + +// Subscribe to a channel +await subscriber.SubscribeAsync(RedisChannel.Literal("notifications"), (channel, message) => +{ + Console.WriteLine($"Received: {message}"); +}); + +// Publish to a channel +await subscriber.PublishAsync(RedisChannel.Literal("notifications"), "Hello, World!"); + +// Pattern-based subscriptions +await subscriber.SubscribeAsync(RedisChannel.Pattern("events:*"), (channel, message) => +{ + Console.WriteLine($"Event on {channel}: {message}"); +}); + +// Unsubscribe +await subscriber.UnsubscribeAsync(RedisChannel.Literal("notifications")); +``` + +**Note:** When the active endpoint changes (due to failover), subscriptions are automatically re-established on the new endpoint. + +## Monitoring Connection Changes + +You can monitor when the active connection changes using the `ConnectionChanged` event: + +```csharp +conn.ConnectionChanged += (sender, args) => +{ + Console.WriteLine($"Connection changed: {args.Type}"); + Console.WriteLine($"Previous: {args.PreviousGroup?.Name ?? "(none)"}"); + Console.WriteLine($"Current: {args.Group.Name}"); +}; +``` + +## Monitoring Member Status + +Each `ConnectionGroupMember` provides status information: + +```csharp +foreach (var member in conn.GetMembers()) +{ + Console.WriteLine($"{member.Name}:"); + Console.WriteLine($" Connected: {member.IsConnected}"); + Console.WriteLine($" Unhealthy: {member.IsUnhealthy}"); + Console.WriteLine($" Weight: {member.Weight}"); + Console.WriteLine($" Latency: {member.Latency}"); +} +``` + +These are the same instances that were passed into `ConnectGroupAsync`. + +## Health Checks + +Configurable health checking monitors the health of all endpoints and automatically routes traffic away from unhealthy instances. + +### Basic Health Check Configuration + +Health checks are configured for all members via `MultiGroupOptions`, and can be overridden per member. Every knob is shown here for orientation, with **the value it already defaults to** - in real code you would set only the ones you want to change: + +```csharp +HealthCheck healthCheck = new HealthCheck.Builder +{ + ProbeCount = 3, // Maximum number of probe attempts per check + ProbeTimeout = TimeSpan.FromSeconds(3), // Timeout for each probe attempt + ProbeInterval = TimeSpan.FromMilliseconds(500), // Delay between failed probes + Probe = HealthCheckProbe.Ping, // Which probe type to use + ProbePolicy = HealthCheckProbePolicy.AllSuccess // Evaluation policy +}; + +MultiGroupOptions options = new MultiGroupOptions.Builder +{ + HealthCheck = healthCheck, + HealthCheckInterval = TimeSpan.FromSeconds(5), // How often checks run (a group-level concern) +}; + +ConnectionGroupMember[] members = [ + new("us-east.redis.example.com:6379", name: "US East") { Weight = 10 }, + new("us-west.redis.example.com:6379", name: "US West") { Weight = 5 } +]; + +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); +``` + +Note that **how often** checks run is `MultiGroupOptions.HealthCheckInterval`, not a property of the check: it is the cadence at which the group re-evaluates *all* members, so a per-member value would be meaningless. + +### Using Default Health Checks + +If you don't specify a health check, the system uses sensible defaults: + +```csharp +// Uses default health check settings +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); + +// Equivalent to: +MultiGroupOptions options = new MultiGroupOptions.Builder { HealthCheck = HealthCheck.Default }; +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); +``` + +A new `Builder` starts from the defaults, so customizing means setting only what differs: + +```csharp +HealthCheck customHealthCheck = new HealthCheck.Builder { ProbeCount = 5 }; // everything else defaulted + +MultiGroupOptions options = new MultiGroupOptions.Builder +{ + HealthCheck = customHealthCheck, + HealthCheckInterval = TimeSpan.FromSeconds(15), +}; +``` + +There is also a `Builder(policy)` overload, for when you want to start from an instance that *isn't* the default - for example, adjusting a policy you were handed: + +```csharp +// take the group's configured check and probe it harder for one member +HealthCheck stricter = new HealthCheck.Builder(conn.Options.HealthCheck) { ProbeCount = 9 }; +``` + +### Health Check Properties + +The `HealthCheck.Builder` class provides several configurable properties: + +| Property | Default | Description | +|----------|---------|-------------| +| `ProbeCount` | 3 | Number of probe operations to perform per health check | +| `ProbeTimeout` | 3 seconds | Maximum time allowed for an individual probe to complete | +| `ProbeInterval` | 500 milliseconds | Delay between consecutive failed probes | +| `Probe` | `Ping` | The probe operation to execute | +| `ProbePolicy` | `AllSuccess` | Policy for evaluating multiple probe results | + +### Per-member health checks, and turning them off + +A member can use its own check - including `HealthCheck.None`, which performs no probes at all and reports `Inconclusive`, leaving that member's selection driven purely by its observed connectivity (and by its circuit-breaker): + +```csharp +ConnectionGroupMember[] members = [ + new("us-east.redis.example.com:6379", name: "US East") { Weight = 10 }, + + // a member we only want to reach for on connectivity grounds - never probe it + new("archive.redis.example.com:6379", name: "Archive") { Weight = 1, HealthCheck = HealthCheck.None }, + + // ...and one we want checked much more aggressively than the rest + new("us-west.redis.example.com:6379", name: "US West") + { + Weight = 5, + HealthCheck = new HealthCheck.Builder { ProbeCount = 5, ProbePolicy = HealthCheckProbePolicy.MajoritySuccess }, + }, +]; +``` + +To disable periodic checking for the whole group, set `MultiGroupOptions.HealthCheckInterval` to `TimeSpan.MaxValue`; the group is then only re-evaluated in response to connection events (such as a tripped circuit-breaker). + +### Built-in Probes + +StackExchange.Redis provides several built-in health check probes: + +#### HealthCheckProbe.Ping + +The simplest probe that executes a `PING` command against the server: + +```csharp +HealthCheck healthCheck = new HealthCheck.Builder +{ + Probe = HealthCheckProbe.Ping +}; +``` + +This is the default and recommended probe for most scenarios as it's lightweight and tests basic connectivity. + +#### HealthCheckProbe.IsConnected + +Checks the connection status without sending any commands: + +```csharp +HealthCheck healthCheck = new HealthCheck.Builder +{ + Probe = HealthCheckProbe.IsConnected +}; +``` + +This is even more lightweight than `Ping` but only verifies the socket connection, not Redis responsiveness. + +#### HealthCheckProbe.StringSet + +Performs a write operation to verify read/write capability: + +```csharp +HealthCheck healthCheck = new HealthCheck.Builder +{ + Probe = HealthCheckProbe.StringSet +}; +``` + +This probe writes a random value to a health check key and verifies it can be retrieved. It's more comprehensive but has higher overhead than `Ping`. Note that this probe automatically skips replica servers. + +### Health Check Policies + +The probe policy determines how multiple probe results are evaluated to determine overall health: + +#### HealthCheckProbePolicy.AnySuccess + +The health check passes if **any** probe succeeds. This provides the most lenient evaluation: + +```csharp +HealthCheck healthCheck = new HealthCheck.Builder +{ + ProbeCount = 3, + ProbePolicy = HealthCheckProbePolicy.AnySuccess +}; +// Healthy if 1 or more of 3 probes succeed +``` + +#### HealthCheckProbePolicy.AllSuccess (Default) + +The health check passes only if **all** probes succeed. This provides the strictest evaluation: + +```csharp +HealthCheck healthCheck = new HealthCheck.Builder +{ + ProbeCount = 3, + ProbePolicy = HealthCheckProbePolicy.AllSuccess +}; +// Healthy only if all 3 probes succeed +``` + +#### HealthCheckProbePolicy.MajoritySuccess + +The health check passes if a **majority** of probes succeed: + +```csharp +HealthCheck healthCheck = new HealthCheck.Builder +{ + ProbeCount = 3, + ProbePolicy = HealthCheckProbePolicy.MajoritySuccess +}; +// Healthy if 2 or more of 3 probes succeed +``` + +### Health Check Behavior + +When a health check fails for a member: +- The member is flagged **unhealthy** (`IsUnhealthy`); this is distinct from `IsConnected` (see *Unhealthy State and Failback* below) +- Traffic is automatically routed to other healthy members based on weight and latency +- The system continues to perform health checks on the unhealthy member +- Once the member recovers and passes health checks, traffic automatically resumes (subject to `FailbackDelay`) + +### Best Practices + +1. **Choose appropriate probe types**: Use `Ping` for most scenarios; use `StringSet` when you need to verify write capability +2. **Balance probe frequency**: More frequent checks provide faster failover but increase load on your Redis servers +3. **Match policy to requirements**: Use `AnySuccess` for resilience, `AllSuccess` for strict validation, `MajoritySuccess` for balance +4. **Increase probe count for critical systems**: More probes with `MajoritySuccess` reduces false positives from transient failures +5. **Set reasonable timeouts**: Ensure `ProbeTimeout` accounts for network latency to your Redis servers +6. **Consider replica behavior**: Write-based probes automatically skip replicas to avoid false negatives + +## Circuit Breakers + +Where health checks *actively* probe each member on a timer, a **circuit breaker** works *passively*: it observes the outcome of the normal traffic already flowing over a connection, and tears that connection down as soon as it decides the connection has become unstable. The two are complementary: + +- A **health check** answers *"is this member reachable?"* by sending its own probes every `Interval`. +- A **circuit breaker** answers *"is the traffic I'm already sending actually succeeding?"* with no extra round-trips, reacting the instant real commands start failing. + +When a circuit breaker trips, it shuts the underlying physical connection down with `ConnectionFailureType.CircuitBreaker`. The connection then reconnects as usual, and - in a connection group - the health check and member-selection logic route traffic to other members until the affected member recovers. A circuit breaker is evaluated **per connection**, so +its state is scoped to exactly the connection whose health it is measuring; a reconnect starts from a clean slate - but breaking the connection is sufficient for the group to notice non-availability. + +### Configuring a Circuit Breaker for a Group + +Circuit breakers are configured for all members via `MultiGroupOptions`, alongside the health check. The setting flows into every member connection: + +```csharp +MultiGroupOptions options = new MultiGroupOptions.Builder +{ + CircuitBreaker = new CircuitBreaker.Builder { FailureRateThreshold = 25 } +}; + +ConnectionGroupMember[] members = [ + new("us-east.redis.example.com:6379", name: "US East") { Weight = 10 }, + new("us-west.redis.example.com:6379", name: "US West") { Weight = 5 } +]; + +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); +``` + +If you don't specify one, `CircuitBreaker.Default` is used automatically: + +```csharp +// these are equivalent +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); + +MultiGroupOptions options = new MultiGroupOptions.Builder { CircuitBreaker = CircuitBreaker.Default }; +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); +``` + +A circuit breaker is also useful *without* a group: set `ConfigurationOptions.CircuitBreaker` and any connection will tear itself down when its traffic starts failing. For a group member, the effective breaker is the member's `CircuitBreaker`, else the member's own `ConfigurationOptions.CircuitBreaker`, else the group default. + +### Tuning the Default Circuit Breaker + +The default circuit breaker uses a rolling time-window: it counts successes and failures over a short window and trips once the failure rate crosses a +threshold, provided enough failures have been seen to be statistically meaningful. Use `CircuitBreaker.Builder` to tune it: + +```csharp +MultiGroupOptions options = new MultiGroupOptions.Builder +{ + CircuitBreaker = new CircuitBreaker.Builder + { + FailureRateThreshold = 25, // trip above 25% failures + MinimumNumberOfFailures = 100, // ...but only after 100 failures in the window + MetricsWindowSize = TimeSpan.FromSeconds(5), // rolling window to measure over + } +}; +``` + +A `Builder` converts implicitly to a `CircuitBreaker`, so it can be assigned directly as above; calling `.Create()` explicitly is equivalent. + +| Property | Default | Description | +|----------|---------|-------------| +| `FailureRateThreshold` | 10 | Percentage of failures within the window that trips the breaker | +| `MinimumNumberOfFailures` | 1000 | Minimum tracked failures in the window before the breaker can trip (avoids acting on tiny samples) | +| `MetricsWindowSize` | 2 seconds | Rolling window over which successes and failures are counted | + +Which faults count against the breaker is decided by *classification*, not by exception type: transient and connection-level errors (and timeouts) count as failures, while application-level errors — for example a `RedisServerException` for a bad command, or a `WRONGTYPE` — are treated as a success for circuit-breaking purposes, since they are not a sign of an unhealthy *connection*. This is derived from the fault's `RedisErrorKind`; to change what counts, override `IsFailure(in FaultContext)` on a custom accumulator (see *Custom circuit breakers* below). + +### Disabling the Circuit Breaker + +Use `CircuitBreaker.None` to opt out entirely: + +```csharp +MultiGroupOptions options = new MultiGroupOptions.Builder +{ + CircuitBreaker = CircuitBreaker.None +}; +``` + +## Automatic Retries + +Health checks and circuit breakers keep the *group* pointed at a healthy member; **automatic retries** deal with the *individual operation* that was in flight when something went wrong. Wrapping a database with `WithRetry(...)` returns a database that transparently re-issues failed operations according to a `RetryPolicy` — riding out transient faults, and (in a connection group) following a failover across to another member, without the caller having to catch-and-retry by hand. + +```csharp +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); + +// wrap the database once; reuse the wrapper like any other IDatabaseAsync. +// the parameterless overload uses the policy configured on the connection (see below) +IDatabaseAsync db = conn.GetDatabase().WithRetry(); + +// a transient fault (e.g. the active member briefly returning LOADING) is retried +// automatically; if the group fails over in the meantime, the retry lands on the new member +var value = await db.StringGetAsync("mykey"); +``` + +> You can call `WithRetry` on any database (`IDatabase` or `IDatabaseAsync`), but the wrapper it returns exposes only the **async** API — there is no synchronous form, since retrying may inherently have delays. It cannot wrap a batch or an existing transaction, nor an already-retrying database. + +> **`asyncState` is not respected.** A database's `asyncState` is stamped onto the task produced by a single dispatch, but a retrying database hands back its own task spanning however many attempts the operation takes, and the per-operation tasks from `retryDb.CreateTransaction()` are durable proxies that outlive any single attempt. Neither can carry it. Rather than dropping the state silently, both refuse it: `conn.GetDatabase(0, asyncState).WithRetry(policy)` throws `InvalidOperationException`, as does `retryDb.CreateTransaction(asyncState)`. Wrap a database obtained without an `asyncState` - note that a key-prefixed view of a state-carrying database is refused too, since it inherits the inner state. + +A retrying database can still *create* a transaction: `retryDb.CreateTransaction()` returns an `ITransactionAsync` whose `ExecuteAsync` is retried as a single unit. Each attempt replays the queued operations (and any `WATCH` constraints) against a fresh `MULTI`/`EXEC` - and, in a connection group, onto whichever member is active at the time - so a transaction can ride out a failover just like a single command; the per-operation tasks handed back at build time resolve from the winning attempt. The retry-category gate (below) applies to the transaction as a whole, using the *most* side-effecting operation in it: a transaction containing an `INCR` counts as `CommandRetryWriteAccumulating`. In practice that gate rarely blocks a transaction, because the faults that a transaction is most likely to hit - a `MULTI`/`EXEC` the server *rejected* wholesale - are known not to have applied anything, and the category is not consulted in that case (see [Known-not-applied faults](#known-not-applied-faults)). + +#### Losing a `WATCH` race + +A transaction with conditions can fail to commit in two different ways, and `Execute`/`ExecuteAsync` reports `false` for both. `ITransaction.WasWatchConflict` (also on `ITransactionAsync`, so it works for retrying transactions too) is what tells them apart: + +- **a condition was not satisfied** - the transaction is abandoned electively, and no `EXEC` is ever sent. `WasWatchConflict` is `false`, and the offending `ConditionResult.WasSatisfied` is `false` too. Re-attempting is pointless: the value really was not what you asserted, so a replay would assert the same thing again and fail the same way. +- **another connection modified a watched key** - between this connection's conditions being evaluated and its `EXEC` arriving, some *other* client wrote to one of the keys being watched on behalf of those conditions, so the server refused the `EXEC`. `WasWatchConflict` is `true`, and `WasSatisfied` is `true` for every condition: your assertions were all correct, you just lost a race with a concurrent writer. This is contention, not a fault - nothing was applied and nothing is broken. + +Note that only *other connections* can cause this. Your own writes on this connection cannot: everything you queue inside the transaction is applied atomically by the `EXEC` itself, after the watch has already been satisfied. + +The second case is the one Redis's `WATCH`/`MULTI`/`EXEC` idiom expects you to retry, so a retrying transaction does exactly that, bounded by `MaxAttemptsOnWatchConflict` (default 3). Each re-attempt re-issues the constraints, so a transaction whose condition has genuinely stopped holding (because the concurrent writer left a value you were not expecting) converges on an ordinary elective abort rather than looping. Because it is contention rather than a fault it gets its own budget: `MaxAttempts` is untouched, `RetryDelay` is not applied (only `JitterMax`), no failover is attempted, and `MaxCommandRetryCategory` does not apply - nothing happened, so there is nothing to double-apply. Set `MaxAttemptsOnWatchConflict = 1` to restore the plain "report `false` and let me deal with it" behaviour. On a retrying transaction, `WasWatchConflict` describes the *final* attempt: `false` if it eventually committed, `true` if it ran out of attempts still losing the race. + +In either case every per-operation task is cancelled, so `await`ing one throws `OperationCanceledException` rather than hanging. + +How likely is this in practice? Much less so than in `redis-cli`-style usage, where a `WATCH` can sit open for as long as the application takes to think. SE.Redis buffers the whole transaction and sends the `WATCH`, the condition reads, the `MULTI`, the queued commands and the `EXEC` as a single dispatch on a multiplexed connection, so a competing writer has only the gap between the condition reads and the `EXEC` landing to squeeze into. Small - but not zero, and on a busy key it will happen. + +### Configuring the retry policy + +Like the health check and the circuit breaker, `RetryPolicy` can be set as a group-wide default - and the parameterless `WithRetry()` picks it up, so callers don't have to thread a policy through their code: + +```csharp +MultiGroupOptions options = new MultiGroupOptions.Builder +{ + RetryPolicy = new RetryPolicy.Builder { MaxAttempts = 5 }, +}; +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + +// uses MultiGroupOptions.RetryPolicy +IDatabaseAsync db = conn.GetDatabase().WithRetry(); +``` + +The same works for a single connection via `ConfigurationOptions.RetryPolicy`. `WithRetry()` resolves in this order: `MultiGroupOptions.RetryPolicy` for a connection group, `ConfigurationOptions.RetryPolicy` for a single connection, else `RetryPolicy.Default`. Pass a policy explicitly - `WithRetry(policy)` - to override that for one database, and use `RetryPolicy.None` to get a wrapper that never retries. + +### RetryPolicy settings + +`RetryPolicy.Builder` controls how many times, how often, and how far an operation is retried: + +| Property | Default | Description | +|----------|---------|-------------| +| `MaxAttempts` | 3 | Total attempts (including the first) before giving up | +| `MaxAttemptsBeforeFailover` | 1 | Attempts against the current member before a retry is allowed to wait for / move to a failover member (only meaningful for multi-group connections) | +| `RetryDelay` | 1 second | Delay between same-server retries | +| `JitterMax` | 0.5 seconds | Upper bound of the additional random delay added to each retry, to avoid stampedes | +| `FailoverDelay` | 5 seconds | Maximum time to wait for a failover, when a retry is gated on one happening | +| `MaxCommandRetryCategory` | `CommandRetryWriteLastWins` | The most side-effecting command category that will be retried (see below) | +| `MaxAttemptsOnWatchConflict` | 3 | Attempts allowed for a *conditional transaction* that keeps losing a `WATCH` race; separate from `MaxAttempts`, and `1` disables re-attempting | + +```csharp +RetryPolicy policy = new RetryPolicy.Builder +{ + MaxAttempts = 5, + RetryDelay = TimeSpan.FromMilliseconds(200), + JitterMax = TimeSpan.FromMilliseconds(100), +}; +IDatabaseAsync db = conn.GetDatabase().WithRetry(policy); +``` + +Only *transient* faults are retried — the same `RedisErrorKind`-based classification the default circuit breaker uses. An application-level error such as `WRONGTYPE`, or an unknown command, is not transient and is never retried, no matter what the policy says. + +### Which operations are safe to retry + +Retrying is not free of consequence: replaying `INCR` after an ambiguous failure could double-count, whereas replaying `GET` is harmless; `SET` is "last wins", so: *usally* fine. Every command therefore carries a **retry category** describing its side-effects, and a policy only retries commands at or below its `MaxCommandRetryCategory`. + +For the built-in typed methods (`StringGet`, `StringSet`, `HashSet`, ...) the library assigns the appropriate category automatically, so retries "just work" within the default policy. + +#### Known-not-applied faults + +The category prices the *ambiguity* of a replay, not the write itself. If we know the operation never took effect, re-issuing it is a first attempt rather than a repeat: it cannot double-apply anything, so the category is not consulted at all and even an `INCR` is retried under the default policy. Two things give us that certainty: + +- **the client never wrote it** - the message was still waiting to be sent, or sitting in the backlog, when the connection failed. +- **the server explicitly rejected it because of its own state** - `LOADING`, `CLUSTERDOWN`, `MASTERDOWN`, `TRYAGAIN`, `MOVED`, `ASK`, `READONLY`, `MISCONF`, `NOREPLICAS`, `BUSY`, `max clients`. The server can only report these *before* running anything. + +`FaultContext.NotApplied` exposes this to custom policies. It is deliberately conservative, and in particular a bare error reply is *not* enough on its own: a Lua script can fail part-way through having already written something, and it propagates the inner error (`WRONGTYPE`, `OOM`, ...) verbatim, so those kinds stay ambiguous. Timeouts, and connection loss after the request was sent, are always ambiguous - which is exactly where `MaxCommandRetryCategory` earns its keep. + +An explicit `CommandRetryNever` is still an absolute veto, as is an operation with no category at all (see below): certainty about *whether* it ran does not tell us that re-running it is meaningful. + +### Custom commands: `Execute` and `ScriptEvaluate` + +The library cannot infer the side-effects of a command it doesn't recognise — and that includes arbitrary commands issued via `Execute`/`ExecuteAsync`, and Lua run via `ScriptEvaluate`/`ScriptEvaluateAsync` (whose effect depends entirely on the script). Such commands are therefore treated **pessimistically**: an uncategorised command defaults to `CommandRetryNever` and is *not* retried. + +Note that `Execute`/`ExecuteAsync` do try to *parse* the command name first, so `Execute("get", key)` is recognised as `GET` and picks up that command's category (read-only) automatically; only genuinely unrecognised command names fall back to `CommandRetryNever`. + +The categories, from safest to most dangerous, are: + +| `CommandFlags` value | Meaning | +|----------------------|---------| +| `CommandRetryAlways` | Always safe to retry, regardless of connection/server state | +| `CommandRetryConnection` | Connection-level or safe metadata (e.g. `CLIENT SETNAME`, `CONFIG GET`) | +| `CommandRetryReadOnly` | Pure reads (e.g. `GET`) | +| `CommandRetryWriteChecked` | Conditional writes (e.g. `SETNX`, `SET ... IFEQ`) | +| `CommandRetryWriteLastWins` | Unconditional overwrite — last-writer-wins (e.g. `SET`) | +| `CommandRetryWriteAccumulating` | Cumulative writes where a retry can double-apply (e.g. `INCR`, `LPUSH`) | +| `CommandRetryServerAdmin` | Server administration (e.g. `CONFIG SET`) | +| `CommandRetryNever` | Never retry | + + +When possible when using ad-hoc commands or script, callers should supply the most appropriate `CommandRetry*` category in the command's `CommandFlags`: + +```csharp +// an arbitrary read-only command: safe to retry +var result = await db.ExecuteAsync("LOLWUT", args: [], flags: CommandFlags.CommandRetryReadOnly); + +// a Lua script that only reads: opt into retries +var value = await db.ScriptEvaluateAsync( + "return redis.call('GET', KEYS[1])", + keys: [key], + flags: CommandFlags.CommandRetryReadOnly); +``` + +Choose the category honestly — it describes what a *replay* would do. If a retry could double-apply a side-effect, use `CommandRetryWriteAccumulating` (or leave it uncategorised) rather than claiming it's a read. +Conversely, if you want more-side-effecting operations retried across the board, raise the policy's `MaxCommandRetryCategory` instead of tagging each call. + +## Unhealthy State and Failback + +Each member tracks two *independent* pieces of state: + +- **`IsConnected`** — the last observed connectivity of the underlying connection. +- **`IsUnhealthy`** — whether the member has been *disabled* by a failing health-check or a tripped circuit breaker. + +A member is only eligible to be selected as the active member when it is **connected _and_ not unhealthy**. Separating the two lets a member that is technically reconnected still be held out of rotation until we're confident it is stable again — which is what `FailbackDelay` controls. + +### How a member becomes unhealthy + +- A **health check** returns `Unhealthy` for it. +- Its **circuit breaker** trips (`ConnectionFailureType.CircuitBreaker`). The failing member is flagged unhealthy immediately on the circuit-breaker fast-path, so traffic routes away without waiting for the next health-check tick. + +Each time a member is (re)marked unhealthy, the time of that failure is recorded. + +### How a member becomes healthy again + +An unhealthy member is cleared in one of three ways: + +1. **Automatically**, once it passes a health check *and* its most recent failure is older than `FailbackDelay` (see below). +2. **Explicitly**, by calling `member.ResetIsUnhealthy()`. +3. **Implicitly**, by calling `IConnectionGroup.TryFailoverTo(member)` — an explicit failover request always clears the target's unhealthy flag first, since the caller is deliberately asking for that member. + +### `FailbackDelay` + +`MultiGroupOptions.FailbackDelay` is the interval a member must remain healthy - measured from its *most recent* failure - before it is automatically returned to rotation: + +```csharp +MultiGroupOptions options = new MultiGroupOptions.Builder +{ + FailbackDelay = TimeSpan.FromMinutes(2), // must be healthy for 2 minutes after its last failure +}; +``` + +Individual members can override this via `ConnectionGroupMember.FailbackDelay`, for example to hold a known-flaky region out of rotation for longer than the rest: + +```csharp +ConnectionGroupMember[] members = [ + new("us-east.redis.example.com:6379", name: "US East") { Weight = 10 }, + new("flaky.redis.example.com:6379", name: "Flaky") { Weight = 5, FailbackDelay = TimeSpan.FromMinutes(10) }, +]; +``` + +| Value | Behavior | +|-------|----------| +| `TimeSpan.Zero` (default) | Immediate failback — the member is eligible again as soon as a health check passes | +| any positive `TimeSpan` | The member must go `FailbackDelay` with no further failures before it is re-selected | +| `TimeSpan.MaxValue` | **Manual mode** — automatic failback is disabled; the member stays out of rotation until `ResetIsUnhealthy()` or `TryFailoverTo(...)` is called | + +This guards against **flapping**: a member that is intermittently failing will keep pushing its "last failure" time forward, so it never satisfies the delay and stays out of rotation until it is genuinely stable. + +> Implementation note: the failback check is pure tick math on the wall clock — the last-failure time and the cutoff (`UtcNow - FailbackDelay`) are both compared as raw `long` UTC ticks, so `DateTimeKind` never enters into it. (`DateTime.Ticks` and `TimeSpan.Ticks` share the same 100 ns unit.) + +### Anti-flap tiebreak in selection + +Member selection ranks candidates by connectivity, explicit override, weight, and latency. When two candidates are otherwise indistinguishable, selection prefers the member that is **already active**, rather than picking arbitrarily. + +## Manual Failover + +In some scenarios, you may need to manually control which member is actively serving traffic, overriding the automatic selection based on weight and latency. The `TryFailoverTo` method allows you to explicitly switch to a specific member or restore automatic selection. + +### Basic Failover to a Specific Member + +```csharp +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); + +// Get the members to find the one you want to fail over to +var groupMembers = conn.GetMembers(); +var targetMember = groupMembers.FirstOrDefault(m => m.Name == "US West"); + +if (targetMember != null) +{ + // Attempt to fail over to the specified member + bool success = conn.TryFailoverTo(targetMember); + + if (success) + { + Console.WriteLine($"Successfully failed over to {targetMember.Name}"); + } + else + { + Console.WriteLine($"Failed to fail over to {targetMember.Name} (member may be disconnected)"); + } +} +``` + +### Restore Automatic Selection + +To remove an explicit failover and return to automatic member selection based on weight and latency: + +```csharp +// Pass null to remove the explicit failover +bool hadExplicitFailover = conn.TryFailoverTo(null); + +if (hadExplicitFailover) +{ + Console.WriteLine("Removed explicit failover, now using automatic selection"); +} +else +{ + Console.WriteLine("No explicit failover was active"); +} +``` + +### Failover Behavior + +The `TryFailoverTo` method has the following behavior: + +- **Returns `true`**: The failover was successful and the specified member is now active (or an explicit override was successfully removed) +- **Returns `false`**: The failover failed because: + - The member is not connected + - The member is not part of this connection group + +When an explicit failover is active: +- The specified member will be preferred for all traffic +- Weight and latency are ignored for member selection +- If the explicitly selected member becomes unavailable, the system automatically falls back to other connected members +- Health checks continue to run on all members + +### Example: Maintenance Mode + +This is particularly useful when performing maintenance on one region and you want to temporarily route all traffic to another: + +```csharp +var members = new ConnectionGroupMember[] +{ + new("us-east.redis.example.com:6379", name: "US East") { Weight = 100 }, + new("us-west.redis.example.com:6379", name: "US West") { Weight = 100 } +}; + +await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); + +// During maintenance on US East, explicitly route to US West +var westMember = conn.GetMembers().First(m => m.Name == "US West"); +if (conn.TryFailoverTo(westMember)) +{ + Console.WriteLine("Traffic now routed to US West for maintenance"); +} + +// ... perform maintenance on US East ... + +// After maintenance, restore automatic selection +if (conn.TryFailoverTo(null)) +{ + Console.WriteLine("Maintenance complete, automatic selection restored"); +} +``` + +### Example: Monitoring Failover Events + +You can monitor when failovers occur using the `ConnectionChanged` event: + +```csharp +conn.ConnectionChanged += (sender, args) => +{ + if (args.Type == GroupConnectionChangedEventArgs.ChangeType.ActiveChanged) + { + Console.WriteLine($"Active member changed from {args.PreviousGroup?.Name ?? "none"} to {args.Group.Name}"); + } +}; + +// Trigger an explicit failover +var member = conn.GetMembers().First(m => m.Name == "Backup"); +conn.TryFailoverTo(member); +// Event will fire: "Active member changed from Primary to Backup" +``` + +### Important Notes + +1. **Connection Required**: You can only fail over to a member that is currently connected (`IsConnected == true`) +2. **Temporary Override**: The explicit failover persists until: + - You call `TryFailoverTo(null)` to remove it + - The connection group is disposed + - The explicitly selected member becomes disconnected (automatic fallback occurs) +3. **Not Persistent**: Explicit failovers are not persisted across application restarts +4. **Thread-Safe**: `TryFailoverTo` is thread-safe and can be called concurrently with normal operations + +## Dynamic Member Management + +You can add or remove members dynamically using the `IConnectionGroup` interface: + +```csharp +// Cast to IConnectionGroup to access dynamic member management +var group = (IConnectionGroup)conn; + +// Add a new member at runtime +var newMember = new ConnectionGroupMember("new-dc.redis.example.com:6379", name: "New Datacenter") +{ + Weight = 5 +}; +await group.AddAsync(newMember); +Console.WriteLine($"Added {newMember.Name} to the group"); + +// Remove a member +var memberToRemove = members[2]; // Reference to an existing member +if (group.Remove(memberToRemove)) +{ + Console.WriteLine($"Removed {memberToRemove.Name} from the group"); +} +else +{ + Console.WriteLine($"Failed to remove {memberToRemove.Name} - member not found"); +} + +// Check current members +var currentMembers = group.GetMembers(); +Console.WriteLine($"Current member count: {currentMembers.Length}"); +foreach (var member in currentMembers) +{ + Console.WriteLine($" - {member.Name} (Weight: {member.Weight}, Connected: {member.IsConnected})"); +} +``` + +### Adding Members During Maintenance + +Add a new datacenter before removing an old one for zero-downtime migrations: + +```csharp +var group = (IConnectionGroup)conn; + +// Add the new datacenter +var newDC = new ConnectionGroupMember("new-location.redis.example.com:6379", name: "New Location") +{ + Weight = 10 // High weight to prefer the new location +}; +await group.AddAsync(newDC); + +// Wait for the new member to be fully connected and healthy +await Task.Delay(TimeSpan.FromSeconds(5)); + +if (newDC.IsConnected) +{ + Console.WriteLine("New datacenter is online and healthy"); + + // Reduce weight of old datacenter + var oldDC = members[0]; + oldDC.Weight = 1; + + // Wait for traffic to shift + await Task.Delay(TimeSpan.FromSeconds(10)); + + // Remove the old datacenter + if (group.Remove(oldDC)) + { + Console.WriteLine("Old datacenter removed successfully"); + } +} +``` + +## Advanced Customization + +The building blocks above cover the common cases. The extension points below let you replace the default health-check, retry, and circuit-breaker behavior with your own logic; most applications will not need them. + +### Custom Health Check Probes + +You can implement custom health check logic by extending `HealthCheckProbe`. Note that care must be used +if the probe involves talking to data via a `RedisKey`, as on "cluster" configurations, it must be ensured that the +key used resolves to the correct server; for this purpose, the `server.InventKey` method can be used: + +A probe receives a `HealthCheckContext`, carrying the `Server` being probed and the `ProbeTimeout` budget: + +```csharp +public abstract class CustomProbe : HealthCheckProbe +{ + public override Task CheckHealthAsync(HealthCheckContext context) + { + // create a random key that routes to the correct server, using + // the specified prefix + RedisKey key = context.Server.InventKey("health-check/"); + // ... + } +} +``` + +Or more conveniently, the key-specific `KeyWriteHealthCheckProbe` encapsulates this logic: + +```csharp +public class CustomWriteProbe : KeyWriteHealthCheckProbe +{ + public override async Task CheckHealthAsync( + HealthCheckContext context, + IDatabaseAsync database, + RedisKey key) + { + try + { + var value = Guid.NewGuid().ToString(); + await database.StringSetAsync(key, value, expiry: context.ProbeTimeout); + bool isMatch = value == await database.StringGetAsync(key); + + return isMatch ? HealthCheckResult.Healthy : HealthCheckResult.Unhealthy; + } + catch + { + return HealthCheckResult.Unhealthy; + } + } +} +``` + +> The context is passed **by value** rather than by `in`, because probes are typically `async`, and async methods cannot take by-ref parameters. The sibling `HealthCheckProbePolicy.Evaluate` is synchronous, so it does take `in HealthCheckProbeContext`. + +### Custom Probe Policies + +In addition to the inbuilt policies, custom policies can be implemented by extending `HealthCheckProbePolicy`. +By checking the properties of the `HealthCheckProbeContext` parameter, your policy can make a determination +about the health of the server - returning `HealthCheckResult.Healthy` or `HealthCheckResult.Unhealthy` as +appropriate. If you return `HealthCheckResult.Inconclusive`, the health check will continue with additional probes. + +#### Example: Require at Least N Successes + +This example demonstrates a policy that requires at least a specified number of successful probes before declaring the endpoint healthy: + +```csharp +public class AtLeastPolicy(int requiredSuccesses) : HealthCheckProbePolicy +{ + public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) + { + // Success if we have at least the required number of successful probes + if (context.Success >= requiredSuccesses) return HealthCheckResult.Healthy; + + // If no more probes remaining, we haven't met our threshold; otherwise: keep trying + return context.Remaining == 0 ? HealthCheckResult.Unhealthy : HealthCheckResult.Inconclusive; + } +} + +// Use the custom policy requiring at least 2 successes +HealthCheck healthCheck = new HealthCheck.Builder +{ + ProbeCount = 5, // Need enough probes to allow for the required successes + ProbePolicy = new AtLeastPolicy(2) +}; + +MultiGroupOptions options = new MultiGroupOptions.Builder +{ + HealthCheck = healthCheck +}; +``` + +This policy ensures that transient successes don't immediately mark an endpoint as healthy. It requires at least the specified number of successful probes, which provides better confidence in the endpoint's stability while still being more lenient than `AllSuccess`. + +### Custom Circuit Breakers + +> This is an advanced extension point; most applications should use `CircuitBreaker.Default` (optionally tuned via `CircuitBreaker.Builder`) or `CircuitBreaker.None`. + +You can implement your own policy by extending `CircuitBreaker` and its `Accumulator`. `CreateAccumulator()` is called once per underlying connection, so each accumulator holds the state for a single connection. +For every completed message the connection calls `ObserveResult`, passing a `FaultContext` that describes the outcome: `IsFault` indicates whether a fault occurred, with the associated `Fault`, `ErrorKind` and `ConnectionFailureType` available for inspection. `IsHealthy()` is consulted separately: return `true` while the connection should be considered **healthy**, or `false` to **trip** the breaker and tear the connection down. + +By default only faults that `IsFailure` regards as genuine failures reach `ObserveResult` *as* failures (transient/connection errors, timeouts, and similar - classified from the fault's `ErrorKind`); everything else - including application-level errors such as a bad command - is passed as a success. Override `IsFailure(in FaultContext)` if you need different rules for what counts: + +```csharp +public sealed class ConsecutiveFailureBreaker(int limit) : CircuitBreaker +{ + public override Accumulator CreateAccumulator() => new Acc(limit); + + private sealed class Acc(int limit) : Accumulator + { + private int _consecutiveFailures; + + // a fault is present iff context.IsFault; a success resets the run + public override void ObserveResult(in FaultContext context) + { + if (context.IsFault) + { + _consecutiveFailures++; + } + else + { + _consecutiveFailures = 0; + } + } + + // healthy until we hit the configured run of consecutive failures + public override bool IsHealthy() => _consecutiveFailures < limit; + + // called if the connection wants to discard accumulated history + public override void Reset() => _consecutiveFailures = 0; + + // (optional) override to change what counts as a failure; the default classifies from ErrorKind + // protected override bool IsFailure(in FaultContext fault) => fault.IsFault; + } +} + +MultiGroupOptions options = new MultiGroupOptions.Builder +{ + CircuitBreaker = new ConsecutiveFailureBreaker(limit: 5) +}; +``` + +Keep `ObserveResult` cheap and thread-safe: it runs on the hot path for every completed message, and may be called concurrently. + +### Custom Retry Policies + +`RetryPolicy` is itself extensible: override `CanRetry(in FaultContext fault)` to make the retry decision yourself. It returns a `RetryResult` - `None` to give up, or a combination of `SameServer` and `FailoverServer` to indicate where a retry may be attempted. +The `FaultContext` gives you the classified `ErrorKind`, the `ConnectionFailureType`, and the command `Flags` (including its retry category) to base the decision on: + +```csharp +public sealed class ReadOnlyOnlyRetryPolicy : RetryPolicy +{ + public override RetryResult CanRetry(in FaultContext fault) + { + // only ever retry pure reads, and only on the same server + if (fault.ErrorKind == RedisErrorKind.Loading + && (fault.Flags & CommandFlags.CommandRetryReadOnly) != 0) + { + return RetryResult.SameServer; + } + // note: base.CanRetry(fault) would apply the default logic + return RetryResult.None; + } +} + +IDatabaseAsync db = conn.GetDatabase().WithRetry(new ReadOnlyOnlyRetryPolicy()); +``` + +A derived policy inherits the default settings; to derive *and* change the settings, take a `Builder` and pass it to the base constructor: + +```csharp +public sealed class ReadOnlyOnlyRetryPolicy(RetryPolicy.Builder builder) : RetryPolicy(builder) +{ + public override RetryResult CanRetry(in FaultContext fault) => /* ... */; +} + +IDatabaseAsync db = conn.GetDatabase().WithRetry( + new ReadOnlyOnlyRetryPolicy(new RetryPolicy.Builder { MaxAttempts = 5 })); +``` diff --git a/docs/exp/SER004.md b/docs/exp/SER004.md index 91f5d87c4..e1e77968b 100644 --- a/docs/exp/SER004.md +++ b/docs/exp/SER004.md @@ -4,6 +4,8 @@ RESPite is an experimental library that provides high-performance low-level RESP It is used as the IO core for StackExchange.Redis v3+. You should not (yet) use it directly unless you have a very good reason to do so. +To suppress this message, add the following to your `csproj` file: + ```xml $(NoWarn);SER004 ``` diff --git a/docs/exp/SER005.md b/docs/exp/SER005.md index 03e7b7bb4..6e446fed3 100644 --- a/docs/exp/SER005.md +++ b/docs/exp/SER005.md @@ -10,6 +10,8 @@ These types are considered slightly more... *mercurial*. We encourage you to use (not just for fun) you might need to update your test code if we tweak something. This should not impact "real" library usage. +To suppress this message, add the following to your `csproj` file: + ```xml $(NoWarn);SER005 ``` diff --git a/docs/exp/SER007.md b/docs/exp/SER007.md new file mode 100644 index 000000000..60fee6dd8 --- /dev/null +++ b/docs/exp/SER007.md @@ -0,0 +1,36 @@ +# Client-side geographic failover + +This feature is used to fail over between multiple redundant endpoints that can each serve your workload - designed for an +**Active-Active** (geo-distributed, bidirectionally-replicated) deployment, but usable over any set of endpoints, including ones +with no replication between them (see the full docs for what that means for your data). It is functionally similar to the +"multi-DB client" concept in other Redis clients (Jedis' `MultiDbClient`, redis-py's `MultiDBClient`); please see +[full docs](/Failover). + +## Status: released and fully supported + +**This is a shipped, supported feature, intended for production use.** It is not a preview, it is not gated on an +unreleased server version, and it does not depend on any server API that is subject to change. Bugs in it are treated as +bugs, and can be reported and addressed in the usual way. No API changes are currently planned. + +The `[Experimental]` marker is nonetheless a real one: while it is present, we reserve the right to adjust this API surface +without the usual backwards-compatibility guarantees. That is the whole reason it is here. Client-side geographic failover +is a large, brand-new public API arriving in a single release, and once a surface that size is locked down it is locked down +for good - so we would rather keep room to correct a design wrinkle found in real-world use than commit to every detail +on day one. + +The marker is expected to be short-lived, and to be removed in a subsequent 3.1.x release once the design has had some +exposure. At that point the diagnostic simply stops being emitted, with no code change needed on your side. + +Suppressing the diagnostic is the normal way to use the feature; doing so acknowledges the above. + +To suppress this message, add the following to your `csproj` file: + +```xml +$(NoWarn);SER007 +``` + +or more granularly / locally in C#: + +``` c# +#pragma warning disable SER007 +``` diff --git a/docs/index.md b/docs/index.md index a21164785..a49d708b2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -35,6 +35,7 @@ Documentation - [Basic Usage](Basics) - getting started and basic usage - [Async Timeouts](AsyncTimeouts) - async timeouts and cancellation - [Configuration](Configuration) - options available when connecting to redis +- [Client-side geographic failover](Failover) (Active-Active / "multi-DB client") - connecting to multiple redundant Redis endpoints for high availability - [Pipelines and Multiplexers](PipelinesMultiplexers) - what is a multiplexer? - [Keys, Values and Channels](KeysValues) - discusses the data-types used on the API - [Transactions](Transactions) - how atomic transactions work in redis diff --git a/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs new file mode 100644 index 000000000..a8df6744c --- /dev/null +++ b/eng/StackExchange.Redis.Build/AutoDatabaseGenerator.cs @@ -0,0 +1,539 @@ +using System.Collections.Immutable; +using System.Reflection.Metadata.Ecma335; +using System.Text; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; + +// readable names for the info we gather; kept as tuples (+ BasicArray) so they have +// value equality and play nicely with incremental generator caching. +using ParamInfo = (string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default); +using MethodInfo = (string Name, string ReturnType, StackExchange.Redis.Build.BasicArray<(string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default)> Parameters, StackExchange.Redis.Build.BasicArray TypeArgs); +using InterfaceInfo = (string Name, string Namespace, StackExchange.Redis.Build.AutoDatabaseGenerator.KnownInterfaces KnownType, StackExchange.Redis.Build.BasicArray<(string Name, string ReturnType, StackExchange.Redis.Build.BasicArray<(string Name, string Type, Microsoft.CodeAnalysis.RefKind RefKind, bool IsParams, bool IsOptional, bool HasDefault, string? Default)> Parameters, StackExchange.Redis.Build.BasicArray TypeArgs)> Methods); +using ClassInfo = (string Name, string Namespace, StackExchange.Redis.Build.AutoDatabaseGenerator.KnownInterfaces Interfaces, bool IsMutator); + +namespace StackExchange.Redis.Build; + +[Generator(LanguageNames.CSharp)] +public class AutoDatabaseGenerator : IIncrementalGenerator +{ + public void Initialize(IncrementalGeneratorInitializationContext ctx) + { + var interfaces = ctx.SyntaxProvider + .CreateSyntaxProvider( + static (node, _) => node is InterfaceDeclarationSyntax decl && FastIndexFilter(decl), ExtractInterfaceMethods) + .Where(pair => pair.Name is { Length: > 0 }) + .Collect(); + + var classes = ctx.SyntaxProvider + .CreateSyntaxProvider( + static (node, _) => node is ClassDeclarationSyntax decl && FastClassFilter(decl), ExtractClasses) + .Where(pair => pair.Name is { Length: > 0 }) + .Collect(); + + ctx.RegisterSourceOutput(interfaces.Combine(classes), static (ctx, content) => Generate(ctx, content.Left, content.Right)); + } + + static KnownInterfaces Identify(string type) => type switch + { + "IDatabase" => KnownInterfaces.IDatabase, + "IDatabaseAsync" => KnownInterfaces.IDatabaseAsync, + "IRedis" => KnownInterfaces.IRedis, + "IRedisAsync" => KnownInterfaces.IRedisAsync, + _ => KnownInterfaces.None, + }; + + private static bool FastIndexFilter(InterfaceDeclarationSyntax decl) // limit to IDatabase, IDatabaseAsync + => Identify(decl.Identifier.ValueText) is not 0; + + private static bool FastClassFilter(ClassDeclarationSyntax decl) // limit to IDatabase, IDatabaseAsync + => decl.AttributeLists.Any(x => x.Attributes.Any(x => x.Name.ToString() is "AutoDatabase" or "AutoDatabaseAttribute")); + + // does this auto-database rewrite keys/channels? if not, the captured-args structs need no mapping members + private static bool IsMutatorInterface(INamedTypeSymbol symbol) => + symbol is + { + TypeKind: TypeKind.Interface, + Name: "IRedisArgsMutator", + ContainingType: null, + ContainingNamespace: + { + Name: "Redis", + ContainingNamespace: + { + Name: "StackExchange", + ContainingNamespace.IsGlobalNamespace: true + } + } + }; + + private static bool IsOurInterface(INamedTypeSymbol symbol) => + symbol is + { + TypeKind: TypeKind.Interface, + Name: "IDatabase" or "IDatabaseAsync" or "IRedis" or "IRedisAsync", + ContainingType: null, + ContainingNamespace: + { + Name: "Redis", + ContainingNamespace: + { + Name: "StackExchange", + ContainingNamespace.IsGlobalNamespace: true + } + } + }; + + private static InterfaceInfo ExtractInterfaceMethods(GeneratorSyntaxContext context, CancellationToken cancel) + { + // note: we deliberately do NOT interpret anything here - just capture the raw shape of every + // method (name, return type, and per-parameter name/type/modifiers/optionality/default) so that + // later passes have everything they might need. + if (context.SemanticModel.GetDeclaredSymbol(context.Node, cancel) is not INamedTypeSymbol iface + || !IsOurInterface(iface)) + { + return default; + } + + var knownType = Identify(iface.Name); + if (knownType is KnownInterfaces.None) return default; + + var methods = new List(); + foreach (var member in iface.GetMembers()) + { + cancel.ThrowIfCancellationRequested(); + if (member is not IMethodSymbol { MethodKind: MethodKind.Ordinary } method) continue; + + var parameters = BasicArray.From(method.Parameters, static p => ( + Name: p.Name, + Type: p.Type.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat), + RefKind: p.RefKind, + IsParams: p.IsParams, + IsOptional: p.IsOptional, + HasDefault: p.HasExplicitDefaultValue, + Default: p.HasExplicitDefaultValue ? FormatDefault(p.ExplicitDefaultValue) : null)); + + BasicArray typeArgs = default; + if (method.IsGenericMethod) + { + typeArgs = BasicArray.From(method.TypeParameters, p => p.Name); + } + methods.Add(( + Name: method.Name, + ReturnType: method.ReturnType.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat), + Parameters: parameters, + TypeArgs: typeArgs)); + } + + var ns = iface.ContainingNamespace.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat); + return (iface.Name, ns, knownType, BasicArray.From(methods)); + } + + private ClassInfo ExtractClasses(GeneratorSyntaxContext context, CancellationToken cancel) + { + // note: we deliberately do NOT interpret anything here - just capture the raw shape of every + // method (name, return type, and per-parameter name/type/modifiers/optionality/default) so that + // later passes have everything they might need. + if (context.SemanticModel.GetDeclaredSymbol(context.Node, cancel) is not INamedTypeSymbol cls + || !HasAutoDatabaseAttrib(cls)) + { + return default; + } + + static bool HasAutoDatabaseAttrib(INamedTypeSymbol symbol) + { + var attribs = symbol.GetAttributes(); + foreach (var attrib in attribs) + { + if (attrib.AttributeClass is + { + Name: "AutoDatabaseAttribute", + ContainingType: null, + ContainingNamespace: + { + Name: "Redis", + ContainingNamespace: + { + Name: "StackExchange", + ContainingNamespace.IsGlobalNamespace: true + } + } + }) + { + return true; + } + } + + return false; + } + + KnownInterfaces known = 0; + foreach (var iFace in cls.Interfaces) + { + if (IsOurInterface(iFace)) + { + switch (iFace.Name) + { + case "IDatabase": + known |= KnownInterfaces.IDatabase | KnownInterfaces.IDatabaseAsync | + KnownInterfaces.IRedis | KnownInterfaces.IRedisAsync; + break; + case "IDatabaseAsync": + known |= KnownInterfaces.IDatabaseAsync | KnownInterfaces.IRedisAsync; + break; + case "IRedis": + known |= KnownInterfaces.IRedis | KnownInterfaces.IRedisAsync; + break; + case "IRedisAsync": + known |= KnownInterfaces.IRedisAsync; + break; + } + } + } + + if (known is KnownInterfaces.None) return default; // nothing to do! + + // AllInterfaces (not Interfaces) so an inherited/explicit implementation still counts + bool isMutator = false; + foreach (var iFace in cls.AllInterfaces) + { + if (IsMutatorInterface(iFace)) + { + isMutator = true; + break; + } + } + + var ns = cls.ContainingNamespace.ToDisplayString(SymbolDisplayFormat.CSharpErrorMessageFormat); + return (cls.Name, ns, known, isMutator); + } + + [Flags] + internal enum KnownInterfaces + { + None = 0, + IDatabase = 1, + IDatabaseAsync = 2, + IRedis = 4, + IRedisAsync = 8, + } + + // methods that don't fit the capture-and-replay shape are left for the caller to implement manually: + // - generic methods (open type args can't be captured into a concrete state struct) + // - the Wait family (synchronization over caller-supplied Tasks, not server calls) + // - IsConnected: a synchronous status probe that IDatabaseAsync carries; it would route through the + // sync Execute funnel, which an async-only database (e.g. RetryDatabase) doesn't provide - and a + // connection check shouldn't be retried in any case + // - IdentifyEndpoint / IdentifyEndpointAsync: a routing lookup ("which endpoint serves this key"), + // not a replayable server command; wrappers want their own behaviour here (e.g. a connection-group + // resolving against the currently-active member and returning null when there is none) + // - streaming returns (IEnumerable / IAsyncEnumerable) whose execution is deferred + private static bool SkipMethod(MethodInfo method) + => !method.TypeArgs.IsEmpty + || method.Name.Contains("Wait") + || method.Name == "IsConnected" + || method.Name.Contains("IdentifyEndpoint") + // transaction/batch factories are not server round-trips and cannot be captured-and-replayed; + // the (few) databases that offer them implement them by hand + || method.Name == "CreateTransaction" + || method.Name == "CreateBatch" + || method.ReturnType.StartsWith("System.Collections.Generic.IEnumerable<", StringComparison.Ordinal) + || method.ReturnType.StartsWith("System.Collections.Generic.IAsyncEnumerable<", StringComparison.Ordinal); + + private const string TaskType = "System.Threading.Tasks.Task"; + + // Task / Task returns route through ExecuteAsync (its own retry policy); everything else + // uses Execute. + private static bool IsAsync(string returnType) + => returnType.StartsWith(TaskType, StringComparison.Ordinal); + + // the retry machinery unwraps the Task and only ever sees the inner result, so key/channel + // (un)mapping must be decided on T, not Task; this strips the Task<...> wrapper from an async + // return type. A bare (non-generic) Task has no result and is returned unchanged (as is any + // non-async return type), which is harmless since neither matches NeedsMap. + private static string StripTask(string returnType) + => IsAsync(returnType) && returnType.StartsWith(TaskType + "<", StringComparison.Ordinal) + ? returnType.Substring(TaskType.Length + 1, returnType.Length - TaskType.Length - 2) + : returnType; + + private static string FormatDefault(object? value) => value switch + { + null => "null", + string s => $"\"{s}\"", + bool b => b ? "true" : "false", + _ => value.ToString() ?? "null", + }; + + private static void Generate(SourceProductionContext ctx, ImmutableArray interfaces, ImmutableArray classes) + { + if (interfaces.IsDefaultOrEmpty | classes.IsDefaultOrEmpty) return; // nothing to do + + // each interface is declared across several partial files, so we get one (identical) entry per + // partial declaration; structural equality lets us collapse those down to one per interface. + var iKeyed = new Dictionary(); + foreach (var t in interfaces) + { + if (t.KnownType is KnownInterfaces.None) continue; + if (!iKeyed.ContainsKey(t.KnownType)) iKeyed.Add(t.KnownType, t); + } + + var sb = new StringBuilder(); + var writer = new CodeWriter(sb); + writer.NewLine().Append("// "); + writer.NewLine().Append("// AutoDatabaseGenerator - explicit interface implementations that funnel every"); + writer.NewLine().Append("// call through Execute(state, projection), capturing the arguments in a generated"); + writer.NewLine().Append("// state struct (one per unique parameter-type signature) to avoid per-call closures."); + writer.NewLine().Append("#nullable enable"); // this needs to be explicit for code-gen + writer.NewLine(); + foreach (var cls in classes.Distinct()) + { + if (!string.IsNullOrWhiteSpace(cls.Namespace)) + { + writer.NewLine().Append("namespace ").Append(cls.Namespace).NewLine().Append("{").Indent(); + } + + // unique parameter-type signatures encountered while emitting this class's methods; + // keyed on the '|'-joined parameter types so distinct methods with the same shape share + // one state struct. tupleDefs[i] holds a representative parameter list for _tuple{i}. + var tupleIndex = new Dictionary(); + var tupleDefs = new List>(); + + // every unique key/channel-bearing result type across all shapes; a single shared + // singleton (emitted after the tuples) implements IRedisArgsResult for each, so + // tuples expose unmap dispatch via UnMapper without ever being cast to an interface + var allReturns = new HashSet(); + + bool isFirst = true; + writer.NewLine().Append("partial class ").Append(cls.Name); + AppendInterfaceDeclaration(KnownInterfaces.IDatabase); + AppendInterfaceDeclaration(KnownInterfaces.IDatabaseAsync); + AppendInterfaceDeclaration(KnownInterfaces.IRedis); + AppendInterfaceDeclaration(KnownInterfaces.IRedisAsync); + writer.NewLine().Append("{").Indent(); + AppendInterfaceMethods(KnownInterfaces.IDatabase); + AppendInterfaceMethods(KnownInterfaces.IDatabaseAsync); + AppendInterfaceMethods(KnownInterfaces.IRedis); + AppendInterfaceMethods(KnownInterfaces.IRedisAsync); + AppendTupleTypes(); + + writer.Outdent().NewLine().Append("}"); + + if (!string.IsNullOrWhiteSpace(cls.Namespace)) + { + writer.Outdent().NewLine().Append("}"); + } + writer.NewLine(); + + + void AppendInterfaceDeclaration(KnownInterfaces knownType) + { + if ((cls.Interfaces & knownType) is 0 | !iKeyed.TryGetValue(knownType, out var iType)) return; + writer.Append(isFirst ? " : " : ", ").Append("global::") + .Append(iType.Namespace).Append('.') .Append(iType.Name); + isFirst = false; + } + + void AppendInterfaceMethods(KnownInterfaces knownType) + { + if ((cls.Interfaces & knownType) is 0 | !iKeyed.TryGetValue(knownType, out var iType)) return; + foreach (var method in iType.Methods) + { + if (SkipMethod(method)) continue; // wonky by nature - left for the caller to implement manually + + writer.NewLine().Append(method.ReturnType).Append(" global::") + .Append(iType.Namespace).Append('.').Append(iType.Name).Append('.').Append(method.Name).Append("("); + bool firstParam = true; + foreach (var p in method.Parameters.Span) + { + if (firstParam) firstParam = false; + else writer.Append(", "); + writer.Append(p.Type).Append(" ").Append(p.Name); + } + writer.Append(')').Indent().NewLine(); + + // async methods route to ExecuteAsync (its own retry policy); everything else uses + // Execute. The state struct is shared regardless of return type - keyed on parameter + // types only - so e.g. ArraySet and ArraySetAsync land on the same _tupleN. The return + // type is Task-stripped so unmapping is keyed on the inner result the retry machinery + // actually sees, not the Task wrapper. + bool isAsync = IsAsync(method.ReturnType); + var simpleResult = StripTask(method.ReturnType); + bool needsMap = NeedsMap(simpleResult); + if (needsMap) allReturns.Add(simpleResult); + int tuple = GetTupleIndex(method.Parameters, needsMap); + writer.Append(isAsync ? "=> ExecuteAsync(new _tuple" : "=> Execute(new _tuple").Append(tuple).Append("("); + firstParam = true; + foreach (var p in method.Parameters.Span) + { + if (firstParam) firstParam = false; + else writer.Append(", "); + writer.Append(p.Name); + } + // the funnels take the state by readonly-ref (see AutoDatabase{Sync|Async}Operation) to avoid + // copying the larger state structs; a lambda only binds to an `in` parameter if it says so, + // hence the explicit modifier (C# 14 allows it without also naming the type) + writer.Append("), static (in state, inner) => inner.").Append(method.Name).Append("("); + for (int i = 0; i < method.Parameters.Length; i++) + { + if (i != 0) writer.Append(", "); + writer.Append("state.Arg").Append(i); + } + writer.Append("));").Outdent().NewLine(); + } + } + + static string GetTupleKey(BasicArray parameters) + { + var keyBuilder = new StringBuilder(); + foreach (var p in parameters.Span) + { + keyBuilder.Append(p.Type).Append('|'); // types-only key; no ref/out on this surface, so type is sufficient + } + + return keyBuilder.ToString(); + } + + bool TupleNeedsMap(BasicArray parameters) => + tupleIndex.TryGetValue(GetTupleKey(parameters), out var found) && found.NeedsMap; + + int GetTupleIndex(BasicArray parameters, bool needsMap) + { + var key = GetTupleKey(parameters); + int index; + if (tupleIndex.TryGetValue(key, out var found)) + { + index = found.Index; + if (needsMap && !found.NeedsMap) + { + found.NeedsMap = true; + tupleIndex[key] = found; + } + } + else + { + index = tupleDefs.Count; + tupleIndex.Add(key, (index, needsMap)); + tupleDefs.Add(parameters); + } + + return index; + } + + // const string CommandFlagsType = "StackExchange.Redis.CommandFlags"; + const string RedisKeyType = "StackExchange.Redis.RedisKey"; + const string RedisChannelType = "StackExchange.Redis.RedisChannel"; + // types that carry key(s) internally without "RedisKey" in their name, so the + // substring check below won't catch them; they rely on a Map extension method + const string StreamPositionType = "StackExchange.Redis.StreamPosition"; + // the Execute/ExecuteAsync escape hatch boxes keys/channels inside a loosely-typed + // arg list; these route through a Map extension that unboxes and rewrites matches + const string ScriptArgArrayType = "object[]"; + const string ScriptArgCollectionType = "System.Collections.Generic.ICollection"; + + static bool NeedsMap(string name) => + name.IndexOf(RedisKeyType, StringComparison.Ordinal) >= 0 + || name.IndexOf(RedisChannelType, StringComparison.Ordinal) >= 0 + || name.IndexOf(StreamPositionType, StringComparison.Ordinal) >= 0 + || name == ScriptArgArrayType + || name == ScriptArgCollectionType + || name.IndexOf("ListPopResult", StringComparison.Ordinal) >= 0 + || name.IndexOf("SortedSetPopResult", StringComparison.Ordinal) >= 0 + || name == ScriptArgCollectionType + "?"; + + void AppendTupleTypes() + { + for (int i = 0; i < tupleDefs.Count; i++) + { + var raw = tupleDefs[i]; + bool needsMap = TupleNeedsMap(raw); + var parameters = raw.Span; + + // key/channel-bearing fields are mutable only when something can rewrite them (i.e. the + // owning database is an IRedisArgsMutator); otherwise every field is readonly + // a non-mutator's captured args never change after capture, so the struct itself can be + // readonly - which also guarantees no defensive copies when read through an `in` ref + writer.NewLine().NewLine().Append(cls.IsMutator ? "private struct _tuple" : "private readonly struct _tuple").Append(i).Append("("); + for (int p = 0; p < parameters.Length; p++) + { + if (p != 0) writer.Append(", "); + writer.Append(parameters[p].Type).Append(" arg").Append(p); + } + + // captured args are a plain struct unless the owning database rewrites keys, in which + // case the mapping members - and the interface that carries them - are emitted below + writer.Append(")"); + if (cls.IsMutator) + { + writer.Append(" : global::StackExchange.Redis.IMappableRedisArgs"); + } + writer.NewLine().Append("{").Indent(); + for (int p = 0; p < parameters.Length; p++) + { + writer.NewLine().Append("public ").Append(cls.IsMutator && NeedsMap(parameters[p].Type) ? "" : "readonly ").Append(parameters[p].Type) + .Append(" Arg").Append(p).Append(" = arg").Append(p).Append(";"); + } + + if (!cls.IsMutator) + { + // nothing rewrites keys here, so there is no Map/UnMapper to emit at all + writer.Outdent().NewLine().Append("}"); + continue; + } + + // Map rewrites each scalar key/channel field directly, and defers container or + // loosely-typed fields to a matching Map extension method + writer.NewLine().Append("public void Map(global::StackExchange.Redis.IRedisArgsMutator mutator)") + .NewLine().Append("{").Indent(); + for (int p = 0; p < parameters.Length; p++) + { + if (NeedsMap(parameters[p].Type)) + { + // key/channel-bearing container (or the loosely-typed script arg list); it + // is the library's job to ensure a suitable Map extension method exists + writer.NewLine().Append("Arg").Append(p).Append(" = mutator.Map(Arg").Append(p).Append(");"); + } + } + writer.Outdent().NewLine().Append("}"); + + // tuples with key/channel-bearing results point UnMapper at the shared singleton + // (which knows how to unmap every such result type); the rest return null so the + // Execute helper skips unmapping entirely - and neither path boxes the struct + writer.NewLine().Append("public readonly object? UnMapper => ") + .Append(needsMap ? "_UnMapper.Instance;" : "null;"); + + writer.Outdent().NewLine().Append("}"); + } + + if (cls.IsMutator && allReturns.Count is not 0) + { + // sorted for deterministic (cache-stable) output + var ordered = new List(allReturns); + ordered.Sort(StringComparer.Ordinal); + + writer.NewLine().NewLine().Append("private sealed class _UnMapper").Indent(); + bool firstIface = true; + foreach (var retType in ordered) + { + writer.NewLine().Append(firstIface ? ": " : ", ") + .Append("global::StackExchange.Redis.IRedisArgsResult<").Append(retType).Append(">"); + firstIface = false; + } + writer.Outdent().NewLine().Append("{").Indent(); + writer.NewLine().Append("public static readonly _UnMapper Instance = new();"); + foreach (var retType in ordered) + { + writer.NewLine().NewLine().Append(retType).Append(' ') + .Append("global::StackExchange.Redis.IRedisArgsResult<") + .Append(retType) + .Append(">.UnMap(global::StackExchange.Redis.IRedisArgsMutator mutator, ") + .Append(retType).Append(" value)") + .Indent().NewLine().Append("=> mutator.UnMap(value);").Outdent(); + } + writer.Outdent().NewLine().Append("}"); + } + } + } + writer.NewLine(); + + ctx.AddSource("AutoDatabase.generated.cs", sb.ToString()); + } +} diff --git a/eng/StackExchange.Redis.Build/BasicArray.cs b/eng/StackExchange.Redis.Build/BasicArray.cs index dc7984c75..0c2df6f92 100644 --- a/eng/StackExchange.Redis.Build/BasicArray.cs +++ b/eng/StackExchange.Redis.Build/BasicArray.cs @@ -38,7 +38,7 @@ public bool Equals(BasicArray other) int i = 0; foreach (ref readonly T el in this.Span) { - if (!_comparer.Equals(el, y[i])) return false; + if (!_comparer.Equals(el, y[i++])) return false; } return true; @@ -58,7 +58,7 @@ public override int GetHashCode() var hash = Length; foreach (ref readonly T el in this.Span) { - _ = (hash * -37) + _comparer.GetHashCode(el); + hash = (hash * -37) + _comparer.GetHashCode(el); } return hash; @@ -82,4 +82,25 @@ public void Add(in T value) public BasicArray Build() => new(elements, Count); } + + public static BasicArray From(ICollection collection) + { + if (collection.Count is 0) return default; + var arr = new T[collection.Count]; + collection.CopyTo(arr, 0); + return new(arr, arr.Length); + } + + public static BasicArray From(ICollection collection, Func selector) + { + if (collection.Count is 0) return default; + var arr = new T[collection.Count]; + int i = 0; + foreach (var item in collection) + { + arr[i++] = selector(item); + } + + return new(arr, i); + } } diff --git a/eng/StackExchange.Redis.Build/CodeWriter.cs b/eng/StackExchange.Redis.Build/CodeWriter.cs index 94d30ef47..8d3091322 100644 --- a/eng/StackExchange.Redis.Build/CodeWriter.cs +++ b/eng/StackExchange.Redis.Build/CodeWriter.cs @@ -14,7 +14,8 @@ internal sealed class CodeWriter(StringBuilder buffer) /// Starts a new line, applying the current indent. public CodeWriter NewLine() { - buffer.AppendLine().Append(' ', _indent * 4); + buffer.AppendLine(); + _lineHasContent = false; return this; } @@ -32,26 +33,41 @@ public CodeWriter Outdent() return this; } + private bool _lineHasContent; + + private void IndentIfNeeded() + { + if (!_lineHasContent) + { + buffer.Append(' ', _indent * 4); + _lineHasContent = true; + } + } + public CodeWriter Append(string? value) { + IndentIfNeeded(); buffer.Append(value); return this; } public CodeWriter Append(char value) { + IndentIfNeeded(); buffer.Append(value); return this; } public CodeWriter Append(int value) { + IndentIfNeeded(); buffer.Append(value); return this; } public CodeWriter Append(long value) { + IndentIfNeeded(); buffer.Append(value); return this; } diff --git a/src/RESPite/Messages/RespReader.cs b/src/RESPite/Messages/RespReader.cs index 413d0174d..2137946d8 100644 --- a/src/RESPite/Messages/RespReader.cs +++ b/src/RESPite/Messages/RespReader.cs @@ -722,6 +722,29 @@ public readonly unsafe bool TryParseScalar( return TryGetSpan(out var span) ? parser(span, out value) : TryParseSlow(parser, out value); } + // same thing as ^^^, but as a utility method for text values, paying UTF8 encode + internal static unsafe bool TryParseScalar( + ReadOnlySpan source, + delegate* managed, out T, bool> parser, + out T value) + { + const int MAX_STACK = 128; + byte[]? lease = null; + int maxLen = RespConstants.UTF8.GetMaxByteCount(source.Length); + Span buffer = maxLen <= MAX_STACK + ? stackalloc byte[MAX_STACK] // prefer fixed size for perf + : (lease = ArrayPool.Shared.Rent(maxLen)); + try + { + var len = RespConstants.UTF8.GetBytes(source, buffer); + return parser(buffer.Slice(0, len), out value); + } + finally + { + if (lease is not null) ArrayPool.Shared.Return(lease); + } + } + [MethodImpl(MethodImplOptions.NoInlining)] private readonly unsafe bool TryParseSlow( delegate* managed, out T, bool> parser, diff --git a/src/RESPite/PublicAPI/Debug/PublicAPI.Shipped.txt b/src/RESPite/PublicAPI/Debug/PublicAPI.Shipped.txt new file mode 100644 index 000000000..804e940b9 --- /dev/null +++ b/src/RESPite/PublicAPI/Debug/PublicAPI.Shipped.txt @@ -0,0 +1,2 @@ +#nullable enable +[SERDBG]RESPite.Messages.RespReader.AggregateEnumerator.Current.get -> RESPite.Messages.RespReader diff --git a/src/RESPite/PublicAPI/PublicAPI.Shipped.txt b/src/RESPite/PublicAPI/PublicAPI.Shipped.txt index 412c52862..eb6ae9cbd 100644 --- a/src/RESPite/PublicAPI/PublicAPI.Shipped.txt +++ b/src/RESPite/PublicAPI/PublicAPI.Shipped.txt @@ -114,7 +114,6 @@ [SER004]RESPite.Messages.RespReader.AggregateEnumerator [SER004]RESPite.Messages.RespReader.AggregateEnumerator.AggregateEnumerator() -> void [SER004]RESPite.Messages.RespReader.AggregateEnumerator.AggregateEnumerator(scoped in RESPite.Messages.RespReader reader) -> void -[SER004]RESPite.Messages.RespReader.AggregateEnumerator.Current.get -> RESPite.Messages.RespReader [SER004]RESPite.Messages.RespReader.AggregateEnumerator.DemandNext() -> void [SER004]RESPite.Messages.RespReader.AggregateEnumerator.FillAll(scoped System.Span target, RESPite.Messages.RespReader.Projection! first, RESPite.Messages.RespReader.Projection! second, System.Func! combine) -> void [SER004]RESPite.Messages.RespReader.AggregateEnumerator.GetEnumerator() -> RESPite.Messages.RespReader.AggregateEnumerator diff --git a/src/RESPite/PublicAPI/Release/PublicAPI.Shipped.txt b/src/RESPite/PublicAPI/Release/PublicAPI.Shipped.txt new file mode 100644 index 000000000..dd1130e4c --- /dev/null +++ b/src/RESPite/PublicAPI/Release/PublicAPI.Shipped.txt @@ -0,0 +1,2 @@ +#nullable enable +[SER004]RESPite.Messages.RespReader.AggregateEnumerator.Current.get -> RESPite.Messages.RespReader diff --git a/src/RESPite/RESPite.csproj b/src/RESPite/RESPite.csproj index 6b9798b0e..8acf9429c 100644 --- a/src/RESPite/RESPite.csproj +++ b/src/RESPite/RESPite.csproj @@ -40,6 +40,8 @@ + + diff --git a/src/RESPite/Shared/AsciiHash.Instance.cs b/src/RESPite/Shared/AsciiHash.Instance.cs index 50ac4d245..93b8258ca 100644 --- a/src/RESPite/Shared/AsciiHash.Instance.cs +++ b/src/RESPite/Shared/AsciiHash.Instance.cs @@ -4,7 +4,12 @@ namespace RESPite; -public readonly partial struct AsciiHash : IEquatable +public readonly partial struct AsciiHash : IEquatable, +#if NET + ISpanFormattable +#else + IFormattable +#endif { // ReSharper disable InconsistentNaming private readonly long _hashCS, _hashUC; @@ -71,4 +76,23 @@ public bool IsCI(ReadOnlySpan value) if (uc != _hashUC | value.Length != len) return false; return len <= MaxBytesHashed || SequenceEqualsCI(Span, value); } + + string IFormattable.ToString(string? format, IFormatProvider? formatProvider) => ToString(); + +#if NET + bool ISpanFormattable.TryFormat( + Span destination, + out int charsWritten, + ReadOnlySpan format, + IFormatProvider? provider) + { + charsWritten = 0; + var source = Span; + if (source.IsEmpty) return true; + if (source.Length > destination.Length) return false; + + charsWritten = Encoding.ASCII.GetChars(source, destination); + return true; + } +#endif } diff --git a/src/RESPite/Shared/Experiments.cs b/src/RESPite/Shared/Experiments.cs index e45379f55..f72ece073 100644 --- a/src/RESPite/Shared/Experiments.cs +++ b/src/RESPite/Shared/Experiments.cs @@ -18,7 +18,7 @@ internal static class Experiments // ReSharper disable InconsistentNaming public const string Respite = "SER004"; public const string UnitTesting = "SER005"; - + public const string GeoRedundantFailover = "SER007"; public const string Server_8_10 = "SER008"; // ReSharper restore InconsistentNaming diff --git a/src/StackExchange.Redis/AutoDatabase.cs b/src/StackExchange.Redis/AutoDatabase.cs new file mode 100644 index 000000000..407a2e3fa --- /dev/null +++ b/src/StackExchange.Redis/AutoDatabase.cs @@ -0,0 +1,223 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; + +namespace StackExchange.Redis; + +[Conditional("DEBUG")] +[AttributeUsage(AttributeTargets.Class | AttributeTargets.Interface, AllowMultiple = false)] +internal sealed class AutoDatabaseAttribute : Attribute +{ +} + +// Implemented by a generated captured-arguments struct only when the owning auto-database implements +// IRedisArgsMutator - i.e. only when something actually rewrites keys/channels. Everything else captures its +// arguments into a plain readonly struct with no interface at all; a funnel that wants to map opts in by +// constraining its TState to this (which is how KeyPrefixedDatabase will work once it moves onto +// [AutoDatabase]). +// +// Map mutates the struct in place, so a funnel that maps MUST take its state BY VALUE - never by `in`, because +// calling a non-readonly member through an `in` reference silently takes a defensive copy, mutates that, and +// throws it away. Invoke it via RedisArgsMutatorExtensions.MapInPlace, which makes that mistake a compile +// error (CS8329) instead of a silent loss of prefixing. (`ref` on the funnel is not an alternative: the +// generated call site passes a constructed temporary, which `ref` rejects with CS1510, and async funnels +// cannot have by-ref parameters at all.) The by-value copy is not waste here - a mapping funnel needs its own +// mutable copy by definition. Only non-mapping funnels take `in`, and their structs are readonly, so for them +// no defensive copy is even possible. +internal interface IMappableRedisArgs +{ + [Obsolete("Use MapInPlace instead; calling Map directly does nothing when the state is held by `in`.")] + void Map(IRedisArgsMutator mutator); + + object? UnMapper { get; } +} + +// The projections used by the auto-database funnels. There are exactly four real shapes, so each is named +// rather than being expressed generically over the target/return type: that lets the type system encode the +// invariants (sync goes to IDatabase, async goes to IDatabaseAsync and returns a Task) instead of leaving +// nonsense combinations like "sync database, Task result" expressible. +// +// They exist at all (rather than Func<,,>/Action<,>) purely to pass the captured state by readonly-reference: +// some of the generated state structs are chunky (a dozen-plus arguments), and by-value would copy the whole +// thing on every call. Note that a lambda only binds to an `in` parameter if it says so - the generator emits +// `static (in state, inner) => ...` accordingly. +// +// TState is necessarily invariant: it is passed by readonly-ref, and variant type parameters cannot carry +// the struct constraint. TResult is covariant on the sync projection (it is the direct return); it cannot be +// on the async one, because there it appears inside the invariant Task. +internal delegate void AutoDatabaseSyncOperation(in TState state, IDatabase database) + where TState : struct; + +internal delegate TResult AutoDatabaseSyncOperation(in TState state, IDatabase database) + where TState : struct; + +internal delegate Task AutoDatabaseAsyncOperation(in TState state, IDatabaseAsync database) + where TState : struct; + +internal delegate Task AutoDatabaseAsyncOperation(in TState state, IDatabaseAsync database) + where TState : struct; + +internal interface IRedisArgsMutator +{ + RedisKey Map(RedisKey key); + RedisChannel Map(RedisChannel channel); + + RedisKey UnMap(RedisKey key); + RedisChannel UnMap(RedisChannel channel); +} + +internal interface IRedisArgsResult +{ + T UnMap(IRedisArgsMutator mutator, T value); +} + +internal static class RedisArgsMutatorExtensions +{ + // these are used by the generated tuple-types via auto-database: each maps the key-bearing parts + // of an argument through the supplied mutator. They hang off IRedisArgsMutator (rather than the + // argument) so a call site reads consistently with the interface's own MapKey/MapChannel. + public static KeyValuePair Map( + this IRedisArgsMutator mutator, + KeyValuePair value) => + new(mutator.Map(value.Key), value.Value); + + // Funnels should map via this rather than calling state.Map(mutator) directly. Mapping mutates the struct + // in place, so it is only correct on a writable variable; the `ref this` here means that applying it to an + // `in` parameter fails to compile (CS8329) instead of silently taking a defensive copy, mutating that, and + // discarding it - which would leave keys unprefixed, i.e. a wrong-keyspace bug rather than a perf nit. + // (Deliberately not named Map: an extension cannot shadow the instance member, so the guard would be lost.) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void MapInPlace(this ref TState state, IRedisArgsMutator mutator) + where TState : struct, IMappableRedisArgs +#pragma warning disable CS0618 // the one sanctioned call site: `state` is a genuine `ref`, so this mutates in place + => state.Map(mutator); +#pragma warning restore CS0618 + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static TResult UnMap(this IRedisArgsMutator mutator, in TState state, TResult result) + where TState : struct, IMappableRedisArgs + { + // note JIT will elide these tests using per-TResult value-type rules + if (typeof(TResult) == typeof(RedisKey)) + { + var tmp = mutator.UnMap(Unsafe.As(ref result)); + return Unsafe.As(ref tmp); + } + + if (typeof(TResult) == typeof(RedisChannel)) + { + var tmp = mutator.UnMap(Unsafe.As(ref result)); + return Unsafe.As(ref tmp); + } + + if (typeof(TResult) == typeof(RedisValue)) + { + return result; // never mapped + } + + return state.UnMapper is IRedisArgsResult unmap ? unmap.UnMap(mutator, result) : result; + } + + [return: NotNullIfNotNull("keys")] + public static RedisKey[]? Map(this IRedisArgsMutator mutator, RedisKey[]? keys) + { + if (keys is null || keys.Length is 0) return keys; + var arr = new RedisKey[keys.Length]; + for (int i = 0; i < arr.Length; i++) + { + arr[i] = mutator.Map(keys[i]); + } + return arr; + } + + [return: NotNullIfNotNull("pairs")] + public static KeyValuePair[]? Map( + this IRedisArgsMutator mutator, + KeyValuePair[]? pairs) + { + if (pairs is null || pairs.Length is 0) return pairs; + var arr = new KeyValuePair[pairs.Length]; + for (int i = 0; i < arr.Length; i++) + { + ref readonly KeyValuePair pair = ref pairs[i]; + arr[i] = new(mutator.Map(pair.Key), pair.Value); + } + return arr; + } + + public static StreamPosition Map(this IRedisArgsMutator mutator, StreamPosition value) => + new(mutator.Map(value.Key), value.Position); + + [return: NotNullIfNotNull("positions")] + public static StreamPosition[]? Map(this IRedisArgsMutator mutator, StreamPosition[]? positions) + { + if (positions is null || positions.Length is 0) return positions; + var arr = new StreamPosition[positions.Length]; + for (int i = 0; i < arr.Length; i++) + { + ref readonly StreamPosition position = ref positions[i]; + arr[i] = new(mutator.Map(position.Key), position.Position); + } + return arr; + } + + // the Execute/ExecuteAsync escape hatch takes a loosely-typed arg list in which any element may + // be a boxed RedisKey or RedisChannel; these rewrite just those entries (mirroring + // KeyPrefixed.ToInner), copying only when there is something to rewrite so the common call allocates nothing. + [return: NotNullIfNotNull("args")] + public static object[]? Map(this IRedisArgsMutator mutator, object[]? args) + { + if (args is null || args.Length is 0) return args; + object[]? copy = null; + for (int i = 0; i < args.Length; i++) + { + if (args[i] is RedisKey key) + { + (copy ??= (object[])args.Clone())[i] = mutator.Map(key); + } + else if (args[i] is RedisChannel channel) + { + (copy ??= (object[])args.Clone())[i] = mutator.Map(channel); + } + } + return copy ?? args; + } + + [return: NotNullIfNotNull("args")] + public static ICollection? Map(this IRedisArgsMutator mutator, ICollection? args) + { + if (args is null || args.Count is 0) return args; + bool any = false; + foreach (var arg in args) + { + if (arg is RedisKey or RedisChannel) + { + any = true; + break; + } + } + if (!any) return args; + + var copy = new object[args.Count]; + int i = 0; + foreach (var arg in args) + { + copy[i++] = arg switch + { + RedisKey key => mutator.Map(key), + RedisChannel channel => mutator.Map(channel), + _ => arg, + }; + } + return copy; + } + + public static SortedSetPopResult UnMap(this IRedisArgsMutator mutator, SortedSetPopResult value) => + value.IsNull ? SortedSetPopResult.Null : new(mutator.Map(value.Key), value.Entries); + + public static ListPopResult UnMap(this IRedisArgsMutator mutator, ListPopResult value) => + value.IsNull ? ListPopResult.Null : new(mutator.Map(value.Key), value.Values); +} diff --git a/src/StackExchange.Redis/Availability/CircuitBreaker.cs b/src/StackExchange.Redis/Availability/CircuitBreaker.cs new file mode 100644 index 000000000..2eb136922 --- /dev/null +++ b/src/StackExchange.Redis/Availability/CircuitBreaker.cs @@ -0,0 +1,359 @@ +using System; +#if !NET8_0_OR_GREATER +using System.Diagnostics; +#endif +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using System.Threading; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Reports connection health by responding to observed success and failure conditions of processed messages. +/// +[Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +public abstract class CircuitBreaker +{ + internal const double DefaultFailureRateThreshold = 10; + internal const int DefaultMinimumNumberOfFailures = 1000; + internal static readonly TimeSpan DefaultMetricsWindowSize = TimeSpan.FromSeconds(2); + + private static readonly CircuitBreaker DefaultInstance = new DefaultCircuitBreaker( +#pragma warning disable SA1114 // Parameter list should follow declaration - false positive: the #if directive splits the argument list +#if NET8_0_OR_GREATER + null, +#endif +#pragma warning restore SA1114 + DefaultFailureRateThreshold, + DefaultMinimumNumberOfFailures, + DefaultMetricsWindowSize); + + /// + /// Default circuit-breaker logic: trips when the failure rate over a short rolling window crosses a threshold. + /// + public static CircuitBreaker Default => DefaultInstance; + + /// + /// No circuit-breaker logic is applied. + /// + public static CircuitBreaker None => NulCircuitBreaker.Instance; + + /// + /// Allows configuration of the default implementation. + /// + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + public sealed class Builder + { + /// + /// Percentage of failures to trigger circuit breaker. + /// + /// Failures are only included if they are of tracked exception types. + public double FailureRateThreshold { get; set; } = DefaultFailureRateThreshold; + + /// + /// Minimum failures before circuit breaker can open. + /// + public int MinimumNumberOfFailures { get; set; } = DefaultMinimumNumberOfFailures; + + /// + /// Time window for collecting metrics. + /// + public TimeSpan MetricsWindowSize { get; set; } = DefaultMetricsWindowSize; + +#if NET8_0_OR_GREATER + /// + /// Time source used to drive the metrics window; when null, the system clock is used. + /// Intended for testing, to make the time-windowed logic deterministic. + /// + internal TimeProvider? TimeProvider { get; set; } +#endif + + /// + /// Create a new circuit-breaker instance. + /// + public CircuitBreaker Create() + { + if (FailureRateThreshold is < 0 or > 100) throw new ArgumentOutOfRangeException(nameof(FailureRateThreshold), FailureRateThreshold, "A percentage between 0 and 100 is required."); + if (MinimumNumberOfFailures < 1) throw new ArgumentOutOfRangeException(nameof(MinimumNumberOfFailures), MinimumNumberOfFailures, "At least one failure is required; use CircuitBreaker.None to disable circuit-breaking."); + if (MetricsWindowSize <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(MetricsWindowSize), MetricsWindowSize, "A positive window is required."); + if (MetricsWindowSize.TotalSeconds > int.MaxValue) throw new ArgumentOutOfRangeException(nameof(MetricsWindowSize), MetricsWindowSize, "The window is too large."); + + if ((FailureRateThreshold is DefaultFailureRateThreshold +#if NET8_0_OR_GREATER + & TimeProvider is null +#endif + & MinimumNumberOfFailures is DefaultMinimumNumberOfFailures) + && MetricsWindowSize == DefaultMetricsWindowSize) + return DefaultInstance; + + return new DefaultCircuitBreaker( +#pragma warning disable SA1114 // Parameter list should follow declaration - false positive: the #if directive splits the argument list +#if NET8_0_OR_GREATER + TimeProvider, +#endif +#pragma warning restore SA1114 + FailureRateThreshold, + MinimumNumberOfFailures, + MetricsWindowSize); + } + + /// + /// Create a new circuit-breaker instance. + /// + public static implicit operator CircuitBreaker(Builder builder) => builder.Create(); + } + + /// + /// Create an object to collate observations for a connection. + /// + public abstract Accumulator CreateAccumulator(); + + internal static bool DefaultIsFailure(in FaultContext fault) + { + if (fault.ConnectionFailureType is not ConnectionFailureType.None) return true; + switch (fault.ErrorKind) + { + // what things *don't* trip the breaker? + case RedisErrorKind.None: // not even flagged + case RedisErrorKind.UnknownCommand: // application failure + case RedisErrorKind.ExecAbort: // transient to one command + case RedisErrorKind.WrongType: // application failure + case RedisErrorKind.NoPermission: // using the wrong keys? + case RedisErrorKind.UnknownError: // not sure what it is, but it starts ERR + case RedisErrorKind.Unknown: // pretty much anything we don't recognize; should we assume this is BAD? + return false; + default: + return true; + } + } + + /// + /// Collates observations for a connection. + /// + public abstract class Accumulator() + { + /// + /// Record a message outcome. + /// + /// struct arg here is in case we want to add more things later + public abstract void ObserveResult(in FaultContext fault); + + /// + /// Indicate whether a given fault should be considered a failure for the . + /// + protected virtual bool IsFailure(in FaultContext fault) => DefaultIsFailure(in fault); + + /// + /// Indicate whether the connection is currently considered healthy, without recording an observation. + /// + /// True if the connection is considered healthy. + public abstract bool IsHealthy(); + + /// + /// Discard all accumulated observations, returning to a clean state. + /// + public abstract void Reset(); + + internal bool Trip(Exception? fault) + { + if (fault is not null) + { + var ctx = new FaultContext(fault); + if (IsFailure(ctx)) + { + ObserveResult(ctx); + return !IsHealthy(); + } + // otherwise, treat as success for the purposes of counting + } + + ObserveResult(in FaultContext.Success); + return false; // never trip through success + } + } + + private sealed class NulCircuitBreaker : CircuitBreaker + { + public static readonly NulCircuitBreaker Instance = new(); + private NulCircuitBreaker() { } + public override Accumulator CreateAccumulator() => NulAccumulator.AccumulatorInstance; + + private sealed class NulAccumulator : Accumulator + { + public static readonly NulAccumulator AccumulatorInstance = new(); + private NulAccumulator() { } + public override void ObserveResult(in FaultContext context) { } + public override bool IsHealthy() => true; + public override void Reset() { } + } + + // note we leave IsConnectionFault alone - that would impact RetryDatabase, where this is the key + } + private sealed class DefaultCircuitBreaker : CircuitBreaker + { + private readonly double _failureRateThreshold; + private readonly int _minimumNumberOfFailures; + + // the metrics window is divided into a fixed number of equal time-slices ("buckets"); + // this lets us keep a rolling count cheaply, evicting whole buckets as they age out, + // rather than storing (and pruning) a timestamp per event + private const int BucketCount = 10; + private readonly long _bucketTicks; // width of one bucket, in high-resolution ticks + +#if NET8_0_OR_GREATER + private readonly TimeProvider _time; +#endif + + // which time-slice ("epoch") are we in right now: the high-resolution timestamp (from + // TimeProvider where available, so tests can drive it; otherwise the Stopwatch clock) + // divided down to bucket width + private long GetEpoch() => +#if NET8_0_OR_GREATER + _time.GetTimestamp() +#else + Stopwatch.GetTimestamp() +#endif + / _bucketTicks; + + public DefaultCircuitBreaker( +#if NET8_0_OR_GREATER + TimeProvider? timeProvider, +#endif + double failureRateThreshold, + int minimumNumberOfFailures, + TimeSpan metricsWindowSize) + { + _failureRateThreshold = failureRateThreshold; + _minimumNumberOfFailures = minimumNumberOfFailures; + +#if NET8_0_OR_GREATER + _time = timeProvider ?? TimeProvider.System; + long frequency = _time.TimestampFrequency; +#else + long frequency = Stopwatch.Frequency; +#endif + long windowTicks = (long)(metricsWindowSize.TotalSeconds * frequency); + _bucketTicks = Math.Max(1, windowTicks / BucketCount); + } + + public override Accumulator CreateAccumulator() => new DefaultAccumulator(this); + + private sealed class DefaultAccumulator(DefaultCircuitBreaker breaker) : Accumulator + { + // ring of buckets; each holds the counts for a single time-slice, tagged with the + // slice ("epoch") it currently represents so we can tell live buckets from stale ones +#if NET8_0_OR_GREATER + // inline it directly into the accumulator + private BucketRing _buckets; // cannot be "readonly", else the indexer is "ref readonly" + [InlineArray(BucketCount)] + private struct BucketRing + { + // beware init rules; we're OK in this case because _buckets is in a field on a heap object, + // but if BucketRing was ever used as a local: [SkipLocalsInit] rules can apply. + private Bucket _element0; + } +#else + // fallback to a separate heap array + private readonly Bucket[] _buckets = new Bucket[BucketCount]; +#endif + + private struct Bucket + { + private long _epoch; + private volatile int _success, _failure; + + // note that Volatile guarantees atomicity (even on x86), so no torn values here + // see https://learn.microsoft.com/dotnet/api/system.threading.volatile + public long Epoch => Volatile.Read(ref _epoch); + + public int Success => _success; + public int Failure => _failure; + + public void Count(long epoch, bool success) + { + if (epoch != Epoch) + { + // epoch rollover; clear the counts first, to prevent anyone over-counting + // in their count loop (under-counting is fine); dropped counts are self-correcting + // if there's an actual problem, stale data misread as current: is not. + _success = _failure = 0; + Volatile.Write(ref _epoch, epoch); + + // if we want to get *super* accurate, we could use bit-packing here and use + // CAS over a 64-bit value, but we'd need to compromise on count upper bounds + // *and* we'd need to consider the max epoch problem - maybe 32 bits for epoch + // and 16 bits for each count, but... let's keep things simple and accept a + // few dropped counts instead, and luxuriate in unreasonably fat epochs and counts. + } + // ReSharper disable ByRefArgumentIsVolatileField + Interlocked.Increment(ref success ? ref _success : ref _failure); + // ReSharper restore ByRefArgumentIsVolatileField + } + + public void Reset() + { + // clear the counts first (as in Count), so a concurrent count-loop reader never + // attributes these stale counts to the epoch; then blank the epoch back to "unused" + _success = _failure = 0; + Volatile.Write(ref _epoch, 0); + } + } + + public override void ObserveResult(in FaultContext result) + { + // which time-slice are we in, and where does it live in the ring? + long epoch = breaker.GetEpoch(); + int index = (int)(epoch % BucketCount); + + // note: to avoid concurrency problems, we're going lock-free here; *technically* this + // might mean we see race oddities during epoch rollovers, but in general that means we're + // miscounting by a tiny amount during intervals when there's enough load to get a race, + // in which case: we're probably fine; also, note that when tracking by endpoint, we're + // only getting results one at a time, so we shouldn't be over-stomping much *anyway* + Span buckets = _buckets; // *not* a payload copy; this is in-place over the data + ref Bucket bucket = ref buckets[index]; + bucket.Count(epoch, success: !result.IsFault); + } + + public override bool IsHealthy() => Evaluate(breaker.GetEpoch()); + + // evaluate health from the buckets still inside the window ending at the given epoch, + // without recording anything; shared by ObserveResult and IsHealthy + private bool Evaluate(long epoch) + { + // sum only the buckets still inside the window; anything older is ignored + // (and contributes nothing even if left un-recycled). empty/never-used buckets + // add zero, so no explicit "unused" sentinel is required + long oldest = epoch - BucketCount + 1; + int failures, total = failures = 0; + Span buckets = _buckets; // *not* a payload copy; this is in-place over the data + foreach (ref readonly Bucket b in buckets) + { + if (b.Epoch < oldest) continue; + + int failure; + failures += failure = b.Failure; // capture to avoid double-read (think epoch rollover) + total += b.Success + failure; + } + + // don't act until we've seen enough failures to be statistically meaningful + if (total is 0 | failures < breaker._minimumNumberOfFailures) + { + return true; + } + double failureRate = (failures * 100d) / total; + return failureRate < breaker._failureRateThreshold; + } + + public override void Reset() + { + Span buckets = _buckets; // in-place over the data, not a copy + foreach (ref Bucket b in buckets) + { + b.Reset(); + } + } + } + } +} diff --git a/src/StackExchange.Redis/Availability/DatabaseExtensions.cs b/src/StackExchange.Redis/Availability/DatabaseExtensions.cs new file mode 100644 index 000000000..d25c9a8e0 --- /dev/null +++ b/src/StackExchange.Redis/Availability/DatabaseExtensions.cs @@ -0,0 +1,48 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Provides availability-related extension methods (such as ) to database instances. +/// +public static class DatabaseExtensions +{ + /// + /// Automatically retry operations when connection failure occurs. This has deep integration with + /// SE.Redis concepts, so can respond to server failover events, apply circuit-breaker rules, and + /// respect command effect categorization. + /// + /// The database to wrap; this must not be a batch, a transaction, an + /// already-retrying database, or a database carrying an asyncState (see remarks). + /// + /// The policy to apply; when (the default), the policy configured for the + /// underlying connection is used - for a connection group, + /// for a single connection, else + /// . + /// + /// + /// asyncState is not supported. A database's asyncState is stamped onto the task + /// produced by a single dispatch, but a retrying database hands back its own task spanning however + /// many attempts the operation takes; the same is true of the per-operation tasks handed out by + /// on such a database. Rather than dropping + /// the state silently, both refuse it: wrapping a database obtained via + /// GetDatabase(db, asyncState) throws, as does supplying an asyncState when creating a + /// transaction from a retrying database. + /// + /// If is a batch, a + /// transaction, already retrying, or carries an asyncState. + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + public static IDatabaseAsync WithRetry(this IDatabaseAsync database, RetryPolicy? retryPolicy = null) + => new RetryDatabase(database, retryPolicy ?? ResolveRetryPolicy(database)); + + // IDatabaseAsync always exposes its multiplexer (via IRedisAsync), so the configured policy is reachable + // without the caller having to thread it through; note IConnectionMultiplexer is a public interface that + // callers may implement or mock, so every step here degrades to the default rather than assuming a type + private static RetryPolicy ResolveRetryPolicy(IDatabaseAsync database) => database.Multiplexer switch + { + IConnectionGroup group => group.Options.RetryPolicy, + IInternalConnectionMultiplexer muxer => muxer.RawConfig.RetryPolicy ?? RetryPolicy.Default, + _ => RetryPolicy.Default, + }; +} diff --git a/src/StackExchange.Redis/Availability/FaultContext.cs b/src/StackExchange.Redis/Availability/FaultContext.cs new file mode 100644 index 000000000..dbb055f15 --- /dev/null +++ b/src/StackExchange.Redis/Availability/FaultContext.cs @@ -0,0 +1,148 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Provides information about a circuit-breaker test. +/// +[Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +public readonly struct FaultContext +{ + private readonly Exception? _fault; + private readonly ConnectionFailureType _connectionFailureType; + private readonly CommandFlags _flags; + + internal static readonly FaultContext Success = default; + + /// + /// Create a new . + /// + /// The fault associated with the operation, or null on success. + public FaultContext(Exception fault) + { + _fault = fault; + + var kind = RedisErrorKind.None; + _connectionFailureType = ConnectionFailureType.None; + var flags = CommandFlags.None; + var status = CommandStatus.Unknown; + switch (fault) + { + case RedisServerException server: + kind = server.Kind; + flags = server.Flags; + break; + case RedisConnectionException connection: + _connectionFailureType = connection.FailureType; + kind = RedisErrorKind.ConnectionFault; + flags = connection.Flags; + status = connection.CommandStatus; + break; + case RedisTimeoutException timeout: + kind = RedisErrorKind.Timeout; + flags = timeout.Flags; + status = timeout.Commandstatus; + break; + case TimeoutException: + kind = RedisErrorKind.Timeout; + break; + } + + NotApplied = IsKnownNotApplied(kind, status); + + if (kind is not RedisErrorKind.None & _connectionFailureType is ConnectionFailureType.None) + { + // fill in some blanks + switch (kind) + { + case RedisErrorKind.Loading: + _connectionFailureType = ConnectionFailureType.Loading; + break; + case RedisErrorKind.NoAuth: + case RedisErrorKind.WrongPass: + _connectionFailureType = ConnectionFailureType.AuthenticationFailure; + break; + } + } + + flags &= Message.UserSelectableFlags; + if ((flags & Message.MaskRetryCategory) is 0) + { + // if no retry category found: assume the worst + flags |= CommandFlags.CommandRetryNever; + } + _flags = flags; + ErrorKind = kind; + } + + /// + /// Indicates whether a fault is present. + /// + [MemberNotNullWhen(true, nameof(Fault))] + public bool IsFault => _fault is not null; // excludes: default(FaultContext) + + /// + /// The fault associated with the operation. + /// + public Exception? Fault => _fault; + + /// + /// Get any command-flags associated with the operation. + /// + public CommandFlags Flags => _flags; + + /// + /// The classified server error condition associated with the fault, if any. + /// + public RedisErrorKind ErrorKind { get; } + + /// + /// Indicates that the operation is known *not* to have been applied by the server - either because it + /// never left the client, or because the server explicitly rejected it due to its own state (still + /// loading, cluster down, writes refused, and so on). Retrying such an operation is a *first* attempt + /// rather than a repeat, so it cannot double-apply a side-effect; therefore + /// ignores in this case. + /// + /// + /// This is deliberately conservative: it is only reported for conditions that the server can *only* + /// raise before running anything (so a Lua script that failed part-way through cannot be mistaken for + /// one that never ran), and for messages the client knows it never wrote. Everything else - notably + /// timeouts, and connection loss after the request was sent - remains ambiguous and is not flagged. + /// + public bool NotApplied { get; } + + /// + /// The connection failure type associated with the fault, if any. + /// + public ConnectionFailureType ConnectionFailureType => _connectionFailureType; + + private static bool IsKnownNotApplied(RedisErrorKind kind, CommandStatus status) + { + // the client never handed it to the socket, so the server cannot have seen it + if (status is CommandStatus.WaitingToBeSent or CommandStatus.WaitingInBacklog) return true; + + // an error *reply* usually means the server declined to run the command, but not always: a Lua + // script can fail part-way through, having already applied earlier writes, and it propagates the + // inner error verbatim (WRONGTYPE, and so on). So we only trust the conditions that describe the + // *server's own state*, which it can only report before running anything. + switch (kind) + { + case RedisErrorKind.Loading: // still loading the dataset + case RedisErrorKind.ClusterDown: // slot not currently served + case RedisErrorKind.MasterDown: // replica cannot serve, primary is unavailable + case RedisErrorKind.TryAgain: // slot mid-migration + case RedisErrorKind.Moved: // wrong node; this one did not run it + case RedisErrorKind.Ask: // ditto, mid-migration + case RedisErrorKind.ReadOnly: // writes refused by a replica + case RedisErrorKind.Misconfigured: // e.g. persistence failing, so writes are refused + case RedisErrorKind.NoReplicas: // not enough replicas to accept the write + case RedisErrorKind.Busy: // a script is hogging the server + case RedisErrorKind.MaxClients: // refused at the door + return true; + default: + return false; + } + } +} diff --git a/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs b/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs new file mode 100644 index 000000000..1ee5dda8a --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheck.Execute.cs @@ -0,0 +1,207 @@ +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; + +namespace StackExchange.Redis.Availability; + +public sealed partial class HealthCheck +{ + /// + /// Evaluate the health of the specified multiplexer, by evaluating all endpoints. + /// + public Task CheckHealthAsync(IConnectionMultiplexer multiplexer) + => !IsEnabled ? HealthCheckProbe.InconclusiveTask + : multiplexer.IsConnected ? CheckHealthCoreAsync(multiplexer) + : HealthCheckProbe.UnhealthyTask; + + private async Task CheckHealthCoreAsync(IConnectionMultiplexer multiplexer) + { + try + { + Task[] pending; + if (multiplexer is IInternalConnectionMultiplexer internalMultiplexer) + { + var snapshot = internalMultiplexer.GetServerSnapshot(); + pending = GetReusablePending(ref _reusablePending, snapshot.Length); + for (int i = 0; i < pending.Length; i++) + { + pending[i] = CheckHealthAsync(snapshot[i].GetRedisServer(null)); + } + } + else + { + var servers = multiplexer.GetServers(); + pending = GetReusablePending(ref _reusablePending, servers.Length); + for (int i = 0; i < pending.Length; i++) + { + pending[i] = CheckHealthAsync(servers[i]); + } + } + var result = await CollateAsync(pending, TotalTimeoutMillis()).ForAwait(); + + // on successful completion (regardless of outcome), we can reuse the pending array + PutReusablePending(ref _reusablePending, ref pending); + return result; + } + catch + { + // definitely unhappy + return HealthCheckResult.Unhealthy; + } + } + + internal int TotalTimeoutMillis() + { + bool valid = TryComputeTotalTimeoutMillis(ProbeCount, ProbeTimeout, ProbeInterval, out int millis); + Debug.Assert(valid, "The probe budget is validated by HealthCheck.Builder.Create, so should always be usable here."); + return millis; + } + + // the total time budget for a full health check, in milliseconds; shared with Builder.Create, which uses + // it to reject a configuration whose budget cannot be expressed (rather than overflowing at check time) + private static bool TryComputeTotalTimeoutMillis(int probeCount, TimeSpan probeTimeout, TimeSpan probeInterval, out int millis) + { + millis = 0; + if (probeCount < 1 || probeTimeout <= TimeSpan.Zero || probeInterval < TimeSpan.Zero) return false; + + try + { + // the first probe doesn't have an interval before it, the rest do + long totalTicks = checked(probeTimeout.Ticks + ((probeTimeout.Ticks + probeInterval.Ticks) * (probeCount - 1))); + long totalMillis = totalTicks / TimeSpan.TicksPerMillisecond; + if (totalMillis is <= 0 or > int.MaxValue) return false; + + millis = (int)totalMillis; + return true; + } + catch (OverflowException) + { + return false; + } + } + + // apply timeout and collation logic to a group of probes + internal static async Task CollateAsync(Task[] probes, int timeoutMilliseconds) + { + var pendingAll = Task.WhenAll(probes).ObserveErrors(); + int success = 0, failure = 0; + + if (await pendingAll.TimeoutAfter(timeoutMilliseconds).ForAwait()) + { + // all completed inside timeout; all results should now be available + for (int i = 0; i < probes.Length; i++) + { + var individualResult = await probes[i].ForAwait(); + switch (individualResult) + { + case HealthCheckResult.Healthy: success++; break; + case HealthCheckResult.Unhealthy: failure++; break; + } + } + } + else + { + // timeout + for (int i = 0; i < probes.Length; i++) + { + _ = probes[i].ObserveErrors(); + } + throw new TimeoutException(); + } + + if (failure > 0) return HealthCheckResult.Unhealthy; + if (success > 0) return HealthCheckResult.Healthy; + return HealthCheckResult.Inconclusive; + } + + private Task[]? _reusablePending; + + // The number of pending tasks is determined by the number of endpoints, which doesn't change frequently + // (if at all); consequently, we can often re-use this buffer between health-checks, as long as we're careful. + internal static Task[] GetReusablePending(ref Task[]? field, int count) + { + var result = Interlocked.Exchange(ref field, null); + if (result is null || result.Length != count) + { + result = count == 0 ? [] : new Task[count]; + } + return result; + } + + internal static void PutReusablePending(ref Task[]? field, ref Task[] value) + { + if (value is { Length: > 0 }) + { + Array.Clear(value, 0, value.Length); + Interlocked.Exchange(ref field, value); + value = []; + } + } + + /// + /// Evaluate the health of an endpoint. + /// + public Task CheckHealthAsync(IServer server) + => !IsEnabled ? HealthCheckProbe.InconclusiveTask + : server.IsConnected ? CheckHealthCoreAsync(server) + : HealthCheckProbe.UnhealthyTask; + + private async Task CheckHealthCoreAsync(IServer server) + { + try + { + int timeout = (int)ProbeTimeout.TotalMilliseconds, success = 0, failure = 0, remaining = ProbeCount; + HealthCheckContext context = new(server, ProbeTimeout); + while (remaining > 0) + { + HealthCheckResult probeResult; + try + { + var pendingProbe = Probe.CheckHealthAsync(context); + probeResult = await pendingProbe.TimeoutAfter(timeout).ForAwait() + ? await pendingProbe.ForAwait() // completed + : HealthCheckResult.Unhealthy; // timeout + } + catch + { + probeResult = HealthCheckResult.Unhealthy; + } + + // update success/failure counts + switch (probeResult) + { + case HealthCheckResult.Healthy: success++; break; + case HealthCheckResult.Unhealthy: failure++; break; + } + HealthCheckProbeContext ctx = new(probeResult, success, failure, --remaining, ProbeInterval); + + // evaluate the policy + var policyResult = ProbePolicy.Evaluate(in ctx); + if (policyResult != HealthCheckResult.Inconclusive) return policyResult; + + if (remaining > 0) + { + // delay between probes when the policy calls for it (today: only after a failure). + // Note: TotalTimeoutMillis() assumes this effective interval equals ProbeInterval; if + // the policy ever returns a variable interval (backoff/jitter), the budget formula must + // be revisited in lockstep, else the outer timeout can fire and mark a member unhealthy. + var delay = ProbePolicy.GetEffectiveProbeInterval(in ctx); + if (delay > TimeSpan.Zero) + { + await Task.Delay(delay).ConfigureAwait(false); + } + } + } + + // we got here without a result + return HealthCheckResult.Inconclusive; + } + catch (Exception ex) + { + // if the health check utterly fails: that isn't a good sign + Debug.WriteLine(ex.Message); + return HealthCheckResult.Unhealthy; + } + } +} diff --git a/src/StackExchange.Redis/Availability/HealthCheck.cs b/src/StackExchange.Redis/Availability/HealthCheck.cs new file mode 100644 index 000000000..c7d562ab1 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheck.cs @@ -0,0 +1,186 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Describes a health check to perform against instances. +/// +/// +/// Instances are immutable and safe to share between members; use to configure +/// a custom check. Note that how often checks run is a group-level concern, configured via +/// , not a property of the check itself. +/// +[Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +public sealed partial class HealthCheck +{ + internal const int DefaultProbeCount = 3; + internal static readonly TimeSpan DefaultProbeTimeout = TimeSpan.FromSeconds(3); + internal static readonly TimeSpan DefaultProbeInterval = TimeSpan.FromMilliseconds(500); + + private static readonly HealthCheck DefaultInstance = new( + enabled: true, + DefaultProbeCount, + DefaultProbeTimeout, + DefaultProbeInterval, + HealthCheckProbe.Ping, + HealthCheckProbePolicy.AllSuccess); + + private static readonly HealthCheck DisabledInstance = new( + enabled: false, + DefaultProbeCount, + DefaultProbeTimeout, + DefaultProbeInterval, + HealthCheckProbe.None, + HealthCheckProbePolicy.AllSuccess); + + /// + /// The default health check: three probes, all of which must succeed. + /// + public static HealthCheck Default => DefaultInstance; + + /// + /// No health check is performed; every check reports , leaving + /// member selection driven purely by the observed connectivity of each member (and by any circuit-breaker). + /// + public static HealthCheck None => DisabledInstance; + + private HealthCheck( + bool enabled, + int probeCount, + TimeSpan probeTimeout, + TimeSpan probeInterval, + HealthCheckProbe probe, + HealthCheckProbePolicy probePolicy) + { + IsEnabled = enabled; + ProbeCount = probeCount; + ProbeTimeout = probeTimeout; + ProbeInterval = probeInterval; + Probe = probe; + ProbePolicy = probePolicy; + } + + /// + public override string ToString() => IsEnabled + ? $"{Probe.GetType().Name} x{ProbeCount} ({ProbePolicy.GetType().Name})" + : "(disabled)"; + + /// + /// Whether this health check performs any probes; false only for . + /// + public bool IsEnabled { get; } + + /// + /// Gets the number of probes to perform for this health check. + /// + public int ProbeCount { get; } + + /// + /// Gets the time that should be allowed for an individual probe to complete. + /// + public TimeSpan ProbeTimeout { get; } + + /// + /// Gets the interval between failed probes. + /// + public TimeSpan ProbeInterval { get; } + + /// + /// Gets the probe to use for this health check. + /// + public HealthCheckProbe Probe { get; } + + /// + /// Gets the policy to use for this health check. + /// + public HealthCheckProbePolicy ProbePolicy { get; } + + /// + /// Allows configuration of a . + /// + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + public sealed class Builder + { + /// + /// Create a builder pre-populated with the default values. + /// + public Builder() + { + } + + /// + /// Create a builder pre-populated from an existing . + /// + public Builder(HealthCheck healthCheck) + { + ProbeCount = healthCheck.ProbeCount; + ProbeTimeout = healthCheck.ProbeTimeout; + ProbeInterval = healthCheck.ProbeInterval; + Probe = healthCheck.Probe; + ProbePolicy = healthCheck.ProbePolicy; + } + + /// + /// The number of probes to perform for this health check. + /// + public int ProbeCount { get; set; } = DefaultProbeCount; + + /// + /// The time that should be allowed for an individual probe to complete. + /// + public TimeSpan ProbeTimeout { get; set; } = DefaultProbeTimeout; + + /// + /// The interval between failed probes. + /// + public TimeSpan ProbeInterval { get; set; } = DefaultProbeInterval; + + /// + /// The probe to use for this health check. + /// + public HealthCheckProbe Probe { get; set; } = HealthCheckProbe.Ping; + + /// + /// The policy to use for this health check. + /// + public HealthCheckProbePolicy ProbePolicy { get; set; } = HealthCheckProbePolicy.AllSuccess; + + /// + /// Create a new health check instance. + /// + public HealthCheck Create() + { + if (ProbeCount < 1) throw new ArgumentOutOfRangeException(nameof(ProbeCount), ProbeCount, "At least one probe is required; use HealthCheck.None to disable health checks."); + if (ProbeTimeout <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(ProbeTimeout), ProbeTimeout, "A positive probe timeout is required."); + if (ProbeInterval < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(ProbeInterval), ProbeInterval, "A non-negative probe interval is required."); + if (Probe is null) throw new ArgumentNullException(nameof(Probe)); + if (ProbePolicy is null) throw new ArgumentNullException(nameof(ProbePolicy)); + + // the total budget is expressed in int milliseconds when the check runs; validate that here, + // rather than letting it overflow into a nonsensical (or negative) timeout later + if (!TryComputeTotalTimeoutMillis(ProbeCount, ProbeTimeout, ProbeInterval, out _)) + { + throw new ArgumentOutOfRangeException(nameof(ProbeTimeout), "The combined probe budget (ProbeCount, ProbeTimeout, ProbeInterval) is too large."); + } + + // prefer the shared default instance when nothing has been customized + if (ProbeCount == DefaultProbeCount + && ProbeTimeout == DefaultProbeTimeout + && ProbeInterval == DefaultProbeInterval + && ReferenceEquals(Probe, HealthCheckProbe.Ping) + && ReferenceEquals(ProbePolicy, HealthCheckProbePolicy.AllSuccess)) + { + return DefaultInstance; + } + + return new HealthCheck(enabled: true, ProbeCount, ProbeTimeout, ProbeInterval, Probe, ProbePolicy); + } + + /// + /// Create a new health check instance. + /// + public static implicit operator HealthCheck(Builder builder) => builder.Create(); + } +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckContext.cs b/src/StackExchange.Redis/Availability/HealthCheckContext.cs new file mode 100644 index 000000000..6d3fd5037 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckContext.cs @@ -0,0 +1,30 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Describes the target of a single health-check probe, and the budget available to it. +/// +/// +/// This is passed by value rather than by in, because probe implementations are +/// typically async, and async methods cannot take by-ref parameters. +/// +[Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +public readonly struct HealthCheckContext(IServer server, TimeSpan probeTimeout) +{ + /// + public override string ToString() => $"{Server?.EndPoint} (timeout: {ProbeTimeout})"; + + /// + /// Gets the server being probed. + /// + public IServer Server => server; + + /// + /// Gets the time allowed for this probe to complete; probes are not required to enforce this + /// themselves (the caller applies it), but may use it to bound any state they create. + /// + public TimeSpan ProbeTimeout => probeTimeout; +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.Connected.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.Connected.cs new file mode 100644 index 000000000..6d058f3a3 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.Connected.cs @@ -0,0 +1,20 @@ +using System.Threading.Tasks; + +namespace StackExchange.Redis.Availability; + +public abstract partial class HealthCheckProbe +{ + /// + /// Report health using the property, without any additional tests. + /// + public static HealthCheckProbe IsConnected => ConnectedProbe.Instance; + + private sealed class ConnectedProbe : HealthCheckProbe + { + public static ConnectedProbe Instance { get; } = new(); + private ConnectedProbe() { } + + public override Task CheckHealthAsync(HealthCheckContext context) + => context.Server.IsConnected ? HealthyTask : UnhealthyTask; + } +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.None.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.None.cs new file mode 100644 index 000000000..c114193e9 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.None.cs @@ -0,0 +1,21 @@ +using System.Threading.Tasks; + +namespace StackExchange.Redis.Availability; + +public abstract partial class HealthCheckProbe +{ + /// + /// Performs no test at all, always reporting ; this is the + /// probe used by , and leaves member selection driven purely by the + /// observed connectivity of each member. + /// + public static HealthCheckProbe None => NoneProbe.Instance; + + private sealed class NoneProbe : HealthCheckProbe + { + public static NoneProbe Instance { get; } = new(); + private NoneProbe() { } + + public override Task CheckHealthAsync(HealthCheckContext context) => InconclusiveTask; + } +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs new file mode 100644 index 000000000..64b1e882d --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.Ping.cs @@ -0,0 +1,23 @@ +using System.Threading.Tasks; + +namespace StackExchange.Redis.Availability; + +public abstract partial class HealthCheckProbe +{ + /// + /// Verify that the server is responsive by sending a PING command. + /// + public static HealthCheckProbe Ping => PingProbe.Instance; + + private sealed class PingProbe : HealthCheckProbe + { + public static PingProbe Instance { get; } = new(); + private PingProbe() { } + + public override async Task CheckHealthAsync(HealthCheckContext context) + { + await context.Server.PingAsync(); + return HealthCheckResult.Healthy; + } + } +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs new file mode 100644 index 000000000..6e1524b7a --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.StringSet.cs @@ -0,0 +1,54 @@ +using System; +using System.Buffers; +using System.Threading.Tasks; + +namespace StackExchange.Redis.Availability; + +public abstract partial class HealthCheckProbe +{ + /// + /// Verify that a string can be successfully set and retrieved. + /// + public static HealthCheckProbe StringSet => StringSetProbe.Instance; + + internal sealed class StringSetProbe : KeyWriteHealthCheckProbe + { + public static StringSetProbe Instance { get; } = new(); + private StringSetProbe() { } + +#if !NET + private static Random SharedRandom { get; } = new(); +#endif + + public override async Task CheckHealthAsync(HealthCheckContext context, IDatabaseAsync database, RedisKey key) + { + // note we use the lock API here because that can selectively choose between appropriate strategies for + // different server versions, including DELEX + const int LEN = 16; + var pooled = ArrayPool.Shared.Rent(LEN); +#if NET + Random.Shared.NextBytes(pooled.AsSpan(0, LEN)); +#else + SharedRandom.NextBytes(pooled); +#endif + var payload = (RedisValue)pooled.AsMemory(0, LEN); + try + { + // write a value to the db + await database.LockTakeAsync( + key: key, + value: payload, + expiry: context.ProbeTimeout, + flags: CommandFlags.FireAndForget).ForAwait(); + + // release from the db if matches (otherwise, we have no clue what happened, so: leave alone) + var success = await database.LockReleaseAsync(key, payload).ForAwait(); + return success ? HealthCheckResult.Healthy : HealthCheckResult.Unhealthy; + } + finally + { + ArrayPool.Shared.Return(pooled); + } + } + } +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbe.cs b/src/StackExchange.Redis/Availability/HealthCheckProbe.cs new file mode 100644 index 000000000..7816d805d --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckProbe.cs @@ -0,0 +1,59 @@ +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Threading.Tasks; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Describes an operation to perform as part of a health check. +/// +[Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +public abstract partial class HealthCheckProbe +{ + /// + /// Check the health of the specified endpoint. + /// + public abstract Task CheckHealthAsync(HealthCheckContext context); + + private static Task? _inconclusive, _healthy, _unhealthy; + + /// + /// Reports a memoized probe that was skipped without being evaluated. + /// + protected internal static Task InconclusiveTask => _inconclusive ??= Task.FromResult(HealthCheckResult.Inconclusive); + + /// + /// Reports a memoized probe that was healthy. + /// + protected internal static Task HealthyTask => _healthy ??= Task.FromResult(HealthCheckResult.Healthy); + + /// + /// Reports a memoized probe that was unhealthy. + /// + protected internal static Task UnhealthyTask => _unhealthy ??= Task.FromResult(HealthCheckResult.Unhealthy); +} + +/// +/// Describes a key-based (write) operation to perform as part of a health check. +/// +[Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +public abstract class KeyWriteHealthCheckProbe : HealthCheckProbe +{ + /// + public sealed override Task CheckHealthAsync(HealthCheckContext context) + { + var server = context.Server; + if (server.IsReplica) return InconclusiveTask; + + RedisKey key = server.InventKey("health-check/"); + if (key.IsNull) return InconclusiveTask; + Debug.Assert(server.Multiplexer.GetServer(key).EndPoint == server.EndPoint, "Key was not routed to the correct endpoint"); + return CheckHealthAsync(context, server.Multiplexer.GetDatabase(), key); + } + + /// + /// Check the health of the specified database using the provided key. + /// + public abstract Task CheckHealthAsync(HealthCheckContext context, IDatabaseAsync database, RedisKey key); +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbeContext.cs b/src/StackExchange.Redis/Availability/HealthCheckProbeContext.cs new file mode 100644 index 000000000..39d19a75e --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckProbeContext.cs @@ -0,0 +1,40 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Represents the context of a health check probe. +/// +[Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +public readonly struct HealthCheckProbeContext(HealthCheckResult result, int success, int failure, int remaining, TimeSpan probeInterval) +{ + /// + public override string ToString() => $"Result: {Result}, Success: {Success}, Failure: {Failure}, Remaining: {Remaining}, ProbeInterval: {ProbeInterval}"; + + /// + /// Gets the most recent result. + /// + public HealthCheckResult Result => result; + + /// + /// Gets the number of successful health checks. + /// + public int Success => success; + + /// + /// Gets the number of failed health checks. + /// + public int Failure => failure; + + /// + /// Gets the number of remaining health checks. + /// + public int Remaining => remaining; + + /// + /// Gets the interval to wait before the next probe attempt. + /// + public TimeSpan ProbeInterval => probeInterval; +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckProbePolicy.cs b/src/StackExchange.Redis/Availability/HealthCheckProbePolicy.cs new file mode 100644 index 000000000..cfbd824b7 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckProbePolicy.cs @@ -0,0 +1,102 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Attempt to evaluate the outcome of a series of health check operations. +/// +[Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +public abstract class HealthCheckProbePolicy +{ + /// + /// Attempt to evaluate the policy given the current context. + /// + /// The state of the probes so far. + /// The result of the policy evaluation. + public abstract HealthCheckResult Evaluate(in HealthCheckProbeContext context); + + /// + /// Get the interval to wait before the next probe attempt. + /// + internal TimeSpan GetEffectiveProbeInterval(in HealthCheckProbeContext context) + { + // if we make this public / overrideable, we will need to think about the max delay timeout + + // only delay between failures + return context.Result is HealthCheckResult.Unhealthy ? context.ProbeInterval : TimeSpan.Zero; + } + + /// + /// Require all probes to succeed. + /// + public static HealthCheckProbePolicy AllSuccess => AllSuccessHealthCheckProbePolicy.Instance; + + /// + /// Require at least one probe to succeed. + /// + public static HealthCheckProbePolicy AnySuccess => AnySuccessHealthCheckProbePolicy.Instance; + + /// + /// Require a majority of probes to succeed. + /// + public static HealthCheckProbePolicy MajoritySuccess => MajoritySuccessHealthCheckProbePolicy.Instance; + + private sealed class AllSuccessHealthCheckProbePolicy : HealthCheckProbePolicy + { + public static readonly AllSuccessHealthCheckProbePolicy Instance = new(); + private AllSuccessHealthCheckProbePolicy() { } + + public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) + { + // Fail as soon as we have any failure + if (context.Failure > 0) return HealthCheckResult.Unhealthy; + + // Succeed only when all probes have succeeded (no remaining) + if (context.Remaining == 0) return HealthCheckResult.Healthy; + + // Can't determine yet + return HealthCheckResult.Inconclusive; + } + } + + private sealed class AnySuccessHealthCheckProbePolicy : HealthCheckProbePolicy + { + public static readonly AnySuccessHealthCheckProbePolicy Instance = new(); + private AnySuccessHealthCheckProbePolicy() { } + + public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) + { + // Succeed as soon as we have any success + if (context.Success > 0) return HealthCheckResult.Healthy; + + // Fail only when all probes have failed (no remaining) + if (context.Remaining == 0) return HealthCheckResult.Unhealthy; + + // Can't determine yet + return HealthCheckResult.Inconclusive; + } + } + + private sealed class MajoritySuccessHealthCheckProbePolicy : HealthCheckProbePolicy + { + public static readonly MajoritySuccessHealthCheckProbePolicy Instance = new(); + private MajoritySuccessHealthCheckProbePolicy() { } + + public override HealthCheckResult Evaluate(in HealthCheckProbeContext context) + { + int total = context.Success + context.Failure + context.Remaining; + int majority = (total / 2) + 1; + + // Succeed as soon as we have enough successes for a majority + if (context.Success >= majority) return HealthCheckResult.Healthy; + + // Fail as soon as we have enough failures to make a majority impossible + if (context.Failure >= majority) return HealthCheckResult.Unhealthy; + + // Can't determine yet + return HealthCheckResult.Inconclusive; + } + } +} diff --git a/src/StackExchange.Redis/Availability/HealthCheckResult.cs b/src/StackExchange.Redis/Availability/HealthCheckResult.cs new file mode 100644 index 000000000..79528f8e5 --- /dev/null +++ b/src/StackExchange.Redis/Availability/HealthCheckResult.cs @@ -0,0 +1,26 @@ +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Indicates the result of a health check. +/// +[Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +public enum HealthCheckResult +{ + /// + /// The health check was skipped or could not be determined. + /// + Inconclusive, + + /// + /// The health check was successful. + /// + Healthy, + + /// + /// The health check failed. + /// + Unhealthy, +} diff --git a/src/StackExchange.Redis/Availability/MultiGroupDatabase.Scans.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Scans.cs new file mode 100644 index 000000000..e9fbe96df --- /dev/null +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.Scans.cs @@ -0,0 +1,47 @@ +using System.Collections.Generic; + +namespace StackExchange.Redis.Availability; + +// Streaming cursor scans (IEnumerable / IAsyncEnumerable) are deferred-execution and don't fit the +// capture-and-replay shape, so [AutoDatabase] skips them; they forward straight to the active member. +internal sealed partial class MultiGroupDatabase +{ + public IEnumerable HashScan(RedisKey key, RedisValue pattern, int pageSize, CommandFlags flags) + => GetActiveDatabase().HashScan(key, pattern, pageSize, flags); + + public IEnumerable HashScan(RedisKey key, RedisValue pattern = default, int pageSize = RedisBase.CursorUtils.DefaultLibraryPageSize, long cursor = RedisBase.CursorUtils.Origin, int pageOffset = 0, CommandFlags flags = CommandFlags.None) + => GetActiveDatabase().HashScan(key, pattern, pageSize, cursor, pageOffset, flags); + + public IEnumerable HashScanNoValues(RedisKey key, RedisValue pattern = default, int pageSize = RedisBase.CursorUtils.DefaultLibraryPageSize, long cursor = RedisBase.CursorUtils.Origin, int pageOffset = 0, CommandFlags flags = CommandFlags.None) + => GetActiveDatabase().HashScanNoValues(key, pattern, pageSize, cursor, pageOffset, flags); + + public IEnumerable SetScan(RedisKey key, RedisValue pattern, int pageSize, CommandFlags flags) + => GetActiveDatabase().SetScan(key, pattern, pageSize, flags); + + public IEnumerable SetScan(RedisKey key, RedisValue pattern = default, int pageSize = RedisBase.CursorUtils.DefaultLibraryPageSize, long cursor = RedisBase.CursorUtils.Origin, int pageOffset = 0, CommandFlags flags = CommandFlags.None) + => GetActiveDatabase().SetScan(key, pattern, pageSize, cursor, pageOffset, flags); + + public IEnumerable SortedSetScan(RedisKey key, RedisValue pattern, int pageSize, CommandFlags flags) + => GetActiveDatabase().SortedSetScan(key, pattern, pageSize, flags); + + public IEnumerable SortedSetScan(RedisKey key, RedisValue pattern = default, int pageSize = RedisBase.CursorUtils.DefaultLibraryPageSize, long cursor = RedisBase.CursorUtils.Origin, int pageOffset = 0, CommandFlags flags = CommandFlags.None) + => GetActiveDatabase().SortedSetScan(key, pattern, pageSize, cursor, pageOffset, flags); + + public IEnumerable VectorSetRangeEnumerate(RedisKey key, RedisValue start = default, RedisValue end = default, long count = 100, Exclude exclude = Exclude.None, CommandFlags flags = CommandFlags.None) + => GetActiveDatabase().VectorSetRangeEnumerate(key, start, end, count, exclude, flags); + + public IAsyncEnumerable HashScanAsync(RedisKey key, RedisValue pattern = default, int pageSize = RedisBase.CursorUtils.DefaultLibraryPageSize, long cursor = RedisBase.CursorUtils.Origin, int pageOffset = 0, CommandFlags flags = CommandFlags.None) + => GetActiveDatabase().HashScanAsync(key, pattern, pageSize, cursor, pageOffset, flags); + + public IAsyncEnumerable HashScanNoValuesAsync(RedisKey key, RedisValue pattern = default, int pageSize = RedisBase.CursorUtils.DefaultLibraryPageSize, long cursor = RedisBase.CursorUtils.Origin, int pageOffset = 0, CommandFlags flags = CommandFlags.None) + => GetActiveDatabase().HashScanNoValuesAsync(key, pattern, pageSize, cursor, pageOffset, flags); + + public IAsyncEnumerable SetScanAsync(RedisKey key, RedisValue pattern = default, int pageSize = RedisBase.CursorUtils.DefaultLibraryPageSize, long cursor = RedisBase.CursorUtils.Origin, int pageOffset = 0, CommandFlags flags = CommandFlags.None) + => GetActiveDatabase().SetScanAsync(key, pattern, pageSize, cursor, pageOffset, flags); + + public IAsyncEnumerable SortedSetScanAsync(RedisKey key, RedisValue pattern = default, int pageSize = RedisBase.CursorUtils.DefaultLibraryPageSize, long cursor = RedisBase.CursorUtils.Origin, int pageOffset = 0, CommandFlags flags = CommandFlags.None) + => GetActiveDatabase().SortedSetScanAsync(key, pattern, pageSize, cursor, pageOffset, flags); + + public IAsyncEnumerable VectorSetRangeEnumerateAsync(RedisKey key, RedisValue start = default, RedisValue end = default, long count = 100, Exclude exclude = Exclude.None, CommandFlags flags = CommandFlags.None) + => GetActiveDatabase().VectorSetRangeEnumerateAsync(key, start, end, count, exclude, flags); +} diff --git a/src/StackExchange.Redis/Availability/MultiGroupDatabase.cs b/src/StackExchange.Redis/Availability/MultiGroupDatabase.cs new file mode 100644 index 000000000..dd51c47c0 --- /dev/null +++ b/src/StackExchange.Redis/Availability/MultiGroupDatabase.cs @@ -0,0 +1,88 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Interfaces; + +namespace StackExchange.Redis.Availability; + +// The bulk of IDatabase/IDatabaseAsync is generated by [AutoDatabase]: every command is captured into a +// state struct and replayed against the currently-active member via the Execute/ExecuteAsync funnels below +// (so a call always lands on whichever member is active *now*). Only the members the generator deliberately +// skips - the Wait family, IsConnected, the batch/transaction factories, and the streaming scans (see +// MultiGroupDatabase.Scans.cs) - plus the properties and internal plumbing are hand-written here. +[AutoDatabase] +internal sealed partial class MultiGroupDatabase(MultiGroupMultiplexer parent, int database, object? asyncState) + : IDatabase, IInternalDatabaseAsync +{ + public object? AsyncState => asyncState; + public int Database => database < 0 ? GetActiveDatabase().Database : database; + + public IConnectionMultiplexer Multiplexer => parent; + + CancellationToken IInternalDatabaseAsync.GetNextFailover() => parent.GetNextFailover(); + + DatabaseFeatureFlags IInternalDatabaseAsync.GetFeatures(out string name) + { + // fold in the currently-active member's features (when there is one) and label with its name + var active = TryGetActiveDatabase(); + var features = active is null ? DatabaseFeatureFlags.None : active.GetFeatures(out _); + name = ((IConnectionGroup)parent).ActiveMember?.Name ?? ""; + return features | DatabaseFeatureFlags.ConnectionGroup | DatabaseFeatureFlags.Failover; + } + + // uses the active member's name (via GetFeatures) when a member is active + public override string ToString() => this.BuildString(); + + // for high DB numbers this might allocate even for null async-state scenarios; unavoidable for now + private IDatabase GetActiveDatabase() => parent.Active.GetDatabase(database, asyncState); + + // non-throwing twin of GetActiveDatabase: returns null when the group currently has no active + // member, for use by members that have an obvious trivial result when disconnected + private IDatabase? TryGetActiveDatabase() => parent.TryGetActive()?.GetDatabase(database, asyncState); + + // ---- [AutoDatabase] funnels --------------------------------------------------------------------- + // the generated explicit interface implementations funnel every captured call through these: we simply + // forward to the currently-active member. Unlike RetryDatabase there is no key mapping to apply (we are + // not rewriting keys), so the captured state's Map/UnMapper is never invoked. + private TResult Execute(in TState state, AutoDatabaseSyncOperation operation) + where TState : struct + => operation(in state, GetActiveDatabase()); + + private void Execute(in TState state, AutoDatabaseSyncOperation operation) + where TState : struct + => operation(in state, GetActiveDatabase()); + + private Task ExecuteAsync(in TState state, AutoDatabaseAsyncOperation operation) + where TState : struct + => operation(in state, GetActiveDatabase()); + + private Task ExecuteAsync(in TState state, AutoDatabaseAsyncOperation operation) + where TState : struct + => operation(in state, GetActiveDatabase()); + + // ---- members the generator deliberately skips (see AutoDatabaseGenerator.SkipMethod) ------------- + public IBatch CreateBatch(object? asyncState = null) + => GetActiveDatabase().CreateBatch(asyncState); + + public ITransaction CreateTransaction(object? asyncState = null) + => GetActiveDatabase().CreateTransaction(asyncState); + + ITransactionAsync IDatabaseAsync.CreateTransaction(object? asyncState) => CreateTransaction(asyncState); + + // status probe, not a server round-trip: trivially "not connected" when there is no active member + public bool IsConnected(RedisKey key, CommandFlags flags = CommandFlags.None) + => TryGetActiveDatabase()?.IsConnected(key, flags) ?? false; + + // routing lookup against the currently-active member; null / NoEndpoint when the group has none + public System.Net.EndPoint? IdentifyEndpoint(RedisKey key = default, CommandFlags flags = CommandFlags.None) + => TryGetActiveDatabase()?.IdentifyEndpoint(key, flags); + + public Task IdentifyEndpointAsync(RedisKey key = default, CommandFlags flags = CommandFlags.None) + => TryGetActiveDatabase()?.IdentifyEndpointAsync(key, flags) ?? MultiGroupMultiplexer.NoEndpoint; + + // the Wait family operates on caller-supplied Tasks, not server calls + public bool TryWait(Task task) => GetActiveDatabase().TryWait(task); + public void Wait(Task task) => GetActiveDatabase().Wait(task); + public T Wait(Task task) => GetActiveDatabase().Wait(task); + public void WaitAll(params Task[] tasks) => GetActiveDatabase().WaitAll(tasks); +} diff --git a/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs new file mode 100644 index 000000000..86cbb6922 --- /dev/null +++ b/src/StackExchange.Redis/Availability/MultiGroupMultiplexer.cs @@ -0,0 +1,1428 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using RESPite; +using StackExchange.Redis.Availability; +using StackExchange.Redis.Maintenance; +using StackExchange.Redis.Profiling; + +namespace StackExchange.Redis +{ + public partial class ConnectionMultiplexer + { + /// + /// Creates a new instance that manages connections to multiple + /// redundant configurations, based on their availability and relative . + /// + /// The initial configurations to connect to. + /// Additional options for configuring this group. + /// The to log to. +#pragma warning disable RS0026 + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + public static Task ConnectGroupAsync( + ConnectionGroupMember[] members, + MultiGroupOptions? options = null, + TextWriter? log = null) +#pragma warning restore RS0026 + { + // create a defensive copy of the array; we don't want callers being able to radically swap things! + members = (ConnectionGroupMember[])members.Clone(); + return MultiGroupMultiplexer.ConnectAsync(members, options ?? MultiGroupOptions.Default, log); + } + + /// + /// Creates a new instance that manages connections to multiple + /// redundant configurations, based on their availability and relative . + /// + /// An initial configuration to connect to. + /// An additional initial configuration to connect to. + /// Additional options for configuring this group. + /// The to log to. + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +#pragma warning disable RS0026 + public static Task ConnectGroupAsync( + ConnectionGroupMember member0, + ConnectionGroupMember member1, + MultiGroupOptions? options = null, + TextWriter? log = null) +#pragma warning restore RS0026 + { + return MultiGroupMultiplexer.ConnectAsync([member0, member1], options ?? MultiGroupOptions.Default, log); + } + } + +#pragma warning disable SA1403 + namespace Availability +#pragma warning restore SA1403 + { + /// + /// A configured member of a . + /// + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +#pragma warning disable RS0016, RS0026 + public sealed partial class ConnectionGroupMember(ConfigurationOptions configuration, string name = "") +#pragma warning restore RS0016, RS0026 + { + /// + /// Create a new from a configuration string. + /// +#pragma warning disable RS0016, RS0026 + public ConnectionGroupMember(string configuration, string name = "") : this( + ConfigurationOptions.Parse(configuration)) +#pragma warning restore RS0016, RS0026 + { + } + + internal ConfigurationOptions Configuration => configuration; + + // all of the simple boolean state for a member is packed into a single flags field, updated atomically + [Flags] + private enum MemberFlags + { + None = 0, + Activated = 1 << 0, // attached to a group; set exactly once (see Init) + Connected = 1 << 1, // last observed health state (see IsConnected) + ExplicitOverrideFlag = 1 << 2, // manual failover target (see ExplicitOverride) + SkipInitialHealthCheck = 1 << 3, // see SkipInitialHealthCheck + Unhealthy = 1 << 4, // disabled by a health-check or circuit-breaker + } + + private int _flags; + + private bool GetFlag(MemberFlags flag) => (Volatile.Read(ref _flags) & (int)flag) != 0; + + private void SetFlag(MemberFlags flag, bool value) + { + int set = value ? (int)flag : 0, clear = value ? 0 : (int)flag; + while (true) + { + int old = Volatile.Read(ref _flags); + int updated = (old & ~clear) | set; + if (updated == old || Interlocked.CompareExchange(ref _flags, updated, old) == old) return; + } + } + + /// + public override string ToString() => Name; + + private ConnectionMultiplexer? _muxer; + + internal ConnectionMultiplexer Multiplexer => _muxer ?? ThrowNoMuxer(); + + internal void SetUnhealthy() + { + // stored as raw UTC ticks and only ever compared as a long against the failback cutoff; + // we never round-trip through a DateTime, so DateTimeKind never enters the picture + Volatile.Write(ref _lastUnhealthyTicks, DateTime.UtcNow.Ticks); + SetFlag(MemberFlags.Unhealthy, true); + } + + // UTC ticks (DateTime.Ticks and TimeSpan.Ticks share the same 100ns unit); long for atomicity + private long _lastUnhealthyTicks; + + /// + /// Clear the flag against this endpoint, allowing it to be reconsidered. + /// + public void ResetIsUnhealthy() => SetFlag(MemberFlags.Unhealthy, false); + + /// + /// Gets whether the endpoint failed a health-check or circuit-breaker test. + /// + public bool IsUnhealthy => GetFlag(MemberFlags.Unhealthy); + + [DoesNotReturn] + private static ConnectionMultiplexer ThrowNoMuxer() => + throw new InvalidOperationException("Member is not connected."); + + // ReSharper disable once NullCoalescingConditionIsAlwaysNotNullAccordingToAPIContract + internal void SetMultiplexer(ConnectionMultiplexer muxer) + => Interlocked.Exchange(ref _muxer, muxer ?? ThrowNoMuxer()); + + internal ConnectionMultiplexer? ClearMultiplexer() => Interlocked.Exchange(ref _muxer, null); + + internal void Init(int index) + { + // add a name if not provided + if (string.IsNullOrWhiteSpace(Name)) + { + var ep = Configuration.EndPoints.FirstOrDefault(); + if (ep is null) + { + Name = index.ToString(); + } + else + { + Name = Format.ToString(ep); + } + } + + // check not already attached (atomic test-and-set of the Activated flag) + while (true) + { + int old = Volatile.Read(ref _flags); + if ((old & (int)MemberFlags.Activated) != 0) + { + throw new InvalidOperationException( + $"Member '{Name}' is already associated with a group, and cannot be reused."); + } + + if (Interlocked.CompareExchange(ref _flags, old | (int)MemberFlags.Activated, old) == old) break; + } + } + + /// + /// Indicates whether this group is currently connected. + /// + public bool IsConnected + { + get => GetFlag(MemberFlags.Connected); + private set => SetFlag(MemberFlags.Connected, value); + } + + /// + /// Indicates whether the initial health-check should be skipped when this member is added to a group. + /// When true, the member is added immediately - as not-yet-connected - and only becomes selectable + /// once it subsequently passes a health-check, rather than blocking the add on an initial probe. This + /// allows adding a member that is not yet healthy. + /// + public bool SkipInitialHealthCheck + { + get => GetFlag(MemberFlags.SkipInitialHealthCheck); + set => SetFlag(MemberFlags.SkipInitialHealthCheck, value); + } + + /// + /// The name of this group member. + /// + public string Name { get; private set; } = name; + + // ---- per-member overrides of the group-wide MultiGroupOptions defaults ---- + // every value on MultiGroupOptions that can vary per member appears here as a nullable + // counterpart; null means "use the group default". These are read when the member is added + // to a group (for the circuit-breaker, which is fixed at connection construction) or on each + // health-check pass (for the rest), so changing them later is only meaningful for the latter. + + /// + /// The health-check to use for this member; when , + /// is used. Use + /// to leave this member's selection driven purely by its observed connectivity. + /// + public HealthCheck? HealthCheck { get; set; } + + /// + /// The circuit-breaker to use for this member; when , the member's own + /// is used, else + /// . This is applied when the member connects, so + /// setting it after the member has been added to a group has no effect on existing connections. + /// + public CircuitBreaker? CircuitBreaker { get; set; } + + /// + /// How long this member must remain healthy, following its most recent failure, before it is + /// eligible to be selected again; when , + /// is used. + /// + public TimeSpan? FailbackDelay { get; set; } + + // the breaker to hand to this member's connections: member override, else its own config, else the group + internal CircuitBreaker? ResolveCircuitBreaker(MultiGroupOptions options) + => CircuitBreaker ?? Configuration.CircuitBreaker ?? options.CircuitBreaker; + + internal HealthCheck ResolveHealthCheck(MultiGroupOptions options) => HealthCheck ?? options.HealthCheck; + + internal TimeSpan ResolveFailbackDelay(MultiGroupOptions options) => FailbackDelay ?? options.FailbackDelay; + + /// + /// The relative weight of this group member; higher is preferred. + /// + public double Weight + { + // avoid "tearing", since we can't rule out this being updated concurrently: Volatile.Read/Write + // on a double are atomic even on 32-bit processors, so no read-modify-write dance is needed + get => Volatile.Read(ref _weight); + set => Volatile.Write(ref _weight, value); + } + + internal bool ExplicitOverride + { + get => GetFlag(MemberFlags.ExplicitOverrideFlag); + set => SetFlag(MemberFlags.ExplicitOverrideFlag, value); + } + + private double _weight = 1.0; + + /// + /// The measured latency to this member. + /// + public TimeSpan Latency => + _latencyTicks is uint.MaxValue ? TimeSpan.MaxValue : TimeSpan.FromTicks(_latencyTicks); + + internal bool ConsiderActive => // "IsConnected && !IsUnhealthy" + ((MemberFlags)Volatile.Read(ref _flags) & (MemberFlags.Connected | MemberFlags.Unhealthy)) is MemberFlags.Connected; + + private uint _latencyTicks = uint.MaxValue; + + internal void SetLatency(uint ticks) => _latencyTicks = ticks; + + internal static uint ToLatencyTicks(TimeSpan latency) + { + long ticks = latency.Ticks; + if (ticks <= 0) + { + return 0; + } + + return ticks > uint.MaxValue ? uint.MaxValue : (uint)ticks; + } + + internal void SetLatency(TimeSpan latency) => SetLatency(ToLatencyTicks(latency)); + + internal static ConnectionGroupMember? Select(ConnectionGroupMember? x, ConnectionGroupMember? y, ConnectionMultiplexer? active) + { + if (x is null) return y; + if (y is null) return x; + + // always prefer a connected endpoint + bool xc = x.IsConnected, yc = y.IsConnected; + if (xc != yc) return xc ? x : y; + + // prefer manual override if only one is overridden + xc = x.ExplicitOverride; + yc = y.ExplicitOverride; + if (xc != yc) return xc ? x : y; + + // prefer higher weight + double xw = x.Weight, yw = y.Weight; + // ReSharper disable once CompareOfFloatsByEqualityOperator + if (xw != yw) return xw > yw ? x : y; + + // then by latency + uint xl = x._latencyTicks, yl = y._latencyTicks; + if (xl != yl) return xl < yl ? x : y; + + // getting hard to choose; is either of them the existing active node? choose that to prevent flapping + if (ReferenceEquals(x._muxer, active)) return x; + if (ReferenceEquals(y._muxer, active)) return y; + + // I've got nothing; choose x arbitrarily + return x; + } + + internal GroupConnectionChangedEventArgs.ChangeType UpdateState(HealthCheckResult result, long failbackFailureCutoffTicks) + { + bool isConnected; + if (_muxer is { IsConnected: true } muxer) + { + isConnected = result is not HealthCheckResult.Unhealthy; + SetLatency(muxer.UpdateLatency()); + } + else + { + isConnected = false; + } + + if (isConnected) + { + if (Volatile.Read(ref _lastUnhealthyTicks) < failbackFailureCutoffTicks) ResetIsUnhealthy(); + } + else + { + SetUnhealthy(); + } + + var oldConnected = IsConnected; + IsConnected = isConnected; + + return isConnected == oldConnected ? GroupConnectionChangedEventArgs.ChangeType.Unknown + : isConnected ? GroupConnectionChangedEventArgs.ChangeType.Reconnected + : GroupConnectionChangedEventArgs.ChangeType.Disconnected; + } + + internal void UpdateLatency() + { + if (_muxer is { } muxer) SetLatency(muxer.UpdateLatency()); + } + } + + internal sealed partial class MultiGroupMultiplexer : IConnectionGroup + { + private ActiveStub _activeStub = new(null); + + private void SetActive(ConnectionMultiplexer? active) + { + ActiveStub? newObj = null; + while (true) + { + var oldObj = Volatile.Read(ref _activeStub); + + // is it already the same? + if (ReferenceEquals(oldObj.Active, active)) + { + newObj?.Dispose(); // never actually released to the world + return; // nothing to do! + } + + newObj ??= new(active); + if (ReferenceEquals(Interlocked.CompareExchange(ref _activeStub, newObj, oldObj), oldObj)) + { + // successful swap; flag the old one as failed-over + oldObj.Cancel(false); + return; + } + + // race? redo from start, but we can keep our newObj if we created one + } + } + + public CancellationToken GetNextFailover() => _activeStub.Token; + + private sealed class ActiveStub(ConnectionMultiplexer? active) : CancellationTokenSource + { + public ConnectionMultiplexer? Active => active; + } + + private ConnectionGroupMember[] _members; + + public override string ToString() + { + var muxer = _activeStub.Active; + ConnectionGroupMember? member = null; + if (muxer is not null) + { + foreach (var m in _members) + { + if (ReferenceEquals(muxer, m.Multiplexer)) + { + member = m; + break; + } + } + } + + return member is null ? "No active connection" : $"Connected to {member.Name}"; + } + + public ReadOnlySpan GetMembers() => _members; + + internal ConnectionMultiplexer Active + { + get + { + return _activeStub.Active ?? Throw(); + + [DoesNotReturn] + static ConnectionMultiplexer Throw() => + throw new InvalidOperationException("All connections are unavailable."); + } + } + + // non-throwing twin of Active, for callers that have a trivial answer when the group is fully down + internal ConnectionMultiplexer? TryGetActive() => _activeStub.Active; + + // a completed "no endpoint" result, shared by the database/subscriber facades when the group is fully down + internal static readonly Task NoEndpoint = Task.FromResult(null); + + private ConnectionGroupMember? GetActiveMember() => GetMember(_activeStub.Active); + + private ConnectionGroupMember? GetMember(ConnectionMultiplexer? muxer) + { + if (muxer is not null) + { + foreach (var member in _members) + { + if (ReferenceEquals(muxer, member.Multiplexer)) + { + return member; + } + } + } + + return null; + } + + ConnectionGroupMember? IConnectionGroup.ActiveMember => GetActiveMember(); + + internal ConnectionGroupMember ActiveMember + { + get + { + return GetActiveMember() ?? Throw(); + + [DoesNotReturn] + static ConnectionGroupMember Throw() => + throw new InvalidOperationException("All connections are unavailable."); + } + } + + internal static async Task ConnectAsync( + ConnectionGroupMember[] members, + MultiGroupOptions options, + TextWriter? log) + { + for (int i = 0; i < members.Length; i++) + { + members[i].Init(i); + } + + var pending = new Task[members.Length]; + for (int i = 0; i < members.Length; i++) + { + var config = members[i].Configuration; + config.AbortOnConnectFail = false; + config.HeartbeatConsistencyChecks = true; + + // note the resolved circuit-breaker is passed *alongside* the configuration rather than + // written into it; see ConnectionMultiplexer.GroupCircuitBreaker + pending[i] = ConnectionMultiplexer.ConnectGroupMemberAsync(config, log, members[i].ResolveCircuitBreaker(options)); + } + + for (int i = 0; i < pending.Length; i++) + { + var muxer = await pending[i].ConfigureAwait(false); + members[i].SetMultiplexer(muxer); + } + + // run initial healthcheck and begin + var result = new MultiGroupMultiplexer(members, options); + await TryHealthCheckAndSelectPreferredGroupAsync(result).ForAwait(); + result.StartPolling(); + return result; + } + + private readonly MultiGroupOptions _options; + + public MultiGroupOptions Options => _options; + + private MultiGroupMultiplexer(ConnectionGroupMember[] members, MultiGroupOptions options) + { + _options = options; + _members = members; + SetActive(null); + + _connectionFailedWithCircuitBreaker = OnMemberConnectionFailed; + // multiplexers should already be attached (ConnectAsync sets them before constructing us) + foreach (var member in members) + { + member.Multiplexer?.ConnectionFailed += _connectionFailedWithCircuitBreaker; + } + } + + internal static async Task TryHealthCheckAndSelectPreferredGroupAsync(object? target) + { + if (target is MultiGroupMultiplexer typed) + { + if (typed.IsDisposed) return false; + + // serialize health-check + select: the poll loop and the circuit-breaker fast-path + // can both land here, and we must not run two overlapping check/select passes (they + // would race _active and emit duplicate change events). If a pass is already running, + // skip - it will select on our behalf, and we converge on the next tick regardless. + if (Interlocked.CompareExchange(ref typed._healthCheckGate, 1, 0) is not 0) + { + return true; // still a live target; keep polling + } + + try + { + await typed.RunHealthCheckAsync().ForAwait(); + typed.SelectPreferredGroup(); + } + catch (Exception ex) + { + typed.OnInternalError(ex, origin: "update group"); + } + finally + { + Volatile.Write(ref typed._healthCheckGate, 0); + } + + return true; // even if we fault: try again + } + + return false; + } + + private void StartPolling() + { + // use a weak-ref to avoid the loop keeping the object alive; capturing the token (rather than + // the muxer) lets us break out of the delay promptly on dispose without resurrecting the target + var cancellationToken = _pollCancellation.Token; + _ = Task.Run(() => PollAsync(new(this), cancellationToken)); + + static async Task PollAsync(WeakReference weakRef, CancellationToken cancellationToken) + { + while (TryGetHealthCheck(weakRef.Target, out var interval)) + { + try + { + await Task.Delay(interval, cancellationToken).ForAwait(); + } + catch (OperationCanceledException) + { + break; // disposed; stop polling without waiting out the interval + } + + if (!await TryHealthCheckAndSelectPreferredGroupAsync(weakRef.Target).ForAwait()) break; + } + } + + static bool TryGetHealthCheck(object? target, out TimeSpan interval) + { + if (target is MultiGroupMultiplexer typed) + { + // note the interval is a group-level concern (how often we re-evaluate the active + // member), not a property of any individual health-check + interval = typed._options.HealthCheckInterval; + return interval > TimeSpan.Zero & interval != TimeSpan.MaxValue; + } + + interval = TimeSpan.Zero; + return false; + } + } + + internal bool IsDisposed => _disposed; + + private Task[]? _reusableHealthCheckBuffer; + private int _healthCheckGate; // 0 = idle, 1 = a check/select pass is in flight (see TryHealthCheckAndSelectPreferredGroupAsync) + + internal async Task RunHealthCheckAsync() + { + if (_disposed) return; + var members = _members; + if (members.Length == 0) return; // nothing to check (and no budget to compute) + + var pending = HealthCheck.GetReusablePending(ref _reusableHealthCheckBuffer, members.Length); + + // members can use different health-checks (see ConnectionGroupMember.HealthCheck), so the + // budget for the whole pass is the largest individual budget + int totalTimeoutMillis = 0; + for (int i = 0; i < members.Length; i++) + { + var muxer = members[i].Multiplexer; + var healthCheck = members[i].ResolveHealthCheck(_options); + totalTimeoutMillis = Math.Max(totalTimeoutMillis, healthCheck.TotalTimeoutMillis()); + pending[i] = healthCheck.CheckHealthAsync(muxer); + } + + await Task.WhenAll(pending).TimeoutAfter(totalTimeoutMillis).ForAwait(); + for (int i = 0; i < pending.Length; i++) + { + HealthCheckResult result; + if (pending[i].IsCompletedSuccessfully) + { + result = await pending[i].ForAwait(); + } + else + { + _ = pending[i].ObserveErrors(); + result = HealthCheckResult.Unhealthy; + } + + var delta = members[i].UpdateState(result, GetFailbackFailureCutoff(members[i])); + if (delta != GroupConnectionChangedEventArgs.ChangeType.Unknown) + { + OnConnectionChanged(delta, members[i]); + } + } + + HealthCheck.PutReusablePending(ref _reusableHealthCheckBuffer, ref pending); + } + + private long GetFailbackFailureCutoff(ConnectionGroupMember member) + { + // the minimum last-observed unhealthy time (as UTC ticks) that we'll allow for reconnect; + // for example, if the FailbackDelay is 2 minutes, and the time is 14:32:55, then the last + // failure must have happened at 14:30:55 or earlier. Pure long tick math on the wall clock: + // DateTime.Ticks and TimeSpan.Ticks are the same 100ns unit, so the subtraction is valid + var delay = member.ResolveFailbackDelay(_options); + if (delay == TimeSpan.MaxValue) return long.MinValue; // manual mode: never auto-reset + + return DateTime.UtcNow.Ticks - delay.Ticks; + } + + internal void SelectPreferredGroup() + { + if (_disposed) return; + var existingActive = _activeStub.Active; + ConnectionGroupMember? preferredMember = null, previousMember = null; + var members = _members; + foreach (var member in members) + { + if (previousMember is null && ReferenceEquals(member.Multiplexer, existingActive)) + { + previousMember = member; + } + + if (member.ConsiderActive) + { + member.UpdateLatency(); // this can change passively + + // (note that when in doubt, we prefer the active muxer, to prevent flapping) + preferredMember = ConnectionGroupMember.Select(preferredMember, member, existingActive); + } + } + + SetActive(preferredMember?.Multiplexer); + + if (preferredMember is not null && !ReferenceEquals(preferredMember, previousMember)) + { + OnConnectionChanged( + GroupConnectionChangedEventArgs.ChangeType.ActiveChanged, + preferredMember, + previousMember); + } + } + + private readonly CancellationTokenSource _pollCancellation = new(); + + private List DropAll() + { + _pollCancellation.Cancel(); // stop the polling loop promptly (idempotent) + SetActive(null); + var members = Interlocked.Exchange(ref _members, []); + if (members.Length is 0) return []; + var muxers = new List(members.Length); + foreach (var member in members) + { + var muxer = member.ClearMultiplexer(); + if (muxer is not null) muxers.Add(muxer); + } + + return muxers; + } + + private bool _disposed; + + public void Dispose() + { + _disposed = true; + foreach (var muxer in DropAll()) + { + muxer.Dispose(); + } + } + + public async ValueTask DisposeAsync() + { + _disposed = true; + foreach (var muxer in DropAll()) + { + await muxer.DisposeAsync(); + } + } + + public string ClientName => Active.ClientName; + public string Configuration => Active.Configuration; + public int TimeoutMilliseconds => Active.TimeoutMilliseconds; + + public long OperationCount + { + get + { + long count = 0; + foreach (var member in _members) + { + count += member.Multiplexer.OperationCount; + } + + return count; + } + } + + [Obsolete] + public bool PreserveAsyncOrder + { + get => Active.PreserveAsyncOrder; + set => Active.PreserveAsyncOrder = value; + } + + // Unlike most members, these intentionally do *not* go via Active (which throws when no member is + // available); callers routinely use IsConnected/IsConnecting as a pre-flight check and expect a + // 'false' result - not an exception - when the entire group is down. + public bool IsConnected => _activeStub.Active?.IsConnected ?? false; + public bool IsConnecting => _activeStub.Active?.IsConnecting ?? false; + + [Obsolete] + public bool IncludeDetailInExceptions + { + get => Active.IncludeDetailInExceptions; + set => Active.IncludeDetailInExceptions = value; + } + + public int StormLogThreshold + { + get => Active.StormLogThreshold; + set => Active.StormLogThreshold = value; + } + + private Func? _profilingSessionProvider; + + public void RegisterProfiler(Func profilingSessionProvider) + { + _profilingSessionProvider = profilingSessionProvider; + foreach (var member in _members) + { + member.Multiplexer.RegisterProfiler(profilingSessionProvider); + } + } + + public ServerCounters GetCounters() => Active.GetCounters(); + + private EventHandler? _errorMessage; + + public event EventHandler? ErrorMessage + { + add + { + if (AddHandler(ref _errorMessage, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ErrorMessage += value; + } + } + } + remove + { + if (RemoveHandler(ref _errorMessage, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ErrorMessage -= value; + } + } + } + } + + /// + /// Add a handler, and return true if this is the *first* handler, which means we should subscribe the dependents. + /// + private static bool AddHandler(ref T? field, T? value) where T : Delegate + { + if (value is null) return false; + while (true) // loop until we win (competition) + { + var oldValue = field; + var newValue = oldValue is null ? value : (T)Delegate.Combine(oldValue, value); + + if (ReferenceEquals(Interlocked.CompareExchange(ref field, newValue, oldValue), oldValue)) + { + return oldValue is null; + } + } + } + + /// + /// Remove a handler, and return true if this is the *last* handler, which means we should unsubscribe the dependents. + /// + private static bool RemoveHandler(ref T? field, T? value) where T : Delegate + { + if (value is null) return false; + while (true) // loop until we win (competition) + { + var oldValue = field; + var newValue = oldValue is null ? null : (T?)Delegate.Remove(oldValue, value); + + if (ReferenceEquals(Interlocked.CompareExchange(ref field, newValue, oldValue), oldValue)) + { + return newValue is null; + } + } + } + + private int _circuitBreakerDebounce = 0; + private void OnMemberConnectionFailed(object? sender, ConnectionFailedEventArgs e) + { + // deliberately scoped to this one failure type; re-probe and re-select promptly rather + // than waiting for the next poll tick, so traffic routes away from the dropped member + if (e.FailureType is ConnectionFailureType.CircuitBreaker + && Interlocked.CompareExchange(ref _circuitBreakerDebounce, 1, 0) is 0) + { + GetMember(sender as ConnectionMultiplexer)?.SetUnhealthy(); + _ = Task.Run(async () => + { + try + { + await TryHealthCheckAndSelectPreferredGroupAsync(this); + } + catch (Exception ex) + { + Debug.WriteLine(ex.Message); + } + finally + { + Volatile.Write(ref _circuitBreakerDebounce, 0); + } + }); + } + + // invoke any custom logic from consumers subscribing to the public events + _connectionFailedExternal?.Invoke(sender, e); + } + + /// + /// Subscribe a child multiplexer to all local event handlers that have subscribers. + /// + private void AddEventHandlers(ConnectionMultiplexer muxer) + { + muxer.ErrorMessage += _errorMessage; + muxer.ConnectionFailed += _connectionFailedWithCircuitBreaker; // always non-null + muxer.InternalError += _internalError; + muxer.ConnectionRestored += _connectionRestored; + muxer.ConfigurationChanged += _configurationChanged; + muxer.ConfigurationChangedBroadcast += _configurationChangedBroadcast; + muxer.ServerMaintenanceEvent += _serverMaintenanceEvent; + muxer.HashSlotMoved += _hashSlotMoved; + } + + /// + /// Unsubscribe a child multiplexer from all local event handlers. + /// + private void RemoveEventHandlers(ConnectionMultiplexer? muxer) + { + if (muxer is null) return; + muxer.ErrorMessage -= _errorMessage; + muxer.ConnectionFailed -= _connectionFailedWithCircuitBreaker; // always non-null + muxer.InternalError -= _internalError; + muxer.ConnectionRestored -= _connectionRestored; + muxer.ConfigurationChanged -= _configurationChanged; + muxer.ConfigurationChangedBroadcast -= _configurationChangedBroadcast; + muxer.ServerMaintenanceEvent -= _serverMaintenanceEvent; + muxer.HashSlotMoved -= _hashSlotMoved; + } + + private readonly EventHandler _connectionFailedWithCircuitBreaker; + + private EventHandler? _connectionFailedExternal; // from public callers + + public event EventHandler? ConnectionFailed + { + // note we *do not* need to hook/unhook the inner connection - we always subscribe to that + add => AddHandler(ref _connectionFailedExternal, value); + remove => RemoveHandler(ref _connectionFailedExternal, value); + } + + private EventHandler? _internalError; + + public event EventHandler? InternalError + { + add + { + if (AddHandler(ref _internalError, value)) + { + foreach (var member in _members) + { + member.Multiplexer.InternalError += value; + } + } + } + remove + { + if (RemoveHandler(ref _internalError, value)) + { + foreach (var member in _members) + { + member.Multiplexer.InternalError -= value; + } + } + } + } + + private EventHandler? _connectionRestored; + + public event EventHandler? ConnectionRestored + { + add + { + if (AddHandler(ref _connectionRestored, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ConnectionRestored += value; + } + } + } + remove + { + if (RemoveHandler(ref _connectionRestored, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ConnectionRestored -= value; + } + } + } + } + + private EventHandler? _configurationChanged; + + public event EventHandler? ConfigurationChanged + { + add + { + if (AddHandler(ref _configurationChanged, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ConfigurationChanged += value; + } + } + } + remove + { + if (RemoveHandler(ref _configurationChanged, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ConfigurationChanged -= value; + } + } + } + } + + private EventHandler? _configurationChangedBroadcast; + + public event EventHandler? ConfigurationChangedBroadcast + { + add + { + if (AddHandler(ref _configurationChangedBroadcast, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ConfigurationChangedBroadcast += value; + } + } + } + remove + { + if (RemoveHandler(ref _configurationChangedBroadcast, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ConfigurationChangedBroadcast -= value; + } + } + } + } + + private EventHandler? _serverMaintenanceEvent; + + public event EventHandler? ServerMaintenanceEvent + { + add + { + if (AddHandler(ref _serverMaintenanceEvent, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ServerMaintenanceEvent += value; + } + } + } + remove + { + if (RemoveHandler(ref _serverMaintenanceEvent, value)) + { + foreach (var member in _members) + { + member.Multiplexer.ServerMaintenanceEvent -= value; + } + } + } + } + + public EndPoint[] GetEndPoints(bool configuredOnly = false) => Active.GetEndPoints(configuredOnly); + + public void Wait(Task task) => Active.Wait(task); + + public T Wait(Task task) => Active.Wait(task); + + public void WaitAll(params Task[] tasks) => Active.WaitAll(tasks); + + private EventHandler? _hashSlotMoved; + + public event EventHandler? HashSlotMoved + { + add + { + if (AddHandler(ref _hashSlotMoved, value)) + { + foreach (var member in _members) + { + member.Multiplexer.HashSlotMoved += value; + } + } + } + remove + { + if (RemoveHandler(ref _hashSlotMoved, value)) + { + foreach (var member in _members) + { + member.Multiplexer.HashSlotMoved -= value; + } + } + } + } + + public int HashSlot(RedisKey key) => Active.HashSlot(key); + + private ISubscriber? _defaultSubscriber; + + public ISubscriber GetSubscriber(object? asyncState = null) + { + if (asyncState is null) return _defaultSubscriber ??= new MultiGroupSubscriber(this, null); + return new MultiGroupSubscriber(this, asyncState); + } + + public IDatabase GetDatabase(int db = -1, object? asyncState = null) + { + if (asyncState is null & db >= -1 & db <= ConnectionMultiplexer.MaxCachedDatabaseInstance) + { + return _databases[db + 1] ??= new MultiGroupDatabase(this, db, null); + } + + return new MultiGroupDatabase(this, db, asyncState); + } + + private readonly IDatabase?[] _databases = + new IDatabase?[ConnectionMultiplexer.MaxCachedDatabaseInstance + 2]; + + public IServer GetServer(string host, int port, object? asyncState = null) + { + Exception ex; + try + { + // try "active" first, and preserve the exception + return Active.GetServer(host, port, asyncState); + } + catch (Exception e) + { + ex = e; + } + + foreach (var member in _members) + { + try + { + return member.Multiplexer.GetServer(host, port, asyncState); + } + catch (Exception e) { Debug.WriteLine(e.Message); } + } + + throw ex; + } + + public IServer GetServer(string hostAndPort, object? asyncState = null) + { + Exception ex; + try + { + // try "active" first, and preserve the exception + return Active.GetServer(hostAndPort, asyncState); + } + catch (Exception e) + { + ex = e; + } + + foreach (var member in _members) + { + try + { + return member.Multiplexer.GetServer(hostAndPort, asyncState); + } + catch (Exception e) { Debug.WriteLine(e.Message); } + } + + throw ex; + } + + public IServer GetServer(IPAddress host, int port) + { + Exception ex; + try + { + // try "active" first, and preserve the exception + return Active.GetServer(host, port); + } + catch (Exception e) + { + ex = e; + } + + foreach (var member in _members) + { + try + { + return member.Multiplexer.GetServer(host, port); + } + catch (Exception e) { Debug.WriteLine(e.Message); } + } + + throw ex; + } + + public IServer GetServer(EndPoint endpoint, object? asyncState = null) + { + Exception ex; + try + { + // try "active" first, and preserve the exception + return Active.GetServer(endpoint, asyncState); + } + catch (Exception e) + { + ex = e; + } + + foreach (var member in _members) + { + try + { + return member.Multiplexer.GetServer(endpoint, asyncState); + } + catch (Exception e) { Debug.WriteLine(e.Message); } + } + + throw ex; + } + + public IServer GetServer(RedisKey key, object? asyncState = null, CommandFlags flags = CommandFlags.None) => + Active.GetServer(key, asyncState, flags); + + public IServer[] GetServers() => Active.GetServers(); + + public Task ConfigureAsync(TextWriter? log = null) => Active.ConfigureAsync(log); + + public bool Configure(TextWriter? log = null) => Active.Configure(log); + + public string GetStatus() => Active.GetStatus(); + + public void GetStatus(TextWriter log) => Active.GetStatus(log); + + public void Close(bool allowCommandsToComplete = true) + { + _disposed = true; + foreach (var member in DropAll()) + { + member.Close(allowCommandsToComplete); + } + } + + public async Task CloseAsync(bool allowCommandsToComplete = true) + { + _disposed = true; + foreach (var member in DropAll()) + { + await member.CloseAsync(allowCommandsToComplete); + } + } + + public string? GetStormLog() => Active.GetStormLog(); + + public void ResetStormLog() => Active.ResetStormLog(); + + public long PublishReconfigure(CommandFlags flags = CommandFlags.None) => Active.PublishReconfigure(flags); + + public Task PublishReconfigureAsync(CommandFlags flags = CommandFlags.None) => + Active.PublishReconfigureAsync(flags); + + public int GetHashSlot(RedisKey key) => Active.GetHashSlot(key); + + public void ExportConfiguration(Stream destination, ExportOptions options = ExportOptions.All) => + Active.ExportConfiguration(destination, options); + + private readonly HashSet _suffixes = new(); // in case we need to add to a new muxer + + public void AddLibraryNameSuffix(string suffix) + { + if (string.IsNullOrWhiteSpace(suffix)) return; // trivial + bool isNew; + lock (_suffixes) + { + isNew = _suffixes.Add(suffix); + } + + if (isNew) + { + foreach (var member in _members) + { + member.Multiplexer.AddLibraryNameSuffix(suffix); + } + } + } + + public event EventHandler? ConnectionChanged; + + private void OnConnectionChanged( + GroupConnectionChangedEventArgs.ChangeType changeType, + ConnectionGroupMember member, + ConnectionGroupMember? previousMember = null) + { + var handler = ConnectionChanged; + if (handler is not null) + { + new GroupConnectionChangedEventArgs(changeType, member, previousMember) + .CompleteAsWorker(handler, this); + } + } + + public async Task AddAsync(ConnectionGroupMember member, TextWriter? log = null) + { + // connect + member.Init(_members.Length); + member.Configuration.AbortOnConnectFail = false; // members are gated by health-checks, not connect-fail + member.Configuration.HeartbeatConsistencyChecks = true; + var muxer = await ConnectionMultiplexer.ConnectGroupMemberAsync( + member.Configuration, log, member.ResolveCircuitBreaker(_options)).ConfigureAwait(false); + member.SetMultiplexer(muxer); + + // unless told otherwise, run an initial health-check so a healthy member can be selected immediately; + // when skipped, the member is added as not-yet-connected and the poll loop brings it online once it + // passes - this is the only way to add a member that is not yet healthy + if (!member.SkipInitialHealthCheck) + { + var health = await member.ResolveHealthCheck(_options).CheckHealthAsync(muxer).ConfigureAwait(false); + member.UpdateState(health, GetFailbackFailureCutoff(member)); + } + + // apply any shared hooks + AddEventHandlers(muxer); // includes circuit-breaker + if (_profilingSessionProvider is not null) muxer.RegisterProfiler(_profilingSessionProvider); + lock (_suffixes) + { + foreach (var suffix in _suffixes) + { + muxer.AddLibraryNameSuffix(suffix); + } + } + + // update the members array + while (true) + { + var arr = _members; + var newArr = new ConnectionGroupMember[arr.Length + 1]; + Array.Copy(arr, 0, newArr, 0, arr.Length); + newArr[arr.Length] = member; + if (Interlocked.CompareExchange(ref _members, newArr, arr) == arr) break; + } + + OnConnectionChanged(GroupConnectionChangedEventArgs.ChangeType.Added, member); + SelectPreferredGroup(); + + // pub/sub + await AddPubSubHandlersAsync(member).ConfigureAwait(false); + } + + public bool TryFailoverTo(ConnectionGroupMember? member) + { + if (member is null) + { + // remove any explicit overrides, returning whether that was an actual change + bool result = false; + foreach (var m in _members) + { + if (m.ExplicitOverride) + { + result = true; // someone was explicitly enabled + m.ExplicitOverride = false; + } + } + + SelectPreferredGroup(); + return result; + } + + var members = _members; + if (!members.Contains(member)) + { + // not one of ours? + return false; + } + + member.ResetIsUnhealthy(); // the user explicitly asked us to consider this node + if (!member.IsConnected) + { + // not allowed + return false; + } + + if (member.ExplicitOverride) + { + // already preferred; no change, but report as success + return true; + } + + // otherwise, deselect everyone else, and select this one + foreach (var m in members) + { + m.ExplicitOverride = ReferenceEquals(m, member); + } + + SelectPreferredGroup(); + return true; + } + + public bool Remove(ConnectionGroupMember member) + { + while (true) + { + var arr = _members; + int index = -1; + for (int i = 0; i < arr.Length; i++) + { + if (ReferenceEquals(arr[i], member)) + { + index = i; + break; + } + } + + if (index == -1) return false; + var newArr = new ConnectionGroupMember[arr.Length - 1]; + if (index > 0) Array.Copy(arr, 0, newArr, 0, index); + if (index < newArr.Length) Array.Copy(arr, index + 1, newArr, index, newArr.Length - index); + if (Interlocked.CompareExchange(ref _members, newArr, arr) == arr) break; + } + + var muxer = member.ClearMultiplexer(); + RemoveEventHandlers(muxer); // includes circuit-breaker + OnConnectionChanged(GroupConnectionChangedEventArgs.ChangeType.Removed, member); + SelectPreferredGroup(); + muxer?.Dispose(); + return true; + } + + internal void OnHeartbeat() // for testing, to update latency etc + { + foreach (var member in _members) + { + member.Multiplexer.OnHeartbeat(); + } + } + + internal void OnInternalError( + Exception exception, + EndPoint? endpoint = null, + ConnectionType connectionType = ConnectionType.None, + string? origin = null) + { + var handler = _internalError; + if (handler is not null) + { + InternalErrorEventArgs args = new(handler, this, endpoint, connectionType, exception, origin); + ConnectionMultiplexer.CompleteAsWorker(args); + } + } + } + } +} diff --git a/src/StackExchange.Redis/Availability/MultiGroupOptions.cs b/src/StackExchange.Redis/Availability/MultiGroupOptions.cs new file mode 100644 index 000000000..1f6f3eccf --- /dev/null +++ b/src/StackExchange.Redis/Availability/MultiGroupOptions.cs @@ -0,0 +1,160 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Configuration options for controlling connections to multiple groups. +/// +/// +/// Instances are immutable; use to configure. Every value here is a group-wide +/// default, and can be overridden per-member by the matching property on +/// (where one exists); the effective value is "member override, else group default". +/// +[Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +public sealed class MultiGroupOptions +{ + internal static readonly TimeSpan DefaultHealthCheckInterval = TimeSpan.FromSeconds(5); + internal static readonly TimeSpan DefaultFailbackDelay = TimeSpan.Zero; + + private static readonly MultiGroupOptions DefaultInstance = new( + HealthCheck.Default, CircuitBreaker.Default, RetryPolicy.Default, DefaultHealthCheckInterval, DefaultFailbackDelay); + + /// + /// Default shared options. + /// + public static MultiGroupOptions Default => DefaultInstance; + + private MultiGroupOptions( + HealthCheck healthCheck, + CircuitBreaker circuitBreaker, + RetryPolicy retryPolicy, + TimeSpan healthCheckInterval, + TimeSpan failbackDelay) + { + HealthCheck = healthCheck; + CircuitBreaker = circuitBreaker; + RetryPolicy = retryPolicy; + HealthCheckInterval = healthCheckInterval; + FailbackDelay = failbackDelay; + } + + /// + public override string ToString() => $"health-check: {HealthCheck} every {HealthCheckInterval}; failback: {FailbackDelay}"; + + /// + /// The health check to use for members of the group when no per-member health check is specified. + /// + public HealthCheck HealthCheck { get; } + + /// + /// The circuit-breaker to use for members of the group when no per-member circuit-breaker is specified. + /// + public CircuitBreaker CircuitBreaker { get; } + + /// + /// The retry policy used by for databases + /// obtained from this group. + /// + public RetryPolicy RetryPolicy { get; } + + /// + /// How frequently health checks are performed, and therefore how frequently the active member is + /// re-evaluated. disables periodic checking entirely (the group is then + /// only re-evaluated in response to connection events such as a tripped circuit-breaker). + /// + public TimeSpan HealthCheckInterval { get; } + + /// + /// If a member has been marked unhealthy by a failing health-check or circuit-breaker, it will not be + /// re-selected as the active member until it has remained healthy for this interval following its most + /// recent failure. When (the default), failback is immediate; when + /// , failback is not automatic and requires explicit use of + /// or . + /// + public TimeSpan FailbackDelay { get; } + + /// + /// Allows configuration of . + /// + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + public sealed class Builder + { + /// + /// Create a builder pre-populated with the default values. + /// + public Builder() + { + } + + /// + /// Create a builder pre-populated from existing options. + /// + public Builder(MultiGroupOptions options) + { + HealthCheck = options.HealthCheck; + CircuitBreaker = options.CircuitBreaker; + RetryPolicy = options.RetryPolicy; + HealthCheckInterval = options.HealthCheckInterval; + FailbackDelay = options.FailbackDelay; + } + + /// + /// The health check to use for members of the group when no per-member health check is specified. + /// + public HealthCheck HealthCheck { get; set; } = HealthCheck.Default; + + /// + /// The circuit-breaker to use for members of the group when no per-member circuit-breaker is specified. + /// + public CircuitBreaker CircuitBreaker { get; set; } = CircuitBreaker.Default; + + /// + /// The retry policy used by for databases + /// obtained from this group. + /// + public RetryPolicy RetryPolicy { get; set; } = RetryPolicy.Default; + + /// + /// How frequently health checks are performed, and therefore how frequently the active member is + /// re-evaluated; disables periodic checking. + /// + public TimeSpan HealthCheckInterval { get; set; } = DefaultHealthCheckInterval; + + /// + /// How long a member must remain healthy, following its most recent failure, before it is eligible to + /// be selected again; requires explicit intervention. + /// + public TimeSpan FailbackDelay { get; set; } = DefaultFailbackDelay; + + /// + /// Create a new options instance. + /// + public MultiGroupOptions Create() + { + if (HealthCheck is null) throw new ArgumentNullException(nameof(HealthCheck)); + if (CircuitBreaker is null) throw new ArgumentNullException(nameof(CircuitBreaker)); + if (RetryPolicy is null) throw new ArgumentNullException(nameof(RetryPolicy)); + if (HealthCheckInterval <= TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(HealthCheckInterval), HealthCheckInterval, "A positive interval is required; use TimeSpan.MaxValue to disable periodic health checks."); + if (FailbackDelay < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(FailbackDelay), FailbackDelay, "A non-negative delay is required."); + + // prefer the shared default instance when nothing has been customized + if (ReferenceEquals(HealthCheck, HealthCheck.Default) + && ReferenceEquals(CircuitBreaker, CircuitBreaker.Default) + && ReferenceEquals(RetryPolicy, RetryPolicy.Default) + && HealthCheckInterval == DefaultHealthCheckInterval + && FailbackDelay == DefaultFailbackDelay) + { + return DefaultInstance; + } + + return new MultiGroupOptions(HealthCheck, CircuitBreaker, RetryPolicy, HealthCheckInterval, FailbackDelay); + } + + /// + /// Create a new options instance. + /// + public static implicit operator MultiGroupOptions(Builder builder) => builder.Create(); + } +} diff --git a/src/StackExchange.Redis/Availability/MultiGroupSubscriber.Tracking.cs b/src/StackExchange.Redis/Availability/MultiGroupSubscriber.Tracking.cs new file mode 100644 index 000000000..a029eef56 --- /dev/null +++ b/src/StackExchange.Redis/Availability/MultiGroupSubscriber.Tracking.cs @@ -0,0 +1,251 @@ +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace StackExchange.Redis.Availability; + +internal sealed partial class MultiGroupSubscriber +{ + // for the actual pub/sub tracking, we need to be more subtle; we need to maintain our *own* map of what is subscribed, + // and only forward messages from the currently live system. + public void Subscribe( + RedisChannel channel, + Action handler, + CommandFlags flags = CommandFlags.None) => parent.SubscribeToAll(new(channel, handler, flags, asyncState)); + + public Task SubscribeAsync( + RedisChannel channel, + Action handler, + CommandFlags flags = CommandFlags.None) => parent.SubscribeToAllAsync(new(channel, handler, flags, asyncState)); + + public ChannelMessageQueue Subscribe( + RedisChannel channel, + CommandFlags flags = CommandFlags.None) + { + var tuple = new MultiGroupMultiplexer.HandlerTuple(channel, flags, asyncState); + parent.SubscribeToAll(tuple); + return tuple.Queue!; + } + + public async Task SubscribeAsync( + RedisChannel channel, + CommandFlags flags = CommandFlags.None) + { + var tuple = new MultiGroupMultiplexer.HandlerTuple(channel, flags, asyncState); + await parent.SubscribeToAllAsync(tuple); + return tuple.Queue!; + } + + // to do this we'd need to track the *filtered* subscriber per node per subscriber per channel + public void Unsubscribe( + RedisChannel channel, + Action? handler = null, + CommandFlags flags = CommandFlags.None) => throw new NotSupportedException("Removing individual pub/sub handlers from multi-group subscribers is not currently supported"); + + public Task UnsubscribeAsync( + RedisChannel channel, + Action? handler = null, + CommandFlags flags = CommandFlags.None) => throw new NotSupportedException("Removing individual pub/sub handlers from multi-group subscribers is not currently supported"); + + public void UnsubscribeAll(CommandFlags flags = CommandFlags.None) => parent.UnsubscribeFromAll(flags, asyncState); + + public Task UnsubscribeAllAsync(CommandFlags flags = CommandFlags.None) => parent.UnsubscribeFromAllAsync(flags, asyncState); +} + +internal sealed partial class MultiGroupMultiplexer +{ + private static Action FilteredHandler(MultiGroupMultiplexer parent, ConnectionGroupMember active, Action handler) + { + // invoke a handler only if the active member is the one we expect + return (channel, value) => + { + if (ReferenceEquals(parent._activeStub.Active, active.Multiplexer)) + { + handler(channel, value); + } + }; + } + + private static void ForwardFilteredMessages( + MultiGroupMultiplexer parent, + ConnectionGroupMember active, + ChannelMessageQueue writeTo, + ChannelMessageQueue readFrom) + { + // Create an async worker; note we can't just use a handler-style callback, because the + // key point of ChannelMessageQueue is to preserve order, and our callback implementation explicitly + // does not guarantee anything about order. + _ = Task.Run(() => ForwardFilteredMessagesAsync(parent, active, writeTo, readFrom)); + static async Task ForwardFilteredMessagesAsync( + MultiGroupMultiplexer parent, + ConnectionGroupMember active, + ChannelMessageQueue writeTo, + ChannelMessageQueue readFrom) + { + try + { + while (await readFrom.WaitToReadAsync()) + { + while (readFrom.TryRead(out var message)) + { + if (ReferenceEquals(parent._activeStub.Active, active.Multiplexer)) + { + // Because of the switchover being imperfect, we can't guarantee exactly one writer, so + // we need to be synchronized; in reality, it will *almost never* be contended, so + // this isn't a bottleneck - so we will pay the price of a lock here, and keep the + // queue in single-writer mode. + writeTo.SynchronizedWrite(message.Channel, message.Message); + } + } + } + } + catch (Exception ex) + { + parent.OnInternalError(ex); + } + } + } + + internal readonly struct HandlerTuple + { + public readonly RedisChannel Channel; + public Action? Handler => _handlerOrQueue as Action; + public ChannelMessageQueue? Queue => _handlerOrQueue as ChannelMessageQueue; + public readonly CommandFlags Flags; + public readonly object? AsyncState; + + private readonly object _handlerOrQueue; + + public HandlerTuple(RedisChannel channel, Action handler, CommandFlags flags, object? asyncState) + { + Channel = channel; + _handlerOrQueue = handler; + Flags = flags; + AsyncState = asyncState; + } + + public HandlerTuple(RedisChannel channel, CommandFlags flags, object? asyncState) + { + Channel = channel; + // note: multi-writer because we can't rule out race conditions at switchover + _handlerOrQueue = new ChannelMessageQueue(channel, null); + Flags = flags; + AsyncState = asyncState; + } + } + + private List _handlers = new(); + + internal void SubscribeToAll(HandlerTuple tuple) + { + lock (_handlers) + { + _handlers.Add(tuple); + } + foreach (var member in _members) + { + var sub = member.Multiplexer.GetSubscriber(tuple.AsyncState); + if (tuple.Handler is not null) + { + sub.Subscribe(tuple.Channel, FilteredHandler(this, member, tuple.Handler), tuple.Flags); + } + else if (tuple.Queue is not null) + { + var from = sub.Subscribe(tuple.Channel, tuple.Flags); + ForwardFilteredMessages(this, member, tuple.Queue, from); + } + } + } + + internal async Task SubscribeToAllAsync(HandlerTuple tuple) + { + lock (_handlers) + { + _handlers.Add(tuple); + } + foreach (var member in _members) + { + var sub = member.Multiplexer.GetSubscriber(tuple.AsyncState); + if (tuple.Handler is not null) + { + await sub.SubscribeAsync(tuple.Channel, FilteredHandler(this, member, tuple.Handler), tuple.Flags); + } + else if (tuple.Queue is not null) + { + var from = await sub.SubscribeAsync(tuple.Channel, tuple.Flags); + ForwardFilteredMessages(this, member, tuple.Queue, from); + } + } + } + + private HandlerTuple[] LeaseHandlers(out int count) + { + HandlerTuple[] lease; + lock (_handlers) + { + count = _handlers.Count; + if (count == 0) return []; + lease = ArrayPool.Shared.Rent(count); + _handlers.CopyTo(lease, 0); + } + + return lease; + } + + private async Task AddPubSubHandlersAsync(ConnectionGroupMember member) + { + // when adding a connection to an established group, add any missing pub/sub handlers + var lease = LeaseHandlers(out var count); + object? asyncState = null; + ISubscriber? sub = null; // try to reuse when possible + for (int i = 0; i < count; i++) + { + var tuple = lease[i]; + if (sub is null || tuple.AsyncState != asyncState) + { + asyncState = tuple.AsyncState; + sub = member.Multiplexer.GetSubscriber(asyncState); + } + if (tuple.Handler is not null) + { + await sub.SubscribeAsync(tuple.Channel, FilteredHandler(this, member, tuple.Handler), tuple.Flags); + } + else if (tuple.Queue is not null) + { + var from = await sub.SubscribeAsync(tuple.Channel, tuple.Flags); + ForwardFilteredMessages(this, member, tuple.Queue, from); + } + } + + // Deliberately *not* in a try/finally: if an await above faults (e.g. timeout/cancellation), the + // in-flight async operation may still hold a reference to this array, so returning it to the pool + // could hand the same buffer to another renter while it is still being read. We only return it on + // the clean path; on the (rare) fault path we simply let it be collected. + ArrayPool.Shared.Return(lease); + } + + public void UnsubscribeFromAll(CommandFlags flags, object? asyncState) + { + lock (_handlers) + { + _handlers.Clear(); + } + foreach (var member in _members) + { + member.Multiplexer.GetSubscriber(asyncState).UnsubscribeAll(flags); + } + } + + public async Task UnsubscribeFromAllAsync(CommandFlags flags, object? asyncState) + { + lock (_handlers) + { + _handlers.Clear(); + } + foreach (var member in _members) + { + await member.Multiplexer.GetSubscriber(asyncState).UnsubscribeAllAsync(flags); + } + } +} diff --git a/src/StackExchange.Redis/Availability/MultiGroupSubscriber.cs b/src/StackExchange.Redis/Availability/MultiGroupSubscriber.cs new file mode 100644 index 000000000..4dbb8e9e3 --- /dev/null +++ b/src/StackExchange.Redis/Availability/MultiGroupSubscriber.cs @@ -0,0 +1,45 @@ +using System; +using System.Net; +using System.Threading.Tasks; + +namespace StackExchange.Redis.Availability; + +internal sealed partial class MultiGroupSubscriber(MultiGroupMultiplexer parent, object? asyncState) : ISubscriber +{ + // for a lot of things, we can defer through to the active implementation + private ISubscriber GetActiveSubscriber() => parent.Active.GetSubscriber(asyncState); + + // non-throwing twin of GetActiveSubscriber: returns null when the group currently has no active + // member, for use by members that have an obvious trivial result when disconnected + private ISubscriber? TryGetActiveSubscriber() => parent.TryGetActive()?.GetSubscriber(asyncState); + + public IConnectionMultiplexer Multiplexer => parent; + + public bool TryWait(Task task) => GetActiveSubscriber().TryWait(task); + + public void Wait(Task task) => GetActiveSubscriber().Wait(task); + + public T Wait(Task task) => GetActiveSubscriber().Wait(task); + + public void WaitAll(params Task[] tasks) => GetActiveSubscriber().WaitAll(tasks); + + public TimeSpan Ping(CommandFlags flags = CommandFlags.None) => GetActiveSubscriber().Ping(flags); + + public Task PingAsync(CommandFlags flags = CommandFlags.None) => GetActiveSubscriber().PingAsync(flags); + + public EndPoint? IdentifyEndpoint(RedisChannel channel, CommandFlags flags = CommandFlags.None) => + TryGetActiveSubscriber()?.IdentifyEndpoint(channel, flags); + + public Task IdentifyEndpointAsync(RedisChannel channel, CommandFlags flags = CommandFlags.None) => + TryGetActiveSubscriber()?.IdentifyEndpointAsync(channel, flags) ?? MultiGroupMultiplexer.NoEndpoint; + + public bool IsConnected(RedisChannel channel = default) => TryGetActiveSubscriber()?.IsConnected() ?? false; + + public long Publish(RedisChannel channel, RedisValue message, CommandFlags flags = CommandFlags.None) + => GetActiveSubscriber().Publish(channel, message, flags); + + public Task PublishAsync(RedisChannel channel, RedisValue message, CommandFlags flags = CommandFlags.None) + => GetActiveSubscriber().PublishAsync(channel, message, flags); + + public EndPoint? SubscribedEndpoint(RedisChannel channel) => TryGetActiveSubscriber()?.SubscribedEndpoint(channel); +} diff --git a/src/StackExchange.Redis/Availability/RetryController.cs b/src/StackExchange.Redis/Availability/RetryController.cs new file mode 100644 index 000000000..a47d79bbc --- /dev/null +++ b/src/StackExchange.Redis/Availability/RetryController.cs @@ -0,0 +1,140 @@ +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Interfaces; + +namespace StackExchange.Redis.Availability; + +/// +/// Holds the retry configuration derived from a (attempt counts, delays, +/// failover gating) and makes the per-attempt retry decision. This is shared by +/// and so the two paths apply identical policy math. +/// +internal sealed class RetryController +{ + private readonly int _maxBeforeFailover, _maxAttempts, _delayMillis, _jitterMillis, _failoverMillis, _maxWatchAttempts; + private readonly RetryPolicy _policy; + + public RetryController(RetryPolicy policy, DatabaseFeatureFlags features) + { + _policy = policy; + + // capture config locally rather than constant cross-object lookups; a RetryPolicy is immutable and + // is validated by RetryPolicy.Builder, so no range checks are needed here + _maxBeforeFailover = (features & DatabaseFeatureFlags.Failover) == 0 ? int.MaxValue : policy.MaxAttemptsBeforeFailover; + _maxAttempts = policy.MaxAttempts; + if (_maxBeforeFailover == _maxAttempts) _maxBeforeFailover = int.MaxValue; // then we'll never look + + _delayMillis = ToMilliseconds(policy.RetryDelay); + _failoverMillis = ToMilliseconds(policy.FailoverDelay); + _jitterMillis = ToMilliseconds(policy.JitterMax); + _maxWatchAttempts = policy.MaxAttemptsOnWatchConflict; + + Debug.Assert(_maxAttempts >= 1 && _maxBeforeFailover >= 1, "attempt counts should be validated by RetryPolicy"); + Debug.Assert(_delayMillis >= 0 && _jitterMillis >= 0 && _failoverMillis >= 0, "delays should be validated by RetryPolicy"); + Debug.Assert(_maxWatchAttempts >= 1, "watch-conflict attempts should be validated by RetryPolicy"); + + static int ToMilliseconds(TimeSpan value) => (int)(value.Ticks / TimeSpan.TicksPerMillisecond); + } + + /// + /// The policy this controller is applying; exposed for tests, which assert how a policy was resolved. + /// + public RetryPolicy Policy => _policy; + + /// + /// How many times a conditional transaction may be attempted when the server keeps rejecting the + /// EXEC due to watch contention; see . + /// + public int MaxWatchConflictAttempts => _maxWatchAttempts; + + /// + /// The pause before re-attempting a transaction that lost a WATCH race. Contention, not a fault: + /// no backoff, just jitter to avoid two callers colliding again in lock-step. + /// + public Task WatchConflictDelayAsync() + => _jitterMillis is 0 + ? Task.CompletedTask + : Task.Delay(ServerSelectionStrategy.SharedRandom.Next(_jitterMillis), CancellationToken.None); + + /// + /// Whether it is ever worth capturing the next-failover token: only when there is more than one + /// attempt and the failover threshold sits below the attempt cap. + /// + public bool TracksFailover => _maxAttempts > 1 & _maxBeforeFailover < _maxAttempts; + + public bool CanRetry( + int attempt, + Exception fault, + ref CancellationToken failover, + out CancellationToken delay) + { + delay = CancellationToken.None; + if (attempt >= _maxAttempts) + { + // all used up + return false; + } + + // ask the retry policy for advice, and mask off the bits we know about + FaultContext ctx = new(fault); + var policy = _policy.CanRetry(ctx) & + (RetryResult.FailoverServer | RetryResult.SameServer); + if (policy is 0) + { + // retry policy says: nope + return false; + } + + if (policy is RetryResult.FailoverServer) + { + // we can *only* retry on a different server; is failover available? + delay = failover; + failover = CancellationToken.None; // only failover once + return delay.CanBeCanceled; + } + + if (attempt == _maxBeforeFailover) + { + // by count, we should really switch over to the failover now; is failover available *and* are we allowed? + delay = failover; + failover = CancellationToken.None; // only failover once + return delay.CanBeCanceled & (policy & RetryResult.FailoverServer) != 0; + } + + // can we pause and retry on the same server? + return (policy & RetryResult.SameServer) != 0; + } + + public Task FailoverOrDelayAsync(CancellationToken delay) + { + if (delay.CanBeCanceled) + { + return AwaitFailover(delay); + } + + // this is just a routine wait between operations; await delay+jitter + return Task.Delay(_delayMillis + ServerSelectionStrategy.SharedRandom.Next(_jitterMillis), CancellationToken.None); + } + + private async Task AwaitFailover(CancellationToken failover) + { + if (!failover.IsCancellationRequested) + { + // failover hasn't happened yet; allow up to "delay" time for that + try + { + await Task.Delay(_failoverMillis, failover).ConfigureAwait(false); + } + catch (OperationCanceledException) when (failover.IsCancellationRequested) + { + // we observed a failover, nice! + } + } + + // either way, we need to add jitter onto that; we can't add in the original delay, because if the failover + // happened before the timeout+jitter, all the awaiters would stampede + await Task.Delay(ServerSelectionStrategy.SharedRandom.Next(_jitterMillis), CancellationToken.None).ConfigureAwait(false); + } +} diff --git a/src/StackExchange.Redis/Availability/RetryDatabase.cs b/src/StackExchange.Redis/Availability/RetryDatabase.cs new file mode 100644 index 000000000..ea037d786 --- /dev/null +++ b/src/StackExchange.Redis/Availability/RetryDatabase.cs @@ -0,0 +1,168 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Interfaces; + +namespace StackExchange.Redis.Availability; + +[AutoDatabase] +internal partial class RetryDatabase : IDatabaseAsync, IInternalDatabaseAsync + // IRedisArgsMutator <==== if we ever want to support key-mapping +{ + // Note: we very deliberately do not include synchronous support for retry; it is inherently delay-ish + + // Note that only transient faults result in retries; this is defined by the RetryPolicy, along with + // understanding the category. The default RetryPolicy works the same as the default CircuitBreaker. + DatabaseFeatureFlags IInternalDatabaseAsync.GetFeatures(out string name) + => _inner.GetFeatures(out name) | DatabaseFeatureFlags.Retry; + + // never: we refuse to wrap a database that carries one (see Validate) + object? IInternalDatabaseAsync.AsyncState => null; + + /// + public override string ToString() => this.BuildString(); + + private readonly IDatabaseAsync _inner; + private readonly RetryController _controller; + + internal RetryPolicy Policy => _controller.Policy; + + public CancellationToken GetNextFailover() + => _controller.TracksFailover ? _inner.GetNextFailover() : CancellationToken.None; + + public RetryDatabase(IDatabaseAsync inner, RetryPolicy policy) + : this(inner, policy, Validate(inner)) + { + } + + private static DatabaseFeatureFlags Validate(IDatabaseAsync inner) + { + // cannot nest retry, and cannot issue retries *inside* a batch/transaction + var features = inner.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction | DatabaseFeatureFlags.Retry); + + // async-state is stamped onto the task that a *single* attempt produces; a retrying database hands + // back its own durable task that spans however many attempts it takes, so it cannot preserve the + // state. Refuse rather than dropping it silently. See also CreateTransaction, below. + if (inner.GetAsyncState() is not null) ThrowAsyncState(); + return features; + } + + internal static void ThrowAsyncState() => throw new InvalidOperationException( + "Retrying databases do not support asyncState; the tasks they hand back are not the tasks that were sent to the server."); + + // test-only: supply the inner database's feature set directly (in particular whether failover is + // available), instead of probing a live inner - so that failover behaviour can be exercised over a + // null inner without a full IDatabaseAsync double. + internal RetryDatabase(IDatabaseAsync inner, RetryPolicy policy, DatabaseFeatureFlags features) + { + _controller = new RetryController(policy, features); + _inner = inner; + } + + ITransactionAsync IDatabaseAsync.CreateTransaction(object? asyncState) + { + // as per the constructor: the per-operation tasks handed out at build time are durable proxies + // that outlive any single attempt, so they cannot carry a per-attempt async-state + if (asyncState is not null) ThrowAsyncState(); + + // the inner database creates the "real" (one-shot) transactions we replay against each attempt + return new RetryTransaction(_inner, _controller); + } + + public int Database => _inner.Database; + + public IConnectionMultiplexer Multiplexer => _inner.Multiplexer; + + // the generated explicit interface implementations funnel every call through these two + // overloads: the arguments are captured in a generated state struct and replayed against + // the inner database via a cacheable static projection (no per-call closure). Retry/failover + // policy will live here in due course; for now it is a straight pass-through. + + // async counterparts (Task / Task); these get their own retry/failover policy in due course. + // note: the state cannot be taken by `in` here (async methods forbid by-ref parameters); + // only the projection takes it by readonly-ref, avoiding a copy per attempt + private async Task ExecuteAsync(TState state, AutoDatabaseAsyncOperation operation) + where TState : struct + { + /* key mapping, not used currently + * state.MapInPlace(this); */ + + int attempt = 0; + TResult result; + // note we need to capture this *before* the attempt - otherwise the failover could happen + // between the failed attempt and fetching this, and we'd miss it + CancellationToken failover = GetNextFailover(); + while (true) + { + try + { + result = await operation(in state, _inner).ConfigureAwait(false); + break; + } + catch (Exception ex) when (_controller.CanRetry(++attempt, ex, ref failover, out var delay)) + { + await _controller.FailoverOrDelayAsync(delay).ConfigureAwait(false); + } + } + /* key mapping, not used currently + // post-process results outside the loop + return this.UnMap(state, result);*/ + return result; + } + + private async Task ExecuteAsync(TState state, AutoDatabaseAsyncOperation operation) + where TState : struct + { + /* key mapping, not used currently + * state.MapInPlace(this); */ + + int attempt = 0; + // note we need to capture this *before* the attempt - otherwise the failover could happen + // between the failed attempt and fetching this, and we'd miss it + CancellationToken failover = GetNextFailover(); + while (true) + { + try + { + await operation(in state, _inner).ConfigureAwait(false); + break; + } + catch (Exception ex) when (_controller.CanRetry(++attempt, ex, ref failover, out var delay)) + { + await _controller.FailoverOrDelayAsync(delay).ConfigureAwait(false); + } + } + // (nothing to post-process) + } + + void IRedisAsync.Wait(Task task) => _inner.Wait(task); + T IRedisAsync.Wait(Task task) => _inner.Wait(task); + void IRedisAsync.WaitAll(Task[] tasks) => _inner.WaitAll(tasks); + bool IRedisAsync.TryWait(Task task) => _inner.TryWait(task); + + // Methods the generator deliberately skips (see AutoDatabaseGenerator.SkipMethod): the Wait + // family, the synchronous IsConnected probe, and the streaming IEnumerable/IAsyncEnumerable scans + // don't fit the capture-and-replay shape, so they are implemented by hand. + // IsConnected is a straight pass-through: it is a cheap status check, not a server round-trip to retry. + bool IDatabaseAsync.IsConnected(RedisKey key, CommandFlags flags) => _inner.IsConnected(key, flags); + + // routing lookup, not a replayable server command - forward straight through (no retry) + Task IDatabaseAsync.IdentifyEndpointAsync(RedisKey key, CommandFlags flags) => _inner.IdentifyEndpointAsync(key, flags); + + // Scans are streaming/cursored, so they can't be captured-and-replayed as a single unit; rather than + // failing outright we forward straight to the inner database - giving up retry, but keeping the scan working. + System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.HashScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => _inner.HashScanAsync(key, pattern, pageSize, cursor, pageOffset, flags); + System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.HashScanNoValuesAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => _inner.HashScanNoValuesAsync(key, pattern, pageSize, cursor, pageOffset, flags); + System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.SetScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => _inner.SetScanAsync(key, pattern, pageSize, cursor, pageOffset, flags); + System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.VectorSetRangeEnumerateAsync(RedisKey key, RedisValue start, RedisValue end, long count, Exclude exclude, CommandFlags flags) => _inner.VectorSetRangeEnumerateAsync(key, start, end, count, exclude, flags); + System.Collections.Generic.IAsyncEnumerable IDatabaseAsync.SortedSetScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => _inner.SortedSetScanAsync(key, pattern, pageSize, cursor, pageOffset, flags); + + /* optional: key mapping + RedisKey IRedisArgsMutator.Map(RedisKey key) => key; + + RedisChannel IRedisArgsMutator.Map(RedisChannel channel) => channel; + + RedisKey IRedisArgsMutator.UnMap(RedisKey key) => key; + RedisChannel IRedisArgsMutator.UnMap(RedisChannel channel) => channel; + */ +} diff --git a/src/StackExchange.Redis/Availability/RetryPolicy.cs b/src/StackExchange.Redis/Availability/RetryPolicy.cs new file mode 100644 index 000000000..b6af57b6a --- /dev/null +++ b/src/StackExchange.Redis/Availability/RetryPolicy.cs @@ -0,0 +1,307 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Configures how messages can be retried due to connection / transient faults. Other faults (such as invalid +/// usage) are not retried. +/// +/// +/// Instances are immutable and safe to share; use to configure the standard policy, or +/// derive from this type and override to make the decision yourself. +/// +[Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +public class RetryPolicy +{ + internal const int DefaultMaxAttempts = 3; + internal const int DefaultMaxAttemptsBeforeFailover = 1; + internal const int DefaultMaxAttemptsOnWatchConflict = 3; + internal const CommandFlags DefaultMaxCommandRetryCategory = CommandFlags.CommandRetryWriteLastWins; + internal static readonly TimeSpan DefaultRetryDelay = TimeSpan.FromSeconds(1); + internal static readonly TimeSpan DefaultJitterMax = TimeSpan.FromMilliseconds(500); + internal static readonly TimeSpan DefaultFailoverDelay = TimeSpan.FromSeconds(5); + + private static readonly RetryPolicy DefaultInstance = new(); + + /// + /// The default retry policy; retries transient faults up to times, for commands + /// at or below . + /// + public static RetryPolicy Default => DefaultInstance; + + /// + /// Never retries anything; useful to disable retries without restructuring calling code. + /// + public static RetryPolicy None => NoRetryPolicy.Instance; + + /// + /// Create a policy using the default settings; intended for use by derived types that override + /// - use to obtain the standard policy. + /// + protected RetryPolicy() + : this(DefaultMaxAttempts, DefaultMaxAttemptsBeforeFailover, DefaultMaxAttemptsOnWatchConflict, DefaultRetryDelay, DefaultJitterMax, DefaultFailoverDelay, DefaultMaxCommandRetryCategory) + { + } + + /// + /// Create a policy using the settings from the supplied ; intended for use by + /// derived types that override . + /// + protected RetryPolicy(Builder builder) + : this( + Validate(builder).MaxAttempts, + builder.MaxAttemptsBeforeFailover, + builder.MaxAttemptsOnWatchConflict, + builder.RetryDelay, + builder.JitterMax, + builder.FailoverDelay, + builder.MaxCommandRetryCategory) + { + } + + private RetryPolicy( + int maxAttempts, + int maxAttemptsBeforeFailover, + int maxAttemptsOnWatchConflict, + TimeSpan retryDelay, + TimeSpan jitterMax, + TimeSpan failoverDelay, + CommandFlags maxCommandRetryCategory) + { + MaxAttempts = maxAttempts; + MaxAttemptsBeforeFailover = maxAttemptsBeforeFailover; + MaxAttemptsOnWatchConflict = maxAttemptsOnWatchConflict; + RetryDelay = retryDelay; + JitterMax = jitterMax; + FailoverDelay = failoverDelay; + MaxCommandRetryCategory = maxCommandRetryCategory; + } + + /// + public override string ToString() => $"{GetType().Name}: {MaxAttempts} attempt(s), up to {MaxCommandRetryCategory}"; + + /// + /// The maximum number of times an operation can be attempted. Defaults to 3. + /// + public int MaxAttempts { get; } + + /// + /// The maximum number of times to retry an operation before waiting for failover; this only currently + /// applies to multi-group connections created via ConnectionMultiplexer.ConnectGroupAsync. + /// Defaults to 1. + /// + public int MaxAttemptsBeforeFailover { get; } + + /// + /// The maximum number of times a *conditional* transaction may be attempted when the only problem is + /// that the server rejected the EXEC because a watched key changed underneath it. Defaults to 3; + /// a value of 1 disables re-attempting such transactions. + /// + /// + /// This is deliberately separate from : a watch conflict is contention, + /// not a fault. Nothing was applied, nothing is broken, and the right response is to re-read the + /// conditions and try again immediately - so no is applied (only + /// ), no failover is attempted, and does + /// not apply. Each re-attempt re-issues the WATCH constraints, so a transaction whose condition + /// has genuinely stopped holding converges on an ordinary elective abort rather than looping. + /// Only transactions with conditions can be affected: without a condition there is no + /// WATCH, so there is nothing to conflict. + /// + public int MaxAttemptsOnWatchConflict { get; } + + /// + /// Gets the time to wait between retries that are *not* dependent on a failover happening. Defaults to 1 second. + /// + public TimeSpan RetryDelay { get; } + + /// + /// Gets the time to wait for a failover, after attempts. Only one + /// failover attempt is awaited. When this applies, is ignored, + /// but is still respected. Defaults to 5 seconds. + /// + public TimeSpan FailoverDelay { get; } + + /// + /// Gets the upper bound for jitter - additional random delay between retries to prevent stampedes. + /// Defaults to 0.5 seconds, meaning between 0 and 0.5 *additional* seconds on top of . + /// + public TimeSpan JitterMax { get; } + + /// + /// Gets the max side-effect category that will be retried; defaults to . + /// + public CommandFlags MaxCommandRetryCategory { get; } + + /// + /// Controls which operations can be repeated, optionally indicating that this should progress to + /// a new server. + /// + public virtual RetryResult CanRetry(in FaultContext fault) + { + var actual = fault.Flags & Message.MaskRetryCategory; + if (actual is 0) actual = CommandFlags.CommandRetryNever; // if not set, assume the worst (as FaultContext does) + + if (actual is CommandFlags.CommandRetryNever) + { + // explicitly disabled at command level + return RetryResult.None; + } + + // the category exists to price the *ambiguity* of a replay: if we know the operation never took + // effect, re-issuing it is a first attempt rather than a repeat, so it cannot double-apply and the + // side-effect scale is irrelevant. (CommandRetryNever above is still an absolute veto.) + if (actual > MaxCommandRetryCategory && !fault.NotApplied) + { + // side-effects are beyond what the policy allows + return RetryResult.None; + } + + if (CircuitBreaker.DefaultIsFailure(in fault)) + { + // assume we can send it everywhere + var result = RetryResult.SameServer | RetryResult.FailoverServer; + if ((fault.Flags & Message.CommandServerSpecific) != 0) + result &= ~RetryResult.FailoverServer; + return result; + } + + // do not retry + return RetryResult.None; + } + + private static Builder Validate(Builder builder) + { + if (builder is null) throw new ArgumentNullException(nameof(builder)); + if (builder.MaxAttempts < 1) throw new ArgumentOutOfRangeException(nameof(builder.MaxAttempts), builder.MaxAttempts, "At least one attempt is required; use RetryPolicy.None to disable retries."); + + // values < 1 can never be hit by the attempt counter (which starts at 1), so they would *silently* + // disable failover rather than erroring + if (builder.MaxAttemptsBeforeFailover < 1) throw new ArgumentOutOfRangeException(nameof(builder.MaxAttemptsBeforeFailover), builder.MaxAttemptsBeforeFailover, "At least one attempt is required before failover."); + + // this one counts *attempts*, so 1 means "try once, do not re-attempt"; zero or negative is + // meaningless rather than a way to say "never execute" + if (builder.MaxAttemptsOnWatchConflict < 1) throw new ArgumentOutOfRangeException(nameof(builder.MaxAttemptsOnWatchConflict), builder.MaxAttemptsOnWatchConflict, "At least one attempt is required; use 1 to disable re-attempting on watch conflicts."); + + if (builder.RetryDelay < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(builder.RetryDelay), builder.RetryDelay, "A non-negative retry delay is required."); + if (builder.JitterMax < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(builder.JitterMax), builder.JitterMax, "A non-negative jitter bound is required."); + if (builder.FailoverDelay < TimeSpan.Zero) throw new ArgumentOutOfRangeException(nameof(builder.FailoverDelay), builder.FailoverDelay, "A non-negative failover delay is required."); + + // the retry loop expresses all three delays in int milliseconds + if (!IsExpressibleAsMilliseconds(builder.RetryDelay)) throw new ArgumentOutOfRangeException(nameof(builder.RetryDelay), builder.RetryDelay, "The retry delay is too large."); + if (!IsExpressibleAsMilliseconds(builder.JitterMax)) throw new ArgumentOutOfRangeException(nameof(builder.JitterMax), builder.JitterMax, "The jitter bound is too large."); + if (!IsExpressibleAsMilliseconds(builder.FailoverDelay)) throw new ArgumentOutOfRangeException(nameof(builder.FailoverDelay), builder.FailoverDelay, "The failover delay is too large."); + + var category = builder.MaxCommandRetryCategory; + if ((category & Message.MaskRetryCategory) is 0 | (category & ~Message.MaskRetryCategory) is not 0) + { + throw new ArgumentException("A single valid CommandRetry* flag should be specified.", nameof(builder.MaxCommandRetryCategory)); + } + + return builder; + + static bool IsExpressibleAsMilliseconds(TimeSpan value) => value.Ticks / TimeSpan.TicksPerMillisecond <= int.MaxValue; + } + + /// + /// Allows configuration of the standard implementation. + /// + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + public sealed class Builder + { + /// + /// Create a builder pre-populated with the default values. + /// + public Builder() + { + } + + /// + /// Create a builder pre-populated from an existing . + /// + public Builder(RetryPolicy policy) + { + MaxAttempts = policy.MaxAttempts; + MaxAttemptsBeforeFailover = policy.MaxAttemptsBeforeFailover; + MaxAttemptsOnWatchConflict = policy.MaxAttemptsOnWatchConflict; + RetryDelay = policy.RetryDelay; + JitterMax = policy.JitterMax; + FailoverDelay = policy.FailoverDelay; + MaxCommandRetryCategory = policy.MaxCommandRetryCategory; + } + + /// + /// The maximum number of times an operation can be attempted. + /// + public int MaxAttempts { get; set; } = DefaultMaxAttempts; + + /// + /// The maximum number of times to retry an operation before waiting for failover. + /// + public int MaxAttemptsBeforeFailover { get; set; } = DefaultMaxAttemptsBeforeFailover; + + /// + public int MaxAttemptsOnWatchConflict { get; set; } = DefaultMaxAttemptsOnWatchConflict; + + /// + /// The time to wait between retries that are *not* dependent on a failover happening. + /// + public TimeSpan RetryDelay { get; set; } = DefaultRetryDelay; + + /// + /// The upper bound for jitter - additional random delay between retries to prevent stampedes. + /// + public TimeSpan JitterMax { get; set; } = DefaultJitterMax; + + /// + /// The time to wait for a failover, after attempts. + /// + public TimeSpan FailoverDelay { get; set; } = DefaultFailoverDelay; + + /// + /// The max side-effect category that will be retried. + /// + public CommandFlags MaxCommandRetryCategory { get; set; } = DefaultMaxCommandRetryCategory; + + /// + /// Create a new retry policy instance. + /// + public RetryPolicy Create() + { + Validate(this); + + // prefer the shared default instance when nothing has been customized + if (MaxAttempts == DefaultMaxAttempts + && MaxAttemptsBeforeFailover == DefaultMaxAttemptsBeforeFailover + && MaxAttemptsOnWatchConflict == DefaultMaxAttemptsOnWatchConflict + && RetryDelay == DefaultRetryDelay + && JitterMax == DefaultJitterMax + && FailoverDelay == DefaultFailoverDelay + && MaxCommandRetryCategory == DefaultMaxCommandRetryCategory) + { + return DefaultInstance; + } + + return new RetryPolicy(MaxAttempts, MaxAttemptsBeforeFailover, MaxAttemptsOnWatchConflict, RetryDelay, JitterMax, FailoverDelay, MaxCommandRetryCategory); + } + + /// + /// Create a new retry policy instance. + /// + public static implicit operator RetryPolicy(Builder builder) => builder.Create(); + } + + private sealed class NoRetryPolicy : RetryPolicy + { + public static readonly NoRetryPolicy Instance = new(); + + // "never retries anything" has to include watch contention: that path does not consult CanRetry + // (nothing was applied, so there is no fault to judge), it is bounded purely by the attempt count + private NoRetryPolicy() : base(new Builder { MaxAttemptsOnWatchConflict = 1 }) + { + } + + public override RetryResult CanRetry(in FaultContext fault) => RetryResult.None; + } +} diff --git a/src/StackExchange.Redis/Availability/RetryResult.cs b/src/StackExchange.Redis/Availability/RetryResult.cs new file mode 100644 index 000000000..2b99ed31a --- /dev/null +++ b/src/StackExchange.Redis/Availability/RetryResult.cs @@ -0,0 +1,28 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis.Availability; + +/// +/// Indicates the result of a query. +/// +[Flags] +[Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +public enum RetryResult +{ + /// + /// None; the operation should not be retried. + /// + None = 0, + + /// + /// The operation can be retried on the same server. + /// + SameServer = 1, + + /// + /// The operation can be retried on a different server after a failover operation. + /// + FailoverServer = 2, +} diff --git a/src/StackExchange.Redis/Availability/RetryTransaction.cs b/src/StackExchange.Redis/Availability/RetryTransaction.cs new file mode 100644 index 000000000..0d87cd47d --- /dev/null +++ b/src/StackExchange.Redis/Availability/RetryTransaction.cs @@ -0,0 +1,293 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Interfaces; + +namespace StackExchange.Redis.Availability; + +// A retryable transaction. Unlike RetryDatabase - which replays a single captured typed call each attempt - +// a transaction is built up across many builder calls plus an Execute, so we *record* each captured call +// (the AutoDatabase (state, projection) pair) and hand the caller a durable, still-incomplete proxy task. +// Only ExecuteAsync actually runs anything: for each attempt it spins up a fresh, one-shot inner transaction +// against the underlying database (which, for a multi-group connection, resolves the currently-active member, +// so a retry after failover lands on the new member), replays every recorded operation and constraint onto +// it, and awaits it. On a clean execution the per-attempt results are forwarded onto the durable proxies; on +// a transient fault the whole attempt is discarded and replayed; on terminal failure the proxies are faulted. +[AutoDatabase] +internal sealed partial class RetryTransaction : IDatabaseAsync, ITransactionAsync +{ + // Note: async-only, exactly like RetryDatabase - retrying is inherently delay-ish. + private readonly IDatabaseAsync _source; + private readonly RetryController _controller; + + private readonly List _ops = new(); + private List? _conditions; + private int _executed; + private volatile bool _watchConflict; + + /// + // reports the *final* attempt's outcome: false if we eventually committed (or aborted electively), + // true if we ran out of watch-conflict attempts still losing the race + public bool WasWatchConflict => _watchConflict; + + public RetryTransaction(IDatabaseAsync source, RetryController controller) + { + _source = source; + _controller = controller; + } + + public int Database => _source.Database; + + public IConnectionMultiplexer Multiplexer => _source.Multiplexer; + + /// + public override string ToString() => "retry-transaction: " + _source; + + // the generated explicit interface implementations funnel every builder call through these two + // overloads, capturing the arguments in a generated state struct plus a cacheable static projection + // (no per-call closure). Here we simply *record* them and return a durable proxy task. + private Task ExecuteAsync(in TState state, AutoDatabaseAsyncOperation operation) + where TState : struct + { + CheckNotExecuted(); + var op = new RecordedOp(state, operation); + _ops.Add(op); + return op.Proxy; + } + + private Task ExecuteAsync(in TState state, AutoDatabaseAsyncOperation operation) + where TState : struct + { + CheckNotExecuted(); + var op = new RecordedVoidOp(state, operation); + _ops.Add(op); + return op.Proxy; + } + + public ConditionResult AddCondition(Condition condition) + { + if (condition is null) throw new ArgumentNullException(nameof(condition)); + CheckNotExecuted(); + var recorded = new RecordedCondition(condition); + (_conditions ??= new List()).Add(recorded); + return recorded.Result; + } + + public async Task ExecuteAsync(CommandFlags flags = CommandFlags.None) + { + if (Interlocked.CompareExchange(ref _executed, 1, 0) != 0) + { + throw new InvalidOperationException("This transaction has already been executed"); + } + + int attempt = 0; + // capture the next-failover token *before* the first attempt - otherwise a failover between a + // failed attempt and re-reading the token could be missed + CancellationToken failover = _controller.TracksFailover ? _source.GetNextFailover() : CancellationToken.None; + + var conditions = _conditions; + + // watch contention gets its own budget; only meaningful when there are conditions to WATCH + int watchAttempt = 0; + int maxWatchAttempts = conditions is null ? 1 : _controller.MaxWatchConflictAttempts; + while (true) + { + // no async-state: RetryDatabase refuses one (the durable proxies below cannot carry it) + var inner = _source.CreateTransaction(); + + // replay the recorded constraints and operations onto this fresh, one-shot transaction; the + // per-attempt tasks they return are forwarded to the durable proxies only on a clean execution + if (conditions is not null) + { + foreach (var c in conditions) c.Replay(inner); + } + foreach (var op in _ops) op.Replay(inner); + + // inject the aggregate retry category onto the EXEC flags so the resulting fault carries it, and + // the shared RetryPolicy/FaultContext logic gates the whole transaction exactly like one command + var category = inner is IInternalTransaction it ? it.GetAggregateRetryCategory() : CommandFlags.CommandRetryNever; + var effectiveFlags = (flags & ~Message.MaskRetryCategory) | category; + + try + { + bool committed = await inner.ExecuteAsync(effectiveFlags).ConfigureAwait(false); + _watchConflict = inner.WasWatchConflict; // surfaced to our own caller, per attempt + + // The server rejected an EXEC we really did issue, because another connection changed a + // watched key: the conditions still held, nothing was applied, and we simply lost a race. + // Re-read and try again (which re-issues the WATCH constraints, so a condition that has + // genuinely stopped holding converges on an elective abort instead of looping). This is + // contention rather than a fault, so it neither consumes the fault budget nor waits for a + // failover, and the side-effect category does not apply - nothing happened. + if (!committed + && _watchConflict + && ++watchAttempt < maxWatchAttempts) + { + foreach (var op in _ops) op.Observe(); + await _controller.WatchConflictDelayAsync().ConfigureAwait(false); + continue; + } + + // clean completion (committed, or aborted); forward the per-attempt outcomes onto the + // durable proxies and we're done + if (conditions is not null) + { + foreach (var c in conditions) c.ForwardSuccess(); + } + foreach (var op in _ops) op.ForwardSuccess(); + return committed; + } + catch (Exception ex) + { + if (_controller.CanRetry(++attempt, ex, ref failover, out var delay)) + { + // discard this attempt; observe its faulted per-attempt tasks so they don't surface as + // unobserved, then wait / failover and replay from the recorded snapshot + foreach (var op in _ops) op.Observe(); + await _controller.FailoverOrDelayAsync(delay).ConfigureAwait(false); + continue; + } + + // out of road: fault the durable proxies and surface the failure to the caller + foreach (var op in _ops) op.Fault(ex); + throw; + } + } + } + + private void CheckNotExecuted() + { + if (Volatile.Read(ref _executed) != 0) + { + throw new InvalidOperationException("Operations cannot be added after the transaction has been executed"); + } + } + + // ---- recorded work ------------------------------------------------------------------------------- + private interface IRecordedOp + { + void Replay(IDatabaseAsync inner); + void ForwardSuccess(); + void Fault(Exception ex); + void Observe(); + } + + private sealed class RecordedOp : IRecordedOp + where TState : struct + { + private readonly TState _state; + private readonly AutoDatabaseAsyncOperation _operation; + private readonly TaskCompletionSource _proxy = new(TaskCreationOptions.RunContinuationsAsynchronously); + private Task? _attempt; + + public RecordedOp(in TState state, AutoDatabaseAsyncOperation operation) + { + _state = state; + _operation = operation; + } + + public Task Proxy => _proxy.Task; + + public void Replay(IDatabaseAsync inner) => _attempt = _operation(in _state, inner); + + public void ForwardSuccess() + { + var attempt = _attempt!; + if (attempt.IsCanceled) _proxy.TrySetCanceled(); + else if (attempt.IsFaulted) _proxy.TrySetException(attempt.Exception!.InnerExceptions); + else _proxy.TrySetResult(attempt.GetAwaiter().GetResult()); + } + + // observe the (faulted) inner attempt before faulting the durable proxy, so the discarded + // per-attempt task doesn't surface as an unobserved exception + public void Fault(Exception ex) + { + Observe(); + _proxy.TrySetException(ex); + } + + public void Observe() => _ = _attempt?.Exception; + } + + private sealed class RecordedVoidOp : IRecordedOp + where TState : struct + { + private readonly TState _state; + private readonly AutoDatabaseAsyncOperation _operation; + private readonly TaskCompletionSource _proxy = new(TaskCreationOptions.RunContinuationsAsynchronously); + private Task? _attempt; + + public RecordedVoidOp(in TState state, AutoDatabaseAsyncOperation operation) + { + _state = state; + _operation = operation; + } + + public Task Proxy => _proxy.Task; + + public void Replay(IDatabaseAsync inner) => _attempt = _operation(in _state, inner); + + public void ForwardSuccess() + { + var attempt = _attempt!; + if (attempt.IsCanceled) _proxy.TrySetCanceled(); + else if (attempt.IsFaulted) _proxy.TrySetException(attempt.Exception!.InnerExceptions); + else _proxy.TrySetResult(true); + } + + // observe the (faulted) inner attempt before faulting the durable proxy, so the discarded + // per-attempt task doesn't surface as an unobserved exception + public void Fault(Exception ex) + { + Observe(); + _proxy.TrySetException(ex); + } + + public void Observe() => _ = _attempt?.Exception; + } + + private sealed class RecordedCondition + { + private readonly Condition _condition; + private ConditionResult? _attempt; + + public RecordedCondition(Condition condition) + { + _condition = condition; + Result = new ConditionResult(condition); + } + + // the durable result handed back to the caller from AddCondition + public ConditionResult Result { get; } + + public void Replay(ITransactionAsync inner) => _attempt = inner.AddCondition(_condition); + + public void ForwardSuccess() + { + if (_attempt is not null) Result.SetSatisfied(_attempt.WasSatisfied); + } + } + + // ---- hand-implemented members the generator deliberately skips ---------------------------------- + // (the Wait family, the synchronous IsConnected probe, and the streaming scans). Wait/IsConnected are + // straight pass-throughs; scans cannot participate in a transaction. + void IRedisAsync.Wait(Task task) => _source.Wait(task); + T IRedisAsync.Wait(Task task) => _source.Wait(task); + void IRedisAsync.WaitAll(Task[] tasks) => _source.WaitAll(tasks); + bool IRedisAsync.TryWait(Task task) => _source.TryWait(task); + + bool IDatabaseAsync.IsConnected(RedisKey key, CommandFlags flags) => _source.IsConnected(key, flags); + + // routing lookup against the underlying database, not a queued transaction operation + Task IDatabaseAsync.IdentifyEndpointAsync(RedisKey key, CommandFlags flags) => _source.IdentifyEndpointAsync(key, flags); + + // nested transactions are not supported (mirrors RedisTransaction) + ITransactionAsync IDatabaseAsync.CreateTransaction(object? asyncState) => throw new NotSupportedException("Nested transactions are not supported"); + + IAsyncEnumerable IDatabaseAsync.HashScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotSupportedException("Scans cannot be used inside a transaction"); + IAsyncEnumerable IDatabaseAsync.HashScanNoValuesAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotSupportedException("Scans cannot be used inside a transaction"); + IAsyncEnumerable IDatabaseAsync.SetScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotSupportedException("Scans cannot be used inside a transaction"); + IAsyncEnumerable IDatabaseAsync.VectorSetRangeEnumerateAsync(RedisKey key, RedisValue start, RedisValue end, long count, Exclude exclude, CommandFlags flags) => throw new NotSupportedException("Scans cannot be used inside a transaction"); + IAsyncEnumerable IDatabaseAsync.SortedSetScanAsync(RedisKey key, RedisValue pattern, int pageSize, long cursor, int pageOffset, CommandFlags flags) => throw new NotSupportedException("Scans cannot be used inside a transaction"); +} diff --git a/src/StackExchange.Redis/ChannelMessageQueue.cs b/src/StackExchange.Redis/ChannelMessageQueue.cs index 7defc35cc..4171ed4b4 100644 --- a/src/StackExchange.Redis/ChannelMessageQueue.cs +++ b/src/StackExchange.Redis/ChannelMessageQueue.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; @@ -34,7 +35,7 @@ public sealed class ChannelMessageQueue : IAsyncEnumerable /// public Task Completion => _queue.Reader.Completion; - internal ChannelMessageQueue(in RedisChannel redisChannel, RedisSubscriber parent) + internal ChannelMessageQueue(in RedisChannel redisChannel, RedisSubscriber? parent) { Channel = redisChannel; _parent = parent; @@ -48,8 +49,22 @@ internal ChannelMessageQueue(in RedisChannel redisChannel, RedisSubscriber paren private void Write(in RedisChannel channel, in RedisValue value) { - var writer = _queue.Writer; - writer.TryWrite(new ChannelMessage(this, channel, value)); + try + { + _queue.Writer.TryWrite(new ChannelMessage(this, channel, value)); + } + catch (Exception ex) + { + Debug.WriteLine("pub/sub ChannelWrite.TryWrite failed: " + ex.Message); + } + } + + internal void SynchronizedWrite(in RedisChannel channel, in RedisValue value) + { + lock (this) + { + Write(channel, value); + } } /// @@ -326,4 +341,7 @@ public async IAsyncEnumerator GetAsyncEnumerator(CancellationTok } } #endif + + internal ValueTask WaitToReadAsync(CancellationToken cancellationToken = default) + => _queue.Reader.WaitToReadAsync(cancellationToken); } diff --git a/src/StackExchange.Redis/ClusterConfiguration.cs b/src/StackExchange.Redis/ClusterConfiguration.cs index c474c3c1b..072c6720f 100644 --- a/src/StackExchange.Redis/ClusterConfiguration.cs +++ b/src/StackExchange.Redis/ClusterConfiguration.cs @@ -374,7 +374,7 @@ public IList Children /// /// Gets whether this node is a replica. /// - [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(IsReplica) + " instead, this will be removed in 3.0.")] + [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(IsReplica) + " instead, this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] public bool IsSlave => IsReplica; diff --git a/src/StackExchange.Redis/Condition.cs b/src/StackExchange.Redis/Condition.cs index fa7e76299..daa9979c1 100644 --- a/src/StackExchange.Redis/Condition.cs +++ b/src/StackExchange.Redis/Condition.cs @@ -953,6 +953,11 @@ internal ConditionResult(Condition condition) internal IEnumerable CreateMessages(int db) => Condition.CreateMessages(db, resultBox); internal IResultBox? GetBox() => resultBox; + + // used by the retry machinery to copy the outcome of a per-attempt condition onto the durable + // ConditionResult that was handed back to the caller when the transaction was built + internal void SetSatisfied(bool value) => wasSatisfied = value; + internal bool UnwrapBox() { if (resultBox != null) diff --git a/src/StackExchange.Redis/ConfigurationOptions.cs b/src/StackExchange.Redis/ConfigurationOptions.cs index eef0bde81..13affadd5 100644 --- a/src/StackExchange.Redis/ConfigurationOptions.cs +++ b/src/StackExchange.Redis/ConfigurationOptions.cs @@ -16,6 +16,7 @@ using Microsoft.Extensions.Logging; using RESPite; using RESPite.Buffers; +using StackExchange.Redis.Availability; using StackExchange.Redis.Configuration; namespace StackExchange.Redis @@ -44,6 +45,12 @@ public static int ParseInt32(string key, string value, int minValue = int.MinVal return tmp; } + public static float ParseSingle(string key, string value) + { + if (!Format.TryParseDouble(value, out double tmp)) throw new ArgumentOutOfRangeException(key, $"Keyword '{key}' requires a numeric value; the value '{value}' is not recognised."); + return (float)tmp; + } + internal static bool ParseBoolean(string key, string value) { if (!Format.TryParseBoolean(value, out bool tmp)) throw new ArgumentOutOfRangeException(key, $"Keyword '{key}' requires a boolean value; the value '{value}' is not recognised."); @@ -344,7 +351,7 @@ public int AsyncTimeout /// /// Indicates whether the connection should be encrypted. /// - [Obsolete("Please use .Ssl instead of .UseSsl, will be removed in 3.0."), + [Obsolete("Please use .Ssl instead of .UseSsl, will be removed in 3.2.", error: true), Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] public bool UseSsl @@ -671,7 +678,7 @@ public TimeSpan HeartbeatInterval /// Use ThreadPriority.AboveNormal for SocketManager reader and writer threads (true by default). /// If , will be used. /// - [Obsolete($"This setting no longer has any effect, please use {nameof(SocketManager.SocketManagerOptions)}.{nameof(SocketManager.SocketManagerOptions.UseHighPrioritySocketThreads)} instead - this setting will be removed in 3.0.")] + [Obsolete($"This setting no longer has any effect, please use {nameof(SocketManager.SocketManagerOptions)}.{nameof(SocketManager.SocketManagerOptions.UseHighPrioritySocketThreads)} instead - this setting will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] public bool HighPrioritySocketThreads { @@ -750,7 +757,7 @@ public string? Password /// /// Specifies whether asynchronous operations should be invoked in a way that guarantees their original delivery order. /// - [Obsolete("Not supported; if you require ordered pub/sub, please see " + nameof(ChannelMessageQueue) + " - this will be removed in 3.0.", false)] + [Obsolete("Not supported; if you require ordered pub/sub, please see " + nameof(ChannelMessageQueue) + " - this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] public bool PreserveAsyncOrder { @@ -798,7 +805,7 @@ public bool ResolveDns /// /// Specifies the time in milliseconds that the system should allow for responses before concluding that the socket is unhealthy. /// - [Obsolete("This setting no longer has any effect, and should not be used - will be removed in 3.0.")] + [Obsolete("This setting no longer has any effect, and should not be used - will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] public int ResponseTimeout { @@ -878,11 +885,12 @@ public string TieBreaker /// /// The size of the output buffer to use. /// - [Obsolete("This setting no longer has any effect, and should not be used - will be removed in 3.0.")] + [Obsolete("This setting no longer has any effect, and should not be used - will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] public int WriteBuffer { get => 0; + // ReSharper disable once ValueParameterNotUsed set { } } @@ -971,6 +979,8 @@ public static ConfigurationOptions Parse(string configuration, bool ignoreUnknow _protocol = _protocol, heartbeatInterval = heartbeatInterval, WriteMode = WriteMode, + CircuitBreaker = CircuitBreaker, + RetryPolicy = RetryPolicy, #if DEBUG OutputLog = OutputLog, #endif @@ -1174,6 +1184,8 @@ private void Clear() Tunnel = null; _protocol = default; WriteMode = default; + CircuitBreaker = null; + RetryPolicy = null; #if DEBUG OutputLog = null; #endif @@ -1372,6 +1384,28 @@ public RedisProtocol? Protocol } internal BufferedStreamWriter.WriteMode WriteMode { get; set; } + + /// + /// The circuit-breaker to apply to physical connections; when null, no breaker is used. + /// + /// + /// For a member of a connection group, the effective breaker is + /// , else this, else + /// . + /// + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + public CircuitBreaker? CircuitBreaker { get; set; } + + /// + /// The retry policy used by for databases + /// obtained from this connection; when null, is used. + /// + /// + /// For a member of a connection group, applies instead. + /// + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + public RetryPolicy? RetryPolicy { get; set; } + internal bool AllowSimulateConnectionFailure { get => IsSet(OptionFlags.AllowSimulateConnectionFailure); diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.Compat.cs b/src/StackExchange.Redis/ConnectionMultiplexer.Compat.cs index 6786e87d2..7108141ac 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.Compat.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.Compat.cs @@ -9,14 +9,14 @@ public partial class ConnectionMultiplexer /// /// No longer used. /// - [Obsolete("No longer used, will be removed in 3.0.")] + [Obsolete("No longer used, will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] public static TaskFactory Factory { get => Task.Factory; set { } } /// /// Gets or sets whether asynchronous operations should be invoked in a way that guarantees their original delivery order. /// - [Obsolete("Not supported; if you require ordered pub/sub, please see " + nameof(ChannelMessageQueue) + ", will be removed in 3.0", false)] + [Obsolete("Not supported; if you require ordered pub/sub, please see " + nameof(ChannelMessageQueue) + ", will be removed in 3.2", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] public bool PreserveAsyncOrder { get => false; set { } } } diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs b/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs index 52ff4ee1a..d651df072 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.Sentinel.cs @@ -181,6 +181,7 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co { throw new RedisConnectionException( ConnectionFailureType.UnableToConnect, + CommandFlags.None, "Sentinel: The ConnectionMultiplexer is not a Sentinel connection. Detected as: " + ServerSelectionStrategy.ServerType); } @@ -216,6 +217,7 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co { throw new RedisConnectionException( ConnectionFailureType.UnableToConnect, + CommandFlags.None, $"Sentinel: Failed connecting to configured primary for service: {config.ServiceName}"); } @@ -283,6 +285,7 @@ public ConnectionMultiplexer GetSentinelMasterConnection(ConfigurationOptions co { throw new RedisConnectionException( ConnectionFailureType.UnableToConnect, + CommandFlags.None, $"Sentinel: Failed connecting to configured primary for service: {config.ServiceName}"); } @@ -430,7 +433,7 @@ internal void SwitchPrimary(EndPoint? switchBlame, ConnectionMultiplexer connect // Get new primary - try twice EndPoint newPrimaryEndPoint = GetConfiguredPrimaryForService(serviceName) ?? GetConfiguredPrimaryForService(serviceName) - ?? throw new RedisConnectionException(ConnectionFailureType.UnableToConnect, $"Sentinel: Failed connecting to switch primary for service: {serviceName}"); + ?? throw new RedisConnectionException(ConnectionFailureType.UnableToConnect, CommandFlags.None, $"Sentinel: Failed connecting to switch primary for service: {serviceName}"); connection.currentSentinelPrimaryEndPoint = newPrimaryEndPoint; diff --git a/src/StackExchange.Redis/ConnectionMultiplexer.cs b/src/StackExchange.Redis/ConnectionMultiplexer.cs index d8e328987..ab704d69e 100644 --- a/src/StackExchange.Redis/ConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/ConnectionMultiplexer.cs @@ -47,6 +47,21 @@ public sealed partial class ConnectionMultiplexer : IInternalConnectionMultiplex internal CommandMap CommandMap { get; } internal EndPointCollection EndPoints { get; } internal ConfigurationOptions RawConfig { get; } + + /// + /// When this multiplexer is a member of a connection group, the group resolves the effective + /// circuit-breaker (member override, else this member's own configuration, else the group default) + /// and supplies it here. This deliberately does *not* write back into : callers + /// may legitimately reuse a single across multiple connections, + /// and a group default must not leak into an unrelated one. + /// + internal Availability.CircuitBreaker? GroupCircuitBreaker { get; private set; } + + /// + /// The circuit-breaker that physical connections for this multiplexer should use, if any. + /// + internal Availability.CircuitBreaker? EffectiveCircuitBreaker => GroupCircuitBreaker ?? RawConfig.CircuitBreaker; + internal ServerSelectionStrategy ServerSelectionStrategy { get; } ServerSelectionStrategy IInternalConnectionMultiplexer.ServerSelectionStrategy => ServerSelectionStrategy; ConnectionMultiplexer IInternalConnectionMultiplexer.UnderlyingMultiplexer => this; @@ -70,7 +85,7 @@ pulse is null unchecked(Environment.TickCount - Volatile.Read(ref lastGlobalHeartbeatTicks)) / 1000; /// - [Obsolete($"Please use {nameof(ConfigurationOptions)}.{nameof(ConfigurationOptions.IncludeDetailInExceptions)} instead - this will be removed in 3.0.")] + [Obsolete($"Please use {nameof(ConfigurationOptions)}.{nameof(ConfigurationOptions.IncludeDetailInExceptions)} instead - this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] public bool IncludeDetailInExceptions { @@ -79,7 +94,7 @@ public bool IncludeDetailInExceptions } /// - [Obsolete($"Please use {nameof(ConfigurationOptions)}.{nameof(ConfigurationOptions.IncludePerformanceCountersInExceptions)} instead - this will be removed in 3.0.")] + [Obsolete($"Please use {nameof(ConfigurationOptions)}.{nameof(ConfigurationOptions.IncludePerformanceCountersInExceptions)} instead - this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] public bool IncludePerformanceCountersInExceptions { @@ -125,10 +140,11 @@ static ConnectionMultiplexer() SetAutodetectFeatureFlags(); } - private ConnectionMultiplexer(ConfigurationOptions configuration, ServerType? serverType = null, EndPointCollection? endpoints = null) + private ConnectionMultiplexer(ConfigurationOptions configuration, ServerType? serverType = null, EndPointCollection? endpoints = null, Availability.CircuitBreaker? groupCircuitBreaker = null) { Interlocked.Increment(ref s_MuxerCreateCount); + GroupCircuitBreaker = groupCircuitBreaker; RawConfig = configuration ?? throw new ArgumentNullException(nameof(configuration)); EndPoints = endpoints ?? RawConfig.EndPoints.Clone(); EndPoints.SetDefaultPorts(serverType, ssl: RawConfig.Ssl); @@ -156,9 +172,9 @@ private ConnectionMultiplexer(ConfigurationOptions configuration, ServerType? se lastHeartbeatTicks = Environment.TickCount; } - private static ConnectionMultiplexer CreateMultiplexer(ConfigurationOptions configuration, ILogger? log, ServerType? serverType, out EventHandler? connectHandler, EndPointCollection? endpoints = null) + private static ConnectionMultiplexer CreateMultiplexer(ConfigurationOptions configuration, ILogger? log, ServerType? serverType, out EventHandler? connectHandler, EndPointCollection? endpoints = null, Availability.CircuitBreaker? groupCircuitBreaker = null) { - var muxer = new ConnectionMultiplexer(configuration, serverType, endpoints); + var muxer = new ConnectionMultiplexer(configuration, serverType, endpoints, groupCircuitBreaker); connectHandler = null; if (log is not null) { @@ -575,7 +591,35 @@ public static Task ConnectAsync(ConfigurationOptions conf : ConnectImplAsync(configuration, log); } - private static async Task ConnectImplAsync(ConfigurationOptions configuration, TextWriter? writer = null, ServerType? serverType = null) + /// + /// Connect a multiplexer that is a member of a connection group, applying the group's resolved + /// circuit-breaker without writing it back into the caller's + /// (which the caller may legitimately reuse for other connections). + /// + internal static Task ConnectGroupMemberAsync(ConfigurationOptions configuration, TextWriter? log, Availability.CircuitBreaker? groupCircuitBreaker) + { + Dependencies.Assert(); + Validate(configuration); + + if (configuration.IsSentinel) + { + // the sentinel path builds the primary connection internally, so we cannot pass the breaker + // down into construction; apply it afterwards - it is picked up by subsequent physical + // connections, and an explicit ConfigurationOptions.CircuitBreaker still applies throughout + return ApplyAfterConnectAsync(SentinelPrimaryConnectAsync(configuration, log), groupCircuitBreaker); + } + + return ConnectImplAsync(configuration, log, groupCircuitBreaker: groupCircuitBreaker); + + static async Task ApplyAfterConnectAsync(Task pending, Availability.CircuitBreaker? groupCircuitBreaker) + { + var muxer = await pending.ForAwait(); + muxer.GroupCircuitBreaker = groupCircuitBreaker; + return muxer; + } + } + + private static async Task ConnectImplAsync(ConfigurationOptions configuration, TextWriter? writer = null, ServerType? serverType = null, Availability.CircuitBreaker? groupCircuitBreaker = null) { IDisposable? killMe = null; EventHandler? connectHandler = null; @@ -587,7 +631,7 @@ private static async Task ConnectImplAsync(ConfigurationO var sw = ValueStopwatch.StartNew(); log?.LogInformationConnectingAsync(RuntimeInformation.FrameworkDescription, Utils.GetLibVersion()); - muxer = CreateMultiplexer(configuration, log, serverType, out connectHandler); + muxer = CreateMultiplexer(configuration, log, serverType, out connectHandler, groupCircuitBreaker: groupCircuitBreaker); killMe = muxer; Interlocked.Increment(ref muxer._connectAttemptCount); bool configured = await muxer.ReconfigureAsync(first: true, reconfigureAll: false, log, null, "connect").ObserveErrors().ForAwait(); @@ -1036,7 +1080,7 @@ public void UnRoot(int token) } } - private void OnHeartbeat() + internal void OnHeartbeat() { try { @@ -1129,7 +1173,7 @@ public IDatabase GetDatabase(int db = -1, object? asyncState = null) } // DB zero is stored separately, since 0-only is a massively common use-case - private const int MaxCachedDatabaseInstance = 16; // 17 items - [0,16] + internal const int MaxCachedDatabaseInstance = 16; // 17 items - [0,16] // Side note: "databases 16" is the default in redis.conf; happy to store one extra to get nice alignment etc private IDatabase? dbCacheZero; private IDatabase[]? dbCacheLow; @@ -1282,6 +1326,8 @@ public long OperationCount } } + internal uint LatencyTicks { get; private set; } = uint.MaxValue; + // note that the RedisChannel->byte[] converter is always direct, so this is not an alloc // (we deal with channels far less frequently, so pay the encoding cost up-front) internal byte[] ChannelPrefix => ((byte[]?)RawConfig.ChannelPrefix) ?? []; @@ -1360,9 +1406,9 @@ internal void GetStatus(ILogger? log) log.LogInformationServerSummary(server.Summary(), server.GetCounters(), server.GetProfile()); } log.LogInformationTimeoutsSummary( - Interlocked.Read(ref syncTimeouts), - Interlocked.Read(ref asyncTimeouts), - Interlocked.Read(ref fireAndForgets), + Volatile.Read(ref syncTimeouts), + Volatile.Read(ref asyncTimeouts), + Volatile.Read(ref fireAndForgets), LastHeartbeatSecondsAgo); } @@ -2034,7 +2080,7 @@ private WriteResult TryPushMessageToBridgeSync(Message message, ResultProcess WriteResult.Success => throw new ArgumentOutOfRangeException(nameof(result), "Be sure to check result isn't successful before calling GetException."), WriteResult.NoConnectionAvailable => ExceptionFactory.NoConnectionAvailable(this, message, server), WriteResult.TimeoutBeforeWrite => ExceptionFactory.Timeout(this, null, message, server, result, bridge), - _ => ExceptionFactory.ConnectionFailure(RawConfig.IncludeDetailInExceptions, ConnectionFailureType.ProtocolFailure, "An unknown error occurred when writing the message", server), + _ => ExceptionFactory.ConnectionFailure(RawConfig.IncludeDetailInExceptions, ConnectionFailureType.ProtocolFailure, message.Flags, "An unknown error occurred when writing the message", server), }; [System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA1816:Dispose methods should call SuppressFinalize", Justification = "Intentional observation")] @@ -2362,5 +2408,35 @@ private Task[] QuitAllServers() long? IInternalConnectionMultiplexer.GetConnectionId(EndPoint endpoint, ConnectionType type) => GetServerEndPoint(endpoint)?.GetBridge(type)?.ConnectionId; + + internal uint UpdateLatency() + { + // Per-server latency is captured passively during the critical handshake (see + // ServerEndPoint.SetLatency), so the values read here are kept current without us issuing + // any extra traffic. We aggregate to the *worst* (max) connected server, so a group is only + // rated as fast as its slowest endpoint. Note that uint.MaxValue doubles as the "not yet + // measured" sentinel: if no connected server has a real measurement we leave the previously + // published value untouched rather than reporting a spurious MaxValue. + var snapshot = GetServerSnapshot(); + uint max = uint.MaxValue; + foreach (var server in snapshot) + { + if (server.IsConnected) + { + var latency = server.LatencyTicks; + if (max is uint.MaxValue || latency > max) + { + max = latency; + } + } + } + + if (max != uint.MaxValue) + { + LatencyTicks = max; + } + + return LatencyTicks; + } } } diff --git a/src/StackExchange.Redis/Enums/ClientFlags.cs b/src/StackExchange.Redis/Enums/ClientFlags.cs index eb687bba6..a6b86312d 100644 --- a/src/StackExchange.Redis/Enums/ClientFlags.cs +++ b/src/StackExchange.Redis/Enums/ClientFlags.cs @@ -89,7 +89,7 @@ public enum ClientFlags : long /// /// The client is a replica in MONITOR mode. /// - [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(ReplicaMonitor) + " instead, this will be removed in 3.0.")] + [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(ReplicaMonitor) + " instead, this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] SlaveMonitor = 1, @@ -101,7 +101,7 @@ public enum ClientFlags : long /// /// The client is a normal replica server. /// - [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(Replica) + " instead, this will be removed in 3.0.")] + [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(Replica) + " instead, this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] Slave = 2, diff --git a/src/StackExchange.Redis/Enums/ClientType.cs b/src/StackExchange.Redis/Enums/ClientType.cs index c2b003d9a..0013d4c79 100644 --- a/src/StackExchange.Redis/Enums/ClientType.cs +++ b/src/StackExchange.Redis/Enums/ClientType.cs @@ -22,7 +22,7 @@ public enum ClientType /// /// Replication connections. /// - [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(Replica) + " instead, this will be removed in 3.0.")] + [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(Replica) + " instead, this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] Slave = 1, diff --git a/src/StackExchange.Redis/Enums/CommandFlags.Category.cs b/src/StackExchange.Redis/Enums/CommandFlags.Category.cs new file mode 100644 index 000000000..02abc3b8b --- /dev/null +++ b/src/StackExchange.Redis/Enums/CommandFlags.Category.cs @@ -0,0 +1,373 @@ +namespace StackExchange.Redis; + +internal static class CommandFlagsExtensions +{ + public static CommandFlags WithCategory(this CommandFlags flags, CommandFlags category) + // if the user hasn't already specified a category: use the category supplied + => ((flags & Message.MaskRetryCategory) is 0) ? flags | (category & Message.MaskRetryCategory) : flags; + + public static CommandFlags WithDefaultCategory(this CommandFlags flags, RedisCommand command) + { + if ((flags & Message.MaskRetryCategory) is 0) + { + // Get the suggested flags; note that the user might have included CommandServerSpecific, + // but we *also* suggest that below - we'll live with it, additively. + // Note also that some commands may have *conditionally* included their category based on + // rules specific to the parameters, for example SCAN 0 is not server specific, + // but SCAN 12341234 *is*. + flags |= DefaultCategory(command); + } + + return flags; + + static CommandFlags DefaultCategory(RedisCommand command) + { + // This is *not* using switch expressions very deliberately, because there are a *lot* of + // options in each; let's keep things vertical rather than horizontal. + switch (command) + { + // ========================================================================== + // CONNECTION / SESSION — node-agnostic, no keyspace side effects, safe to + // replay on a fresh connection. + // ========================================================================== + case RedisCommand.PING: + case RedisCommand.ECHO: + case RedisCommand.AUTH: + case RedisCommand.HELLO: + case RedisCommand.SELECT: + case RedisCommand.QUIT: + case RedisCommand.SUBSCRIBE: + case RedisCommand.UNSUBSCRIBE: + case RedisCommand.PSUBSCRIBE: + case RedisCommand.PUNSUBSCRIBE: + case RedisCommand.SSUBSCRIBE: + case RedisCommand.SUNSUBSCRIBE: + case RedisCommand.INFO: + case RedisCommand.TIME: + case RedisCommand.DBSIZE: + case RedisCommand.LASTSAVE: + case RedisCommand.COMMAND: + return CommandFlags.CommandRetryConnection; + + // CLIENT etc often use server-specific IDs + case RedisCommand.CLIENT: // note some can be considered admin, overridden locally + return CommandFlags.CommandRetryConnection | Message.CommandServerSpecific; + + // ========================================================================== + // READ-ONLY — no mutation, always safe to retry. + // ========================================================================== + case RedisCommand.GET: + case RedisCommand.MGET: + case RedisCommand.STRLEN: + case RedisCommand.GETRANGE: + case RedisCommand.EXISTS: + case RedisCommand.TYPE: + case RedisCommand.TTL: + case RedisCommand.PTTL: + case RedisCommand.EXPIRETIME: + case RedisCommand.PEXPIRETIME: + case RedisCommand.KEYS: + case RedisCommand.RANDOMKEY: + case RedisCommand.DUMP: + case RedisCommand.TOUCH: // technically bumps LRU/LFU state, but that's not a "real" side effect worth blocking retries over + case RedisCommand.OBJECT: + case RedisCommand.MEMORY: + case RedisCommand.SORT_RO: + case RedisCommand.SORT: // ignoring the STORE variant + case RedisCommand.LCS: + case RedisCommand.GETEX: // ignoring the TTL-mutating option variants + case RedisCommand.HGET: + case RedisCommand.HMGET: + case RedisCommand.HGETALL: + case RedisCommand.HKEYS: + case RedisCommand.HVALS: + case RedisCommand.HLEN: + case RedisCommand.HEXISTS: + case RedisCommand.HSTRLEN: + case RedisCommand.HRANDFIELD: + case RedisCommand.HPTTL: + case RedisCommand.HEXPIRETIME: + case RedisCommand.HPEXPIRETIME: + case RedisCommand.HGETEX: // ignoring TTL-mutating option variants + case RedisCommand.LLEN: + case RedisCommand.LRANGE: + case RedisCommand.LINDEX: + case RedisCommand.LPOS: + case RedisCommand.SMEMBERS: + case RedisCommand.SCARD: + case RedisCommand.SISMEMBER: + case RedisCommand.SMISMEMBER: + case RedisCommand.SRANDMEMBER: + case RedisCommand.SDIFF: + case RedisCommand.SINTER: + case RedisCommand.SINTERCARD: + case RedisCommand.SUNION: + case RedisCommand.ZCARD: + case RedisCommand.ZSCORE: + case RedisCommand.ZMSCORE: + case RedisCommand.ZRANK: + case RedisCommand.ZREVRANK: + case RedisCommand.ZCOUNT: + case RedisCommand.ZLEXCOUNT: + case RedisCommand.ZRANGE: + case RedisCommand.ZREVRANGE: + case RedisCommand.ZRANGEBYSCORE: + case RedisCommand.ZREVRANGEBYSCORE: + case RedisCommand.ZRANGEBYLEX: + case RedisCommand.ZREVRANGEBYLEX: + case RedisCommand.ZRANDMEMBER: + case RedisCommand.ZDIFF: + case RedisCommand.ZINTER: + case RedisCommand.ZUNION: + case RedisCommand.ZINTERCARD: + case RedisCommand.BITCOUNT: + case RedisCommand.BITPOS: + case RedisCommand.GETBIT: + case RedisCommand.PFCOUNT: + case RedisCommand.GEOPOS: + case RedisCommand.GEODIST: + case RedisCommand.GEOHASH: + case RedisCommand.GEOSEARCH: + case RedisCommand.XLEN: + case RedisCommand.XRANGE: + case RedisCommand.XREVRANGE: + case RedisCommand.XREAD: // group-less read only; XREADGROUP is handled separately below + case RedisCommand.XPENDING: + case RedisCommand.XINFO: + case RedisCommand.EVAL_RO: + case RedisCommand.EVALSHA_RO: + return CommandFlags.CommandRetryReadOnly; + + // PUBSUB/SCAN/etc are *basically* read-only, but make limited sense between nodes + case RedisCommand.PUBSUB: + case RedisCommand.SCAN: + case RedisCommand.ZSCAN: + case RedisCommand.SSCAN: + case RedisCommand.HSCAN: + return CommandFlags.CommandRetryReadOnly | Message.CommandServerSpecific; + + // ========================================================================== + // WRITE - CHECKED — inherently conditional/idempotent; a retry either + // no-ops or fails in a way that leaves the end-state identical. + // ========================================================================== + case RedisCommand.SETNX: + case RedisCommand.MSETNX: + case RedisCommand.HSETNX: + case RedisCommand.DEL: + case RedisCommand.UNLINK: + case RedisCommand.PERSIST: + case RedisCommand.RENAMENX: + case RedisCommand.COPY: // ignoring REPLACE — default behavior fails if dest exists + case RedisCommand.MOVE: + case RedisCommand.RESTORE: // ignoring REPLACE + case RedisCommand.GETDEL: + case RedisCommand.HGETDEL: + case RedisCommand.HPERSIST: + case RedisCommand.LTRIM: + case RedisCommand.SADD: + case RedisCommand.SREM: + case RedisCommand.SMOVE: + case RedisCommand.ZREM: + case RedisCommand.ZREMRANGEBYRANK: + case RedisCommand.ZREMRANGEBYSCORE: + case RedisCommand.ZREMRANGEBYLEX: + case RedisCommand.HDEL: + case RedisCommand.PFADD: + case RedisCommand.XDEL: + case RedisCommand.XTRIM: + case RedisCommand.XACK: + case RedisCommand.XGROUP: + case RedisCommand.XNACK: + return CommandFlags.CommandRetryWriteChecked; + + // ========================================================================== + // WRITE - LAST WINS — unconditional overwrite of a specific value/state; + // repeating with the same args always converges to the same final value. + // ========================================================================== + case RedisCommand.SET: + case RedisCommand.GETSET: + case RedisCommand.MSET: + case RedisCommand.SETEX: + case RedisCommand.PSETEX: + case RedisCommand.SETRANGE: + case RedisCommand.SETBIT: + case RedisCommand.BITOP: + case RedisCommand.RENAME: + case RedisCommand.EXPIRE: + case RedisCommand.PEXPIRE: + case RedisCommand.EXPIREAT: + case RedisCommand.PEXPIREAT: + case RedisCommand.HSET: + case RedisCommand.HMSET: + case RedisCommand.HEXPIRE: + case RedisCommand.HPEXPIRE: + case RedisCommand.HEXPIREAT: + case RedisCommand.HPEXPIREAT: + case RedisCommand.LSET: + case RedisCommand.ZADD: // ignoring INCR option + case RedisCommand.ZRANGESTORE: + case RedisCommand.ZUNIONSTORE: + case RedisCommand.ZINTERSTORE: + case RedisCommand.ZDIFFSTORE: + case RedisCommand.SDIFFSTORE: + case RedisCommand.SINTERSTORE: + case RedisCommand.SUNIONSTORE: + case RedisCommand.GEOADD: + case RedisCommand.GEOSEARCHSTORE: + case RedisCommand.GEORADIUS: // because of store scenarios + case RedisCommand.GEORADIUSBYMEMBER: + case RedisCommand.XCLAIM: + case RedisCommand.XAUTOCLAIM: + return CommandFlags.CommandRetryWriteLastWins; + + // ========================================================================== + // WRITE - ACCUMULATING — effect compounds with every additional call. + // ========================================================================== + case RedisCommand.INCR: + case RedisCommand.DECR: + case RedisCommand.INCRBY: + case RedisCommand.DECRBY: + case RedisCommand.INCRBYFLOAT: + case RedisCommand.APPEND: + case RedisCommand.HINCRBY: + case RedisCommand.HINCRBYFLOAT: + case RedisCommand.ZINCRBY: + case RedisCommand.LPUSH: + case RedisCommand.RPUSH: + case RedisCommand.LPUSHX: + case RedisCommand.RPUSHX: + case RedisCommand.LINSERT: + case RedisCommand.LREM: + case RedisCommand.LPOP: + case RedisCommand.RPOP: + case RedisCommand.RPOPLPUSH: + case RedisCommand.LMOVE: + case RedisCommand.LMPOP: + case RedisCommand.SPOP: + case RedisCommand.ZPOPMIN: + case RedisCommand.ZPOPMAX: + case RedisCommand.ZMPOP: + case RedisCommand.XADD: + case RedisCommand.PFMERGE: // destination accumulates union each call + return CommandFlags.CommandRetryWriteAccumulating; + + // ========================================================================== + // SERVER ADMIN / NODE-SPECIFIC + // ========================================================================== + case RedisCommand.REPLICAOF: + case RedisCommand.SLAVEOF: + case RedisCommand.BGSAVE: + case RedisCommand.BGREWRITEAOF: + case RedisCommand.SAVE: + case RedisCommand.SHUTDOWN: + case RedisCommand.FLUSHALL: + case RedisCommand.FLUSHDB: + case RedisCommand.SWAPDB: + case RedisCommand.MIGRATE: + case RedisCommand.DEBUG: + case RedisCommand.MONITOR: + case RedisCommand.CONFIG: // note CONFIG GET con be considered more safe + case RedisCommand.SLOWLOG: + case RedisCommand.LATENCY: + case RedisCommand.SCRIPT: + case RedisCommand.CLUSTER: // note: some like MYID can be considered more safe + return CommandFlags.CommandRetryServerAdmin | Message.CommandServerSpecific; + + // ========================================================================== + // NEVER — transactions, arbitrary scripts, and blocking/destructive or + // fire-and-forget commands where a lost ack makes blind retry dangerous. + // ========================================================================== + case RedisCommand.MULTI: + case RedisCommand.EXEC: + case RedisCommand.DISCARD: + case RedisCommand.WATCH: + case RedisCommand.UNWATCH: + case RedisCommand.PUBLISH: + case RedisCommand.SPUBLISH: + case RedisCommand.XREADGROUP: + case RedisCommand.BLPOP: + case RedisCommand.BRPOP: + case RedisCommand.BRPOPLPUSH: + return CommandFlags.CommandRetryNever; + + // ========================================================================== + // scripts / modules / functions; we're going to assume nothing too weird, + // at worst similar to INCR; but it is *hoped* that callers will supply hints. + // ========================================================================== + case RedisCommand.EVAL: + case RedisCommand.EVALSHA: + return CommandFlags.CommandRetryWriteAccumulating; + + // ---- CONNECTION / SESSION ---- + case RedisCommand.ASKING: // cluster ASK redirection flag - per-connection state + case RedisCommand.READONLY: // cluster client read-routing flag - per-connection state + case RedisCommand.READWRITE: // cluster client read-routing flag - per-connection state + case RedisCommand.ROLE: // replication role report, like INFO - no keyspace effect + return CommandFlags.CommandRetryConnection; + + // ---- READ-ONLY ---- + case RedisCommand.ARCOUNT: + case RedisCommand.ARINFO: // structure metadata, like INFO + case RedisCommand.ARGET: + case RedisCommand.ARGETRANGE: + case RedisCommand.ARGREP: + case RedisCommand.ARLASTITEMS: + case RedisCommand.ARLEN: + case RedisCommand.ARMGET: + case RedisCommand.ARSCAN: + case RedisCommand.AROP: + case RedisCommand.DIGEST: // computes a hash of existing data, doesn't mutate it + case RedisCommand.VCARD: + case RedisCommand.VDIM: + case RedisCommand.VEMB: + case RedisCommand.VGETATTR: + case RedisCommand.VINFO: + case RedisCommand.VISMEMBER: + case RedisCommand.VLINKS: + case RedisCommand.VRANDMEMBER: + case RedisCommand.VRANGE: + case RedisCommand.VSIM: + return CommandFlags.CommandRetryReadOnly; + + // ---- WRITE - CHECKED ---- + case RedisCommand.DELEX: // conditional/expiry-aware delete - converges same as DEL + case RedisCommand.VADD: // idempotent add-member, like SADD + case RedisCommand.VREM: // idempotent remove-member, like SREM/ZREM + case RedisCommand.XACKDEL: // ack+delete, converges like XACK/XDEL combined + case RedisCommand.XDELEX: + return CommandFlags.CommandRetryWriteChecked; + + // ---- WRITE - LAST WINS ---- + case RedisCommand.ARDEL: + case RedisCommand.ARDELRANGE: + case RedisCommand.ARMSET: + case RedisCommand.ARSEEK: // repositions a cursor to an explicit point - overwrite semantics + case RedisCommand.ARSET: + case RedisCommand.HSETEX: // HSET + expiry, unconditional overwrite + case RedisCommand.MSETEX: + case RedisCommand.VSETATTR: // unconditional attribute overwrite + case RedisCommand.XCFGSET: + return CommandFlags.CommandRetryWriteLastWins; + + // ---- WRITE - ACCUMULATING ---- + case RedisCommand.ARRING: // presumed create/configure-ring, unconditional define + case RedisCommand.ARINSERT: // ring-buffer insert, compounds like a push + case RedisCommand.ARNEXT: // advances a cursor/position with each call + case RedisCommand.INCREX: // counter semantics + expiry, still accumulating + return CommandFlags.CommandRetryWriteAccumulating; + + // ---- SERVER ADMIN / NODE-SPECIFIC ---- + case RedisCommand.HOTKEYS: // diagnostic/introspection, node-local + case RedisCommand.SENTINEL: + case RedisCommand.SYNC: // replication stream handshake + return CommandFlags.CommandRetryServerAdmin | Message.CommandServerSpecific; + + // if we don't recognize it: default to the most pessimistic + case RedisCommand.NONE: + case RedisCommand.UNKNOWN: + default: + return CommandFlags.CommandRetryNever; + } + } + } +} diff --git a/src/StackExchange.Redis/Enums/CommandFlags.cs b/src/StackExchange.Redis/Enums/CommandFlags.cs index ff69fb750..b1b99b672 100644 --- a/src/StackExchange.Redis/Enums/CommandFlags.cs +++ b/src/StackExchange.Redis/Enums/CommandFlags.cs @@ -1,11 +1,27 @@ using System; using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using RESPite; namespace StackExchange.Redis { /// /// Behaviour markers associated with a given command. /// + /* + WARNING: the *declaration order* of the members below is load-bearing, and is not the obvious/natural order. + + This enum has duplicate values (the obsolete "slave" aliases, and PreferMaster which is zero like None) and + multi-bit values (PreferReplica/DemandReplica occupy a 2-bit region), both of which [Flags] formatting handles + badly. Which of two same-valued names ToString() picks depends on where they land in the *sorted* name/value + arrays that the runtime builds, and that sort is unstable - so it is a function of declaration order, member + count, and runtime. The order here was determined empirically to give sane output on both .NET Framework and + modern .NET (see FormatTests.CommandFlagsFormatting for the expectations). + + Consequence: adding or removing members can silently perturb ToString() for *existing* values. If + CommandFlagsFormatting starts failing after such a change, juggling the declaration order (in particular the + positions of the obsolete PreferSlave/DemandSlave aliases) is the fix, not a change to the values. + */ [Flags] [System.Diagnostics.CodeAnalysis.SuppressMessage("Design", "CA1069:Enums values should not be duplicated", Justification = "Compatibility")] public enum CommandFlags @@ -18,7 +34,7 @@ public enum CommandFlags /// /// From 2.0, this flag is not used. /// - [Obsolete("From 2.0, this flag is not used, this will be removed in 3.0.", false)] + [Obsolete("From 2.0, this flag is not used, this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] HighPriority = 1, @@ -28,65 +44,45 @@ public enum CommandFlags /// FireAndForget = 2, - /// - /// This operation should be performed on the primary if it is available, but read operations may - /// be performed on a replica if no primary is available. This is the default option. - /// - PreferMaster = 0, + // note: the two obsolete aliases below are declared *before* their replacements, deliberately; see the + // warning above the enum - this is what makes ToString() prefer the "replica" names over the legacy ones -#if NET8_0_OR_GREATER /// /// This operation should be performed on the replica if it is available, but will be performed on /// a primary if no replicas are available. Suitable for read operations only. /// - [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(PreferReplica) + " instead, this will be removed in 3.0.")] + [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(PreferReplica) + " instead, this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] PreferSlave = 8, -#endif /// - /// This operation should only be performed on the primary. - /// - DemandMaster = 4, - -#if !NET8_0_OR_GREATER - /// - /// This operation should be performed on the replica if it is available, but will be performed on - /// a primary if no replicas are available. Suitable for read operations only. + /// This operation should only be performed on a replica. Suitable for read operations only. /// - [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(PreferReplica) + " instead, this will be removed in 3.0.")] + [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(DemandReplica) + " instead, this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] - PreferSlave = 8, -#endif + DemandSlave = 12, /// - /// This operation should be performed on the replica if it is available, but will be performed on - /// a primary if no replicas are available. Suitable for read operations only. + /// This operation should be performed on the primary if it is available, but read operations may + /// be performed on a replica if no primary is available. This is the default option. /// - PreferReplica = 8, // note: we're using a 2-bit set here, which [Flags] formatting hates; position is doing the best we can for reasonable outcomes here + PreferMaster = 0, -#if NET8_0_OR_GREATER /// - /// This operation should only be performed on a replica. Suitable for read operations only. + /// This operation should only be performed on the primary. /// - [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(DemandReplica) + " instead, this will be removed in 3.0.")] - [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] - DemandSlave = 12, -#endif + DemandMaster = 4, /// - /// This operation should only be performed on a replica. Suitable for read operations only. + /// This operation should be performed on the replica if it is available, but will be performed on + /// a primary if no replicas are available. Suitable for read operations only. /// - DemandReplica = 12, // note: we're using a 2-bit set here, which [Flags] formatting hates; position is doing the best we can for reasonable outcomes here + PreferReplica = 8, // note: we're using a 2-bit set here, which [Flags] formatting hates; position is doing the best we can for reasonable outcomes here -#if !NET8_0_OR_GREATER /// /// This operation should only be performed on a replica. Suitable for read operations only. /// - [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(DemandReplica) + " instead, this will be removed in 3.0.")] - [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] - DemandSlave = 12, -#endif + DemandReplica = 12, // note: we're using a 2-bit set here, which [Flags] formatting hates; position is doing the best we can for reasonable outcomes here // 16: reserved for additional "demand/prefer" options @@ -111,5 +107,79 @@ public enum CommandFlags // 2048: Use subscription connection type; never user-specified, so not visible on the public API // 4096: Identifies handshake completion messages; never user-specified, so not visible on the public API + + /* + Command side-effect/retry logic: + The values below reserve a chunk of bits (bits 13-17, i.e. << 13) for command + retry logic/categorization; by default, nothing in this chunk will be passed and + the library will supply a value based on the command being issued. The caller can + override, though - ultimately it is their data/server! They will need to supply a + suitable value for Execute[Async] etc, otherwise we'll assume the worst. + + The math here is intended so that we can specify a numeric "max" that we'll + retry, and can filter with <=, so the higher numbers have more side-effects. + As such, this region is not flags per-se, and we are intentionally leaving gaps + to slide other things in later. Note that 0 is the implicit "not specified" value + (distinct from CommandRetryAlways), and must be resolved to a concrete category + downstream before any <= comparison. CommandServerSpecific (bit 18) is a genuine + single-bit flag, orthogonal to the severity ladder. + */ + + /// + /// The command is always safe to retry, regardless of connection or server state. + /// + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + CommandRetryAlways = 1 << 13, // pre-shift value 1 + + /// + /// The command relates to the connection or to safe metadata (for example CLIENT SETNAME, + /// COMMAND, or CONFIG GET) and can be retried at the connection level. + /// + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + CommandRetryConnection = 4 << 13, // pre-shift value 4 + + /// + /// The command only reads data (for example GET) and can be safely retried. + /// + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + CommandRetryReadOnly = 8 << 13, // pre-shift value 8 + + /// + /// The command writes data conditionally (for example SETNX or SET ... IFEQ), so a retry + /// is checked against server state. + /// + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + CommandRetryWriteChecked = 12 << 13, // pre-shift value 12 + + /// + /// The command writes data such that a retry simply overwrites (last-writer-wins, for example SET). + /// + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + CommandRetryWriteLastWins = 16 << 13, // pre-shift value 16 + + /// + /// The command writes data cumulatively (for example INCR or LPOP), so a retry can + /// double-apply and change the result. + /// + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + CommandRetryWriteAccumulating = 20 << 13, // pre-shift value 20 + + /// + /// The command performs server administration (for example REPLICAOF or CONFIG SET); these + /// are commonly also endpoint-specific (the internal server-specific flag). + /// + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + CommandRetryServerAdmin = 24 << 13, // pre-shift value 24 + + /// + /// The command should never be retried. + /// + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + CommandRetryNever = 31 << 13, // pre-shift value 31 (the full retry-category region) + + // 262144 (bit 18): "server specific" - the command is tied to a specific endpoint and must never be + // retried on a different endpoint (for example cursor-based operations); orthogonal to the retry-category + // region. Internal-only (see Message.CommandServerSpecific): the wrapper database cannot yet express + // endpoint-stickiness over a *range* of operations, so this is not (currently) on the public API. } } diff --git a/src/StackExchange.Redis/Enums/ConnectionFailureType.cs b/src/StackExchange.Redis/Enums/ConnectionFailureType.cs index 55eeacef6..9f0df5ba0 100644 --- a/src/StackExchange.Redis/Enums/ConnectionFailureType.cs +++ b/src/StackExchange.Redis/Enums/ConnectionFailureType.cs @@ -1,4 +1,7 @@ -namespace StackExchange.Redis +using System.Diagnostics.CodeAnalysis; +using RESPite; + +namespace StackExchange.Redis { /// /// The known types of connection failure. @@ -59,5 +62,11 @@ public enum ConnectionFailureType /// High-integrity mode was enabled, and a failure was detected. /// ResponseIntegrityFailure, + + /// + /// The associated with this connection detected instability. + /// + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + CircuitBreaker, } } diff --git a/src/StackExchange.Redis/Enums/RedisCommand.cs b/src/StackExchange.Redis/Enums/RedisCommand.cs index dadf01033..55769105f 100644 --- a/src/StackExchange.Redis/Enums/RedisCommand.cs +++ b/src/StackExchange.Redis/Enums/RedisCommand.cs @@ -317,6 +317,9 @@ internal static partial class RedisCommandMetadata [AsciiHash(CaseSensitive = false)] public static partial bool TryParseCI(ReadOnlySpan command, out RedisCommand value); + + [AsciiHash] + public static partial bool TryFormat(RedisCommand command, out string format); } // ReSharper restore InconsistentNaming diff --git a/src/StackExchange.Redis/Enums/ReplicationChangeOptions.cs b/src/StackExchange.Redis/Enums/ReplicationChangeOptions.cs index 897ebbb6c..785ac722f 100644 --- a/src/StackExchange.Redis/Enums/ReplicationChangeOptions.cs +++ b/src/StackExchange.Redis/Enums/ReplicationChangeOptions.cs @@ -28,7 +28,7 @@ public enum ReplicationChangeOptions /// /// Issue a REPLICAOF to all other known nodes, making this primary of all. /// - [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(ReplicateToOtherEndpoints) + " instead, this will be removed in 3.0.")] + [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(ReplicateToOtherEndpoints) + " instead, this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] EnslaveSubordinates = 4, diff --git a/src/StackExchange.Redis/ExceptionFactory.cs b/src/StackExchange.Redis/ExceptionFactory.cs index 380ac0c0f..0f6ab3cb5 100644 --- a/src/StackExchange.Redis/ExceptionFactory.cs +++ b/src/StackExchange.Redis/ExceptionFactory.cs @@ -1,4 +1,5 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Security.Authentication; using System.Text; @@ -33,9 +34,9 @@ internal static Exception TooManyArgs(string command, int argCount) internal static Exception CommandHasWhitespace(string command) => new RedisCommandException($"The command '{command}' contains whitespace and would be sent as a single unknown token; pass each word as a separate argument, for example Execute(\"ACL\", \"SETUSER\", \"x\") rather than Execute(\"ACL SETUSER x\")."); - internal static Exception ConnectionFailure(bool includeDetail, ConnectionFailureType failureType, string message, ServerEndPoint? server) + internal static Exception ConnectionFailure(bool includeDetail, ConnectionFailureType failureType, CommandFlags flags, string message, ServerEndPoint? server) { - var ex = new RedisConnectionException(failureType, message); + var ex = new RedisConnectionException(failureType, flags, message); if (includeDetail) AddExceptionDetail(ex, null, server, null); return ex; } @@ -154,7 +155,7 @@ internal static Exception NoConnectionAvailable( data = new List>(); AddCommonDetail(data, sb, message, multiplexer, server); } - var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, sb.ToString(), innerException, message?.Status ?? CommandStatus.Unknown); + var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, message?.Flags ?? CommandFlags.None, sb.ToString(), innerException, message?.Status ?? CommandStatus.Unknown); if (multiplexer.RawConfig.IncludeDetailInExceptions) { CopyDataToException(data, ex); @@ -164,6 +165,37 @@ internal static Exception NoConnectionAvailable( return ex; } + /// + /// A connection failure produces a single exception describing the *connection*, which is then handed + /// to every message that was in flight. Sharing one instance across many unrelated callers is dubious + /// in itself ( is mutable, so each caller sees the others' additions), and + /// it discards the per-message detail the retry machinery needs: the command's retry category, and + /// whether this particular message had actually been written. Give each message its own. + /// + internal static Exception PerMessage(Exception shared, Message message) + { + // only connection failures get shared like this; anything else already describes one operation + if (shared is not RedisConnectionException conn + || (conn.Flags == message.Flags && conn.CommandStatus == message.Status)) + { + return shared; + } + + var ex = new RedisConnectionException( + conn.FailureType, + message.Flags, + conn.Message, + conn.InnerException, + message.Status); + foreach (DictionaryEntry entry in conn.Data) + { + ex.Data[entry.Key] = entry.Value; + } + ex.Data[DataSentStatusKey] = message.Status; // ...and correct this one for *this* message + if (conn.HelpLink is not null) ex.HelpLink = conn.HelpLink; + return ex; + } + internal static Exception? PopulateInnerExceptions(ReadOnlySpan serverSnapshot) { var innerExceptions = new List(); @@ -283,12 +315,13 @@ internal static Exception Timeout(ConnectionMultiplexer multiplexer, string? bas // If we're from a backlog timeout scenario, we log a more intuitive connection exception for the timeout...because the timeout was a symptom // and we have a more direct cause: we had no connection to send it on. + var msgFlags = message?.Flags ?? CommandFlags.CommandRetryNever; Exception ex = logConnectionException && lastConnectionException is not null - ? new RedisConnectionException(lastConnectionException.FailureType, sb.ToString(), lastConnectionException, message?.Status ?? CommandStatus.Unknown) + ? new RedisConnectionException(lastConnectionException.FailureType, msgFlags, sb.ToString(), lastConnectionException, message?.Status ?? CommandStatus.Unknown) { HelpLink = TimeoutHelpLink, } - : new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown) + : new RedisTimeoutException(msgFlags, sb.ToString(), message?.Status ?? CommandStatus.Unknown) { HelpLink = TimeoutHelpLink, }; @@ -448,7 +481,7 @@ internal static Exception UnableToConnect(ConnectionMultiplexer muxer, string? f sb.Append(' ').Append(failureMessage.Trim()); } - return new RedisConnectionException(failureType, sb.ToString(), inner); + return new RedisConnectionException(failureType, CommandFlags.None, sb.ToString(), inner); } } } diff --git a/src/StackExchange.Redis/Exceptions.cs b/src/StackExchange.Redis/Exceptions.cs index 1f1c973ce..405a0ea4e 100644 --- a/src/StackExchange.Redis/Exceptions.cs +++ b/src/StackExchange.Redis/Exceptions.cs @@ -25,7 +25,6 @@ public RedisCommandException(string message, Exception innerException) : base(me #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] #endif private RedisCommandException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { } } @@ -41,8 +40,19 @@ public sealed partial class RedisTimeoutException : TimeoutException /// /// The message for the exception. /// The command status, as of when the timeout happened. - public RedisTimeoutException(string message, CommandStatus commandStatus) : base(message) + [Obsolete("Prefer the overload that specifies CommandFlags")] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] + public RedisTimeoutException(string message, CommandStatus commandStatus) : this(CommandFlags.CommandRetryNever, message, commandStatus) { } + + /// + /// Creates a new . + /// + /// The command-flags associated with the faulting operation. + /// The message for the exception. + /// The command status, as of when the timeout happened. + public RedisTimeoutException(CommandFlags flags, string message, CommandStatus commandStatus) : base(message) { + Flags = flags; Commandstatus = commandStatus; } @@ -51,9 +61,13 @@ public RedisTimeoutException(string message, CommandStatus commandStatus) : base /// public CommandStatus Commandstatus { get; } + /// + /// The command-flags associated with the faulting operation (including its retry category). + /// + public CommandFlags Flags { get; } + #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] #endif private RedisTimeoutException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { @@ -67,7 +81,7 @@ private RedisTimeoutException(SerializationInfo info, StreamingContext ctx) : ba /// Serialization context. #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] #endif public override void GetObjectData(SerializationInfo info, StreamingContext context) { @@ -87,7 +101,9 @@ public sealed partial class RedisConnectionException : RedisException /// /// The type of connection failure. /// The message for the exception. - public RedisConnectionException(ConnectionFailureType failureType, string message) : this(failureType, message, null, CommandStatus.Unknown) { } + [Obsolete("Prefer the overload that specifies CommandFlags")] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] + public RedisConnectionException(ConnectionFailureType failureType, string message) : this(failureType, CommandFlags.CommandRetryNever, message, null, CommandStatus.Unknown) { } /// /// Creates a new . @@ -95,18 +111,33 @@ public RedisConnectionException(ConnectionFailureType failureType, string messag /// The type of connection failure. /// The message for the exception. /// The inner exception. - public RedisConnectionException(ConnectionFailureType failureType, string message, Exception? innerException) : this(failureType, message, innerException, CommandStatus.Unknown) { } + [Obsolete("Prefer the overload that specifies CommandFlags")] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] + public RedisConnectionException(ConnectionFailureType failureType, string message, Exception? innerException) : this(failureType, CommandFlags.CommandRetryNever, message, innerException, CommandStatus.Unknown) { } + + /// + /// Creates a new . + /// + /// The type of connection failure. + /// The message for the exception. + /// The inner exception. + /// The status of the command. + [Obsolete("Prefer the overload that specifies CommandFlags")] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] + public RedisConnectionException(ConnectionFailureType failureType, string message, Exception? innerException, CommandStatus commandStatus) : this(failureType, CommandFlags.CommandRetryNever, message, innerException, commandStatus) { } /// /// Creates a new . /// /// The type of connection failure. + /// The command-flags associated with the faulting operation. /// The message for the exception. /// The inner exception. /// The status of the command. - public RedisConnectionException(ConnectionFailureType failureType, string message, Exception? innerException, CommandStatus commandStatus) : base(message, innerException) + public RedisConnectionException(ConnectionFailureType failureType, CommandFlags flags, string message, Exception? innerException = null, CommandStatus commandStatus = CommandStatus.Unknown) : base(message, innerException) { FailureType = failureType; + Flags = flags; CommandStatus = commandStatus; } @@ -115,6 +146,11 @@ public RedisConnectionException(ConnectionFailureType failureType, string messag /// public ConnectionFailureType FailureType { get; } + /// + /// The command-flags associated with the faulting operation (including its retry category). + /// + public CommandFlags Flags { get; } + /// /// Status of the command while communicating with Redis. /// @@ -122,7 +158,6 @@ public RedisConnectionException(ConnectionFailureType failureType, string messag #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] #endif private RedisConnectionException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { @@ -137,7 +172,7 @@ private RedisConnectionException(SerializationInfo info, StreamingContext ctx) : /// Serialization context. #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] #endif public override void GetObjectData(SerializationInfo info, StreamingContext context) { @@ -173,7 +208,6 @@ public RedisException(string message, Exception? innerException) : base(message, /// Serialization context. #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] #endif protected RedisException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { } } @@ -188,12 +222,35 @@ public sealed partial class RedisServerException : RedisException /// Creates a new . /// /// The message for the exception. - public RedisServerException(string message) : base(message) { } + [Obsolete("Specify Kind and CommandFlags when possible")] + [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] + public RedisServerException(string message) : this(RedisErrorKind.Unknown, CommandFlags.CommandRetryNever, message) { } + + /// + /// Creates a new . + /// + /// The categorized meaning of the error. + /// The command-flags associated with the faulting operation. + /// The message for the exception. + public RedisServerException(RedisErrorKind kind, CommandFlags flags, string message) : base(message) + { + Kind = kind; + Flags = flags; + } #if NET8_0_OR_GREATER [Obsolete(Obsoletions.LegacyFormatterImplMessage, DiagnosticId = Obsoletions.LegacyFormatterImplDiagId)] - [EditorBrowsable(EditorBrowsableState.Never)] #endif private RedisServerException(SerializationInfo info, StreamingContext ctx) : base(info, ctx) { } + + /// + /// Identifies the kind of error received. + /// + public RedisErrorKind Kind { get; } + + /// + /// The command-flags associated with the faulting operation (including its retry category). + /// + public CommandFlags Flags { get; } } } diff --git a/src/StackExchange.Redis/Interfaces/IConnectionGroup.cs b/src/StackExchange.Redis/Interfaces/IConnectionGroup.cs new file mode 100644 index 000000000..0b0b53478 --- /dev/null +++ b/src/StackExchange.Redis/Interfaces/IConnectionGroup.cs @@ -0,0 +1,131 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.Text; +using System.Threading.Tasks; +using RESPite; + +// ReSharper disable once CheckNamespace +namespace StackExchange.Redis.Availability; + +/// +/// A group of connections to redis servers, that manages connections to multiple +/// servers, routing traffic based on the availability of the servers and their +/// relative . +/// +[Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +public interface IConnectionGroup : IConnectionMultiplexer +{ + /// + /// A change occured to one of the connection groups. + /// + event EventHandler? ConnectionChanged; + + /// + /// Adds a new member to the group. + /// + Task AddAsync(ConnectionGroupMember member, TextWriter? log = null); + + /// + /// Attempt to explicitly switch to the specified member, or remove an explicit failover if is . + /// A member specified via this method will be preferred over other members (ignoring and ) + /// until the failover is removed, or the connection is no longer reachable. + /// + /// The member to switch to, or to remove an explicit failover. + /// if the failover was successful (i.e. the member can be used), otherwise. + /// This will implicitly include . + bool TryFailoverTo(ConnectionGroupMember? member); + + /// + /// Removes a member from the group. + /// + bool Remove(ConnectionGroupMember member); + + /// + /// Get the members of the group. + /// + ReadOnlySpan GetMembers(); + + /// + /// Gets the currently active member. + /// + ConnectionGroupMember? ActiveMember { get; } + + /// + /// Gets the group-wide options this group was created with; these are immutable, and are the defaults + /// against which each 's overrides are resolved. + /// + MultiGroupOptions Options { get; } +} + +/// +/// Represents a change to a connection group. +/// +[Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +public class GroupConnectionChangedEventArgs(GroupConnectionChangedEventArgs.ChangeType type, ConnectionGroupMember group, ConnectionGroupMember? previousGroup = null) : EventArgs, ICompletable +{ + /// + /// The group relating to the change. For , this is the new group. + /// + public ConnectionGroupMember Group => group; + + /// + /// The previous group relating to the change, if applicable. + /// + public ConnectionGroupMember? PreviousGroup => previousGroup; + + /// + /// The type of change that occurred. + /// + public ChangeType Type => type; + + private EventHandler? _handler; + private object? _sender; + + /// + /// The type of change that occurred. + /// + public enum ChangeType + { + /// + /// Unused. + /// + Unknown = 0, + + /// + /// A new connection group was added. + /// + Added = 1, + + /// + /// A connection group was removed. + /// + Removed = 2, + + /// + /// A connection group became disconnected. + /// + Disconnected = 3, + + /// + /// A connection group became reconnected. + /// + Reconnected = 4, + + /// + /// The active connection group changed, changing how traffic is routed. + /// + ActiveChanged = 5, + } + + internal void CompleteAsWorker(EventHandler handler, object sender) + { + _handler = handler; + _sender = sender; + ConnectionMultiplexer.CompleteAsWorker(this); + } + + void ICompletable.AppendStormLog(StringBuilder sb) { } + + bool ICompletable.TryComplete(bool isAsync) => ConnectionMultiplexer.TryCompleteHandler(_handler, _sender!, this, isAsync); +} diff --git a/src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs b/src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs index 96b4ce8f6..617a63f25 100644 --- a/src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs +++ b/src/StackExchange.Redis/Interfaces/IConnectionMultiplexer.cs @@ -4,6 +4,7 @@ using System.IO; using System.Net; using System.Threading.Tasks; +using StackExchange.Redis.Availability; using StackExchange.Redis.Maintenance; using StackExchange.Redis.Profiling; using static StackExchange.Redis.ConnectionMultiplexer; @@ -76,7 +77,7 @@ public interface IConnectionMultiplexer : IDisposable, IAsyncDisposable /// /// Should exceptions include identifiable details? (key names, additional annotations). /// - [Obsolete($"Please use {nameof(ConfigurationOptions)}.{nameof(ConfigurationOptions.IncludeDetailInExceptions)} instead - this will be removed in 3.0.")] + [Obsolete($"Please use {nameof(ConfigurationOptions)}.{nameof(ConfigurationOptions.IncludeDetailInExceptions)} instead - this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] bool IncludeDetailInExceptions { get; set; } diff --git a/src/StackExchange.Redis/Interfaces/IDatabase.cs b/src/StackExchange.Redis/Interfaces/IDatabase.cs index 22202ebc4..038439a0b 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabase.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabase.cs @@ -16,7 +16,7 @@ public partial interface IDatabase : IRedis, IDatabaseAsync /// /// The numeric identifier of this database. /// - int Database { get; } + new int Database { get; } /// /// Allows creation of a group of operations that will be sent to the server as a single unit, @@ -32,7 +32,8 @@ public partial interface IDatabase : IRedis, IDatabaseAsync /// /// The async object state to be passed into the created . /// The created transaction. - ITransaction CreateTransaction(object? asyncState = null); + // hides IDatabaseAsync.CreateTransaction, refining the return type from ITransactionAsync to ITransaction + new ITransaction CreateTransaction(object? asyncState = null); /// /// Atomically transfer a key from a source Redis instance to a destination Redis instance. diff --git a/src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs b/src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs index a49033152..f80aa744a 100644 --- a/src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs +++ b/src/StackExchange.Redis/Interfaces/IDatabaseAsync.cs @@ -14,6 +14,19 @@ namespace StackExchange.Redis /// public partial interface IDatabaseAsync : IRedisAsync { + /// + int Database { get; } + + /// + /// Creates a transaction exposing only asynchronous completion; see . + /// + /// The async state to set on the created transaction. + /// Unlike , this offers no synchronous + /// execution, so it is usable from async-only databases such as one created via + /// . + [Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] + ITransactionAsync CreateTransaction(object? asyncState = null); + /// /// Indicates whether the instance can communicate with the server (resolved using the supplied key and optional flags). /// diff --git a/src/StackExchange.Redis/Interfaces/IInternalDatabaseAsync.cs b/src/StackExchange.Redis/Interfaces/IInternalDatabaseAsync.cs new file mode 100644 index 000000000..6aea5db57 --- /dev/null +++ b/src/StackExchange.Redis/Interfaces/IInternalDatabaseAsync.cs @@ -0,0 +1,87 @@ +using System; +using System.Threading; + +namespace StackExchange.Redis.Interfaces; + +[Flags] +internal enum DatabaseFeatureFlags +{ + None = 0, + Cluster = 1 << 0, + ConnectionGroup = 1 << 1, + Batch = 1 << 2, + Transaction = 1 << 3, + KeyPrefix = 1 << 4, + Retry = 1 << 5, + Unknown = 1 << 6, + Failover = 1 << 7, +} + +internal interface IInternalDatabaseAsync : IDatabaseAsync +{ + DatabaseFeatureFlags GetFeatures(out string name); + CancellationToken GetNextFailover(); + + /// + /// The async-state that this database stamps onto the tasks it hands out, if any; wrappers that + /// cannot preserve it (see RetryDatabase) need to know when one is present rather than + /// dropping it silently. + /// + object? AsyncState { get; } +} + +/// +/// Exposes transaction-level detail needed by the retry machinery. Implemented by the concrete +/// transaction types so a RetryTransaction can inspect the transaction it is replaying against. +/// +internal interface IInternalTransaction +{ + /// + /// The most side-effecting retry category across all queued operations (excluding WATCH + /// constraints); this describes what replaying the whole transaction would do. + /// + CommandFlags GetAggregateRetryCategory(); +} + +internal static class InternalDatabaseExtension +{ + internal static DatabaseFeatureFlags GetFeatures(this IDatabaseAsync database, out string name) + { + if (database is IInternalDatabaseAsync idb) + { + return idb.GetFeatures(out name); + } + + name = ""; + return DatabaseFeatureFlags.Unknown; + } + + internal static string BuildString(this IDatabaseAsync database) + { + var features = database.GetFeatures(out string name); + return string.IsNullOrEmpty(name) ? features.ToString() : $"{name}: {features}"; + } + + internal static DatabaseFeatureFlags RejectFlags(this IDatabaseAsync database, DatabaseFeatureFlags incompatible) + { + // note: returns *all* the features of the database provided + var features = database.GetFeatures(out _); + var overlap = features & incompatible; + if (overlap is not 0) Throw(overlap); + return features; + + static void Throw(DatabaseFeatureFlags overlap) => throw new InvalidOperationException( + $"This operation is not compatible with feature(s): {overlap}"); + } + + internal static object? GetAsyncState(this IDatabaseAsync database) + => database is IInternalDatabaseAsync ida ? ida.AsyncState : null; + + internal static CancellationToken GetNextFailover(this IDatabaseAsync database) + { + // get a CT that represents the next failover; you might be asking "shouldn't that be a Task getter?" - no, + // because Task *does* have ContinueWith, but it doesn't have any mechanism to *undo* that; conversely, + // CancellationToken is expressly designed with that intent, with Register(..) being scoped. + return database is IInternalDatabaseAsync ida ? ida.GetNextFailover() : CancellationToken.None; + } +} diff --git a/src/StackExchange.Redis/Interfaces/IServer.cs b/src/StackExchange.Redis/Interfaces/IServer.cs index 1959f15d8..986e82b8c 100644 --- a/src/StackExchange.Redis/Interfaces/IServer.cs +++ b/src/StackExchange.Redis/Interfaces/IServer.cs @@ -41,7 +41,7 @@ public partial interface IServer : IRedis /// /// Gets whether the connected server is a replica. /// - [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(IsReplica) + " instead, this will be removed in 3.0.")] + [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(IsReplica) + " instead, this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] bool IsSlave { get; } @@ -53,7 +53,7 @@ public partial interface IServer : IRedis /// /// Explicitly opt in for replica writes on writable replica. /// - [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(AllowReplicaWrites) + " instead, this will be removed in 3.0.")] + [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(AllowReplicaWrites) + " instead, this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] bool AllowSlaveWrites { get; set; } @@ -399,7 +399,7 @@ public partial interface IServer : IRedis Task LastSaveAsync(CommandFlags flags = CommandFlags.None); /// - [Obsolete("Please use " + nameof(MakePrimaryAsync) + ", this will be removed in 3.0.")] + [Obsolete("Please use " + nameof(MakePrimaryAsync) + ", this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] void MakeMaster(ReplicationChangeOptions options, TextWriter? log = null); @@ -502,17 +502,17 @@ public partial interface IServer : IRedis void Shutdown(ShutdownMode shutdownMode = ShutdownMode.Default, CommandFlags flags = CommandFlags.None); /// - [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(ReplicaOfAsync) + " instead, this will be removed in 3.0.")] + [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(ReplicaOfAsync) + " instead, this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] void SlaveOf(EndPoint master, CommandFlags flags = CommandFlags.None); /// - [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(ReplicaOfAsync) + " instead, this will be removed in 3.0.")] + [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(ReplicaOfAsync) + " instead, this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] Task SlaveOfAsync(EndPoint master, CommandFlags flags = CommandFlags.None); /// - [Obsolete("Please use " + nameof(ReplicaOfAsync) + ", this will be removed in 3.0.")] + [Obsolete("Please use " + nameof(ReplicaOfAsync) + ", this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] void ReplicaOf(EndPoint master, CommandFlags flags = CommandFlags.None); @@ -767,12 +767,12 @@ public partial interface IServer : IRedis Task[][]> SentinelMastersAsync(CommandFlags flags = CommandFlags.None); /// - [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(SentinelReplicas) + " instead, this will be removed in 3.0.")] + [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(SentinelReplicas) + " instead, this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] KeyValuePair[][] SentinelSlaves(string serviceName, CommandFlags flags = CommandFlags.None); /// - [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(SentinelReplicasAsync) + " instead, this will be removed in 3.0.")] + [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(SentinelReplicasAsync) + " instead, this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] Task[][]> SentinelSlavesAsync(string serviceName, CommandFlags flags = CommandFlags.None); @@ -810,6 +810,11 @@ public partial interface IServer : IRedis /// Task[][]> SentinelSentinelsAsync(string serviceName, CommandFlags flags = CommandFlags.None); + + /// + /// Invent a random key that will resolve to this server. + /// + RedisKey InventKey(RedisKey prefix = default); } internal static class IServerExtensions diff --git a/src/StackExchange.Redis/Interfaces/ITransaction.cs b/src/StackExchange.Redis/Interfaces/ITransaction.cs index 21c66968a..d8fdaf4c7 100644 --- a/src/StackExchange.Redis/Interfaces/ITransaction.cs +++ b/src/StackExchange.Redis/Interfaces/ITransaction.cs @@ -14,13 +14,15 @@ namespace StackExchange.Redis /// Note that on a cluster, it may be required that all keys involved in the transaction (including constraints) are in the same hash-slot. /// /// - public interface ITransaction : IBatch + public interface ITransaction : IBatch, ITransactionAsync { /// /// Adds a precondition for this transaction. /// /// The condition to add to the transaction. - ConditionResult AddCondition(Condition condition); + // re-declared (rather than inherited from ITransactionAsync) so this interface's surface is + // self-contained and unambiguous for existing callers + new ConditionResult AddCondition(Condition condition); /// /// Execute the batch operation, sending all queued commands to the server. @@ -32,6 +34,6 @@ public interface ITransaction : IBatch /// Execute the batch operation, sending all queued commands to the server. /// /// The command flags to use. - Task ExecuteAsync(CommandFlags flags = CommandFlags.None); + new Task ExecuteAsync(CommandFlags flags = CommandFlags.None); } } diff --git a/src/StackExchange.Redis/Interfaces/ITransactionAsync.cs b/src/StackExchange.Redis/Interfaces/ITransactionAsync.cs new file mode 100644 index 000000000..14548c7d7 --- /dev/null +++ b/src/StackExchange.Redis/Interfaces/ITransactionAsync.cs @@ -0,0 +1,53 @@ +using System.Threading.Tasks; + +namespace StackExchange.Redis; + +/// +/// Represents a group of operations that will be sent to the server as a single unit, +/// and processed on the server as a single unit, exposing only asynchronous completion. +/// This is the async-only counterpart to , used where synchronous +/// execution is not offered - for example, a retrying database created via +/// , where execution may inherently involve delays. +/// +/// +/// Transactions can also include constraints (implemented via WATCH). +/// +/// +public interface ITransactionAsync : IDatabaseAsync +{ + /// + /// Adds a precondition for this transaction. + /// + /// The condition to add to the transaction. + ConditionResult AddCondition(Condition condition); + + /// + /// Execute the transaction, sending all queued commands to the server. + /// + /// The command flags to use. + Task ExecuteAsync(CommandFlags flags = CommandFlags.None); + + /// + /// Whether the transaction failed to commit because the *server* rejected the EXEC: every + /// condition held, so MULTI/EXEC really was issued, but a key being watched on behalf of + /// those conditions was modified by another connection in the meantime. + /// + /// + /// A transaction that does not commit reports false from Execute for two quite + /// different reasons, and this is what tells them apart: + /// + /// A condition was not satisfied, so the transaction was abandoned without ever + /// issuing an EXEC. The value genuinely was not what was asserted, and re-running would assert + /// the same thing again. This property is false, and the offending + /// is also false. + /// A watched key was changed by a different connection between the + /// conditions being evaluated and the EXEC arriving. This property is true, every + /// is true, and re-reading and trying again is the + /// expected response - this is what the WATCH/MULTI/EXEC idiom is for. + /// + /// Either way nothing was applied, and every queued operation's task is cancelled. This can only + /// be true for a transaction that has conditions (without one there is no WATCH, so there + /// is nothing to conflict over), and is false before the transaction has been executed. + /// + bool WasWatchConflict { get; } +} diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs index 0d5afed5d..c889a2c57 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixed.cs @@ -3,11 +3,13 @@ using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Net; +using System.Threading; using System.Threading.Tasks; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis.KeyspaceIsolation { - internal partial class KeyPrefixed : IDatabaseAsync where TInner : IDatabaseAsync + internal partial class KeyPrefixed : IDatabaseAsync, IInternalDatabaseAsync where TInner : IDatabaseAsync { internal KeyPrefixed(TInner inner, byte[] keyPrefix) { @@ -17,10 +19,33 @@ internal KeyPrefixed(TInner inner, byte[] keyPrefix) public IConnectionMultiplexer Multiplexer => Inner.Multiplexer; + public int Database => Inner.Database; + + // default: this wrapper is not a full database (it is the base for the batch/transaction wrappers too), + // so it cannot start a transaction; KeyPrefixedDatabase reimplements this to prefix a real one + ITransactionAsync IDatabaseAsync.CreateTransaction(object? asyncState) + => throw new NotSupportedException("Transactions cannot be created here"); + internal TInner Inner { get; } internal byte[] Prefix { get; } + DatabaseFeatureFlags IInternalDatabaseAsync.GetFeatures(out string name) + => Inner.GetFeatures(out name) | GetDatabaseFeatures(); + + CancellationToken IInternalDatabaseAsync.GetNextFailover() => Inner.GetNextFailover(); + + // this wrapper does not stamp its own async-state; it inherits whatever the inner database uses + object? IInternalDatabaseAsync.AsyncState => Inner.GetAsyncState(); + + // the flags contributed by this wrapper itself (on top of the inner database); the batch and + // transaction subclasses override to fold in their own flag, mirroring RedisDatabase/RedisBatch/ + // RedisTransaction rather than relying on the inner instance to carry it. + private protected virtual DatabaseFeatureFlags GetDatabaseFeatures() + => DatabaseFeatureFlags.KeyPrefix; + + public override string ToString() => this.BuildString(); + public Task DebugObjectAsync(RedisKey key, CommandFlags flags = CommandFlags.None) => Inner.DebugObjectAsync(ToInner(key), flags); diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedBatch.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedBatch.cs index 6f5679a66..710e8cabd 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedBatch.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedBatch.cs @@ -1,8 +1,15 @@ -namespace StackExchange.Redis.KeyspaceIsolation +using StackExchange.Redis.Interfaces; + +namespace StackExchange.Redis.KeyspaceIsolation { internal sealed class KeyPrefixedBatch : KeyPrefixed, IBatch { - public KeyPrefixedBatch(IBatch inner, byte[] prefix) : base(inner, prefix) { } + public KeyPrefixedBatch(IBatch inner, byte[] prefix) : base(inner, prefix) + { + } + + private protected override DatabaseFeatureFlags GetDatabaseFeatures() + => base.GetDatabaseFeatures() | DatabaseFeatureFlags.Batch; public void Execute() => Inner.Execute(); } diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs index ed3dc952e..cbf7c72fc 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedDatabase.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Net; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis.KeyspaceIsolation { @@ -10,13 +11,22 @@ public KeyPrefixedDatabase(IDatabase inner, byte[] prefix) : base(inner, prefix) { } - public IBatch CreateBatch(object? asyncState = null) => - new KeyPrefixedBatch(Inner.CreateBatch(asyncState), Prefix); + public IBatch CreateBatch(object? asyncState = null) + { + // reject nesting based on *this* database (the thing being wrapped), not the freshly + // created inner batch - which trivially carries the Batch flag and would always trip + this.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction); + return new KeyPrefixedBatch(Inner.CreateBatch(asyncState), Prefix); + } - public ITransaction CreateTransaction(object? asyncState = null) => - new KeyPrefixedTransaction(Inner.CreateTransaction(asyncState), Prefix); + public ITransaction CreateTransaction(object? asyncState = null) + { + this.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction); + return new KeyPrefixedTransaction(Inner.CreateTransaction(asyncState), Prefix); + } - public int Database => Inner.Database; + // reimplement the async-only IDatabaseAsync slot (the base throws); we prefix a real transaction + ITransactionAsync IDatabaseAsync.CreateTransaction(object? asyncState) => CreateTransaction(asyncState); public RedisValue DebugObject(RedisKey key, CommandFlags flags = CommandFlags.None) => Inner.DebugObject(ToInner(key), flags); diff --git a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedTransaction.cs b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedTransaction.cs index 89703ba6a..3658d65ee 100644 --- a/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedTransaction.cs +++ b/src/StackExchange.Redis/KeyspaceIsolation/KeyPrefixedTransaction.cs @@ -1,10 +1,22 @@ using System.Threading.Tasks; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis.KeyspaceIsolation { - internal sealed class KeyPrefixedTransaction : KeyPrefixed, ITransaction + internal sealed class KeyPrefixedTransaction : KeyPrefixed, ITransaction, IInternalTransaction { - public KeyPrefixedTransaction(ITransaction inner, byte[] prefix) : base(inner, prefix) { } + public KeyPrefixedTransaction(ITransaction inner, byte[] prefix) : base(inner, prefix) + { + } + + private protected override DatabaseFeatureFlags GetDatabaseFeatures() + => base.GetDatabaseFeatures() | DatabaseFeatureFlags.Transaction; + + CommandFlags IInternalTransaction.GetAggregateRetryCategory() + => Inner is IInternalTransaction it ? it.GetAggregateRetryCategory() : CommandFlags.CommandRetryNever; + + /// + public bool WasWatchConflict => Inner.WasWatchConflict; public ConditionResult AddCondition(Condition condition) => Inner.AddCondition(condition.MapKeys(GetMapFunction())); diff --git a/src/StackExchange.Redis/Message.cs b/src/StackExchange.Redis/Message.cs index 8b4470de2..0eceec89e 100644 --- a/src/StackExchange.Redis/Message.cs +++ b/src/StackExchange.Redis/Message.cs @@ -59,7 +59,10 @@ internal abstract partial class Message : ICompletable internal const CommandFlags InternalCallFlag = (CommandFlags)128, - NoFlushFlag = (CommandFlags)1024; + NoFlushFlag = (CommandFlags)1024, + // "server specific" (bit 18): tied to a specific endpoint, never retry elsewhere. Not (yet) a + // public CommandFlags member - see the note on the hidden bit-18 value in CommandFlags.cs. + CommandServerSpecific = (CommandFlags)(1 << 18); protected RedisCommand command; @@ -73,17 +76,22 @@ internal const CommandFlags | CommandFlags.PreferMaster | CommandFlags.PreferReplica; - private const CommandFlags UserSelectableFlags = CommandFlags.None + // the 5-bit retry-category severity region (bits 13-17); numerically equal to CommandRetryNever. + // deliberately excludes CommandServerSpecific (bit 18), which is an orthogonal flag, not part of + // the <=-comparable severity ladder. + internal const CommandFlags MaskRetryCategory = CommandFlags.CommandRetryNever; + + internal const CommandFlags UserSelectableFlags = CommandFlags.None | CommandFlags.DemandMaster | CommandFlags.DemandReplica | CommandFlags.PreferMaster | CommandFlags.PreferReplica -#pragma warning disable CS0618 // Type or member is obsolete - | CommandFlags.HighPriority -#pragma warning restore CS0618 + | (CommandFlags)1 // CommandFlags.HighPriority; obsolete-as-error, but still tolerated from callers | CommandFlags.FireAndForget | CommandFlags.NoRedirect | CommandFlags.NoScriptCache + | MaskRetryCategory // caller may override the retry category... + | CommandServerSpecific // ...and the server-specific flag | NoFlushFlag; // we'll allow this one even though not advertised private IResultBox? resultBox; @@ -120,7 +128,9 @@ protected Message(int db, CommandFlags flags, RedisCommand command) bool primaryOnly = command.IsPrimaryOnly(); Db = db; this.command = command; - Flags = flags & UserSelectableFlags; + // apply the user-selectable flags, then fill in the default retry-category for this command + // (WithDefaultCategory is a no-op if the caller already specified a CommandRetry* category) + Flags = (flags & UserSelectableFlags).WithDefaultCategory(command); if (primaryOnly) SetPrimaryOnly(); CreatedDateTime = DateTime.UtcNow; @@ -522,18 +532,21 @@ public string ToStringCommandOnly() => bool ICompletable.TryComplete(bool isAsync) { - Complete(); + Complete(null); return true; } - public void Complete() + public void Complete(PhysicalConnection? connection) { // Ensure we can never call Complete on the same resultBox from two threads by grabbing it now var currBox = Interlocked.Exchange(ref resultBox, null); // set the completion/performance data performance?.SetCompleted(); - + if (currBox is not null) + { + connection?.ObserveMessageResult(currBox.Fault); + } currBox?.ActivateContinuations(); } @@ -635,6 +648,12 @@ internal static CommandFlags GetPrimaryReplicaFlags(CommandFlags flags) return flags & MaskPrimaryServerPreference; } + internal static CommandFlags GetRetryCategory(CommandFlags flags) + { + // isolate the retry-category region; 0 here means "not specified" (resolved downstream) + return flags & MaskRetryCategory; + } + internal static bool RequiresDatabase(RedisCommand command) { switch (command) @@ -745,10 +764,10 @@ internal void Fail(ConnectionFailureType failure, Exception? innerException, str resultProcessor?.ConnectionFail(this, failure, innerException, annotation, muxer); } - internal virtual void SetExceptionAndComplete(Exception exception, PhysicalBridge? bridge) + internal virtual void SetExceptionAndComplete(Exception exception, PhysicalConnection? connection) { resultBox?.SetException(exception); - Complete(); + Complete(connection); } internal bool TrySetResult(T value) diff --git a/src/StackExchange.Redis/PhysicalBridge.cs b/src/StackExchange.Redis/PhysicalBridge.cs index ed43629f4..049824536 100644 --- a/src/StackExchange.Redis/PhysicalBridge.cs +++ b/src/StackExchange.Redis/PhysicalBridge.cs @@ -105,9 +105,9 @@ public enum State : byte public long SubscriptionCount => physical?.SubscriptionCount ?? 0; internal State ConnectionState => (State)state; - internal bool IsBeating => Interlocked.CompareExchange(ref beating, 0, 0) == 1; + internal bool IsBeating => Volatile.Read(ref beating) == 1; - internal long OperationCount => Interlocked.Read(ref operationCount); + internal long OperationCount => Volatile.Read(ref operationCount); public RedisCommand LastCommand { get; private set; } @@ -133,7 +133,7 @@ public void Dispose() { isDisposed = true; // If there's anything in the backlog and we're being torn down - exfil it immediately (e.g. so all awaitables complete) - AbandonPendingBacklog(new ObjectDisposedException("Connection is being disposed")); + AbandonPendingBacklog(new ObjectDisposedException("Connection is being disposed"), null); try { _backlogAutoReset?.Set(); @@ -152,7 +152,7 @@ public void Dispose() isDisposed = true; // make damn sure we don't true to resurrect // If there's anything in the backlog and we're being torn down - exfil it immediately (e.g. so all awaitables complete) - AbandonPendingBacklog(new ObjectDisposedException("Connection is being finalized")); + AbandonPendingBacklog(new ObjectDisposedException("Connection is being finalized"), null); // shouldn't *really* touch managed objects // in a finalizer, but we need to kill that socket, @@ -192,7 +192,7 @@ private WriteResult QueueOrFailMessage(Message message) // Anything else goes in the bin - we're just not ready for you yet message.Cancel(); Multiplexer.OnMessageFaulted(message, null); - message.Complete(); + message.Complete(null); return WriteResult.NoConnectionAvailable; } @@ -200,7 +200,7 @@ private WriteResult FailDueToNoConnection(Message message) { message.Cancel(); Multiplexer.OnMessageFaulted(message, null); - message.Complete(); + message.Complete(null); return WriteResult.NoConnectionAvailable; } @@ -259,9 +259,9 @@ internal void AppendProfile(StringBuilder sb) var clone = new long[ProfileLogSamples + 1]; for (int i = 0; i < ProfileLogSamples; i++) { - clone[i] = Interlocked.Read(ref profileLog[i]); + clone[i] = Volatile.Read(ref profileLog[i]); } - clone[ProfileLogSamples] = Interlocked.Read(ref operationCount); + clone[ProfileLogSamples] = Volatile.Read(ref operationCount); Array.Sort(clone); sb.Append(' ').Append(clone[0]); for (int i = 1; i < clone.Length; i++) @@ -282,9 +282,9 @@ internal void AppendProfile(StringBuilder sb) internal void GetCounters(ConnectionCounters counters) { counters.OperationCount = OperationCount; - counters.SocketCount = Interlocked.Read(ref socketCount); - counters.WriterCount = Interlocked.CompareExchange(ref activeWriters, 0, 0); - counters.NonPreferredEndpointCount = Interlocked.Read(ref nonPreferredEndpointCount); + counters.SocketCount = Volatile.Read(ref socketCount); + counters.WriterCount = Volatile.Read(ref activeWriters); + counters.NonPreferredEndpointCount = Volatile.Read(ref nonPreferredEndpointCount); counters.PendingUnsentItems = Volatile.Read(ref _backlogCurrentEnqueued); physical?.GetCounters(counters); } @@ -342,7 +342,7 @@ public override string ToString() => internal BridgeStatus GetStatus() => new() { - MessagesSinceLastHeartbeat = (int)(Interlocked.Read(ref operationCount) - Interlocked.Read(ref profileLastLog)), + MessagesSinceLastHeartbeat = (int)(Volatile.Read(ref operationCount) - Volatile.Read(ref profileLastLog)), ConnectedAt = ConnectedAt, IsWriterActive = !_singleWriter.IsAvailable, BacklogMessagesPending = _backlog.Count, @@ -462,7 +462,7 @@ internal void OnConnectionFailed(PhysicalConnection connection, ConnectionFailur // If we're configured to, fail all pending backlogged messages if (Multiplexer.RawConfig.BacklogPolicy?.AbortPendingOnConnectionFailure == true) { - AbandonPendingBacklog(innerException); + AbandonPendingBacklog(innerException, connection); } if (reportNextFailure) @@ -511,12 +511,16 @@ internal void OnDisconnected(ConnectionFailureType failureType, PhysicalConnecti } } - private void AbandonPendingBacklog(Exception ex) + private void AbandonPendingBacklog(Exception ex, PhysicalConnection? connection) { while (BacklogTryDequeue(out Message? next)) { - Multiplexer.OnMessageFaulted(next, ex); - next.SetExceptionAndComplete(ex, this); + // as in PhysicalConnection.RecordMessageFailed: don't hand the same exception to every + // message. A backlogged message has provably never been written, which is exactly what the + // retry machinery needs to know, and it is lost if we share the connection-level instance. + var perMessage = ExceptionFactory.PerMessage(ex, next); + Multiplexer.OnMessageFaulted(next, perMessage); + next.SetExceptionAndComplete(perMessage, connection); } } @@ -551,7 +555,7 @@ internal void OnFullyEstablished(PhysicalConnection connection, string source) private bool DueForConnectRetry() { int connectTimeMilliseconds = unchecked(Environment.TickCount - Volatile.Read(ref connectStartTicks)); - return Multiplexer.RawConfig.ReconnectRetryPolicy.ShouldRetry(Interlocked.Read(ref connectTimeoutRetryCount), connectTimeMilliseconds); + return Multiplexer.RawConfig.ReconnectRetryPolicy.ShouldRetry(Volatile.Read(ref connectTimeoutRetryCount), connectTimeMilliseconds); } internal void OnHeartbeat(bool ifConnectedOnly) @@ -572,9 +576,9 @@ internal void OnHeartbeat(bool ifConnectedOnly) if (!runThisTime) return; uint index = (uint)Interlocked.Increment(ref profileLogIndex); - long newSampleCount = Interlocked.Read(ref operationCount); - Interlocked.Exchange(ref profileLog[index % ProfileLogSamples], newSampleCount); - Interlocked.Exchange(ref profileLastLog, newSampleCount); + long newSampleCount = Volatile.Read(ref operationCount); + Volatile.Write(ref profileLog[index % ProfileLogSamples], newSampleCount); + Volatile.Write(ref profileLastLog, newSampleCount); #pragma warning disable CS0420 // A reference to a volatile field will not be treated as volatile // ReSharper disable once LocalVariableHidesMember @@ -613,7 +617,7 @@ internal void OnHeartbeat(bool ifConnectedOnly) // Track that we should reset the count on the next disconnect, but not do so in a loop, reset // the connect-retry-count (used for backoff decay etc), and remove any non-responsive flag. shouldResetConnectionRetryCount = true; - Interlocked.Exchange(ref connectTimeoutRetryCount, 0); + Volatile.Write(ref connectTimeoutRetryCount, 0); tmp.BridgeCouldBeNull?.ServerEndPoint?.ClearUnselectable(UnselectableFlags.DidNotRespond); } tmp.OnBridgeHeartbeat(out int asyncTimeoutThisHeartbeat, out int syncTimeoutThisHeartbeat); @@ -687,7 +691,7 @@ internal void OnHeartbeat(bool ifConnectedOnly) if (shouldResetConnectionRetryCount) { shouldResetConnectionRetryCount = false; - Interlocked.Exchange(ref connectTimeoutRetryCount, 0); + Volatile.Write(ref connectTimeoutRetryCount, 0); } if (!ifConnectedOnly && DueForConnectRetry()) { @@ -700,7 +704,7 @@ internal void OnHeartbeat(bool ifConnectedOnly) } break; default: - Interlocked.Exchange(ref connectTimeoutRetryCount, 0); + Volatile.Write(ref connectTimeoutRetryCount, 0); break; } } @@ -802,7 +806,7 @@ private WriteResult WriteMessageInsideLock(PhysicalConnection physical, Message // killed the underlying connection Trace("Unable to write to server"); message.Fail(ConnectionFailureType.ProtocolFailure, null, "failure before write: " + result.ToString(), Multiplexer); - message.Complete(); + message.Complete(physical); return result; } // The parent message (next) may be returned from GetMessages @@ -1022,7 +1026,7 @@ private void CheckBacklogForTimeouts() // Tell the message it has failed // Note: Attempting to *avoid* reentrancy/deadlock issues by not holding the lock while completing messages. var ex = Multiplexer.GetException(WriteResult.TimeoutBeforeWrite, message, ServerEndPoint, this); - message.SetExceptionAndComplete(ex, this); + message.SetExceptionAndComplete(ex, null); } } } @@ -1098,7 +1102,7 @@ private void ProcessBacklog() } var ex = ExceptionFactory.Timeout(Multiplexer, "The message was in the backlog when connection was disposed", message, ServerEndPoint, WriteResult.TimeoutBeforeWrite, this); - message.SetExceptionAndComplete(ex, this); + message.SetExceptionAndComplete(ex, null); } } } @@ -1248,7 +1252,7 @@ private WriteResult TimedOutBeforeWrite(Message message) { message.Cancel(); Multiplexer.OnMessageFaulted(message, null); - message.Complete(); + message.Complete(null); return WriteResult.TimeoutBeforeWrite; } @@ -1380,8 +1384,8 @@ private async ValueTask CompleteWriteAndReleaseLockAsync( private WriteResult HandleWriteException(PhysicalConnection? physical, Message message, Exception ex) { - var inner = new RedisConnectionException(ConnectionFailureType.InternalFailure, "Failed to write", ex); - message.SetExceptionAndComplete(inner, this); + var inner = new RedisConnectionException(ConnectionFailureType.InternalFailure, message.Flags, "Failed to write", ex); + message.SetExceptionAndComplete(inner, physical); // Tear down the physical connection. A write that throws may have left a partial frame on the // wire, and continuing to use the same socket would let the next reply match the wrong message // in the response queue. Forcing a reconnect drains the in-flight queue with failures and @@ -1642,7 +1646,7 @@ private WriteResult WriteMessageToServerInsideWriteLock(PhysicalConnection conne { Trace("Write failed: " + ex.Message); message.Fail(ConnectionFailureType.InternalFailure, ex, null, Multiplexer); - message.Complete(); + message.Complete(connection); // This failed without actually writing; we're OK with that... unless there's a transaction if (connection?.TransactionActive == true) @@ -1657,7 +1661,7 @@ private WriteResult WriteMessageToServerInsideWriteLock(PhysicalConnection conne { Trace("Write failed: " + ex.Message); message.Fail(ConnectionFailureType.InternalFailure, ex, null, Multiplexer); - message.Complete(); + message.Complete(connection); // We're not sure *what* happened here - probably an IOException; kill the connection connection?.RecordConnectionFailed(ConnectionFailureType.InternalFailure, ex); diff --git a/src/StackExchange.Redis/PhysicalConnection.Read.cs b/src/StackExchange.Redis/PhysicalConnection.Read.cs index 0de5487cd..ad6973eca 100644 --- a/src/StackExchange.Redis/PhysicalConnection.Read.cs +++ b/src/StackExchange.Redis/PhysicalConnection.Read.cs @@ -632,7 +632,7 @@ private void MatchNextResult(ReadOnlySpan frame) if (_awaitingToken is not null && (msg = Interlocked.Exchange(ref _awaitingToken, null)) is not null) { _readStatus = ReadStatus.ResponseSequenceCheck; - if (!ProcessHighIntegrityResponseToken(msg, frame, BridgeCouldBeNull)) + if (!ProcessHighIntegrityResponseToken(msg, frame, this)) { RecordConnectionFailed(ConnectionFailureType.ResponseIntegrityFailure, origin: nameof(ReadStatus.ResponseSequenceCheck)); } @@ -678,7 +678,7 @@ static void Throw(ReadOnlySpan frame, ConnectionType connection, RedisProt if (highIntegrityToken is 0) { // can't complete yet if needs checksum - msg.Complete(); + msg.Complete(this); } } else @@ -694,7 +694,7 @@ static void Throw(ReadOnlySpan frame, ConnectionType connection, RedisProt _readStatus = ReadStatus.MatchResultComplete; _activeMessage = null; - static bool ProcessHighIntegrityResponseToken(Message message, ReadOnlySpan frame, PhysicalBridge? bridge) + static bool ProcessHighIntegrityResponseToken(Message message, ReadOnlySpan frame, PhysicalConnection? connection) { bool isValid = false; var reader = new RespReader(frame); @@ -716,12 +716,12 @@ static bool ProcessHighIntegrityResponseToken(Message message, ReadOnlySpan queue, [NotNullWhen(true)] out Messa } } - private void RecordMessageFailed(Message next, Exception? ex, string? origin, PhysicalBridge? bridge) + private void RecordMessageFailed(Message next, Exception? ex, string? origin, PhysicalConnection? connection) { if (next.Command == RedisCommand.QUIT && next.TrySetResult(true)) { // fine, death of a socket is close enough - next.Complete(); + next.Complete(this); } else { - if (bridge != null) + // the connection-level exception is shared across every message being failed here; give this + // one its own, carrying *its* flags and sent-status so retry policy can reason about it + if (ex is not null) ex = ExceptionFactory.PerMessage(ex, next); + + var bridge = connection?.BridgeCouldBeNull; + if (bridge is not null) { bridge.Trace("Failing: " + next); bridge.Multiplexer?.OnMessageFaulted(next, ex, origin); } - next.SetExceptionAndComplete(ex!, bridge); + next.SetExceptionAndComplete(ex!, connection); } } @@ -599,7 +607,7 @@ internal void EnqueueInsideWriteLock(Message next, bool enforceMuxer = true) // we can still process it to avoid making things worse/more complex, // but: we can't reliably assume this works, so: shout now! next.Cancel(); - next.Complete(); + next.Complete(null); } bool wasEmpty; @@ -754,7 +762,7 @@ internal void OnBridgeHeartbeat(out int asyncTimeoutDetected, out int syncTimeou lock (_writtenAwaitingResponse) { - if (_writtenAwaitingResponse.Count != 0 && BridgeCouldBeNull is PhysicalBridge bridge) + if (_writtenAwaitingResponse.Count != 0 && BridgeCouldBeNull is { } bridge) { var server = bridge.ServerEndPoint; var multiplexer = bridge.Multiplexer; @@ -773,7 +781,7 @@ internal void OnBridgeHeartbeat(out int asyncTimeoutDetected, out int syncTimeou : $"Timeout awaiting response ({elapsed}ms elapsed, timeout is {timeout}ms)"; var timeoutEx = ExceptionFactory.Timeout(multiplexer, baseErrorMessage, msg, server); multiplexer.OnMessageFaulted(msg, timeoutEx); - msg.SetExceptionAndComplete(timeoutEx, bridge); // tell the message that it is doomed + msg.SetExceptionAndComplete(timeoutEx, this); // tell the message that it is doomed multiplexer.OnAsyncTimeout(); asyncTimeoutDetected++; } @@ -803,6 +811,14 @@ internal void OnBridgeHeartbeat(out int asyncTimeoutDetected, out int syncTimeou } } } + + // backstop for a tripped circuit-breaker: normally the trip worker actuates the teardown, but + // if it hasn't been scheduled yet (and the connection has gone quiet) the heartbeat does it. + // Must be outside the lock above - RecordConnectionFailed takes that same lock. + if (Volatile.Read(ref _circuitBreakerState) is CircuitBreakerTripped) + { + CheckCircuitBreakerTrip(); + } } internal void OnInternalError(Exception exception, [CallerMemberName] string? origin = null) @@ -1110,7 +1126,7 @@ private void OnDebugAbort() var bridge = BridgeCouldBeNull; if (bridge == null || !bridge.Multiplexer.AllowConnect) { - throw new RedisConnectionException(ConnectionFailureType.InternalFailure, "Aborting (AllowConnect: False)"); + throw new RedisConnectionException(ConnectionFailureType.InternalFailure, CommandFlags.None, "Aborting (AllowConnect: False)"); } } @@ -1178,6 +1194,53 @@ internal bool HasPendingCallerFacingItems() if (lockTaken) Monitor.Exit(_writtenAwaitingResponse); } } + + public void ObserveMessageResult(Exception? fault) + { + // Hot path - runs per completed message. Let the breaker observe the outcome (ObserveResult + // returns true while healthy); if it trips and we're the *first* to notice, hand off the + // teardown rather than doing it inline: RecordConnectionFailed can fail the whole backlog and + // build a detailed exception, which we don't want to pay for on the completion thread. + if (circuitBreaker is { } cb && cb.Trip(fault) + && Interlocked.CompareExchange(ref _circuitBreakerState, CircuitBreakerTripped, CircuitBreakerHealthy) is CircuitBreakerHealthy) + { + // hand off to a worker; the heartbeat (see OnBridgeHeartbeat) is a backstop in case the + // pool is slow to schedule us and the connection then goes quiet - either way the actual + // teardown happens exactly once, via CheckCircuitBreakerTrip. A cached static callback with + // the connection as state avoids any per-trip delegate/closure allocation. + ThreadPool.QueueUserWorkItem(s_CheckCircuitBreakerTrip, this); + } + } + + private static readonly WaitCallback s_CheckCircuitBreakerTrip = static state => + { + var connection = (PhysicalConnection)state!; + try + { + connection.CheckCircuitBreakerTrip(); + } + catch (Exception ex) + { + connection.OnInternalError(ex); + } + }; + + // Actuates a pending circuit-breaker teardown at most once. Called from both the trip worker and + // the heartbeat backstop; whoever wins the tripped->actuated transition does the work. Routing via + // RecordConnectionFailed (not a bare Shutdown) is what surfaces the failure as a ConnectionFailed + // event, which is what a connection group reacts to when re-routing away from this member. + private void CheckCircuitBreakerTrip() + { + if (Interlocked.CompareExchange(ref _circuitBreakerState, CircuitBreakerActuated, CircuitBreakerTripped) is CircuitBreakerTripped) + { + RecordConnectionFailed(ConnectionFailureType.CircuitBreaker); + } + } + + private CircuitBreaker.Accumulator? circuitBreaker; + + private int _circuitBreakerState; // transitions strictly Healthy -> Tripped -> Actuated + private const int CircuitBreakerHealthy = 0, CircuitBreakerTripped = 1, CircuitBreakerActuated = 2; } internal sealed class DummyHighIntegrityMessage : Message diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Shipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Shipped.txt index a663ada89..605f04040 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Shipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Shipped.txt @@ -1,4 +1,4 @@ -#nullable enable +#nullable enable abstract StackExchange.Redis.RedisResult.IsNull.get -> bool override StackExchange.Redis.ChannelMessage.Equals(object? obj) -> bool override StackExchange.Redis.ChannelMessage.GetHashCode() -> int @@ -2542,3 +2542,318 @@ static StackExchange.Redis.Lease.Create(int length, System.Buffers.MemoryPool static StackExchange.Redis.RedisValue.implicit operator StackExchange.Redis.RedisValue(System.Buffers.ReadOnlySequence value) -> StackExchange.Redis.RedisValue static StackExchange.Redis.RedisValue.implicit operator System.Buffers.ReadOnlySequence(StackExchange.Redis.RedisValue value) -> System.Buffers.ReadOnlySequence static StackExchange.Redis.ValueCondition.CalculateDigest(in System.Buffers.ReadOnlySequence value) -> StackExchange.Redis.ValueCondition +abstract StackExchange.Redis.Configuration.LoggingTunnel.Log(System.IO.Stream! stream, System.Net.EndPoint! endpoint, StackExchange.Redis.ConnectionType connectionType) -> System.IO.Stream! +override StackExchange.Redis.Configuration.LoggingTunnel.BeforeAuthenticateAsync(System.Net.EndPoint! endpoint, StackExchange.Redis.ConnectionType connectionType, System.Net.Sockets.Socket? socket, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.ValueTask +override StackExchange.Redis.Configuration.LoggingTunnel.BeforeSocketConnectAsync(System.Net.EndPoint! endPoint, StackExchange.Redis.ConnectionType connectionType, System.Net.Sockets.Socket? socket, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.ValueTask +override StackExchange.Redis.Configuration.LoggingTunnel.GetSocketConnectEndpointAsync(System.Net.EndPoint! endpoint, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.ValueTask +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.BeginRead(byte[]! buffer, int offset, int count, System.AsyncCallback? callback, object? state) -> System.IAsyncResult! +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.BeginWrite(byte[]! buffer, int offset, int count, System.AsyncCallback? callback, object? state) -> System.IAsyncResult! +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.CanRead.get -> bool +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.CanSeek.get -> bool +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.CanTimeout.get -> bool +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.CanWrite.get -> bool +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Close() -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.EndRead(System.IAsyncResult! asyncResult) -> int +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.EndWrite(System.IAsyncResult! asyncResult) -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Flush() -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.FlushAsync(System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Length.get -> long +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Position.get -> long +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Position.set -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Read(byte[]! buffer, int offset, int count) -> int +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.ReadAsync(byte[]! buffer, int offset, int count, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.ReadByte() -> int +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.ReadTimeout.get -> int +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.ReadTimeout.set -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Seek(long offset, System.IO.SeekOrigin origin) -> long +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.SetLength(long value) -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Write(byte[]! buffer, int offset, int count) -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.WriteAsync(byte[]! buffer, int offset, int count, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.WriteByte(byte value) -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.WriteTimeout.get -> int +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.WriteTimeout.set -> void +StackExchange.Redis.Configuration.LoggingTunnel +StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream +StackExchange.Redis.Configuration.LoggingTunnel.LoggingTunnel(StackExchange.Redis.ConfigurationOptions? options = null, StackExchange.Redis.Configuration.Tunnel? tail = null) -> void +static StackExchange.Redis.Configuration.LoggingTunnel.DefaultFormatCommand(StackExchange.Redis.RedisResult! value) -> string! +static StackExchange.Redis.Configuration.LoggingTunnel.DefaultFormatResponse(StackExchange.Redis.RedisResult! value) -> string! +static StackExchange.Redis.Configuration.LoggingTunnel.LogToDirectory(StackExchange.Redis.ConfigurationOptions! options, string! path) -> void +static StackExchange.Redis.Configuration.LoggingTunnel.ReplayAsync(string! path, System.Action! pair) -> System.Threading.Tasks.Task! +static StackExchange.Redis.Configuration.LoggingTunnel.ReplayAsync(System.IO.Stream! out, System.IO.Stream! in, System.Action! pair) -> System.Threading.Tasks.Task! +static StackExchange.Redis.Configuration.LoggingTunnel.ValidateAsync(string! path) -> System.Threading.Tasks.Task! +static StackExchange.Redis.Configuration.LoggingTunnel.ValidateAsync(System.IO.Stream! stream) -> System.Threading.Tasks.Task! +StackExchange.Redis.IDatabase.ListMove(StackExchange.Redis.RedisKey sourceKey, StackExchange.Redis.RedisKey destinationKey, StackExchange.Redis.ListSide sourceSide, StackExchange.Redis.ListSide destinationSide, long count, StackExchange.Redis.ListMoveCount mode = StackExchange.Redis.ListMoveCount.UpTo, StackExchange.Redis.ListMoveOrder order = StackExchange.Redis.ListMoveOrder.Bulk, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.RedisValue[]? +StackExchange.Redis.IDatabase.SetCombineLength(StackExchange.Redis.SetOperation operation, StackExchange.Redis.RedisKey[]! keys, long limit = 0, bool approximate = false, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> long +StackExchange.Redis.IDatabase.StreamRead(StackExchange.Redis.StreamPosition[]! streamPositions, int? countPerStream = null, int? maxCount = null, int? maxSize = null, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.RedisStream[]! +StackExchange.Redis.IDatabase.StreamReadGroup(StackExchange.Redis.StreamPosition[]! streamPositions, StackExchange.Redis.RedisValue groupName, StackExchange.Redis.RedisValue consumerName, int? countPerStream = null, bool noAck = false, System.TimeSpan? claimMinIdleTime = null, int? maxCount = null, int? maxSize = null, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.RedisStream[]! +StackExchange.Redis.IDatabaseAsync.ListMoveAsync(StackExchange.Redis.RedisKey sourceKey, StackExchange.Redis.RedisKey destinationKey, StackExchange.Redis.ListSide sourceSide, StackExchange.Redis.ListSide destinationSide, long count, StackExchange.Redis.ListMoveCount mode = StackExchange.Redis.ListMoveCount.UpTo, StackExchange.Redis.ListMoveOrder order = StackExchange.Redis.ListMoveOrder.Bulk, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +StackExchange.Redis.IDatabaseAsync.SetCombineLengthAsync(StackExchange.Redis.SetOperation operation, StackExchange.Redis.RedisKey[]! keys, long limit = 0, bool approximate = false, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +StackExchange.Redis.IDatabaseAsync.StreamReadAsync(StackExchange.Redis.StreamPosition[]! streamPositions, int? countPerStream = null, int? maxCount = null, int? maxSize = null, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +StackExchange.Redis.IDatabaseAsync.StreamReadGroupAsync(StackExchange.Redis.StreamPosition[]! streamPositions, StackExchange.Redis.RedisValue groupName, StackExchange.Redis.RedisValue consumerName, int? countPerStream = null, bool noAck = false, System.TimeSpan? claimMinIdleTime = null, int? maxCount = null, int? maxSize = null, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +StackExchange.Redis.ListMoveCount +StackExchange.Redis.ListMoveCount.Exactly = 1 -> StackExchange.Redis.ListMoveCount +StackExchange.Redis.ListMoveCount.UpTo = 0 -> StackExchange.Redis.ListMoveCount +StackExchange.Redis.ListMoveOrder +StackExchange.Redis.ListMoveOrder.Bulk = 0 -> StackExchange.Redis.ListMoveOrder +StackExchange.Redis.ListMoveOrder.OneByOne = 1 -> StackExchange.Redis.ListMoveOrder +[SER008]StackExchange.Redis.HashImport +[SER008]StackExchange.Redis.HashImport.Dispose() -> void +[SER008]StackExchange.Redis.HashImport.DisposeAsync() -> System.Threading.Tasks.ValueTask +[SER008]StackExchange.Redis.IDatabase.HashImport(StackExchange.Redis.RedisKey key, StackExchange.Redis.HashImport! fieldSet, System.ReadOnlyMemory values, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> void +[SER008]StackExchange.Redis.IDatabaseAsync.HashImportAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.HashImport! fieldSet, System.ReadOnlyMemory values, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +[SER008]static StackExchange.Redis.HashImport.Create(params StackExchange.Redis.RedisValue[]! fields) -> StackExchange.Redis.HashImport! +[SER008]static StackExchange.Redis.HashImport.Create(System.ReadOnlyMemory fields) -> StackExchange.Redis.HashImport! +StackExchange.Redis.RedisFeatures.HashImport.get -> bool +StackExchange.Redis.IDatabaseAsync.Database.get -> int +[SER007]StackExchange.Redis.Availability.RetryPolicy +[SER007]StackExchange.Redis.Availability.RetryPolicy.FailoverDelay.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.RetryPolicy.JitterMax.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxAttempts.get -> int +[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsBeforeFailover.get -> int +[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxAttemptsOnWatchConflict.get -> int +[SER007]StackExchange.Redis.Availability.RetryPolicy.MaxCommandRetryCategory.get -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryDelay.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicy() -> void +StackExchange.Redis.IServer.InventKey(StackExchange.Redis.RedisKey prefix = default(StackExchange.Redis.RedisKey)) -> StackExchange.Redis.RedisKey +StackExchange.Redis.Availability.DatabaseExtensions +[SER007]static StackExchange.Redis.Availability.DatabaseExtensions.WithRetry(this StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.Availability.RetryPolicy? retryPolicy = null) -> StackExchange.Redis.IDatabaseAsync! +[SER007]StackExchange.Redis.IDatabaseAsync.CreateTransaction(object? asyncState = null) -> StackExchange.Redis.ITransactionAsync! +StackExchange.Redis.ITransactionAsync +StackExchange.Redis.ITransactionAsync.AddCondition(StackExchange.Redis.Condition! condition) -> StackExchange.Redis.ConditionResult! +StackExchange.Redis.ITransactionAsync.ExecuteAsync(StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! +StackExchange.Redis.ITransactionAsync.WasWatchConflict.get -> bool +[SER007]StackExchange.Redis.Availability.CircuitBreaker +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Accumulator +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Accumulator.Accumulator() -> void +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.Builder() -> void +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.Create() -> StackExchange.Redis.Availability.CircuitBreaker! +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.FailureRateThreshold.get -> double +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.FailureRateThreshold.set -> void +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.MetricsWindowSize.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.MetricsWindowSize.set -> void +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.MinimumNumberOfFailures.get -> int +[SER007]StackExchange.Redis.Availability.CircuitBreaker.Builder.MinimumNumberOfFailures.set -> void +[SER007]StackExchange.Redis.Availability.CircuitBreaker.CircuitBreaker() -> void +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.ConnectionGroupMember(StackExchange.Redis.ConfigurationOptions! configuration, string! name = "") -> void +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.ConnectionGroupMember(string! configuration, string! name = "") -> void +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.IsConnected.get -> bool +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.Latency.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.Name.get -> string! +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.SkipInitialHealthCheck.get -> bool +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.SkipInitialHealthCheck.set -> void +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.Weight.get -> double +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.Weight.set -> void +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType.ActiveChanged = 5 -> StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType.Added = 1 -> StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType.Disconnected = 3 -> StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType.Reconnected = 4 -> StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType.Removed = 2 -> StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType.Unknown = 0 -> StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.Group.get -> StackExchange.Redis.Availability.ConnectionGroupMember! +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.GroupConnectionChangedEventArgs(StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType type, StackExchange.Redis.Availability.ConnectionGroupMember! group, StackExchange.Redis.Availability.ConnectionGroupMember? previousGroup = null) -> void +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.PreviousGroup.get -> StackExchange.Redis.Availability.ConnectionGroupMember? +[SER007]StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.Type.get -> StackExchange.Redis.Availability.GroupConnectionChangedEventArgs.ChangeType +[SER007]StackExchange.Redis.Availability.HealthCheck +[SER007]StackExchange.Redis.Availability.HealthCheck.ProbeCount.get -> int +[SER007]StackExchange.Redis.Availability.HealthCheck.ProbeInterval.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.HealthCheck.ProbeTimeout.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.IConnectionGroup +[SER007]StackExchange.Redis.Availability.IConnectionGroup.ActiveMember.get -> StackExchange.Redis.Availability.ConnectionGroupMember? +[SER007]StackExchange.Redis.Availability.IConnectionGroup.AddAsync(StackExchange.Redis.Availability.ConnectionGroupMember! member, System.IO.TextWriter? log = null) -> System.Threading.Tasks.Task! +[SER007]StackExchange.Redis.Availability.IConnectionGroup.ConnectionChanged -> System.EventHandler? +[SER007]StackExchange.Redis.Availability.IConnectionGroup.GetMembers() -> System.ReadOnlySpan +[SER007]StackExchange.Redis.Availability.IConnectionGroup.Remove(StackExchange.Redis.Availability.ConnectionGroupMember! member) -> bool +[SER007]StackExchange.Redis.Availability.IConnectionGroup.TryFailoverTo(StackExchange.Redis.Availability.ConnectionGroupMember? member) -> bool +[SER007]StackExchange.Redis.Availability.MultiGroupOptions +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.CircuitBreaker.get -> StackExchange.Redis.Availability.CircuitBreaker! +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.HealthCheck.get -> StackExchange.Redis.Availability.HealthCheck! +[SER007]static StackExchange.Redis.ConnectionMultiplexer.ConnectGroupAsync(StackExchange.Redis.Availability.ConnectionGroupMember! member0, StackExchange.Redis.Availability.ConnectionGroupMember! member1, StackExchange.Redis.Availability.MultiGroupOptions? options = null, System.IO.TextWriter? log = null) -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.ConnectionMultiplexer.ConnectGroupAsync(StackExchange.Redis.Availability.ConnectionGroupMember![]! members, StackExchange.Redis.Availability.MultiGroupOptions? options = null, System.IO.TextWriter? log = null) -> System.Threading.Tasks.Task! +[SER007]StackExchange.Redis.ConfigurationOptions.CircuitBreaker.get -> StackExchange.Redis.Availability.CircuitBreaker? +[SER007]StackExchange.Redis.ConfigurationOptions.CircuitBreaker.set -> void +[SER007]StackExchange.Redis.ConnectionFailureType.CircuitBreaker = 11 -> StackExchange.Redis.ConnectionFailureType +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.IsUnhealthy.get -> bool +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.ResetIsUnhealthy() -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.FailbackDelay.get -> System.TimeSpan +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.IsHealthy() -> bool +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.Reset() -> void +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.CreateAccumulator() -> StackExchange.Redis.Availability.CircuitBreaker.Accumulator! +[SER007]static StackExchange.Redis.Availability.CircuitBreaker.Default.get -> StackExchange.Redis.Availability.CircuitBreaker! +[SER007]static StackExchange.Redis.Availability.CircuitBreaker.None.get -> StackExchange.Redis.Availability.CircuitBreaker! +[SER007]static StackExchange.Redis.Availability.CircuitBreaker.Builder.implicit operator StackExchange.Redis.Availability.CircuitBreaker!(StackExchange.Redis.Availability.CircuitBreaker.Builder! builder) -> StackExchange.Redis.Availability.CircuitBreaker! +[SER007]static StackExchange.Redis.Availability.HealthCheck.Default.get -> StackExchange.Redis.Availability.HealthCheck! +[SER007]static StackExchange.Redis.Availability.MultiGroupOptions.Default.get -> StackExchange.Redis.Availability.MultiGroupOptions! +[SER007]override StackExchange.Redis.Availability.ConnectionGroupMember.ToString() -> string! +[SER007]StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.None = 0 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Unknown = 1 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.Availability.FaultContext +[SER007]StackExchange.Redis.Availability.FaultContext.ConnectionFailureType.get -> StackExchange.Redis.ConnectionFailureType +[SER007]StackExchange.Redis.Availability.FaultContext.ErrorKind.get -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.Availability.FaultContext.Fault.get -> System.Exception? +[SER007]StackExchange.Redis.Availability.FaultContext.FaultContext() -> void +[SER007]StackExchange.Redis.Availability.FaultContext.FaultContext(System.Exception! fault) -> void +[SER007]StackExchange.Redis.Availability.FaultContext.Flags.get -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.Availability.FaultContext.IsFault.get -> bool +[SER007]StackExchange.Redis.Availability.FaultContext.NotApplied.get -> bool +[SER007]StackExchange.Redis.RedisErrorKind.Ask = 15 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Busy = 12 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.ClusterDown = 6 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.ConnectionFault = 3 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.CrossSlot = 16 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.ExecAbort = 23 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Loading = 5 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.MasterDown = 7 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.MaxClients = 13 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Misconfigured = 10 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Moved = 14 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.NoAuth = 17 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.NoPermission = 19 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.NoReplicas = 9 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.NoScript = 25 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.NotPermitted = 20 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.OutOfMemory = 11 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.ReadOnly = 24 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.Timeout = 4 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.TryAgain = 8 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.UnknownCommand = 21 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.UnknownError = 2 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.WrongPass = 18 -> StackExchange.Redis.RedisErrorKind +[SER007]StackExchange.Redis.RedisErrorKind.WrongType = 22 -> StackExchange.Redis.RedisErrorKind +[SER007]abstract StackExchange.Redis.Availability.CircuitBreaker.Accumulator.ObserveResult(in StackExchange.Redis.Availability.FaultContext fault) -> void +[SER007]virtual StackExchange.Redis.Availability.CircuitBreaker.Accumulator.IsFailure(in StackExchange.Redis.Availability.FaultContext fault) -> bool +[SER007]StackExchange.Redis.CommandFlags.CommandRetryAlways = 8192 -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryConnection = 32768 -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryReadOnly = 65536 -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryWriteChecked = StackExchange.Redis.CommandFlags.CommandRetryConnection | StackExchange.Redis.CommandFlags.CommandRetryReadOnly -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins = 131072 -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryWriteAccumulating = StackExchange.Redis.CommandFlags.CommandRetryConnection | StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryServerAdmin = StackExchange.Redis.CommandFlags.CommandRetryReadOnly | StackExchange.Redis.CommandFlags.CommandRetryWriteLastWins -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.CommandFlags.CommandRetryNever = 253952 -> StackExchange.Redis.CommandFlags +StackExchange.Redis.RedisConnectionException.Flags.get -> StackExchange.Redis.CommandFlags +StackExchange.Redis.RedisConnectionException.RedisConnectionException(StackExchange.Redis.ConnectionFailureType failureType, StackExchange.Redis.CommandFlags flags, string! message, System.Exception? innerException = null, StackExchange.Redis.CommandStatus commandStatus = StackExchange.Redis.CommandStatus.Unknown) -> void +StackExchange.Redis.RedisServerException.Flags.get -> StackExchange.Redis.CommandFlags +StackExchange.Redis.RedisServerException.Kind.get -> StackExchange.Redis.RedisErrorKind +StackExchange.Redis.RedisServerException.RedisServerException(StackExchange.Redis.RedisErrorKind kind, StackExchange.Redis.CommandFlags flags, string! message) -> void +StackExchange.Redis.RedisTimeoutException.Flags.get -> StackExchange.Redis.CommandFlags +StackExchange.Redis.RedisTimeoutException.RedisTimeoutException(StackExchange.Redis.CommandFlags flags, string! message, StackExchange.Redis.CommandStatus commandStatus) -> void +[SER007]abstract StackExchange.Redis.Availability.HealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheckContext context) -> System.Threading.Tasks.Task! +[SER007]abstract StackExchange.Redis.Availability.HealthCheckProbePolicy.Evaluate(in StackExchange.Redis.Availability.HealthCheckProbeContext context) -> StackExchange.Redis.Availability.HealthCheckResult +[SER007]abstract StackExchange.Redis.Availability.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheckContext context, StackExchange.Redis.IDatabaseAsync! database, StackExchange.Redis.RedisKey key) -> System.Threading.Tasks.Task! +[SER007]override sealed StackExchange.Redis.Availability.KeyWriteHealthCheckProbe.CheckHealthAsync(StackExchange.Redis.Availability.HealthCheckContext context) -> System.Threading.Tasks.Task! +[SER007]override StackExchange.Redis.Availability.HealthCheckContext.ToString() -> string! +[SER007]override StackExchange.Redis.Availability.HealthCheckProbeContext.ToString() -> string! +[SER007]override StackExchange.Redis.Availability.HealthCheck.ToString() -> string! +[SER007]override StackExchange.Redis.Availability.MultiGroupOptions.ToString() -> string! +[SER007]override StackExchange.Redis.Availability.RetryPolicy.ToString() -> string! +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.CircuitBreaker.get -> StackExchange.Redis.Availability.CircuitBreaker? +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.CircuitBreaker.set -> void +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.FailbackDelay.get -> System.TimeSpan? +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.FailbackDelay.set -> void +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.HealthCheck.get -> StackExchange.Redis.Availability.HealthCheck? +[SER007]StackExchange.Redis.Availability.ConnectionGroupMember.HealthCheck.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.Builder() -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.Builder(StackExchange.Redis.Availability.HealthCheck! healthCheck) -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.Create() -> StackExchange.Redis.Availability.HealthCheck! +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.ProbeCount.get -> int +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.ProbeCount.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.Probe.get -> StackExchange.Redis.Availability.HealthCheckProbe! +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.ProbeInterval.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.ProbeInterval.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.ProbePolicy.get -> StackExchange.Redis.Availability.HealthCheckProbePolicy! +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.ProbePolicy.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.Probe.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.ProbeTimeout.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.HealthCheck.Builder.ProbeTimeout.set -> void +[SER007]StackExchange.Redis.Availability.HealthCheck.CheckHealthAsync(StackExchange.Redis.IConnectionMultiplexer! multiplexer) -> System.Threading.Tasks.Task! +[SER007]StackExchange.Redis.Availability.HealthCheck.CheckHealthAsync(StackExchange.Redis.IServer! server) -> System.Threading.Tasks.Task! +[SER007]StackExchange.Redis.Availability.HealthCheckContext +[SER007]StackExchange.Redis.Availability.HealthCheckContext.HealthCheckContext() -> void +[SER007]StackExchange.Redis.Availability.HealthCheckContext.HealthCheckContext(StackExchange.Redis.IServer! server, System.TimeSpan probeTimeout) -> void +[SER007]StackExchange.Redis.Availability.HealthCheckContext.ProbeTimeout.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.HealthCheckContext.Server.get -> StackExchange.Redis.IServer! +[SER007]StackExchange.Redis.Availability.HealthCheck.IsEnabled.get -> bool +[SER007]StackExchange.Redis.Availability.HealthCheckProbe +[SER007]StackExchange.Redis.Availability.HealthCheckProbeContext +[SER007]StackExchange.Redis.Availability.HealthCheckProbeContext.Failure.get -> int +[SER007]StackExchange.Redis.Availability.HealthCheckProbeContext.HealthCheckProbeContext() -> void +[SER007]StackExchange.Redis.Availability.HealthCheckProbeContext.HealthCheckProbeContext(StackExchange.Redis.Availability.HealthCheckResult result, int success, int failure, int remaining, System.TimeSpan probeInterval) -> void +[SER007]StackExchange.Redis.Availability.HealthCheckProbeContext.ProbeInterval.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.HealthCheckProbeContext.Remaining.get -> int +[SER007]StackExchange.Redis.Availability.HealthCheckProbeContext.Result.get -> StackExchange.Redis.Availability.HealthCheckResult +[SER007]StackExchange.Redis.Availability.HealthCheckProbeContext.Success.get -> int +[SER007]StackExchange.Redis.Availability.HealthCheck.Probe.get -> StackExchange.Redis.Availability.HealthCheckProbe! +[SER007]StackExchange.Redis.Availability.HealthCheckProbe.HealthCheckProbe() -> void +[SER007]StackExchange.Redis.Availability.HealthCheckProbePolicy +[SER007]StackExchange.Redis.Availability.HealthCheck.ProbePolicy.get -> StackExchange.Redis.Availability.HealthCheckProbePolicy! +[SER007]StackExchange.Redis.Availability.HealthCheckProbePolicy.HealthCheckProbePolicy() -> void +[SER007]StackExchange.Redis.Availability.HealthCheckResult +[SER007]StackExchange.Redis.Availability.HealthCheckResult.Healthy = 1 -> StackExchange.Redis.Availability.HealthCheckResult +[SER007]StackExchange.Redis.Availability.HealthCheckResult.Inconclusive = 0 -> StackExchange.Redis.Availability.HealthCheckResult +[SER007]StackExchange.Redis.Availability.HealthCheckResult.Unhealthy = 2 -> StackExchange.Redis.Availability.HealthCheckResult +[SER007]StackExchange.Redis.Availability.IConnectionGroup.Options.get -> StackExchange.Redis.Availability.MultiGroupOptions! +[SER007]StackExchange.Redis.Availability.KeyWriteHealthCheckProbe +[SER007]StackExchange.Redis.Availability.KeyWriteHealthCheckProbe.KeyWriteHealthCheckProbe() -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.Builder() -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.Builder(StackExchange.Redis.Availability.MultiGroupOptions! options) -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.CircuitBreaker.get -> StackExchange.Redis.Availability.CircuitBreaker! +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.CircuitBreaker.set -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.Create() -> StackExchange.Redis.Availability.MultiGroupOptions! +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.FailbackDelay.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.FailbackDelay.set -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.HealthCheck.get -> StackExchange.Redis.Availability.HealthCheck! +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.HealthCheckInterval.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.HealthCheckInterval.set -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.HealthCheck.set -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.RetryPolicy.get -> StackExchange.Redis.Availability.RetryPolicy! +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.Builder.RetryPolicy.set -> void +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.HealthCheckInterval.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.MultiGroupOptions.RetryPolicy.get -> StackExchange.Redis.Availability.RetryPolicy! +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.Builder() -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.Builder(StackExchange.Redis.Availability.RetryPolicy! policy) -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.Create() -> StackExchange.Redis.Availability.RetryPolicy! +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.FailoverDelay.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.FailoverDelay.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.JitterMax.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.JitterMax.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.MaxAttemptsBeforeFailover.get -> int +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.MaxAttemptsBeforeFailover.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.MaxAttemptsOnWatchConflict.get -> int +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.MaxAttemptsOnWatchConflict.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.MaxAttempts.get -> int +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.MaxAttempts.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.MaxCommandRetryCategory.get -> StackExchange.Redis.CommandFlags +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.MaxCommandRetryCategory.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.RetryDelay.get -> System.TimeSpan +[SER007]StackExchange.Redis.Availability.RetryPolicy.Builder.RetryDelay.set -> void +[SER007]StackExchange.Redis.Availability.RetryPolicy.RetryPolicy(StackExchange.Redis.Availability.RetryPolicy.Builder! builder) -> void +[SER007]StackExchange.Redis.Availability.RetryResult +[SER007]StackExchange.Redis.Availability.RetryResult.FailoverServer = 2 -> StackExchange.Redis.Availability.RetryResult +[SER007]StackExchange.Redis.Availability.RetryResult.None = 0 -> StackExchange.Redis.Availability.RetryResult +[SER007]StackExchange.Redis.Availability.RetryResult.SameServer = 1 -> StackExchange.Redis.Availability.RetryResult +[SER007]StackExchange.Redis.ConfigurationOptions.RetryPolicy.get -> StackExchange.Redis.Availability.RetryPolicy? +[SER007]StackExchange.Redis.ConfigurationOptions.RetryPolicy.set -> void +[SER007]static StackExchange.Redis.Availability.HealthCheck.Builder.implicit operator StackExchange.Redis.Availability.HealthCheck!(StackExchange.Redis.Availability.HealthCheck.Builder! builder) -> StackExchange.Redis.Availability.HealthCheck! +[SER007]static StackExchange.Redis.Availability.HealthCheck.None.get -> StackExchange.Redis.Availability.HealthCheck! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbe.HealthyTask.get -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbe.InconclusiveTask.get -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbe.IsConnected.get -> StackExchange.Redis.Availability.HealthCheckProbe! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbe.None.get -> StackExchange.Redis.Availability.HealthCheckProbe! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbe.Ping.get -> StackExchange.Redis.Availability.HealthCheckProbe! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbePolicy.AllSuccess.get -> StackExchange.Redis.Availability.HealthCheckProbePolicy! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbePolicy.AnySuccess.get -> StackExchange.Redis.Availability.HealthCheckProbePolicy! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbePolicy.MajoritySuccess.get -> StackExchange.Redis.Availability.HealthCheckProbePolicy! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbe.StringSet.get -> StackExchange.Redis.Availability.HealthCheckProbe! +[SER007]static StackExchange.Redis.Availability.HealthCheckProbe.UnhealthyTask.get -> System.Threading.Tasks.Task! +[SER007]static StackExchange.Redis.Availability.MultiGroupOptions.Builder.implicit operator StackExchange.Redis.Availability.MultiGroupOptions!(StackExchange.Redis.Availability.MultiGroupOptions.Builder! builder) -> StackExchange.Redis.Availability.MultiGroupOptions! +[SER007]static StackExchange.Redis.Availability.RetryPolicy.Builder.implicit operator StackExchange.Redis.Availability.RetryPolicy!(StackExchange.Redis.Availability.RetryPolicy.Builder! builder) -> StackExchange.Redis.Availability.RetryPolicy! +[SER007]static StackExchange.Redis.Availability.RetryPolicy.Default.get -> StackExchange.Redis.Availability.RetryPolicy! +[SER007]static StackExchange.Redis.Availability.RetryPolicy.None.get -> StackExchange.Redis.Availability.RetryPolicy! +[SER007]virtual StackExchange.Redis.Availability.RetryPolicy.CanRetry(in StackExchange.Redis.Availability.FaultContext fault) -> StackExchange.Redis.Availability.RetryResult diff --git a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt index a21ad1c69..7dc5c5811 100644 --- a/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/StackExchange.Redis/PublicAPI/PublicAPI.Unshipped.txt @@ -1,23 +1 @@ #nullable enable -StackExchange.Redis.IDatabase.ListMove(StackExchange.Redis.RedisKey sourceKey, StackExchange.Redis.RedisKey destinationKey, StackExchange.Redis.ListSide sourceSide, StackExchange.Redis.ListSide destinationSide, long count, StackExchange.Redis.ListMoveCount mode = StackExchange.Redis.ListMoveCount.UpTo, StackExchange.Redis.ListMoveOrder order = StackExchange.Redis.ListMoveOrder.Bulk, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.RedisValue[]? -StackExchange.Redis.IDatabase.SetCombineLength(StackExchange.Redis.SetOperation operation, StackExchange.Redis.RedisKey[]! keys, long limit = 0, bool approximate = false, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> long -StackExchange.Redis.IDatabase.StreamRead(StackExchange.Redis.StreamPosition[]! streamPositions, int? countPerStream = null, int? maxCount = null, int? maxSize = null, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.RedisStream[]! -StackExchange.Redis.IDatabase.StreamReadGroup(StackExchange.Redis.StreamPosition[]! streamPositions, StackExchange.Redis.RedisValue groupName, StackExchange.Redis.RedisValue consumerName, int? countPerStream = null, bool noAck = false, System.TimeSpan? claimMinIdleTime = null, int? maxCount = null, int? maxSize = null, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> StackExchange.Redis.RedisStream[]! -StackExchange.Redis.IDatabaseAsync.ListMoveAsync(StackExchange.Redis.RedisKey sourceKey, StackExchange.Redis.RedisKey destinationKey, StackExchange.Redis.ListSide sourceSide, StackExchange.Redis.ListSide destinationSide, long count, StackExchange.Redis.ListMoveCount mode = StackExchange.Redis.ListMoveCount.UpTo, StackExchange.Redis.ListMoveOrder order = StackExchange.Redis.ListMoveOrder.Bulk, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! -StackExchange.Redis.IDatabaseAsync.SetCombineLengthAsync(StackExchange.Redis.SetOperation operation, StackExchange.Redis.RedisKey[]! keys, long limit = 0, bool approximate = false, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! -StackExchange.Redis.IDatabaseAsync.StreamReadAsync(StackExchange.Redis.StreamPosition[]! streamPositions, int? countPerStream = null, int? maxCount = null, int? maxSize = null, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! -StackExchange.Redis.IDatabaseAsync.StreamReadGroupAsync(StackExchange.Redis.StreamPosition[]! streamPositions, StackExchange.Redis.RedisValue groupName, StackExchange.Redis.RedisValue consumerName, int? countPerStream = null, bool noAck = false, System.TimeSpan? claimMinIdleTime = null, int? maxCount = null, int? maxSize = null, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! -StackExchange.Redis.ListMoveCount -StackExchange.Redis.ListMoveCount.Exactly = 1 -> StackExchange.Redis.ListMoveCount -StackExchange.Redis.ListMoveCount.UpTo = 0 -> StackExchange.Redis.ListMoveCount -StackExchange.Redis.ListMoveOrder -StackExchange.Redis.ListMoveOrder.Bulk = 0 -> StackExchange.Redis.ListMoveOrder -StackExchange.Redis.ListMoveOrder.OneByOne = 1 -> StackExchange.Redis.ListMoveOrder -[SER008]StackExchange.Redis.HashImport -[SER008]StackExchange.Redis.HashImport.Dispose() -> void -[SER008]StackExchange.Redis.HashImport.DisposeAsync() -> System.Threading.Tasks.ValueTask -[SER008]StackExchange.Redis.IDatabase.HashImport(StackExchange.Redis.RedisKey key, StackExchange.Redis.HashImport! fieldSet, System.ReadOnlyMemory values, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> void -[SER008]StackExchange.Redis.IDatabaseAsync.HashImportAsync(StackExchange.Redis.RedisKey key, StackExchange.Redis.HashImport! fieldSet, System.ReadOnlyMemory values, StackExchange.Redis.CommandFlags flags = StackExchange.Redis.CommandFlags.None) -> System.Threading.Tasks.Task! -[SER008]static StackExchange.Redis.HashImport.Create(params StackExchange.Redis.RedisValue[]! fields) -> StackExchange.Redis.HashImport! -[SER008]static StackExchange.Redis.HashImport.Create(System.ReadOnlyMemory fields) -> StackExchange.Redis.HashImport! -StackExchange.Redis.RedisFeatures.HashImport.get -> bool diff --git a/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Shipped.txt b/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Shipped.txt index bec516e5c..cec3e2742 100644 --- a/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Shipped.txt +++ b/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Shipped.txt @@ -3,3 +3,8 @@ StackExchange.Redis.ConfigurationOptions.SslClientAuthenticationOptions.get -> S StackExchange.Redis.ConfigurationOptions.SslClientAuthenticationOptions.set -> void System.Runtime.CompilerServices.IsExternalInit (forwarded, contained in System.Runtime) StackExchange.Redis.ConfigurationOptions.SetUserPemCertificate(string! userCertificatePath, string? userKeyPath = null) -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.DisposeAsync() -> System.Threading.Tasks.ValueTask +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Read(System.Span buffer) -> int +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.ReadAsync(System.Memory buffer, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.ValueTask +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.Write(System.ReadOnlySpan buffer) -> void +override StackExchange.Redis.Configuration.LoggingTunnel.LoggingDuplexStream.WriteAsync(System.ReadOnlyMemory buffer, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.ValueTask diff --git a/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt b/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt new file mode 100644 index 000000000..ab058de62 --- /dev/null +++ b/src/StackExchange.Redis/PublicAPI/net6.0/PublicAPI.Unshipped.txt @@ -0,0 +1 @@ +#nullable enable diff --git a/src/StackExchange.Redis/RedisBatch.cs b/src/StackExchange.Redis/RedisBatch.cs index 9863bc018..a7783edcd 100644 --- a/src/StackExchange.Redis/RedisBatch.cs +++ b/src/StackExchange.Redis/RedisBatch.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis { @@ -8,7 +9,14 @@ internal sealed class RedisBatch : RedisDatabase, IBatch { private List? pending; - public RedisBatch(RedisDatabase wrapped, object? asyncState) : base(wrapped.multiplexer, wrapped.Database, asyncState ?? wrapped.AsyncState) { } + public RedisBatch(RedisDatabase wrapped, object? asyncState) + : base(wrapped.multiplexer, wrapped.Database, asyncState ?? wrapped.AsyncState) + { + wrapped.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction); + } + + private protected override DatabaseFeatureFlags GetDatabaseFeatures() + => base.GetDatabaseFeatures() | DatabaseFeatureFlags.Batch; public void Execute() { @@ -123,13 +131,13 @@ internal override Task ExecuteAsync(Message? message, ResultProcessor? internal override T ExecuteSync(Message? message, ResultProcessor? processor, ServerEndPoint? server = null, T? defaultValue = default) where T : default => throw new NotSupportedException("ExecuteSync cannot be used inside a batch"); - private static void FailNoServer(ConnectionMultiplexer muxer, List messages) + private static void FailNoServer(ConnectionMultiplexer muxer, List? messages) { - if (messages == null) return; + if (messages is null) return; foreach (var msg in messages) { msg.Fail(ConnectionFailureType.UnableToResolvePhysicalConnection, null, "unable to write batch", muxer); - msg.Complete(); + msg.Complete(null); } } } diff --git a/src/StackExchange.Redis/RedisDatabase.cs b/src/StackExchange.Redis/RedisDatabase.cs index 1ac58b4a9..4e2dd61be 100644 --- a/src/StackExchange.Redis/RedisDatabase.cs +++ b/src/StackExchange.Redis/RedisDatabase.cs @@ -5,12 +5,14 @@ using System.Net; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Threading; using System.Threading.Tasks; using RESPite.Messages; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis { - internal partial class RedisDatabase : RedisBase, IDatabase + internal partial class RedisDatabase : RedisBase, IDatabase, IInternalDatabaseAsync { internal RedisDatabase(ConnectionMultiplexer multiplexer, int db, object? asyncState) : base(multiplexer, asyncState) @@ -22,6 +24,20 @@ internal RedisDatabase(ConnectionMultiplexer multiplexer, int db, object? asyncS public int Database { get; } + DatabaseFeatureFlags IInternalDatabaseAsync.GetFeatures(out string name) + { + name = multiplexer.ClientName; + return GetDatabaseFeatures(); + } + + CancellationToken IInternalDatabaseAsync.GetNextFailover() => CancellationToken.None; + + // the base feature set for this database; wrapping databases (batch, transaction, ...) override + // to fold in their own flag. Cluster is intrinsic to the underlying connection, so it lives here. + private protected virtual DatabaseFeatureFlags GetDatabaseFeatures() + => multiplexer.ServerSelectionStrategy.ServerType == ServerType.Cluster + ? DatabaseFeatureFlags.Cluster : DatabaseFeatureFlags.None; + public IBatch CreateBatch(object? asyncState) { if (this is IBatch) throw new NotSupportedException("Nested batches are not supported"); @@ -34,6 +50,10 @@ public ITransaction CreateTransaction(object? asyncState) return new RedisTransaction(this, asyncState); } + // the async-only IDatabaseAsync slot; the return type differs (ITransactionAsync vs ITransaction), + // so it does not implicitly bind to the public method above + ITransactionAsync IDatabaseAsync.CreateTransaction(object? asyncState) => CreateTransaction(asyncState); + private ITransaction? CreateTransactionIfAvailable(object? asyncState) { var map = multiplexer.CommandMap; diff --git a/src/StackExchange.Redis/RedisErrorKind.cs b/src/StackExchange.Redis/RedisErrorKind.cs new file mode 100644 index 000000000..2a60f76f6 --- /dev/null +++ b/src/StackExchange.Redis/RedisErrorKind.cs @@ -0,0 +1,235 @@ +using System; +using System.Buffers; +using System.Diagnostics.CodeAnalysis; +using System.Text; +using RESPite; +using RESPite.Messages; + +namespace StackExchange.Redis; + +/// +/// Well-known server error conditions, identified from the error-reply prefix (and in some cases the +/// message text). Used to classify faults consistently - in particular to decide whether a fault is +/// transient (worth retrying / awaiting failover) or permanent (retrying will never help). +/// +[Experimental(Experiments.GeoRedundantFailover, UrlFormat = Experiments.UrlFormat)] +public enum RedisErrorKind +{ + /// + /// No error condition; the reply was not an error. + /// + [AsciiHash("")] + None = 0, + + /// + /// The error was not recognized as one of the well-known conditions and did not start with ERR. + /// + [AsciiHash("")] + Unknown, + + /// + /// The error was not recognized as one of the well-known conditions, but started with ERR. + /// + [AsciiHash("")] + UnknownError, + + /// + /// The error was due to a connection fault, categorized separately by . + /// + [AsciiHash("")] + ConnectionFault, + + /// + /// The operation timed out; this is a client-side condition and does not correspond to a server reply. + /// + [AsciiHash("")] + Timeout, + + // --- availability / typically transient (retry or failover may recover) --- + + /// + /// LOADING - the server is still loading its dataset into memory and is not yet ready. + /// + Loading, + + /// + /// CLUSTERDOWN - the cluster is down; the hash slot is not currently being served. + /// + ClusterDown, + + /// + /// MASTERDOWN - the link with the primary is down and replica-serve-stale-data is no. + /// + MasterDown, + + /// + /// TRYAGAIN - a multi-key operation spans slots that are being migrated; the client should retry. + /// + TryAgain, + + /// + /// NOREPLICAS - not enough healthy replicas to satisfy the configured min-replicas-to-write. + /// + NoReplicas, + + /// + /// MISCONF - RDB/AOF persistence is misconfigured and writes are currently refused. + /// + [AsciiHash("MISCONF")] + Misconfigured, + + /// + /// OOM - the command was refused because used memory is above the maxmemory limit. + /// + [AsciiHash("OOM")] + OutOfMemory, + + /// + /// BUSY - a script or function is running and blocking the server (needs SCRIPT KILL etc.). + /// + Busy, + + /// + /// MAXCLIENTS - the configured maximum number of client connections has been reached. + /// + MaxClients, + + // --- cluster slot routing --- + + /// + /// MOVED - the hash slot has permanently moved to another endpoint. + /// + Moved, + + /// + /// ASK - the hash slot is temporarily served by another endpoint during migration. + /// + Ask, + + /// + /// CROSSSLOT - the keys in a multi-key operation do not all hash to the same slot. + /// + CrossSlot, + + // --- authentication / authorization (will not recover without a credential or ACL change) --- + + /// + /// NOAUTH - authentication is required before commands can be issued. + /// + NoAuth, + + /// + /// WRONGPASS - the supplied username/password pair was rejected. + /// + WrongPass, + + /// + /// NOPERM - the authenticated ACL user is not permitted to run this command/key/channel. + /// + [AsciiHash("NOPERM")] + NoPermission, + + /// + /// ERR operation not permitted - the operation is not permitted in the current context. + /// + [AsciiHash("")] // matched via the "ERR " branch, not the first token + NotPermitted, + + // --- client / usage errors (permanent - retry and failover will never help) --- + + /// + /// ERR unknown command - the command is not known to the server (typo, disabled, or unsupported version). + /// + [AsciiHash("ERR")] // matched via the "ERR " branch, not the first token + UnknownCommand, + + /// + /// WRONGTYPE - the operation was applied against a key holding an incompatible value type. + /// + WrongType, + + /// + /// EXECABORT - the transaction was discarded because of an earlier error while queuing commands. + /// + ExecAbort, + + /// + /// READONLY - a write was attempted against a read-only replica. + /// + ReadOnly, + + /// + /// NOSCRIPT - no matching script for EVALSHA; the script must be re-loaded. + /// + NoScript, +} + +internal static partial class RedisErrorKindMetadata +{ + [AsciiHash(CaseSensitive = false)] + private static partial bool TryParseFirstTokenCI(ReadOnlySpan value, out RedisErrorKind kind); + + /// + /// Classifies the error currently held by . The caller is assumed to have + /// already established that this is an error, so an unrecognized reply yields + /// (never ). + /// + internal static unsafe RedisErrorKind Classify(in RespReader reader) + => reader.TryParseScalar(&TryParse, out RedisErrorKind kind) ? kind : RedisErrorKind.Unknown; + + internal static unsafe RedisErrorKind Classify(string message) + => RespReader.TryParseScalar(message.AsSpan(), &TryParse, out RedisErrorKind kind) ? kind : RedisErrorKind.Unknown; + + internal static bool TryParse(ReadOnlySpan value, out RedisErrorKind kind) + { + var space = value.IndexOf((byte)' '); + // Deal with the exact matches on the first token first - many errors may or may not + // have descriptive text after the leading token. + var firstToken = space > 0 ? value.Slice(0, space) : value; + if (TryParseFirstTokenCI(firstToken, out kind)) + { + // check for more specific "ERR ..." scenarios + if (kind is RedisErrorKind.UnknownError & space > 0) + { + // get the message text after "ERR " + value = value.Slice(space + 1); + + // some ERR conditions can be identified further, noting that the text may or + // may not have some tokens - sometimes we need partial match. + var valueHash = AsciiHash.HashUC(value); + if (value.Length is OperationNotPermitted.Length && OperationNotPermitted.IsCI(value, valueHash)) + { + kind = RedisErrorKind.NotPermitted; + } + else if (value.Length >= UnknownCommand.Length && + AsciiHash.SequenceEqualsCI(value.Slice(0, UnknownCommand.Length), UnknownCommand.U8)) + { + kind = RedisErrorKind.UnknownCommand; + } + } + return true; + } + + kind = value.IsEmpty ? RedisErrorKind.None : RedisErrorKind.Unknown; + return true; + } + + [AsciiHash("operation not permitted")] + private static partial class OperationNotPermitted { } + + [AsciiHash("unknown command")] + private static partial class UnknownCommand { } + + /* while these are recognizable, we issue SELECT *on behalf* of the user + and react internally, so reporting them seems... unnecessary. + + [AsciiHash("DB index is out of range")] + private static partial class DbIndexOutOfRange { } + + [AsciiHash("invalid DB index")] + private static partial class InvalidDbIndex { } + + [AsciiHash("SELECT is not allowed in cluster mode")] + private static partial class SelectNotAllowedInClusterMode { } + */ +} diff --git a/src/StackExchange.Redis/RedisFeatures.cs b/src/StackExchange.Redis/RedisFeatures.cs index b5233d843..a2a0c0726 100644 --- a/src/StackExchange.Redis/RedisFeatures.cs +++ b/src/StackExchange.Redis/RedisFeatures.cs @@ -245,7 +245,7 @@ public RedisFeatures(Version version) public bool ScriptingDatabaseSafe => Version.IsAtLeast(v2_8_12); /// - [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(HyperLogLogCountReplicaSafe) + " instead, this will be removed in 3.0.")] + [Obsolete("Starting with Redis version 5, Redis has moved to 'replica' terminology. Please use " + nameof(HyperLogLogCountReplicaSafe) + " instead, this will be removed in 3.2.", error: true)] [Browsable(false), EditorBrowsable(EditorBrowsableState.Never)] public bool HyperLogLogCountSlaveSafe => HyperLogLogCountReplicaSafe; diff --git a/src/StackExchange.Redis/RedisResult.cs b/src/StackExchange.Redis/RedisResult.cs index 78ed649bc..1d496468f 100644 --- a/src/StackExchange.Redis/RedisResult.cs +++ b/src/StackExchange.Redis/RedisResult.cs @@ -558,7 +558,10 @@ internal override RedisValue AsRedisValue() private sealed class ErrorRedisResult : RedisResult { private readonly string value; + private RedisErrorKind kind; // lazily parsed from the value + private RedisErrorKind Kind => kind is RedisErrorKind.None ? GetKind() : kind; + private RedisErrorKind GetKind() => kind = RedisErrorKindMetadata.Classify(value); public ErrorRedisResult(string? value, ResultType type) : base(type) { this.value = value ?? throw new ArgumentNullException(nameof(value)); @@ -570,30 +573,30 @@ public ErrorRedisResult(string? value, ResultType type) : base(type) type = null; return value; } - internal override bool AsBoolean() => throw new RedisServerException(value); - internal override bool[] AsBooleanArray() => throw new RedisServerException(value); - internal override byte[] AsByteArray() => throw new RedisServerException(value); - internal override byte[][] AsByteArrayArray() => throw new RedisServerException(value); - internal override double AsDouble() => throw new RedisServerException(value); - internal override double[] AsDoubleArray() => throw new RedisServerException(value); - internal override int AsInt32() => throw new RedisServerException(value); - internal override int[] AsInt32Array() => throw new RedisServerException(value); - internal override long AsInt64() => throw new RedisServerException(value); - internal override ulong AsUInt64() => throw new RedisServerException(value); - internal override long[] AsInt64Array() => throw new RedisServerException(value); - internal override ulong[] AsUInt64Array() => throw new RedisServerException(value); - internal override bool? AsNullableBoolean() => throw new RedisServerException(value); - internal override double? AsNullableDouble() => throw new RedisServerException(value); - internal override int? AsNullableInt32() => throw new RedisServerException(value); - internal override long? AsNullableInt64() => throw new RedisServerException(value); - internal override ulong? AsNullableUInt64() => throw new RedisServerException(value); - internal override RedisKey AsRedisKey() => throw new RedisServerException(value); - internal override RedisKey[] AsRedisKeyArray() => throw new RedisServerException(value); - internal override RedisResult[] AsRedisResultArray() => throw new RedisServerException(value); - internal override RedisValue AsRedisValue() => throw new RedisServerException(value); - internal override RedisValue[] AsRedisValueArray() => throw new RedisServerException(value); - internal override string? AsString() => throw new RedisServerException(value); - internal override string?[]? AsStringArray() => throw new RedisServerException(value); + internal override bool AsBoolean() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override bool[] AsBooleanArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override byte[] AsByteArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override byte[][] AsByteArrayArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override double AsDouble() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override double[] AsDoubleArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override int AsInt32() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override int[] AsInt32Array() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override long AsInt64() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override ulong AsUInt64() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override long[] AsInt64Array() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override ulong[] AsUInt64Array() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override bool? AsNullableBoolean() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override double? AsNullableDouble() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override int? AsNullableInt32() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override long? AsNullableInt64() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override ulong? AsNullableUInt64() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override RedisKey AsRedisKey() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override RedisKey[] AsRedisKeyArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override RedisResult[] AsRedisResultArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override RedisValue AsRedisValue() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override RedisValue[] AsRedisValueArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override string? AsString() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); + internal override string?[]? AsStringArray() => throw new RedisServerException(Kind, CommandFlags.CommandRetryNever, value); } private sealed class SingleRedisResult : RedisResult, IConvertible diff --git a/src/StackExchange.Redis/RedisServer.cs b/src/StackExchange.Redis/RedisServer.cs index 61918347f..86ffa894a 100644 --- a/src/StackExchange.Redis/RedisServer.cs +++ b/src/StackExchange.Redis/RedisServer.cs @@ -53,6 +53,18 @@ public bool AllowReplicaWrites public Version Version => server.Version; + public RedisKey InventKey(RedisKey prefix = default) + { + var guid = Guid.NewGuid(); + if (server.ServerType is ServerType.Cluster) + { + var hashTag = multiplexer.ServerSelectionStrategy.GetHashTag(server); + if (string.IsNullOrEmpty(hashTag)) return RedisKey.Null; + return prefix.Append($"{guid}:{{{hashTag}}}"); + } + return prefix.Append(guid.ToString()); + } + public void ClientKill(EndPoint endpoint, CommandFlags flags = CommandFlags.None) { var msg = Message.Create(-1, flags, RedisCommand.CLIENT, RedisLiterals.KILL, Format.ToString(endpoint).AsRedisValue()); diff --git a/src/StackExchange.Redis/RedisTransaction.cs b/src/StackExchange.Redis/RedisTransaction.cs index 0c369e988..b23ade3d5 100644 --- a/src/StackExchange.Redis/RedisTransaction.cs +++ b/src/StackExchange.Redis/RedisTransaction.cs @@ -5,17 +5,48 @@ using System.Threading; using System.Threading.Tasks; using RESPite.Messages; +using StackExchange.Redis.Interfaces; namespace StackExchange.Redis { - internal sealed class RedisTransaction : RedisDatabase, ITransaction + internal sealed class RedisTransaction : RedisDatabase, ITransaction, IInternalTransaction { private List? _conditions; private List? _pending; + private TransactionMessage? _lastMessage; private object SyncLock => this; + /// + // set by TransactionProcessor when the server answers EXEC with a null array + public bool WasWatchConflict => _lastMessage?.WasWatchConflict == true; + + // combine the retry categories of all queued operations, taking the most side-effecting (numerically + // highest) - this is what a replay of the whole transaction would do. WATCH constraints are *not* + // included: they live in _conditions, not _pending, and re-issuing them is what makes replay safe. + CommandFlags IInternalTransaction.GetAggregateRetryCategory() + { + var result = CommandFlags.None; + lock (SyncLock) + { + var list = _pending; + if (list != null) + { + foreach (var q in list) + { + var cat = q.Wrapped.Flags & Message.MaskRetryCategory; + if (cat > result) result = cat; + } + } + } + return result; + } + + private protected override DatabaseFeatureFlags GetDatabaseFeatures() + => base.GetDatabaseFeatures() | DatabaseFeatureFlags.Transaction; + public RedisTransaction(RedisDatabase wrapped, object? asyncState) : base(wrapped.multiplexer, wrapped.Database, asyncState ?? wrapped.AsyncState) { + wrapped.RejectFlags(DatabaseFeatureFlags.Batch | DatabaseFeatureFlags.Transaction); // need to check we can reliably do this... var commandMap = multiplexer.CommandMap; commandMap.AssertAvailable(RedisCommand.MULTI); @@ -169,7 +200,7 @@ private void QueueMessage(Message message) } processor = TransactionProcessor.Default; - return new TransactionMessage(Database, flags, cond, work); + return _lastMessage = new TransactionMessage(Database, flags, cond, work); } private sealed class QueuedMessage : Message @@ -237,21 +268,32 @@ public TransactionMessage(int db, CommandFlags flags, List? con this.conditions = (conditions?.Count > 0) ? conditions.ToArray() : Array.Empty(); } - internal override void SetExceptionAndComplete(Exception exception, PhysicalBridge? bridge) + internal override void SetExceptionAndComplete(Exception exception, PhysicalConnection? connection) { var inner = InnerOperations; - if (inner != null) + // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract + if (inner is not null) { for (int i = 0; i < inner.Length; i++) { - inner[i]?.Wrapped?.SetExceptionAndComplete(exception, bridge); + inner[i]?.Wrapped?.SetExceptionAndComplete(exception, connection); } } - base.SetExceptionAndComplete(exception, bridge); + base.SetExceptionAndComplete(exception, connection); } public bool IsAborted => command != RedisCommand.EXEC; + // the server rejected an EXEC we really did issue, because a watched key moved; volatile + // because it is written on the read loop and observed by whoever awaited the transaction + public bool WasWatchConflict + { + get => Volatile.Read(ref _watchConflict); + set => Volatile.Write(ref _watchConflict, value); + } + + private bool _watchConflict; + public override void AppendStormLog(StringBuilder sb) { base.AppendStormLog(sb); @@ -432,7 +474,7 @@ public IEnumerable GetMessages(PhysicalConnection connection) { var inner = op.Wrapped; inner.Cancel(); - inner.Complete(); + inner.Complete(connection); } } connection.Trace("End of transaction: " + Command); @@ -479,12 +521,13 @@ public override bool SetResult(PhysicalConnection connection, Message message, r reader.MovePastBof(); if (reader.IsError && message is TransactionMessage tran) { + var errorKind = RedisErrorKindMetadata.Classify(reader); string error = reader.ReadString()!; foreach (var op in tran.InnerOperations) { var inner = op.Wrapped; - ServerFail(inner, error); - inner.Complete(); + ServerFail(inner, errorKind, error); + inner.Complete(connection); } } return base.SetResult(connection, message, ref copy); @@ -500,16 +543,23 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes if (reader.IsNull) // EXEC returned with a NULL { - if (tran.IsAborted) + // the server refused to apply the transaction because a watched key changed + // ("WATCH drift"); nothing was applied, so every queued operation must be + // brought to a terminal state - otherwise the caller's tasks hang forever. + // (in the electively-aborted case they were already cancelled in GetMessages; + // Cancel/Complete are idempotent, so doing it again is harmless) + muxer?.OnTransactionLog("Aborting wrapped messages (failed watch)"); + connection.Trace("Server aborted due to failed WATCH"); + + // distinguish "the server refused an EXEC we issued" from "we chose not to issue + // one"; only the former is worth re-attempting (see IInternalTransaction) + tran.WasWatchConflict = !tran.IsAborted; + + foreach (var op in wrapped) { - muxer?.OnTransactionLog("Aborting wrapped messages (failed watch)"); - connection.Trace("Server aborted due to failed WATCH"); - foreach (var op in wrapped) - { - var inner = op.Wrapped; - inner.Cancel(); - inner.Complete(); - } + var inner = op.Wrapped; + inner.Cancel(); + inner.Complete(connection); } SetResult(message, false); return true; @@ -539,7 +589,7 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes muxer?.OnTransactionLog($"> got {iter.Value.GetOverview()} for {inner.CommandAndKey}"); if (inner.ComputeResult(connection, ref iter.Value)) { - inner.Complete(); + inner.Complete(connection); } } @@ -553,10 +603,10 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes // the pending tasks foreach (var op in wrapped) { - if (op?.Wrapped is Message inner) + if (op?.Wrapped is { } inner) { inner.Fail(ConnectionFailureType.ProtocolFailure, null, "Transaction failure", muxer); - inner.Complete(); + inner.Complete(connection); } } } diff --git a/src/StackExchange.Redis/RespReaderExtensions.cs b/src/StackExchange.Redis/RespReaderExtensions.cs index 2107d0397..5683208eb 100644 --- a/src/StackExchange.Redis/RespReaderExtensions.cs +++ b/src/StackExchange.Redis/RespReaderExtensions.cs @@ -234,13 +234,6 @@ internal bool AnyNull() } } -#if !NET - extension(Task task) - { - public bool IsCompletedSuccessfully => task.Status is TaskStatus.RanToCompletion; - } -#endif - private static readonly int MaxCanonicalLength = Math.Max(Format.MaxInt64TextLen, Format.MaxDoubleTextLen); // Recognizes the canonical decimal text of an integer (signed Int64 or, for non-negative values up to diff --git a/src/StackExchange.Redis/ResultBox.cs b/src/StackExchange.Redis/ResultBox.cs index 20b76ba15..160493cc1 100644 --- a/src/StackExchange.Redis/ResultBox.cs +++ b/src/StackExchange.Redis/ResultBox.cs @@ -6,6 +6,7 @@ namespace StackExchange.Redis { internal interface IResultBox { + Exception? Fault { get; } bool IsAsync { get; } bool IsFaulted { get; } void SetException(Exception ex); @@ -24,6 +25,7 @@ internal abstract class SimpleResultBox : IResultBox bool IResultBox.IsAsync => false; bool IResultBox.IsFaulted => _exception != null; + Exception? IResultBox.Fault => _exception; void IResultBox.SetException(Exception exception) => _exception = exception ?? CancelledException; void IResultBox.Cancel() => _exception = CancelledException; @@ -95,6 +97,7 @@ private TaskResultBox(object? asyncState, TaskCreationOptions creationOptions) : bool IResultBox.IsAsync => true; bool IResultBox.IsFaulted => _exception != null; + Exception? IResultBox.Fault => _exception; void IResultBox.Cancel() => _exception = SimpleResultBox.CancelledException; diff --git a/src/StackExchange.Redis/ResultProcessor.Literals.cs b/src/StackExchange.Redis/ResultProcessor.Literals.cs index 7d50cc09e..3deaa7027 100644 --- a/src/StackExchange.Redis/ResultProcessor.Literals.cs +++ b/src/StackExchange.Redis/ResultProcessor.Literals.cs @@ -8,16 +8,6 @@ internal partial class Literals { #pragma warning disable CS8981, SA1300, SA1134 // forgive naming etc // ReSharper disable InconsistentNaming - [AsciiHash] internal static partial class NOAUTH { } - [AsciiHash] internal static partial class WRONGPASS { } - [AsciiHash] internal static partial class NOSCRIPT { } - [AsciiHash] internal static partial class MOVED { } - [AsciiHash] internal static partial class ASK { } - [AsciiHash] internal static partial class READONLY { } - [AsciiHash] internal static partial class LOADING { } - [AsciiHash("ERR operation not permitted")] - internal static partial class ERR_not_permitted { } - // Result processor literals [AsciiHash] internal static partial class OK diff --git a/src/StackExchange.Redis/ResultProcessor.cs b/src/StackExchange.Redis/ResultProcessor.cs index 14acb54c0..45a09cfbc 100644 --- a/src/StackExchange.Redis/ResultProcessor.cs +++ b/src/StackExchange.Redis/ResultProcessor.cs @@ -210,15 +210,15 @@ public void ConnectionFail(Message message, ConnectionFailureType fail, Exceptio sb.Append(", "); sb.Append(annotation); } - var ex = new RedisConnectionException(fail, sb.ToString(), innerException); + var ex = new RedisConnectionException(fail, message?.Flags ?? CommandFlags.None, sb.ToString(), innerException); SetException(message, ex); } public static void ConnectionFail(Message message, ConnectionFailureType fail, string errorMessage) => - SetException(message, new RedisConnectionException(fail, errorMessage)); + SetException(message, new RedisConnectionException(fail, message.Flags, errorMessage)); - public static void ServerFail(Message message, string errorMessage) => - SetException(message, new RedisServerException(errorMessage)); + public static void ServerFail(Message message, RedisErrorKind kind, string errorMessage) => + SetException(message, new RedisServerException(kind, message.Flags, errorMessage)); public static void SetException(Message? message, Exception ex) { @@ -263,22 +263,24 @@ private bool HandleCommonError(Message message, RespReader reader, PhysicalConne { connection.OnDetailLog($"applying common error-handling: {reader.GetOverview()}"); var bridge = connection.BridgeCouldBeNull; - if (reader.StartsWith(Literals.NOAUTH.U8)) + var errorKind = RedisErrorKindMetadata.Classify(reader); + switch (errorKind) { - bridge?.Multiplexer.SetAuthSuspect(new RedisServerException("NOAUTH Returned - connection has not yet authenticated")); - } - else if (reader.StartsWith(Literals.WRONGPASS.U8)) - { - bridge?.Multiplexer.SetAuthSuspect(new RedisServerException(reader.GetOverview())); + case RedisErrorKind.NoAuth: + bridge?.Multiplexer.SetAuthSuspect(new RedisServerException(errorKind, message.Flags, "NOAUTH Returned - connection has not yet authenticated")); + break; + case RedisErrorKind.WrongPass: + bridge?.Multiplexer.SetAuthSuspect(new RedisServerException(errorKind, message.Flags, reader.GetOverview())); + break; } var server = bridge?.ServerEndPoint; bool log = !message.IsInternalCall; - bool isMoved = reader.StartsWith(Literals.MOVED.U8); + bool isMoved = errorKind == RedisErrorKind.Moved; bool wasNoRedirect = (message.Flags & CommandFlags.NoRedirect) != 0; string? err = string.Empty; bool unableToConnectError = false; - if (isMoved || reader.StartsWith(Literals.ASK.U8)) + if (isMoved || errorKind == RedisErrorKind.Ask) { connection.OnDetailLog($"redirect via {(isMoved ? "MOVED" : "ASK")} to '{reader.ReadString()}'"); message.SetResponseReceived(); @@ -361,7 +363,7 @@ private bool HandleCommonError(Message message, RespReader reader, PhysicalConne } else { - ServerFail(message, err); + ServerFail(message, errorKind, err); } return true; @@ -852,7 +854,7 @@ public override bool SetResult(PhysicalConnection connection, Message message, r { var copy = reader; reader.MovePastBof(); - if (reader.IsError && reader.StartsWith(Literals.READONLY.U8)) + if (reader.IsError && RedisErrorKindMetadata.Classify(reader) == RedisErrorKind.ReadOnly) { var bridge = connection.BridgeCouldBeNull; if (bridge != null) @@ -2249,7 +2251,7 @@ public override bool SetResult(PhysicalConnection connection, Message message, r { var copy = reader; reader.MovePastBof(); - if (reader.IsError && reader.StartsWith(Literals.NOSCRIPT.U8)) + if (reader.IsError && RedisErrorKindMetadata.Classify(reader) == RedisErrorKind.NoScript) { // scripts are not flushed individually, so assume the entire script cache is toast ("SCRIPT FLUSH") connection.BridgeCouldBeNull?.ServerEndPoint?.FlushScriptCache(); message.SetScriptUnavailable(); @@ -3152,38 +3154,33 @@ protected override bool SetResultCore(PhysicalConnection connection, Message mes } } - private sealed class TracerProcessor : ResultProcessor + private sealed class TracerProcessor(bool establishConnection) : ResultProcessor { - private readonly bool establishConnection; - - public TracerProcessor(bool establishConnection) - { - this.establishConnection = establishConnection; - } - public override bool SetResult(PhysicalConnection connection, Message message, ref RespReader reader) { reader.MovePastBof(); bool isError = reader.IsError; var copy = reader; + connection.BridgeCouldBeNull?.ServerEndPoint?.SetLatency(message.CreatedDateTime); connection.BridgeCouldBeNull?.Multiplexer.OnInfoMessage($"got '{reader.Prefix}' for '{message.CommandAndKey}' on '{connection}'"); var final = base.SetResult(connection, message, ref reader); if (isError) { reader = copy; // rewind and re-parse - if (reader.StartsWith(Literals.ERR_not_permitted.U8) || reader.StartsWith(Literals.NOAUTH.U8)) + var errorKind = RedisErrorKindMetadata.Classify(reader); + if (errorKind is RedisErrorKind.NotPermitted or RedisErrorKind.NoAuth) { connection.RecordConnectionFailed(ConnectionFailureType.AuthenticationFailure, new Exception(reader.GetOverview() + " Verify if the Redis password provided is correct. Attempted command: " + message.Command)); } - else if (reader.StartsWith(Literals.LOADING.U8)) + else if (errorKind == RedisErrorKind.Loading) { connection.RecordConnectionFailed(ConnectionFailureType.Loading); } else { - connection.RecordConnectionFailed(ConnectionFailureType.ProtocolFailure, new RedisServerException(reader.GetOverview())); + connection.RecordConnectionFailed(ConnectionFailureType.ProtocolFailure, new RedisServerException(RedisErrorKind.ConnectionFault, message.Flags, reader.GetOverview())); } } diff --git a/src/StackExchange.Redis/ServerEndPoint.cs b/src/StackExchange.Redis/ServerEndPoint.cs index f191c1c3b..202eaf876 100644 --- a/src/StackExchange.Redis/ServerEndPoint.cs +++ b/src/StackExchange.Redis/ServerEndPoint.cs @@ -10,6 +10,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using StackExchange.Redis.Availability; using static StackExchange.Redis.PhysicalBridge; namespace StackExchange.Redis @@ -1144,6 +1145,20 @@ internal bool HasPendingCallerFacingItems() return subscription?.HasPendingCallerFacingItems() ?? false; } + public void SetLatency(DateTime startTime) + { + try + { + LatencyTicks = ConnectionGroupMember.ToLatencyTicks(DateTime.UtcNow - startTime); + } + catch (Exception ex) + { + Debug.WriteLine(ex.Message); + } + } + + internal uint LatencyTicks { get; private set; } = uint.MaxValue; + private ProductVariant _productVariant = ProductVariant.Redis; private string _productVersion = ""; diff --git a/src/StackExchange.Redis/ServerSelectionStrategy.HashTags.cs b/src/StackExchange.Redis/ServerSelectionStrategy.HashTags.cs new file mode 100644 index 000000000..4cbd4538c --- /dev/null +++ b/src/StackExchange.Redis/ServerSelectionStrategy.HashTags.cs @@ -0,0 +1,86 @@ +using System; +using System.Diagnostics; +using System.Text; + +namespace StackExchange.Redis; + +internal sealed partial class ServerSelectionStrategy +{ + // pre-computed hash-tags for each slot + private static class HashTags + { + private static readonly string[] Cache = Populate(); + public static ReadOnlySpan Tags => Cache; + public static string Get(int slot) => Cache[slot]; + + private static string[] Populate() + { + // Via testing, we know that 3 characters is sufficient to populate all slots + // using a total of 48643 operations - this is acceptable (same order-of-magnitude as the slot count). + var slots = new string?[TotalSlots]; + + // using an alphabet of the visible ASCII characters, excluding { and } (used to denote hash-tags) + ReadOnlySpan alphabet = "!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz|~"u8; + Debug.WriteLine($"Alphabet: '{Encoding.ASCII.GetString(alphabet)}', {alphabet.Length} chars"); + + Span threeChars = stackalloc byte[3]; + Span twoChars = threeChars.Slice(0, 2); + Span oneChar = threeChars.Slice(0, 1); + + int operations = 0; + int remaining = slots.Length; + + bool Test(ReadOnlySpan span) + { + var slot = GetClusterSlot(span); + operations++; + if (slots[slot] is { } existing) + { + // prefer smaller tags (but doesn't change the outcome) + if (span.Length < existing.Length) + { + slots[slot] = Encoding.ASCII.GetString(span); + } + } + else + { + // new value for this slot + slots[slot] = Encoding.ASCII.GetString(span); + return --remaining == 0; + } + + return false; + } + for (int i = 0; i < alphabet.Length; i++) + { + oneChar[0] = alphabet[i]; + + // Test single character keys + if (i == 0) Test(oneChar); + + for (int j = 0; j < alphabet.Length; j++) + { + twoChars[1] = alphabet[j]; + + // Test two characters keys + if (i == 0) Test(twoChars); + + for (int k = 0; k < alphabet.Length; k++) + { + threeChars[2] = alphabet[k]; + + // Test three characters - we know this is the only possible exit location + if (Test(threeChars)) + { + Debug.WriteLine($"Populated all hash-tag slots in {operations} operations"); + return slots!; + } + } + } + } + + throw new InvalidOperationException( + $"Failed to populate hash-tag cache after {operations} operations, {remaining} slots remaining"); + } + } +} diff --git a/src/StackExchange.Redis/ServerSelectionStrategy.cs b/src/StackExchange.Redis/ServerSelectionStrategy.cs index 5025b732d..80c9b9efe 100644 --- a/src/StackExchange.Redis/ServerSelectionStrategy.cs +++ b/src/StackExchange.Redis/ServerSelectionStrategy.cs @@ -8,7 +8,7 @@ namespace StackExchange.Redis { - internal sealed class ServerSelectionStrategy + internal sealed partial class ServerSelectionStrategy { public const int NoSlot = -1, MultipleSlots = -2; private const int RedisClusterSlotCount = 16384; @@ -52,9 +52,9 @@ internal sealed class ServerSelectionStrategy private int anyStartOffset = SharedRandom.Next(); // initialize to a random value so routing isn't uniform #if NET - private static Random SharedRandom => Random.Shared; + internal static Random SharedRandom => Random.Shared; #else - private static Random SharedRandom { get; } = new(); + internal static Random SharedRandom { get; } = new(); #endif private ServerEndPoint[]? map; @@ -418,5 +418,29 @@ internal bool CanServeSlot(ServerEndPoint server, int slot) } return false; } + + /// + /// Gets a string that can be used as a hash-tag to reference a specific slot. + /// + internal string GetHashTag(ServerEndPoint endpoint) + { + if (map is { } arr) + { + // inefficient way of finding a slot for a given endpoint, but: it'll work + for (int i = 0; i < arr.Length; i++) + { + if (arr[i] == endpoint) + { + return HashTags.Get(i); + } + } + } + return ""; + } + + /// + /// Gets a string that can be used as a hash-tag to reference a specific slot. + /// + internal static string GetHashTag(int slot) => slot < 0 ? "" : HashTags.Get(slot); } } diff --git a/src/StackExchange.Redis/StackExchange.Redis.csproj b/src/StackExchange.Redis/StackExchange.Redis.csproj index 705b07eec..51d4ca902 100644 --- a/src/StackExchange.Redis/StackExchange.Redis.csproj +++ b/src/StackExchange.Redis/StackExchange.Redis.csproj @@ -60,4 +60,19 @@ + + + + MultiGroupDatabase.cs + + + HealthCheck.cs + + + HealthCheckProbe.cs + + + MultiGroupSubscriber.cs + + \ No newline at end of file diff --git a/src/StackExchange.Redis/TaskExtensions.cs b/src/StackExchange.Redis/TaskExtensions.cs index 989c0861e..344dff75c 100644 --- a/src/StackExchange.Redis/TaskExtensions.cs +++ b/src/StackExchange.Redis/TaskExtensions.cs @@ -26,6 +26,11 @@ internal static Task ObserveErrors(this Task task) } #if !NET + extension(Task task) + { + public bool IsCompletedSuccessfully => task.Status is TaskStatus.RanToCompletion; + } + // suboptimal polyfill version of the .NET 6+ API, but reasonable for light use internal static Task WaitAsync(this Task task, CancellationToken cancellationToken) { diff --git a/tests/StackExchange.Redis.Tests/ActiveActiveIntegrationTests.cs b/tests/StackExchange.Redis.Tests/ActiveActiveIntegrationTests.cs new file mode 100644 index 000000000..406999a27 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/ActiveActiveIntegrationTests.cs @@ -0,0 +1,45 @@ +using System; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using StackExchange.Redis.Tests.Helpers; +using Xunit; + +namespace StackExchange.Redis.Tests; + +public class ActiveActiveIntegrationTests(ITestOutputHelper output) +{ + [Fact] + public async Task ProductionActiveActive() + { + var config = TestConfig.Current; + var eps = config.ActiveActiveEndpoints; + Assert.SkipUnless(eps is { Length: > 0 }, "no active:active endpoints"); + + var writer = new TextWriterOutputHelper(output); + ConnectionGroupMember[] members = Array.ConvertAll(eps, x => new ConnectionGroupMember(x)); + var muxer = await ConnectionMultiplexer.ConnectGroupAsync(members, log: writer); + + Assert.True(muxer.IsConnected); + var db = muxer.GetDatabase(); + Assert.True(db.IsConnected(default)); + await db.PingAsync(); + + Task last = Task.CompletedTask; + var ttl = TimeSpan.FromMinutes(5); + for (int i = 0; i < 100; i++) + { + RedisKey key = Guid.NewGuid().ToString(); + _ = db.StringSetAsync(key, i, ttl, flags: CommandFlags.FireAndForget); + last.RedisFireAndForget(); // in case of fault + last = db.StringGetAsync(key); // observe the last + } + + await last; + + foreach (var member in muxer.GetMembers()) + { + output?.WriteLine($"{member.Name}: {member.Latency.TotalMilliseconds}us"); + } + output?.WriteLine($"Active: {muxer.ActiveMember?.Name}"); + } +} diff --git a/tests/StackExchange.Redis.Tests/AvailabilityConfigTests.cs b/tests/StackExchange.Redis.Tests/AvailabilityConfigTests.cs new file mode 100644 index 000000000..4328fcc58 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/AvailabilityConfigTests.cs @@ -0,0 +1,333 @@ +using System; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// Covers the shape shared by every Availability configuration type: an immutable policy with static +/// Default/None, configured through a nested Builder that validates in Create() and collapses onto the +/// shared default when nothing was customized. +/// +public class AvailabilityConfigTests +{ + // ---- HealthCheck ---- + [Fact] + public void HealthCheck_UntouchedBuilder_CollapsesOntoDefault() + { + Assert.Same(HealthCheck.Default, new HealthCheck.Builder().Create()); + Assert.Same(HealthCheck.Default, new HealthCheck.Builder(HealthCheck.Default).Create()); + } + + [Fact] + public void HealthCheck_BuilderRoundTripsExistingInstance() + { + HealthCheck original = new HealthCheck.Builder + { + ProbeCount = 7, + ProbeTimeout = TimeSpan.FromSeconds(11), + ProbeInterval = TimeSpan.FromMilliseconds(250), + Probe = HealthCheckProbe.IsConnected, + ProbePolicy = HealthCheckProbePolicy.MajoritySuccess, + }; + + // the copy constructor is the replacement for the old Clone() + var copy = new HealthCheck.Builder(original).Create(); + + Assert.NotSame(original, copy); + Assert.Equal(original.ProbeCount, copy.ProbeCount); + Assert.Equal(original.ProbeTimeout, copy.ProbeTimeout); + Assert.Equal(original.ProbeInterval, copy.ProbeInterval); + Assert.Same(original.Probe, copy.Probe); + Assert.Same(original.ProbePolicy, copy.ProbePolicy); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void HealthCheck_RejectsNonPositiveProbeCount(int probeCount) + { + var builder = new HealthCheck.Builder { ProbeCount = probeCount }; + var ex = Assert.Throws(() => builder.Create()); + Assert.Equal(nameof(HealthCheck.Builder.ProbeCount), ex.ParamName); + } + + [Fact] + public void HealthCheck_RejectsNonPositiveProbeTimeout() + { + var builder = new HealthCheck.Builder { ProbeTimeout = TimeSpan.Zero }; + var ex = Assert.Throws(() => builder.Create()); + Assert.Equal(nameof(HealthCheck.Builder.ProbeTimeout), ex.ParamName); + } + + [Fact] + public void HealthCheck_RejectsNegativeProbeInterval() + { + var builder = new HealthCheck.Builder { ProbeInterval = TimeSpan.FromMilliseconds(-1) }; + var ex = Assert.Throws(() => builder.Create()); + Assert.Equal(nameof(HealthCheck.Builder.ProbeInterval), ex.ParamName); + } + + [Fact] + public void HealthCheck_RejectsUnrepresentableTotalBudget() + { + // ProbeCount x ProbeTimeout has to fit in int milliseconds; this used to overflow silently + var builder = new HealthCheck.Builder { ProbeCount = 1000, ProbeTimeout = TimeSpan.FromDays(30) }; + Assert.Throws(() => builder.Create()); + } + + [Fact] + public void HealthCheck_None_IsDisabledAndStable() + { + Assert.Same(HealthCheck.None, HealthCheck.None); + Assert.NotSame(HealthCheck.None, HealthCheck.Default); + Assert.False(HealthCheck.None.IsEnabled); + Assert.True(HealthCheck.Default.IsEnabled); + } + + [Fact] + public async Task HealthCheck_None_ReportsInconclusiveWithoutProbing() + { + // a null server would throw if the probe were actually invoked + Assert.Equal(HealthCheckResult.Inconclusive, await HealthCheck.None.CheckHealthAsync(server: null!)); + } + + // ---- RetryPolicy ---- + [Fact] + public void RetryPolicy_UntouchedBuilder_CollapsesOntoDefault() + { + Assert.Same(RetryPolicy.Default, new RetryPolicy.Builder().Create()); + Assert.Same(RetryPolicy.Default, new RetryPolicy.Builder(RetryPolicy.Default).Create()); + } + + [Fact] + public void RetryPolicy_BuilderRoundTripsExistingInstance() + { + RetryPolicy original = new RetryPolicy.Builder + { + MaxAttempts = 9, + MaxAttemptsBeforeFailover = 4, + RetryDelay = TimeSpan.FromMilliseconds(123), + JitterMax = TimeSpan.FromMilliseconds(45), + FailoverDelay = TimeSpan.FromSeconds(6), + MaxCommandRetryCategory = CommandFlags.CommandRetryWriteAccumulating, + }; + + var copy = new RetryPolicy.Builder(original).Create(); + + Assert.NotSame(original, copy); + Assert.Equal(original.MaxAttempts, copy.MaxAttempts); + Assert.Equal(original.MaxAttemptsBeforeFailover, copy.MaxAttemptsBeforeFailover); + Assert.Equal(original.RetryDelay, copy.RetryDelay); + Assert.Equal(original.JitterMax, copy.JitterMax); + Assert.Equal(original.FailoverDelay, copy.FailoverDelay); + Assert.Equal(original.MaxCommandRetryCategory, copy.MaxCommandRetryCategory); + } + + [Fact] + public void RetryPolicy_RejectsZeroAttempts() + { + var builder = new RetryPolicy.Builder { MaxAttempts = 0 }; + var ex = Assert.Throws(() => builder.Create()); + Assert.Equal(nameof(RetryPolicy.Builder.MaxAttempts), ex.ParamName); + } + + [Fact] + public void RetryPolicy_RejectsZeroAttemptsBeforeFailover() + { + // previously this silently disabled failover, and only threw later, from WithRetry + var builder = new RetryPolicy.Builder { MaxAttemptsBeforeFailover = 0 }; + var ex = Assert.Throws(() => builder.Create()); + Assert.Equal(nameof(RetryPolicy.Builder.MaxAttemptsBeforeFailover), ex.ParamName); + } + + [Fact] + public void RetryPolicy_RejectsNegativeDelays() + { + Assert.Equal( + nameof(RetryPolicy.Builder.RetryDelay), + Assert.Throws(() => new RetryPolicy.Builder { RetryDelay = TimeSpan.FromTicks(-1) }.Create()).ParamName); + Assert.Equal( + nameof(RetryPolicy.Builder.JitterMax), + Assert.Throws(() => new RetryPolicy.Builder { JitterMax = TimeSpan.FromTicks(-1) }.Create()).ParamName); + Assert.Equal( + nameof(RetryPolicy.Builder.FailoverDelay), + Assert.Throws(() => new RetryPolicy.Builder { FailoverDelay = TimeSpan.FromTicks(-1) }.Create()).ParamName); + } + + [Theory] + [InlineData(CommandFlags.None)] // no category at all + [InlineData(CommandFlags.FireAndForget)] // not a category + [InlineData(CommandFlags.CommandRetryReadOnly | CommandFlags.PreferReplica)] // category plus noise + public void RetryPolicy_RejectsInvalidRetryCategory(CommandFlags flags) + { + var builder = new RetryPolicy.Builder { MaxCommandRetryCategory = flags }; + var ex = Assert.Throws(() => builder.Create()); + Assert.Equal(nameof(RetryPolicy.Builder.MaxCommandRetryCategory), ex.ParamName); + } + + [Fact] + public void RetryPolicy_None_NeverRetries() + { + Assert.Same(RetryPolicy.None, RetryPolicy.None); + Assert.NotSame(RetryPolicy.None, RetryPolicy.Default); + + // a transient, retryable fault on a read-only command: the default policy retries, None does not + var fault = new FaultContext(new RedisConnectionException(ConnectionFailureType.SocketFailure, CommandFlags.None, "boom")); + Assert.Equal(RetryResult.None, RetryPolicy.None.CanRetry(in fault)); + } + + // ---- CircuitBreaker ---- + [Fact] + public void CircuitBreaker_RejectsOutOfRangeThreshold() + { + Assert.Equal( + nameof(CircuitBreaker.Builder.FailureRateThreshold), + Assert.Throws(() => new CircuitBreaker.Builder { FailureRateThreshold = 101 }.Create()).ParamName); + Assert.Equal( + nameof(CircuitBreaker.Builder.FailureRateThreshold), + Assert.Throws(() => new CircuitBreaker.Builder { FailureRateThreshold = -1 }.Create()).ParamName); + } + + [Fact] + public void CircuitBreaker_RejectsInvalidWindowAndMinimum() + { + Assert.Equal( + nameof(CircuitBreaker.Builder.MinimumNumberOfFailures), + Assert.Throws(() => new CircuitBreaker.Builder { MinimumNumberOfFailures = 0 }.Create()).ParamName); + Assert.Equal( + nameof(CircuitBreaker.Builder.MetricsWindowSize), + Assert.Throws(() => new CircuitBreaker.Builder { MetricsWindowSize = TimeSpan.Zero }.Create()).ParamName); + } + + // ---- MultiGroupOptions ---- + [Fact] + public void MultiGroupOptions_UntouchedBuilder_CollapsesOntoDefault() + { + Assert.Same(MultiGroupOptions.Default, new MultiGroupOptions.Builder().Create()); + Assert.Same(MultiGroupOptions.Default, new MultiGroupOptions.Builder(MultiGroupOptions.Default).Create()); + } + + [Fact] + public void MultiGroupOptions_DefaultsAreTheSharedPolicyDefaults() + { + var options = MultiGroupOptions.Default; + Assert.Same(HealthCheck.Default, options.HealthCheck); + Assert.Same(CircuitBreaker.Default, options.CircuitBreaker); + Assert.Same(RetryPolicy.Default, options.RetryPolicy); + Assert.Equal(TimeSpan.FromSeconds(5), options.HealthCheckInterval); + Assert.Equal(TimeSpan.Zero, options.FailbackDelay); + } + + [Fact] + public void MultiGroupOptions_RejectsInvalidIntervals() + { + Assert.Equal( + nameof(MultiGroupOptions.Builder.HealthCheckInterval), + Assert.Throws(() => new MultiGroupOptions.Builder { HealthCheckInterval = TimeSpan.Zero }.Create()).ParamName); + Assert.Equal( + nameof(MultiGroupOptions.Builder.FailbackDelay), + Assert.Throws(() => new MultiGroupOptions.Builder { FailbackDelay = TimeSpan.FromTicks(-1) }.Create()).ParamName); + + // MaxValue is the documented "never" sentinel for both, and must remain legal + MultiGroupOptions ok = new MultiGroupOptions.Builder + { + HealthCheckInterval = TimeSpan.MaxValue, + FailbackDelay = TimeSpan.MaxValue, + }; + Assert.Equal(TimeSpan.MaxValue, ok.HealthCheckInterval); + Assert.Equal(TimeSpan.MaxValue, ok.FailbackDelay); + } + + [Fact] + public void MultiGroupOptions_BuilderConvertsImplicitly() + { + // every Builder in the namespace supports this, so options can be written inline at the call-site + MultiGroupOptions options = new MultiGroupOptions.Builder { FailbackDelay = TimeSpan.FromMinutes(2) }; + Assert.Equal(TimeSpan.FromMinutes(2), options.FailbackDelay); + } + + // ---- per-member override resolution ---- + [Fact] + public void Member_ResolvesGroupDefaultsWhenNoOverride() + { + var member = new ConnectionGroupMember("localhost:6379"); + var options = MultiGroupOptions.Default; + + Assert.Same(options.HealthCheck, member.ResolveHealthCheck(options)); + Assert.Same(options.CircuitBreaker, member.ResolveCircuitBreaker(options)); + Assert.Equal(options.FailbackDelay, member.ResolveFailbackDelay(options)); + } + + [Fact] + public void Member_OverridesBeatGroupDefaults() + { + HealthCheck memberCheck = new HealthCheck.Builder { ProbeCount = 1 }; + CircuitBreaker memberBreaker = new CircuitBreaker.Builder { FailureRateThreshold = 42 }; + var member = new ConnectionGroupMember("localhost:6379") + { + HealthCheck = memberCheck, + CircuitBreaker = memberBreaker, + FailbackDelay = TimeSpan.FromMinutes(3), + }; + + var options = MultiGroupOptions.Default; + Assert.Same(memberCheck, member.ResolveHealthCheck(options)); + Assert.Same(memberBreaker, member.ResolveCircuitBreaker(options)); + Assert.Equal(TimeSpan.FromMinutes(3), member.ResolveFailbackDelay(options)); + } + + [Fact] + public void Member_CircuitBreakerFallsBackToItsOwnConfigurationBeforeTheGroup() + { + // precedence is: member override, then the member's own ConfigurationOptions, then the group default + CircuitBreaker fromConfig = new CircuitBreaker.Builder { FailureRateThreshold = 42 }; + var config = ConfigurationOptions.Parse("localhost:6379"); + config.CircuitBreaker = fromConfig; + + var member = new ConnectionGroupMember(config); + Assert.Same(fromConfig, member.ResolveCircuitBreaker(MultiGroupOptions.Default)); + + CircuitBreaker fromMember = new CircuitBreaker.Builder { FailureRateThreshold = 13 }; + member.CircuitBreaker = fromMember; + Assert.Same(fromMember, member.ResolveCircuitBreaker(MultiGroupOptions.Default)); + } + + [Fact] + public void GroupDefaultsAreNotWrittenBackIntoCallerConfiguration() + { + // callers may legitimately reuse a ConfigurationOptions across connections, so resolving a group + // default must not mutate it (this used to be a `config.CircuitBreaker ??= options.CircuitBreaker`) + var config = ConfigurationOptions.Parse("localhost:6379"); + var member = new ConnectionGroupMember(config); + + Assert.Same(MultiGroupOptions.Default.CircuitBreaker, member.ResolveCircuitBreaker(MultiGroupOptions.Default)); + Assert.Null(config.CircuitBreaker); + } + + // ---- WithRetry() policy resolution ---- + [Fact] + public async Task WithRetry_UsesConfiguredPolicyForASingleConnection() + { + RetryPolicy configured = new RetryPolicy.Builder { MaxAttempts = 7 }; + var config = ConfigurationOptions.Parse("localhost:6379"); + config.RetryPolicy = configured; + config.AbortOnConnectFail = false; + + await using var muxer = await ConnectionMultiplexer.ConnectAsync(config); + var retrying = Assert.IsType(muxer.GetDatabase().WithRetry()); + Assert.Same(configured, retrying.Policy); + } + + [Fact] + public async Task WithRetry_FallsBackToDefaultWhenNoneConfigured() + { + var config = ConfigurationOptions.Parse("localhost:6379"); + config.AbortOnConnectFail = false; + + await using var muxer = await ConnectionMultiplexer.ConnectAsync(config); + var retrying = Assert.IsType(muxer.GetDatabase().WithRetry()); + Assert.Same(RetryPolicy.Default, retrying.Policy); + } +} diff --git a/tests/StackExchange.Redis.Tests/BufferedStreamWriterTests.cs b/tests/StackExchange.Redis.Tests/BufferedStreamWriterTests.cs index 932153db0..e1c9bceeb 100644 --- a/tests/StackExchange.Redis.Tests/BufferedStreamWriterTests.cs +++ b/tests/StackExchange.Redis.Tests/BufferedStreamWriterTests.cs @@ -215,7 +215,7 @@ private sealed class ObservedStream : Stream private TaskCompletionSource? _blockedFlush, _allowFlush; public Exception? WriteException { get; set; } - public long BytesWritten => Interlocked.Read(ref _bytesWritten); + public long BytesWritten => Volatile.Read(ref _bytesWritten); public int SyncWriteCount => Volatile.Read(ref _syncWriteCount); public int AsyncWriteCount => Volatile.Read(ref _asyncWriteCount); public byte[] WrittenBytes diff --git a/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs b/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs new file mode 100644 index 000000000..81ca78ee0 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/CircuitBreakerServerTests.cs @@ -0,0 +1,88 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using Xunit; + +namespace StackExchange.Redis.Tests; + +public class CircuitBreakerServerTests(ITestOutputHelper output) : TestBase(output) +{ + [Fact] + public async Task CircuitBreakerObservesMessageResults() + { + using var server = new InProcessTestServer(); + + // take the template options from the server, and slot in our test breaker *before* connecting, + // so every physical connection yanks an accumulator from it during init + var config = server.GetClientConfig(); + var breaker = new CountingCircuitBreaker(); + config.CircuitBreaker = breaker; + + using var client = await ConnectionMultiplexer.ConnectAsync(config); + var db = client.GetDatabase(); + + // some successful operations (these, plus handshake traffic, count as non-fault observations) + RedisKey key = Me(); + await db.StringSetAsync(key, "abc"); + Assert.Equal("abc", await db.StringGetAsync(key)); + await db.StringGetAsync(key); + + var successesAfterGetSets = breaker.Successes; + + // knock the server offline: it now replies LOADING to every command, which (unlike an + // application-level "unknown command") is a genuine availability fault the breaker observes. + // flip it straight back off so no background heartbeat can observe a second LOADING reply - + // the fault for our command is recorded synchronously as it completes, before the await returns. + server.IsLoading = true; + var fault = await Assert.ThrowsAsync(() => db.StringGetAsync(key)); + server.IsLoading = false; + Assert.Equal(RedisErrorKind.Loading, fault.Kind); + Output.WriteLine($"loading fault: {fault.GetType().Name}: {fault.Message}"); + + Output.WriteLine($"observed successes={breaker.Successes}, failures={breaker.Failures}, lastFault={breaker.LastFault?.GetType().Name}"); + + // the get/sets were observed as successes + Assert.True(successesAfterGetSets > 0, "expected the successful operations to be observed"); + + // exactly one fault (the LOADING reply), captured as the clean server error. A healthy + // breaker must NOT tear the connection down: a regression there shows up here as a + // RedisConnectionException (and extra faults) rather than this RedisServerException. + Assert.Equal(1, breaker.Failures); + var serverFault = Assert.IsType(breaker.LastFault); + Assert.Equal(RedisErrorKind.Loading, serverFault.Kind); + } + + // a minimal breaker for tests: shares counters across all accumulators it creates, so we can + // observe traffic across every physical connection; it never trips (always reports healthy) + private sealed class CountingCircuitBreaker : CircuitBreaker + { + private int _successes, _failures; + + public int Successes => Volatile.Read(ref _successes); + public int Failures => Volatile.Read(ref _failures); + public Exception? LastFault { get; private set; } + + public override Accumulator CreateAccumulator() => new CountingAccumulator(this); + + private sealed class CountingAccumulator(CountingCircuitBreaker owner) : Accumulator + { + public override void ObserveResult(in FaultContext context) + { + if (context.IsFault) + { + Interlocked.Increment(ref owner._failures); + owner.LastFault = context.Fault; + } + else + { + Interlocked.Increment(ref owner._successes); + } + } + + public override bool IsHealthy() => true; + + public override void Reset() { } + } + } +} diff --git a/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs b/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs new file mode 100644 index 000000000..ab0b15545 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/CircuitBreakerUnitTests.cs @@ -0,0 +1,193 @@ +using System; +using StackExchange.Redis.Availability; +using Xunit; +#if NET8_0_OR_GREATER +using System.Threading; +#endif + +namespace StackExchange.Redis.Tests; + +public class CircuitBreakerUnitTests +{ + [Fact] + public void Builder_AllDefaults_ReturnsSharedDefaultInstance() + { + // a builder that hasn't been touched should collapse onto the shared default instance... + var a = new CircuitBreaker.Builder().Create(); + var b = new CircuitBreaker.Builder().Create(); + + Assert.Same(a, b); + Assert.Same(CircuitBreaker.Default, a); + } + + [Fact] + public void Builder_NonDefaults_ReturnsDistinctValidInstances() + { + // ...but as soon as any knob is changed, we get a fresh, distinct instance per Create() + CircuitBreaker.Builder Configured() => new() { FailureRateThreshold = 42 }; + + var a = Configured().Create(); + var b = Configured().Create(); + + Assert.NotNull(a); + Assert.NotNull(b); + Assert.NotSame(a, b); + Assert.NotSame(CircuitBreaker.Default, a); + } + + [Fact] + public void None_IsDistinctFromDefault_ButStable() + { + Assert.NotNull(CircuitBreaker.None); + Assert.Same(CircuitBreaker.None, CircuitBreaker.None); + Assert.NotSame(CircuitBreaker.Default, CircuitBreaker.None); + } + + [Fact] + public void None_IsAlwaysHealthy() + { + var acc = CircuitBreaker.None.CreateAccumulator(); + // even a solid wall of tracked failures never trips the no-op breaker + Assert.True(Record(acc, 10_000, new RedisTimeoutException(CommandFlags.None, "boom", CommandStatus.Unknown))); + } + +#if NET8_0_OR_GREATER + // the time-windowed logic needs a controllable clock; TimeProvider is only available on net8.0+, + // and we don't want to pull the BCL shim in just for down-level test coverage. + [Fact] + public void BelowMinimumFailures_StaysHealthy() + { + var time = new ManualTimeProvider(); + // threshold is trivially low (1%), so the *only* thing keeping us healthy is the minimum-count gate + var acc = Build(time, failureRateThreshold: 1, minimumNumberOfFailures: 10).CreateAccumulator(); + + // nine tracked failures: one short of the minimum, so we withhold judgement + Assert.True(Record(acc, 9, Timeout())); + } + + [Fact] + public void AboveThreshold_Trips() + { + var time = new ManualTimeProvider(); + var acc = Build(time, failureRateThreshold: 50, minimumNumberOfFailures: 10).CreateAccumulator(); + + // 20 tracked failures, 0 successes -> 100% failure rate, well past both gates + Assert.False(Record(acc, 20, Timeout())); + } + + [Fact] + public void BelowThreshold_StaysHealthy() + { + var time = new ManualTimeProvider(); + var acc = Build(time, failureRateThreshold: 50, minimumNumberOfFailures: 10).CreateAccumulator(); + + Record(acc, 10, Timeout()); // enough failures to clear the minimum-count gate + Record(acc, 190); // but drowned out by successes -> 5% failure rate + + // a pure health read confirms we're comfortably under the 50% threshold + Assert.True(acc.IsHealthy()); + } + + [Fact] + public void UntrackedExceptions_CountAsSuccess() + { + var time = new ManualTimeProvider(); + // default tracking set (null) == RedisConnectionException + RedisTimeoutException only + var acc = Build(time, failureRateThreshold: 1, minimumNumberOfFailures: 1).CreateAccumulator(); + + // a flood of *untracked* failures must not trip the breaker... + Assert.True(Record(acc, 100, new InvalidOperationException("not tracked"))); + + // ...whereas the same volume of tracked failures does + Assert.False(Record(acc, 100, Timeout())); + } + + [Fact] + public void OldFailures_AgeOutOfTheWindow() + { + var time = new ManualTimeProvider(); + var acc = Build(time, failureRateThreshold: 50, minimumNumberOfFailures: 1).CreateAccumulator(); + + // saturate the window with failures -> tripped + Assert.False(Record(acc, 100, Timeout())); + + // step past the whole window; the earlier failures should no longer count + time.Advance(TimeSpan.FromSeconds(11)); + + // the window is now empty of in-range failures -> healthy again + Assert.True(acc.IsHealthy()); + } + + [Fact] + public void IsHealthy_ReflectsStateWithoutObserving() + { + var time = new ManualTimeProvider(); + var acc = Build(time, failureRateThreshold: 50, minimumNumberOfFailures: 1).CreateAccumulator(); + + Assert.True(acc.IsHealthy()); // nothing observed yet + + Assert.False(Record(acc, 100, Timeout())); // trip it via observations + + // the context-free overload reports the same verdict, purely by reading the window + Assert.False(acc.IsHealthy()); + } + + [Fact] + public void Reset_DiscardsHistory() + { + var time = new ManualTimeProvider(); + var acc = Build(time, failureRateThreshold: 50, minimumNumberOfFailures: 1).CreateAccumulator(); + + // trip it wide open... + Assert.False(Record(acc, 100, Timeout())); + + // ...then wipe the slate; the prior failures are forgotten + acc.Reset(); + + // an empty window reads as healthy again + Assert.True(acc.IsHealthy()); + } + + private static CircuitBreaker Build( + TimeProvider time, + double failureRateThreshold, + int minimumNumberOfFailures) + => new CircuitBreaker.Builder + { + FailureRateThreshold = failureRateThreshold, + MinimumNumberOfFailures = minimumNumberOfFailures, + MetricsWindowSize = TimeSpan.FromSeconds(10), + TimeProvider = time, + }.Create(); + + /// + /// A hand-cranked whose clock only moves when we tell it to, + /// so the bucketed metrics window is fully deterministic. + /// + private sealed class ManualTimeProvider : TimeProvider + { + private long _timestamp; + + // one tick == 100ns, matching TimeSpan; keeps Advance(TimeSpan) a straight addition + public override long TimestampFrequency => TimeSpan.TicksPerSecond; + + public override long GetTimestamp() => Volatile.Read(ref _timestamp); + + public void Advance(TimeSpan by) => Interlocked.Add(ref _timestamp, by.Ticks); + } +#endif + + private static RedisTimeoutException Timeout() => new(CommandFlags.None, "timeout", CommandStatus.Unknown); + + private static bool Record(CircuitBreaker.Accumulator accumulator, int count, Exception? fault = null) + { + // Trip applies the IsFailure gate (a fault the breaker doesn't consider a failure is counted as a + // success), then records via ObserveResult; call it rather than ObserveResult directly so the gate runs + for (int i = 0; i < count; i++) + { + accumulator.Trip(fault); + } + + return accumulator.IsHealthy(); + } +} diff --git a/tests/StackExchange.Redis.Tests/ClusterTests.cs b/tests/StackExchange.Redis.Tests/ClusterTests.cs index 118e0f975..af58346e9 100644 --- a/tests/StackExchange.Redis.Tests/ClusterTests.cs +++ b/tests/StackExchange.Redis.Tests/ClusterTests.cs @@ -689,9 +689,9 @@ public async Task AccessRandomKeys() var counters = server.GetCounters(); Log(counters.ToString()); } - int final = Interlocked.CompareExchange(ref total, 0, 0); + int final = Volatile.Read(ref total); Assert.Equal(COUNT, final); - Assert.Equal(0, Interlocked.CompareExchange(ref slotMovedCount, 0, 0)); + Assert.Equal(0, Volatile.Read(ref slotMovedCount)); } [Theory] diff --git a/tests/StackExchange.Redis.Tests/ConfigTests.cs b/tests/StackExchange.Redis.Tests/ConfigTests.cs index 89ffd851e..00fe6ad1b 100644 --- a/tests/StackExchange.Redis.Tests/ConfigTests.cs +++ b/tests/StackExchange.Redis.Tests/ConfigTests.cs @@ -69,6 +69,7 @@ orderby name "CertificateSelection", "CertificateValidation", "ChannelPrefix", + "CircuitBreaker", "ClientName", "commandMap", "configChannel", @@ -93,6 +94,7 @@ orderby name "RequestBufferPool", "ResponseBufferPool", "responseTimeout", + "RetryPolicy", "ServiceName", "SocketManager", #if !NETFRAMEWORK @@ -728,6 +730,39 @@ public async Task BeforeSocketConnect() Assert.True(subscriptionSocket.DontFragment); } + /// + /// Reads a property that is obsolete-as-error, which the compiler will not let us name directly + /// (and #pragma warning disable cannot suppress, since it is an error rather than a warning). + /// + private static T GetObsoleteProperty(object target, string name) + { + var type = target.GetType(); + var property = type.GetProperty(name); + if (property is null) + { + throw new ArgumentException($"Property '{name}' was not found on '{type.FullName}'; has it been renamed or removed?", nameof(name)); + } + + var value = property.GetValue(target); + if (value is null) + { + // null is a perfectly good value unless the caller asked for a non-nullable value type + if (typeof(T).IsValueType && Nullable.GetUnderlyingType(typeof(T)) is null) + { + throw new ArgumentException($"Property '{type.FullName}.{name}' is null; expected '{typeof(T).Name}'.", nameof(name)); + } + + return default!; + } + + if (value is not T typed) + { + throw new ArgumentException($"Property '{type.FullName}.{name}' is of type '{property.PropertyType.Name}' with value '{value}'; expected '{typeof(T).Name}'.", nameof(name)); + } + + return typed; + } + [Fact] public async Task MutableOptions() { @@ -753,21 +788,21 @@ public async Task MutableOptions() options.CommandMap = CommandMap.Envoyproxy; Assert.NotSame(options.CommandMap, conn.CommandMap); -#pragma warning disable CS0618 // Type or member is obsolete + // note: the ConnectionMultiplexer-level versions of these are [Obsolete(..., error: true)], so they + // can only be reached via reflection - see GetObsoleteProperty // Defaults true Assert.True(options.IncludeDetailInExceptions); - Assert.True(conn.IncludeDetailInExceptions); + Assert.True(GetObsoleteProperty(conn, nameof(options.IncludeDetailInExceptions))); options.IncludeDetailInExceptions = false; Assert.False(options.IncludeDetailInExceptions); - Assert.False(conn.IncludeDetailInExceptions); + Assert.False(GetObsoleteProperty(conn, nameof(options.IncludeDetailInExceptions))); // Defaults false Assert.False(options.IncludePerformanceCountersInExceptions); - Assert.False(conn.IncludePerformanceCountersInExceptions); + Assert.False(GetObsoleteProperty(conn, nameof(options.IncludePerformanceCountersInExceptions))); options.IncludePerformanceCountersInExceptions = true; Assert.True(options.IncludePerformanceCountersInExceptions); - Assert.True(conn.IncludePerformanceCountersInExceptions); -#pragma warning restore CS0618 + Assert.True(GetObsoleteProperty(conn, nameof(options.IncludePerformanceCountersInExceptions))); var newName = Guid.NewGuid().ToString(); options.ClientName = newName; diff --git a/tests/StackExchange.Redis.Tests/ConnectionShutdownTests.cs b/tests/StackExchange.Redis.Tests/ConnectionShutdownTests.cs index 6fdc90cea..c05ecdec9 100644 --- a/tests/StackExchange.Redis.Tests/ConnectionShutdownTests.cs +++ b/tests/StackExchange.Redis.Tests/ConnectionShutdownTests.cs @@ -27,8 +27,8 @@ public async Task ShutdownRaisesConnectionFailureAndRestore() }; var db = conn.GetDatabase(); await db.PingAsync(); - Assert.Equal(0, Interlocked.CompareExchange(ref failed, 0, 0)); - Assert.Equal(0, Interlocked.CompareExchange(ref restored, 0, 0)); + Assert.Equal(0, Volatile.Read(ref failed)); + Assert.Equal(0, Volatile.Read(ref restored)); await Task.Delay(1).ForAwait(); // To make compiler happy in Release conn.AllowConnect = false; @@ -39,13 +39,13 @@ public async Task ShutdownRaisesConnectionFailureAndRestore() db.Ping(CommandFlags.FireAndForget); await Task.Delay(250).ForAwait(); - Assert.Equal(2, Interlocked.CompareExchange(ref failed, 0, 0)); - Assert.Equal(0, Interlocked.CompareExchange(ref restored, 0, 0)); + Assert.Equal(2, Volatile.Read(ref failed)); + Assert.Equal(0, Volatile.Read(ref restored)); conn.AllowConnect = true; db.Ping(CommandFlags.FireAndForget); await Task.Delay(1500).ForAwait(); - Assert.Equal(2, Interlocked.CompareExchange(ref failed, 0, 0)); - Assert.Equal(2, Interlocked.CompareExchange(ref restored, 0, 0)); + Assert.Equal(2, Volatile.Read(ref failed)); + Assert.Equal(2, Volatile.Read(ref restored)); watch.Stop(); } } diff --git a/tests/StackExchange.Redis.Tests/ControllableProbe.cs b/tests/StackExchange.Redis.Tests/ControllableProbe.cs new file mode 100644 index 000000000..4d5c5ae23 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/ControllableProbe.cs @@ -0,0 +1,18 @@ +using System.Net; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; + +namespace StackExchange.Redis.Tests; + +// A health-check probe whose verdict is driven by the test: nominated endpoints report unhealthy on +// demand, everything else is healthy. This keeps a specific member deselected deterministically, even +// after its physical connection reconnects underneath us. +internal sealed class ControllableProbe : HealthCheckProbe +{ + private volatile EndPoint? _down; + + public void MarkDown(EndPoint endpoint) => _down = endpoint; + + public override Task CheckHealthAsync(HealthCheckContext context) + => Equals(context.Server.EndPoint, _down) ? UnhealthyTask : HealthyTask; +} diff --git a/tests/StackExchange.Redis.Tests/DeprecatedTests.cs b/tests/StackExchange.Redis.Tests/DeprecatedTests.cs index ab909ea16..1da30ca4c 100644 --- a/tests/StackExchange.Redis.Tests/DeprecatedTests.cs +++ b/tests/StackExchange.Redis.Tests/DeprecatedTests.cs @@ -1,4 +1,5 @@ -using System; +using System; +using System.Reflection; using Xunit; namespace StackExchange.Redis.Tests; @@ -8,63 +9,75 @@ namespace StackExchange.Redis.Tests; /// public class DeprecatedTests(ITestOutputHelper output) : TestBase(output) { -#pragma warning disable CS0618 // Type or member is obsolete + // note: everything under test here is [Obsolete(..., error: true)], so it cannot be named directly - not + // even via nameof, and #pragma cannot suppress an error; reflection is the only way to reach these members + private static PropertyInfo AssertObsoleteAsError(string name) + { + var property = typeof(ConfigurationOptions).GetProperty(name) + ?? throw new MissingMemberException(nameof(ConfigurationOptions), name); + var obsolete = property.GetCustomAttribute(); + Assert.NotNull(obsolete); + Assert.True(obsolete.IsError, $"{name} should be obsolete as an error"); + return property; + } + + private static T Get(PropertyInfo property, ConfigurationOptions options) => (T)property.GetValue(options)!; + [Fact] public void HighPrioritySocketThreads() { - Assert.True(Attribute.IsDefined(typeof(ConfigurationOptions).GetProperty(nameof(ConfigurationOptions.HighPrioritySocketThreads))!, typeof(ObsoleteAttribute))); + var property = AssertObsoleteAsError("HighPrioritySocketThreads"); var options = ConfigurationOptions.Parse("name=Hello"); - Assert.False(options.HighPrioritySocketThreads); + Assert.False(Get(property, options)); options = ConfigurationOptions.Parse("highPriorityThreads=true"); Assert.Equal("", options.ToString()); - Assert.False(options.HighPrioritySocketThreads); + Assert.False(Get(property, options)); options = ConfigurationOptions.Parse("highPriorityThreads=false"); Assert.Equal("", options.ToString()); - Assert.False(options.HighPrioritySocketThreads); + Assert.False(Get(property, options)); } [Fact] public void PreserveAsyncOrder() { - Assert.True(Attribute.IsDefined(typeof(ConfigurationOptions).GetProperty(nameof(ConfigurationOptions.PreserveAsyncOrder))!, typeof(ObsoleteAttribute))); + var property = AssertObsoleteAsError("PreserveAsyncOrder"); var options = ConfigurationOptions.Parse("name=Hello"); - Assert.False(options.PreserveAsyncOrder); + Assert.False(Get(property, options)); options = ConfigurationOptions.Parse("preserveAsyncOrder=true"); Assert.Equal("", options.ToString()); - Assert.False(options.PreserveAsyncOrder); + Assert.False(Get(property, options)); options = ConfigurationOptions.Parse("preserveAsyncOrder=false"); Assert.Equal("", options.ToString()); - Assert.False(options.PreserveAsyncOrder); + Assert.False(Get(property, options)); } [Fact] public void WriteBufferParse() { - Assert.True(Attribute.IsDefined(typeof(ConfigurationOptions).GetProperty(nameof(ConfigurationOptions.WriteBuffer))!, typeof(ObsoleteAttribute))); + var property = AssertObsoleteAsError("WriteBuffer"); var options = ConfigurationOptions.Parse("name=Hello"); - Assert.Equal(0, options.WriteBuffer); + Assert.Equal(0, Get(property, options)); options = ConfigurationOptions.Parse("writeBuffer=8092"); - Assert.Equal(0, options.WriteBuffer); + Assert.Equal(0, Get(property, options)); } [Fact] public void ResponseTimeout() { - Assert.True(Attribute.IsDefined(typeof(ConfigurationOptions).GetProperty(nameof(ConfigurationOptions.ResponseTimeout))!, typeof(ObsoleteAttribute))); + var property = AssertObsoleteAsError("ResponseTimeout"); var options = ConfigurationOptions.Parse("name=Hello"); - Assert.Equal(0, options.ResponseTimeout); + Assert.Equal(0, Get(property, options)); options = ConfigurationOptions.Parse("responseTimeout=1000"); - Assert.Equal(0, options.ResponseTimeout); + Assert.Equal(0, Get(property, options)); } -#pragma warning restore CS0618 } diff --git a/tests/StackExchange.Redis.Tests/FailoverTests.cs b/tests/StackExchange.Redis.Tests/FailoverTests.cs index e1222394c..da470631c 100644 --- a/tests/StackExchange.Redis.Tests/FailoverTests.cs +++ b/tests/StackExchange.Redis.Tests/FailoverTests.cs @@ -89,7 +89,7 @@ public async Task ConfigVerifyReceiveConfigChangeBroadcast() await GetServer(receiverConn).PingAsync(); await Task.Delay(1000).ConfigureAwait(false); Assert.True(count == -1 || count >= 2, "subscribers"); - Assert.True(Interlocked.CompareExchange(ref total, 0, 0) >= 1, "total (1st)"); + Assert.True(Volatile.Read(ref total) >= 1, "total (1st)"); Interlocked.Exchange(ref total, 0); @@ -100,7 +100,7 @@ public async Task ConfigVerifyReceiveConfigChangeBroadcast() await Task.Delay(1000).ConfigureAwait(false); await GetServer(receiverConn).PingAsync(); await GetServer(receiverConn).PingAsync(); - Assert.True(Interlocked.CompareExchange(ref total, 0, 0) >= 1, "total (2nd)"); + Assert.True(Volatile.Read(ref total) >= 1, "total (2nd)"); } [Fact] @@ -339,17 +339,17 @@ public async Task SubscriptionsSurvivePrimarySwitchAsync() Log(" SubA ping: " + subA.Ping()); Log(" SubB ping: " + subB.Ping()); // If redis is under load due to this suite, it may take a moment to send across. - await UntilConditionAsync(TimeSpan.FromSeconds(5), () => Interlocked.Read(ref aCount) == 2 && Interlocked.Read(ref bCount) == 2).ForAwait(); + await UntilConditionAsync(TimeSpan.FromSeconds(5), () => Volatile.Read(ref aCount) == 2 && Volatile.Read(ref bCount) == 2).ForAwait(); - Assert.Equal(2, Interlocked.Read(ref aCount)); - Assert.Equal(2, Interlocked.Read(ref bCount)); - Assert.Equal(0, Interlocked.Read(ref primaryChanged)); + Assert.Equal(2, Volatile.Read(ref aCount)); + Assert.Equal(2, Volatile.Read(ref bCount)); + Assert.Equal(0, Volatile.Read(ref primaryChanged)); try { - Interlocked.Exchange(ref primaryChanged, 0); - Interlocked.Exchange(ref aCount, 0); - Interlocked.Exchange(ref bCount, 0); + Volatile.Write(ref primaryChanged, 0); + Volatile.Write(ref aCount, 0); + Volatile.Write(ref bCount, 0); Log("Changing primary..."); using (var sw = new StringWriter()) { @@ -410,17 +410,17 @@ public async Task SubscriptionsSurvivePrimarySwitchAsync() await subA.PingAsync(); await subB.PingAsync(); Log("Ping Complete. Checking..."); - await UntilConditionAsync(TimeSpan.FromSeconds(10), () => Interlocked.Read(ref aCount) == 2 && Interlocked.Read(ref bCount) == 2).ForAwait(); + await UntilConditionAsync(TimeSpan.FromSeconds(10), () => Volatile.Read(ref aCount) == 2 && Volatile.Read(ref bCount) == 2).ForAwait(); Log("Counts so far:"); - Log(" aCount: " + Interlocked.Read(ref aCount)); - Log(" bCount: " + Interlocked.Read(ref bCount)); - Log(" primaryChanged: " + Interlocked.Read(ref primaryChanged)); + Log(" aCount: " + Volatile.Read(ref aCount)); + Log(" bCount: " + Volatile.Read(ref bCount)); + Log(" primaryChanged: " + Volatile.Read(ref primaryChanged)); - Assert.Equal(2, Interlocked.Read(ref aCount)); - Assert.Equal(2, Interlocked.Read(ref bCount)); + Assert.Equal(2, Volatile.Read(ref aCount)); + Assert.Equal(2, Volatile.Read(ref bCount)); // Expect 12, because a sees a, but b sees a and b due to replication, but contenders may add their own - Assert.True(Interlocked.CompareExchange(ref primaryChanged, 0, 0) >= 12); + Assert.True(Volatile.Read(ref primaryChanged) >= 12); } catch { diff --git a/tests/StackExchange.Redis.Tests/HashTagUnitTests.cs b/tests/StackExchange.Redis.Tests/HashTagUnitTests.cs new file mode 100644 index 000000000..91b63d980 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/HashTagUnitTests.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Xunit; + +namespace StackExchange.Redis.Tests; + +public class HashTagUnitTests +{ + [Fact] + public void TestHashTagCoverage() + { + HashSet uniques = []; + Assert.Equal("", ServerSelectionStrategy.GetHashTag(ServerSelectionStrategy.NoSlot)); + Assert.Equal("", ServerSelectionStrategy.GetHashTag(ServerSelectionStrategy.MultipleSlots)); + Span buffer = stackalloc byte[3]; + for (int i = 0; i < ServerSelectionStrategy.TotalSlots; i++) + { + var tag = ServerSelectionStrategy.GetHashTag(i); + Assert.False(string.IsNullOrEmpty(tag)); + Assert.True(uniques.Add(tag)); + + var len = Encoding.ASCII.GetBytes(tag, buffer); + var slot = ServerSelectionStrategy.GetClusterSlot(buffer.Slice(0, len)); + Assert.Equal(i, slot); + } + Assert.Equal(ServerSelectionStrategy.TotalSlots, uniques.Count); + } +} diff --git a/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs b/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs new file mode 100644 index 000000000..c8a30030e --- /dev/null +++ b/tests/StackExchange.Redis.Tests/HealthCheckPolicyUnitTests.cs @@ -0,0 +1,108 @@ +using System; +using StackExchange.Redis.Availability; +using Xunit; + +namespace StackExchange.Redis.Tests; + +public class HealthCheckPolicyUnitTests +{ + [Theory] + [InlineData(0, 0, 5, HealthCheckResult.Inconclusive)] // No results yet + [InlineData(1, 0, 0, HealthCheckResult.Healthy)] // One success, no more probes + [InlineData(0, 1, 0, HealthCheckResult.Unhealthy)] // One failure, no more probes + [InlineData(2, 1, 0, HealthCheckResult.Healthy)] // Mixed results, success wins + [InlineData(1, 2, 0, HealthCheckResult.Healthy)] // Mixed results, success wins + [InlineData(5, 0, 0, HealthCheckResult.Healthy)] // All successes + [InlineData(0, 5, 0, HealthCheckResult.Unhealthy)] // All failures + [InlineData(1, 0, 2, HealthCheckResult.Healthy)] // Early success + [InlineData(0, 1, 2, HealthCheckResult.Inconclusive)] // Early failure but more probes remain + public void AnySuccess_EvaluatesCorrectly(int success, int failure, int remaining, HealthCheckResult expected) + { + var policy = HealthCheckProbePolicy.AnySuccess; + var context = new HealthCheckProbeContext(HealthCheckResult.Inconclusive, success, failure, remaining, TimeSpan.Zero); + + var result = policy.Evaluate(context); + + Assert.Equal(expected, result); + } + + [Theory] + [InlineData(0, 0, 5, HealthCheckResult.Inconclusive)] // No results yet + [InlineData(1, 0, 0, HealthCheckResult.Healthy)] // One success, no more probes + [InlineData(0, 1, 0, HealthCheckResult.Unhealthy)] // One failure, no more probes + [InlineData(2, 1, 0, HealthCheckResult.Unhealthy)] // Mixed results, one failure is enough + [InlineData(1, 2, 0, HealthCheckResult.Unhealthy)] // Mixed results, one failure is enough + [InlineData(5, 0, 0, HealthCheckResult.Healthy)] // All successes + [InlineData(0, 5, 0, HealthCheckResult.Unhealthy)] // All failures + [InlineData(1, 0, 2, HealthCheckResult.Inconclusive)] // Success but more probes remain + [InlineData(0, 1, 2, HealthCheckResult.Unhealthy)] // Early failure + [InlineData(4, 0, 1, HealthCheckResult.Inconclusive)] // Multiple successes but still waiting + public void AllSuccess_EvaluatesCorrectly(int success, int failure, int remaining, HealthCheckResult expected) + { + var policy = HealthCheckProbePolicy.AllSuccess; + var context = new HealthCheckProbeContext(HealthCheckResult.Inconclusive, success, failure, remaining, TimeSpan.Zero); + + var result = policy.Evaluate(context); + + Assert.Equal(expected, result); + } + + [Theory] + // Total 5 probes: need 3 for majority + [InlineData(0, 0, 5, HealthCheckResult.Inconclusive)] // No results yet + [InlineData(3, 0, 2, HealthCheckResult.Healthy)] // Reached majority (3/5) + [InlineData(2, 0, 3, HealthCheckResult.Inconclusive)] // Not yet majority + [InlineData(0, 3, 2, HealthCheckResult.Unhealthy)] // Majority impossible (3 failures) + [InlineData(2, 2, 1, HealthCheckResult.Inconclusive)] // Tied, one more probe + [InlineData(3, 2, 0, HealthCheckResult.Healthy)] // Majority achieved (3/5) + [InlineData(2, 3, 0, HealthCheckResult.Unhealthy)] // Majority failed (3/5) + [InlineData(5, 0, 0, HealthCheckResult.Healthy)] // All successes + [InlineData(0, 5, 0, HealthCheckResult.Unhealthy)] // All failures + + // Total 3 probes: need 2 for majority + [InlineData(0, 0, 3, HealthCheckResult.Inconclusive)] // No results yet (3 total) + [InlineData(2, 0, 1, HealthCheckResult.Healthy)] // Reached majority (2/3) + [InlineData(1, 0, 2, HealthCheckResult.Inconclusive)] // Not yet majority (3 total) + [InlineData(0, 2, 1, HealthCheckResult.Unhealthy)] // Majority impossible (2 failures of 3) + [InlineData(2, 1, 0, HealthCheckResult.Healthy)] // Majority achieved (2/3) + [InlineData(1, 2, 0, HealthCheckResult.Unhealthy)] // Majority failed (2/3) + + // Total 1 probe: need 1 for majority + [InlineData(0, 0, 1, HealthCheckResult.Inconclusive)] // No results yet (1 total) + [InlineData(1, 0, 0, HealthCheckResult.Healthy)] // Majority achieved (1/1) + [InlineData(0, 1, 0, HealthCheckResult.Unhealthy)] // Majority failed (1/1) + + // Total 6 probes: need 4 for majority + [InlineData(4, 0, 2, HealthCheckResult.Healthy)] // Reached majority (4/6) + [InlineData(3, 0, 3, HealthCheckResult.Inconclusive)] // Not yet majority (6 total) + [InlineData(0, 4, 2, HealthCheckResult.Unhealthy)] // Majority impossible (4 failures) + [InlineData(3, 3, 0, HealthCheckResult.Inconclusive)] // Tied, neither side has majority (3/6 is not >=4) + public void MajoritySuccess_EvaluatesCorrectly(int success, int failure, int remaining, HealthCheckResult expected) + { + var policy = HealthCheckProbePolicy.MajoritySuccess; + var context = new HealthCheckProbeContext(HealthCheckResult.Inconclusive, success, failure, remaining, TimeSpan.Zero); + + var result = policy.Evaluate(context); + + Assert.Equal(expected, result); + } + + [Fact] + public void Policies_AreSingletons() + { + var any1 = HealthCheckProbePolicy.AnySuccess; + var any2 = HealthCheckProbePolicy.AnySuccess; + Assert.NotNull(any1); + Assert.Same(any1, any2); + + var all1 = HealthCheckProbePolicy.AllSuccess; + var all2 = HealthCheckProbePolicy.AllSuccess; + Assert.NotNull(all1); + Assert.Same(all1, all2); + + var maj1 = HealthCheckProbePolicy.MajoritySuccess; + var maj2 = HealthCheckProbePolicy.MajoritySuccess; + Assert.NotNull(maj1); + Assert.Same(maj1, maj2); + } +} diff --git a/tests/StackExchange.Redis.Tests/Helpers/TestConfig.cs b/tests/StackExchange.Redis.Tests/Helpers/TestConfig.cs index c0194d5a6..59520eab4 100644 --- a/tests/StackExchange.Redis.Tests/Helpers/TestConfig.cs +++ b/tests/StackExchange.Redis.Tests/Helpers/TestConfig.cs @@ -130,5 +130,6 @@ public class Config public int ProxyPort { get; set; } = 7015; public string ProxyServerAndPort => ProxyServer + ":" + ProxyPort.ToString(); + public string[] ActiveActiveEndpoints { get; set; } = []; } } diff --git a/tests/StackExchange.Redis.Tests/InProcessTestServer.cs b/tests/StackExchange.Redis.Tests/InProcessTestServer.cs index 7dcc5f7c9..2f6833ace 100644 --- a/tests/StackExchange.Redis.Tests/InProcessTestServer.cs +++ b/tests/StackExchange.Redis.Tests/InProcessTestServer.cs @@ -196,10 +196,10 @@ public override void OnFlush(RedisClient client, int messages, long bytes) } */ - public override TypedRedisValue OnUnknownCommand(in RedisClient client, in RedisRequest request, ReadOnlySpan command) + public override TypedRedisValue OnUnknownCommand(RedisClient client, in RedisRequest request, ReadOnlySpan command) { _log?.WriteLine($"[{client}] unknown command: {Encoding.ASCII.GetString(command)}"); - return base.OnUnknownCommand(in client, in request, command); + return base.OnUnknownCommand(client, in request, command); } public override void OnClientConnected(RedisClient client, object state) @@ -403,4 +403,20 @@ protected override void Dispose(bool disposing) if (disposing) _server.Dispose(); } */ + // written by the test thread, read by the server's own thread(s): store as ticks so the read/write can + // be made explicitly visible (a plain TimeSpan field can be missed indefinitely) + public void SetLatency(TimeSpan latency) => Volatile.Write(ref _latencyTicks, latency.Ticks); + + private long _latencyTicks; + + protected override ValueTask ClientPauseAsync(RedisClient client, in RedisRequest request) + { + var latency = TimeSpan.FromTicks(Volatile.Read(ref _latencyTicks)); + if (latency > TimeSpan.Zero & request.KnownCommand != RedisCommand.QUIT) + { + Log($"[{client}] holding {request.Command} response by {latency.TotalMilliseconds}ms"); + return new(Task.Delay(latency)); + } + return base.ClientPauseAsync(client, request); + } } diff --git a/tests/StackExchange.Redis.Tests/KeyPrefixedDatabaseTests.cs b/tests/StackExchange.Redis.Tests/KeyPrefixedDatabaseTests.cs index 7ab5fa85f..a93c425dc 100644 --- a/tests/StackExchange.Redis.Tests/KeyPrefixedDatabaseTests.cs +++ b/tests/StackExchange.Redis.Tests/KeyPrefixedDatabaseTests.cs @@ -64,13 +64,6 @@ public void DebugObject() mock.Received().DebugObject("prefix:key", CommandFlags.None); } - [Fact] - public void Get_Database() - { - mock.Database.Returns(123); - Assert.Equal(123, prefixed.Database); - } - [Fact] public void HashDecrement_1() { diff --git a/tests/StackExchange.Redis.Tests/LockingTests.cs b/tests/StackExchange.Redis.Tests/LockingTests.cs index 52d03bb83..52d498a25 100644 --- a/tests/StackExchange.Redis.Tests/LockingTests.cs +++ b/tests/StackExchange.Redis.Tests/LockingTests.cs @@ -58,7 +58,7 @@ void Inner(object? obj) ThreadPool.QueueUserWorkItem(Inner, conn2.GetDatabase(db)); evt.WaitOne(8000); } - Assert.Equal(0, Interlocked.CompareExchange(ref errorCount, 0, 0)); + Assert.Equal(0, Volatile.Read(ref errorCount)); Assert.Equal(0, bgErrorCount); } diff --git a/tests/StackExchange.Redis.Tests/LoggerTests.cs b/tests/StackExchange.Redis.Tests/LoggerTests.cs index 31ed3fae0..6839eea5e 100644 --- a/tests/StackExchange.Redis.Tests/LoggerTests.cs +++ b/tests/StackExchange.Redis.Tests/LoggerTests.cs @@ -137,7 +137,7 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except _output.WriteLine(logLine); } - public long CallCount => Interlocked.Read(ref _callCount); + public long CallCount => Volatile.Read(ref _callCount); public override string ToString() { lock (sb) diff --git a/tests/StackExchange.Redis.Tests/MultiGroupTests/BasicMultiGroupTests.cs b/tests/StackExchange.Redis.Tests/MultiGroupTests/BasicMultiGroupTests.cs new file mode 100644 index 000000000..23eaaa7bd --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MultiGroupTests/BasicMultiGroupTests.cs @@ -0,0 +1,392 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using StackExchange.Redis.Tests.Helpers; +using Xunit; + +namespace StackExchange.Redis.Tests.MultiGroupTests; + +[RunPerProtocol] +public class BasicMultiGroupTests(ITestOutputHelper log) +{ + private sealed class Capture(ITestOutputHelper log) + { + private readonly List _seen = []; + + public void Seen(string source, RedisChannel channel, RedisValue value) + { + string message = $"[{source}] {channel}: {value}"; + lock (_seen) + { + _seen.Add(message); + } + } + + public void Reset() + { + lock (_seen) + { + _seen.Clear(); + } + } + + public void AssertTakeAny(string value) + { + lock (_seen) + { + Assert.True(_seen.Remove(value), $"Expected to find '{value}', but did not"); + } + } + + public void AssertTakeFirst(string value) + { + lock (_seen) + { + Assert.NotEmpty(_seen); + Assert.Equal(value, _seen[0]); + _seen.RemoveAt(0); // not concerned by perf here + } + } + + public async ValueTask AwaitAsync(ISubscriber sub, int expected) + { + for (int i = 0; i < 10; i++) + { + lock (_seen) + { + if (_seen.Count >= expected) + { + Assert.Equal(expected, _seen.Count); + log.WriteLine("Messages:"); + foreach (var item in _seen) + { + log.WriteLine(item); + } + return; + } + } + log.WriteLine($"Waiting for {expected} messages, got {i}, pausing..."); + await Task.Delay(10, TestContext.Current.CancellationToken); + await sub.PingAsync(); + } + + int actual; + lock (_seen) + { + actual = _seen.Count; + } + throw new TimeoutException($"Timed out waiting for {expected} messages, got {actual}"); + } + + public Task WriteSeen(string source, ChannelMessageQueue queue) => + Task.Run(async () => + { + await foreach (var msg in queue) + { + Seen(source, msg.Channel, msg.Message); + } + }); + } + protected TextWriter Log { get; } = new TextWriterOutputHelper(log); + + public enum InbuiltProbe + { + IsConnected, + Ping, + StringSet, + } + + [Theory] + [InlineData(InbuiltProbe.IsConnected, ServerType.Standalone)] + [InlineData(InbuiltProbe.Ping, ServerType.Standalone)] + [InlineData(InbuiltProbe.StringSet, ServerType.Standalone)] + [InlineData(InbuiltProbe.IsConnected, ServerType.Cluster)] + [InlineData(InbuiltProbe.Ping, ServerType.Cluster)] + [InlineData(InbuiltProbe.StringSet, ServerType.Cluster)] + public async Task SelectByWeight(InbuiltProbe probe, ServerType serverType) + { + HealthCheck healthCheck = new HealthCheck.Builder + { + Probe = probe switch + { + InbuiltProbe.IsConnected => HealthCheckProbe.IsConnected, + InbuiltProbe.Ping => HealthCheckProbe.Ping, + InbuiltProbe.StringSet => HealthCheckProbe.StringSet, + _ => throw new ArgumentOutOfRangeException(nameof(probe)), + }, + }; + + EndPoint germany = new DnsEndPoint("germany", 6379); + EndPoint canada = new DnsEndPoint("canada", 6379); + EndPoint tokyo = new DnsEndPoint("tokyo", 6379); + + using var server0 = new InProcessTestServer(log, endpoint: germany) { ServerType = serverType }; + using var server1 = new InProcessTestServer(log, endpoint: canada) { ServerType = serverType }; + using var server2 = new InProcessTestServer(log, endpoint: tokyo) { ServerType = serverType }; + + ConnectionGroupMember[] members = [ + new(server0.GetClientConfig()) { Weight = 2 }, + new(server1.GetClientConfig()) { Weight = 9 }, + new(server2.GetClientConfig()) { Weight = 3 }, + ]; + MultiGroupOptions options = new MultiGroupOptions.Builder { HealthCheck = healthCheck }; + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + Assert.True(conn.IsConnected); + var typed = Assert.IsType(conn); + + // (R.4.1) If multiple member databases are configured, then I want to failover to the one with the highest weight. + var db = conn.GetDatabase(); + var ep = await db.IdentifyEndpointAsync(); + Assert.Equal(canada, ep); + + // change weight and update + members[1].Weight = 1; + typed.SelectPreferredGroup(); + ep = await db.IdentifyEndpointAsync(); + Assert.Equal(tokyo, ep); + + WriteLatency(conn); + } + + private void WriteLatency(IConnectionGroup conn) + { + var typed = Assert.IsType(conn); + foreach (var member in conn.GetMembers()) + { + member.UpdateLatency(); + log.WriteLine($"{member.Name}: {member.Latency.TotalMilliseconds}us"); + } + log.WriteLine($"Active: {typed.Active}"); + } + + [Fact] + public async Task SelectByLatency() + { + EndPoint germany = new DnsEndPoint("germany", 6379); + EndPoint canada = new DnsEndPoint("canada", 6379); + EndPoint tokyo = new DnsEndPoint("tokyo", 6379); + + using var server0 = new InProcessTestServer(log, endpoint: germany); + using var server1 = new InProcessTestServer(log, endpoint: canada); + using var server2 = new InProcessTestServer(log, endpoint: tokyo); + + static ConfigurationOptions Check(ConfigurationOptions options) + { + // options.HeartbeatConsistencyChecks = true; + return options; + } + ConnectionGroupMember[] members = [ + new(Check(server0.GetClientConfig())), + new(Check(server1.GetClientConfig())), + new(Check(server2.GetClientConfig())), + ]; + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); + conn.ConnectionChanged += (_, args) => log.WriteLine($"Connection changed: {args.Type}, from {args.PreviousGroup?.Name ?? "(nil)"} to {args.Group.Name}"); + + Assert.True(conn.IsConnected); + server0.SetLatency(TimeSpan.FromMilliseconds(10)); + server1.SetLatency(TimeSpan.Zero); + server2.SetLatency(TimeSpan.FromMilliseconds(15)); + var typed = Assert.IsType(conn); + typed.OnHeartbeat(); // update latencies + await Task.Delay(100); // allow time to settle + typed.SelectPreferredGroup(); + WriteLatency(typed); + + // select lowest latency + var db = conn.GetDatabase(); + var ep = await db.IdentifyEndpointAsync(); + Assert.Equal(canada, ep); + + // change latency and update + server0.SetLatency(TimeSpan.FromMilliseconds(10)); + server1.SetLatency(TimeSpan.FromMilliseconds(10)); + server2.SetLatency(TimeSpan.Zero); + typed.OnHeartbeat(); // update latencies + await Task.Delay(100); // allow time to settle + typed.SelectPreferredGroup(); + ep = await db.IdentifyEndpointAsync(); + WriteLatency(typed); + Assert.Equal(tokyo, ep); + } + + [Fact] + public async Task PubSubRouted() + { + EndPoint germany = new DnsEndPoint("germany", 6379); + EndPoint canada = new DnsEndPoint("canada", 6379); + EndPoint tokyo = new DnsEndPoint("tokyo", 6379); + + using var server0 = new InProcessTestServer(log, endpoint: germany); + Assert.SkipUnless(server0.GetClientConfig().CommandMap.IsAvailable(RedisCommand.PUBLISH), "PUBLISH is not available"); + using var server1 = new InProcessTestServer(log, endpoint: canada); + using var server2 = new InProcessTestServer(log, endpoint: tokyo); + + Capture capture = new(log); + + RedisChannel channel = RedisChannel.Literal("chan"); + var pub0 = (await server0.ConnectAsync()).GetSubscriber(); + await pub0.SubscribeAsync(channel, (x, y) => capture.Seen(nameof(pub0), x, y)); + var pub1 = (await server1.ConnectAsync()).GetSubscriber(); + await pub1.SubscribeAsync(channel, (x, y) => capture.Seen(nameof(pub1), x, y)); + var pub2 = (await server2.ConnectAsync()).GetSubscriber(); + await pub2.SubscribeAsync(channel, (x, y) => capture.Seen(nameof(pub2), x, y)); + + ConnectionGroupMember[] members = [ + new(server0.GetClientConfig()) { Weight = 2 }, + new(server1.GetClientConfig()) { Weight = 9 }, + new(server2.GetClientConfig()) { Weight = 3 }, + ]; + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); + Assert.True(conn.IsConnected); + var typed = Assert.IsType(conn); + var multi = conn.GetSubscriber(); + await multi.SubscribeAsync(channel, (x, y) => capture.Seen(nameof(conn), x, y)); + + // (R.4.1) If multiple member databases are configured, then I want to failover to the one with the highest weight. + var db = conn.GetDatabase(); + var ep = await db.IdentifyEndpointAsync(); + Assert.Equal(canada, ep); + + // now publish via all 4 options, see what happens + capture.Reset(); + await pub0.PublishAsync(channel, "abc"); + await pub1.PublishAsync(channel, "def"); + await pub2.PublishAsync(channel, "ghi"); + await multi.PublishAsync(channel, "jkl"); + + // we're expecting just canada, so: + await capture.AwaitAsync(multi, 6); + capture.AssertTakeAny("[pub0] chan: abc"); + capture.AssertTakeAny("[pub1] chan: def"); + capture.AssertTakeAny("[pub2] chan: ghi"); + capture.AssertTakeAny("[conn] chan: def"); // receives the message from pub1 + capture.AssertTakeAny("[conn] chan: jkl"); // receives the message from itself + capture.AssertTakeAny("[pub1] chan: jkl"); // received the message from the multi-group + + // change weight and update + members[1].Weight = 1; + typed.SelectPreferredGroup(); + ep = await db.IdentifyEndpointAsync(); + Assert.Equal(tokyo, ep); + + capture.Reset(); + log.WriteLine("Publishing..."); + await pub0.PublishAsync(channel, "abc"); + await pub1.PublishAsync(channel, "def"); + await pub2.PublishAsync(channel, "ghi"); + await multi.PublishAsync(channel, "jkl"); + + // now we're expecting just tokyo, so: + await capture.AwaitAsync(multi, 6); + capture.AssertTakeAny("[pub0] chan: abc"); + capture.AssertTakeAny("[pub1] chan: def"); + capture.AssertTakeAny("[pub2] chan: ghi"); + capture.AssertTakeAny("[conn] chan: ghi"); // receives the message from pub2 + capture.AssertTakeAny("[conn] chan: jkl"); // receives the message from itself + capture.AssertTakeAny("[pub2] chan: jkl"); // received the message from the multi-group + } + + [Fact] + public async Task PubSubOrderedRouted() + { + EndPoint germany = new DnsEndPoint("germany", 6379); + EndPoint canada = new DnsEndPoint("canada", 6379); + EndPoint tokyo = new DnsEndPoint("tokyo", 6379); + + using var server0 = new InProcessTestServer(log, endpoint: germany); + Assert.SkipUnless( + server0.GetClientConfig().CommandMap.IsAvailable(RedisCommand.PUBLISH), + "PUBLISH is not available"); + using var server1 = new InProcessTestServer(log, endpoint: canada); + using var server2 = new InProcessTestServer(log, endpoint: tokyo); + + Capture capture = new(log); + + RedisChannel channel = RedisChannel.Literal("chan"); + var pub0 = (await server0.ConnectAsync()).GetSubscriber(); + _ = capture.WriteSeen(nameof(pub0), await pub0.SubscribeAsync(channel)); + var pub1 = (await server1.ConnectAsync()).GetSubscriber(); + _ = capture.WriteSeen(nameof(pub1), await pub1.SubscribeAsync(channel)); + var pub2 = (await server2.ConnectAsync()).GetSubscriber(); + _ = capture.WriteSeen(nameof(pub2), await pub2.SubscribeAsync(channel)); + + ConnectionGroupMember[] members = + [ + new(server0.GetClientConfig()) { Weight = 2 }, + new(server1.GetClientConfig()) { Weight = 9 }, + new(server2.GetClientConfig()) { Weight = 3 }, + ]; + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members); + Assert.True(conn.IsConnected); + var typed = Assert.IsType(conn); + var multi = conn.GetSubscriber(); + _ = capture.WriteSeen(nameof(conn), await multi.SubscribeAsync(channel)); + + // (R.4.1) If multiple member databases are configured, then I want to failover to the one with the highest weight. + var db = conn.GetDatabase(); + var ep = await db.IdentifyEndpointAsync(); + Assert.Equal(canada, ep); + + // now publish via all 4 options, see what happens + capture.Reset(); + await pub0.PublishAsync(channel, "abc"); + await pub1.PublishAsync(channel, "def"); + await pub2.PublishAsync(channel, "ghi"); + for (int i = 0; i < 5; i++) + { + await multi.PublishAsync(channel, $"jkl{i}"); + } + + // we're expecting just canada, so: + await capture.AwaitAsync(multi, 14); + capture.AssertTakeAny("[pub0] chan: abc"); + capture.AssertTakeAny("[pub1] chan: def"); + capture.AssertTakeAny("[pub2] chan: ghi"); + for (int i = 0; i < 5; i++) + { + capture.AssertTakeAny($"[pub1] chan: jkl{i}"); // received the message from the multi-group + } + // these should be ordered + capture.AssertTakeFirst("[conn] chan: def"); // receives the message from pub1 + for (int i = 0; i < 5; i++) + { + capture.AssertTakeFirst($"[conn] chan: jkl{i}"); // receives the message from itself + } + + // change weight and update + members[1].Weight = 1; + typed.SelectPreferredGroup(); + ep = await db.IdentifyEndpointAsync(); + Assert.Equal(tokyo, ep); + + capture.Reset(); + await pub0.PublishAsync(channel, "abc"); + await pub1.PublishAsync(channel, "def"); + await pub2.PublishAsync(channel, "ghi"); + for (int i = 0; i < 5; i++) + { + await multi.PublishAsync(channel, $"jkl{i}"); + } + + // now we're expecting just tokyo, so: + await capture.AwaitAsync(multi, 14); + capture.AssertTakeAny("[pub0] chan: abc"); + capture.AssertTakeAny("[pub1] chan: def"); + capture.AssertTakeAny("[pub2] chan: ghi"); + for (int i = 0; i < 5; i++) + { + capture.AssertTakeAny($"[pub2] chan: jkl{i}"); // received the message from the multi-group + } + + // these should be ordered + capture.AssertTakeFirst("[conn] chan: ghi"); // receives the message from pub2 + for (int i = 0; i < 5; i++) + { + capture.AssertTakeFirst($"[conn] chan: jkl{i}"); // receives the message from itself + } + } +} diff --git a/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs b/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs new file mode 100644 index 000000000..86d3dc01f --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MultiGroupTests/CircuitBreakerRerouteTests.cs @@ -0,0 +1,132 @@ +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using Xunit; + +namespace StackExchange.Redis.Tests.MultiGroupTests; + +[RunPerProtocol] +public class CircuitBreakerRerouteTests(ITestOutputHelper log) : TestBase(log) +{ + // A circuit-breaker trip must steer the group away from the affected member *promptly* - i.e. via + // the shim's ConnectionFailed(CircuitBreaker) fast-path, not by waiting for the next health-check + // poll. To isolate that mechanism we make the poll interval enormous, so the *only* thing that can + // reroute inside the test window is the fast-path; and we hold the tripped member unhealthy via a + // controllable probe, so the reroute is deterministic even though the physical connection reconnects + // immediately after being torn down. + [Fact] + public async Task CircuitBreakerTrip_ReroutesAwayFromMember() + { + EndPoint alpha = new DnsEndPoint("alpha", 6379); + EndPoint bravo = new DnsEndPoint("bravo", 6379); + EndPoint charlie = new DnsEndPoint("charlie", 6379); + + using var serverA = new InProcessTestServer(Output, endpoint: alpha); + using var serverB = new InProcessTestServer(Output, endpoint: bravo); + using var serverC = new InProcessTestServer(Output, endpoint: charlie); + + var breaker = new FlipBreaker(); + var probe = new ControllableProbe(); + + // only member A carries the trippable breaker; B and C are left with the default (which requires + // an implausible number of failures to trip, so they never interfere) + var configA = serverA.GetClientConfig(); + configA.CircuitBreaker = breaker; + + ConnectionGroupMember[] members = + [ + new(configA, "A") { Weight = 9 }, // highest weight -> initially active + new(serverB.GetClientConfig(), "B") { Weight = 3 }, // preferred failover target + new(serverC.GetClientConfig(), "C") { Weight = 1 }, + ]; + + MultiGroupOptions options = new MultiGroupOptions.Builder + { + // enormous, so the poll loop cannot be what reroutes us during the test + HealthCheckInterval = TimeSpan.FromMinutes(30), + HealthCheck = new HealthCheck.Builder + { + Probe = probe, + ProbeCount = 1, + ProbeTimeout = TimeSpan.FromSeconds(5), + }, + }; + + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + var typed = Assert.IsType(conn); + + // completes the first time we see a ConnectionFailed(CircuitBreaker); we await this (rather than + // polling a counter) so the assertion is deterministic and not subject to event-timing races + var circuitBreakerEvents = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + typed.ConnectionFailed += (_, e) => + { + Log($"ConnectionFailed: {e.FailureType} @ {e.EndPoint}"); + if (e.FailureType == ConnectionFailureType.CircuitBreaker) + { + circuitBreakerEvents.TrySetResult(true); + } + }; + + // sanity: A (highest weight) is the active member to begin with + Assert.True(conn.IsConnected); + Assert.Same(members[0], conn.ActiveMember); + + // arm the breaker and hold A unhealthy, then drive a *faulting* command to the active member (A): + // the fault is what the breaker evaluates, and a tripped breaker tears the connection down + probe.MarkDown(alpha); + breaker.Trip(); + var db = conn.GetDatabase(); + var fault = await Assert.ThrowsAnyAsync(() => db.ExecuteAsync("nonesuch")); + Log($"observed fault: {fault.GetType().Name}: {fault.Message}"); + + // wait (briefly) for the fast-path to react; nothing else can move us within this window + var moved = await WaitForActiveAsync(conn, notMember: members[0], timeout: TimeSpan.FromSeconds(5)); + + Assert.True(moved, "expected the circuit-breaker trip to reroute away from member A"); + Assert.Same(members[1], conn.ActiveMember); // B is the next-highest weight + + // a ConnectionFailed event with FailureType == CircuitBreaker must have fired; await it (with a + // timeout linked to the ambient test cancellation) rather than racing on a counter read + using var cbCts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + cbCts.CancelAfter(TimeSpan.FromSeconds(5)); + Assert.True(await circuitBreakerEvents.Task.WaitAsync(cbCts.Token), "expected a ConnectionFailed event with FailureType == CircuitBreaker"); + } + + private static async Task WaitForActiveAsync( + IConnectionGroup conn, ConnectionGroupMember notMember, TimeSpan timeout) + { + var deadline = DateTime.UtcNow + timeout; + while (DateTime.UtcNow < deadline) + { + if (!ReferenceEquals(conn.ActiveMember, notMember) && conn.ActiveMember is not null) + { + return true; + } + await Task.Delay(50, TestContext.Current.CancellationToken); + } + return false; + } + + // trips on demand: healthy until Trip() is called; the trip only actuates on a *fault* observation + // (successes are never evaluated, by design), which the test provides via a bad command + private sealed class FlipBreaker : CircuitBreaker + { + private volatile bool _tripped; + public void Trip() => _tripped = true; + + public override Accumulator CreateAccumulator() => new Acc(this); + + private sealed class Acc(FlipBreaker owner) : Accumulator + { + // this breaker trips on demand regardless of *what* faulted, so treat every observed fault as a + // failure - otherwise Trip's IsFailure gate would filter out non-failure faults (e.g. an unknown + // command) before the tripped state is ever consulted + protected override bool IsFailure(in FaultContext fault) => true; + public override void ObserveResult(in FaultContext context) { } + public override bool IsHealthy() => !owner._tripped; + public override void Reset() { } + } + } +} diff --git a/tests/StackExchange.Redis.Tests/MultiGroupTests/GroupConfigResolutionTests.cs b/tests/StackExchange.Redis.Tests/MultiGroupTests/GroupConfigResolutionTests.cs new file mode 100644 index 000000000..2aca640ba --- /dev/null +++ b/tests/StackExchange.Redis.Tests/MultiGroupTests/GroupConfigResolutionTests.cs @@ -0,0 +1,102 @@ +using System; +using System.Net; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using Xunit; + +namespace StackExchange.Redis.Tests.MultiGroupTests; + +/// +/// Verifies how a live group resolves its configuration: that group defaults reach the members, that a +/// per-member override wins, and that none of this mutates the caller's . +/// +public class GroupConfigResolutionTests(ITestOutputHelper log) +{ + [Fact] + public async Task GroupExposesItsOptions() + { + using var server0 = new InProcessTestServer(log, endpoint: new DnsEndPoint("alpha", 6379)); + using var server1 = new InProcessTestServer(log, endpoint: new DnsEndPoint("beta", 6379)); + + MultiGroupOptions options = new MultiGroupOptions.Builder + { + FailbackDelay = TimeSpan.FromMinutes(4), + RetryPolicy = new RetryPolicy.Builder { MaxAttempts = 6 }, + }; + + ConnectionGroupMember[] members = [new(server0.GetClientConfig()), new(server1.GetClientConfig())]; + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + + Assert.Same(options, conn.Options); + Assert.Equal(TimeSpan.FromMinutes(4), conn.Options.FailbackDelay); + } + + [Fact] + public async Task WithRetryUsesTheGroupPolicy() + { + using var server0 = new InProcessTestServer(log, endpoint: new DnsEndPoint("alpha", 6379)); + using var server1 = new InProcessTestServer(log, endpoint: new DnsEndPoint("beta", 6379)); + + RetryPolicy groupPolicy = new RetryPolicy.Builder { MaxAttempts = 6, RetryDelay = TimeSpan.Zero }; + MultiGroupOptions options = new MultiGroupOptions.Builder { RetryPolicy = groupPolicy }; + + ConnectionGroupMember[] members = [new(server0.GetClientConfig()), new(server1.GetClientConfig())]; + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + + // the parameterless overload resolves the policy from the group it is attached to + var retrying = Assert.IsType(conn.GetDatabase().WithRetry()); + Assert.Same(groupPolicy, retrying.Policy); + + // ...and an explicit policy still wins + RetryPolicy explicitPolicy = new RetryPolicy.Builder { MaxAttempts = 2 }; + var explicitlyRetrying = Assert.IsType(conn.GetDatabase().WithRetry(explicitPolicy)); + Assert.Same(explicitPolicy, explicitlyRetrying.Policy); + } + + [Fact] + public async Task GroupCircuitBreakerReachesMembersWithoutMutatingCallerConfig() + { + using var server0 = new InProcessTestServer(log, endpoint: new DnsEndPoint("alpha", 6379)); + using var server1 = new InProcessTestServer(log, endpoint: new DnsEndPoint("beta", 6379)); + + CircuitBreaker groupBreaker = new CircuitBreaker.Builder { FailureRateThreshold = 42 }; + CircuitBreaker memberBreaker = new CircuitBreaker.Builder { FailureRateThreshold = 13 }; + + var config0 = server0.GetClientConfig(); + var config1 = server1.GetClientConfig(); + + MultiGroupOptions options = new MultiGroupOptions.Builder { CircuitBreaker = groupBreaker }; + ConnectionGroupMember[] members = [new(config0), new(config1) { CircuitBreaker = memberBreaker }]; + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + + // the group default reached the first member's connection, and the override reached the second + Assert.Same(groupBreaker, AsMultiplexer(members[0]).EffectiveCircuitBreaker); + Assert.Same(memberBreaker, AsMultiplexer(members[1]).EffectiveCircuitBreaker); + + // ...and neither was written back into the caller's configuration, which remains reusable + Assert.Null(config0.CircuitBreaker); + Assert.Null(config1.CircuitBreaker); + + static ConnectionMultiplexer AsMultiplexer(ConnectionGroupMember member) => member.Multiplexer; + } + + [Fact] + public async Task DisabledHealthCheckLeavesMemberSelectableOnConnectivityAlone() + { + using var server0 = new InProcessTestServer(log, endpoint: new DnsEndPoint("alpha", 6379)); + using var server1 = new InProcessTestServer(log, endpoint: new DnsEndPoint("beta", 6379)); + + // HealthCheck.None performs no probes and reports Inconclusive, which is not Unhealthy - so a + // connected member stays eligible, and the higher weight still wins + MultiGroupOptions options = new MultiGroupOptions.Builder { HealthCheck = HealthCheck.None }; + ConnectionGroupMember[] members = [ + new(server0.GetClientConfig(), "alpha") { Weight = 1 }, + new(server1.GetClientConfig(), "beta") { Weight = 9 }, + ]; + + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + Assert.True(conn.IsConnected); + Assert.Equal("beta", conn.ActiveMember?.Name); + Assert.All(members, member => Assert.False(member.IsUnhealthy)); + } +} diff --git a/tests/StackExchange.Redis.Tests/NamingTests.cs b/tests/StackExchange.Redis.Tests/NamingTests.cs index 784be86cd..5a6e71d54 100644 --- a/tests/StackExchange.Redis.Tests/NamingTests.cs +++ b/tests/StackExchange.Redis.Tests/NamingTests.cs @@ -168,6 +168,12 @@ private void CheckMethod(MethodInfo method, bool isAsync) { case nameof(IDatabaseAsync.IsConnected): return; + case nameof(IDatabaseAsync.CreateTransaction) when isAsync: + // IDatabaseAsync.CreateTransaction is a synchronous *factory*: it hands back an + // ITransactionAsync rather than performing an async operation, so neither the *Async + // suffix nor the Task-returning rule applies. IDatabase.CreateTransaction is unaffected + // and still goes through the checks below. + return; case nameof(IDatabase.CreateBatch): case nameof(IDatabase.CreateTransaction): case nameof(IDatabase.IdentifyEndpoint): diff --git a/tests/StackExchange.Redis.Tests/PubSubTests.cs b/tests/StackExchange.Redis.Tests/PubSubTests.cs index 65e1ee574..b4ba7b5de 100644 --- a/tests/StackExchange.Redis.Tests/PubSubTests.cs +++ b/tests/StackExchange.Redis.Tests/PubSubTests.cs @@ -720,16 +720,16 @@ public async Task TestMultipleSubscribersGetMessage() await Task.WhenAll(tA, tB).ForAwait(); Assert.Equal(2, pub.Publish(channel, "message")); await AllowReasonableTimeToPublishAndProcess().ForAwait(); - Assert.Equal(1, Interlocked.CompareExchange(ref gotA, 0, 0)); - Assert.Equal(1, Interlocked.CompareExchange(ref gotB, 0, 0)); + Assert.Equal(1, Volatile.Read(ref gotA)); + Assert.Equal(1, Volatile.Read(ref gotB)); // and unsubscribe... tA = listenA.UnsubscribeAsync(channel); await tA; Assert.Equal(1, pub.Publish(channel, "message")); await AllowReasonableTimeToPublishAndProcess().ForAwait(); - Assert.Equal(1, Interlocked.CompareExchange(ref gotA, 0, 0)); - Assert.Equal(2, Interlocked.CompareExchange(ref gotB, 0, 0)); + Assert.Equal(1, Volatile.Read(ref gotA)); + Assert.Equal(2, Volatile.Read(ref gotB)); } [Fact] @@ -762,7 +762,7 @@ public async Task Issue38() await AllowReasonableTimeToPublishAndProcess().ForAwait(); Assert.Equal(6, total); // sent - Assert.Equal(6, Interlocked.CompareExchange(ref count, 0, 0)); // received + Assert.Equal(6, Volatile.Read(ref count)); // received } internal static Task AllowReasonableTimeToPublishAndProcess() => Task.Delay(500); @@ -787,8 +787,8 @@ public async Task TestPartialSubscriberGetMessage() Assert.Equal(2, pub.Publish(prefix + "channel", "message")); #pragma warning restore CS0618 await AllowReasonableTimeToPublishAndProcess().ForAwait(); - Assert.Equal(1, Interlocked.CompareExchange(ref gotA, 0, 0)); - Assert.Equal(1, Interlocked.CompareExchange(ref gotB, 0, 0)); + Assert.Equal(1, Volatile.Read(ref gotA)); + Assert.Equal(1, Volatile.Read(ref gotB)); // and unsubscibe... #pragma warning disable CS0618 @@ -797,8 +797,8 @@ public async Task TestPartialSubscriberGetMessage() Assert.Equal(1, pub.Publish(prefix + "channel", "message")); #pragma warning restore CS0618 await AllowReasonableTimeToPublishAndProcess().ForAwait(); - Assert.Equal(2, Interlocked.CompareExchange(ref gotA, 0, 0)); - Assert.Equal(1, Interlocked.CompareExchange(ref gotB, 0, 0)); + Assert.Equal(2, Volatile.Read(ref gotA)); + Assert.Equal(1, Volatile.Read(ref gotB)); } [Fact] diff --git a/tests/StackExchange.Redis.Tests/RetryPolicyUnitTests.cs b/tests/StackExchange.Redis.Tests/ReconnectRetryPolicyUnitTests.cs similarity index 99% rename from tests/StackExchange.Redis.Tests/RetryPolicyUnitTests.cs rename to tests/StackExchange.Redis.Tests/ReconnectRetryPolicyUnitTests.cs index 78036e635..b3ea620f5 100644 --- a/tests/StackExchange.Redis.Tests/RetryPolicyUnitTests.cs +++ b/tests/StackExchange.Redis.Tests/ReconnectRetryPolicyUnitTests.cs @@ -11,7 +11,7 @@ namespace StackExchange.Redis.Tests; -public class RetryPolicyUnitTests(ITestOutputHelper log) +public class ReconnectRetryPolicyUnitTests(ITestOutputHelper log) { [Theory] [InlineData(FailureMode.Success)] diff --git a/tests/StackExchange.Redis.Tests/RedisTestConfig.json b/tests/StackExchange.Redis.Tests/RedisTestConfig.json index c652a4583..31ae07818 100644 --- a/tests/StackExchange.Redis.Tests/RedisTestConfig.json +++ b/tests/StackExchange.Redis.Tests/RedisTestConfig.json @@ -3,4 +3,8 @@ //"PrimaryServer": "[::1]", //"ReplicaServer": "[::1]", //"SecureServer": "[::1]" + //"ActiveActiveEndpoints": [ + // "somewhere,password=sssshhh", + // "somewhereElse,password=sssshhh" + //] } \ No newline at end of file diff --git a/tests/StackExchange.Redis.Tests/ResultBoxTests.cs b/tests/StackExchange.Redis.Tests/ResultBoxTests.cs index adb1b309f..99c8358e8 100644 --- a/tests/StackExchange.Redis.Tests/ResultBoxTests.cs +++ b/tests/StackExchange.Redis.Tests/ResultBoxTests.cs @@ -46,7 +46,7 @@ public void SyncResultBox() Assert.Equal(0, Volatile.Read(ref activated)); // check that complete signals continuation - msg.Complete(); + msg.Complete(null); Thread.Sleep(100); Assert.Equal(1, Volatile.Read(ref activated)); @@ -81,7 +81,7 @@ public void TaskResultBox() Assert.False(tcs.Task.IsCompleted); // check that complete signals continuation - msg.Complete(); + msg.Complete(null); Thread.Sleep(100); Assert.True(tcs.Task.IsCompleted); diff --git a/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs new file mode 100644 index 000000000..c1d6e1d51 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/RetryTests/CommandRetryPolicyUnitTests.cs @@ -0,0 +1,237 @@ +using System.Threading; +using StackExchange.Redis.Availability; +using StackExchange.Redis.Interfaces; +using Xunit; + +namespace StackExchange.Redis.Tests.RetryTests; + +public class CommandRetryPolicyUnitTests +{ + // --- RetryPolicy.CanRetry: spoofed fault scenarios ------------------------------- + + // Builds a FaultContext for a spoofed server error of the given kind, carrying the given + // command-flags, and asks the policy whether it may be retried. + private static RetryResult CanRetry(RedisErrorKind kind, CommandFlags flags, RetryPolicy? policy = null) + { + // the exception carries both the Kind and the command-flags; FaultContext reads them back + var fault = new FaultContext(new RedisServerException(kind, flags, kind.ToString())); + return (policy ?? RetryPolicy.Default).CanRetry(in fault); + } + + // As above, but for an *ambiguous* fault: a timeout on a request we know was sent. We have no idea + // whether the server applied it, so this is the case the retry-category exists to price. + private static RetryResult CanRetryAmbiguous(CommandFlags flags, RetryPolicy? policy = null) + { + var fault = new FaultContext(new RedisTimeoutException(flags, "timeout", CommandStatus.Sent)); + return (policy ?? RetryPolicy.Default).CanRetry(in fault); + } + + // The command's retry-category is checked against the policy's max category: the default max is + // CommandRetryWriteLastWins, so anything at-or-below that is in-range, and anything with more + // side-effects is not. Using an ambiguous (timeout-after-send) fault, since that is where the + // category actually bites - see CanRetry_NotAppliedBypassesCategory for the other half. + [Theory] + [InlineData(CommandFlags.CommandRetryAlways, true)] + [InlineData(CommandFlags.CommandRetryConnection, true)] + [InlineData(CommandFlags.CommandRetryReadOnly, true)] + [InlineData(CommandFlags.CommandRetryWriteChecked, true)] + [InlineData(CommandFlags.CommandRetryWriteLastWins, true)] // == default max + [InlineData(CommandFlags.CommandRetryWriteAccumulating, false)] // beyond default max + [InlineData(CommandFlags.CommandRetryServerAdmin, false)] + [InlineData(CommandFlags.CommandRetryNever, false)] + [InlineData(CommandFlags.None, false)] // unspecified => assume the worst => not retried + public void CanRetry_CategoryVersusDefaultMax(CommandFlags category, bool expectRetry) + { + var result = CanRetryAmbiguous(category); + Assert.Equal(expectRetry, result != RetryResult.None); + } + + // When the fault proves the operation never took effect (here: the server was still LOADING, so it + // rejected the command outright), a replay is a first attempt rather than a repeat - it cannot + // double-apply anything, so the side-effect category is irrelevant and the cap is bypassed. An + // explicit CommandRetryNever is still an absolute veto, as is an unspecified category. + [Theory] + [InlineData(CommandFlags.CommandRetryWriteAccumulating, true)] // beyond the default cap, but safe here + [InlineData(CommandFlags.CommandRetryServerAdmin, true)] + [InlineData(CommandFlags.CommandRetryNever, false)] // never means never + [InlineData(CommandFlags.None, false)] // we don't know what it is; don't guess + public void CanRetry_NotAppliedBypassesCategory(CommandFlags category, bool expectRetry) + { + // sanity: the same category against an ambiguous fault of the same "retryability" is refused + if (expectRetry) Assert.Equal(RetryResult.None, CanRetryAmbiguous(category)); + + var result = CanRetry(RedisErrorKind.Loading, category); + Assert.Equal(expectRetry, result != RetryResult.None); + } + + // The other source of certainty: the client knows it never wrote the message. Same conclusion, even + // though the fault itself (a connection failure) is otherwise ambiguous. + [Theory] + [InlineData(CommandStatus.WaitingToBeSent, true)] + [InlineData(CommandStatus.WaitingInBacklog, true)] + [InlineData(CommandStatus.Sent, false)] // may or may not have been applied + [InlineData(CommandStatus.Unknown, false)] + public void CanRetry_UnsentMessageBypassesCategory(CommandStatus status, bool expectRetry) + { + var fault = new FaultContext(new RedisConnectionException( + ConnectionFailureType.SocketFailure, + CommandFlags.CommandRetryWriteAccumulating, // beyond the default cap + "boom", + innerException: null, + commandStatus: status)); + + Assert.Equal(expectRetry, fault.NotApplied); + Assert.Equal(expectRetry, RetryPolicy.Default.CanRetry(in fault) != RetryResult.None); + } + + // With an in-range category (== default max), the outcome is decided purely by whether the error + // is transient: LOADING is worth retrying, WRONGTYPE is an application error that will not fix itself. + [Theory] + [InlineData(RedisErrorKind.Loading, true)] // still loading the dataset - transient + [InlineData(RedisErrorKind.ClusterDown, true)] // slot temporarily unserved - transient + [InlineData(RedisErrorKind.WrongType, false)] // wrong data type - application error + [InlineData(RedisErrorKind.NoPermission, false)] // ACL - application error + public void CanRetry_ErrorKindGatesRetry_WhenInRange(RedisErrorKind kind, bool expectRetry) + { + var result = CanRetry(kind, CommandFlags.CommandRetryWriteLastWins); + Assert.Equal(expectRetry, result != RetryResult.None); + } + + // "never" and "always" adjust only the category range - they do not override the error-kind check: + // an "always" command still won't retry an application error, and a "never" command won't retry even + // a transient one. + [Theory] + [InlineData(CommandFlags.CommandRetryAlways, RedisErrorKind.Loading, true)] + [InlineData(CommandFlags.CommandRetryAlways, RedisErrorKind.WrongType, false)] + [InlineData(CommandFlags.CommandRetryNever, RedisErrorKind.Loading, false)] + [InlineData(CommandFlags.CommandRetryNever, RedisErrorKind.WrongType, false)] + public void CanRetry_NeverAndAlwaysAffectRangeNotErrorKind(CommandFlags category, RedisErrorKind kind, bool expectRetry) + { + var result = CanRetry(kind, category); + Assert.Equal(expectRetry, result != RetryResult.None); + } + + // When a retry is permitted, it normally offers both the same server and a failover server; but a + // "server specific" (sticky) command must not move endpoints, so only the same-server option remains. + [Theory] + [InlineData(CommandFlags.None, RetryResult.SameServer | RetryResult.FailoverServer)] + [InlineData(Message.CommandServerSpecific, RetryResult.SameServer)] + public void CanRetry_ServerSpecificRestrictsToSameServer(CommandFlags extra, RetryResult expected) + { + // in-range category (== default max) + transient error => a retry is offered; the sticky flag + // only changes *where* the retry may go, not *whether* it happens. + var result = CanRetry(RedisErrorKind.Loading, CommandFlags.CommandRetryWriteLastWins | extra); + Assert.Equal(expected, result); + } + + // The sticky (server-specific) flag lives outside the retry-category region, so it is masked off + // before the category-vs-max comparison and must not change the range verdict (retry-at-all vs none) + // - it only affects the same/failover choice, covered above. + [Theory] + [InlineData(CommandFlags.CommandRetryReadOnly, true)] // in range + [InlineData(CommandFlags.CommandRetryServerAdmin, false)] // beyond default max + public void CanRetry_ServerSpecificDoesNotAffectRange(CommandFlags category, bool expectRetry) + { + var withoutFlag = CanRetryAmbiguous(category); + var withFlag = CanRetryAmbiguous(category | Message.CommandServerSpecific); + + Assert.Equal(expectRetry, withoutFlag != RetryResult.None); + Assert.Equal(expectRetry, withFlag != RetryResult.None); + } + + // --- RetryDatabase.CanRetry: attempt accounting ---------------------------------- + + // With max-attempts-before-failover pinned equal to max-attempts, the failover path is disabled, so + // this exercises pure same-server attempt counting. A transient LOADING fault on an in-range command + // means the policy would allow a retry; the only gate is the attempt counter: with MaxAttempts=3, + // attempts 1 and 2 may retry, attempt 3 is exhausted. Because we never fail over, the out "delay" is + // never cancellable (that is how "don't wait for failover" is expressed) and the ref "failover" is + // left untouched. + [Theory] + [InlineData(1, true)] + [InlineData(2, true)] + [InlineData(3, false)] + public void RetryDatabase_CanRetry_MaxAttempts_NoFailover(int attempt, bool expected) + { + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 3, MaxAttemptsBeforeFailover = 3 }; + var controller = new RetryController(policy, DatabaseFeatureFlags.None); // CanRetry never touches any database + + using var cts = new CancellationTokenSource(); + var token = cts.Token; + var failover = token; + + var fault = new RedisServerException(RedisErrorKind.Loading, CommandFlags.CommandRetryWriteLastWins, "LOADING"); + var result = controller.CanRetry(attempt, fault, ref failover, out var delay); + + Assert.Equal(expected, result); + Assert.False(delay.CanBeCanceled); // never waiting for a failover + Assert.Equal(token, failover); // ref failover untouched + } + + // MaxAttempts=4 with failover enabled after 2 attempts. A single "failover" token is threaded through + // the sequence to observe the state machine: attempts 1..3 return true, attempt 4 is exhausted. The + // interesting step is attempt 2 (== MaxAttemptsBeforeFailover): it still returns true, but now hands the + // failover token back as "delay" and clears the ref (we fail over only once); attempt 3 therefore sees + // no failover token and drops back to a plain same-server retry. + [Fact] + public void RetryDatabase_CanRetry_FailoverAtThreshold() + { + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 4, MaxAttemptsBeforeFailover = 2 }; + // failover is only armed when the inner database advertises the feature; supply it explicitly + var controller = new RetryController(policy, DatabaseFeatureFlags.Failover); + + using var cts = new CancellationTokenSource(); + var token = cts.Token; + var failover = token; + var fault = new RedisServerException(RedisErrorKind.Loading, CommandFlags.CommandRetryWriteLastWins, "LOADING"); + + // attempt 1: plain same-server retry; failover token not yet consumed + Assert.True(controller.CanRetry(1, fault, ref failover, out var delay)); + Assert.False(delay.CanBeCanceled); + Assert.Equal(token, failover); + + // attempt 2 (== MaxAttemptsBeforeFailover): still a retry, but now fail over - "delay" becomes the + // failover token and the ref is cleared to None so it only fires once + Assert.True(controller.CanRetry(2, fault, ref failover, out delay)); + Assert.Equal(token, delay); + Assert.True(delay.CanBeCanceled); + Assert.Equal(CancellationToken.None, failover); + + // attempt 3: failover already spent (ref is None) -> back to a same-server retry + Assert.True(controller.CanRetry(3, fault, ref failover, out delay)); + Assert.False(delay.CanBeCanceled); + Assert.Equal(CancellationToken.None, failover); + + // attempt 4: no retries left + Assert.False(controller.CanRetry(4, fault, ref failover, out delay)); + Assert.False(delay.CanBeCanceled); + } + + // As above, but with the sticky (server-specific) flag set: the policy now permits only same-server + // retries, so there is no failover option at the threshold. Current behaviour: CanRetry returns *false* + // at attempt 2 - the command gives up rather than continuing on the same server - because the threshold + // branch (attempt == MaxAttemptsBeforeFailover) requires FailoverServer permission and does not fall + // back to a same-server retry. It also consumes the failover token as a side-effect of that branch. + [Fact] + public void RetryDatabase_CanRetry_ServerSpecific_CannotFailover() + { + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 4, MaxAttemptsBeforeFailover = 2 }; + var controller = new RetryController(policy, DatabaseFeatureFlags.Failover); + + using var cts = new CancellationTokenSource(); + var token = cts.Token; + var failover = token; + const CommandFlags flags = CommandFlags.CommandRetryWriteLastWins | Message.CommandServerSpecific; + var fault = new RedisServerException(RedisErrorKind.Loading, flags, "LOADING"); + + // attempt 1: same-server retry; failover token untouched + Assert.True(controller.CanRetry(1, fault, ref failover, out var delay)); + Assert.False(delay.CanBeCanceled); + Assert.Equal(token, failover); + + // attempt 2 (== MaxAttemptsBeforeFailover): sticky forbids failover -> gives up (false), even though + // attempts remain; the failover token is still consumed to None as a side-effect of the branch + Assert.False(controller.CanRetry(2, fault, ref failover, out delay)); + Assert.Equal(CancellationToken.None, failover); + } +} diff --git a/tests/StackExchange.Redis.Tests/RetryTests/ConnectionFaultDetailTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/ConnectionFaultDetailTests.cs new file mode 100644 index 000000000..95781c82c --- /dev/null +++ b/tests/StackExchange.Redis.Tests/RetryTests/ConnectionFaultDetailTests.cs @@ -0,0 +1,153 @@ +using StackExchange.Redis.Availability; +using Xunit; + +namespace StackExchange.Redis.Tests.RetryTests; + +/// +/// When a connection dies, one exception is built to describe the *connection*, and it used to be handed +/// verbatim to every message that was in flight. Sharing one exception instance across unrelated callers is +/// dubious in itself (Exception.Data is mutable), and it discards the per-message facts the retry +/// machinery needs: the command's retry category, and whether this particular message had actually been +/// written. Without them nothing at all is retryable after a connection failure, not even a plain +/// GET. is what splits them apart. +/// +public class ConnectionFaultDetailTests +{ + private static RedisConnectionException SharedConnectionFault() + { + // as built by PhysicalConnection.RecordConnectionFailed: describes the connection, knows nothing + // about any individual message + var ex = new RedisConnectionException( + ConnectionFailureType.SocketClosed, + CommandFlags.None, + "SocketClosed on 127.0.0.1:6379/Interactive"); + ex.Data["Redis-Version"] = "1.2.3"; + ex.Data["Redis-Server"] = "127.0.0.1:6379"; + return ex; + } + + private static Message Read() => Message.Create(0, CommandFlags.None, RedisCommand.GET, (RedisKey)"key"); + + private static Message AccumulatingWrite() => Message.Create(0, CommandFlags.None, RedisCommand.INCR, (RedisKey)"key"); + + // A message that was written before the socket died: the outcome is genuinely ambiguous, so the sent + // status must survive as-is, but the command's category has to come through - otherwise the policy sees + // "no category" and refuses to retry even a pure read. + [Fact] + public void SentMessage_CarriesCategoryAndSentStatus() + { + var shared = SharedConnectionFault(); + var message = Read(); + message.SetRequestSent(); + + var per = Assert.IsType(ExceptionFactory.PerMessage(shared, message)); + + Assert.NotSame(shared, per); + Assert.Equal(ConnectionFailureType.SocketClosed, per.FailureType); + Assert.Equal(CommandStatus.Sent, per.CommandStatus); + Assert.Equal(CommandFlags.CommandRetryReadOnly, per.Flags & Message.MaskRetryCategory); + + var ctx = new FaultContext(per); + Assert.False(ctx.NotApplied); // it was on the wire; we cannot know whether the server ran it + Assert.NotEqual(RetryResult.None, RetryPolicy.Default.CanRetry(in ctx)); // ...but a read is safe + + // for contrast: the shared exception the message used to receive is retryable for nothing at all + var sharedCtx = new FaultContext(shared); + Assert.Equal(RetryResult.None, RetryPolicy.Default.CanRetry(in sharedCtx)); + } + + // Same situation, accumulating write: the category comes through and correctly *blocks* the retry, since + // a replay could double-apply. The caller can still opt in by raising the cap. + [Fact] + public void SentAccumulatingWrite_RemainsGatedByCategory() + { + var message = AccumulatingWrite(); + message.SetRequestSent(); + + var per = Assert.IsType(ExceptionFactory.PerMessage(SharedConnectionFault(), message)); + Assert.Equal(CommandFlags.CommandRetryWriteAccumulating, per.Flags & Message.MaskRetryCategory); + + var ctx = new FaultContext(per); + Assert.False(ctx.NotApplied); + Assert.Equal(RetryResult.None, RetryPolicy.Default.CanRetry(in ctx)); + + RetryPolicy permissive = new RetryPolicy.Builder { MaxCommandRetryCategory = CommandFlags.CommandRetryWriteAccumulating }; + Assert.NotEqual(RetryResult.None, permissive.CanRetry(in ctx)); + } + + // A message that never left the client - still waiting to be written, or sitting in the backlog - is + // *provably* unapplied, which is the one case where even an accumulating write can be safely re-issued. + // That fact lives on the message, so sharing the connection's exception threw it away. + [Theory] + [InlineData(false)] // never handed to the bridge + [InlineData(true)] // queued in the backlog awaiting a healthy connection + public void UnsentMessage_IsKnownNotApplied(bool backlogged) + { + var message = AccumulatingWrite(); + if (backlogged) message.SetBacklogged(); + + var per = Assert.IsType(ExceptionFactory.PerMessage(SharedConnectionFault(), message)); + + Assert.Equal(backlogged ? CommandStatus.WaitingInBacklog : CommandStatus.WaitingToBeSent, per.CommandStatus); + + var ctx = new FaultContext(per); + Assert.True(ctx.NotApplied); + // accumulating, i.e. beyond the default cap - but nothing was applied, so there is nothing to repeat + Assert.NotEqual(RetryResult.None, RetryPolicy.Default.CanRetry(in ctx)); + } + + // The connection-level diagnostics are the useful part of these exceptions, so they have to come across; + // but the dictionaries must be independent, or one caller's annotations show up on another's exception. + [Fact] + public void SharedDiagnosticsAreCopied_ButNotShared() + { + var shared = SharedConnectionFault(); + var first = ExceptionFactory.PerMessage(shared, Read()); + var second = ExceptionFactory.PerMessage(shared, AccumulatingWrite()); + + Assert.NotSame(first, second); + Assert.Equal(shared.Message, first.Message); + Assert.Equal("1.2.3", first.Data["Redis-Version"]); + Assert.Equal("1.2.3", second.Data["Redis-Version"]); + + first.Data["mine"] = "only-mine"; + Assert.False(second.Data.Contains("mine")); + Assert.False(shared.Data.Contains("mine")); + } + + // The per-message status is recorded in the diagnostic data too, so a user reading the exception's Data + // sees this message's status rather than whatever the connection-level exception happened to say. + [Fact] + public void SentStatusIsRecordedInDiagnosticData() + { + var shared = SharedConnectionFault(); + shared.Data["request-sent-status"] = CommandStatus.Unknown; + + var message = Read(); + message.SetRequestSent(); + var per = ExceptionFactory.PerMessage(shared, message); + + Assert.Equal(CommandStatus.Sent, per.Data["request-sent-status"]); + } + + // Only the shared connection-failure shape needs splitting; anything else already describes a single + // operation, and an exception that already matches the message is passed straight through (no needless + // allocation on a teardown that may be failing thousands of messages). + [Fact] + public void UnrelatedOrAlreadyMatchingExceptions_ArePassedThrough() + { + var message = Read(); + message.SetRequestSent(); + + var serverFault = new RedisServerException(RedisErrorKind.Loading, message.Flags, "LOADING"); + Assert.Same(serverFault, ExceptionFactory.PerMessage(serverFault, message)); + + var alreadySpecific = new RedisConnectionException( + ConnectionFailureType.SocketClosed, + message.Flags, + "already describes this message", + innerException: null, + commandStatus: CommandStatus.Sent); + Assert.Same(alreadySpecific, ExceptionFactory.PerMessage(alreadySpecific, message)); + } +} diff --git a/tests/StackExchange.Redis.Tests/RetryTests/RetryControllerTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/RetryControllerTests.cs new file mode 100644 index 000000000..313d40c30 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/RetryTests/RetryControllerTests.cs @@ -0,0 +1,206 @@ +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using StackExchange.Redis.Interfaces; +using Xunit; + +namespace StackExchange.Redis.Tests.RetryTests; + +// Configuration validation (which lives on RetryPolicy.Builder, since a RetryPolicy is immutable and +// validated on construction) and the wait/failover timing state machine of RetryController; neither needs a +// server, or even an inner database - CanRetry and the delays never touch one. +public class RetryControllerTests +{ + // A failover threshold below 1 could never be reached by the attempt counter (which starts at 1), so + // it would *silently* disable failover; that is rejected up front. + [Fact] + public void Policy_RejectsUnreachableFailoverThreshold() + => Assert.Throws(() => new RetryPolicy.Builder { MaxAttemptsBeforeFailover = 0 }.Create()); + + // DatabaseFeatureFlags is internal, so theories take a bool and map here + private static DatabaseFeatureFlags Features(bool withFailover) + => withFailover ? DatabaseFeatureFlags.Failover : DatabaseFeatureFlags.None; + + // Negative durations are nonsense for a delay; each is validated separately. + [Fact] + public void Policy_RejectsNegativeDurations() + { + var negative = TimeSpan.FromMilliseconds(-1); + Assert.Throws(() => new RetryPolicy.Builder { RetryDelay = negative }.Create()); + Assert.Throws(() => new RetryPolicy.Builder { JitterMax = negative }.Create()); + Assert.Throws(() => new RetryPolicy.Builder { FailoverDelay = negative }.Create()); + } + + // The watch-contention budget counts *attempts*, so 1 means "try once, do not re-attempt"; zero or + // negative is meaningless rather than a way to say "never execute". + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void Policy_RejectsNonPositiveWatchAttempts(int attempts) + => Assert.Throws(() => new RetryPolicy.Builder { MaxAttemptsOnWatchConflict = attempts }.Create()); + + // ...and 1 is accepted, since that is how re-attempting is switched off + [Fact] + public void Policy_AcceptsSingleWatchAttempt() + => Assert.Equal(1, new RetryPolicy.Builder { MaxAttemptsOnWatchConflict = 1 }.Create().MaxAttemptsOnWatchConflict); + + // The category cap must name exactly one of the CommandRetry* values: an empty value, or one that + // strays outside the category bits, is a usage error rather than something to interpret. + [Fact] + public void Policy_RejectsNonCategoryMaxCommandRetryCategory() + { + Assert.Throws(() => new RetryPolicy.Builder { MaxCommandRetryCategory = CommandFlags.None }.Create()); + Assert.Throws(() => new RetryPolicy.Builder { MaxCommandRetryCategory = CommandFlags.FireAndForget }.Create()); + Assert.Throws( + () => new RetryPolicy.Builder { MaxCommandRetryCategory = CommandFlags.CommandRetryReadOnly | CommandFlags.FireAndForget }.Create()); + + RetryPolicy valid = new RetryPolicy.Builder { MaxCommandRetryCategory = CommandFlags.CommandRetryAlways }; + Assert.Equal(CommandFlags.CommandRetryAlways, valid.MaxCommandRetryCategory); + } + + // RetryPolicy.None means *nothing* is re-attempted - including watch contention, which is bounded by + // an attempt count rather than by CanRetry (nothing was applied, so there is no fault to judge). + [Fact] + public void NonePolicy_DisablesWatchReattempts() + { + Assert.Equal(1, RetryPolicy.None.MaxAttemptsOnWatchConflict); + Assert.Equal(DefaultMaxAttemptsOnWatchConflict, RetryPolicy.Default.MaxAttemptsOnWatchConflict); + } + + private const int DefaultMaxAttemptsOnWatchConflict = 3; + + // Round-tripping a policy through a builder must preserve the watch budget along with everything else. + [Fact] + public void Policy_RoundTripsThroughBuilder() + { + RetryPolicy original = new RetryPolicy.Builder { MaxAttemptsOnWatchConflict = 7, MaxAttempts = 4 }; + var copy = new RetryPolicy.Builder(original).Create(); + + Assert.Equal(7, copy.MaxAttemptsOnWatchConflict); + Assert.Equal(4, copy.MaxAttempts); + } + + // Contention is not a fault, so there is no backoff - only jitter, to stop two callers colliding again + // in lock-step. With jitter disabled the re-attempt is immediate. + [Fact] + public async Task WatchConflictDelay_HasNoBackoff() + { + var controller = new RetryController( + new RetryPolicy.Builder + { + RetryDelay = TimeSpan.FromMilliseconds(LongMillis), + FailoverDelay = TimeSpan.FromMilliseconds(LongMillis), + JitterMax = TimeSpan.Zero, + }, + DatabaseFeatureFlags.Failover); + + var watch = Stopwatch.StartNew(); + await controller.WatchConflictDelayAsync(); + Assert.True(watch.ElapsedMilliseconds < ShortMillis, $"returned after {watch.ElapsedMilliseconds}ms"); + } + + // Capturing the "next failover" token costs something, so we only do it when a failover could + // actually be waited on: the database must offer failover, there must be more than one attempt, and + // the threshold must sit strictly below the attempt cap (at the cap it can never be reached). + [Theory] + [InlineData(3, 1, true, true)] + [InlineData(3, 1, false, false)] // no failover available + [InlineData(1, 1, true, false)] // single attempt: nothing to retry + [InlineData(3, 3, true, false)] // threshold == cap: unreachable + [InlineData(3, 4, true, false)] // threshold beyond cap: unreachable + public void TracksFailover_OnlyWhenReachable(int maxAttempts, int beforeFailover, bool withFailover, bool expected) + { + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = maxAttempts, MaxAttemptsBeforeFailover = beforeFailover }; + Assert.Equal(expected, new RetryController(policy, Features(withFailover)).TracksFailover); + } + + // MaxAttempts = 1 means "try once": the very first failure is already exhausted. + [Fact] + public void SingleAttempt_NeverRetries() + { + var controller = new RetryController(new RetryPolicy.Builder { MaxAttempts = 1 }, DatabaseFeatureFlags.Failover); + using var cts = new CancellationTokenSource(); + var failover = cts.Token; + var fault = new RedisServerException(RedisErrorKind.Loading, CommandFlags.CommandRetryReadOnly, "LOADING"); + + Assert.False(controller.CanRetry(1, fault, ref failover, out var delay)); + Assert.False(delay.CanBeCanceled); + } + + // --- FailoverOrDelayAsync ----------------------------------------------------------------------- + // Deliberately coarse thresholds: we are distinguishing "waited for the configured period" from + // "returned as soon as it could", not measuring the clock. + private const int LongMillis = 2000, ShortMillis = 1000; + + // No failover token: this is a routine pause between same-server attempts, so it waits RetryDelay. + [Fact] + public async Task Delay_WithoutFailoverToken_WaitsRetryDelay() + { + var controller = new RetryController( + new RetryPolicy.Builder { RetryDelay = TimeSpan.FromMilliseconds(LongMillis), JitterMax = TimeSpan.Zero }, + DatabaseFeatureFlags.None); + + var watch = Stopwatch.StartNew(); + await controller.FailoverOrDelayAsync(CancellationToken.None); + Assert.True(watch.ElapsedMilliseconds >= ShortMillis, $"returned after {watch.ElapsedMilliseconds}ms"); + } + + // A failover token that has *already* fired: there is nothing to wait for, so only jitter applies - + // and in particular RetryDelay is deliberately ignored on the failover path. + [Fact] + public async Task Delay_WithFiredFailoverToken_ReturnsImmediately() + { + var controller = new RetryController( + new RetryPolicy.Builder + { + RetryDelay = TimeSpan.FromMilliseconds(LongMillis), + FailoverDelay = TimeSpan.FromMilliseconds(LongMillis), + JitterMax = TimeSpan.Zero, + }, + DatabaseFeatureFlags.Failover); + + using var cts = new CancellationTokenSource(); + cts.Cancel(); // Cancel, not CancelAsync: this project also targets net481 + + var watch = Stopwatch.StartNew(); + await controller.FailoverOrDelayAsync(cts.Token); + Assert.True(watch.ElapsedMilliseconds < ShortMillis, $"returned after {watch.ElapsedMilliseconds}ms"); + } + + // A failover that arrives while we are waiting: we stop waiting as soon as it lands, rather than + // sitting out the whole FailoverDelay. + [Fact] + public async Task Delay_WhenFailoverArrives_StopsWaiting() + { + var controller = new RetryController( + new RetryPolicy.Builder { FailoverDelay = TimeSpan.FromMilliseconds(LongMillis * 4), JitterMax = TimeSpan.Zero }, + DatabaseFeatureFlags.Failover); + + using var cts = new CancellationTokenSource(); + var watch = Stopwatch.StartNew(); + var pending = controller.FailoverOrDelayAsync(cts.Token); + cts.Cancel(); // Cancel, not CancelAsync: this project also targets net481 + await pending; + + Assert.True(watch.ElapsedMilliseconds < ShortMillis, $"returned after {watch.ElapsedMilliseconds}ms"); + } + + // A failover that never arrives: we give it FailoverDelay and then proceed anyway (retrying on the + // original server is better than giving up). + [Fact] + public async Task Delay_WhenFailoverNeverArrives_ProceedsAfterFailoverDelay() + { + var controller = new RetryController( + new RetryPolicy.Builder { FailoverDelay = TimeSpan.FromMilliseconds(LongMillis), JitterMax = TimeSpan.Zero }, + DatabaseFeatureFlags.Failover); + + using var cts = new CancellationTokenSource(); + var watch = Stopwatch.StartNew(); + await controller.FailoverOrDelayAsync(cts.Token); + + Assert.True(watch.ElapsedMilliseconds >= ShortMillis, $"returned after {watch.ElapsedMilliseconds}ms"); + Assert.False(cts.IsCancellationRequested); // no failover ever happened + } +} diff --git a/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs new file mode 100644 index 000000000..8f2b6b623 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/RetryTests/RetryEndToEndTests.cs @@ -0,0 +1,850 @@ +using System; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using StackExchange.Redis.Server; +using Xunit; + +namespace StackExchange.Redis.Tests.RetryTests; + +[RunPerProtocol] +public class RetryEndToEndTests(ITestOutputHelper log) : TestBase(log) +{ + // End-to-end: a server that answers the first couple of GETs with a transient LOADING error, then + // serves normally. Wrapping the database with .WithRetry should transparently ride through the LOADING + // responses; we can then observe (via the server's counter) that it really did take three GETs before + // one succeeded. + [Fact] + public async Task WithRetry_RidesOutTransientLoading() + { + using var server = new LoadingServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + Assert.True(conn.IsConnected); + + var db = conn.GetDatabase(); + + RedisKey key = "retry:loading"; + Assert.True(await db.StringSetAsync(key, "hello")); // seed the value before we start failing GETs + + // queue up two LOADING responses; the third GET should succeed + server.LoadingOps = 2; + + // zero delay/jitter so the test isn't paying the default ~1s retry backoff between attempts + RetryPolicy policy = new RetryPolicy.Builder + { + MaxAttempts = 3, + RetryDelay = TimeSpan.Zero, + JitterMax = TimeSpan.Zero, + }; + var retryDb = db.WithRetry(policy); + + var value = await retryDb.StringGetAsync(key); + + Assert.Equal("hello", value); // retries rode out the LOADING responses + Assert.Equal(0, server.LoadingOps); // both LOADING responses were consumed + Assert.Equal(3, server.GetOpsReceived); // 2 x LOADING + 1 x success + } + + // Retries are bounded: once MaxAttempts is used up the original server fault is surfaced to the + // caller unchanged (not wrapped, not swallowed), and the server saw exactly MaxAttempts requests. + [Fact] + public async Task WithRetry_WhenAttemptsExhausted_ThrowsOriginalFault() + { + using var server = new LoadingServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var db = conn.GetDatabase(); + RedisKey key = "retry:exhaust"; + Assert.True(await db.StringSetAsync(key, "hello")); + + server.LoadingOps = 100; // more than we will ever attempt + + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; + var ex = await Assert.ThrowsAsync(async () => await db.WithRetry(policy).StringGetAsync(key)); + + Assert.Equal(RedisErrorKind.Loading, ex.Kind); + Assert.Equal(3, server.GetOpsReceived); // tried exactly MaxAttempts times + } + + // A fault that will not fix itself is not worth repeating: WRONGTYPE is an application error, so it + // surfaces on the first attempt regardless of how many attempts the policy allows. + [Fact] + public async Task WithRetry_NonTransientFault_IsNotRetried() + { + using var server = new LoadingServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var db = conn.GetDatabase(); + RedisKey key = "retry:wrongtype"; + Assert.True(await db.StringSetAsync(key, "hello")); + + server.LoadingOps = 100; + server.ErrorText = "WRONGTYPE Operation against a key holding the wrong kind of value"; + + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; + var ex = await Assert.ThrowsAsync(async () => await db.WithRetry(policy).StringGetAsync(key)); + + Assert.Equal(RedisErrorKind.WrongType, ex.Kind); + Assert.Equal(1, server.GetOpsReceived); // gave up immediately + } + + // An ad-hoc command whose *name* is recognised gets that command's category for free, so a plain + // Execute("get", ...) is retried like any other read - the caller does not have to say anything. + [Fact] + public async Task WithRetry_AdHocCommand_InheritsKnownCommandCategory() + { + using var server = new LoadingServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var db = conn.GetDatabase(); + RedisKey key = "retry:adhoc:known"; + Assert.True(await db.StringSetAsync(key, "hello")); + + server.LoadingOps = 2; + + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; + var result = await db.WithRetry(policy).ExecuteAsync("get", [key]); + + Assert.Equal("hello", result.AsString()); + Assert.Equal(3, server.GetOpsReceived); // recognised as GET, i.e. read-only, so retried + } + + // A command the library does *not* recognise could do anything, so it is treated pessimistically and + // never retried; a caller who knows better can say so via the flags. + [Theory] + [InlineData(CommandFlags.None, 1)] // unrecognised: assume the worst, do not retry + [InlineData(CommandFlags.CommandRetryReadOnly, 3)] // caller asserts it is a pure read + public async Task WithRetry_UnrecognisedCommand_RespectsSuppliedCategory(CommandFlags flags, int expectedAttempts) + { + using var server = new LoadingServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var db = conn.GetDatabase(); + server.LoadingOps = 2; // the third attempt would succeed, if we are allowed a third + + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; + var retryDb = db.WithRetry(policy); + + if (expectedAttempts == 1) + { + await Assert.ThrowsAsync(async () => await retryDb.ExecuteAsync("notarealcommand", [], flags)); + } + else + { + var result = await retryDb.ExecuteAsync("notarealcommand", [], flags); + Assert.Equal("made-up-ok", result.AsString()); + } + + Assert.Equal(expectedAttempts, server.UnknownOpsReceived); + } + + // Multi-group + WithRetry: two backends hold *different* values for the same key. The group is weighted + // towards A, so a normal read returns A's value. When A drops into LOADING, the faults trip A's + // (deliberately hair-trigger) circuit-breaker; the group reroutes to B, and the retry wrapper rides that + // failover transparently - the very same StringGet now returns B's value, with no caller intervention and + // no configuration change. + [Fact] + public async Task WithRetry_FailsOverBetweenGroupsOnLoading() + { + EndPoint alpha = new DnsEndPoint("alpha", 6379); + EndPoint bravo = new DnsEndPoint("bravo", 6379); + using var serverA = new InProcessTestServer(Output, endpoint: alpha); + using var serverB = new InProcessTestServer(Output, endpoint: bravo); + + RedisKey key = "retry:multigroup"; + + // seed each backend with its own distinct value (direct connections, so each cache gets its own) + await using (var seedA = await serverA.ConnectAsync()) + { + Assert.True(await seedA.GetDatabase().StringSetAsync(key, "from-A")); + } + await using (var seedB = await serverB.ConnectAsync()) + { + Assert.True(await seedB.GetDatabase().StringSetAsync(key, "from-B")); + } + + var probe = new ControllableProbe(); + + // A carries a hair-trigger breaker (trips on the first fault); B keeps the default (never trips here) + var configA = serverA.GetClientConfig(); + configA.CircuitBreaker = new CircuitBreaker.Builder + { + MinimumNumberOfFailures = 1, + FailureRateThreshold = 1, + }; + + ConnectionGroupMember[] members = + [ + new(configA, "A") { Weight = 9 }, // highest weight -> initially active + new(serverB.GetClientConfig(), "B") { Weight = 1 }, // failover target + ]; + + MultiGroupOptions options = new MultiGroupOptions.Builder + { + HealthCheckInterval = TimeSpan.FromMinutes(30), // huge: the breaker fast-path is what reroutes us + HealthCheck = new HealthCheck.Builder + { + Probe = probe, + ProbeCount = 1, + ProbeTimeout = TimeSpan.FromSeconds(5), + }, + }; + + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + Assert.True(conn.IsConnected); + Assert.Same(members[0], conn.ActiveMember); // A is active (highest weight) + + // failover enabled, plenty of attempts, no artificial delay between them + RetryPolicy policy = new RetryPolicy.Builder + { + MaxAttempts = 20, + MaxAttemptsBeforeFailover = 1, + RetryDelay = TimeSpan.Zero, + JitterMax = TimeSpan.Zero, + }; + var db = conn.GetDatabase().WithRetry(policy); + + // normal read: routed to the active (weighted) member, A + string? before = await db.StringGetAsync(key); + Assert.Equal("from-A", before); + + // knock A into LOADING and hold it down; nothing else about the setup changes + serverA.IsLoading = true; + probe.MarkDown(alpha); + + // the *same* call now transparently rides the circuit-break failover across to B + string? after = await db.StringGetAsync(key); + Assert.Equal("from-B", after); + Assert.Same(members[1], conn.ActiveMember); // we really did move to B + } + + // The unhappy path of the failover threshold: the attempt count says "wait for a failover now", but no + // failover ever comes (the member stays nominally healthy - a couple of LOADING replies are not enough + // to trip the default breaker). We wait out FailoverDelay and then carry on retrying the original + // member rather than giving up, and the group never moves. + [Fact] + public async Task WithRetry_WhenFailoverNeverArrives_KeepsRetryingSameMember() + { + EndPoint alpha = new DnsEndPoint("alpha", 6379); + EndPoint bravo = new DnsEndPoint("bravo", 6379); + using var serverA = new LoadingServer(Output, endpoint: alpha); + using var serverB = new InProcessTestServer(Output, endpoint: bravo); + + RedisKey key = "retry:nofailover"; + await using (var seedA = await serverA.ConnectAsync()) + { + Assert.True(await seedA.GetDatabase().StringSetAsync(key, "from-A")); + } + + ConnectionGroupMember[] members = + [ + new(serverA.GetClientConfig(), "A") { Weight = 9 }, // highest weight -> active, and stays active + new(serverB.GetClientConfig(), "B") { Weight = 1 }, + ]; + + MultiGroupOptions options = new MultiGroupOptions.Builder + { + HealthCheckInterval = TimeSpan.FromMinutes(30), + HealthCheck = new HealthCheck.Builder + { + Probe = new ControllableProbe(), // never marked down + ProbeCount = 1, + ProbeTimeout = TimeSpan.FromSeconds(5), + }, + }; + + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + Assert.Same(members[0], conn.ActiveMember); + + // failover is armed after the first attempt, but nothing will ever trigger it; keep the wait short + RetryPolicy policy = new RetryPolicy.Builder + { + MaxAttempts = 5, + MaxAttemptsBeforeFailover = 1, + FailoverDelay = TimeSpan.FromMilliseconds(200), + RetryDelay = TimeSpan.Zero, + JitterMax = TimeSpan.Zero, + }; + var db = conn.GetDatabase().WithRetry(policy); + + serverA.LoadingOps = 2; // two transient faults, then A answers normally + + Assert.Equal("from-A", await db.StringGetAsync(key)); + Assert.Equal(3, serverA.GetOpsReceived); // 2 x LOADING + 1 x success, all on A + Assert.Same(members[0], conn.ActiveMember); // never moved to B + } + + // End-to-end for a retryable transaction: a server that fails the first EXEC with a transient LOADING + // error (discarding that attempt's queued commands), then commits the second. A transaction created via + // the retrying database should ride this out: the per-operation tasks handed out at build time resolve + // from the winning (second) attempt, the value is actually written, and the server saw two EXECs. + [Fact] + public async Task WithRetry_Transaction_RidesOutTransientExec() + { + using var server = new ExecFailServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + Assert.True(conn.IsConnected); + + var db = conn.GetDatabase(); + + RedisKey key = "retry:tran"; + Assert.True(await db.StringSetAsync(key, "seed")); // seed a value we expect the transaction to overwrite + + server.FailExecOps = 1; // fail the first EXEC; the second should commit + + RetryPolicy policy = new RetryPolicy.Builder + { + MaxAttempts = 3, + RetryDelay = TimeSpan.Zero, + JitterMax = TimeSpan.Zero, + }; + var retryDb = db.WithRetry(policy); + + var tran = retryDb.CreateTransaction(); + var setTask = tran.StringSetAsync(key, "committed"); + var getTask = tran.StringGetAsync(key); + bool committed = await tran.ExecuteAsync(); + + Assert.True(committed); // rode out the transient EXEC failure + Assert.True(await setTask); // per-op proxy resolved from the winning attempt + Assert.Equal("committed", await getTask); // read reflects the committed value + Assert.Equal("committed", await db.StringGetAsync(key)); // and it really landed on the server + Assert.Equal(0, server.FailExecOps); // the transient failure was consumed + Assert.Equal(2, server.ExecOpsReceived); // 1 x LOADING + 1 x commit + } + + // A transaction's effective retry category is the most side-effecting of its operations, so an INCR + // makes the whole thing "accumulating" - beyond what the default policy (capped at write-last-wins) + // would normally repeat. But a LOADING reply to EXEC *proves* the server discarded the transaction + // wholesale, so replaying it cannot double-count: the category cap does not apply, and even the + // default policy rides it out. Without that distinction, most interesting transactions (i.e. the ones + // that mutate something) would never be retryable at all. + [Fact] + public async Task WithRetry_Transaction_RejectedExec_RetriesRegardlessOfCategory() + { + using var server = new ExecFailServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + Assert.True(conn.IsConnected); + + var db = conn.GetDatabase(); + RedisKey key = "retry:tran:incr"; + + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; + Assert.Equal(CommandFlags.CommandRetryWriteLastWins, policy.MaxCommandRetryCategory); // i.e. the default + + server.FailExecOps = 1; + var tran = db.WithRetry(policy).CreateTransaction(); + var incr = tran.StringIncrementAsync(key); + + Assert.True(await tran.ExecuteAsync()); + Assert.Equal(1, await incr); // the discarded attempt applied nothing -> incremented exactly once + Assert.Equal(2, server.ExecOpsReceived); // 1 x LOADING + 1 x commit + } + + // The other half of that story: when the outcome is genuinely *ambiguous*, the category still bites. + // OOM is deliberately *not* treated as "known not applied": a Lua script can hit the memory limit + // part-way through, having already written something, and it reports the inner error verbatim - so from + // the client's side an OOM reply proves nothing about whether the command took effect. Here the server + // models exactly that: it applies the INCR and *then* reports OOM. Under the default cap the fault + // surfaces after one attempt; raising the cap opts into replaying it, and the value shows the + // triple-count that the cap exists to prevent. + [Theory] + [InlineData(false, 1)] // default cap: not repeated + [InlineData(true, 3)] // caller opted in: repeated, and every attempt landed + public async Task WithRetry_AmbiguousFault_IsStillGatedByCategory(bool allowAccumulating, int expectedValue) + { + using var server = new AppliedThenFailedServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var db = conn.GetDatabase(); + RedisKey key = "retry:ambiguous:incr"; + + RetryPolicy policy = new RetryPolicy.Builder + { + MaxAttempts = 3, + RetryDelay = TimeSpan.Zero, + JitterMax = TimeSpan.Zero, + MaxCommandRetryCategory = allowAccumulating + ? CommandFlags.CommandRetryWriteAccumulating + : RetryPolicy.Default.MaxCommandRetryCategory, + }; + + var ex = await Assert.ThrowsAsync(async () => await db.WithRetry(policy).StringIncrementAsync(key)); + Assert.Equal(RedisErrorKind.OutOfMemory, ex.Kind); + + Assert.Equal(expectedValue, server.IncrOpsReceived); + server.FailIncr = false; + Assert.Equal(expectedValue, (long)await db.StringGetAsync(key)); // and each one really did apply + } + + // A WATCH constraint is replayed on every attempt. Here the condition is satisfied, and the first EXEC + // hits a transient LOADING; the transaction should ride it out and the *durable* ConditionResult handed + // back at build time should reflect the winning attempt (satisfied). Confirms the condition survives the + // discard+replay and that its outcome is forwarded onto the caller-visible result. + [Fact] + public async Task WithRetry_Transaction_SatisfiedCondition_RidesOutTransientExec() + { + using var server = new ExecFailServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + Assert.True(conn.IsConnected); + + var db = conn.GetDatabase(); + RedisKey key = "retry:tran:cond"; + Assert.True(await db.StringSetAsync(key, "seed")); + + server.FailExecOps = 1; // fail the first EXEC; the condition must still hold on the replay + + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; + var tran = db.WithRetry(policy).CreateTransaction(); + var cond = tran.AddCondition(Condition.StringEqual(key, "seed")); // satisfied on both attempts + var setTask = tran.StringSetAsync(key, "committed"); + bool committed = await tran.ExecuteAsync(); + + Assert.True(committed); // rode out the transient EXEC failure + Assert.True(cond.WasSatisfied); // durable condition result forwarded from the winning attempt + Assert.True(await setTask); // per-op proxy resolved + Assert.Equal("committed", await db.StringGetAsync(key)); // the write really landed + Assert.Equal(2, server.ExecOpsReceived); // 1 x LOADING + 1 x commit -> the WATCH replayed + } + + // A WATCH constraint that is *not* satisfied is a business outcome, not a transient fault: the transaction + // aborts electively (no EXEC is ever issued), the per-operation proxies are cancelled, and - crucially - + // it is NOT retried. Confirms the ForwardSuccess cancellation branch and that a failed WATCH doesn't loop. + [Fact] + public async Task WithRetry_Transaction_UnsatisfiedCondition_AbortsWithoutRetry() + { + using var server = new ExecFailServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + Assert.True(conn.IsConnected); + + var db = conn.GetDatabase(); + RedisKey key = "retry:tran:cond:fail"; + Assert.True(await db.StringSetAsync(key, "seed")); + + server.FailExecOps = 0; // no transient fault; the condition itself aborts the transaction + + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; + var tran = db.WithRetry(policy).CreateTransaction(); + var cond = tran.AddCondition(Condition.StringEqual(key, "different")); // NOT satisfied + var setTask = tran.StringSetAsync(key, "committed"); + bool committed = await tran.ExecuteAsync(); + + Assert.False(committed); // electively aborted via the failed WATCH + Assert.False(cond.WasSatisfied); + await Assert.ThrowsAnyAsync(async () => await setTask); // proxy cancelled, not hung + Assert.Equal("seed", await db.StringGetAsync(key)); // nothing was written + Assert.Equal(0, server.ExecOpsReceived); // never EXEC'd, and (the point) never retried + } + + // A committed transaction can still carry a per-command error (e.g. INCR against a non-numeric string + // errors while EXEC itself succeeds). committed is true, the good op resolves, and only the offending op's + // proxy faults - exercising the ForwardSuccess faulted branch, which the transient-EXEC tests never hit. + [Fact] + public async Task WithRetry_Transaction_PerOpError_FaultsOnlyThatProxy() + { + using var server = new ExecFailServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + Assert.True(conn.IsConnected); + + var db = conn.GetDatabase(); + RedisKey badKey = "retry:tran:notnum"; + RedisKey goodKey = "retry:tran:str"; + Assert.True(await db.StringSetAsync(badKey, "abc")); // non-numeric: INCR will error at EXEC time + + server.FailExecOps = 0; // EXEC commits; one queued op errors at execution time + + // allow accumulating so the INCR doesn't gate retries - though nothing here retries anyway (EXEC commits) + RetryPolicy policy = new RetryPolicy.Builder + { + MaxAttempts = 3, + RetryDelay = TimeSpan.Zero, + JitterMax = TimeSpan.Zero, + MaxCommandRetryCategory = CommandFlags.CommandRetryWriteAccumulating, + }; + var tran = db.WithRetry(policy).CreateTransaction(); + var badTask = tran.StringIncrementAsync(badKey); // errors at EXEC: INCR on a non-numeric value + var goodTask = tran.StringSetAsync(goodKey, "ok"); + bool committed = await tran.ExecuteAsync(); + + Assert.True(committed); // EXEC still committed + Assert.True(await goodTask); // the good op resolved + await Assert.ThrowsAsync(async () => await badTask); // only the bad op faulted + Assert.Equal("ok", await db.StringGetAsync(goodKey)); // the good write landed + Assert.Equal(1, server.ExecOpsReceived); // committed first time; no retry + } + + // Multi-group + a retryable transaction: A fails every EXEC and is knocked out, so the group reroutes to B. + // Because RetryTransaction builds a *fresh* inner transaction against the currently-active member on each + // attempt, the replay lands on B and commits there - the transaction analogue of the single-command + // WithRetry_FailsOverBetweenGroupsOnLoading test. + [Fact] + public async Task WithRetry_Transaction_FailsOverBetweenGroups() + { + EndPoint alpha = new DnsEndPoint("alpha", 6379); + EndPoint bravo = new DnsEndPoint("bravo", 6379); + using var serverA = new ExecFailServer(Output, endpoint: alpha); // EXEC always fails on A + using var serverB = new InProcessTestServer(Output, endpoint: bravo); + + RedisKey key = "retry:tran:failover"; + + var probe = new ControllableProbe(); + + // A carries a hair-trigger breaker (trips on the first fault); B keeps the default + var configA = serverA.GetClientConfig(); + configA.CircuitBreaker = new CircuitBreaker.Builder + { + MinimumNumberOfFailures = 1, + FailureRateThreshold = 1, + }; + + ConnectionGroupMember[] members = + [ + new(configA, "A") { Weight = 9 }, // highest weight -> initially active + new(serverB.GetClientConfig(), "B") { Weight = 1 }, // failover target + ]; + + MultiGroupOptions options = new MultiGroupOptions.Builder + { + HealthCheckInterval = TimeSpan.FromMinutes(30), // huge: the breaker fast-path is what reroutes us + HealthCheck = new HealthCheck.Builder + { + Probe = probe, + ProbeCount = 1, + ProbeTimeout = TimeSpan.FromSeconds(5), + }, + }; + + await using var conn = await ConnectionMultiplexer.ConnectGroupAsync(members, options); + Assert.True(conn.IsConnected); + Assert.Same(members[0], conn.ActiveMember); // A is active (highest weight) + + RetryPolicy policy = new RetryPolicy.Builder + { + MaxAttempts = 20, + MaxAttemptsBeforeFailover = 1, + RetryDelay = TimeSpan.Zero, + JitterMax = TimeSpan.Zero, + }; + var db = conn.GetDatabase().WithRetry(policy); + + // A will fail every EXEC; knock it out so the group reroutes to B + serverA.FailExecOps = int.MaxValue; + probe.MarkDown(alpha); + + var tran = db.CreateTransaction(); + var setTask = tran.StringSetAsync(key, "committed-on-B"); + bool committed = await tran.ExecuteAsync(); + + Assert.True(committed); // rode the failover across to B and committed there + Assert.True(await setTask); + Assert.Same(members[1], conn.ActiveMember); // we really did move to B + + // the write landed on B, not A + await using var checkB = await serverB.ConnectAsync(); + Assert.Equal("committed-on-B", await checkB.GetDatabase().StringGetAsync(key)); + } + + // Replay has to be repeatable, not just possible once: two transient EXEC failures in a row, with a + // mixed bag of operations, and every proxy still resolves from the third (winning) attempt. + [Fact] + public async Task WithRetry_Transaction_ReplaysRepeatedly() + { + using var server = new ExecFailServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var db = conn.GetDatabase(); + RedisKey key = "retry:tran:repeat", counter = "retry:tran:repeat:count", list = "retry:tran:repeat:list"; + Assert.True(await db.StringSetAsync(key, "seed")); + + server.FailExecOps = 2; // fail twice; the third EXEC commits + + RetryPolicy policy = new RetryPolicy.Builder + { + MaxAttempts = 4, + RetryDelay = TimeSpan.Zero, + JitterMax = TimeSpan.Zero, + MaxCommandRetryCategory = CommandFlags.CommandRetryWriteAccumulating, // INCR/LPUSH are accumulating + }; + var tran = db.WithRetry(policy).CreateTransaction(); + var cond = tran.AddCondition(Condition.StringEqual(key, "seed")); + var set = tran.StringSetAsync(key, "committed"); + var incr = tran.StringIncrementAsync(counter); + var push = tran.ListLeftPushAsync(list, "item"); + var get = tran.StringGetAsync(key); + + Assert.True(await tran.ExecuteAsync()); + + Assert.True(cond.WasSatisfied); + Assert.True(await set); + Assert.Equal(1, await incr); // the two discarded attempts applied nothing + Assert.Equal(1, await push); + Assert.Equal("committed", await get); + Assert.Equal(3, server.ExecOpsReceived); // 2 x LOADING + 1 x commit + } + + // When a transaction runs out of attempts, the failure must reach *every* per-operation proxy as well + // as the ExecuteAsync caller; a proxy left unresolved would hang the caller forever. + [Fact] + public async Task WithRetry_Transaction_WhenAttemptsExhausted_FaultsEveryProxy() + { + using var server = new ExecFailServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var db = conn.GetDatabase(); + RedisKey key = "retry:tran:exhaust"; + + server.FailExecOps = 100; // never succeeds + + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 2, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; + var tran = db.WithRetry(policy).CreateTransaction(); + var set = tran.StringSetAsync(key, "never"); + var get = tran.StringGetAsync(key); + + var fault = await Assert.ThrowsAsync(async () => await tran.ExecuteAsync()); + Assert.Equal(RedisErrorKind.Loading, fault.Kind); + + // both proxies carry the same terminal fault rather than being left pending + Assert.Same(fault, await Assert.ThrowsAsync(async () => await set)); + Assert.Same(fault, await Assert.ThrowsAsync(async () => await get)); + Assert.Equal(2, server.ExecOpsReceived); // exactly MaxAttempts + } + + // WATCH drift under retry: the condition holds, so EXEC really is issued, but the server refuses it + // because a watched key moved. Nothing was applied and nothing faulted - we simply lost a race - so + // the transaction is re-attempted (re-reading the condition), and the second attempt commits. This is + // contention, not a fault, so the fault budget is untouched and the side-effect category is irrelevant. + [Fact] + public async Task WithRetry_Transaction_WatchDrift_IsReattempted() + { + using var server = new ExecFailServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + Assert.True(conn.IsConnected); + + var db = conn.GetDatabase(); + RedisKey key = "retry:tran:drift"; + Assert.True(await db.StringSetAsync(key, "seed")); + + server.DriftKey = key; + server.DriftOps = 1; // the next EXEC observes a concurrent write to the watched key + + // MaxAttempts = 1: no *fault* retries at all, proving the watch budget is separate + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 1, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; + var tran = db.WithRetry(policy).CreateTransaction(); + var cond = tran.AddCondition(Condition.StringEqual(key, "seed")); + var setTask = tran.StringSetAsync(key, "committed"); + + var execute = tran.ExecuteAsync(); + if (await Task.WhenAny(execute, Task.Delay(5000)) != execute) + { + Assert.Fail("ExecuteAsync never completed"); + } + + Assert.True(await execute); // rode out the lost race + Assert.True(cond.WasSatisfied); + Assert.True(await setTask); + Assert.False(tran.WasWatchConflict); // reports the *final* attempt: we got there in the end + Assert.Equal(2, server.ExecOpsReceived); // 1 x watch conflict + 1 x commit + Assert.Equal("committed", await db.StringGetAsync(key)); + } + + // Watch contention is bounded, and the bound is opt-out-able: with MaxAttemptsOnWatchConflict = 1 a + // conflict aborts exactly as it did before. ExecuteAsync reports false with every condition satisfied + // (which is what distinguishes drift from an elective abort), and the per-operation proxies are + // cancelled rather than left dangling - the case that used to hang the caller outright. + [Fact] + public async Task WithRetry_Transaction_WatchDrift_CanBeDisabled() + { + using var server = new ExecFailServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var db = conn.GetDatabase(); + RedisKey key = "retry:tran:drift:off"; + Assert.True(await db.StringSetAsync(key, "seed")); + + server.DriftKey = key; + server.DriftOps = 1; + + RetryPolicy policy = new RetryPolicy.Builder + { + MaxAttempts = 3, + MaxAttemptsOnWatchConflict = 1, // i.e. do not re-attempt on contention + RetryDelay = TimeSpan.Zero, + JitterMax = TimeSpan.Zero, + }; + var tran = db.WithRetry(policy).CreateTransaction(); + var cond = tran.AddCondition(Condition.StringEqual(key, "seed")); + var setTask = tran.StringSetAsync(key, "committed"); + + var execute = tran.ExecuteAsync(); + if (await Task.WhenAny(execute, Task.Delay(5000)) != execute) + { + Assert.Fail("ExecuteAsync never completed"); + } + + Assert.False(await execute); + Assert.True(cond.WasSatisfied); // the condition held; the server-side WATCH is what killed it + Assert.True(tran.WasWatchConflict); // ...and the caller can see exactly that + Assert.Equal(1, server.ExecOpsReceived); + await Assert.ThrowsAnyAsync(async () => await setTask); + Assert.Equal("seed", await db.StringGetAsync(key)); // nothing was applied + } + + // Contention that never clears must not loop forever: the server conflicts on every EXEC, so we give + // up after MaxAttemptsOnWatchConflict attempts and report the ordinary "did not commit" outcome. + [Fact] + public async Task WithRetry_Transaction_PersistentWatchDrift_GivesUp() + { + using var server = new ExecFailServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var db = conn.GetDatabase(); + RedisKey key = "retry:tran:drift:forever"; + Assert.True(await db.StringSetAsync(key, "seed")); + + server.DriftKey = key; + server.DriftOps = int.MaxValue; // every EXEC loses the race + + RetryPolicy policy = new RetryPolicy.Builder + { + MaxAttempts = 3, + MaxAttemptsOnWatchConflict = 4, + RetryDelay = TimeSpan.Zero, + JitterMax = TimeSpan.Zero, + }; + var tran = db.WithRetry(policy).CreateTransaction(); + var cond = tran.AddCondition(Condition.StringEqual(key, "seed")); + var setTask = tran.StringSetAsync(key, "committed"); + + Assert.False(await tran.ExecuteAsync()); + Assert.True(cond.WasSatisfied); + Assert.True(tran.WasWatchConflict); // still losing the race when we ran out of attempts + Assert.Equal(4, server.ExecOpsReceived); // bounded by MaxAttemptsOnWatchConflict + await Assert.ThrowsAnyAsync(async () => await setTask); + Assert.Equal("seed", await db.StringGetAsync(key)); + } + + // A transaction with no conditions has no WATCH, so it can never lose a watch race; the watch budget + // must not be spent on the ordinary "aborted" path. (Belt and braces: the aggregate outcome here is a + // clean commit, so this mostly guards against the budget logic firing on a false positive.) + [Fact] + public async Task WithRetry_Transaction_WithoutConditions_IgnoresWatchBudget() + { + using var server = new ExecFailServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var db = conn.GetDatabase(); + RedisKey key = "retry:tran:nocond"; + + RetryPolicy policy = new RetryPolicy.Builder { MaxAttempts = 1, MaxAttemptsOnWatchConflict = 5, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; + var tran = db.WithRetry(policy).CreateTransaction(); + var set = tran.StringSetAsync(key, "committed"); + + Assert.True(await tran.ExecuteAsync()); + Assert.True(await set); + Assert.Equal(1, server.ExecOpsReceived); + } + + // An in-proc server that *applies* each INCR and then reports OOM: the write happened, but the client + // has no way to know that. Counts the applications so a test can tell "the client gave up" from "the + // client repeated a write it could not account for". + private sealed class AppliedThenFailedServer(ITestOutputHelper? log) : InProcessTestServer(log) + { + private int _incrOpsReceived; + + public int IncrOpsReceived => Volatile.Read(ref _incrOpsReceived); + + public bool FailIncr { get; set; } = true; + + protected override TypedRedisValue Incr(RedisClient client, in RedisRequest request) + { + var applied = base.Incr(client, in request); + if (!FailIncr) return applied; + + Interlocked.Increment(ref _incrOpsReceived); + return TypedRedisValue.Error("OOM command not allowed when used memory > 'maxmemory'"); + } + } + + // An in-proc server that fails the first FailExecOps EXEC operations with a transient LOADING error, + // discarding that attempt's queued commands so nothing is applied, then commits normally. It can + // also simulate WATCH drift (a concurrent write to DriftKey immediately before EXEC is processed), + // which is a clean server-side rejection rather than a fault. + private sealed class ExecFailServer(ITestOutputHelper? log, EndPoint? endpoint = null) : InProcessTestServer(log, endpoint) + { + public int ExecOpsReceived { get; private set; } + + public int FailExecOps { get; set; } + + public int DriftOps { get; set; } + + public RedisKey DriftKey { get; set; } + + protected override TypedRedisValue Exec(RedisClient client, in RedisRequest request) + { + ExecOpsReceived++; + + if (FailExecOps > 0) + { + FailExecOps--; + client.Discard(); // drop this attempt's queued commands; nothing is applied + return TypedRedisValue.Error("LOADING Redis is loading the dataset in memory"); + } + + if (DriftOps > 0) + { + DriftOps--; + client.Touch(client.Database, DriftKey); // as if another connection wrote the watched key + } + + return base.Exec(client, in request); + } + } + + // An in-proc server that fails the first LoadingOps GET operations with a transient LOADING error + // (decrementing the counter each time), then serves normally. Every GET bumps GetOpsReceived so the + // test can confirm how many attempts actually reached the server. + private sealed class LoadingServer(ITestOutputHelper? log, EndPoint? endpoint = null) : InProcessTestServer(log, endpoint) + { + // the server core processes operations under a lock (single-threaded, like Redis), so plain fields + // are fine here + public int GetOpsReceived { get; private set; } + + public int LoadingOps { get; set; } + + // the error to reply with; transient by default, but overridable so the same harness can present + // a fault that is *not* worth retrying + public string ErrorText { get; set; } = "LOADING Redis is loading the dataset in memory"; + + public int UnknownOpsReceived { get; private set; } + + // a command the *client* cannot categorise; answered from the same LoadingOps budget so the retry + // decision is isolated to the category, not the error kind + public override TypedRedisValue OnUnknownCommand(RedisClient client, in RedisRequest request, ReadOnlySpan command) + { + UnknownOpsReceived++; + + if (LoadingOps > 0) + { + LoadingOps--; + return TypedRedisValue.Error(ErrorText); + } + + return TypedRedisValue.SimpleString("made-up-ok"); + } + + protected override TypedRedisValue Get(RedisClient client, in RedisRequest request) + { + GetOpsReceived++; + + // while LOADING ops remain, consume one and reply with a transient LOADING error + if (LoadingOps > 0) + { + LoadingOps--; + return TypedRedisValue.Error(ErrorText); + } + + return base.Get(client, in request); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/RetryTests/RetryGuardTests.cs b/tests/StackExchange.Redis.Tests/RetryTests/RetryGuardTests.cs new file mode 100644 index 000000000..64ea62f7e --- /dev/null +++ b/tests/StackExchange.Redis.Tests/RetryTests/RetryGuardTests.cs @@ -0,0 +1,182 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using StackExchange.Redis.Availability; +using StackExchange.Redis.KeyspaceIsolation; +using Xunit; + +namespace StackExchange.Redis.Tests.RetryTests; + +// The rejection rules and hand-written members of the retry wrappers: what WithRetry refuses to wrap, +// what a retrying transaction refuses to do, and the members that are deliberately *not* retried +// (status probes, routing lookups, streaming scans) and so are implemented by hand. +public class RetryGuardTests(ITestOutputHelper log) : TestBase(log) +{ + private static RetryPolicy Policy() => new RetryPolicy.Builder { MaxAttempts = 3, RetryDelay = TimeSpan.Zero, JitterMax = TimeSpan.Zero }; + + // Retrying inside a batch or a transaction makes no sense (the individual operations are not being + // dispatched yet), and retry cannot be nested. All three are refused at wrap time. + [Fact] + public async Task WithRetry_RefusesBatchTransactionAndNesting() + { + using var server = new InProcessTestServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + var db = conn.GetDatabase(); + + Assert.Throws(() => db.CreateBatch().WithRetry(Policy())); + Assert.Throws(() => db.CreateTransaction().WithRetry(Policy())); + Assert.Throws(() => db.WithRetry(Policy()).WithRetry(Policy())); + } + + // A database's asyncState is stamped onto the task produced by a single dispatch. A retrying database + // hands back its own task spanning however many attempts it takes, so it cannot carry that state - + // and silently dropping it would be worse than refusing. + [Fact] + public async Task WithRetry_RefusesAsyncState() + { + using var server = new InProcessTestServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + object state = new(); + var ex = Assert.Throws(() => conn.GetDatabase(0, state).WithRetry(Policy())); + Log(ex.Message); + + // ...including via a key-prefixed view of such a database, which inherits the inner state + Assert.Throws(() => conn.GetDatabase(0, state).WithKeyPrefix("p:").WithRetry(Policy())); + + // and the same applies to a transaction created *from* a retrying database + var retryDb = conn.GetDatabase().WithRetry(Policy()); + Assert.Throws(() => retryDb.CreateTransaction(state)); + + // no state: fine + Assert.NotNull(conn.GetDatabase().WithKeyPrefix("p:").WithRetry(Policy())); + Assert.NotNull(retryDb.CreateTransaction()); + } + + // Key-prefixing composes with retry: the prefix is applied when the operation is captured, so it + // survives being replayed. (Only one nesting order is expressible, since WithKeyPrefix needs an + // IDatabase and WithRetry produces an async-only database.) + [Fact] + public async Task WithRetry_ComposesWithKeyPrefix() + { + using var server = new InProcessTestServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var db = conn.GetDatabase().WithKeyPrefix("pfx:").WithRetry(Policy()); + Assert.True(await db.StringSetAsync("key", "value")); + + // visible under the prefixed name from an unprefixed database + Assert.Equal("value", await conn.GetDatabase().StringGetAsync("pfx:key")); + } + + // The transaction lifecycle guards: execute once, and do not accept work afterwards. + [Fact] + public async Task RetryTransaction_RefusesReuse() + { + using var server = new InProcessTestServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var tran = conn.GetDatabase().WithRetry(Policy()).CreateTransaction(); + var set = tran.StringSetAsync("guard:key", "value"); + Assert.True(await tran.ExecuteAsync()); + Assert.True(await set); + + await Assert.ThrowsAsync(async () => await tran.ExecuteAsync()); + Assert.Throws(() => { _ = tran.StringSetAsync("guard:key", "again"); }); + Assert.Throws(() => tran.AddCondition(Condition.KeyExists("guard:key"))); + } + + // Nested transactions, and the cursored scans, cannot participate in a transaction at all. + [Fact] + public async Task RetryTransaction_RefusesNestingAndScans() + { + using var server = new InProcessTestServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var tran = conn.GetDatabase().WithRetry(Policy()).CreateTransaction(); + + Assert.Throws(() => tran.CreateTransaction()); + Assert.Throws(() => tran.HashScanAsync("k")); + Assert.Throws(() => tran.HashScanNoValuesAsync("k")); + Assert.Throws(() => tran.SetScanAsync("k")); + Assert.Throws(() => tran.SortedSetScanAsync("k")); + Assert.Throws(() => tran.VectorSetRangeEnumerateAsync("k", "a", "z")); + } + + // Scans *can* be used on a retrying database: they are cursored, so they cannot be captured and + // replayed as a unit, but rather than refusing them outright we forward straight through (giving up + // retry, keeping the scan working). Needs a real server: the managed one has no *SCAN support. + [Fact] + public async Task WithRetry_ForwardsScans() + { + await using var conn = Create(); + + var inner = conn.GetDatabase(); + RedisKey hash = Me() + ":hash", set = Me() + ":set", zset = Me() + ":zset"; + await inner.KeyDeleteAsync([hash, set, zset]); + await inner.HashSetAsync(hash, [new HashEntry("a", "1"), new HashEntry("b", "2")]); + await inner.SetAddAsync(set, ["x", "y"]); + await inner.SortedSetAddAsync(zset, [new SortedSetEntry("p", 1), new SortedSetEntry("q", 2)]); + + var db = inner.WithRetry(Policy()); + + Assert.Equal(2, await CountAsync(db.HashScanAsync(hash))); + Assert.Equal(2, await CountAsync(db.HashScanNoValuesAsync(hash))); + Assert.Equal(2, await CountAsync(db.SetScanAsync(set))); + Assert.Equal(2, await CountAsync(db.SortedSetScanAsync(zset))); + } + + // Commands with no return value (a bare Task, not Task) go through their own funnel in the retry + // database, and their own recorded-operation type inside a retrying transaction. Needs a real server: + // the managed one implements no void-shaped command, so the fault/replay variant of this cannot + // currently be driven in-process. + [Fact] + public async Task WithRetry_HandlesVoidOperations() + { + await using var conn = Create(); + + var inner = conn.GetDatabase(); + RedisKey key = Me(); + await inner.KeyDeleteAsync(key); + await inner.ListRightPushAsync(key, ["a", "b", "c", "d"]); + + var db = inner.WithRetry(Policy()); + await db.ListTrimAsync(key, 0, 1); // Task, not Task + Assert.Equal(2, await db.ListLengthAsync(key)); + + // and the same shape recorded into (and replayed by) a retrying transaction + var tran = db.CreateTransaction(); + var trim = tran.ListTrimAsync(key, 0, 0); + var length = tran.ListLengthAsync(key); + Assert.True(await tran.ExecuteAsync()); + + await trim; // the void proxy resolved rather than hanging + Assert.True(trim.IsCompletedSuccessfully); + Assert.Equal(1, await length); + } + + // The cheap status/routing members are pass-throughs rather than retried round-trips. + [Fact] + public async Task WithRetry_ForwardsProbes() + { + using var server = new InProcessTestServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var inner = conn.GetDatabase(); + var db = inner.WithRetry(Policy()); + RedisKey key = "probe:key"; + + Assert.Equal(inner.Database, db.Database); + Assert.Same(conn, db.Multiplexer); + Assert.True(db.IsConnected(key)); + Assert.NotNull(await db.IdentifyEndpointAsync(key)); + Log(db.ToString()!); // exercises the feature-flag description + } + + private static async Task CountAsync(IAsyncEnumerable source) + { + int count = 0; + await foreach (var _ in source) count++; + return count; + } +} diff --git a/tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj b/tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj index c14c149d6..f20a2646a 100644 --- a/tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj +++ b/tests/StackExchange.Redis.Tests/StackExchange.Redis.Tests.csproj @@ -1,6 +1,6 @@  - + net481;net8.0;net10.0 Exe StackExchange.Redis.Tests @@ -10,7 +10,7 @@ enable true - + $(DefineConstants);BUILD_CURRENT diff --git a/tests/StackExchange.Redis.Tests/TransactionWatchDriftTests.cs b/tests/StackExchange.Redis.Tests/TransactionWatchDriftTests.cs new file mode 100644 index 000000000..7b4f1bc26 --- /dev/null +++ b/tests/StackExchange.Redis.Tests/TransactionWatchDriftTests.cs @@ -0,0 +1,189 @@ +using System.Net; +using System.Threading.Tasks; +using RESPite.Messages; +using StackExchange.Redis.Server; +using Xunit; + +namespace StackExchange.Redis.Tests; + +/// +/// Covers the "WATCH drift" outcome of a conditional transaction: every condition was satisfied, so +/// MULTI/EXEC really was issued, but a watched key changed underneath us and the server +/// answered EXEC with a null array. This is distinct from an *elective* abort (a condition that +/// failed, where no EXEC is sent at all) and, unlike that case, it can only be produced by a +/// concurrent write - so it needs the in-process server to drive it deterministically. +/// +[RunPerProtocol] +public class TransactionWatchDriftTests(ITestOutputHelper log) : TestBase(log) +{ + // A null array is not an empty array: EXEC answering *-1 (RESP2) / _ (RESP3) means "watch failed", + // whereas *0 means "a transaction of zero commands committed". The managed server used to collapse + // the former into the latter, which made the whole drift path untestable (and, client-side, it + // surfaced as a protocol failure instead). + [Fact] + public void NullArrayIsDistinctFromEmptyArray() + { + var nullArray = TypedRedisValue.NullArray(RespPrefix.Array); + Assert.True(nullArray.IsNullArray); + Assert.True(nullArray.IsNullValueOrArray); + Assert.True(nullArray.Span.IsEmpty); + + var emptyArray = TypedRedisValue.EmptyArray(RespPrefix.Array); + Assert.False(emptyArray.IsNullArray); + Assert.False(emptyArray.IsNullValueOrArray); + Assert.True(emptyArray.Span.IsEmpty); + } + + // The headline case: the condition holds, EXEC is issued, and the server rejects it because the + // watched key moved. Execute reports false (nothing was applied) and - the part that regressed - + // every queued operation's task must reach a terminal state (cancelled), not hang forever. + [Fact] + public async Task WatchDrift_AbortsAndCancelsQueuedOperations() + { + using var server = new WatchDriftServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var db = conn.GetDatabase(); + RedisKey key = "drift:cancel"; + Assert.True(await db.StringSetAsync(key, "seed")); + + server.DriftKey = key; + server.DriftOps = 1; // the next EXEC observes a concurrent write to the watched key + + var tran = db.CreateTransaction(); + var cond = tran.AddCondition(Condition.StringEqual(key, "seed")); + var setTask = tran.StringSetAsync(key, "committed"); + var incrTask = tran.StringIncrementAsync("drift:counter"); + + Assert.False(await tran.ExecuteAsync()); // EXEC returned a null array + Assert.True(cond.WasSatisfied); // the *condition* held; the server-side WATCH is what killed it + Assert.True(tran.WasWatchConflict); // ...and this is how a caller tells the two apart + Assert.Equal(1, server.ExecOpsReceived); + + // both per-operation tasks must complete (as cancelled); before the fix they sat forever in + // WaitingForActivation, so assert with a timeout rather than awaiting them directly + await AssertCancelledAsync(setTask); + await AssertCancelledAsync(incrTask); + + Assert.Equal("seed", await db.StringGetAsync(key)); // nothing was applied + Assert.False(await db.KeyExistsAsync("drift:counter")); + } + + // Same shape, but confirming the *elective* abort still behaves: the condition fails, no EXEC is + // ever issued, and the queued operations are cancelled. This is the path that already worked; it is + // here so the two outcomes are pinned side by side (they are indistinguishable from Execute's bool + // alone - WasWatchConflict, or inspecting the ConditionResults, is what separates them). + [Fact] + public async Task FailedCondition_AbortsElectively_WithoutExec() + { + using var server = new WatchDriftServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var db = conn.GetDatabase(); + RedisKey key = "drift:elective"; + Assert.True(await db.StringSetAsync(key, "seed")); + + var tran = db.CreateTransaction(); + Assert.False(tran.WasWatchConflict); // false before execution, too + + var cond = tran.AddCondition(Condition.StringEqual(key, "different")); + var setTask = tran.StringSetAsync(key, "committed"); + + Assert.False(await tran.ExecuteAsync()); + Assert.False(cond.WasSatisfied); // this is what distinguishes an elective abort from drift + Assert.False(tran.WasWatchConflict); // we chose not to issue an EXEC; nobody raced us + Assert.Equal(0, server.ExecOpsReceived); // never even asked + + await AssertCancelledAsync(setTask); + Assert.Equal("seed", await db.StringGetAsync(key)); + } + + // A transaction with conditions but no operations: drift still aborts it, and there are no + // per-operation tasks to cancel. Guards the zero-length inner-operations edge in the processor. + [Fact] + public async Task WatchDrift_ConditionOnlyTransaction_Aborts() + { + using var server = new WatchDriftServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var db = conn.GetDatabase(); + RedisKey key = "drift:condonly"; + Assert.True(await db.StringSetAsync(key, "seed")); + + server.DriftKey = key; + server.DriftOps = 1; + + var tran = db.CreateTransaction(); + var cond = tran.AddCondition(Condition.StringEqual(key, "seed")); + + Assert.False(await tran.ExecuteAsync()); + Assert.True(cond.WasSatisfied); + Assert.True(tran.WasWatchConflict); + Assert.Equal(1, server.ExecOpsReceived); + } + + // A transaction that commits cleanly must not report a conflict. + [Fact] + public async Task SatisfiedCondition_Commits_WithoutConflict() + { + using var server = new WatchDriftServer(Output); + await using var conn = await server.ConnectAsync(log: Writer); + + var db = conn.GetDatabase(); + RedisKey key = "drift:clean"; + Assert.True(await db.StringSetAsync(key, "seed")); + + var tran = db.CreateTransaction(); + var cond = tran.AddCondition(Condition.StringEqual(key, "seed")); + var setTask = tran.StringSetAsync(key, "committed"); + + Assert.True(await tran.ExecuteAsync()); + Assert.True(cond.WasSatisfied); + Assert.False(tran.WasWatchConflict); + Assert.True(await setTask); + } + + private async Task AssertCancelledAsync(Task task) + { + var completed = await Task.WhenAny(task, Task.Delay(5000)); + if (completed != task) + { + Log($"task did not complete; status: {task.Status}"); + Assert.Fail($"queued operation never completed (status: {task.Status})"); + } + + await Assert.ThrowsAnyAsync(async () => await task); + } + + // An in-process server that, for the next DriftOps EXEC operations, simulates a concurrent write to + // DriftKey immediately before the EXEC is processed. Touch is exactly what a real write from another + // connection would do, so the transaction is doomed by the server's own WATCH bookkeeping and EXEC + // replies with a null array - no special-casing of the reply itself. + // + // Driving this from a genuinely separate connection is not practical: SE.Redis does not issue the WATCH + // when AddCondition is called, it issues WATCH, the condition reads, MULTI, the queued commands and + // EXEC as one dispatch. So the window an interloper has to squeeze into is the gap between the + // condition reads and the EXEC landing, within a single flush - which is the point of the feature, but + // makes it useless as a test lever. Injecting the Touch server-side reproduces the same state exactly. + private sealed class WatchDriftServer(ITestOutputHelper? log, EndPoint? endpoint = null) : InProcessTestServer(log, endpoint) + { + public int ExecOpsReceived { get; private set; } + + public int DriftOps { get; set; } + + public RedisKey DriftKey { get; set; } + + protected override TypedRedisValue Exec(RedisClient client, in RedisRequest request) + { + ExecOpsReceived++; + + if (DriftOps > 0) + { + DriftOps--; + client.Touch(client.Database, DriftKey); + } + + return base.Exec(client, in request); + } + } +} diff --git a/tests/StackExchange.Redis.Tests/WithKeyPrefixTests.cs b/tests/StackExchange.Redis.Tests/WithKeyPrefixTests.cs index acbef74cf..b70d215ee 100644 --- a/tests/StackExchange.Redis.Tests/WithKeyPrefixTests.cs +++ b/tests/StackExchange.Redis.Tests/WithKeyPrefixTests.cs @@ -106,6 +106,7 @@ public async Task ConditionTest() var prefix = Me() + ":"; var foo = raw.WithKeyPrefix(prefix); + Output.WriteLine($"prefixed db features: {foo}"); // should be KeyPrefix (not Transaction/Batch) raw.KeyDelete(prefix + "abc", CommandFlags.FireAndForget); raw.KeyDelete(prefix + "i", CommandFlags.FireAndForget); diff --git a/toys/StackExchange.Redis.Server/MemoryCacheRedisServer.cs b/toys/StackExchange.Redis.Server/MemoryCacheRedisServer.cs index a8c0c22d1..62c8989b6 100644 --- a/toys/StackExchange.Redis.Server/MemoryCacheRedisServer.cs +++ b/toys/StackExchange.Redis.Server/MemoryCacheRedisServer.cs @@ -145,16 +145,26 @@ protected override RedisValue Get(int database, in RedisKey key) return RedisValue.Unbox(val); } - protected override void Set(int database, in RedisKey key, in RedisValue value) - => GetDb(database)[key] = value.Box(); - - protected override void SetEx(int database, in RedisKey key, TimeSpan expiration, in RedisValue value) + protected override bool Set(int database, in RedisKey key, in RedisValue value, TimeSpan? expiration = null, SetFlags flags = SetFlags.None) { var db = GetDb(database); + switch (flags & (SetFlags.NX | SetFlags.XX)) + { + case SetFlags.NX when Exists(database, key): return false; + case SetFlags.XX when !Exists(database, key): return false; + case SetFlags.NX | SetFlags.XX: throw new ArgumentOutOfRangeException(nameof(flags)); + } + + if (expiration is null) + { + db[key] = value.Box(); + return true; + } var now = Time(); - var absolute = now + expiration; + var absolute = now + expiration.Value; if (absolute <= now) db.Remove(key); else db[key] = new ExpiringValue(value.Box(), absolute); + return true; } protected override bool Del(int database, in RedisKey key) diff --git a/toys/StackExchange.Redis.Server/RedisServer.cs b/toys/StackExchange.Redis.Server/RedisServer.cs index a46705c22..77ad4f1ed 100644 --- a/toys/StackExchange.Redis.Server/RedisServer.cs +++ b/toys/StackExchange.Redis.Server/RedisServer.cs @@ -160,8 +160,18 @@ protected override void AppendStats(StringBuilder sb) public string Password { get; set; } = ""; + /// + /// When set, every command is rejected with a LOADING error, mimicking a server that is + /// still loading its dataset into memory. + /// + public bool IsLoading { get; set; } + public override TypedRedisValue Execute(RedisClient client, in RedisRequest request) { + if (IsLoading) + { + return TypedRedisValue.Error("LOADING"); + } var pw = Password; if (!string.IsNullOrEmpty(pw) & !client.IsAuthenticated) { @@ -352,7 +362,7 @@ protected virtual TypedRedisValue SetEx(RedisClient client, in RedisRequest requ RedisKey key = request.GetKey(1); int seconds = request.GetInt32(2); var value = request.GetValue(3); - SetEx(client.Database, key, TimeSpan.FromSeconds(seconds), value); + Set(client.Database, key, value, TimeSpan.FromSeconds(seconds)); return TypedRedisValue.OK; } @@ -431,12 +441,6 @@ protected virtual TypedRedisValue Exec(RedisClient client, in RedisRequest reque return results; } - protected virtual void SetEx(int database, in RedisKey key, TimeSpan timeout, in RedisValue value) - { - Set(database, key, value); - Expire(database, key, timeout); - } - [RedisCommand(3, nameof(RedisCommand.CLIENT), "setname", LockFree = true)] protected virtual TypedRedisValue ClientSetname(RedisClient client, in RedisRequest request) { @@ -993,13 +997,37 @@ protected virtual TypedRedisValue Get(RedisClient client, in RedisRequest reques protected virtual RedisValue Get(int database, in RedisKey key) => throw new NotSupportedException(); - [RedisCommand(3)] + [RedisCommand(-3)] protected virtual TypedRedisValue Set(RedisClient client, in RedisRequest request) { - Set(client.Database, request.GetKey(1), request.GetValue(2)); - return TypedRedisValue.OK; + TimeSpan? expiry = null; + var key = request.GetKey(1); + var value = request.GetValue(2); + SetFlags flags = SetFlags.None; + for (int i = 3; i < request.Count; i++) + { + if (request.IsString(i, "nx"u8) || request.IsString(i, "NX"u8)) flags |= SetFlags.NX; + else if (request.IsString(i, "xx"u8) || request.IsString(i, "XX"u8)) flags |= SetFlags.XX; + else if (request.IsString(i, "ex"u8) || request.IsString(i, "EX"u8)) expiry = TimeSpan.FromSeconds(request.GetInt32(++i)); + else if (request.IsString(i, "px"u8) || request.IsString(i, "PX"u8)) expiry = TimeSpan.FromMilliseconds(request.GetInt32(++i)); + else return TypedRedisValue.Error("ERR syntax error"); + } + const SetFlags BOTH = SetFlags.NX | SetFlags.XX; + if ((flags & BOTH) == BOTH) return TypedRedisValue.Error("ERR Invalid flags combination"); + var result = Set(client.Database, request.GetKey(1), request.GetValue(2), expiry, flags); + return result ? TypedRedisValue.OK : TypedRedisValue.BulkString(RedisValue.Null); } - protected virtual void Set(int database, in RedisKey key, in RedisValue value) => throw new NotSupportedException(); + + [Flags] + public enum SetFlags + { + None = 0, + NX = 1, + XX = 2, + } + + protected virtual bool Set(int database, in RedisKey key, in RedisValue value, TimeSpan? expiry = null, SetFlags flags = SetFlags.None) => throw new NotSupportedException(); + [RedisCommand(1)] protected new virtual TypedRedisValue Shutdown(RedisClient client, in RedisRequest request) { diff --git a/toys/StackExchange.Redis.Server/RespServer.cs b/toys/StackExchange.Redis.Server/RespServer.cs index 51996721e..cb98ead2b 100644 --- a/toys/StackExchange.Redis.Server/RespServer.cs +++ b/toys/StackExchange.Redis.Server/RespServer.cs @@ -331,6 +331,7 @@ public async Task RunClientAsync(IDuplexPipe pipe, RedisServer.Node node = null, } else { + await ClientPauseAsync(client, request); await client.AddOutboundAsync(response); } client.ResetAfterRequest(); @@ -397,6 +398,7 @@ internal static string GetUtf8String(in ReadOnlySequence buffer) charCount += Encoding.UTF8.GetChars(segment.Span, target.Slice(charCount)); } } + const string CR = "\u240D", LF = "\u240A", CRLF = CR + LF; string s = target.Slice(0, charCount).ToString() .Replace("\r\n", CRLF).Replace("\r", CR).Replace("\n", LF); @@ -416,6 +418,8 @@ public virtual void Log(string message) } } + protected virtual ValueTask ClientPauseAsync(RedisClient client, in RedisRequest request) => default; + protected object ServerSyncLock => this; private long _totalCommandsProcesed, _totalErrorCount; @@ -427,7 +431,7 @@ public virtual void ResetCounters() _totalCommandsProcesed = _totalErrorCount = _totalClientCount = 0; } - public virtual TypedRedisValue OnUnknownCommand(in RedisClient client, in RedisRequest request, ReadOnlySpan command) + public virtual TypedRedisValue OnUnknownCommand(RedisClient client, in RedisRequest request, ReadOnlySpan command) { return request.CommandNotFound(); } diff --git a/toys/StackExchange.Redis.Server/StackExchange.Redis.Server.csproj b/toys/StackExchange.Redis.Server/StackExchange.Redis.Server.csproj index 4151146bf..fed77b54c 100644 --- a/toys/StackExchange.Redis.Server/StackExchange.Redis.Server.csproj +++ b/toys/StackExchange.Redis.Server/StackExchange.Redis.Server.csproj @@ -15,5 +15,6 @@ + diff --git a/toys/StackExchange.Redis.Server/TypedRedisValue.cs b/toys/StackExchange.Redis.Server/TypedRedisValue.cs index d3264a1ec..da5d33a42 100644 --- a/toys/StackExchange.Redis.Server/TypedRedisValue.cs +++ b/toys/StackExchange.Redis.Server/TypedRedisValue.cs @@ -194,14 +194,17 @@ private TypedRedisValue(TypedRedisValue[] oversizedItems, int count, RespPrefix if (oversizedItems == null) { if (count != 0) throw new ArgumentOutOfRangeException(nameof(count)); - oversizedItems = []; - } - else - { - if (count < 0 || count > oversizedItems.Length) throw new ArgumentOutOfRangeException(nameof(count)); - if (count == 0) oversizedItems = []; + + // a *null* array is not the same as an empty array; keep the value null so that + // IsNullArray reports true and we emit *-1 / _ rather than *0 + _value = RedisValue.Null; + Type = type; + return; } + if (count < 0 || count > oversizedItems.Length) throw new ArgumentOutOfRangeException(nameof(count)); + if (count == 0) oversizedItems = []; + _value = RedisValue.CreateForeign(oversizedItems, 0, count); Type = type; } diff --git a/version.json b/version.json index 461b0972f..ab7b283f6 100644 --- a/version.json +++ b/version.json @@ -1,7 +1,7 @@ { - "version": "3.0", - "versionHeightOffset": 0, - "versionHeightOffsetAppliesTo": "3.0", + "version": "3.1", + "versionHeightOffset": -2, + "versionHeightOffsetAppliesTo": "3.1", "assemblyVersion": "3.0", "publicReleaseRefSpec": [ "^refs/heads/main$",