Imported from organvm-iv-taxis/a-i--skills (skills/development/resilience-patterns/SKILL.md). Install upstream with npx skills add organvm-iv-taxis/a-i--skills --skill resilience-patterns. Copyright stays with the author (MIT).

Resilience Patterns

Build systems that survive partial failures and degrade gracefully.

Core Patterns

Retry with Exponential Backoff

import asyncio
import random
from functools import wraps

def retry(max_attempts: int = 3, base_delay: float = 1.0, max_delay: float = 30.0):
    def decorator(func):
        @wraps(func)
        async def wrapper(*args, **kwargs):
            for attempt in range(max_attempts):
                try:
                    return await func(*args, **kwargs)
                except Exception as e:
                    if attempt == max_attempts - 1:
                        raise
                    delay = min(base_delay * (2 ** attempt) + random.uniform(0, 1), max_delay)
                    await asyncio.sleep(delay)
        return wrapper
    return decorator

@retry(max_attempts=3, base_delay=1.0)
async def fetch_data(url: str) -> dict:
    async with httpx.AsyncClient() as client:
        response = await client.get(url, timeout=10)
        response.raise_for_status()
        return response.json()

Circuit Breaker

import time
from enum import Enum

class CircuitState(Enum):
    CLOSED = "closed"       # Normal operation
    OPEN = "open"           # Failing, reject requests
    HALF_OPEN = "half_open" # Testing recovery

class CircuitBreaker:
    def __init__(self, failure_threshold: int = 5, recovery_timeout: float = 30.0):
        self.failure_threshold = failure_threshold
        self.recovery_timeout = recovery_timeout
        self.state = CircuitState.CLOSED
        self.failure_count = 0
        self.last_failure_time = 0.0

    async def call(self, func, *args, **kwargs):
        if self.state == CircuitState.OPEN:
            if time.time() - self.last_failure_time > self.recovery_timeout:
                self.state = CircuitState.HALF_OPEN
            else:
                raise CircuitOpenError(f"Circuit open, retry after {self.recovery_timeout}s")

        try:
            result = await func(*args, **kwargs)
            self._on_success()
            return result
        except Exception as e:
            self._on_failure()
            raise

    def _on_success(self):
        self.failure_count = 0
        self.state = CircuitState.CLOSED

    def _on_failure(self):
        self.failure_count += 1
        self.last_failure_time = time.time()
        if self.failure_count >= self.failure_threshold:
            self.state = CircuitState.OPEN

Timeout

async def with_timeout(coro, seconds: float):
    try:
        return await asyncio.wait_for(coro, timeout=seconds)
    except asyncio.TimeoutError:
        raise TimeoutError(f"Operation timed out after {seconds}s")

Bulkhead (Resource Isolation)

class Bulkhead:
    """Limit concurrent access to a resource."""
    def __init__(self, max_concurrent: int = 10):
        self.semaphore = asyncio.Semaphore(max_concurrent)

    async def execute(self, func, *args, **kwargs):
        async with self.semaphore:
            return await func(*args, **kwargs)

# Isolate different downstream services
payment_bulkhead = Bulkhead(max_concurrent=5)
inventory_bulkhead = Bulkhead(max_concurrent=20)

Fallback

async def get_user_profile(user_id: str) -> dict:
    try:
        return await primary_service.get_profile(user_id)
    except ServiceUnavailable:
        try:
            return await cache.get_profile(user_id)  # Stale cache fallback
        except CacheMiss:
            return {"user_id": user_id, "name": "Unknown", "_fallback": True}

Composition

Chain patterns for defense in depth:

Request → Timeout → Bulkhead → Circuit Breaker → Retry → Service Call

class ResilientClient:
    def __init__(self):
        self.circuit = CircuitBreaker(failure_threshold=5)
        self.bulkhead = Bulkhead(max_concurrent=10)

    @retry(max_attempts=3, base_delay=0.5)
    async def call(self, url: str) -> dict:
        return await with_timeout(
            self.bulkhead.execute(
                self.circuit.call, self._do_request, url
            ),
            seconds=15
        )

    async def _do_request(self, url: str) -> dict:
        async with httpx.AsyncClient() as client:
            response = await client.get(url, timeout=5)
            response.raise_for_status()
            return response.json()

Queue-Based Load Leveling

import asyncio
from collections import deque

class RateLimiter:
    def __init__(self, rate: int, per: float = 1.0):
        self.rate = rate
        self.per = per
        self.tokens = rate
        self.last_refill = time.time()
        self.lock = asyncio.Lock()

    async def acquire(self):
        async with self.lock:
            now = time.time()
            elapsed = now - self.last_refill
            self.tokens = min(self.rate, self.tokens + elapsed * (self.rate / self.per))
            self.last_refill = now
            if self.tokens >= 1:
                self.tokens -= 1
                return True
            return False

Health Check Patterns

from enum import Enum

class HealthStatus(Enum):
    HEALTHY = "healthy"
    DEGRADED = "degraded"
    UNHEALTHY = "unhealthy"

async def health_check() -> dict:
    checks = {
        "database": check_database(),
        "cache": check_cache(),
        "external_api": check_external_api(),
    }
    results = {}
    for name, check in checks.items():
        try:
            await asyncio.wait_for(check, timeout=5)
            results[name] = HealthStatus.HEALTHY
        except Exception:
            results[name] = HealthStatus.UNHEALTHY

    overall = (
        HealthStatus.HEALTHY if all(v == HealthStatus.HEALTHY for v in results.values())
        else HealthStatus.DEGRADED if any(v == HealthStatus.HEALTHY for v in results.values())
        else HealthStatus.UNHEALTHY
    )
    return {"status": overall.value, "checks": {k: v.value for k, v in results.items()}}

Failure Mode Analysis

Idempotency

async def process_payment(idempotency_key: str, amount: float):
    existing = await db.get_by_idempotency_key(idempotency_key)
    if existing:
        return existing  # Already processed

    result = await payment_gateway.charge(amount)
    await db.store(idempotency_key=idempotency_key, result=result)
    return result

Anti-Patterns

• Retry without backoff — Creates thundering herd on recovering services
• Retry on non-transient errors — 400 errors will never succeed on retry
• No timeout — Hanging requests consume resources indefinitely
• Cascading retries — Each layer retrying multiplies total attempts exponentially
• Circuit breaker too sensitive — Single failure shouldn't trip the circuit
• Ignoring partial failures — Assume any external call can fail