Functions — Retry Policies, Dead Letter Functions, and Reliability

The Problem

Your functions fail intermittently:

• Transient network errors cause function failures
• No retry mechanism leads to message loss
• Failed messages disappear without trace
• Need consistent retry behavior across functions

Solution Implementation

Step 1: Configure Built-in Retry Policy

// host.json
{
  "version": "2.0",
  "retry": {
    "strategy": "exponentialBackoff",
    "maxRetryCount": 5,
    "delayInterval": "00:00:05",
    "minimumInterval": "00:00:01",
    "maximumInterval": "00:01:00"
  },
  "extensions": {
    "serviceBus": {
      "prefetchCount": 100,
      "messageHandlerOptions": {
        "autoComplete": false,
        "maxConcurrentCalls": 32,
        "maxAutoRenewDuration": "00:05:00"
      }
    }
  }
}

Step 2: Queue Trigger with Retry

// queue-trigger/index.ts
import { AzureFunction, Context, QueueMessage } from "@azure/functions";

const queueTrigger: AzureFunction = async function (context: Context, message: QueueMessage): Promise<void> {
    const messageId = message.messageId;
    const dequeueCount = message.dequeueCount;
    
    context.log(`Processing message ${messageId}, attempt ${dequeueCount}`);
    
    try {
        // Process the message
        await processMessage(message);
        
        context.log(`Successfully processed message ${messageId}`);
    }
    catch (error) {
        context.log.error(`Error processing message ${messageId}: ${error}`);
        
        // Throw to trigger retry (built-in retry will handle)
        throw error;
    }
};

async function processMessage(message: QueueMessage): Promise<void> {
    const data = JSON.parse(message.messageText);
    
    // Simulate processing
    if (data.shouldFail) {
        throw new Error("Transient failure");
    }
    
    await doWork(data);
}

export default queueTrigger;

// function.json
{
  "bindings": [
    {
      "name": "message",
      "type": "queueTrigger",
      "direction": "in",
      "queueName": "orders",
      "connection": "AzureWebJobsStorage"
    }
  ],
  "retry": {
    "strategy": "exponentialBackoff",
    "maxRetryCount": 5,
    "delayInterval": "00:00:05"
  }
}

Step 3: Service Bus Trigger with Dead Letter

// servicebus-trigger/index.ts
import { AzureFunction, Context, ServiceBusMessage } from "@azure/functions";

const serviceBusTrigger: AzureFunction = async function (context: Context, message: ServiceBusMessage): Promise<void> {
    const messageId = message.messageId;
    const deliveryCount = message.deliveryCount;
    
    context.log(`Processing SB message ${messageId}, attempt ${deliveryCount}`);
    
    try {
        await processMessage(message);
        
        // Success - complete the message
        context.log(`Success: ${messageId}`);
    }
    catch (error) {
        context.log.error(`Failed: ${messageId}, error: ${error}`);
        
        // Check if we've exhausted retries
        if (deliveryCount >= 5) {
            // Send to dead letter queue
            context.log.error(`Max retries exceeded for ${messageId}, sending to DLQ`);
            throw new Error("SEND_TO_DLQ"); // Special handling
        }
        
        // Throw to retry
        throw error;
    }
};

async function processMessage(message: ServiceBusMessage): Promise<void> {
    const order = message.body;
    
    if (order.isCorrupted) {
        throw new Error("Cannot process corrupted order");
    }
    
    await callExternalApi(order);
}

export default serviceBusTrigger;

Step 4: Dead Letter Function

// deadletter-trigger/index.ts
import { AzureFunction, Context, ServiceBusMessage } from "@azure/functions";

const deadLetterTrigger: AzureFunction = async function (context: Context, message: ServiceBusMessage): Promise<void> {
    context.log("Processing dead letter message");
    
    const messageId = message.messageId;
    const deadLetterReason = message.deadLetterReason;
    const deadLetterError = message.deadLetterErrorDescription;
    
    context.log(`DLQ Message ID: ${messageId}`);
    context.log(`Reason: ${deadLetterReason}`);
    context.log(`Error: ${deadLetterError}`);
    
    try {
        // Parse original message
        const originalMessage = JSON.parse(message.messageText);
        
        // Analyze and take action
        const action = determineAction(deadLetterReason, originalMessage);
        
        await takeAction(action, originalMessage);
        
        // Archive to permanent storage
        await archiveDeadLetter({
            messageId,
            reason: deadLetterReason,
            error: deadLetterError,
            message: originalMessage,
            processedAt: new Date().toISOString()
        });
    }
    catch (error) {
        context.log.error(`Error processing DLQ: ${error}`);
        throw error;
    }
};

function determineAction(reason: string, message: any): string {
    if (reason?.includes("MaxDeliveryCount")) {
        return "alert_operations";
    }
    if (reason?.includes("Timeout")) {
        return "retry_with_backoff";
    }
    if (reason?.includes("Validation")) {
        return "notify_sender";
    }
    return "archive";
}

async function takeAction(action: string, message: any): Promise<void> {
    switch (action) {
        case "alert_operations":
            await sendAlert("Dead letter pile growing", message);
            break;
        case "notify_sender":
            await notifySender(message);
            break;
    }
}

export default deadLetterTrigger;

Step 5: Custom Retry Logic

// custom-retry/index.ts
import { AzureFunction, Context, HttpRequest } from "@azure/functions";

const httpTrigger: AzureFunction = async function (context: Context, req: HttpRequest): Promise<void> {
    const maxRetries = 3;
    const retryDelay = 1000;
    
    let lastError: Error | null = null;
    
    for (let attempt = 0; attempt < maxRetries; attempt++) {
        try {
            const result = await callUnreliableService();
            context.res = { status: 200, body: result };
            return;
        }
        catch (error) {
            lastError = error as Error;
            context.log.warn(`Attempt ${attempt + 1} failed: ${error}`);
            
            if (attempt < maxRetries - 1) {
                // Exponential backoff
                const delay = retryDelay * Math.pow(2, attempt);
                await sleep(delay);
            }
        }
    }
    
    context.log.error(`All ${maxRetries} attempts failed`);
    context.res = { 
        status: 503, 
        body: { error: "service_unavailable", message: lastError?.message } 
    };
};

function sleep(ms: number): Promise<void> {
    return new Promise(resolve => setTimeout(resolve, ms));
}

export default httpTrigger;

Step 6: Circuit Breaker Pattern

// circuit-breaker.ts
class CircuitBreaker {
    private state: "closed" | "open" | "half-open" = "closed";
    private failures = 0;
    private lastFailureTime = 0;
    private threshold = 5;
    private timeout = 30000;
    
    async execute<T>(operation: () => Promise<T>): Promise<T> {
        if (this.state === "open") {
            if (Date.now() - this.lastFailureTime > this.timeout) {
                this.state = "half-open";
            } else {
                throw new Error("Circuit breaker is open");
            }
        }
        
        try {
            const result = await operation();
            
            if (this.state === "half-open") {
                this.state = "closed";
                this.failures = 0;
            }
            
            return result;
        }
        catch (error) {
            this.failures++;
            this.lastFailureTime = Date.now();
            
            if (this.failures >= this.threshold) {
                this.state = "open";
            }
            
            throw error;
        }
    }
}

Summary

• Configure built-in retry with exponential backoff
• Use dead letter functions to handle failed messages
• Implement custom retry logic when needed
• Add circuit breaker for external service calls
• Monitor retry and DLQ metrics