Monitoring & Observability
Complete Guide

Your Application (.NET 8 / Functions / AKS pods)
        │
        │  emits via OpenTelemetry SDK or App Insights SDK
        ▼
┌────────────────────────────────────────────────────────┐
│              Azure Monitor (the platform)              │
│                                                        │
│  ┌─────────────────────┐   ┌────────────────────────┐  │
│  │ Application Insights│   │   Log Analytics        │  │
│  │  (APM + traces +    │   │   Workspace            │  │
│  │   exceptions +      │   │   (logs from ALL       │  │
│  │   custom events)    │   │    Azure resources)    │  │
│  └─────────────────────┘   └────────────────────────┘  │
│                                                        │
│  ┌─────────────────────┐   ┌────────────────────────┐  │
│  │  Metrics Store      │   │   Alerts Engine        │  │
│  │  (time-series,      │   │   (rules, action       │  │
│  │   auto-collected    │   │    groups, PagerDuty)  │  │
│  │   from all services)│   │                        │  │
│  └─────────────────────┘   └────────────────────────┘  │
└────────────────────────────────────────────────────────┘
        │
        ▼
Workbooks · Dashboards · Grafana · Power BI

// NuGet: Microsoft.ApplicationInsights.AspNetCore
// NuGet: Azure.Monitor.OpenTelemetry.AspNetCore  ← modern OpenTelemetry-based

// ── Option A: Azure Monitor OpenTelemetry (recommended for new projects) ──
var builder = WebApplication.CreateBuilder(args);

builder.Services.AddOpenTelemetry()
    .UseAzureMonitor(options =>
    {
        // Connection string from Key Vault / App Settings — never hardcode
        options.ConnectionString =
            builder.Configuration["ApplicationInsights:ConnectionString"];

        // Sampling — see Section 02c
        options.SamplingRatio = 0.1f; // 10% sampling in production
    });

// ── Option B: Classic App Insights SDK ────────────────────────────────
builder.Services.AddApplicationInsightsTelemetry(options =>
{
    options.ConnectionString =
        builder.Configuration["ApplicationInsights:ConnectionString"];
    options.EnableAdaptiveSampling = true;
    options.EnableHeartbeat        = true;
    options.EnableDebugLogger      = false; // Off in production
});

// ── appsettings.json ──────────────────────────────────────────────────
// {
//   "ApplicationInsights": {
//     "ConnectionString": "@Microsoft.KeyVault(VaultName=myVault;SecretName=ai-conn-string)"
//   }
// }
//
// Connection string format:
// InstrumentationKey=<key>;IngestionEndpoint=https://eastus-8.in.applicationinsights.azure.com/;...

// NuGet: Microsoft.ApplicationInsights.WorkerService
builder.Services.AddApplicationInsightsTelemetryWorkerService(options =>
{
    options.ConnectionString =
        builder.Configuration["ApplicationInsights:ConnectionString"];
});

// Azure Functions — auto-configured via host.json
// host.json:
// {
//   "logging": {
//     "applicationInsights": {
//       "samplingSettings": { "isEnabled": true, "maxTelemetryItemsPerSecond": 20 }
//     }
//   }
// }

// ITelemetryInitializer — runs on every telemetry item before it's sent
public class ServiceTelemetryInitializer : ITelemetryInitializer
{
    private readonly IHttpContextAccessor _http;

    public ServiceTelemetryInitializer(IHttpContextAccessor http) => _http = http;

    public void Initialize(ITelemetry telemetry)
    {
        // Tag every item with service metadata
        telemetry.Context.Cloud.RoleName     = "orders-api";
        telemetry.Context.Cloud.RoleInstance = Environment.MachineName;

        // Add custom global dimensions
        if (telemetry is ISupportProperties props)
        {
            props.Properties["ServiceVersion"] = "1.4.2";
            props.Properties["Environment"]    = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") ?? "Unknown";

            // Propagate correlation IDs from inbound request headers
            var ctx = _http.HttpContext;
            if (ctx is not null)
            {
                var correlationId = ctx.Request.Headers["X-Correlation-Id"].FirstOrDefault();
                if (!string.IsNullOrEmpty(correlationId))
                    props.Properties["CorrelationId"] = correlationId;

                var userId = ctx.User.FindFirst("oid")?.Value;
                if (!string.IsNullOrEmpty(userId))
                    props.Properties["UserId"] = userId;
            }
        }
    }
}

// Register in DI
services.AddSingleton<ITelemetryInitializer, ServiceTelemetryInitializer>();

public class OrderService(TelemetryClient telemetry, ILogger<OrderService> logger)
{
    public async Task<Order> CreateOrderAsync(CreateOrderCommand cmd, CancellationToken ct)
    {
        // ── Custom Event — business milestone ─────────────────────────
        telemetry.TrackEvent("OrderCreated", new Dictionary<string, string>
        {
            ["CustomerId"] = cmd.CustomerId.ToString(),
            ["Region"]     = cmd.Region,
            ["ItemCount"]  = cmd.LineItems.Count.ToString(),
        },
        new Dictionary<string, double>
        {
            ["OrderAmount"] = (double)cmd.TotalAmount,
        });

        // ── Custom Metric — numeric measurement ───────────────────────
        telemetry.TrackMetric("Order.TotalAmount", (double)cmd.TotalAmount,
            new Dictionary<string, string> { ["Region"] = cmd.Region });

        // ── Custom Dependency — any external call ─────────────────────
        var startTime = DateTimeOffset.UtcNow;
        var timer     = Stopwatch.StartNew();
        bool success  = false;
        try
        {
            var result = await _inventoryClient.ReserveItemsAsync(cmd.LineItems, ct);
            success = result.IsSuccess;
            return result.Value;
        }
        catch (Exception ex)
        {
            // ── Exception with extra context ──────────────────────────
            telemetry.TrackException(ex, new Dictionary<string, string>
            {
                ["OrderId"]    = cmd.OrderId.ToString(),
                ["CustomerId"] = cmd.CustomerId.ToString(),
                ["Operation"]  = "CreateOrder",
            });
            throw;
        }
        finally
        {
            timer.Stop();
            telemetry.TrackDependency(
                dependencyTypeName: "gRPC",
                target:   "inventory-api",
                dependencyName: "InventoryClient.ReserveItems",
                data:     $"CustomerId={cmd.CustomerId}",
                startTime: startTime,
                duration:  timer.Elapsed,
                resultCode: success ? "200" : "500",
                success:   success);
        }
    }
}

// ── Structured logging — ILogger feeds into App Insights traces ───────
logger.LogInformation(
    "Order {OrderId} created for customer {CustomerId} — amount {Amount:C}",
    order.Id, order.CustomerId, order.TotalAmount);

// ── Using scopes for correlated log groups ────────────────────────────
using (logger.BeginScope(new Dictionary<string, object>
{
    ["OrderId"]    = order.Id,
    ["CustomerId"] = order.CustomerId,
    ["RequestId"]  = Activity.Current?.TraceId.ToString() ?? ""
}))
{
    logger.LogInformation("Starting payment processing");
    await ProcessPaymentAsync(order, ct);
    logger.LogInformation("Payment processed successfully");
}

// Use IOperationHolder to group related telemetry items
// All items inside the using block share the same operation ID
public async Task ProcessMessageAsync(ServiceBusReceivedMessage message)
{
    using var operation = telemetry.StartOperation<RequestTelemetry>(
        "ServiceBus.ProcessOrder");

    operation.Telemetry.Properties["MessageId"]  = message.MessageId;
    operation.Telemetry.Properties["QueueName"]  = "orders";

    try
    {
        var order = JsonSerializer.Deserialize<OrderCreatedEvent>(message.Body);
        await ProcessOrderInternalAsync(order!);

        operation.Telemetry.Success    = true;
        operation.Telemetry.ResponseCode = "200";
    }
    catch (Exception ex)
    {
        operation.Telemetry.Success    = false;
        operation.Telemetry.ResponseCode = "500";
        telemetry.TrackException(ex);
        throw;
    }
}

// Adaptive sampling — recommended for most services
services.AddApplicationInsightsTelemetry(options =>
{
    options.EnableAdaptiveSampling = true;
});

services.Configure<TelemetryConfiguration>(config =>
{
    var adaptiveSamplingProcessor = new AdaptiveSamplingTelemetryProcessor(
        new SamplingPercentageEstimatorSettings
        {
            MaxTelemetryItemsPerSecond = 5,     // Target: max 5 events/sec per instance
            MinSamplingPercentage      = 0.1,   // Never sample less than 0.1%
            MaxSamplingPercentage      = 100,   // Full sampling when low traffic
            EvaluationInterval         = TimeSpan.FromSeconds(15),
            SamplingPercentageDecreaseTimeout = TimeSpan.FromMinutes(2),
            SamplingPercentageIncreaseTimeout = TimeSpan.FromMinutes(15),
        },
        next: config.TelemetryProcessorChainBuilder.Build());

    config.TelemetryProcessorChainBuilder
        .Use(_ => adaptiveSamplingProcessor)
        .Build();
});

// ── Fixed-rate sampling — predictable volume ──────────────────────────
services.Configure<TelemetryConfiguration>(config =>
{
    config.TelemetryProcessorChainBuilder
        .UseSampling(samplingPercentage: 10) // Sample 10% of operations
        .Build();
});

// ── Exclude specific telemetry from sampling ──────────────────────────
public class ExcludeHealthCheckFilter : ITelemetryProcessor
{
    private readonly ITelemetryProcessor _next;

    public ExcludeHealthCheckFilter(ITelemetryProcessor next) => _next = next;

    public void Process(ITelemetry item)
    {
        // Don't send health check telemetry to App Insights at all
        if (item is RequestTelemetry req &&
            req.Url?.AbsolutePath.StartsWith("/health") == true)
            return;

        _next.Process(item);
    }
}

services.Configure<TelemetryConfiguration>(config =>
{
    config.TelemetryProcessorChainBuilder
        .Use(next => new ExcludeHealthCheckFilter(next))
        .UseSampling(10)
        .Build();
});

// Custom availability test via Azure Function (for private endpoints)
[FunctionName("AvailabilityTest")]
public async Task Run([TimerTrigger("0 */5 * * * *")] TimerInfo timer)
{
    var testName    = "Orders API — Create Order Flow";
    var runLocation = "East US";
    var startTime   = DateTimeOffset.UtcNow;
    var timer2      = Stopwatch.StartNew();
    bool success    = false;
    string message  = "";

    try
    {
        // Step 1: Authenticate
        var token = await GetTestTokenAsync();

        // Step 2: Create a test order
        _httpClient.DefaultRequestHeaders.Authorization =
            new AuthenticationHeaderValue("Bearer", token);

        var response = await _httpClient.PostAsJsonAsync(
            "/api/v1/orders",
            new { customerId = TestCustomerId, lineItems = TestLineItems });

        success = response.IsSuccessStatusCode;
        message = $"Status: {(int)response.StatusCode}";

        // Step 3: Verify the order exists
        if (success)
        {
            var order = await response.Content.ReadFromJsonAsync<OrderResponse>();
            var getResponse = await _httpClient.GetAsync($"/api/v1/orders/{order!.OrderId}");
            success = getResponse.IsSuccessStatusCode;
            message += $" | GET: {(int)getResponse.StatusCode}";
        }
    }
    catch (Exception ex)
    {
        success = false;
        message = ex.Message;
    }
    finally
    {
        timer2.Stop();
        _telemetry.TrackAvailability(
            name:      testName,
            timeStamp: startTime,
            duration:  timer2.Elapsed,
            runLocation: runLocation,
            success:   success,
            message:   message);
    }
}

// ── App Insights KQL — Failure rate by operation in last 1h ─────────
requests
| where timestamp > ago(1h)
| summarize
    Total    = count(),
    Failed   = countif(success == false),
    P95_ms   = percentile(duration, 95),
    P99_ms   = percentile(duration, 99)
  by name
| extend FailureRate = round(100.0 * Failed / Total, 2)
| where Total > 10
| order by FailureRate desc

// ── Slow dependency calls (SQL > 1s) ──────────────────────────────────
dependencies
| where timestamp > ago(1h)
| where type == "SQL"
| where duration > 1000
| project timestamp, name, data, duration, success, resultCode
| order by duration desc
| take 50

// ── Top exceptions in last 24h ────────────────────────────────────────
exceptions
| where timestamp > ago(24h)
| summarize Count = count() by type, outerMessage
| order by Count desc
| take 20

// ── User journey — trace one operation end-to-end ─────────────────────
let opId = "abc123def456";
union requests, dependencies, exceptions, traces
| where operation_Id == opId
| project timestamp, itemType, name, duration, success, message, type
| order by timestamp asc

# Create Log Analytics Workspace
az monitor log-analytics workspace create \
  --resource-group myRG \
  --workspace-name myPlatformWorkspace \
  --location eastus \
  --retention-time 90 \
  --sku PerGB2018

# Get workspace ID (for App Insights linking)
WORKSPACE_ID=$(az monitor log-analytics workspace show \
  --resource-group myRG \
  --workspace-name myPlatformWorkspace \
  --query id --output tsv)

# Create Application Insights linked to the workspace
az monitor app-insights component create \
  --app myOrdersApi \
  --resource-group myRG \
  --location eastus \
  --workspace "$WORKSPACE_ID" \
  --kind web

# Enable diagnostic settings for Service Bus → Workspace
az monitor diagnostic-settings create \
  --resource "/subscriptions/<sub>/resourceGroups/<rg>/providers/Microsoft.ServiceBus/namespaces/<ns>" \
  --name "sb-diagnostics" \
  --workspace "$WORKSPACE_ID" \
  --logs '[{"category":"OperationalLogs","enabled":true},{"category":"VNetAndIPFilteringLogs","enabled":true}]' \
  --metrics '[{"category":"AllMetrics","enabled":true}]'

// ── Filtering ─────────────────────────────────────────────────────────
TableName
| where TimeGenerated > ago(1h)                   // Time filter (always first!)
| where Level == "Error"                           // Exact match
| where Message contains "timeout"                // Case-insensitive substring
| where Message has "OrderId"                      // Faster than contains for words
| where StatusCode between (500 .. 599)            // Numeric range
| where isnotempty(CorrelationId)                  // Not null/empty
| where Properties.region in ("EU", "US")         // Value in list

// ── Projection ────────────────────────────────────────────────────────
| project TimeGenerated, Level, Message, Properties  // Select columns
| project-away TenantId, SubscriptionId              // Drop columns
| extend DurationSec = Duration / 1000.0             // Add computed column
| parse Message with "OrderId=" OrderId " amount=" Amount:double " region=" Region

// ── Aggregation ───────────────────────────────────────────────────────
| summarize
    Count       = count(),
    ErrorCount  = countif(Level == "Error"),
    AvgDuration = avg(Duration),
    P95Duration = percentile(Duration, 95),
    MaxDuration = max(Duration)
  by bin(TimeGenerated, 5m), ServiceName

// ── Sorting & Limiting ────────────────────────────────────────────────
| order by TimeGenerated desc
| top 100 by Duration desc        // Top N by a column

// ── Joining tables ────────────────────────────────────────────────────
requests
| join kind=leftouter (
    exceptions
    | project operation_Id, exceptionType = type, exceptionMsg = outerMessage
) on operation_Id

// ── String operations ─────────────────────────────────────────────────
| extend OrderId = extract("OrderId=([a-f0-9-]+)", 1, Message)
| extend Domain  = split(Email, "@")[1]
| where ServiceName startswith "order"
| where Url matches regex @"/api/v[0-9]+/orders/[a-f0-9-]+"

// ── Time operations ───────────────────────────────────────────────────
| extend HourOfDay  = hourofday(TimeGenerated)
| extend DayOfWeek  = dayofweek(TimeGenerated)
| where TimeGenerated between (datetime(2026-05-01) .. datetime(2026-05-08))

// ── Error rate trend (5-min buckets) ─────────────────────────────────
requests
| where TimeGenerated > ago(3h)
| summarize
    Total  = count(),
    Failed = countif(success == false)
  by bin(TimeGenerated, 5m), cloud_RoleName
| extend ErrorRate = round(100.0 * Failed / Total, 2)
| render timechart

// ── P50/P95/P99 latency by endpoint ──────────────────────────────────
requests
| where TimeGenerated > ago(1h)
| summarize
    P50 = percentile(duration, 50),
    P95 = percentile(duration, 95),
    P99 = percentile(duration, 99),
    Count = count()
  by name
| where Count > 100
| order by P99 desc

// ── Dependency failures — which external calls are breaking? ──────────
dependencies
| where TimeGenerated > ago(1h)
| where success == false
| summarize
    FailureCount = count(),
    AvgDuration  = avg(duration),
    Targets      = make_set(target)
  by type, name
| order by FailureCount desc

// ── Exceptions — top 10 with sample messages ─────────────────────────
exceptions
| where TimeGenerated > ago(24h)
| summarize
    Count   = count(),
    Sample  = any(outerMessage),
    LastSeen = max(TimeGenerated)
  by type
| top 10 by Count desc

// ── Slow SQL queries ──────────────────────────────────────────────────
dependencies
| where TimeGenerated > ago(1h)
| where type == "SQL"
| where duration > 500
| project TimeGenerated, name, data, duration, resultCode
| order by duration desc
| take 20

// ── Container restart loop detection ─────────────────────────────────
ContainerLog
| where TimeGenerated > ago(1h)
| where LogEntry contains "OOMKilled" or LogEntry contains "CrashLoopBackOff"
| summarize Restarts = count() by ContainerID, _ResourceId
| where Restarts > 3
| order by Restarts desc

// ── Alerting query: error spike detection (> 2x baseline) ───────────
let baseline = requests
    | where TimeGenerated between (ago(2h) .. ago(1h))
    | summarize BaselineErrors = countif(success == false);
let current = requests
    | where TimeGenerated > ago(1h)
    | summarize CurrentErrors = countif(success == false);
current
| cross join baseline
| where CurrentErrors > BaselineErrors * 2 and CurrentErrors > 10
| project CurrentErrors, BaselineErrors,
          Increase = round(100.0 * (CurrentErrors - BaselineErrors) / BaselineErrors, 1)

// ── User impact analysis — how many users hit errors? ─────────────────
requests
| where TimeGenerated > ago(1h)
| where success == false
| summarize
    FailedRequests = count(),
    AffectedUsers  = dcount(user_Id),
    AffectedOps    = dcount(operation_Id)
  by name, resultCode
| order by AffectedUsers desc

// ── Funnel analysis — order creation drop-off ─────────────────────────
let step1 = customEvents | where name == "CartViewed"     | summarize s1 = dcount(user_Id);
let step2 = customEvents | where name == "CheckoutStarted" | summarize s2 = dcount(user_Id);
let step3 = customEvents | where name == "OrderSubmitted"  | summarize s3 = dcount(user_Id);
let step4 = customEvents | where name == "OrderConfirmed"  | summarize s4 = dcount(user_Id);
step1 | cross join step2 | cross join step3 | cross join step4
| project
    CartViewed        = s1,
    CheckoutStarted   = s2,
    OrderSubmitted    = s3,
    OrderConfirmed    = s4,
    CheckoutConversion = round(100.0 * s2 / s1, 1),
    SubmitConversion   = round(100.0 * s3 / s2, 1),
    FinalConversion    = round(100.0 * s4 / s1, 1)

// ── Cross-workspace query — correlate App Insights + Security logs ────
workspace("SecurityWorkspace").SecurityEvent
| where TimeGenerated > ago(1h)
| where EventID == 4625 // Failed login
| join kind=inner (
    app("OrdersAppInsights").requests
    | where success == false
    | project operation_Id, clientIP = client_IP
) on $left.IpAddress == $right.clientIP
| project TimeGenerated, IpAddress, Account, operation_Id

// ── Dynamic columns from JSON properties ──────────────────────────────
customEvents
| where name == "OrderCreated"
| extend
    OrderId  = tostring(customDimensions["OrderId"]),
    Amount   = todouble(customDimensions["OrderAmount"]),
    Region   = tostring(customDimensions["Region"])
| summarize Revenue = sum(Amount), Orders = count() by Region, bin(TimeGenerated, 1h)
| render timechart

using System.Diagnostics.Metrics;

// Define meters and instruments (static — created once)
public static class OrdersMetrics
{
    private static readonly Meter Meter = new("Orders.Api", "1.0.0");

    // Counter — monotonically increasing
    public static readonly Counter<long> OrdersCreated =
        Meter.CreateCounter<long>(
            name: "orders.created.total",
            unit: "orders",
            description: "Total number of orders created");

    // Histogram — distribution of values (latency, sizes)
    public static readonly Histogram<double> OrderProcessingDuration =
        Meter.CreateHistogram<double>(
            name: "orders.processing.duration",
            unit: "ms",
            description: "Time to process an order end-to-end");

    // ObservableGauge — current value from a callback
    public static readonly ObservableGauge<int> PendingOrders =
        Meter.CreateObservableGauge<int>(
            name: "orders.pending.count",
            observeValue: () => OrderRepository.GetPendingCount(),
            unit: "orders",
            description: "Current number of pending orders");

    // UpDownCounter — can increase and decrease
    public static readonly UpDownCounter<int> ActiveConnections =
        Meter.CreateUpDownCounter<int>(
            name: "orders.active.connections",
            unit: "connections");
}

// Usage in command handler
public async Task<Result<OrderResponse>> Handle(CreateOrderCommand cmd, CancellationToken ct)
{
    var sw = Stopwatch.StartNew();
    try
    {
        var order = Order.Create(cmd.CustomerId, cmd.LineItems);
        await _repository.SaveAsync(order, ct);

        // Record with dimensions (tags)
        OrdersMetrics.OrdersCreated.Add(1,
            new KeyValuePair<string, object?>("region",   cmd.Region),
            new KeyValuePair<string, object?>("channel",  cmd.Channel),
            new KeyValuePair<string, object?>("priority", cmd.IsPriority ? "high" : "normal"));

        return Result.Success(OrderResponse.From(order));
    }
    finally
    {
        sw.Stop();
        OrdersMetrics.OrderProcessingDuration.Record(sw.Elapsed.TotalMilliseconds,
            new KeyValuePair<string, object?>("region", cmd.Region));
    }
}

// Register meters with OTel
builder.Services.AddOpenTelemetry()
    .WithMetrics(m => m
        .AddMeter("Orders.Api")
        .AddAspNetCoreInstrumentation()
        .AddHttpClientInstrumentation()
        .AddRuntimeInstrumentation()           // GC, thread pool, heap
        .AddProcessInstrumentation()           // CPU, memory
        .AddOtlpExporter()                     // → Azure Monitor / Prometheus
        .AddPrometheusExporter());             // /metrics endpoint for Prometheus scraping

# ── Step 1: Create Action Group (who gets notified) ─────────────────
az monitor action-group create \
  --resource-group myRG \
  --name platform-oncall \
  --short-name oncall \
  --action email oncall-email ops-team@contoso.com \
  --action webhook pagerduty https://events.pagerduty.com/integration/<key>/enqueue

# ── Step 2: Metric Alert — Service Bus DLQ > 0 ──────────────────────
az monitor metrics alert create \
  --name "ServiceBus-DLQ-Alert" \
  --resource-group myRG \
  --scopes "/subscriptions/<sub>/resourceGroups/<rg>/providers/Microsoft.ServiceBus/namespaces/<ns>" \
  --condition "avg DeadletteredMessages > 0" \
  --window-size 5m \
  --evaluation-frequency 1m \
  --severity 2 \
  --action "/subscriptions/<sub>/resourceGroups/myRG/providers/microsoft.insights/actionGroups/platform-oncall" \
  --description "Messages in Service Bus dead-letter queue"

# ── Step 3: Log Alert — 5xx errors spike ─────────────────────────────
az monitor scheduled-query create \
  --name "High-Error-Rate-Alert" \
  --resource-group myRG \
  --scopes "<app-insights-resource-id>" \
  --condition-query "requests | where timestamp > ago(5m) | summarize FailRate = countif(success==false)*100.0/count() | where FailRate > 5" \
  --condition-time-aggregation Count \
  --condition-operator GreaterThan \
  --condition-threshold 0 \
  --evaluation-period 5 \
  --evaluation-frequency 1 \
  --severity 1 \
  --action-groups "/subscriptions/<sub>/resourceGroups/myRG/providers/microsoft.insights/actionGroups/platform-oncall"

// Workbook ARM template structure (simplified)
{
  "type": "microsoft.insights/workbooks",
  "properties": {
    "displayName": "Orders API — SLO Dashboard",
    "serializedData": {
      "version": "Notebook/1.0",
      "items": [
        {
          "type": 9,          // Parameters
          "content": {
            "parameters": [
              {
                "id": "timeRange",
                "version": "KqlParameterItem/1.0",
                "name": "TimeRange",
                "type": 4,
                "value": { "durationMs": 3600000 }
              },
              {
                "id": "service",
                "name": "ServiceName",
                "type": 2,             // Drop-down
                "query": "requests | summarize by cloud_RoleName"
              }
            ]
          }
        },
        {
          "type": 3,          // Query item (chart)
          "content": {
            "query": "requests | where cloud_RoleName == '{ServiceName}' | summarize ErrorRate = countif(success==false)*100.0/count() by bin(TimeGenerated, 5m) | render timechart",
            "visualization": "timechart",
            "title": "Error Rate % — {ServiceName}"
          }
        }
      ]
    }
  }
}

// Bicep — apply diagnostic settings to every resource via module
module diagnostics 'modules/diagnostics.bicep' = {
  name: 'orders-api-diagnostics'
  params: {
    resourceId: ordersAppService.id
    workspaceId: logAnalyticsWorkspace.id
    retentionDays: 90
  }
}

// modules/diagnostics.bicep
resource diagnosticSetting 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = {
  name: 'send-to-workspace'
  scope: resourceId
  properties: {
    workspaceId: workspaceId
    logs: [
      { categoryGroup: 'allLogs',   enabled: true,  retentionPolicy: { days: retentionDays, enabled: true } }
    ]
    metrics: [
      { category: 'AllMetrics', enabled: true, retentionPolicy: { days: retentionDays, enabled: true } }
    ]
  }
}

// NuGet packages needed:
// OpenTelemetry.Extensions.Hosting
// OpenTelemetry.Instrumentation.AspNetCore
// OpenTelemetry.Instrumentation.Http
// OpenTelemetry.Instrumentation.EntityFrameworkCore
// OpenTelemetry.Instrumentation.Runtime
// OpenTelemetry.Exporter.OpenTelemetryProtocol
// Azure.Monitor.OpenTelemetry.AspNetCore

var builder = WebApplication.CreateBuilder(args);

builder.Services.AddOpenTelemetry()
    // ── Resource metadata (common tags on all signals) ─────────────
    .ConfigureResource(resource => resource
        .AddService(
            serviceName:    "orders-api",
            serviceVersion: "2.1.0",
            serviceInstanceId: Environment.MachineName)
        .AddAttributes(new Dictionary<string, object>
        {
            ["deployment.environment"] = builder.Environment.EnvironmentName,
            ["service.namespace"]      = "com.myplatform",
            ["k8s.pod.name"]           = Environment.GetEnvironmentVariable("HOSTNAME") ?? "",
        }))

    // ── Tracing ────────────────────────────────────────────────────
    .WithTracing(tracing => tracing
        .AddAspNetCoreInstrumentation(options =>
        {
            options.RecordException        = true;
            options.EnrichWithHttpRequest  = (activity, request) =>
            {
                activity.SetTag("http.request.body_size",
                    request.ContentLength ?? 0);
            };
            options.EnrichWithHttpResponse = (activity, response) =>
            {
                activity.SetTag("http.response.body_size",
                    response.ContentLength ?? 0);
            };
            // Don't trace health checks — too noisy
            options.Filter = ctx =>
                !ctx.Request.Path.StartsWithSegments("/health");
        })
        .AddHttpClientInstrumentation(options =>
        {
            options.RecordException = true;
            // Redact auth headers from traces
            options.EnrichWithHttpRequestMessage = (activity, request) =>
            {
                request.Headers.Remove("Authorization");
            };
        })
        .AddEntityFrameworkCoreInstrumentation(options =>
        {
            options.SetDbStatementForText        = true;  // Capture SQL
            options.SetDbStatementForStoredProcedure = true;
        })
        .AddSource("Orders.Api")           // Custom ActivitySource
        .AddSource("MassTransit")          // MassTransit instrumentation
        .AddOtlpExporter(options =>
        {
            options.Endpoint = new Uri(
                builder.Configuration["Otel:Endpoint"] ?? "http://localhost:4317");
            options.Protocol = OtlpExportProtocol.Grpc;
        }))

    // ── Metrics ────────────────────────────────────────────────────
    .WithMetrics(metrics => metrics
        .AddAspNetCoreInstrumentation()
        .AddHttpClientInstrumentation()
        .AddRuntimeInstrumentation()
        .AddProcessInstrumentation()
        .AddMeter("Orders.Api")
        .AddOtlpExporter())

    // ── Logging ────────────────────────────────────────────────────
    .WithLogging(logging => logging
        .AddOtlpExporter());

// Also export to Azure Monitor (Application Insights) simultaneously
builder.Services.AddOpenTelemetry().UseAzureMonitor();

// Define ActivitySource (one per service or logical domain)
public static class OrdersTracing
{
    public static readonly ActivitySource Source =
        new("Orders.Api", "2.1.0");
}

// Create manual spans for business operations
public async Task<Result<OrderResponse>> Handle(
    CreateOrderCommand cmd, CancellationToken ct)
{
    // Start a new span — child of the current ambient activity
    using var activity = OrdersTracing.Source.StartActivity(
        "CreateOrder",
        ActivityKind.Internal);

    // Tag the span with business context
    activity?.SetTag("order.customer_id", cmd.CustomerId.ToString());
    activity?.SetTag("order.region",      cmd.Region);
    activity?.SetTag("order.item_count",  cmd.LineItems.Count);
    activity?.SetTag("order.channel",     cmd.Channel);

    try
    {
        // Nested span for inventory validation
        using var inventorySpan = OrdersTracing.Source.StartActivity(
            "ValidateInventory",
            ActivityKind.Client);
        inventorySpan?.SetTag("inventory.product_count", cmd.LineItems.Count);

        var inventory = await _inventoryClient.ValidateAsync(cmd.LineItems, ct);
        inventorySpan?.SetTag("inventory.all_available", inventory.AllAvailable);

        if (!inventory.AllAvailable)
        {
            activity?.SetStatus(ActivityStatusCode.Error, "Insufficient inventory");
            return Result.Failure<OrderResponse>("Some items are out of stock");
        }

        var order = Order.Create(cmd.CustomerId, cmd.LineItems);
        activity?.SetTag("order.id", order.Id.ToString());

        // Add an event (point-in-time annotation on the span)
        activity?.AddEvent(new ActivityEvent("OrderPersisted", tags:
            new ActivityTagsCollection { ["order.id"] = order.Id.ToString() }));

        activity?.SetStatus(ActivityStatusCode.Ok);
        return Result.Success(OrderResponse.From(order));
    }
    catch (Exception ex)
    {
        // Record exception on span
        activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
        activity?.RecordException(ex);
        throw;
    }
}

// W3C traceparent header format
traceparent: 00-<trace-id>-<parent-span-id>-<flags>

// Example:
traceparent: 00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01
//            ^  ^                                ^                ^
//            version  128-bit trace ID           64-bit span ID   flags (01=sampled)

// W3C tracestate — vendor-specific data
tracestate: az=<app-insights-specific-data>

// ── HttpClient propagates context automatically with OTel ───────────
// Nothing to do — OTel HttpClient instrumentation injects traceparent

// ── Manual propagation for Service Bus messages ───────────────────────
// Sender — inject current trace context into message properties
public async Task SendOrderEventAsync(OrderCreatedEvent evt)
{
    var message = new ServiceBusMessage(JsonSerializer.SerializeToUtf8Bytes(evt))
    {
        MessageId   = evt.OrderId.ToString(),
        ContentType = "application/json",
    };

    // Inject W3C trace context into message ApplicationProperties
    var propagator = Propagators.DefaultTextMapPropagator;
    propagator.Inject(
        new PropagationContext(Activity.Current?.Context ?? default, Baggage.Current),
        message.ApplicationProperties,
        (props, key, value) => props[key] = value);

    await _sender.SendMessageAsync(message);
}

// Receiver — extract trace context from message and set as parent
public async Task ProcessMessageAsync(ServiceBusReceivedMessage message)
{
    // Extract context from message properties
    var propagator = Propagators.DefaultTextMapPropagator;
    var parentContext = propagator.Extract(
        default,
        message.ApplicationProperties,
        (props, key) =>
        {
            if (props.TryGetValue(key, out var value))
                return [value?.ToString() ?? ""];
            return [];
        });

    // Start child span linked to the sender's trace
    using var activity = OrdersTracing.Source.StartActivity(
        "ServiceBus.ProcessOrder",
        ActivityKind.Consumer,
        parentContext.ActivityContext);

    activity?.SetTag("messaging.system",      "servicebus");
    activity?.SetTag("messaging.destination", "orders");
    activity?.SetTag("messaging.message_id",  message.MessageId);

    await ProcessOrderAsync(message.Body.ToString());
}

// Set baggage at entry point (API Gateway / BFF)
Baggage.Current = Baggage.SetBaggage("tenant.id",    request.TenantId);
Baggage.Current = Baggage.SetBaggage("user.id",      user.GetUserId());
Baggage.Current = Baggage.SetBaggage("feature.flags","beta-checkout=true");

// Read baggage anywhere in the call chain (propagated automatically)
var tenantId   = Baggage.GetBaggage("tenant.id");
var featureFlags = Baggage.GetBaggage("feature.flags");

// All spans in this trace automatically inherit baggage values
// Useful for: multi-tenant filtering, feature flag tracking,
//             A/B test attribution, compliance tagging

// ── Head-based sampling via OTel SDK ─────────────────────────────────

// 1. Always sample errors, sample 10% of successful traces
builder.Services.AddOpenTelemetry()
    .WithTracing(t => t
        .SetSampler(new CompositeApplicationSampler(
            // Always sample if there's an error on the span
            errorSampler: new AlwaysOnSampler(),
            // 10% of everything else
            defaultSampler: new TraceIdRatioBasedSampler(0.10)
        ))
    );

// Custom sampler — always sample slow requests
public class SlowRequestSampler : Sampler
{
    private readonly double _threshold;

    public SlowRequestSampler(double threshold) => _threshold = threshold;

    public override SamplingResult ShouldSample(in SamplingParameters parameters)
    {
        // Always sample if parent says so (preserve parent decision)
        if (parameters.ParentContext.TraceFlags.HasFlag(ActivityTraceFlags.Recorded))
            return new SamplingResult(SamplingDecision.RecordAndSample);

        // Sample based on URL priority
        var httpPath = parameters.Tags?
            .FirstOrDefault(t => t.Key == "http.target").Value?.ToString();

        // Always trace payment and checkout flows
        if (httpPath?.Contains("/checkout") == true ||
            httpPath?.Contains("/payments") == true)
            return new SamplingResult(SamplingDecision.RecordAndSample);

        // 5% sample for everything else
        return new SamplingResult(
            Random.Shared.NextDouble() < 0.05
                ? SamplingDecision.RecordAndSample
                : SamplingDecision.Drop);
    }
}

# docker-compose.override.yml — dev environment
services:
  jaeger:
    image: jaegertracing/all-in-one:latest
    ports:
      - "16686:16686"   # Jaeger UI
      - "4317:4317"     # OTLP gRPC receiver
      - "4318:4318"     # OTLP HTTP receiver
    environment:
      - COLLECTOR_OTLP_ENABLED=true
    networks: [platform]

  otel-collector:
    image: otel/opentelemetry-collector-contrib:latest
    command: ["--config=/etc/otel-collector-config.yaml"]
    volumes:
      - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
    ports:
      - "4319:4317"     # Receive from apps
    networks: [platform]

# otel-collector-config.yaml
# receivers:
#   otlp:
#     protocols: { grpc: { endpoint: 0.0.0.0:4317 } }
# exporters:
#   jaeger:
#     endpoint: jaeger:14250
#   azuremonitor:
#     connection_string: "${AI_CONNECTION_STRING}"
# service:
#   pipelines:
#     traces:
#       receivers: [otlp]
#       exporters: [jaeger, azuremonitor]

// appsettings.Development.json — point to local Jaeger
{
  "Otel": {
    "Endpoint": "http://localhost:4317"
  },
  "ApplicationInsights": {
    "ConnectionString": ""
  }
}

// appsettings.Production.json — Azure Monitor
{
  "Otel": {
    "Endpoint": "https://eastus-8.in.applicationinsights.azure.com/"
  },
  "ApplicationInsights": {
    "ConnectionString": "@Microsoft.KeyVault(VaultName=myVault;SecretName=ai-conn-string)"
  }
}

// ── Availability SLO — % of successful requests ──────────────────────
let sloWindow = 30d;
let sloTarget = 99.5; // 99.5% availability
requests
| where TimeGenerated > ago(sloWindow)
| where name !startswith "GET /health"   // Exclude health checks from SLO
| summarize
    Total   = count(),
    Success = countif(success == true)
| extend
    AvailabilitySLI = round(100.0 * Success / Total, 3),
    ErrorBudget_pct = 100.0 - sloTarget,
    ActualErrors    = Total - Success,
    BudgetedErrors  = round(Total * (1 - sloTarget / 100.0), 0),
    BudgetRemaining = round(Total * (1 - sloTarget / 100.0), 0) - (Total - Success)
| extend SLO_Met = AvailabilitySLI >= sloTarget

// ── Latency SLO — P99 must be under 500ms ─────────────────────────────
requests
| where TimeGenerated > ago(30d)
| where name == "POST /api/v1/orders"
| summarize
    TotalRequests     = count(),
    WithinTarget      = countif(duration < 500),
    P50_ms = percentile(duration, 50),
    P95_ms = percentile(duration, 95),
    P99_ms = percentile(duration, 99)
| extend
    SLI_latency = round(100.0 * WithinTarget / TotalRequests, 3),
    SLO_target  = 99.0,
    SLO_Met     = (100.0 * WithinTarget / TotalRequests) >= 99.0

// ── Error budget burn rate alert ──────────────────────────────────────
// Alert when burning through budget more than 2x faster than sustainable
let budget_pct   = 0.5;   // 0.5% of requests can fail
let burn_window  = 1h;
let total_window = 30d;
let totalRequests = toscalar(requests | where TimeGenerated > ago(total_window) | count);
let budgetTotal   = totalRequests * (budget_pct / 100.0);
let recentErrors  = toscalar(
    requests | where TimeGenerated > ago(burn_window) | where success == false | count);
let sustainableBurnPerHour = budgetTotal / (30.0 * 24.0);
recentErrors > sustainableBurnPerHour * 2  // Alert: burning 2x too fast

## Alert: High Error Rate — Orders API

**Severity:** Sev 1
**SLO Impact:** Yes — consuming error budget at >2x sustainable rate

### What this means
More than 5% of POST /api/v1/orders requests are failing over the last 5 minutes.
Users cannot place orders.

### Immediate Diagnosis (< 5 minutes)
1. Check Application Map in App Insights — which dependency is failing?
   Link: https://portal.azure.com/#resource/<ai-resource-id>/applicationMap

2. Run this query in Log Analytics:
   ```kql
   exceptions
   | where timestamp > ago(30m)
   | where cloud_RoleName == "orders-api"
   | summarize count() by outerMessage
   | order by count_ desc
   ```

3. Check recent deployments:
   Link: https://dev.azure.com/<org>/<project>/_release

### Common Causes & Fixes
| Symptom | Cause | Fix |
|---------|-------|-----|
| SQL timeout errors | DB CPU spike | Scale up SQL, check missing indexes |
| 503 from inventory-api | Inventory service down | Check inventory pod status in AKS |
| 401 from payment-api | Token expiry / MI issue | Restart pod, check MI role assignments |
| OutOfMemory | Memory leak in new deploy | Roll back deployment |

### Escalation
- Engineering on-call: PagerDuty → orders-api rotation
- Incident commander: Slack #incidents