Distributed Tracing System - API Design

Overview

The API surface of a distributed tracing system serves three distinct user groups: instrumented services (submitting spans), developers (querying traces for debugging), and platform teams (managing configuration and monitoring system health). The design must support high-throughput ingestion with minimal latency impact on applications, while providing rich query capabilities for trace analysis. We follow OpenTelemetry standards for interoperability with the broader observability ecosystem.

Span Ingestion API

OpenTelemetry Protocol (OTLP) - Primary Ingestion

// gRPC service definition (preferred for high-throughput)
service TraceService {
  rpc Export(ExportTraceServiceRequest) returns (ExportTraceServiceResponse);
}

message ExportTraceServiceRequest {
  repeated ResourceSpans resource_spans = 1;
}

message ResourceSpans {
  Resource resource = 1;
  repeated ScopeSpans scope_spans = 2;
}

message ScopeSpans {
  InstrumentationScope scope = 1;
  repeated Span spans = 2;
}

# HTTP/JSON equivalent (for environments without gRPC support)
POST /v1/traces
Content-Type: application/json
Authorization: Bearer <api-key>
X-Tenant-ID: team-payments

{
  "resourceSpans": [
    {
      "resource": {
        "attributes": [
          { "key": "service.name", "value": { "stringValue": "order-service" } },
          { "key": "service.version", "value": { "stringValue": "2.1.0" } },
          { "key": "host.name", "value": { "stringValue": "prod-order-7b4f9" } },
          { "key": "deployment.environment", "value": { "stringValue": "production" } }
        ]
      },
      "scopeSpans": [
        {
          "scope": {
            "name": "io.opentelemetry.contrib.http",
            "version": "1.24.0"
          },
          "spans": [
            {
              "traceId": "5b8aa5a2d2c872e8321cf37308d69df2",
              "spanId": "051581bf3cb55c13",
              "parentSpanId": "ab1f3c9e7d2b4a5f",
              "name": "GET /api/orders/{id}",
              "kind": 2,
              "startTimeUnixNano": "1705312200000000000",
              "endTimeUnixNano": "1705312200150000000",
              "attributes": [
                { "key": "http.method", "value": { "stringValue": "GET" } },
                { "key": "http.status_code", "value": { "intValue": "200" } },
                { "key": "http.url", "value": { "stringValue": "/api/orders/12345" } }
              ],
              "events": [
                {
                  "timeUnixNano": "1705312200050000000",
                  "name": "cache.miss",
                  "attributes": [
                    { "key": "cache.key", "value": { "stringValue": "order:12345" } }
                  ]
                }
              ],
              "status": { "code": 1 }
            }
          ]
        }
      ]
    }
  ]
}

# Response
HTTP/1.1 200 OK
Content-Type: application/json

{
  "partialSuccess": {
    "rejectedSpans": 0,
    "errorMessage": ""
  }
}

Jaeger Thrift Format (Legacy Compatibility)

POST /api/traces
Content-Type: application/x-thrift
X-Jaeger-Batch-Size: 100

# Binary Thrift payload containing jaeger.thrift Batch structure
# Used by older Jaeger clients and agents

Zipkin JSON Format (Compatibility)

POST /api/v2/spans
Content-Type: application/json

[
  {
    "traceId": "5b8aa5a2d2c872e8321cf37308d69df2",
    "id": "051581bf3cb55c13",
    "parentId": "ab1f3c9e7d2b4a5f",
    "name": "get /api/orders/{id}",
    "timestamp": 1705312200000000,
    "duration": 150000,
    "kind": "SERVER",
    "localEndpoint": {
      "serviceName": "order-service",
      "ipv4": "10.0.1.42",
      "port": 8080
    },
    "remoteEndpoint": {
      "serviceName": "api-gateway",
      "ipv4": "10.0.1.10"
    },
    "tags": {
      "http.method": "GET",
      "http.status_code": "200"
    }
  }
]

# Response
HTTP/1.1 202 Accepted

Ingestion Response Codes

200 OK: All spans accepted
202 Accepted: Spans queued for processing (async)
207 Multi-Status: Partial success (some spans rejected)
400 Bad Request: Malformed payload
401 Unauthorized: Invalid API key
413 Payload Too Large: Batch exceeds 5MB limit
429 Too Many Requests: Rate limit exceeded (include Retry-After header)
503 Service Unavailable: Collector overloaded (client should buffer and retry)

Trace Query API

Get Trace by ID

GET /api/v1/traces/{traceId}
Authorization: Bearer <user-token>

# Optional parameters
?raw=false          # If true, return raw spans without assembly
&max_spans=1000     # Limit spans returned for very large traces

# Response 200 OK
{
  "traceId": "5b8aa5a2d2c872e8321cf37308d69df2",
  "spans": [
    {
      "spanId": "051581bf3cb55c13",
      "parentSpanId": "ab1f3c9e7d2b4a5f",
      "operationName": "GET /api/orders/{id}",
      "serviceName": "order-service",
      "serviceColor": "#4CAF50",
      "startTime": 1705312200000000,
      "duration": 150000,
      "tags": [
        { "key": "http.method", "value": "GET", "type": "string" },
        { "key": "http.status_code", "value": 200, "type": "int64" }
      ],
      "logs": [
        {
          "timestamp": 1705312200050000,
          "fields": [
            { "key": "event", "value": "cache.miss" },
            { "key": "cache.key", "value": "order:12345" }
          ]
        }
      ],
      "warnings": [],
      "childSpanCount": 3
    }
  ],
  "traceMetadata": {
    "duration": 245000,
    "spanCount": 7,
    "serviceCount": 4,
    "services": [
      { "name": "api-gateway", "spanCount": 1 },
      { "name": "order-service", "spanCount": 2 },
      { "name": "payment-service", "spanCount": 2 },
      { "name": "inventory-service", "spanCount": 2 }
    ],
    "depth": 4,
    "hasErrors": false
  }
}

Search Traces

GET /api/v1/traces
Authorization: Bearer <user-token>

# Query Parameters
?service=order-service           # Required: filter by service
&operation=GET /api/orders/{id}  # Optional: filter by operation
&start=1705312200000000          # Required: start time (epoch microseconds)
&end=1705398600000000            # Required: end time
&minDuration=100ms               # Optional: minimum trace duration
&maxDuration=5s                  # Optional: maximum trace duration
&tags=http.status_code:500       # Optional: tag filter (repeatable)
&tags=error:true                 # Multiple tags are AND-ed
&limit=20                        # Results per page (max 100)
&offset=0                        # Pagination offset

# Response 200 OK
{
  "traces": [
    {
      "traceId": "5b8aa5a2d2c872e8321cf37308d69df2",
      "rootService": "api-gateway",
      "rootOperation": "GET /api/orders/{id}",
      "startTime": 1705312200000000,
      "duration": 245000,
      "spanCount": 7,
      "serviceCount": 4,
      "services": [
        { "name": "api-gateway", "spanCount": 1 },
        { "name": "order-service", "spanCount": 2 }
      ],
      "hasErrors": false
    }
  ],
  "total": 1523,
  "limit": 20,
  "offset": 0
}

Advanced Search (Structured Query)

POST /api/v1/traces/search
Content-Type: application/json
Authorization: Bearer <user-token>

{
  "query": {
    "service": "order-service",
    "operation": "GET /api/orders/{id}",
    "timeRange": {
      "start": "2024-01-15T00:00:00Z",
      "end": "2024-01-15T23:59:59Z"
    },
    "duration": {
      "min": "100ms",
      "max": "5s"
    },
    "tags": {
      "http.status_code": "500",
      "customer.tier": "premium"
    },
    "hasError": true,
    "minSpanCount": 5
  },
  "sort": {
    "field": "duration",
    "order": "desc"
  },
  "pagination": {
    "limit": 20,
    "cursor": "eyJsYXN0X3RyYWNlIjoiYWJjMTIzIn0="
  }
}

# Response 200 OK
{
  "traces": [...],
  "total": 342,
  "cursor": "eyJsYXN0X3RyYWNlIjoiZGVmNDU2In0="
}

Service Dependency API

Get Service Graph

GET /api/v1/dependencies
Authorization: Bearer <user-token>

?start=1705312200000000
&end=1705398600000000
&service=order-service    # Optional: focus on specific service

# Response 200 OK
{
  "nodes": [
    {
      "service": "api-gateway",
      "type": "HTTP",
      "spanCount": 150000,
      "errorRate": 0.002
    },
    {
      "service": "order-service",
      "type": "HTTP",
      "spanCount": 120000,
      "errorRate": 0.005
    },
    {
      "service": "payment-service",
      "type": "gRPC",
      "spanCount": 80000,
      "errorRate": 0.001
    }
  ],
  "edges": [
    {
      "source": "api-gateway",
      "target": "order-service",
      "protocol": "HTTP",
      "callCount": 120000,
      "errorCount": 600,
      "avgDurationMs": 45.2,
      "p99DurationMs": 250.0
    },
    {
      "source": "order-service",
      "target": "payment-service",
      "protocol": "gRPC",
      "callCount": 80000,
      "errorCount": 80,
      "avgDurationMs": 30.5,
      "p99DurationMs": 180.0
    }
  ],
  "timeRange": {
    "start": "2024-01-15T00:00:00Z",
    "end": "2024-01-15T23:59:59Z"
  }
}

Get Service Operations

GET /api/v1/services/{serviceName}/operations
Authorization: Bearer <user-token>

# Response 200 OK
{
  "service": "order-service",
  "operations": [
    {
      "name": "GET /api/orders/{id}",
      "spanKind": "SERVER",
      "avgDuration": 45200,
      "callRate": 500.0,
      "errorRate": 0.005,
      "lastSeen": "2024-01-15T23:59:00Z"
    },
    {
      "name": "POST /api/orders",
      "spanKind": "SERVER",
      "avgDuration": 120000,
      "callRate": 100.0,
      "errorRate": 0.01,
      "lastSeen": "2024-01-15T23:58:00Z"
    }
  ]
}

Analytics API

Latency Percentiles Over Time

GET /api/v1/analytics/latency
Authorization: Bearer <user-token>

?service=order-service
&operation=GET /api/orders/{id}
&start=1705312200000000
&end=1705398600000000
&granularity=5m              # Bucket size: 1m, 5m, 15m, 1h, 1d
&percentiles=50,95,99        # Which percentiles to compute

# Response 200 OK
{
  "service": "order-service",
  "operation": "GET /api/orders/{id}",
  "granularity": "5m",
  "dataPoints": [
    {
      "timestamp": "2024-01-15T10:00:00Z",
      "p50": 42000,
      "p95": 180000,
      "p99": 450000,
      "count": 2500,
      "errorCount": 12
    },
    {
      "timestamp": "2024-01-15T10:05:00Z",
      "p50": 45000,
      "p95": 200000,
      "p99": 520000,
      "count": 2600,
      "errorCount": 8
    }
  ]
}

RED Metrics (Rate, Errors, Duration)

GET /api/v1/analytics/red
Authorization: Bearer <user-token>

?service=order-service
&start=1705312200000000
&end=1705398600000000
&granularity=1m

# Response 200 OK
{
  "service": "order-service",
  "metrics": [
    {
      "timestamp": "2024-01-15T10:00:00Z",
      "rate": 520.5,
      "errorRate": 0.005,
      "duration": {
        "avg": 48000,
        "p50": 42000,
        "p95": 180000,
        "p99": 450000
      }
    }
  ]
}

Error Analysis

GET /api/v1/analytics/errors
Authorization: Bearer <user-token>

?service=order-service
&start=1705312200000000
&end=1705398600000000
&groupBy=operation,status_code

# Response 200 OK
{
  "errorGroups": [
    {
      "operation": "POST /api/orders",
      "statusCode": 500,
      "count": 145,
      "percentage": 0.8,
      "firstSeen": "2024-01-15T08:30:00Z",
      "lastSeen": "2024-01-15T10:15:00Z",
      "exampleTraceIds": [
        "5b8aa5a2d2c872e8321cf37308d69df2",
        "7c9bb6b3e3d983f9432dg48419e70eg3"
      ]
    }
  ]
}

Comparison API

Compare Traces

POST /api/v1/traces/compare
Content-Type: application/json
Authorization: Bearer <user-token>

{
  "baselineTraceId": "5b8aa5a2d2c872e8321cf37308d69df2",
  "comparisonTraceId": "7c9bb6b3e3d983f9432dg48419e70eg3"
}

# Response 200 OK
{
  "baseline": {
    "traceId": "5b8aa5a2...",
    "duration": 150000,
    "spanCount": 7
  },
  "comparison": {
    "traceId": "7c9bb6b3...",
    "duration": 450000,
    "spanCount": 9
  },
  "differences": {
    "durationDelta": 300000,
    "additionalSpans": [
      {
        "service": "payment-service",
        "operation": "retry-payment",
        "duration": 200000
      }
    ],
    "slowerSpans": [
      {
        "service": "order-service",
        "operation": "validate-inventory",
        "baselineDuration": 20000,
        "comparisonDuration": 80000,
        "delta": 60000
      }
    ]
  }
}

Detect Regressions

POST /api/v1/analytics/regressions
Content-Type: application/json
Authorization: Bearer <user-token>

{
  "service": "order-service",
  "operation": "GET /api/orders/{id}",
  "baselineWindow": {
    "start": "2024-01-14T00:00:00Z",
    "end": "2024-01-14T23:59:59Z"
  },
  "comparisonWindow": {
    "start": "2024-01-15T00:00:00Z",
    "end": "2024-01-15T23:59:59Z"
  },
  "thresholds": {
    "p50DeltaPercent": 20,
    "p99DeltaPercent": 50,
    "errorRateDeltaPercent": 100
  }
}

# Response 200 OK
{
  "regressions": [
    {
      "metric": "p99_latency",
      "baseline": 250000,
      "current": 520000,
      "deltaPercent": 108,
      "severity": "HIGH",
      "possibleCauses": [
        {
          "service": "inventory-service",
          "operation": "check-stock",
          "baselineP99": 50000,
          "currentP99": 200000,
          "contribution": 0.65
        }
      ]
    }
  ]
}

Admin API

Sampling Configuration

GET /api/v1/admin/sampling
Authorization: Bearer <admin-token>

# Response 200 OK
{
  "defaultStrategy": {
    "type": "probabilistic",
    "param": 0.01
  },
  "serviceStrategies": [
    {
      "service": "api-gateway",
      "type": "ratelimiting",
      "param": 5.0
    },
    {
      "service": "payment-service",
      "type": "probabilistic",
      "param": 0.1
    }
  ],
  "operationStrategies": [
    {
      "service": "order-service",
      "operation": "/health",
      "type": "probabilistic",
      "param": 0.001
    }
  ]
}

PUT /api/v1/admin/sampling
Content-Type: application/json
Authorization: Bearer <admin-token>

{
  "defaultStrategy": {
    "type": "probabilistic",
    "param": 0.01
  },
  "serviceStrategies": [
    {
      "service": "payment-service",
      "type": "probabilistic",
      "param": 0.5
    }
  ]
}

Retention Policies

GET /api/v1/admin/retention
Authorization: Bearer <admin-token>

# Response 200 OK
{
  "policies": [
    {
      "name": "default",
      "duration": "7d",
      "appliesTo": "all"
    },
    {
      "name": "errors",
      "duration": "30d",
      "condition": "status_code = ERROR"
    },
    {
      "name": "slow-traces",
      "duration": "14d",
      "condition": "duration > 5s"
    }
  ]
}

SDK / Client Library Interface

Auto-Instrumentation (Zero-Code)

// Java - OpenTelemetry auto-instrumentation agent
// Run with: java -javaagent:opentelemetry-javaagent.jar -jar myapp.jar
// Automatically instruments: HTTP clients, JDBC, gRPC, Kafka, Redis

// Configuration via environment variables:
// OTEL_SERVICE_NAME=order-service
// OTEL_EXPORTER_OTLP_ENDPOINT=http://collector:4317
// OTEL_TRACES_SAMPLER=parentbased_traceidratio
// OTEL_TRACES_SAMPLER_ARG=0.01

Manual Instrumentation

# Python SDK - Manual span creation
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter

# Setup
provider = TracerProvider()
processor = BatchSpanProcessor(
    OTLPSpanExporter(endpoint="http://collector:4317"),
    max_queue_size=2048,
    max_export_batch_size=512,
    schedule_delay_millis=5000
)
provider.add_span_processor(processor)
trace.set_tracer_provider(provider)

tracer = trace.get_tracer("order-service", "2.1.0")

# Creating spans
@tracer.start_as_current_span("process_order")
def process_order(order_id: str):
    span = trace.get_current_span()
    span.set_attribute("order.id", order_id)
    span.set_attribute("order.type", "standard")

    try:
        result = validate_inventory(order_id)
        span.set_attribute("inventory.available", result.available)
        return result
    except Exception as e:
        span.set_status(trace.StatusCode.ERROR, str(e))
        span.record_exception(e)
        raise

# Context propagation across services
from opentelemetry.propagators import inject, extract

def call_downstream_service(url, payload):
    headers = {}
    inject(headers)  # Injects traceparent header
    response = requests.post(url, json=payload, headers=headers)
    return response

def handle_incoming_request(request):
    context = extract(request.headers)  # Extracts trace context
    with tracer.start_as_current_span("handle_request", context=context):
        return process(request)

Go SDK Example

package main

import (
    "context"
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/trace"
)

var tracer = otel.Tracer("order-service")

func ProcessOrder(ctx context.Context, orderID string) error {
    ctx, span := tracer.Start(ctx, "ProcessOrder",
        trace.WithAttributes(
            attribute.String("order.id", orderID),
        ),
    )
    defer span.End()

    // Child span for database call
    ctx, dbSpan := tracer.Start(ctx, "db.query",
        trace.WithSpanKind(trace.SpanKindClient),
    )
    dbSpan.SetAttributes(
        attribute.String("db.system", "postgresql"),
        attribute.String("db.statement", "SELECT * FROM orders WHERE id = $1"),
    )
    result, err := db.QueryContext(ctx, "SELECT * FROM orders WHERE id = $1", orderID)
    if err != nil {
        dbSpan.RecordError(err)
        dbSpan.SetStatus(codes.Error, err.Error())
    }
    dbSpan.End()

    return nil
}

Rate Limiting and Backpressure

Ingestion Rate Limits:
- Per API key: 100,000 spans/sec
- Per service: 50,000 spans/sec
- Per collector instance: 500,000 spans/sec total
- Burst allowance: 2x rate for 30 seconds

Query Rate Limits:
- Per user: 100 queries/min
- Per organization: 1,000 queries/min
- Complex queries (aggregations): 10/min per user

Backpressure Signals:
- HTTP 429 with Retry-After header
- gRPC RESOURCE_EXHAUSTED status
- Exponential backoff recommended: 1s, 2s, 4s, 8s (max 60s)

This API design provides a complete interface for distributed tracing that is compatible with OpenTelemetry standards while offering rich query and analytics capabilities for production debugging workflows.