AI Platform on AWS EKS: Bedrock + vLLM Hybrid Architecture

┌─────────────────────────────────────────────────────────────┐
│                     Application Layer                        │
│  ┌──────────────┐  ┌──────────────┐  ┌──────────────┐      │
│  │   API Client │  │   AI Agent   │  │  Batch Jobs  │      │
│  └──────┬───────┘  └──────┬───────┘  └──────┬───────┘      │
│         │                 │                 │                │
│         └─────────────────┼─────────────────┘                │
│                           │                                  │
└───────────────────────────┼──────────────────────────────────┘
                            │
                   ┌────────▼────────┐
                   │  Unified API     │  (FastAPI Gateway)
                   │  Gateway (EKS)   │  - Authentication
                   └────────┬─────────┘  - Routing logic
                            │            - Rate limiting
              ┌─────────────┴─────────────┐
              │                           │
      ┌───────▼────────┐         ┌───────▼────────┐
      │  AWS Bedrock    │         │   vLLM Cluster │
      │  (Managed)      │         │   (Self-hosted)│
      │                 │         │                 │
      │ - Claude Sonnet │         │ - Llama 3 70B  │
      │ - Claude Opus   │         │ - Mistral 7B   │
      │ - Titan         │         │ - Custom LoRA  │
      └─────────────────┘         └────────┬───────┘
                                           │
                                  ┌────────▼────────┐
                                  │   GPU Nodes     │
                                  │   (g5.12xlarge) │
                                  │   AutoScaling   │
                                  └─────────────────┘

apiVersion: eksctl.io/v1alpha5
kind: ClusterConfig

metadata:
  name: ai-platform
  region: us-west-2
  version: "1.28"

iam:
  withOIDC: true

vpc:
  cidr: 10.0.0.0/16
  nat:
    gateway: HighlyAvailable

managedNodeGroups:
  # CPU node group for API gateway
  - name: cpu-workers
    instanceType: c6i.2xlarge
    minSize: 2
    maxSize: 10
    desiredCapacity: 3
    
    labels:
      workload: api-gateway
    
    tags:
      k8s.io/cluster-autoscaler/enabled: "true"
      k8s.io/cluster-autoscaler/ai-platform: "owned"
    
    iam:
      withAddonPolicies:
        autoScaler: true
        cloudWatch: true
        albIngress: true

  # GPU node group for vLLM
  - name: gpu-inference
    instanceType: g5.12xlarge  # 4x A10G GPUs
    minSize: 0
    maxSize: 20
    desiredCapacity: 2
    
    labels:
      workload: vllm-inference
      gpu: "true"
    
    taints:
      - key: nvidia.com/gpu
        value: "true"
        effect: NoSchedule
    
    # Use spot instances for cost savings
    instancesDistribution:
      instanceTypes:
        - g5.12xlarge
        - g5.24xlarge
      onDemandBaseCapacity: 1
      onDemandPercentageAboveBaseCapacity: 0
      spotInstancePools: 3
    
    tags:
      k8s.io/cluster-autoscaler/enabled: "true"
      k8s.io/cluster-autoscaler/ai-platform: "owned"

addons:
  - name: vpc-cni
  - name: coredns
  - name: kube-proxy
  - name: aws-ebs-csi-driver

cloudWatch:
  clusterLogging:
    enableTypes:
      - api
      - audit
      - authenticator
      - controllerManager
      - scheduler

# Create cluster
eksctl create cluster -f eks-cluster.yaml

# Install NVIDIA device plugin
kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.14.5/nvidia-device-plugin.yml

# Install cluster autoscaler
eksctl create iamserviceaccount \
  --cluster=ai-platform \
  --namespace=kube-system \
  --name=cluster-autoscaler \
  --attach-policy-arn=arn:aws:iam::aws:policy/AutoScalingFullAccess \
  --approve

kubectl apply -f https://raw.githubusercontent.com/kubernetes/autoscaler/master/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml

# vllm-llama3-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-llama3-70b
  namespace: ai-platform
  labels:
    app: vllm-llama3
    model: llama-3-70b
spec:
  replicas: 2
  selector:
    matchLabels:
      app: vllm-llama3
  template:
    metadata:
      labels:
        app: vllm-llama3
    spec:
      nodeSelector:
        workload: vllm-inference
      
      tolerations:
      - key: nvidia.com/gpu
        operator: Equal
        value: "true"
        effect: NoSchedule
      
      initContainers:
      # Download model from S3 on startup
      - name: download-model
        image: amazon/aws-cli
        command:
        - sh
        - -c
        - |
          aws s3 sync s3://ai-models/llama-3-70b-instruct /models/llama-3-70b --no-progress
        volumeMounts:
        - name: model-cache
          mountPath: /models
        env:
        - name: AWS_REGION
          value: us-west-2
      
      containers:
      - name: vllm
        image: vllm/vllm-openai:v0.3.1
        
        command:
        - python3
        - -m
        - vllm.entrypoints.openai.api_server
        - --model
        - /models/llama-3-70b
        - --tensor-parallel-size
        - "4"
        - --max-model-len
        - "8192"
        - --gpu-memory-utilization
        - "0.95"
        - --enable-prefix-caching
        - --disable-log-stats
        - --host
        - "0.0.0.0"
        - --port
        - "8000"
        
        ports:
        - containerPort: 8000
          name: http
        
        resources:
          requests:
            nvidia.com/gpu: 4
            memory: 160Gi
            cpu: 32
          limits:
            nvidia.com/gpu: 4
            memory: 160Gi
        
        volumeMounts:
        - name: model-cache
          mountPath: /models
        - name: shm
          mountPath: /dev/shm
        
        env:
        - name: VLLM_WORKER_MULTIPROC_METHOD
          value: spawn
        
        livenessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 600
          periodSeconds: 30
          timeoutSeconds: 10
        
        readinessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 600
          periodSeconds: 10
      
      volumes:
      - name: model-cache
        emptyDir:
          sizeLimit: 200Gi
      - name: shm
        emptyDir:
          medium: Memory
          sizeLimit: 32Gi
---
apiVersion: v1
kind: Service
metadata:
  name: vllm-llama3-service
  namespace: ai-platform
  labels:
    app: vllm-llama3
spec:
  selector:
    app: vllm-llama3
  ports:
  - port: 8000
    targetPort: 8000
  type: ClusterIP

# Deploy vLLM
kubectl apply -f vllm-llama3-deployment.yaml

# Wait for pods ready
kubectl wait --for=condition=ready pod -l app=vllm-llama3 -n ai-platform --timeout=600s

# Test inference
kubectl port-forward -n ai-platform svc/vllm-llama3-service 8000:8000

curl http://localhost:8000/v1/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "/models/llama-3-70b",
    "prompt": "Explain Kubernetes in one sentence:",
    "max_tokens": 50
  }'

# api_gateway.py
from fastapi import FastAPI, HTTPException, Header
from pydantic import BaseModel
import boto3
import httpx
import time
from enum import Enum

app = FastAPI(title="AI Platform Gateway")

# Backends
bedrock_client = boto3.client("bedrock-runtime", region_name="us-west-2")
vllm_endpoint = "http://vllm-llama3-service.ai-platform.svc.cluster.local:8000"

class ModelBackend(str, Enum):
    BEDROCK = "bedrock"
    VLLM = "vllm"

class CompletionRequest(BaseModel):
    prompt: str
    model: str = "auto"  # auto, bedrock/claude-3-5-sonnet, vllm/llama-3-70b
    max_tokens: int = 512
    temperature: float = 0.7

class CompletionResponse(BaseModel):
    text: str
    model: str
    backend: ModelBackend
    latency_ms: float
    estimated_cost_usd: float

# Model routing configuration
ROUTING_CONFIG = {
    "bedrock/claude-3-5-sonnet": {
        "backend": ModelBackend.BEDROCK,
        "bedrock_model_id": "anthropic.claude-3-5-sonnet-20240620-v1:0",
        "cost_per_1k_input": 0.003,
        "cost_per_1k_output": 0.015,
    },
    "bedrock/claude-opus-4": {
        "backend": ModelBackend.BEDROCK,
        "bedrock_model_id": "anthropic.claude-opus-4-20250514-v1:0",
        "cost_per_1k_input": 0.015,
        "cost_per_1k_output": 0.075,
    },
    "vllm/llama-3-70b": {
        "backend": ModelBackend.VLLM,
        "vllm_model": "/models/llama-3-70b",
        "cost_per_1k_input": 0.0003,  # Amortized infra cost
        "cost_per_1k_output": 0.0003,
    },
}

def route_model(request: CompletionRequest) -> str:
    """Determine which model to use."""
    
    if request.model != "auto":
        return request.model
    
    # Auto-routing logic
    prompt_len = len(request.prompt.split())
    
    # Complex queries to Bedrock
    if any(kw in request.prompt.lower() for kw in ["analyze", "explain", "design", "compare"]):
        return "bedrock/claude-3-5-sonnet"
    
    # Short queries to vLLM
    if prompt_len < 100:
        return "vllm/llama-3-70b"
    
    # Default to vLLM for cost
    return "vllm/llama-3-70b"

async def call_bedrock(model_config: dict, prompt: str, max_tokens: int, temperature: float) -> dict:
    """Call AWS Bedrock."""
    
    body = {
        "anthropic_version": "bedrock-2023-05-31",
        "messages": [{"role": "user", "content": prompt}],
        "max_tokens": max_tokens,
        "temperature": temperature,
    }
    
    import json
    response = bedrock_client.invoke_model(
        modelId=model_config["bedrock_model_id"],
        body=json.dumps(body),
    )
    
    result = json.loads(response["body"].read())
    
    return {
        "text": result["content"][0]["text"],
        "input_tokens": result["usage"]["input_tokens"],
        "output_tokens": result["usage"]["output_tokens"],
    }

async def call_vllm(model_config: dict, prompt: str, max_tokens: int, temperature: float) -> dict:
    """Call vLLM endpoint."""
    
    async with httpx.AsyncClient(timeout=60.0) as client:
        response = await client.post(
            f"{vllm_endpoint}/v1/completions",
            json={
                "model": model_config["vllm_model"],
                "prompt": prompt,
                "max_tokens": max_tokens,
                "temperature": temperature,
            },
        )
        response.raise_for_status()
        result = response.json()
    
    return {
        "text": result["choices"][0]["text"],
        "input_tokens": result["usage"]["prompt_tokens"],
        "output_tokens": result["usage"]["completion_tokens"],
    }

@app.post("/v1/completions", response_model=CompletionResponse)
async def create_completion(
    request: CompletionRequest,
    authorization: str = Header(None),
) -> CompletionResponse:
    """Unified completion endpoint."""
    
    # Authenticate (simplified)
    if not authorization or not authorization.startswith("Bearer "):
        raise HTTPException(status_code=401, detail="Invalid authorization")
    
    start_time = time.perf_counter()
    
    # Route to model
    model_key = route_model(request)
    
    if model_key not in ROUTING_CONFIG:
        raise HTTPException(status_code=400, detail=f"Unknown model: {model_key}")
    
    model_config = ROUTING_CONFIG[model_key]
    backend = model_config["backend"]
    
    # Call backend
    try:
        if backend == ModelBackend.BEDROCK:
            result = await call_bedrock(
                model_config,
                request.prompt,
                request.max_tokens,
                request.temperature,
            )
        else:  # VLLM
            result = await call_vllm(
                model_config,
                request.prompt,
                request.max_tokens,
                request.temperature,
            )
        
        # Calculate cost
        cost_usd = (
            (result["input_tokens"] / 1000) * model_config["cost_per_1k_input"] +
            (result["output_tokens"] / 1000) * model_config["cost_per_1k_output"]
        )
        
        latency_ms = (time.perf_counter() - start_time) * 1000
        
        # Log metrics (Prometheus)
        from prometheus_client import Counter, Histogram
        request_counter = Counter("api_requests_total", "Total requests", ["model", "backend"])
        latency_histogram = Histogram("api_latency_seconds", "Request latency", ["model", "backend"])
        
        request_counter.labels(model=model_key, backend=backend.value).inc()
        latency_histogram.labels(model=model_key, backend=backend.value).observe(latency_ms / 1000)
        
        return CompletionResponse(
            text=result["text"],
            model=model_key,
            backend=backend,
            latency_ms=latency_ms,
            estimated_cost_usd=cost_usd,
        )
    
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/health")
async def health():
    """Health check."""
    return {"status": "healthy"}

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=8080)

# api-gateway-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: api-gateway
  namespace: ai-platform
spec:
  replicas: 3
  selector:
    matchLabels:
      app: api-gateway
  template:
    metadata:
      labels:
        app: api-gateway
    spec:
      nodeSelector:
        workload: api-gateway
      
      serviceAccountName: api-gateway-sa
      
      containers:
      - name: gateway
        image: <your-registry>/ai-gateway:latest
        
        ports:
        - containerPort: 8080
          name: http
        
        env:
        - name: AWS_REGION
          value: us-west-2
        - name: VLLM_ENDPOINT
          value: http://vllm-llama3-service.ai-platform.svc.cluster.local:8000
        
        resources:
          requests:
            memory: 512Mi
            cpu: 500m
          limits:
            memory: 1Gi
            cpu: 1000m
        
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
          periodSeconds: 10
        
        readinessProbe:
          httpGet:
            path: /health
            port: 8080
          periodSeconds: 5
---
apiVersion: v1
kind: Service
metadata:
  name: api-gateway
  namespace: ai-platform
spec:
  selector:
    app: api-gateway
  ports:
  - port: 80
    targetPort: 8080
  type: LoadBalancer

# advanced_routing.py
from dataclasses import dataclass
from typing import Optional
import numpy as np

@dataclass
class RoutingDecision:
    model: str
    confidence: float
    reason: str

class ModelRouter:
    """Advanced model routing logic."""
    
    def __init__(self):
        self.complexity_classifier = self._load_complexity_classifier()
    
    def route(
        self,
        prompt: str,
        user_tier: str = "free",
        latency_sla_ms: Optional[int] = None,
    ) -> RoutingDecision:
        """Route request to optimal model."""
        
        # Classify query complexity
        complexity = self._classify_complexity(prompt)
        
        # Check user tier
        if user_tier == "enterprise":
            # Enterprise users get best quality
            if complexity == "high":
                return RoutingDecision(
                    model="bedrock/claude-opus-4",
                    confidence=0.95,
                    reason="High complexity + enterprise tier",
                )
            else:
                return RoutingDecision(
                    model="bedrock/claude-3-5-sonnet",
                    confidence=0.90,
                    reason="Enterprise tier",
                )
        
        # Free/Pro users: optimize for cost
        if complexity == "low":
            return RoutingDecision(
                model="vllm/mistral-7b",  # Cheapest, fastest
                confidence=0.85,
                reason="Low complexity query",
            )
        
        elif complexity == "medium":
            # vLLM Llama 3 70B handles most queries well
            return RoutingDecision(
                model="vllm/llama-3-70b",
                confidence=0.80,
                reason="Medium complexity, cost-optimized",
            )
        
        else:  # high complexity
            # Use Bedrock for quality
            return RoutingDecision(
                model="bedrock/claude-3-5-sonnet",
                confidence=0.90,
                reason="High complexity requires reasoning",
            )
    
    def _classify_complexity(self, prompt: str) -> str:
        """Classify query complexity (low/medium/high)."""
        
        # Simple heuristics (replace with ML classifier in production)
        prompt_lower = prompt.lower()
        
        # High complexity indicators
        if any(kw in prompt_lower for kw in [
            "analyze", "design", "architect", "compare multiple",
            "explain why", "what are the tradeoffs",
        ]):
            return "high"
        
        # Low complexity indicators
        if any(kw in prompt_lower for kw in [
            "what is", "define", "list", "who", "when", "where",
        ]):
            return "low"
        
        # Default to medium
        return "medium"
    
    def _load_complexity_classifier(self):
        """Load ML classifier (placeholder)."""
        # In production: load fine-tuned classifier
        return None

# Usage
router = ModelRouter()

queries = [
    ("What is Kubernetes?", "free"),
    ("Design a distributed caching architecture", "free"),
    ("Explain Kubernetes networking", "enterprise"),
]

for query, tier in queries:
    decision = router.route(query, user_tier=tier)
    print(f"{tier:10} | {decision.model:25} | {query[:50]}")
    print(f"           Reason: {decision.reason}\n")

# cost_tracker.py
from datetime import datetime, timezone
from dataclasses import dataclass
import asyncpg

@dataclass
class RequestCost:
    timestamp: datetime
    model: str
    backend: str
    input_tokens: int
    output_tokens: int
    cost_usd: float
    latency_ms: float
    user_id: str

class CostTracker:
    """Track costs across backends."""
    
    def __init__(self, db_pool):
        self.db = db_pool
    
    async def record_request(self, cost: RequestCost) -> None:
        """Record request cost."""
        
        await self.db.execute(
            """
            INSERT INTO request_costs 
            (timestamp, model, backend, input_tokens, output_tokens, cost_usd, latency_ms, user_id)
            VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
            """,
            cost.timestamp, cost.model, cost.backend,
            cost.input_tokens, cost.output_tokens,
            cost.cost_usd, cost.latency_ms, cost.user_id,
        )
    
    async def get_cost_breakdown(self, days: int = 7) -> dict:
        """Analyze costs by backend and model."""
        
        from datetime import timedelta
        start_date = datetime.now(timezone.utc) - timedelta(days=days)
        
        rows = await self.db.fetch(
            """
            SELECT 
                backend,
                model,
                COUNT(*) as request_count,
                SUM(cost_usd) as total_cost,
                AVG(cost_usd) as avg_cost_per_request,
                AVG(latency_ms) as avg_latency_ms
            FROM request_costs
            WHERE timestamp >= $1
            GROUP BY backend, model
            ORDER BY total_cost DESC
            """,
            start_date,
        )
        
        results = {}
        for row in rows:
            results[f"{row['backend']}/{row['model']}"] = {
                "request_count": row["request_count"],
                "total_cost_usd": float(row["total_cost"]),
                "avg_cost_per_request": float(row["avg_cost_per_request"]),
                "avg_latency_ms": float(row["avg_latency_ms"]),
            }
        
        return results
    
    async def calculate_savings(self, days: int = 7) -> dict:
        """Calculate savings from routing vs 100% Bedrock."""
        
        from datetime import timedelta
        start_date = datetime.now(timezone.utc) - timedelta(days=days)
        
        # Actual cost
        actual_cost = await self.db.fetchval(
            "SELECT SUM(cost_usd) FROM request_costs WHERE timestamp >= $1",
            start_date,
        )
        
        # Hypothetical cost if all on Bedrock Claude
        total_requests = await self.db.fetchval(
            "SELECT COUNT(*) FROM request_costs WHERE timestamp >= $1",
            start_date,
        )
        
        avg_bedrock_cost = 0.02  # $0.02 per request estimate
        hypothetical_bedrock_cost = total_requests * avg_bedrock_cost
        
        savings = hypothetical_bedrock_cost - float(actual_cost)
        savings_pct = (savings / hypothetical_bedrock_cost) * 100
        
        return {
            "actual_cost_usd": float(actual_cost),
            "hypothetical_bedrock_cost_usd": hypothetical_bedrock_cost,
            "savings_usd": savings,
            "savings_percentage": savings_pct,
        }

# Usage example
async def analyze_costs():
    pool = await asyncpg.create_pool(database="ai_platform")
    tracker = CostTracker(pool)
    
    breakdown = await tracker.get_cost_breakdown(days=7)
    print("Cost breakdown (last 7 days):")
    for model, stats in breakdown.items():
        print(f"  {model:30} ${stats['total_cost_usd']:8.2f} ({stats['request_count']:,} requests)")
    
    savings = await tracker.calculate_savings(days=7)
    print(f"\nSavings vs 100% Bedrock: ${savings['savings_usd']:.2f} ({savings['savings_percentage']:.1f}%)")

# metrics.py
from prometheus_client import Counter, Histogram, Gauge

# Request metrics
requests_total = Counter(
    "ai_requests_total",
    "Total AI requests",
    ["model", "backend", "status"],
)

request_latency = Histogram(
    "ai_request_latency_seconds",
    "Request latency",
    ["model", "backend"],
    buckets=[0.1, 0.5, 1.0, 2.0, 5.0, 10.0],
)

request_cost = Histogram(
    "ai_request_cost_usd",
    "Request cost in USD",
    ["model", "backend"],
    buckets=[0.0001, 0.001, 0.01, 0.1, 1.0],
)

# Token metrics
tokens_processed = Counter(
    "ai_tokens_total",
    "Total tokens processed",
    ["model", "backend", "type"],  # type: input/output
)

# Backend health
backend_available = Gauge(
    "ai_backend_available",
    "Backend availability (1=up, 0=down)",
    ["backend"],
)

# Queue depth
queue_depth = Gauge(
    "ai_queue_depth",
    "Number of queued requests",
    ["backend"],
)

# grafana-dashboard.json (excerpt)
{
  "dashboard": {
    "title": "AI Platform Overview",
    "panels": [
      {
        "title": "Requests per Second by Backend",
        "targets": [
          {"expr": "rate(ai_requests_total[5m])"}
        ]
      },
      {
        "title": "Cost per Hour",
        "targets": [
          {"expr": "sum(rate(ai_request_cost_usd[1h])) * 3600"}
        ]
      },
      {
        "title": "P95 Latency by Model",
        "targets": [
          {"expr": "histogram_quantile(0.95, ai_request_latency_seconds_bucket)"}
        ]
      },
      {
        "title": "Backend Distribution",
        "targets": [
          {"expr": "sum(rate(ai_requests_total[5m])) by (backend)"}
        ]
      }
    ]
  }
}