DevOps & Infrastructure

┌────────┐   ┌────────┐   ┌────────┐   ┌────────┐   ┌────────┐
│  Code  │──►│ Build  │──►│  Test  │──►│ Deploy │──►│Monitor │
│ Commit │   │        │   │        │   │        │   │        │
└────────┘   └────────┘   └────────┘   └────────┘   └────────┘
                CI                           CD

name: CI/CD Pipeline

on:
  push:
    branches: [main]       # Trigger on merges to main
  pull_request:
    branches: [main]       # Trigger on PRs targeting main (catches issues before merge)

jobs:
  build-and-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3      # Pull the code
      
      - name: Setup Node.js
        uses: actions/setup-node@v3
        with:
          node-version: '18'           # Pin to specific version -- avoid "works on my machine"
          
      - name: Install dependencies
        run: npm ci                    # Use 'ci' not 'install' -- deterministic, lockfile-based
        
      - name: Run tests
        run: npm test                  # Gate: if tests fail, pipeline stops here
        
      - name: Build
        run: npm run build             # Gate: catches TypeScript/build errors
  
  deploy:
    needs: build-and-test              # Only runs if build-and-test succeeds
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/main'  # Only deploy from main, not from PR branches
    steps:
      - name: Deploy to production
        run: |
          # In practice, this calls a deploy script, pushes a Docker image,
          # or triggers a GitOps sync (ArgoCD, Flux). Never run raw shell
          # commands against production servers.
          echo "Deploying to production..."

┌─────────────────────────────────────────────────┐
│                    Host OS                       │
│  ┌─────────────────────────────────────────────┐│
│  │              Docker Engine                  ││
│  │  ┌───────────┐ ┌───────────┐ ┌───────────┐ ││
│  │  │ Container │ │ Container │ │ Container │ ││
│  │  │  ┌─────┐  │ │  ┌─────┐  │ │  ┌─────┐  │ ││
│  │  │  │ App │  │ │  │ App │  │ │  │ App │  │ ││
│  │  │  └─────┘  │ │  └─────┘  │ │  └─────┘  │ ││
│  │  │   Libs    │ │   Libs    │ │   Libs    │ ││
│  │  └───────────┘ └───────────┘ └───────────┘ ││
│  └─────────────────────────────────────────────┘│
└─────────────────────────────────────────────────┘

# 1. Use specific version, not 'latest' -- 'latest' is a moving target
#    that breaks builds unpredictably. Pin the exact version.
#    Use -alpine for smaller images (~5x smaller than debian-based).
FROM node:18-alpine

# 2. Set working directory -- all subsequent commands run from here
WORKDIR /app

# 3. Copy ONLY dependency files first -- this is the cache optimization trick.
#    If package.json has not changed, Docker reuses the cached layer and
#    skips the expensive npm install step entirely. This alone can cut
#    build times from 5 minutes to 30 seconds on subsequent builds.
COPY package*.json ./

# 4. Install production dependencies only (skip devDependencies)
RUN npm ci --only=production

# 5. NOW copy application code -- this layer changes on every build,
#    but all layers above it are cached if dependencies did not change.
COPY . .

# 6. Security: never run as root inside containers. If the app is
#    compromised, the attacker only has limited 'node' user permissions.
USER node

# 7. Document the port (does not actually publish it -- that's done at runtime)
EXPOSE 3000

# 8. Health check -- lets Docker/K8s know if the container is healthy.
#    Unhealthy containers are automatically restarted.
HEALTHCHECK --interval=30s --timeout=3s \
  CMD curl -f http://localhost:3000/health || exit 1

# 9. Use exec form (JSON array), not shell form -- ensures proper signal handling
#    so the container shuts down gracefully on SIGTERM.
CMD ["node", "server.js"]

version: '3.8'

services:
  web:
    build: .                                  # Build from local Dockerfile
    ports:
      - "3000:3000"                           # host:container port mapping
    environment:
      - DATABASE_URL=postgres://db:5432/app   # 'db' is the service name (Docker DNS)
    depends_on:
      - db                                    # Start db before web (order, not readiness)
      - redis
    
  db:
    image: postgres:14                        # Use official image, pin version
    volumes:
      - postgres_data:/var/lib/postgresql/data  # Persist data across container restarts
    environment:
      - POSTGRES_PASSWORD=secret              # For local dev only -- use secrets in prod
      
  redis:
    image: redis:alpine                       # Alpine variant for smaller image

volumes:
  postgres_data:                              # Named volume survives docker compose down

┌────────────────────────────────────────────────────────┐
│                      Cluster                           │
│  ┌──────────────────────────────────────────────────┐  │
│  │                 Control Plane                    │  │
│  │  ┌──────────┐ ┌──────────┐ ┌──────────────────┐ │  │
│  │  │ API      │ │ Scheduler│ │ Controller       │ │  │
│  │  │ Server   │ │          │ │ Manager          │ │  │
│  │  └──────────┘ └──────────┘ └──────────────────┘ │  │
│  └──────────────────────────────────────────────────┘  │
│  ┌──────────────────────────────────────────────────┐  │
│  │                    Nodes                         │  │
│  │  ┌─────────────────┐  ┌─────────────────┐       │  │
│  │  │     Node 1      │  │     Node 2      │       │  │
│  │  │ ┌─────┐ ┌─────┐ │  │ ┌─────┐ ┌─────┐ │       │  │
│  │  │ │ Pod │ │ Pod │ │  │ │ Pod │ │ Pod │ │       │  │
│  │  │ └─────┘ └─────┘ │  │ └─────┘ └─────┘ │       │  │
│  │  └─────────────────┘  └─────────────────┘       │  │
│  └──────────────────────────────────────────────────┘  │
└────────────────────────────────────────────────────────┘

# Deployment -- manages a set of identical pods (your application instances)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-app
spec:
  replicas: 3                     # K8s maintains exactly 3 pods at all times
  selector:
    matchLabels:
      app: web                    # "This deployment manages pods labeled app=web"
  template:
    metadata:
      labels:
        app: web
    spec:
      containers:
      - name: web
        image: myapp:1.0.0        # Always use specific tags, never 'latest'
        ports:
        - containerPort: 3000
        resources:
          requests:               # Minimum resources guaranteed to the pod
            memory: "128Mi"       # Used by scheduler to find a node with enough capacity
            cpu: "100m"           # 100 millicores = 0.1 CPU cores
          limits:                 # Maximum resources allowed -- pod is killed if exceeded
            memory: "256Mi"       # OOMKilled if the container exceeds this
            cpu: "200m"           # Throttled (not killed) if it exceeds this
---
# Service -- provides stable networking for the ephemeral pods above.
# Pods come and go (new IPs each time), but the Service gives them a
# permanent address that other services can rely on.
apiVersion: v1
kind: Service
metadata:
  name: web-service
spec:
  selector:
    app: web                      # Routes traffic to all pods with label app=web
  ports:
  - port: 80                     # Port the Service exposes
    targetPort: 3000              # Port the container listens on
  type: LoadBalancer              # Provisions a cloud load balancer (AWS ALB, GCP LB)

# main.tf -- This file IS your infrastructure. Delete the server in the console?
# Run 'terraform apply' and it comes back exactly as specified. No guessing,
# no "what was that setting again?" moments at 3 AM.
provider "aws" {
  region = "us-east-1"
}

# Compute -- a single web server instance.
# In practice, you would use an Auto Scaling Group behind an ALB,
# but this illustrates the core concept: infrastructure as a text file.
resource "aws_instance" "web" {
  ami           = "ami-0c55b159cbfafe1f0"  # Amazon Linux 2 -- pin the AMI ID
  instance_type = "t2.micro"                # ~$8/month -- right-size for dev/staging
  
  tags = {
    Name        = "web-server"
    Environment = "production"              # Always tag for cost tracking and filtering
    ManagedBy   = "terraform"               # So humans know not to edit this in the console
  }
}

# Storage -- versioned S3 bucket for static assets.
# Versioning lets you recover from accidental deletes or overwrites.
resource "aws_s3_bucket" "static" {
  bucket = "my-static-assets"
  
  versioning {
    enabled = true  # Every overwrite creates a new version -- undo is trivial
  }
}

# Output values -- useful for piping into other tools or scripts.
# After 'terraform apply', these print to stdout and are stored in state.
output "web_server_ip" {
  value = aws_instance.web.public_ip
}

from prometheus_client import Counter, Histogram, Gauge

# Counter -- cumulative, only increases. Resets to 0 on restart.
# Use for: total requests, errors, bytes processed.
# Prometheus calculates rate of change: rate(http_requests_total[5m])
http_requests_total = Counter(
    'http_requests_total',
    'Total HTTP requests',
    ['method', 'endpoint', 'status']  # Labels for dimensional filtering
)

# Histogram -- tracks distribution of values in configurable buckets.
# Use for: request durations, response sizes. Enables percentile queries.
# The buckets define the resolution: more buckets = more precision = more storage.
request_duration = Histogram(
    'http_request_duration_seconds',
    'Request duration in seconds',
    ['endpoint'],
    buckets=[.01, .05, .1, .5, 1, 5]  # Choose buckets around your SLO thresholds
)

# Gauge -- value that can go up and down. Current state, not cumulative.
# Use for: active connections, queue depth, temperature, memory usage.
active_connections = Gauge(
    'active_connections',
    'Number of active connections'
)

# Usage -- instrument your endpoints to automatically emit metrics.
@app.route('/api/users')
def get_users():
    with request_duration.labels(endpoint='/api/users').time():  # Auto-measure duration
        result = fetch_users()
    http_requests_total.labels(
        method='GET', endpoint='/api/users', status='200'
    ).inc()  # Increment the counter
    return result

import structlog
import json

logger = structlog.get_logger()

# ❌ Bad: Unstructured logs -- looks fine in development, useless at scale.
# How do you query "all orders from user 123 that took over 1s"? You cannot.
print(f"User {user_id} created order {order_id}")

# ✅ Good: Structured JSON logs -- every field is queryable and filterable.
# In Elasticsearch/Loki/Splunk, you can write: user_id="123" AND duration_ms>1000
logger.info(
    "order_created",
    user_id=user_id,
    order_id=order_id,
    total=order.total,
    items_count=len(order.items),
    duration_ms=duration
)

# Output (JSON) -- machines love this, log aggregators index every field:
# {"event": "order_created", "user_id": "123", "order_id": "456", 
#  "total": 99.99, "items_count": 3, "duration_ms": 45, 
#  "timestamp": "2024-01-15T10:30:00Z"}

┌─────────────┐     ┌─────────────┐     ┌─────────────────────┐
│  Developer  │────►│    Git      │────►│    GitOps Agent     │
│  Push       │     │  Repository │     │ (ArgoCD/Flux)       │
└─────────────┘     └─────────────┘     └──────────┬──────────┘
                                                   │
                                                   ▼
                                        ┌─────────────────────┐
                                        │    Kubernetes       │
                                        │    Cluster          │
                                        └─────────────────────┘

apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: my-app
  namespace: argocd                            # ArgoCD lives in its own namespace
spec:
  project: default
  source:
    repoURL: https://github.com/org/repo.git  # Git repo = single source of truth
    targetRevision: HEAD                        # Track the latest commit on default branch
    path: k8s/overlays/production               # Kustomize overlay for production config
  destination:
    server: https://kubernetes.default.svc      # The cluster ArgoCD is managing
    namespace: production
  syncPolicy:
    automated:
      prune: true       # Delete resources from cluster if removed from Git
      selfHeal: true    # Revert manual cluster changes to match Git (the thermostat)
    syncOptions:
      - CreateNamespace=true   # Create the namespace if it does not exist

Before: Blue is live, Green has the new version staged and tested
┌─────────────────────────────────────────────────┐
│  Load Balancer  ───────────►  Blue (v1) ✓      │
│                               Green (v2)        │
└─────────────────────────────────────────────────┘

After: One DNS/LB change flips all traffic to Green instantly
┌─────────────────────────────────────────────────┐
│  Load Balancer  ───────────►  Blue (v1)        │
│                 ───────────►  Green (v2) ✓     │
└─────────────────────────────────────────────────┘
Rollback: Flip back to Blue in seconds if Green has issues

┌──────────────────────────────────────────────────┐
│                  Load Balancer                   │
│                       │                          │
│          ┌────────────┴────────────┐            │
│          │                         │            │
│          ▼ (90%)                   ▼ (10%)      │
│    ┌──────────┐              ┌──────────┐      │
│    │  v1.0    │              │  v1.1    │      │
│    │ (stable) │              │ (canary) │      │
│    └──────────┘              └──────────┘      │
└──────────────────────────────────────────────────┘

Gradually increase canary traffic: 10% → 25% → 50% → 100%
At each step: compare error rates, latency, and business metrics.
If canary looks healthy, promote. If not, roll back automatically.

# Kubernetes rolling update strategy -- fine-tune the rollout speed
# and safety margin for your specific application.
spec:
  replicas: 4
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1        # At most 5 pods during update (4 desired + 1 extra)
      maxUnavailable: 1  # At least 3 pods always serving traffic
  # Readiness probe is CRITICAL for rolling updates -- without it, K8s
  # considers a pod "ready" the instant its container starts, even if
  # the app needs 30 seconds to warm up. Result: traffic hits an
  # uninitialized app and users see errors during deployment.
  template:
    spec:
      containers:
      - name: web
        readinessProbe:
          httpGet:
            path: /health        # Your app must return 200 when ready
            port: 3000
          initialDelaySeconds: 5  # Wait 5s before first check
          periodSeconds: 10       # Check every 10s

import hvac

# Initialize client
client = hvac.Client(url='https://vault.example.com')
client.token = os.environ['VAULT_TOKEN']

# Read secret
secret = client.secrets.kv.v2.read_secret_version(
    path='myapp/database'
)
db_password = secret['data']['data']['password']

# Dynamic secrets (auto-rotated)
creds = client.secrets.database.generate_credentials(
    name='my-role'
)
print(f"Username: {creds['data']['username']}")
print(f"Password: {creds['data']['password']}")
# Credentials automatically expire and rotate

# This Custom Resource tells the External Secrets Operator:
# "Go fetch the password from Vault, and create a K8s Secret called
# 'database-secret' with it. Refresh every hour in case it rotated."
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: database-credentials
spec:
  refreshInterval: 1h                  # Re-sync from Vault every hour (catches rotations)
  secretStoreRef:
    kind: ClusterSecretStore
    name: vault-backend                # Points to your Vault instance configuration
  target:
    name: database-secret              # The K8s Secret name your pods will reference
  data:
  - secretKey: password                # Key inside the K8s Secret
    remoteRef:
      key: secret/data/production/database   # Path in Vault
      property: password                      # Field within the Vault secret

# Error budget calculation
monthly_minutes = 30 * 24 * 60  # 43,200 minutes

slo_99_9 = 0.999
error_budget_minutes = monthly_minutes * (1 - slo_99_9)  # 43.2 minutes

slo_99_99 = 0.9999
error_budget_minutes = monthly_minutes * (1 - slo_99_99)  # 4.32 minutes

modules/
├── vpc/
│   ├── main.tf
│   ├── variables.tf
│   ├── outputs.tf
│   └── README.md
├── eks/
│   ├── main.tf
│   ├── variables.tf
│   └── outputs.tf
└── rds/
    ├── main.tf
    ├── variables.tf
    └── outputs.tf

environments/
├── dev/
│   ├── main.tf
│   ├── terraform.tfvars
│   └── backend.tf
├── staging/
│   └── ...
└── production/
    └── ...

# backend.tf -- remote state with locking (non-negotiable for team use)
terraform {
  backend "s3" {
    bucket         = "my-terraform-state"       # Dedicated bucket, versioned
    key            = "production/terraform.tfstate"
    region         = "us-east-1"
    encrypt        = true                       # State contains secrets -- encrypt at rest
    dynamodb_table = "terraform-locks"          # Prevents concurrent modifications
  }
}