# Create the parent directory for the MLOps training and model serving lab manifests
mkdir -p ~/enterprise-mlops-lab/training
mkdir -p ~/enterprise-mlops-lab/serving/triton-model-repo
mkdir -p ~/enterprise-mlops-lab/autoscaling
cd ~/enterprise-mlops-lab
 
# Verify the directory structure was created successfully
pwd

cat << 'EOF' > training/pytorch-distributed-job.yaml
apiVersion: kubeflow.org/v1
kind: PyTorchJob
metadata:
  name: production-recommendation-model-training
  namespace: production
  labels:
    tier: ai-training
spec:
  # Master Pod Cleanup Policy: Delete worker pods the moment Master finishes!
  # (Unlocks expensive GPUs instantly and eliminates hardware stranding!)
  cleanPodPolicy: Running
  pytorchReplicaSpecs:
    # 1. Master Pod Replica Specification!
    Master:
      replicas: 1
      restartPolicy: OnFailure
      template:
        spec:
          containers:
          - name: pytorch
            image: myregistry.com/ai/recommendation-model:v2.0.0
            command: ["python3", "-m", "torch.distributed.run", "train.py"]
            resources:
              limits:
                nvidia.com/gpu: 1
    # 2. Worker Pod Replica Specification!
    Worker:
      replicas: 15 # Deploy 15 worker pods (Total WORLD_SIZE = 16!)
      restartPolicy: OnFailure
      template:
        spec:
          containers:
          - name: pytorch
            image: myregistry.com/ai/recommendation-model:v2.0.0
            command: ["python3", "-m", "torch.distributed.run", "train.py"]
            resources:
              limits:
                nvidia.com/gpu: 1
EOF

cat << 'EOF' > serving/vllm-serving-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: vllm-inference-engine
  namespace: production
  labels:
    tier: ai-inference
spec:
  replicas: 4
  selector:
    matchLabels:
      app: vllm-engine
  template:
    metadata:
      labels:
        app: vllm-engine
    spec:
      # ==============================================================================
      # INIT CONTAINER: S3 Model Puller with NVMe HostPath Caching!
      # ==============================================================================
      initContainers:
      - name: s3-model-puller
        image: peakai/s5cmd:v2.2.2 # Utilize s5cmd (Blazing fast C++ S3 client!)
        command:
        - "/bin/sh"
        - "-c"
        - |
          MODEL_DIR="/var/cache/models/llama3-70b-prod"
          mkdir -p "$MODEL_DIR"
          
          # Inspect local NVMe host path cache! If weights exist, skip S3 download entirely!
          if [ "$(ls -A $MODEL_DIR 2>/dev/null)" ]; then
              echo "[CACHE EVALUATION]: CACHE HIT! Model weights already exist on host NVMe."
              exit 0
          else
              echo "[CACHE EVALUATION]: CACHE MISS! Downloading weights from S3 Model Registry..."
              s5cmd --stat cp "s3://enterprise-model-registry/llama3-70b-prod/*" "$MODEL_DIR/"
          fi
        volumeMounts:
        - name: model-cache-volume
          mountPath: /var/cache/models
        env:
        - name: AWS_REGION
          value: "us-east-1"
 
      # ==============================================================================
      # MAIN CONTAINER: vLLM Serving Engine (Mounts NVMe Cache Instantly!)
      # ==============================================================================
      containers:
      - name: vllm-container
        image: vllm/vllm-openai:v0.4.2 # Lightweight 500MB serving image (Instant Pull!)
        command: ["python3", "-m", "vllm.entrypoints.openai.api_server"]
        args:
        - "--model=/var/cache/models/llama3-70b-prod"
        # Master Memory Governance: Pre-allocate 90% of VRAM for PagedAttention KV Cache!
        # (Eliminates VRAM fragmentation entirely and prevents CUDA OOM crashes!)
        - "--gpu-memory-utilization=0.90"
        - "--max-model-len=4096"
        - "--tensor-parallel-size=2" # Split 70B model across 2 physical GPUs!
        resources:
          limits:
            nvidia.com/gpu: 2 # Request 2 physical NVIDIA H100 GPUs!
        ports:
        - containerPort: 8000
        volumeMounts:
        - name: model-cache-volume
          mountPath: /var/cache/models
        readinessProbe:
          httpGet:
            path: /health
            port: 8000
          initialDelaySeconds: 180 # Allow 3 minutes for 140GB model weights to load!
          periodSeconds: 10
      
      # Master Storage Governance: Mount /var/cache/models directly from host NVMe filesystem!
      # (Slashes container startup latency from 50 minutes down to 3 seconds!)
      volumes:
      - name: model-cache-volume
        hostPath:
          path: /var/cache/models
          type: DirectoryOrCreate
EOF

cat << 'EOF' > serving/triton-model-repo/config.pbtxt
name: "production-recommendation-engine"
platform: "tensorrt_plan"
max_batch_size: 128 # Enable Dynamic Batching! Allow up to 128 requests per batch!
 
# Define exact Dynamic Batching queue delay window!
# (Triton will pause for 5 milliseconds to group incoming user requests into a single batch!)
dynamic_batching {
  max_queue_delay_microseconds: 5000
}
 
instance_group [
  {
    count: 1
    kind: KIND_GPU # Execute model on physical GPU silicon!
  }
]
EOF

cat << 'EOF' > autoscaling/vllm-keda-autoscaler.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: vllm-keda-autoscaler
  namespace: production
  labels:
    tier: ai-autoscaling
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: vllm-inference-engine # Target our pristine vLLM Deployment!
  # Master Financial Governance: Scale to ZERO pods when zero users are online!
  # (Unlocks expensive GPUs instantly and eliminates cloud computing waste!)
  minReplicaCount: 0
  maxReplicaCount: 20 # Enable dynamic KEDA autoscaling up to 20 pods!
  cooldownPeriod: 300 # Wait 5 minutes after traffic drops before scaling down
  pollingInterval: 15 # Poll Prometheus metrics every 15 seconds
  triggers:
  # ==============================================================================
  # TRIGGER 1: Request Queue Depth (vllm:num_requests_waiting > 10)
  # ==============================================================================
  - type: prometheus
    metadata:
      serverAddress: http://prometheus-k8s.monitoring.svc.cluster.local:9090
      # PromQL Math: Average number of requests waiting in vLLM execution queue
      query: sum(vllm:num_requests_waiting{namespace="production"})
      threshold: '10' # Scale up new pods when > 10 requests are waiting in queue!
 
  # ==============================================================================
  # TRIGGER 2: Physical GPU Silicon Utilization (DCGM_FI_DEV_GPU_UTIL > 80%)
  # ==============================================================================
  - type: prometheus
    metadata:
      serverAddress: http://prometheus-k8s.monitoring.svc.cluster.local:9090
      # PromQL Math: Average physical GPU silicon utilization across pods
      query: avg(DCGM_FI_DEV_GPU_UTIL{namespace="production"})
      threshold: '80' # Scale up new pods when physical GPU utilization exceeds 80%!
EOF

# 1. Verify cleanPodPolicy in the Kubeflow PyTorchJob manifest
cat training/pytorch-distributed-job.yaml | grep -E "cleanPodPolicy:.*Running"
 
# 2. Verify PagedAttention memory utilization flag in the vLLM deployment manifest
cat serving/vllm-serving-deployment.yaml | grep -E "gpu-memory-utilization.*0.90"
 
# 3. Verify hostPath NVMe caching mount in the vLLM deployment manifest
cat serving/vllm-serving-deployment.yaml | grep -E "hostPath:.*" -A 2
 
# 4. Verify Dynamic Batching queue delay window in the Triton configuration manifest
cat serving/triton-model-repo/config.pbtxt | grep -E "max_queue_delay_microseconds:.*5000"
 
# 5. Verify scale-to-zero minReplicaCount in the KEDA ScaledObject manifest
cat autoscaling/vllm-keda-autoscaler.yaml | grep -E "minReplicaCount:.*0"
 
# 6. Verify request queue depth PromQL query in the KEDA ScaledObject manifest
cat autoscaling/vllm-keda-autoscaler.yaml | grep -E "query:.*vllm:num_requests_waiting"

  cleanPodPolicy: Running
        - "--gpu-memory-utilization=0.90"
      - name: model-cache-volume
        hostPath:
          path: /var/cache/models
  max_queue_delay_microseconds: 5000
  minReplicaCount: 0
      query: sum(vllm:num_requests_waiting{namespace="production"})

# Safely remove the enterprise MLOps lab directory
rm -rf ~/enterprise-mlops-lab
 
# Verify the directory was removed successfully
ls -la ~/enterprise-mlops-lab 2>/dev/null || echo "Cleanup complete. Directory successfully removed."