# Create the parent directory for the AI infrastructure and GPU management lab manifests
mkdir -p ~/enterprise-ai-lab/sharing
mkdir -p ~/enterprise-ai-lab/operator
mkdir -p ~/enterprise-ai-lab/networking
mkdir -p ~/enterprise-ai-lab/monitoring
cd ~/enterprise-ai-lab
 
# Verify the directory structure was created successfully
pwd

cat << 'EOF' > sharing/time-slicing-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: nvidia-time-slicing-config
  namespace: kube-system
data:
  any: |-
    version: v1
    flags:
      # Do not rename resource strings; keep advertising generic nvidia.com/gpu!
      renameByDefault: false
    sharing:
      timeSlicing:
        # Enable Time-Slicing! Allow up to 4 pods to share a single physical GPU concurrently!
        # (Multiplies 1 physical GPU into 4 virtual nvidia.com/gpu resources!)
        resources:
        - name: nvidia.com/gpu
          replicas: 4
EOF

cat << 'EOF' > operator/cluster-policy.yaml
apiVersion: nvidia.com/v1
kind: ClusterPolicy
metadata:
  name: cluster-policy
spec:
  # 1. Containerized NVIDIA Driver Configuration!
  driver:
    enabled: true
    version: "550.54.15" # Specify exact proprietary driver version to deploy across nodes!
    image: "driver"
    repository: "nvcr.io/nvidia"
    manager:
      env:
      # Enable Dynamic Kernel Module Support (DKMS) to ensure driver survives kernel updates!
      - name: ENABLE_DKMS
        value: "true"
 
  # 2. NVIDIA Container Toolkit Configuration!
  toolkit:
    enabled: true
    version: "v1.15.0-ubuntu22.04"
    installDir: "/usr/local/nvidia" # Mount path on host worker node!
 
  # 3. NVIDIA k8s-device-plugin Configuration!
  devicePlugin:
    enabled: true
    version: "v0.15.0"
    config:
      # Mount our Time-Slicing ConfigMap to eliminate hardware stranding!
      name: "nvidia-time-slicing-config"
      default: "any"
 
  # 4. NVIDIA DCGM Exporter Configuration (GPU Hardware Telemetry)!
  dcgmExporter:
    enabled: true
    version: "3.3.5-3.4.0-ubuntu22.04"
    serviceMonitor:
      enabled: true # Automatically generate Prometheus Operator ServiceMonitor!
 
  # 5. Multi-Instance GPU (MIG) Configuration!
  mig:
    strategy: single # Set to single or mixed depending on cluster multi-tenancy requirements
EOF

cat << 'EOF' > networking/ib-sriov-cni.conflist
{
  "cniVersion": "0.3.1",
  "name": "ib-sriov-network",
  "plugins": [
    {
      # Utilize HostDevice CNI plugin! Bypasses virtual ethernet (veth) pairs entirely!
      "type": "host-device",
      # Specify exact physical InfiniBand Virtual Function (VF) device name on host!
      "device": "ib0",
      "ipam": {
        "type": "host-local",
        "subnet": "192.168.100.0/24",
        "routes": [
          { "dst": "0.0.0.0/0" }
        ]
      }
    }
  ]
}
EOF

cat << 'EOF' > monitoring/dcgm-alerts.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: production-gpu-hardware-alerts
  namespace: production
  labels:
    release: prometheus-stack # Mandatory label matching Prometheus Operator selector!
spec:
  groups:
  - name: dcgm-gpu-alerts
    rules:
    # ==============================================================================
    # ALERT 1: VRAM EXHAUSTION (Alerts BEFORE a fatal CUDA OOM crash occurs!)
    # ==============================================================================
    - alert: GpuVramExhaustionWarning
      # PromQL Math: Used VRAM / Total VRAM * 100 > 95%
      expr: >-
        DCGM_FI_DEV_FB_USED / DCGM_FI_DEV_FB_TOTAL * 100 > 95.0
      for: 2m
      labels:
        severity: warning # Skips PagerDuty! Sends peaceful Slack warning!
        tier: ai-platform
      annotations:
        summary: "GPU VRAM utilization exceeded 95% on node {{ $labels.instance }} (GPU {{ $labels.gpu }})"
        description: "AI workload is approaching maximum High Bandwidth Memory capacity. Danger of CUDA Out-Of-Memory (OOM) crash."
        runbook_url: "https://wiki.mycompany.com/runbooks/gpu-vram-exhaustion"
 
    # ==============================================================================
    # ALERT 2: XID HARDWARE FAULT (Fatal Silicon Error -> PAGERDUTY PHONE CALL!)
    # ==============================================================================
    - alert: GpuXidHardwareFault
      # PromQL Math: Any active XID error code > 0 (e.g., XID 43: GPU fell off PCIe bus!)
      expr: DCGM_FI_DEV_XID_ERRORS > 0
      for: 1m
      labels:
        severity: critical # Master routing label! Triggers PagerDuty phone call!
        tier: ai-platform
      annotations:
        summary: "Fatal NVIDIA XID Error Code {{ $value }} detected on node {{ $labels.instance }} (GPU {{ $labels.gpu }})"
        description: "Underlying physical GPU silicon encountered a fatal hardware or kernel driver fault (e.g., XID 43 bus disconnect or XID 48 ECC corruption)."
        runbook_url: "https://wiki.mycompany.com/runbooks/gpu-xid-faults"
 
    # ==============================================================================
    # ALERT 3: THERMAL THROTTLING (Overheating -> PEACEFUL SLACK WARNING!)
    # ==============================================================================
    - alert: GpuThermalThrottlingWarning
      # PromQL Math: Physical GPU core temperature > 85 Celsius
      expr: DCGM_FI_DEV_GPU_TEMP > 85
      for: 3m
      labels:
        severity: warning
        tier: ai-platform
      annotations:
        summary: "GPU Core Temperature exceeded 85C on node {{ $labels.instance }} (GPU {{ $labels.gpu }})"
        runbook_url: "https://wiki.mycompany.com/runbooks/gpu-thermal-throttling"
EOF

# 1. Verify Time-Slicing multiplier in the sharing ConfigMap manifest
cat sharing/time-slicing-config.yaml | grep -E "replicas:.*4"
 
# 2. Verify DKMS enabler in the ClusterPolicy manifest
cat operator/cluster-policy.yaml | grep -E "ENABLE_DKMS.*true"
 
# 3. Verify HostDevice plugin type in the CNI configuration manifest
cat networking/ib-sriov-cni.conflist | grep -E "type.*host-device"
 
# 4. Verify InfiniBand Virtual Function device name in the CNI manifest
cat networking/ib-sriov-cni.conflist | grep -E "device.*ib0"
 
# 5. Verify XID hardware fault PromQL math in the DCGM alerting manifest
cat monitoring/dcgm-alerts.yaml | grep -E "DCGM_FI_DEV_XID_ERRORS.*>.*0"

          replicas: 4
      - name: ENABLE_DKMS
        value: "true"
      "type": "host-device",
      "device": "ib0",
      expr: DCGM_FI_DEV_XID_ERRORS > 0

# Safely remove the enterprise AI infrastructure lab directory
rm -rf ~/enterprise-ai-lab
 
# Verify the directory was removed successfully
ls -la ~/enterprise-ai-lab 2>/dev/null || echo "Cleanup complete. Directory successfully removed."