apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: nvidia-device-plugin-daemonset
  namespace: kube-system
spec:
  selector:
    matchLabels:
      name: nvidia-device-plugin-ds
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        name: nvidia-device-plugin-ds
    spec:
      tolerations:
        - key: nvidia.com/gpu
          operator: Exists
          effect: NoSchedule
      priorityClassName: system-node-critical
      containers:
        - name: nvidia-device-plugin-ctr
          image: nvcr.io/nvidia/k8s-device-plugin:v0.17.0
          env:
            - name: FAIL_ON_INIT_ERROR
              value: "false"
          securityContext:
            allowPrivilegeEscalation: false
            capabilities:
              drop: ["ALL"]
          volumeMounts:
            - name: device-plugin
              mountPath: /var/lib/kubelet/device-plugins
      volumes:
        - name: device-plugin
          hostPath:
            path: /var/lib/kubelet/device-plugins

# Oder direkt via kubectl:
kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/v0.17.0/deployments/static/nvidia-device-plugin.yml

# Pruefen, ob GPUs erkannt werden:
kubectl get nodes -o json | jq '.items[] | {name: .metadata.name, gpus: .status.allocatable["nvidia.com/gpu"]}'

# Helm Repository hinzufuegen
helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
helm repo update

# GPU Operator installieren
helm install gpu-operator nvidia/gpu-operator \
  --namespace gpu-operator \
  --create-namespace \
  --set driver.enabled=true \
  --set toolkit.enabled=true \
  --set devicePlugin.enabled=true \
  --set dcgmExporter.enabled=true

# Status pruefen
kubectl get pods -n gpu-operator

# Erwartete Pods:
# nvidia-driver-daemonset-*
# nvidia-container-toolkit-daemonset-*
# nvidia-device-plugin-daemonset-*
# nvidia-dcgm-exporter-*
# gpu-feature-discovery-*

apiVersion: v1
kind: Pod
metadata:
  name: gpu-test
spec:
  restartPolicy: Never
  containers:
    - name: cuda-test
      image: nvcr.io/nvidia/cuda:12.3.1-base-ubuntu22.04
      command: ["nvidia-smi"]
      resources:
        limits:
          nvidia.com/gpu: 1

kubectl apply -f gpu-test.yaml
kubectl logs gpu-test
# Sollte die GPU-Informationen anzeigen (Modell, Memory, Treiber-Version)

apiVersion: batch/v1
kind: Job
metadata:
  name: pytorch-training
spec:
  backoffLimit: 2
  template:
    spec:
      restartPolicy: OnFailure
      containers:
        - name: trainer
          image: nvcr.io/nvidia/pytorch:24.01-py3
          command:
            - python
            - -c
            - |
              import torch
              print(f"CUDA available: {torch.cuda.is_available()}")
              print(f"GPU: {torch.cuda.get_device_name(0)}")
              # Ihr Training-Script hier
          resources:
            limits:
              nvidia.com/gpu: 1
              memory: 16Gi
              cpu: "4"
            requests:
              memory: 8Gi
              cpu: "2"
          volumeMounts:
            - name: data
              mountPath: /data
            - name: models
              mountPath: /models
      volumes:
        - name: data
          persistentVolumeClaim:
            claimName: training-data
        - name: models
          persistentVolumeClaim:
            claimName: model-storage

# GPU-Nodes mit Taint versehen
kubectl taint nodes gpu-node-1 nvidia.com/gpu=present:NoSchedule
kubectl taint nodes gpu-node-2 nvidia.com/gpu=present:NoSchedule

# Labels setzen fuer Node Affinity
kubectl label nodes gpu-node-1 gpu-type=a100
kubectl label nodes gpu-node-2 gpu-type=t4

apiVersion: v1
kind: Pod
metadata:
  name: gpu-inference
spec:
  tolerations:
    - key: nvidia.com/gpu
      operator: Equal
      value: present
      effect: NoSchedule
  affinity:
    nodeAffinity:
      requiredDuringSchedulingIgnoredDuringExecution:
        nodeSelectorTerms:
          - matchExpressions:
              - key: gpu-type
                operator: In
                values:
                  - a100
  containers:
    - name: inference
      image: your-inference-image:latest
      resources:
        limits:
          nvidia.com/gpu: 1

apiVersion: v1
kind: ResourceQuota
metadata:
  name: gpu-quota-team-ml
  namespace: team-ml
spec:
  hard:
    requests.nvidia.com/gpu: "4"
    limits.nvidia.com/gpu: "4"
---
apiVersion: v1
kind: ResourceQuota
metadata:
  name: gpu-quota-team-cv
  namespace: team-cv
spec:
  hard:
    requests.nvidia.com/gpu: "2"
    limits.nvidia.com/gpu: "2"

helm install dcgm-exporter nvidia/dcgm-exporter \
  --namespace monitoring \
  --create-namespace \
  --set serviceMonitor.enabled=true

# GPU nicht erkannt? Drei Checks:
# 1. Device Plugin laeuft?
kubectl get pods -n kube-system -l name=nvidia-device-plugin-ds

# 2. Node hat GPU-Kapazitaet?
kubectl describe node <gpu-node> | grep -A5 "Allocatable"

# 3. Pod haengt in Pending?
kubectl describe pod <gpu-pod> | tail -20
# Haeufige Ursachen: Insufficient nvidia.com/gpu, fehlende Toleration, ResourceQuota