feat(server): add credential storage drivers for Provider v2

.github/workflows/branch-e2e.yml

@@ -26,6 +26,7 @@ jobs:
       run_core_e2e: ${{ steps.labels.outputs.run_core_e2e }}
       run_gpu_e2e: ${{ steps.labels.outputs.run_gpu_e2e }}
       run_kubernetes_ha_e2e: ${{ steps.labels.outputs.run_kubernetes_ha_e2e }}
+      run_kubernetes_credential_drivers_e2e: ${{ steps.labels.outputs.run_kubernetes_credential_drivers_e2e }}
       run_any_e2e: ${{ steps.labels.outputs.run_any_e2e }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -44,6 +45,7 @@ jobs:
               run_core_e2e="$(jq -r 'index("test:e2e") != null' <<< "$LABELS_JSON")"
               run_gpu_e2e="$(jq -r 'index("test:e2e-gpu") != null' <<< "$LABELS_JSON")"
               run_kubernetes_ha_e2e="$(jq -r 'index("test:e2e-kubernetes") != null' <<< "$LABELS_JSON")"
+              run_kubernetes_credential_drivers_e2e="$(jq -r 'index("test:e2e-kubernetes") != null' <<< "$LABELS_JSON")"
               ;;
             merge_group)
               # Merge groups have no PR labels. When GPU E2E is required as documented
@@ -52,14 +54,16 @@ jobs:
               run_core_e2e=true
               run_gpu_e2e=true
               run_kubernetes_ha_e2e=false
+              run_kubernetes_credential_drivers_e2e=true
               ;;
             *)
               run_core_e2e=true
               run_gpu_e2e=true
               run_kubernetes_ha_e2e=true
+              run_kubernetes_credential_drivers_e2e=true
               ;;
           esac
-          if [ "$run_core_e2e" = "true" ] || [ "$run_gpu_e2e" = "true" ] || [ "$run_kubernetes_ha_e2e" = "true" ]; then
+          if [ "$run_core_e2e" = "true" ] || [ "$run_gpu_e2e" = "true" ] || [ "$run_kubernetes_ha_e2e" = "true" ] || [ "$run_kubernetes_credential_drivers_e2e" = "true" ]; then
             run_any_e2e=true
           else
             run_any_e2e=false
@@ -68,6 +72,7 @@ jobs:
             echo "run_core_e2e=$run_core_e2e"
             echo "run_gpu_e2e=$run_gpu_e2e"
             echo "run_kubernetes_ha_e2e=$run_kubernetes_ha_e2e"
+            echo "run_kubernetes_credential_drivers_e2e=$run_kubernetes_credential_drivers_e2e"
             echo "run_any_e2e=$run_any_e2e"
           } >> "$GITHUB_OUTPUT"
 
@@ -192,6 +197,20 @@ jobs:
       external-postgres-secret: openshell-ha-pg
       cli-artifact-prefix: rust-binary-cli
 
+  kubernetes-credential-drivers-e2e:
+    needs: [pr_metadata, build-gateway, build-supervisor, build-cli]
+    if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true'
+    permissions:
+      actions: read
+      contents: read
+      packages: read
+    uses: ./.github/workflows/e2e-kubernetes-test.yml
+    with:
+      image-tag: ${{ github.sha }}
+      job-name: Kubernetes Credential Drivers E2E
+      e2e-task: e2e:kubernetes:credential-drivers
+      cli-artifact-prefix: rust-binary-cli
+
   core-e2e-result:
     name: Core E2E result
     needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e]
@@ -254,6 +273,33 @@ jobs:
           done
           exit "$failed"
 
+  kubernetes-credential-drivers-e2e-result:
+    name: Kubernetes Credential Drivers E2E result
+    needs: [pr_metadata, build-gateway, build-supervisor, kubernetes-credential-drivers-e2e]
+    if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Verify Kubernetes credential drivers E2E jobs
+        env:
+          BUILD_GATEWAY_RESULT: ${{ needs.build-gateway.result }}
+          BUILD_SUPERVISOR_RESULT: ${{ needs.build-supervisor.result }}
+          KUBERNETES_CREDENTIAL_DRIVERS_E2E_RESULT: ${{ needs.kubernetes-credential-drivers-e2e.result }}
+        run: |
+          set -euo pipefail
+          failed=0
+          for item in \
+            "build-gateway:$BUILD_GATEWAY_RESULT" \
+            "build-supervisor:$BUILD_SUPERVISOR_RESULT" \
+            "kubernetes-credential-drivers-e2e:$KUBERNETES_CREDENTIAL_DRIVERS_E2E_RESULT"; do
+            name="${item%%:*}"
+            result="${item#*:}"
+            if [ "$result" != "success" ]; then
+              echo "::error::$name concluded $result"
+              failed=1
+            fi
+          done
+          exit "$failed"
+
   kubernetes-ha-e2e-result:
     name: Kubernetes HA E2E result
     needs: [pr_metadata, build-gateway, build-supervisor, build-cli, kubernetes-ha-e2e]

.github/workflows/e2e-kubernetes-test.yml

@@ -37,6 +37,11 @@ on:
         required: false
         type: string
         default: "v0.5.0"
+      e2e-task:
+        description: "mise task to run for the Kubernetes e2e job"
+        required: false
+        type: string
+        default: "e2e:kubernetes"
       mise-version:
         description: "mise version to install on the bare Kubernetes e2e runner"
         required: false
@@ -130,12 +135,13 @@ jobs:
             kind load image-archive "$archive" --name "$KIND_CLUSTER_NAME"
           done
 
-      - name: Run Kubernetes E2E (Rust smoke)
+      - name: Run Kubernetes E2E
         env:
           AGENT_SANDBOX_VERSION: ${{ inputs.agent-sandbox-version }}
           OPENSHELL_E2E_KUBE_CONTEXT: kind-${{ env.KIND_CLUSTER_NAME }}
           OPENSHELL_E2E_KUBE_EXTRA_VALUES: ${{ inputs.extra-helm-values }}
           OPENSHELL_E2E_KUBE_EXTERNAL_POSTGRES_SECRET: ${{ inputs.external-postgres-secret }}
           IMAGE_TAG: ${{ inputs.image-tag }}
           OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell
-        run: mise run --no-deps --skip-deps e2e:kubernetes
+          E2E_TASK: ${{ inputs.e2e-task }}
+        run: mise run --no-deps --skip-deps "$E2E_TASK"

.github/workflows/e2e-label-help.yml

@@ -51,7 +51,7 @@ jobs:
               status_summary="The matching required CI gate status on this PR will flip green automatically once the run finishes."
               ;;
             test:e2e-kubernetes)
-              suite_summary="Kubernetes HA E2E"
+              suite_summary="Kubernetes HA and credential-driver E2E"
               build_summary="gateway and supervisor images"
               status_summary="This is an optional proof-of-life suite; failures are visible in the workflow run but do not publish a required CI gate status."
               ;;

AGENTS.md

@@ -41,6 +41,9 @@ These pipelines connect skills into end-to-end workflows. Individual skill files
 | `crates/openshell-sdk/` | Shared client SDK | Async Rust gateway client (gRPC transport, TLS, OIDC refresh, edge tunnel); consumed by CLI, TUI, and `@openshell/sdk` |
 | `crates/openshell-providers/` | Provider management | Credential provider backends |
 | `crates/openshell-tui/` | Terminal UI | Ratatui-based dashboard for monitoring |
+| `crates/openshell-driver-db-credstore/` | Default encrypted DB credential store | In-process `CredentialDriver` backend for AES-256-GCM encrypted database credential storage |
+| `crates/openshell-driver-kubernetes-secrets/` | Kubernetes Secrets credential driver | In-process `CredentialDriver` backend for OpenShell-managed K8s Secret storage |
+| `crates/openshell-driver-vault/` | Vault credential driver | In-process `CredentialDriver` backend for Vault-compatible KV storage |
 | `crates/openshell-driver-kubernetes/` | Kubernetes compute driver | In-process `ComputeDriver` backend for K8s sandbox pods |
 | `crates/openshell-driver-docker/` | Docker compute driver | In-process `ComputeDriver` backend for local Docker sandbox containers |
 | `crates/openshell-driver-podman/` | Podman compute driver | In-process `ComputeDriver` backend for local Podman sandbox containers |

CI.md

@@ -18,10 +18,11 @@ Three opt-in labels enable the long-running E2E suites:
   suites in `Branch E2E Checks`
 - `test:e2e-gpu` runs GPU E2E in `Branch E2E Checks`
 - `test:e2e-kubernetes` runs Kubernetes E2E with the HA Helm overlay
-  (`replicaCount: 2` and bundled PostgreSQL) in `Branch E2E Checks`
+  (`replicaCount: 2` and bundled PostgreSQL) and the credential-driver suite
+  (Kubernetes Secrets plus Vault) in `Branch E2E Checks`
 
 When multiple labels are present, `Branch E2E Checks` builds the shared gateway and supervisor images once, builds one CLI artifact per runner architecture, builds the Linux VM driver artifact once, and fans out all enabled suites in parallel. Docker, Podman, GPU, Rust, Python, MCP, and VM E2E jobs reuse the matching prebuilt gateway and CLI binaries instead of compiling additional debug binaries in each job; Kubernetes E2E consumes the gateway image directly and reuses the prebuilt CLI. VM E2E also reuses the prebuilt VM driver artifact and falls back to local VM-driver/runtime preparation for local runs or workflow invocations that omit the artifact.
-The `OpenShell / E2E` and `OpenShell / GPU E2E` required statuses are evaluated from separate suite result jobs inside that workflow. `test:e2e-kubernetes` is optional while HA behavior is under active iteration: failures are visible in the workflow run but do not publish a required CI gate status.
+The `OpenShell / E2E` and `OpenShell / GPU E2E` required statuses are evaluated from separate suite result jobs inside that workflow. `test:e2e-kubernetes` is optional while Kubernetes HA and credential-driver behavior are under active iteration: failures are visible in the workflow run but do not publish a required CI gate status.
 
 The GitHub ruleset should require the `OpenShell / ...` statuses published by `Required CI Gates`, not the push-triggered workflow jobs directly.
 
@@ -135,7 +136,7 @@ The bot's full administrator documentation is internal to NVIDIA. The only comma
 | File | Role |
 |---|---|
 | `.github/workflows/branch-checks.yml` | Required non-E2E checks. Triggers on `push: pull-request/[0-9]+` for PR mirrors and `merge_group` for queued merges. |
-| `.github/workflows/branch-e2e.yml` | Standard, GPU, and Kubernetes HA E2E. PR mirror pushes use `test:e2e`, `test:e2e-gpu`, and `test:e2e-kubernetes` labels; merge groups run core and GPU E2E. |
+| `.github/workflows/branch-e2e.yml` | Standard, GPU, Kubernetes HA, and Kubernetes credential-driver E2E. PR mirror pushes use `test:e2e`, `test:e2e-gpu`, and `test:e2e-kubernetes` labels; merge groups run core and GPU E2E. |
 | `.github/workflows/helm-lint.yml` | Helm chart validation. PR mirror pushes skip lint jobs unless Helm inputs changed; merge groups always validate Helm because they represent the final integration state. |
 | `.github/actions/pr-gate/action.yml` | Composite action that resolves PR metadata and verifies the required label is set for PR mirror pushes. Non-push events are allowed through. |
 | `.github/actions/pr-merge-base/action.yml` | Composite action that resolves and fetches the merge-base commit for `pull-request/<N>` push workflows. |

Cargo.toml

@@ -91,6 +91,7 @@ tokio-tungstenite = { version = "0.26", features = ["rustls-tls-native-roots"] }
 base64 = "0.22"
 
 # Crypto / Auth
+ring = "0.17"
 sha2 = "0.10"
 rand = "0.9"
 jsonwebtoken = "9"

TESTING.md

@@ -150,6 +150,7 @@ Suites:
 - Common suite (`--features e2e`) - driver-neutral CLI behavior, sandbox lifecycle, sync, port forwarding, policy, and provider tests.
 - Docker suite (`--features e2e-docker`) - common suite plus Docker-only coverage such as Dockerfile image builds, Docker preflight checks, and managed Docker gateway resume.
 - Docker GPU suite (`--features e2e-docker-gpu`) - Docker suite plus GPU sandbox smoke coverage.
+- Kubernetes credential-driver suite (`--features e2e-kubernetes-credential-drivers`) - targeted Kubernetes Secrets and Vault provider credential storage coverage.
 - VM suite (`--features e2e-vm`) - runs e2e tests on a VM.
 
 GPU device-selection tests compare OpenShell sandboxes against a plain Docker or
@@ -180,6 +181,14 @@ Run the VM-backed Rust CLI e2e suite:
 mise run e2e:vm
 ```
 
+Run the targeted Kubernetes credential-driver e2e suite. This deploys an
+OpenBao fixture for the Vault-compatible driver path and validates Kubernetes
+Secrets and Vault storage backends one at a time:
+
+```shell
+mise run e2e:kubernetes:credential-drivers
+```
+
 Run a single test directly with cargo:
 
 ```shell
@@ -210,3 +219,4 @@ The harness (`e2e/rust/src/harness/`) provides:
 | `OPENSHELL_GATEWAY` | Override active gateway name for E2E tests |
 | `OPENSHELL_GATEWAY_ENDPOINT` | Run E2E tests against an existing plaintext HTTP gateway endpoint |
 | `OPENSHELL_E2E_DRIVER` | Driver name exported by the e2e gateway wrapper (`docker`, `podman`, or `vm`) |
+| `OPENSHELL_E2E_CREDENTIAL_DRIVERS` | Enables the Kubernetes credential-driver fixture path in `e2e/with-kube-gateway.sh` |

architecture/gateway.md

@@ -272,9 +272,14 @@ default WAL journal mode), which mirror the same sensitive contents.
 Persisted state includes sandboxes, providers, provider credential refresh
 state, SSH sessions, policy revisions, settings, inference configuration, and
 deployment records. Provider refresh material is stored as a separate object
-scoped to the provider instance through `objects.scope`; the provider record
-keeps only the current injectable credential values and optional per-credential
-expiry timestamps. A refresh normally mints one credential, but a strategy may
+scoped to the provider instance through `objects.scope`. Provider records keep
+inline credential values only for legacy records created before credential
+driver storage. New provider writes keep driver-owned credential handles and
+optional per-credential expiry timestamps. When no external credential driver
+is configured, gateways use server-owned encrypted database credential storage
+for defense in depth. Multi-replica deployments can use that default with a
+shared database and shared key-encryption key, or opt into an external backend such as Vault
+or Kubernetes Secrets. A refresh normally mints one credential, but a strategy may
 co-mint several (AWS STS mints the access key, secret key, and session token in
 one call); the refresh state pins the resolved set of env keys it owns so
 collision checks reserve all of them before the first mint.