Integration with SandD

api/v1alpha1/nodepool_types.go

@@ -136,9 +136,9 @@ const (
 type FailoverPolicy struct {
 	// BlocklistTTL is the BASE duration a failed placement is excluded before the
 	// provider becomes a candidate for it again. The controller adds a random jitter
-	// (up to a minute) on top so Pods that failed for the same reason do not all
-	// retry the just-freed candidate in lockstep, so the effective exclusion is this
-	// value plus that jitter.
+	// (up to 30s) on top so Pods that failed for the same reason do not all retry the
+	// just-freed candidate in lockstep, so the effective exclusion is this value plus
+	// that jitter.
 	// +kubebuilder:default="30s"
 	BlocklistTTL metav1.Duration `json:"blocklistTTL,omitempty"`
 }

cmd/main.go

@@ -351,7 +351,18 @@ func registerProviders(ctx context.Context, c client.Client) {
 	// delivered via a Secret), and one account-global credential authorizes every
 	// region. Registration only fails (and is a non-fatal skip) if the price catalog
 	// cannot load — region config can no longer make it fail.
-	if p, err := awsprovider.NewSDKClient(ctx, awsRegionSource(c)); err != nil {
+	// SandD daemon (opt-in): if SANDD_TUNNEL_AUTHKEY is set, every AWS workload runs
+	// the SandD daemon inside its container in tunnel mode, so commands and
+	// interactive shells can be run in the user's own environment over the mesh with
+	// no inbound access. Absent the authkey the zero SanddConfig injects nothing, so
+	// this is off by default. The authkey is a secret (delivered via a mounted
+	// Secret) and is NEVER logged.
+	sanddCfg := provider.SanddConfig{
+		AuthKey:       os.Getenv("SANDD_TUNNEL_AUTHKEY"),
+		ControlServer: os.Getenv("SANDD_TUNNEL_SERVER"),
+		ServerURL:     os.Getenv("SANDD_SERVER_URL"),
+	}
+	if p, err := awsprovider.NewSDKClient(ctx, awsRegionSource(c), sanddCfg); err != nil {
 		setupLog.Info("skipping AWS provider registration", "reason", err.Error())
 	} else {
 		provider.Register(p)

config/crd/bases/nebula.inftyai.com_nodepools.yaml

@@ -102,9 +102,9 @@ spec:
                     description: |-
                       BlocklistTTL is the BASE duration a failed placement is excluded before the
                       provider becomes a candidate for it again. The controller adds a random jitter
-                      (up to a minute) on top so Pods that failed for the same reason do not all
-                      retry the just-freed candidate in lockstep, so the effective exclusion is this
-                      value plus that jitter.
+                      (up to 30s) on top so Pods that failed for the same reason do not all retry the
+                      just-freed candidate in lockstep, so the effective exclusion is this value plus
+                      that jitter.
                     type: string
                 type: object
               providers:

config/samples/deployment.yaml

@@ -28,7 +28,7 @@ metadata:
   labels:
     app.kubernetes.io/managed-by: nebula
 spec:
-  replicas: 8
+  replicas: 2
   selector:
     matchLabels:
       app: gpu-workload-sample
@@ -38,7 +38,7 @@ spec:
         app: gpu-workload-sample
         nebula.inftyai.com/enabled: "true"
         nebula.inftyai.com/nodepool: sample
-        nebula.inftyai.com/accelerator-type: a100-40gb
+        nebula.inftyai.com/accelerator-type: t4
     spec:
       # Do NOT set nodeName or a provider nodeSelector yourself — the placement
       # controller fills the nodeSelector in when it ungates the Pod. Setting
@@ -65,4 +65,4 @@ spec:
             # GPU count. Standard extended resource, so the scheduler's fit check
             # and provisioning read the same number. 8 => 8x the accelerator-type
             # above.
-            nvidia.com/gpu: "8"
+            nvidia.com/gpu: "1"

config/samples/nebula_v1alpha1_nodepool.yaml

@@ -26,4 +26,4 @@ spec:
   # Inner axis: within the active capacity tier.
   strategy: Ordered
   failover:
-    blocklistTTL: 3m
+    blocklistTTL: 30s

pkg/provider/aws/aws.go

@@ -142,6 +142,11 @@ type InstanceSpec struct {
 	Region string
 	// Tags carry Nebula identity; ClaimTagKey holds the NodeClaim name.
 	Tags map[string]string
+	// Sandd, when Enabled, makes the workload's docker run start the SandD daemon
+	// INSIDE the container (via an entrypoint shim) so commands and interactive
+	// shells run in the user's own env/cwd/code over the tunnel. A zero value injects
+	// nothing. See buildUserData/writeSanddEntrypoint and provider.SanddConfig.
+	Sandd provider.SanddConfig
 }
 
 // EC2Instance is the adapter-level view of one EC2 instance as observed.
@@ -211,6 +216,9 @@ type Provider struct {
 	// regionSource reports the NodePool-declared regions to sweep in List/Offerings.
 	// May be nil in tests, in which case sweepRegions uses only the cache keys.
 	regionSource RegionSource
+	// sandd is the optional SandD daemon config stamped onto every InstanceSpec so
+	// buildUserData can bootstrap it. Zero value => disabled (no injection).
+	sandd provider.SanddConfig
 
 	mu      sync.Mutex
 	clients map[string]Client // region -> Client, populated lazily by clientFor
@@ -225,11 +233,16 @@ type Provider struct {
 // (admission requires each aws pool to list ≥1 region, and placement stamps it onto
 // the ProvisionRequest), and observed instances report their region from the
 // region-pinned client — so nothing needs a fallback, and no AWS_REGION env is read.
-func New(newClient ClientFactory, cat catalog.Lookup, regionSource RegionSource) *Provider {
+//
+// sandd is the optional SandD daemon config baked into every instance's user-data;
+// its zero value disables injection, so tests and the non-SandD path pass
+// provider.SanddConfig{}.
+func New(newClient ClientFactory, cat catalog.Lookup, regionSource RegionSource, sandd provider.SanddConfig) *Provider {
 	return &Provider{
 		Base:         catalog.Base{ProviderName: provider.ProviderAWS, Catalog: cat},
 		newClient:    newClient,
 		regionSource: regionSource,
+		sandd:        sandd,
 		clients:      make(map[string]Client),
 	}
 }
@@ -244,6 +257,7 @@ func newSingleRegion(client Client, cat catalog.Lookup, region string) *Provider
 		func(context.Context, string) (Client, error) { return client, nil },
 		cat,
 		func() []string { return []string{region} },
+		provider.SanddConfig{}, // single-region test convenience: no SandD injection
 	)
 	// Pre-seed the cache so even a stray region lookup returns the fake rather than
 	// invoking the (constant) factory.
@@ -714,6 +728,9 @@ func (p *Provider) instanceSpecFromPod(pod *corev1.Pod, req provider.ProvisionRe
 		Spot:          req.CapacityType == nebulav1alpha1.CapacitySpot,
 		Region:        req.Region,
 		Tags:          map[string]string{ClaimTagKey: req.ClaimName},
+		// SandD daemon (opt-in): keyed by the claim name so a daemon that dials home
+		// correlates 1:1 with this instance's NodeClaim. Zero value => no injection.
+		Sandd: p.sandd,
 	}, nil
 }
 

pkg/provider/aws/client.go

@@ -30,6 +30,7 @@ import (
 	smithy "github.com/aws/smithy-go"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
+	"github.com/InftyAI/Nebula/pkg/provider"
 	"github.com/InftyAI/Nebula/pkg/provider/catalog"
 )
 
@@ -204,7 +205,14 @@ var _ Client = (*sdkClient)(nil)
 //
 // The catalog is loaded via catalog.Load() (embedded CSV / mounted ConfigMap),
 // identical to the other adapters.
-func NewSDKClient(_ context.Context, regionSource RegionSource) (*Provider, error) {
+//
+// sandd is the OPTIONAL SandD daemon config: its zero value (empty AuthKey) leaves
+// the bootstrap untouched, so passing provider.SanddConfig{} is a no-op. When set,
+// every workload this provider launches runs the SandD daemon inside its container
+// in tunnel mode (see buildUserData), the workload's command-execution/shell
+// channel. Unlike credentials, the auth key IS accepted here — it is delivered to
+// the controller as a secret and stamped into the (base64) user-data.
+func NewSDKClient(_ context.Context, regionSource RegionSource, sandd provider.SanddConfig) (*Provider, error) {
 	cat, err := catalog.Load()
 	if err != nil {
 		return nil, fmt.Errorf("aws: load price catalog: %w", err)
@@ -213,7 +221,7 @@ func NewSDKClient(_ context.Context, regionSource RegionSource) (*Provider, erro
 	factory := func(ctx context.Context, region string) (Client, error) {
 		return newSDKClientForRegion(ctx, region)
 	}
-	return New(factory, cat, regionSource), nil
+	return New(factory, cat, regionSource, sandd), nil
 }
 
 // newSDKClientForRegion builds one region-pinned sdkClient: it loads SDK config for

pkg/provider/aws/client_test.go

@@ -347,6 +347,110 @@ func TestBuildUserData_QuotesHostileValues(t *testing.T) {
 	}
 }
 
+// TestBuildUserData_SanddDisabled: the zero SanddConfig is the default, so a spec
+// that does not opt in must render EXACTLY the plain bootstrap — no sandd/tunnel
+// tokens leak in, so existing clusters are unaffected.
+func TestBuildUserData_SanddDisabled(t *testing.T) {
+	encoded, err := buildUserData(InstanceSpec{Image: "img"})
+	if err != nil {
+		t.Fatalf("buildUserData: %v", err)
+	}
+	raw, _ := base64.StdEncoding.DecodeString(encoded)
+	script := string(raw)
+	if strings.Contains(script, "sandd") || strings.Contains(script, "tunnel") {
+		t.Fatalf("sandd disabled but bootstrap injected it:\n%s", script)
+	}
+}
+
+// TestBuildUserData_SanddEnabled: when opted in, sandd runs INSIDE the container via
+// an entrypoint shim so its shells see the user's env/cwd/code. The shim must
+// override the image ENTRYPOINT with /bin/sh, carry daemon config as container env
+// (incl. the claim-derived DAEMON_ID), start the daemon fail-open + backgrounded,
+// and exec the user's effective command so the workload becomes PID 1.
+func TestBuildUserData_SanddEnabled(t *testing.T) {
+	spec := InstanceSpec{
+		Image:   "img",
+		Command: []string{"python", "train.py"},
+		Args:    []string{"--epochs", "3"},
+		Tags:    map[string]string{ClaimTagKey: "claim-abc"},
+		Sandd: provider.SanddConfig{
+			AuthKey:       "tskey-secret",
+			ControlServer: "http://headscale:8080",
+			ServerURL:     "ws://100.64.0.1:8765/ws",
+		},
+	}
+	encoded, err := buildUserData(spec)
+	if err != nil {
+		t.Fatalf("buildUserData: %v", err)
+	}
+	raw, _ := base64.StdEncoding.DecodeString(encoded)
+	script := string(raw)
+
+	// The shim commandeers the container ENTRYPOINT as /bin/sh.
+	if !strings.Contains(script, "--entrypoint '/bin/sh'") {
+		t.Fatalf("sandd shim must override the image entrypoint with /bin/sh:\n%s", script)
+	}
+
+	// Daemon config is delivered as container env (so a process INSIDE the container
+	// sees it), including the claim-derived DAEMON_ID and the tunnel key/server.
+	for _, want := range []string{
+		"-e 'DAEMON_ID=claim-abc'",
+		"-e 'SERVER_URL=ws://100.64.0.1:8765/ws'",
+		"-e 'SANDD_TUNNEL_AUTHKEY=tskey-secret'",
+		"-e 'SANDD_TUNNEL_SERVER=http://headscale:8080'",
+		"-e 'SANDD_BINARY_URL=" + sanddBinaryURL + "'",
+	} {
+		if !strings.Contains(script, want) {
+			t.Fatalf("sandd shim missing container env %q:\n%s", want, script)
+		}
+	}
+
+	// Fail-open + backgrounded: the daemon bring-up is a `( set +e ... ) || true`
+	// subshell ending in `&`, so a failed fetch/start cannot stop the workload.
+	if !strings.Contains(script, "set +e") || !strings.Contains(script, ") || true") {
+		t.Fatalf("sandd bring-up must be fail-open (set +e + || true):\n%s", script)
+	}
+	if !strings.Contains(script, "&\n") {
+		t.Fatalf("sandd daemon must be backgrounded:\n%s", script)
+	}
+
+	// The shim execs the user's effective command so the workload is PID 1, and that
+	// command (image then argv) is rendered after the shim payload in kubelet order:
+	// Command[0]+Command[1:] then Args.
+	if !strings.Contains(script, `exec "$@"`) {
+		t.Fatalf("shim must exec the user command as PID 1:\n%s", script)
+	}
+	if !strings.Contains(script, "'img' '-c'") {
+		t.Fatalf("shim payload must follow the image on the run line:\n%s", script)
+	}
+	if !strings.Contains(script, "'sandd-shim' 'python' 'train.py' '--epochs' '3'\n") {
+		t.Fatalf("user effective command not rendered after the shim in order:\n%s", script)
+	}
+}
+
+// TestBuildUserData_SanddQuotesHostileValues: sandd config values are shell-quoted
+// like every other injected value, so a hostile auth key/URL cannot break out.
+func TestBuildUserData_SanddQuotesHostileValues(t *testing.T) {
+	spec := InstanceSpec{
+		Image: "img",
+		Tags:  map[string]string{ClaimTagKey: "c"},
+		Sandd: provider.SanddConfig{
+			AuthKey:       "k'; rm -rf /; echo '",
+			ControlServer: "http://h:8080",
+			ServerURL:     "ws://s:8765/ws",
+		},
+	}
+	encoded, err := buildUserData(spec)
+	if err != nil {
+		t.Fatalf("buildUserData: %v", err)
+	}
+	raw, _ := base64.StdEncoding.DecodeString(encoded)
+	script := string(raw)
+	if strings.Contains(script, "\nrm -rf /") {
+		t.Fatalf("hostile sandd value escaped quoting:\n%s", script)
+	}
+}
+
 func TestClassifyEC2Error(t *testing.T) {
 	tests := []struct {
 		name      string

pkg/provider/aws/translate.go

@@ -52,6 +52,7 @@ func buildUserData(spec InstanceSpec) (string, error) {
 	var b strings.Builder
 	b.WriteString("#!/bin/bash\n")
 	b.WriteString("set -euo pipefail\n")
+
 	// Pull first so an image error surfaces before we try to run.
 	fmt.Fprintf(&b, "docker pull %s\n", shellQuote(spec.Image))
 
@@ -70,14 +71,27 @@ func buildUserData(spec InstanceSpec) (string, error) {
 	for _, k := range sortedKeys(spec.Env) {
 		fmt.Fprintf(&b, " -e %s", shellQuote(k+"="+spec.Env[k]))
 	}
-	// runArgs are everything that follows the image: the entrypoint's own arguments
-	// (Command[1:]) then the CMD arguments (Args), in that order.
+
+	// SandD daemon (opt-in): run it INSIDE the container so its shells/commands see
+	// the user's own env, cwd, filesystem and code (a host-side daemon would only see
+	// the host). We cannot assume the user's image has sandd, so a shell entrypoint
+	// shim fetches the static musl binary at boot, starts it backgrounded, then execs
+	// the user's real program — see writeSanddEntrypoint. This overrides the image's
+	// ENTRYPOINT with the shim, so the shim (not Docker) is responsible for launching
+	// the workload with the right Command/Args; runArgs is left empty in that case.
 	var runArgs []string
-	if len(spec.Command) > 0 {
-		fmt.Fprintf(&b, " --entrypoint %s", shellQuote(spec.Command[0]))
-		runArgs = append(runArgs, spec.Command[1:]...)
+	if spec.Sandd.Enabled() {
+		runArgs = writeSanddEntrypoint(&b, spec)
+	} else {
+		// No shim: map Command/Args straight onto Docker's --entrypoint/CMD as before.
+		// runArgs are everything that follows the image: the entrypoint's own arguments
+		// (Command[1:]) then the CMD arguments (Args), in that order.
+		if len(spec.Command) > 0 {
+			fmt.Fprintf(&b, " --entrypoint %s", shellQuote(spec.Command[0]))
+			runArgs = append(runArgs, spec.Command[1:]...)
+		}
+		runArgs = append(runArgs, spec.Args...)
 	}
-	runArgs = append(runArgs, spec.Args...)
 
 	b.WriteString(" " + shellQuote(spec.Image))
 	for _, arg := range runArgs {
@@ -88,6 +102,77 @@ func buildUserData(spec InstanceSpec) (string, error) {
 	return base64.StdEncoding.EncodeToString([]byte(b.String())), nil
 }
 
+// sanddBinaryURL is the statically-linked (musl) SandD daemon release asset. Being
+// static, this one binary runs in any container image regardless of its libc, so the
+// shim can fetch it into an arbitrary user image at boot. It is pinned to the same
+// asset name install.sh resolves; amd64 matches the x86_64 GPU instance types.
+const sanddBinaryURL = "https://github.com/InftyAI/SandD/releases/latest/download/sandd-linux-amd64"
+
+// sanddShimScript is the container ENTRYPOINT shim (run as `/bin/sh -c <script>`).
+// It fetches the static sandd binary, starts it backgrounded in tunnel mode, then
+// execs the user's real program so the daemon shares the WORKLOAD's namespace — its
+// shells and commands see the user's env, cwd, filesystem and code.
+//
+// Load-bearing properties:
+//   - FAIL-OPEN: the whole daemon bring-up is a `( set +e ; ... ) || true` subshell,
+//     so a failed fetch/start can never stop the workload from launching.
+//   - BACKGROUNDED: sandd is a long-lived process started with `&`; the shim then
+//     `exec "$@"` REPLACES the shell with the user's program, so the workload is PID 1
+//     (signals/exit codes behave as without the shim) and the shell adds no overhead.
+//
+// Config arrives as container env (SERVER_URL/DAEMON_ID are read by sandd natively;
+// the tunnel key/server are referenced here as flags) so no secret is embedded in
+// this script literal. Requires /bin/sh and curl-or-wget in the image (documented in
+// writeSanddEntrypoint); distroless/scratch images have neither and are unsupported.
+const sanddShimScript = `( set +e
+  BIN=/tmp/.sandd
+  curl -fsSL "$SANDD_BINARY_URL" -o "$BIN" 2>/dev/null || wget -qO "$BIN" "$SANDD_BINARY_URL" 2>/dev/null
+  chmod +x "$BIN" 2>/dev/null
+  "$BIN" --tunnel --tunnel-authkey "$SANDD_TUNNEL_AUTHKEY" --tunnel-server "$SANDD_TUNNEL_SERVER" >/tmp/sandd.log 2>&1 &
+) || true
+exec "$@"`
+
+// writeSanddEntrypoint appends the in-container SandD shim to a `docker run` line: it
+// emits the `-e` env carrying daemon config, overrides the image ENTRYPOINT with
+// `/bin/sh` running sanddShimScript, and returns the post-image argv (the shim's
+// `-c` payload plus the user's effective command) for buildUserData to render.
+//
+// Because the shim commandeers --entrypoint, the image's OWN ENTRYPOINT is bypassed:
+// the shim can only relaunch the workload from what Nebula can see, i.e. the Pod's
+// Command+Args. A Pod that sets neither (relying on the image's baked-in ENTRYPOINT)
+// cannot be faithfully reconstructed here — a known limitation of the in-container
+// model; such Pods should set an explicit command.
+//
+// The auth key is delivered as container env, so it is visible to the workload
+// itself (the tenant owns this container). That is inherent to running the daemon
+// INSIDE the container — the container must hold the credential to dial home — and is
+// acceptable only because the daemon is single-tenant (one per container).
+func writeSanddEntrypoint(b *strings.Builder, spec InstanceSpec) []string {
+	cfg := spec.Sandd
+	// Daemon config as env: SERVER_URL/DAEMON_ID are the names sandd reads natively;
+	// the rest the shim references. Rendered via the same shellQuote path as user env
+	// so hostile values cannot break out. (Not sorted with user env: these are fixed.)
+	for _, kv := range [][2]string{
+		{"SERVER_URL", cfg.ServerURL},
+		{"DAEMON_ID", spec.Tags[ClaimTagKey]},
+		{"SANDD_TUNNEL_AUTHKEY", cfg.AuthKey},
+		{"SANDD_TUNNEL_SERVER", cfg.ControlServer},
+		{"SANDD_BINARY_URL", sanddBinaryURL},
+	} {
+		fmt.Fprintf(b, " -e %s", shellQuote(kv[0]+"="+kv[1]))
+	}
+	fmt.Fprintf(b, " --entrypoint %s", shellQuote("/bin/sh"))
+
+	// `sh -c <script> <arg0> <argv...>`: arg0 ("sandd-shim") becomes $0, and the
+	// user's effective argv fills $@, which the shim `exec "$@"`s. Command[0] is the
+	// program and Command[1:]+Args its arguments — the same effective argv the kubelet
+	// would run, just launched by the shim instead of Docker's --entrypoint/CMD.
+	runArgs := []string{"-c", sanddShimScript, "sandd-shim"}
+	runArgs = append(runArgs, spec.Command...)
+	runArgs = append(runArgs, spec.Args...)
+	return runArgs
+}
+
 // sortedKeys returns m's keys in sorted order, for deterministic rendering.
 func sortedKeys(m map[string]string) []string {
 	keys := make([]string, 0, len(m))