diff --git a/api/v1alpha1/nodepool_types.go b/api/v1alpha1/nodepool_types.go index 779de1b..a954007 100644 --- a/api/v1alpha1/nodepool_types.go +++ b/api/v1alpha1/nodepool_types.go @@ -136,9 +136,9 @@ const ( type FailoverPolicy struct { // BlocklistTTL is the BASE duration a failed placement is excluded before the // provider becomes a candidate for it again. The controller adds a random jitter - // (up to a minute) on top so Pods that failed for the same reason do not all - // retry the just-freed candidate in lockstep, so the effective exclusion is this - // value plus that jitter. + // (up to 30s) on top so Pods that failed for the same reason do not all retry the + // just-freed candidate in lockstep, so the effective exclusion is this value plus + // that jitter. // +kubebuilder:default="30s" BlocklistTTL metav1.Duration `json:"blocklistTTL,omitempty"` } diff --git a/cmd/main.go b/cmd/main.go index d70e105..89e31e1 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -351,7 +351,18 @@ func registerProviders(ctx context.Context, c client.Client) { // delivered via a Secret), and one account-global credential authorizes every // region. Registration only fails (and is a non-fatal skip) if the price catalog // cannot load — region config can no longer make it fail. - if p, err := awsprovider.NewSDKClient(ctx, awsRegionSource(c)); err != nil { + // SandD daemon (opt-in): if SANDD_TUNNEL_AUTHKEY is set, every AWS workload runs + // the SandD daemon inside its container in tunnel mode, so commands and + // interactive shells can be run in the user's own environment over the mesh with + // no inbound access. Absent the authkey the zero SanddConfig injects nothing, so + // this is off by default. The authkey is a secret (delivered via a mounted + // Secret) and is NEVER logged. + sanddCfg := provider.SanddConfig{ + AuthKey: os.Getenv("SANDD_TUNNEL_AUTHKEY"), + ControlServer: os.Getenv("SANDD_TUNNEL_SERVER"), + ServerURL: os.Getenv("SANDD_SERVER_URL"), + } + if p, err := awsprovider.NewSDKClient(ctx, awsRegionSource(c), sanddCfg); err != nil { setupLog.Info("skipping AWS provider registration", "reason", err.Error()) } else { provider.Register(p) diff --git a/config/crd/bases/nebula.inftyai.com_nodepools.yaml b/config/crd/bases/nebula.inftyai.com_nodepools.yaml index 61e03c6..ddfa71f 100644 --- a/config/crd/bases/nebula.inftyai.com_nodepools.yaml +++ b/config/crd/bases/nebula.inftyai.com_nodepools.yaml @@ -102,9 +102,9 @@ spec: description: |- BlocklistTTL is the BASE duration a failed placement is excluded before the provider becomes a candidate for it again. The controller adds a random jitter - (up to a minute) on top so Pods that failed for the same reason do not all - retry the just-freed candidate in lockstep, so the effective exclusion is this - value plus that jitter. + (up to 30s) on top so Pods that failed for the same reason do not all retry the + just-freed candidate in lockstep, so the effective exclusion is this value plus + that jitter. type: string type: object providers: diff --git a/config/samples/deployment.yaml b/config/samples/deployment.yaml index f45239b..f3e62cd 100644 --- a/config/samples/deployment.yaml +++ b/config/samples/deployment.yaml @@ -28,7 +28,7 @@ metadata: labels: app.kubernetes.io/managed-by: nebula spec: - replicas: 8 + replicas: 2 selector: matchLabels: app: gpu-workload-sample @@ -38,7 +38,7 @@ spec: app: gpu-workload-sample nebula.inftyai.com/enabled: "true" nebula.inftyai.com/nodepool: sample - nebula.inftyai.com/accelerator-type: a100-40gb + nebula.inftyai.com/accelerator-type: t4 spec: # Do NOT set nodeName or a provider nodeSelector yourself — the placement # controller fills the nodeSelector in when it ungates the Pod. Setting @@ -65,4 +65,4 @@ spec: # GPU count. Standard extended resource, so the scheduler's fit check # and provisioning read the same number. 8 => 8x the accelerator-type # above. - nvidia.com/gpu: "8" + nvidia.com/gpu: "1" diff --git a/config/samples/nebula_v1alpha1_nodepool.yaml b/config/samples/nebula_v1alpha1_nodepool.yaml index 7b2c852..8295ac8 100644 --- a/config/samples/nebula_v1alpha1_nodepool.yaml +++ b/config/samples/nebula_v1alpha1_nodepool.yaml @@ -26,4 +26,4 @@ spec: # Inner axis: within the active capacity tier. strategy: Ordered failover: - blocklistTTL: 3m + blocklistTTL: 30s diff --git a/pkg/provider/aws/aws.go b/pkg/provider/aws/aws.go index c061ec4..0d46c58 100644 --- a/pkg/provider/aws/aws.go +++ b/pkg/provider/aws/aws.go @@ -142,6 +142,11 @@ type InstanceSpec struct { Region string // Tags carry Nebula identity; ClaimTagKey holds the NodeClaim name. Tags map[string]string + // Sandd, when Enabled, makes the workload's docker run start the SandD daemon + // INSIDE the container (via an entrypoint shim) so commands and interactive + // shells run in the user's own env/cwd/code over the tunnel. A zero value injects + // nothing. See buildUserData/writeSanddEntrypoint and provider.SanddConfig. + Sandd provider.SanddConfig } // EC2Instance is the adapter-level view of one EC2 instance as observed. @@ -211,6 +216,9 @@ type Provider struct { // regionSource reports the NodePool-declared regions to sweep in List/Offerings. // May be nil in tests, in which case sweepRegions uses only the cache keys. regionSource RegionSource + // sandd is the optional SandD daemon config stamped onto every InstanceSpec so + // buildUserData can bootstrap it. Zero value => disabled (no injection). + sandd provider.SanddConfig mu sync.Mutex clients map[string]Client // region -> Client, populated lazily by clientFor @@ -225,11 +233,16 @@ type Provider struct { // (admission requires each aws pool to list ≥1 region, and placement stamps it onto // the ProvisionRequest), and observed instances report their region from the // region-pinned client — so nothing needs a fallback, and no AWS_REGION env is read. -func New(newClient ClientFactory, cat catalog.Lookup, regionSource RegionSource) *Provider { +// +// sandd is the optional SandD daemon config baked into every instance's user-data; +// its zero value disables injection, so tests and the non-SandD path pass +// provider.SanddConfig{}. +func New(newClient ClientFactory, cat catalog.Lookup, regionSource RegionSource, sandd provider.SanddConfig) *Provider { return &Provider{ Base: catalog.Base{ProviderName: provider.ProviderAWS, Catalog: cat}, newClient: newClient, regionSource: regionSource, + sandd: sandd, clients: make(map[string]Client), } } @@ -244,6 +257,7 @@ func newSingleRegion(client Client, cat catalog.Lookup, region string) *Provider func(context.Context, string) (Client, error) { return client, nil }, cat, func() []string { return []string{region} }, + provider.SanddConfig{}, // single-region test convenience: no SandD injection ) // Pre-seed the cache so even a stray region lookup returns the fake rather than // invoking the (constant) factory. @@ -714,6 +728,9 @@ func (p *Provider) instanceSpecFromPod(pod *corev1.Pod, req provider.ProvisionRe Spot: req.CapacityType == nebulav1alpha1.CapacitySpot, Region: req.Region, Tags: map[string]string{ClaimTagKey: req.ClaimName}, + // SandD daemon (opt-in): keyed by the claim name so a daemon that dials home + // correlates 1:1 with this instance's NodeClaim. Zero value => no injection. + Sandd: p.sandd, }, nil } diff --git a/pkg/provider/aws/client.go b/pkg/provider/aws/client.go index 0ee0629..4b70865 100644 --- a/pkg/provider/aws/client.go +++ b/pkg/provider/aws/client.go @@ -30,6 +30,7 @@ import ( smithy "github.com/aws/smithy-go" logf "sigs.k8s.io/controller-runtime/pkg/log" + "github.com/InftyAI/Nebula/pkg/provider" "github.com/InftyAI/Nebula/pkg/provider/catalog" ) @@ -204,7 +205,14 @@ var _ Client = (*sdkClient)(nil) // // The catalog is loaded via catalog.Load() (embedded CSV / mounted ConfigMap), // identical to the other adapters. -func NewSDKClient(_ context.Context, regionSource RegionSource) (*Provider, error) { +// +// sandd is the OPTIONAL SandD daemon config: its zero value (empty AuthKey) leaves +// the bootstrap untouched, so passing provider.SanddConfig{} is a no-op. When set, +// every workload this provider launches runs the SandD daemon inside its container +// in tunnel mode (see buildUserData), the workload's command-execution/shell +// channel. Unlike credentials, the auth key IS accepted here — it is delivered to +// the controller as a secret and stamped into the (base64) user-data. +func NewSDKClient(_ context.Context, regionSource RegionSource, sandd provider.SanddConfig) (*Provider, error) { cat, err := catalog.Load() if err != nil { return nil, fmt.Errorf("aws: load price catalog: %w", err) @@ -213,7 +221,7 @@ func NewSDKClient(_ context.Context, regionSource RegionSource) (*Provider, erro factory := func(ctx context.Context, region string) (Client, error) { return newSDKClientForRegion(ctx, region) } - return New(factory, cat, regionSource), nil + return New(factory, cat, regionSource, sandd), nil } // newSDKClientForRegion builds one region-pinned sdkClient: it loads SDK config for diff --git a/pkg/provider/aws/client_test.go b/pkg/provider/aws/client_test.go index 544a161..43c2cdf 100644 --- a/pkg/provider/aws/client_test.go +++ b/pkg/provider/aws/client_test.go @@ -347,6 +347,110 @@ func TestBuildUserData_QuotesHostileValues(t *testing.T) { } } +// TestBuildUserData_SanddDisabled: the zero SanddConfig is the default, so a spec +// that does not opt in must render EXACTLY the plain bootstrap — no sandd/tunnel +// tokens leak in, so existing clusters are unaffected. +func TestBuildUserData_SanddDisabled(t *testing.T) { + encoded, err := buildUserData(InstanceSpec{Image: "img"}) + if err != nil { + t.Fatalf("buildUserData: %v", err) + } + raw, _ := base64.StdEncoding.DecodeString(encoded) + script := string(raw) + if strings.Contains(script, "sandd") || strings.Contains(script, "tunnel") { + t.Fatalf("sandd disabled but bootstrap injected it:\n%s", script) + } +} + +// TestBuildUserData_SanddEnabled: when opted in, sandd runs INSIDE the container via +// an entrypoint shim so its shells see the user's env/cwd/code. The shim must +// override the image ENTRYPOINT with /bin/sh, carry daemon config as container env +// (incl. the claim-derived DAEMON_ID), start the daemon fail-open + backgrounded, +// and exec the user's effective command so the workload becomes PID 1. +func TestBuildUserData_SanddEnabled(t *testing.T) { + spec := InstanceSpec{ + Image: "img", + Command: []string{"python", "train.py"}, + Args: []string{"--epochs", "3"}, + Tags: map[string]string{ClaimTagKey: "claim-abc"}, + Sandd: provider.SanddConfig{ + AuthKey: "tskey-secret", + ControlServer: "http://headscale:8080", + ServerURL: "ws://100.64.0.1:8765/ws", + }, + } + encoded, err := buildUserData(spec) + if err != nil { + t.Fatalf("buildUserData: %v", err) + } + raw, _ := base64.StdEncoding.DecodeString(encoded) + script := string(raw) + + // The shim commandeers the container ENTRYPOINT as /bin/sh. + if !strings.Contains(script, "--entrypoint '/bin/sh'") { + t.Fatalf("sandd shim must override the image entrypoint with /bin/sh:\n%s", script) + } + + // Daemon config is delivered as container env (so a process INSIDE the container + // sees it), including the claim-derived DAEMON_ID and the tunnel key/server. + for _, want := range []string{ + "-e 'DAEMON_ID=claim-abc'", + "-e 'SERVER_URL=ws://100.64.0.1:8765/ws'", + "-e 'SANDD_TUNNEL_AUTHKEY=tskey-secret'", + "-e 'SANDD_TUNNEL_SERVER=http://headscale:8080'", + "-e 'SANDD_BINARY_URL=" + sanddBinaryURL + "'", + } { + if !strings.Contains(script, want) { + t.Fatalf("sandd shim missing container env %q:\n%s", want, script) + } + } + + // Fail-open + backgrounded: the daemon bring-up is a `( set +e ... ) || true` + // subshell ending in `&`, so a failed fetch/start cannot stop the workload. + if !strings.Contains(script, "set +e") || !strings.Contains(script, ") || true") { + t.Fatalf("sandd bring-up must be fail-open (set +e + || true):\n%s", script) + } + if !strings.Contains(script, "&\n") { + t.Fatalf("sandd daemon must be backgrounded:\n%s", script) + } + + // The shim execs the user's effective command so the workload is PID 1, and that + // command (image then argv) is rendered after the shim payload in kubelet order: + // Command[0]+Command[1:] then Args. + if !strings.Contains(script, `exec "$@"`) { + t.Fatalf("shim must exec the user command as PID 1:\n%s", script) + } + if !strings.Contains(script, "'img' '-c'") { + t.Fatalf("shim payload must follow the image on the run line:\n%s", script) + } + if !strings.Contains(script, "'sandd-shim' 'python' 'train.py' '--epochs' '3'\n") { + t.Fatalf("user effective command not rendered after the shim in order:\n%s", script) + } +} + +// TestBuildUserData_SanddQuotesHostileValues: sandd config values are shell-quoted +// like every other injected value, so a hostile auth key/URL cannot break out. +func TestBuildUserData_SanddQuotesHostileValues(t *testing.T) { + spec := InstanceSpec{ + Image: "img", + Tags: map[string]string{ClaimTagKey: "c"}, + Sandd: provider.SanddConfig{ + AuthKey: "k'; rm -rf /; echo '", + ControlServer: "http://h:8080", + ServerURL: "ws://s:8765/ws", + }, + } + encoded, err := buildUserData(spec) + if err != nil { + t.Fatalf("buildUserData: %v", err) + } + raw, _ := base64.StdEncoding.DecodeString(encoded) + script := string(raw) + if strings.Contains(script, "\nrm -rf /") { + t.Fatalf("hostile sandd value escaped quoting:\n%s", script) + } +} + func TestClassifyEC2Error(t *testing.T) { tests := []struct { name string diff --git a/pkg/provider/aws/translate.go b/pkg/provider/aws/translate.go index ccb0444..b843fc7 100644 --- a/pkg/provider/aws/translate.go +++ b/pkg/provider/aws/translate.go @@ -52,6 +52,7 @@ func buildUserData(spec InstanceSpec) (string, error) { var b strings.Builder b.WriteString("#!/bin/bash\n") b.WriteString("set -euo pipefail\n") + // Pull first so an image error surfaces before we try to run. fmt.Fprintf(&b, "docker pull %s\n", shellQuote(spec.Image)) @@ -70,14 +71,27 @@ func buildUserData(spec InstanceSpec) (string, error) { for _, k := range sortedKeys(spec.Env) { fmt.Fprintf(&b, " -e %s", shellQuote(k+"="+spec.Env[k])) } - // runArgs are everything that follows the image: the entrypoint's own arguments - // (Command[1:]) then the CMD arguments (Args), in that order. + + // SandD daemon (opt-in): run it INSIDE the container so its shells/commands see + // the user's own env, cwd, filesystem and code (a host-side daemon would only see + // the host). We cannot assume the user's image has sandd, so a shell entrypoint + // shim fetches the static musl binary at boot, starts it backgrounded, then execs + // the user's real program — see writeSanddEntrypoint. This overrides the image's + // ENTRYPOINT with the shim, so the shim (not Docker) is responsible for launching + // the workload with the right Command/Args; runArgs is left empty in that case. var runArgs []string - if len(spec.Command) > 0 { - fmt.Fprintf(&b, " --entrypoint %s", shellQuote(spec.Command[0])) - runArgs = append(runArgs, spec.Command[1:]...) + if spec.Sandd.Enabled() { + runArgs = writeSanddEntrypoint(&b, spec) + } else { + // No shim: map Command/Args straight onto Docker's --entrypoint/CMD as before. + // runArgs are everything that follows the image: the entrypoint's own arguments + // (Command[1:]) then the CMD arguments (Args), in that order. + if len(spec.Command) > 0 { + fmt.Fprintf(&b, " --entrypoint %s", shellQuote(spec.Command[0])) + runArgs = append(runArgs, spec.Command[1:]...) + } + runArgs = append(runArgs, spec.Args...) } - runArgs = append(runArgs, spec.Args...) b.WriteString(" " + shellQuote(spec.Image)) for _, arg := range runArgs { @@ -88,6 +102,77 @@ func buildUserData(spec InstanceSpec) (string, error) { return base64.StdEncoding.EncodeToString([]byte(b.String())), nil } +// sanddBinaryURL is the statically-linked (musl) SandD daemon release asset. Being +// static, this one binary runs in any container image regardless of its libc, so the +// shim can fetch it into an arbitrary user image at boot. It is pinned to the same +// asset name install.sh resolves; amd64 matches the x86_64 GPU instance types. +const sanddBinaryURL = "https://github.com/InftyAI/SandD/releases/latest/download/sandd-linux-amd64" + +// sanddShimScript is the container ENTRYPOINT shim (run as `/bin/sh -c