performance improment

api/v1alpha1/nodepool_types.go

@@ -134,9 +134,12 @@ const (
 // type, so a failed H100 request does not block A100 requests on the same
 // provider) for BlocklistTTL, then reconsiders it. This spec only tunes that.
 type FailoverPolicy struct {
-	// BlocklistTTL is how long a failed placement is excluded before the
-	// provider becomes a candidate for it again.
-	// +kubebuilder:default="3m"
+	// BlocklistTTL is the BASE duration a failed placement is excluded before the
+	// provider becomes a candidate for it again. The controller adds a random jitter
+	// (up to a minute) on top so Pods that failed for the same reason do not all
+	// retry the just-freed candidate in lockstep, so the effective exclusion is this
+	// value plus that jitter.
+	// +kubebuilder:default="30s"
 	BlocklistTTL metav1.Duration `json:"blocklistTTL,omitempty"`
 }
 

config/crd/bases/nebula.inftyai.com_nodepools.yaml

@@ -98,10 +98,13 @@ spec:
                   RunPod reports no capacity) is temporarily excluded and re-tried.
                 properties:
                   blocklistTTL:
-                    default: 3m
+                    default: 30s
                     description: |-
-                      BlocklistTTL is how long a failed placement is excluded before the
-                      provider becomes a candidate for it again.
+                      BlocklistTTL is the BASE duration a failed placement is excluded before the
+                      provider becomes a candidate for it again. The controller adds a random jitter
+                      (up to a minute) on top so Pods that failed for the same reason do not all
+                      retry the just-freed candidate in lockstep, so the effective exclusion is this
+                      value plus that jitter.
                     type: string
                 type: object
               providers:

config/samples/deployment.yaml

@@ -38,7 +38,7 @@ spec:
         app: gpu-workload-sample
         nebula.inftyai.com/enabled: "true"
         nebula.inftyai.com/nodepool: sample
-        nebula.inftyai.com/accelerator-type: l4
+        nebula.inftyai.com/accelerator-type: a100-40gb
     spec:
       # Do NOT set nodeName or a provider nodeSelector yourself — the placement
       # controller fills the nodeSelector in when it ungates the Pod. Setting

pkg/provider/aws/aws.go

@@ -126,8 +126,13 @@ type InstanceSpec struct {
 	// Image is the container image, from the Pod's first container. The Client is
 	// responsible for launching it (e.g. via a GPU AMI + user-data, or ECS/EKS).
 	Image string
-	// Command is the container command+args, from the Pod.
+	// Command is the Pod container's command, kept SEPARATE from Args to preserve
+	// Kubernetes container semantics: command overrides the image ENTRYPOINT (Docker
+	// --entrypoint), not its CMD. Empty means "use the image's own ENTRYPOINT".
 	Command []string
+	// Args is the Pod container's args, appended after the entrypoint just as CMD
+	// arguments. Empty means "use the image's own CMD".
+	Args []string
 	// Env is the environment, flattened from the Pod's container env.
 	Env map[string]string
 	// Spot requests interruptible capacity when true (OnDemand otherwise).
@@ -703,7 +708,8 @@ func (p *Provider) instanceSpecFromPod(pod *corev1.Pod, req provider.ProvisionRe
 	return InstanceSpec{
 		InstanceTypes: instanceTypes,
 		Image:         c.Image,
-		Command:       append(append([]string{}, c.Command...), c.Args...),
+		Command:       append([]string{}, c.Command...),
+		Args:          append([]string{}, c.Args...),
 		Env:           env,
 		Spot:          req.CapacityType == nebulav1alpha1.CapacitySpot,
 		Region:        req.Region,

pkg/provider/aws/aws_test.go

@@ -195,8 +195,13 @@ func TestProvision_MapsAcceleratorToInstanceType(t *testing.T) {
 	if got := f.lastSpec.Tags[ClaimTagKey]; got != "claim-a" {
 		t.Fatalf("claim tag = %q, want claim-a", got)
 	}
-	if len(f.lastSpec.Command) != 2 || f.lastSpec.Command[0] != "run" {
-		t.Fatalf("command = %v", f.lastSpec.Command)
+	// Command and Args ride through SEPARATELY (no flattening), preserving the
+	// Pod's Kubernetes container semantics for the user-data renderer.
+	if len(f.lastSpec.Command) != 1 || f.lastSpec.Command[0] != "run" {
+		t.Fatalf("command = %v, want [run]", f.lastSpec.Command)
+	}
+	if len(f.lastSpec.Args) != 1 || f.lastSpec.Args[0] != "--flag" {
+		t.Fatalf("args = %v, want [--flag]", f.lastSpec.Args)
 	}
 }
 

pkg/provider/aws/client.go

@@ -295,6 +295,42 @@ func (c *sdkClient) RunInstance(ctx context.Context, spec InstanceSpec) (string,
 	log := logf.FromContext(ctx).WithName("aws-run").WithValues(
 		"region", c.region, "instanceTypes", spec.InstanceTypes, "spot", spec.Spot)
 
+	// Prune subnets whose AZ does not OFFER a requested type before building the grid,
+	// so we never emit a (type, subnet) override EC2 would reject outright with
+	// InvalidFleetConfiguration (e.g. g6.48xlarge in us-east-1e). Fail-open: a lookup
+	// error yields nil offerings and offeredSubnets keeps every subnet, so a transient
+	// DescribeInstanceTypeOfferings failure can never shrink the grid below today's.
+	// Done BEFORE any launch template is created so a region that offers none of the
+	// requested types costs neither the template create/delete pair nor a CreateFleet.
+	offerings := c.instanceTypeAZs(ctx, spec.InstanceTypes)
+
+	// One override per (instance type, subnet) pair => EC2 searches that whole grid
+	// for capacity in a single call, landing on whichever pair is available. The
+	// instance types are interchangeable alternates the accelerator maps to (primary
+	// first); spanning them broadens the launch beyond a single type's per-AZ
+	// capacity. The launch template carries no instance type — it lives here in the
+	// overrides — so a template built once serves every type. No subnets (no default
+	// VPC) => one override per type with the subnet unset, letting EC2 pick it (still
+	// no cross-AZ search).
+	overrides := fleetOverrides(spec.InstanceTypes, c.subnets, offerings)
+	log.V(1).Info("preparing instant fleet",
+		"candidateInstanceTypes", len(spec.InstanceTypes), "candidateSubnets", len(c.subnets),
+		"overrides", len(overrides))
+
+	// No override survived the prune: every requested type is offered in NO configured
+	// AZ of this region (a known-empty offered set — offerings are stable and we just
+	// queried them). CreateFleet would only bounce with InvalidFleetConfiguration, so
+	// short-circuit to a region-scoped no-capacity (NOT zone-local: no sibling AZ can
+	// help) and let region/tier failover route elsewhere — without the doomed API call
+	// or its Warning event. A fail-open (unknown) offered set never lands here: it
+	// keeps every subnet, so overrides is non-empty.
+	if len(overrides) == 0 {
+		log.Info("skipping instant fleet: no requested type offered in any configured AZ of this region")
+		return "", wrapNoCapacity(
+			fmt.Errorf("aws: none of %v offered in any availability zone of %s", spec.InstanceTypes, c.region),
+			spec.Spot, false /* zoneLocal */)
+	}
+
 	// Opportunistically reap any ephemeral launch template a prior provision leaked
 	// (crash between create and delete). Best-effort and staleness-gated, so it
 	// cannot touch a concurrent provision's own template; see sweepStaleLaunchTemplates.
@@ -315,25 +351,7 @@ func (c *sdkClient) RunInstance(ctx context.Context, spec InstanceSpec) (string,
 		}
 	}()
 
-	// Prune subnets whose AZ does not OFFER a requested type before building the grid,
-	// so we never emit a (type, subnet) override EC2 would reject outright with
-	// InvalidFleetConfiguration (e.g. g6.48xlarge in us-east-1e). Fail-open: a lookup
-	// error yields nil offerings and offeredAZs keeps every subnet, so a transient
-	// DescribeInstanceTypeOfferings failure can never shrink the grid below today's.
-	offerings := c.instanceTypeAZs(ctx, spec.InstanceTypes)
-
-	// One override per (instance type, subnet) pair => EC2 searches that whole grid
-	// for capacity in a single call, landing on whichever pair is available. The
-	// instance types are interchangeable alternates the accelerator maps to (primary
-	// first); spanning them broadens the launch beyond a single type's per-AZ
-	// capacity. The launch template carries no instance type — it lives here in the
-	// overrides — so a template built once serves every type. No subnets (no default
-	// VPC) => one override per type with the subnet unset, letting EC2 pick it (still
-	// no cross-AZ search).
-	overrides := fleetOverrides(spec.InstanceTypes, c.subnets, offerings)
-	log.V(1).Info("launching via instant fleet",
-		"candidateInstanceTypes", len(spec.InstanceTypes), "candidateSubnets", len(c.subnets),
-		"overrides", len(overrides))
+	log.V(1).Info("launching via instant fleet", "overrides", len(overrides))
 
 	out, err := c.ec2.CreateFleet(ctx, &ec2.CreateFleetInput{
 		Type: ec2types.FleetTypeInstant, // synchronous: instances/errors in the response
@@ -524,9 +542,16 @@ func fleetOverrides(
 }
 
 // offeredSubnets filters subnets to those in an AZ that offers it, per offerings.
-// It fails open: an unknown offered set (nil — lookup skipped or failed) keeps every
-// subnet, and a filter that would remove ALL subnets also keeps every subnet, so
-// pruning only ever narrows a launch that still has at least one usable AZ left.
+// The two outcomes turn on whether the offered set is KNOWN:
+//   - unknown (nil — lookup skipped or failed): fail open, keep every subnet, so a
+//     transient DescribeInstanceTypeOfferings failure never shrinks the launch.
+//   - known: keep only the subnets whose AZ offers the type. When that intersection
+//     is EMPTY the type is offered in no configured AZ of this region, so return nil
+//     (no usable subnet) rather than the old fail-open "keep all". RunInstance reads
+//     an all-types-empty override set as a region no-capacity and skips the doomed
+//     CreateFleet — this is the source of the InvalidFleetConfiguration event churn
+//     (e.g. p4d.24xlarge in us-west-1, offered in neither AZ). A known-empty result
+//     is trustworthy: offerings are stable and we just queried them.
 func offeredSubnets(it string, subnets []subnet, offerings map[string]map[string]bool) []subnet {
 	azs := offerings[it]
 	if azs == nil {
@@ -538,10 +563,7 @@ func offeredSubnets(it string, subnets []subnet, offerings map[string]map[string
 			kept = append(kept, sn)
 		}
 	}
-	if len(kept) == 0 {
-		return subnets // pruning would zero the launch: keep all, let EC2 decide
-	}
-	return kept
+	return kept // may be empty: type offered in no configured AZ of this region
 }
 
 // fleetCapacityType maps the Spot flag to the fleet's default target-capacity

pkg/provider/aws/client_test.go

@@ -247,6 +247,7 @@ func TestBuildUserData(t *testing.T) {
 	spec := InstanceSpec{
 		Image:   "myrepo/train:v1",
 		Command: []string{"python", "train.py"},
+		Args:    []string{"--epochs", "3"},
 		Env:     map[string]string{"B": "2", "A": "1"},
 	}
 	encoded, err := buildUserData(spec)
@@ -269,8 +270,55 @@ func TestBuildUserData(t *testing.T) {
 	if idxA, idxB := strings.Index(script, "'A=1'"), strings.Index(script, "'B=2'"); idxA < 0 || idxB < 0 || idxA > idxB {
 		t.Fatalf("env not rendered in sorted order:\n%s", script)
 	}
-	if !strings.Contains(script, "'python' 'train.py'") {
-		t.Fatalf("script missing command:\n%s", script)
+	// Kubernetes semantics: Command[0] overrides the image ENTRYPOINT (--entrypoint),
+	// Command[1:] are its leading args, and Args follow as CMD arguments — the whole
+	// tail rendered after the image, in that order.
+	if !strings.Contains(script, "--entrypoint 'python'") {
+		t.Fatalf("command[0] not mapped to --entrypoint:\n%s", script)
+	}
+	if !strings.Contains(script, "'myrepo/train:v1' 'train.py' '--epochs' '3'") {
+		t.Fatalf("command tail/args not rendered after the image in order:\n%s", script)
+	}
+}
+
+// TestBuildUserData_ArgsWithoutCommand covers a Pod that sets args but not command:
+// the image's own ENTRYPOINT must run (no --entrypoint emitted) with args appended
+// as CMD, exactly as a kubelet would launch it.
+func TestBuildUserData_ArgsWithoutCommand(t *testing.T) {
+	spec := InstanceSpec{
+		Image: "srv:v2",
+		Args:  []string{"--port", "8080"},
+	}
+	encoded, err := buildUserData(spec)
+	if err != nil {
+		t.Fatalf("buildUserData: %v", err)
+	}
+	raw, _ := base64.StdEncoding.DecodeString(encoded)
+	script := string(raw)
+
+	if strings.Contains(script, "--entrypoint") {
+		t.Fatalf("no command was set; --entrypoint must not be emitted:\n%s", script)
+	}
+	if !strings.Contains(script, "'srv:v2' '--port' '8080'") {
+		t.Fatalf("args not appended after the image as CMD:\n%s", script)
+	}
+}
+
+// TestBuildUserData_NoCommandOrArgs covers a bare image: the image's own
+// ENTRYPOINT/CMD run, so the run line is just the image with no trailing tokens.
+func TestBuildUserData_NoCommandOrArgs(t *testing.T) {
+	encoded, err := buildUserData(InstanceSpec{Image: "bare:v1"})
+	if err != nil {
+		t.Fatalf("buildUserData: %v", err)
+	}
+	raw, _ := base64.StdEncoding.DecodeString(encoded)
+	script := string(raw)
+
+	if strings.Contains(script, "--entrypoint") {
+		t.Fatalf("no command was set; --entrypoint must not be emitted:\n%s", script)
+	}
+	if !strings.Contains(script, "--gpus all 'bare:v1'\n") {
+		t.Fatalf("bare image must run with no trailing command/args:\n%s", script)
 	}
 }
 
@@ -405,14 +453,16 @@ func TestFleetOverrides_UnknownOfferingsDoesNotPrune(t *testing.T) {
 	}
 }
 
-func TestFleetOverrides_EmptyOfferedSetKeepsAll(t *testing.T) {
+func TestFleetOverrides_KnownEmptyOfferedSetPrunesAll(t *testing.T) {
 	subnets := []subnet{{id: "sn-a", az: "us-east-1a"}, {id: "sn-e", az: "us-east-1e"}}
-	// Resolved-but-empty offered set (type offered in no listed AZ) would zero the
-	// launch; fail-open keeps all subnets so a stale map can never wedge provisioning.
+	// Resolved-but-empty offered set: the type is offered in NO configured AZ. Unlike
+	// an unknown (nil) set, this is trustworthy, so it prunes to nothing rather than
+	// failing open — RunInstance reads the empty grid as a region no-capacity and skips
+	// the doomed CreateFleet instead of letting EC2 bounce it.
 	offerings := map[string]map[string]bool{"g6.48xlarge": {}}
 	got := fleetOverrides([]string{"g6.48xlarge"}, subnets, offerings)
-	if len(got) != 2 {
-		t.Fatalf("an empty offered set must keep all subnets (fail-open); got %d", len(got))
+	if len(got) != 0 {
+		t.Fatalf("a known-empty offered set must prune every subnet; got %d", len(got))
 	}
 }
 
@@ -454,6 +504,38 @@ func TestSDKRunInstance_PrunesUnofferedAZAndCaches(t *testing.T) {
 	}
 }
 
+// When a requested type is offered in NO configured AZ of the region (a known-empty
+// offered set, e.g. p4d.24xlarge in us-west-1), RunInstance must short-circuit to a
+// region-scoped no-capacity WITHOUT creating a launch template or calling CreateFleet
+// — the fleet would only bounce with InvalidFleetConfiguration. The error must carry
+// ErrNoCapacity (so failover proceeds) but NOT errZoneLocal (no sibling AZ can help).
+func TestSDKRunInstance_NoOfferedAZShortCircuits(t *testing.T) {
+	f := &fakeEC2{
+		fleetOut:      fleetWith("i-1"), // would succeed IF a fleet were launched
+		offeringPages: []*ec2.DescribeInstanceTypeOfferingsOutput{offeringPage()},
+	}
+	c := newSDKClient(f)
+	c.subnets = []subnet{{id: "sn-a", az: "us-west-1a"}, {id: "sn-b", az: "us-west-1b"}}
+
+	_, err := c.RunInstance(context.Background(), InstanceSpec{
+		InstanceTypes: []string{"p4d.24xlarge"}, Image: "img", Spot: true,
+		Tags: map[string]string{ClaimTagKey: "c"},
+	})
+	if !errors.Is(err, provider.ErrNoCapacity) {
+		t.Fatalf("want ErrNoCapacity, got %v", err)
+	}
+	if errors.Is(err, errZoneLocal) {
+		t.Fatal("a region-wide unavailability must not be zone-local (no sibling AZ helps)")
+	}
+	// No doomed API calls: neither a launch template nor a fleet.
+	if f.fleetCalls != 0 {
+		t.Fatalf("CreateFleet must be skipped when no AZ offers the type; got %d calls", f.fleetCalls)
+	}
+	if len(f.createdLTs) != 0 {
+		t.Fatalf("no launch template must be created on the short-circuit; got %v", f.createdLTs)
+	}
+}
+
 // A burst of concurrent provisions for the same NEW type must coalesce into ONE
 // DescribeInstanceTypeOfferings (single-flight): the first caller runs the lookup
 // while the rest wait on it, so the shared cache is populated with a single call
@@ -591,7 +673,10 @@ func TestSDKRunInstance_SpotSetsCapacityType(t *testing.T) {
 // One override per discovered subnet: EC2 does the per-AZ capacity search inside
 // the single fleet call rather than the adapter sweeping RunInstances per zone.
 func TestSDKRunInstance_OneOverridePerSubnet(t *testing.T) {
-	f := &fakeEC2{fleetOut: fleetWith("i-1")}
+	f := &fakeEC2{
+		fleetOut:      fleetWith("i-1"),
+		offeringPages: []*ec2.DescribeInstanceTypeOfferingsOutput{offeringPage("us-east-1a", "us-east-1b", "us-east-1c")},
+	}
 	c := &sdkClient{ec2: f, region: testRegion, amiID: "ami-123", subnets: []subnet{
 		{id: "subnet-a", az: "us-east-1a"},
 		{id: "subnet-b", az: "us-east-1b"},
@@ -624,7 +709,14 @@ func TestSDKRunInstance_OneOverridePerSubnet(t *testing.T) {
 // capacity. The launch template carries no instance type — it lives in the
 // overrides — so one template serves every candidate type.
 func TestSDKRunInstance_SpansInstanceTypesAndSubnets(t *testing.T) {
-	f := &fakeEC2{fleetOut: fleetWith("i-1")}
+	f := &fakeEC2{
+		fleetOut: fleetWith("i-1"),
+		// One offering page per type lookup (p5 then p4d), each offered in both AZs.
+		offeringPages: []*ec2.DescribeInstanceTypeOfferingsOutput{
+			offeringPage("us-east-1a", "us-east-1b"),
+			offeringPage("us-east-1a", "us-east-1b"),
+		},
+	}
 	c := &sdkClient{ec2: f, region: testRegion, amiID: "ami-123", subnets: []subnet{
 		{id: "subnet-a", az: "us-east-1a"},
 		{id: "subnet-b", az: "us-east-1b"},

pkg/provider/aws/translate.go

@@ -55,15 +55,32 @@ func buildUserData(spec InstanceSpec) (string, error) {
 	// Pull first so an image error surfaces before we try to run.
 	fmt.Fprintf(&b, "docker pull %s\n", shellQuote(spec.Image))
 
-	// docker run --gpus all -d, with env and command appended. Env keys are sorted
-	// so the rendered script is deterministic (stable across reconciles and easy to
-	// assert in tests).
+	// docker run --rm --gpus all, with env, then the image, then the workload's
+	// command/args. Kubernetes container semantics are preserved by mapping the two
+	// Pod fields the way the kubelet does, NOT by concatenating them:
+	//   - Command (Pod command) overrides the image ENTRYPOINT => Docker --entrypoint,
+	//     which takes a single program; Command[0] is the entrypoint and Command[1:]
+	//     become leading arguments (Docker only lets --entrypoint carry the program).
+	//   - Args (Pod args) are appended after as CMD arguments.
+	// When Command is empty the image's own ENTRYPOINT runs; when Args is empty the
+	// image's own CMD runs — so an image with a baked-in entrypoint launches exactly
+	// as it would under a kubelet. Env keys are sorted so the rendered script is
+	// deterministic (stable across reconciles and easy to assert in tests).
 	b.WriteString("docker run --rm --gpus all")
 	for _, k := range sortedKeys(spec.Env) {
 		fmt.Fprintf(&b, " -e %s", shellQuote(k+"="+spec.Env[k]))
 	}
+	// runArgs are everything that follows the image: the entrypoint's own arguments
+	// (Command[1:]) then the CMD arguments (Args), in that order.
+	var runArgs []string
+	if len(spec.Command) > 0 {
+		fmt.Fprintf(&b, " --entrypoint %s", shellQuote(spec.Command[0]))
+		runArgs = append(runArgs, spec.Command[1:]...)
+	}
+	runArgs = append(runArgs, spec.Args...)
+
 	b.WriteString(" " + shellQuote(spec.Image))
-	for _, arg := range spec.Command {
+	for _, arg := range runArgs {
 		b.WriteString(" " + shellQuote(arg))
 	}
 	b.WriteString("\n")