Add custom image family and apibootstrap

In an analogous fashion to the aws provider this exposes a userData field in the nodeclass API.
In addition to the also added custom image family type this will enable use cases for custom images and boostrapping mechanisims

pkg/apis/v1alpha2/aksnodeclass.go

@@ -24,6 +24,13 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+// TODO(enxebre): make this its own type, e.g. type ImageFamily string.
+const (
+	Ubuntu2204ImageFamily string = "Ubuntu2204"
+	AzureLinuxImageFamily string = "AzureLinux"
+	CustomImageFamily     string = "Custom"
+)
+
 // AKSNodeClassSpec is the top level specification for the AKS Karpenter Provider.
 // This will contain configuration necessary to launch instances in AKS.
 type AKSNodeClassSpec struct {
@@ -41,7 +48,7 @@ type AKSNodeClassSpec struct {
 	ImageID *string `json:"-"`
 	// ImageFamily is the image family that instances use.
 	// +kubebuilder:default=Ubuntu2204
-	// +kubebuilder:validation:Enum:={Ubuntu2204,AzureLinux}
+	// +kubebuilder:validation:Enum:={Ubuntu2204,AzureLinux,Custom}
 	ImageFamily *string `json:"imageFamily,omitempty"`
 	// Tags to be applied on Azure resources like instances.
 	// +optional
@@ -56,6 +63,11 @@ type AKSNodeClassSpec struct {
 	// +kubebuilder:validation:Minimum:=0
 	// +optional
 	MaxPods *int32 `json:"maxPods,omitempty"`
+	// userData to be applied to the provisioned nodes.
+	// It must be in the appropriate format based on the AMIFamily in use.
+	// Karpenter will use this content for the VM OSProfile When the ImageFamily is "Custom".
+	// +optional
+	UserData *string `json:"userData,omitempty"`
 }
 
 // KubeletConfiguration defines args to be used when configuring kubelet on provisioned nodes.

pkg/apis/v1alpha2/labels.go

@@ -108,8 +108,3 @@ var (
 	AnnotationAKSNodeClassHash        = apis.Group + "/aksnodeclass-hash"
 	AnnotationAKSNodeClassHashVersion = apis.Group + "/aksnodeclass-hash-version"
 )
-
-const (
-	Ubuntu2204ImageFamily = "Ubuntu2204"
-	AzureLinuxImageFamily = "AzureLinux"
-)

pkg/providers/imagefamily/azlinux.go

@@ -35,6 +35,8 @@ const (
 	AzureLinuxGen2ArmImageDefinition = "V2gen2arm64"
 )
 
+var _ ImageFamily = (*AzureLinux)(nil)
+
 type AzureLinux struct {
 	Options *parameters.StaticParameters
 }
@@ -83,7 +85,7 @@ func (u AzureLinux) DefaultImages() []DefaultImageOutput {
 }
 
 // UserData returns the default userdata script for the image Family
-func (u AzureLinux) ScriptlessCustomData(kubeletConfig *bootstrap.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType) bootstrap.Bootstrapper {
+func (u AzureLinux) ScriptlessCustomData(kubeletConfig *bootstrap.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType, apiUserData *string) bootstrap.Bootstrapper {
 	return bootstrap.AKS{
 		Options: bootstrap.Options{
 			ClusterName:      u.Options.ClusterName,

pkg/providers/imagefamily/bootstrap/apibootstrap.go

@@ -0,0 +1,19 @@
+package bootstrap
+
+import (
+	"encoding/base64"
+)
+
+var _ Bootstrapper = (*APIbootstrap)(nil) // assert AKS implements Bootstrapper
+
+type APIbootstrap struct {
+	UserData *string
+}
+
+func (a APIbootstrap) Script() (string, error) {
+	if a.UserData == nil {
+		return "", nil
+	}
+
+	return base64.StdEncoding.EncodeToString([]byte(*a.UserData)), nil
+}

pkg/providers/imagefamily/custom.go

@@ -0,0 +1,56 @@
+/*
+Portions Copyright (c) Microsoft Corporation.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package imagefamily
+
+import (
+	"github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2"
+	"github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily/bootstrap"
+	"github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily/customscriptsbootstrap"
+	"github.com/Azure/karpenter-provider-azure/pkg/providers/launchtemplate/parameters"
+	v1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/karpenter/pkg/cloudprovider"
+)
+
+var _ ImageFamily = (*Custom)(nil)
+
+type Custom struct {
+	Options *parameters.StaticParameters
+}
+
+func (c Custom) Name() string {
+	return v1alpha2.CustomImageFamily
+}
+
+func (c Custom) DefaultImages() []DefaultImageOutput {
+	return []DefaultImageOutput{
+		{
+			Distro: "Custom",
+		},
+	}
+}
+
+// UserData returns the default userdata script for the image Family
+func (c Custom) ScriptlessCustomData(kubeletConfig *bootstrap.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType, apiUserData *string) bootstrap.Bootstrapper {
+	return bootstrap.APIbootstrap{
+		UserData: apiUserData,
+	}
+}
+
+// UserData returns the default userdata script for the image Family
+func (c Custom) CustomScriptsNodeBootstrapping(kubeletConfig *bootstrap.KubeletConfiguration, taints []v1.Taint, startupTaints []v1.Taint, labels map[string]string, instanceType *cloudprovider.InstanceType, imageDistro string, storageProfile string) customscriptsbootstrap.Bootstrapper {
+	return customscriptsbootstrap.ProvisionClientBootstrap{}
+}

pkg/providers/imagefamily/resolver.go

@@ -50,6 +50,7 @@ type ImageFamily interface {
 		labels map[string]string,
 		caBundle *string,
 		instanceType *cloudprovider.InstanceType,
+		apiUserData *string,
 	) bootstrap.Bootstrapper
 	CustomScriptsNodeBootstrapping(
 		kubeletConfig *bootstrap.KubeletConfiguration,
@@ -114,6 +115,7 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass,
 			staticParameters.Labels,
 			staticParameters.CABundle,
 			instanceType,
+			nodeClass.Spec.UserData,
 		),
 		CustomScriptsNodeBootstrapping: imageFamily.CustomScriptsNodeBootstrapping(
 			prepareKubeletConfiguration(instanceType, nodeClass),
@@ -159,6 +161,8 @@ func getImageFamily(familyName *string, parameters *template.StaticParameters) I
 		return &Ubuntu2204{Options: parameters}
 	case v1alpha2.AzureLinuxImageFamily:
 		return &AzureLinux{Options: parameters}
+	case v1alpha2.CustomImageFamily:
+		return &Custom{}
 	default:
 		return &Ubuntu2204{Options: parameters}
 	}

pkg/providers/imagefamily/ubuntu_2204.go

@@ -35,6 +35,8 @@ const (
 	Ubuntu2204Gen2ArmImageDefinition = "2204gen2arm64containerd"
 )
 
+var _ ImageFamily = (*Ubuntu2204)(nil)
+
 type Ubuntu2204 struct {
 	Options *parameters.StaticParameters
 }
@@ -83,7 +85,7 @@ func (u Ubuntu2204) DefaultImages() []DefaultImageOutput {
 }
 
 // UserData returns the default userdata script for the image Family
-func (u Ubuntu2204) ScriptlessCustomData(kubeletConfig *bootstrap.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType) bootstrap.Bootstrapper {
+func (u Ubuntu2204) ScriptlessCustomData(kubeletConfig *bootstrap.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType, apiUserData *string) bootstrap.Bootstrapper {
 	return bootstrap.AKS{
 		Options: bootstrap.Options{
 			ClusterName:      u.Options.ClusterName,