edgelesssys · msanft · Dec 8, 2023 · Nov 27, 2023 · Nov 27, 2023 · Nov 27, 2023
diff --git a/.github/actions/constellation_create/action.yml b/.github/actions/constellation_create/action.yml
@@ -53,6 +53,12 @@ inputs:
   selfManagedInfra:
     description: "Use self-managed infrastructure instead of infrastructure created by the Constellation CLI."
     required: true
+  marketplaceImageVersion:
+    description: "Marketplace OS image version. Used instead of osImage."
+    required: false
+  force:
+    description: "Set the force-flag on apply to ignore version mismatches."
+    required: false
 
 outputs:
   kubeconfig:
@@ -97,6 +103,13 @@ runs:
         yq eval -i "(.image) = \"${imageInput}\"" constellation-conf.yaml
         echo "image=${imageInput}" | tee -a "$GITHUB_OUTPUT"
 
+    - name: Set marketplace image flag (Azure)
+      if: inputs.marketplaceImageVersion != '' && inputs.cloudProvider == 'azure'
+      shell: bash
+      run: |
+        yq eval -i "(.provider.azure.useMarketplaceImage) = true" constellation-conf.yaml
+        yq eval -i "(.image) = \"${{ inputs.marketplaceImageVersion }}\"" constellation-conf.yaml
+
     - name: Update measurements for non-stable images
       if: inputs.fetchMeasurements
       shell: bash
@@ -163,11 +176,18 @@ runs:
         kubernetesVersion: ${{ inputs.kubernetesVersion }}
         selfManagedInfra: ${{ inputs.selfManagedInfra }}
 
+    - name: Set force flag
+      id: set-force-flag
+      if: inputs.force == 'true'
+      shell: bash
+      run: |
+        echo "flag=--force" | tee -a $GITHUB_OUTPUT
+
     - name: Constellation init
       id: constellation-init
       shell: bash
       run: |
-        constellation apply --skip-phases=infrastructure --debug
+        constellation apply --skip-phases=infrastructure --debug ${{ steps.set-force-flag.outputs.flag }}
         echo "KUBECONFIG=$(pwd)/constellation-admin.conf" | tee -a $GITHUB_OUTPUT
 
     - name: Wait for nodes to join and become ready

diff --git a/.github/actions/e2e_test/action.yml b/.github/actions/e2e_test/action.yml
@@ -80,6 +80,12 @@ inputs:
     description: "Access key for s3proxy"
   s3SecretKey:
     description: "Secret key for s3proxy"
+  marketplaceImageVersion:
+    description: "Marketplace OS image version. Used instead of osImage."
+    required: false
+  force:
+    description: "Set the force-flag on apply to ignore version mismatches."
+    required: false
 
 outputs:
   kubeconfig:
@@ -266,6 +272,8 @@ runs:
         internalLoadBalancer: ${{ inputs.internalLoadBalancer }}
         test: ${{ inputs.test }}
         selfManagedInfra: ${{ inputs.selfManagedInfra }}
+        marketplaceImageVersion: ${{ inputs.marketplaceImageVersion }}
+        force: ${{ inputs.force }}
 
     - name: Deploy log- and metrics-collection (Kubernetes)
       id: deploy-logcollection

diff --git a/.github/workflows/e2e-test-marketplace-image.yml b/.github/workflows/e2e-test-marketplace-image.yml
@@ -0,0 +1,87 @@
+name: e2e test marketplace image
+
+on:
+  workflow_dispatch:
+    inputs:
+      nodeCount:
+        description: "Number of nodes to use in the cluster. Given in format `<control-plane nodes>:<worker nodes>`."
+        default: "3:2"
+        type: string
+      cloudProvider:
+        description: "Which cloud provider to use."
+        type: choice
+        options:
+          - "azure"
+        default: "azure"
+        required: true
+      runner:
+        description: "Architecture of the runner that executes the CLI"
+        type: choice
+        options:
+          - "ubuntu-22.04"
+          - "macos-12"
+        default: "ubuntu-22.04"
+      test:
+        description: "The test to run."
+        type: choice
+        options:
+          - "sonobuoy quick"
+          - "sonobuoy full"
+          - "autoscaling"
+          - "lb"
+          - "perf-bench"
+          - "verify"
+          - "recover"
+          - "malicious join"
+          - "nop"
+        required: true
+      kubernetesVersion:
+        description: "Kubernetes version to create the cluster from."
+        default: "1.27"
+        required: true
+      cliVersion:
+        description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref."
+        type: string
+        default: ""
+        required: false
+      marketplaceImageVersion:
+        description: "Marketplace image version to use in the cluster's nodes. Needs to be a release semver."
+        type: string
+        default: ""
+        required: true
+      machineType:
+        description: "Override VM machine type. Leave as 'default' or empty to use the default VM type for the selected cloud provider."
+        type: string
+        default: "default"
+        required: false
+      regionZone:
+        description: "Region or zone to create the cluster in. Leave empty for default region/zone."
+        type: string
+      git-ref:
+        description: "Git ref to checkout."
+        type: string
+        default: "head"
+        required: false
+
+jobs:
+  e2e-test:
+    permissions:
+      id-token: write
+      checks: write
+      contents: read
+      packages: write
+    secrets: inherit
+    uses: ./.github/workflows/e2e-test.yml
+    with:
+      nodeCount: ${{ inputs.nodeCount }}
+      cloudProvider: ${{ inputs.cloudProvider }}
+      runner: ${{ inputs.runner }}
+      test: ${{ inputs.test }}
+      kubernetesVersion: ${{ inputs.kubernetesVersion }}
+      cliVersion: ${{ inputs.cliVersion }}
+      imageVersion: ${{ inputs.marketplaceImageVersion }}
+      machineType: ${{ inputs.machineType }}
+      regionZone: ${{ inputs.regionZone }}
+      git-ref: ${{ inputs.git-ref }}
+      marketplaceImageVersion: ${{ inputs.marketplaceImageVersion }}
+      force: true
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
@@ -116,6 +116,12 @@ on:
         description: "Use self-managed infrastructure."
         type: boolean
         default: false
+      marketplaceImageVersion:
+        description: "Marketplace image version to use."
+        type: string
+      force:
+        description: "Use the force-flag when applying to ignore version mismatches."
+        type: boolean
 
 jobs:
   split-nodeCount:
@@ -238,6 +244,8 @@ jobs:
           selfManagedInfra: ${{ inputs.selfManagedInfra }}
           s3AccessKey: ${{ secrets.AWS_ACCESS_KEY_ID_S3PROXY }}
           s3SecretKey: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3PROXY }}
+          marketplaceImageVersion: ${{ inputs.marketplaceImageVersion }}
+          force: ${{ inputs.force }}
 
       - name: Always terminate cluster
         if: always()

@@ -30,6 +30,7 @@ go_library(
         "//internal/file",
         "//internal/imagefetcher",
         "//internal/maa",
+        "//internal/mpimage",
         "//internal/role",
         "//internal/state",
     ],

@@ -131,7 +131,7 @@ func (a *Applier) terraformApplyVars(ctx context.Context, conf *config.Config) (
 		ctx,
 		conf.GetProvider(),
 		conf.GetAttestationConfig().GetVariant(),
-		conf.Image, conf.GetRegion(),
+		conf.Image, conf.GetRegion(), conf.UseMarketplaceImage(),
 	)
 	if err != nil {
 		return nil, fmt.Errorf("fetching image reference: %w", err)
@@ -141,7 +141,7 @@ func (a *Applier) terraformApplyVars(ctx context.Context, conf *config.Config) (
 	case cloudprovider.AWS:
 		return awsTerraformVars(conf, imageRef), nil
 	case cloudprovider.Azure:
-		return azureTerraformVars(conf, imageRef), nil
+		return azureTerraformVars(conf, imageRef)
 	case cloudprovider.GCP:
 		return gcpTerraformVars(conf, imageRef), nil
 	case cloudprovider.OpenStack:

@@ -20,7 +20,7 @@ import (
 type imageFetcher interface {
 	FetchReference(ctx context.Context,
 		provider cloudprovider.Provider, attestationVariant variant.Variant,
-		image, region string,
+		image, region string, useMarketplaceImage bool,
 	) (string, error)
 }
 

@@ -124,7 +124,7 @@ type stubImageFetcher struct {
 
 func (f *stubImageFetcher) FetchReference(_ context.Context,
 	_ cloudprovider.Provider, _ variant.Variant,
-	_, _ string,
+	_, _ string, _ bool,
 ) (string, error) {
 	return f.reference, f.fetchReferenceErr
 }

@@ -24,6 +24,7 @@ import (
 	"github.com/edgelesssys/constellation/v2/internal/config"
 	"github.com/edgelesssys/constellation/v2/internal/constants"
 	"github.com/edgelesssys/constellation/v2/internal/file"
+	"github.com/edgelesssys/constellation/v2/internal/mpimage"
 	"github.com/edgelesssys/constellation/v2/internal/role"
 )
 
@@ -127,7 +128,7 @@ func normalizeAzureURIs(vars *terraform.AzureClusterVariables) *terraform.AzureC
 
 // azureTerraformVars provides variables required to execute the Terraform scripts.
 // It should be the only place to declare the Azure variables.
-func azureTerraformVars(conf *config.Config, imageRef string) *terraform.AzureClusterVariables {
+func azureTerraformVars(conf *config.Config, imageRef string) (*terraform.AzureClusterVariables, error) {
 	nodeGroups := make(map[string]terraform.AzureNodeGroup)
 	for groupName, group := range conf.NodeGroups {
 		zones := strings.Split(group.Zone, ",")
@@ -147,7 +148,6 @@ func azureTerraformVars(conf *config.Config, imageRef string) *terraform.AzureCl
 		Name:                 conf.Name,
 		NodeGroups:           nodeGroups,
 		Location:             conf.Provider.Azure.Location,
-		ImageID:              imageRef,
 		CreateMAA:            toPtr(conf.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})),
 		Debug:                toPtr(conf.IsDebugCluster()),
 		ConfidentialVM:       toPtr(conf.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})),
@@ -158,8 +158,31 @@ func azureTerraformVars(conf *config.Config, imageRef string) *terraform.AzureCl
 		InternalLoadBalancer: conf.InternalLoadBalancer,
 	}
 
+	if conf.UseMarketplaceImage() {
+		image, err := mpimage.NewFromURI(imageRef)
+		if err != nil {
+			return nil, fmt.Errorf("parsing marketplace image URI: %w", err)
+		}
+
+		azureImage, ok := image.(mpimage.AzureMarketplaceImage)
+		if !ok {
+			return nil, fmt.Errorf("expected Azure marketplace image, got %T", image)
+		}
+
+		// If a marketplace image is used, only the marketplace reference is required.
+		vars.MarketplaceImage = terraform.AzureMarketplaceImageVariables{
+			Publisher: azureImage.Publisher,
+			Product:   azureImage.Offer,
+			Name:      azureImage.SKU,
+			Version:   azureImage.Version,
+		}
+	} else {
+		// If not, we need to specify the exact CommunityGalleries/.. image reference.
+		vars.ImageID = imageRef
+	}
+
 	vars = normalizeAzureURIs(vars)
-	return vars
+	return vars, nil
 }
 
 func azureTerraformIAMVars(conf *config.Config, oldVars terraform.AzureIAMVariables) *terraform.AzureIAMVariables {

diff --git a/cli/internal/cmd/apply.go b/cli/internal/cmd/apply.go
@@ -629,7 +629,7 @@ func (a *applyCmd) runNodeImageUpgrade(cmd *cobra.Command, conf *config.Config)
 	provider := conf.GetProvider()
 	attestationVariant := conf.GetAttestationConfig().GetVariant()
 	region := conf.GetRegion()
-	imageReference, err := a.imageFetcher.FetchReference(cmd.Context(), provider, attestationVariant, conf.Image, region)
+	imageReference, err := a.imageFetcher.FetchReference(cmd.Context(), provider, attestationVariant, conf.Image, region, conf.UseMarketplaceImage())
 	if err != nil {
 		return fmt.Errorf("fetching image reference: %w", err)
 	}
@@ -846,6 +846,6 @@ type applier interface {
 type imageFetcher interface {
 	FetchReference(ctx context.Context,
 		provider cloudprovider.Provider, attestationVariant variant.Variant,
-		image, region string,
+		image, region string, useMarketplaceImage bool,
 	) (string, error)
 }
@@ -389,7 +389,7 @@ type stubImageFetcher struct {
 
 func (f *stubImageFetcher) FetchReference(_ context.Context,
 	_ cloudprovider.Provider, _ variant.Variant,
-	_, _ string,
+	_, _ string, _ bool,
 ) (string, error) {
 	return f.reference, f.fetchReferenceErr
 }
@@ -209,6 +209,8 @@ type AzureClusterVariables struct {
 	CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
 	// InternalLoadBalancer is true if an internal load balancer should be created.
 	InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"`
+	// MarketplaceImage is the (optional) Azure Marketplace image to use.
+	MarketplaceImage AzureMarketplaceImageVariables `hcl:"marketplace_image" cty:"marketplace_image"`
 }
 
 // GetCreateMAA gets the CreateMAA variable.
@@ -250,6 +252,18 @@ type AzureIAMVariables struct {
 	ResourceGroup string `hcl:"resource_group_name" cty:"resource_group_name"`
 }
 
+// AzureMarketplaceImageVariables is a configuration for specifying an Azure Marketplace image.
+type AzureMarketplaceImageVariables struct {
+	// Publisher is the publisher ID of the image.
+	Publisher string `hcl:"publisher" cty:"publisher"`
+	// Product is the product ID of the image.
+	Product string `hcl:"product" cty:"product"`
+	// Name is the name of the image.
+	Name string `hcl:"name" cty:"name"`
+	// Version is the version of the image.
+	Version string `hcl:"version" cty:"version"`
+}
+
 // String returns a string representation of the IAM-specific variables, formatted as Terraform variables.
 func (v *AzureIAMVariables) String() string {
 	f := hclwrite.NewEmptyFile()

@@ -193,6 +193,12 @@ func TestAzureClusterVariables(t *testing.T) {
 		Debug:                to.Ptr(true),
 		Location:             "eu-central-1",
 		CustomEndpoint:       "example.com",
+		MarketplaceImage: AzureMarketplaceImageVariables{
+			Publisher: "edgelesssys",
+			Product:   "constellation",
+			Name:      "constellation",
+			Version:   "2.13.0",
+		},
 	}
 
 	// test that the variables are correctly rendered
@@ -216,6 +222,12 @@ node_groups = {
 }
 custom_endpoint        = "example.com"
 internal_load_balancer = false
+marketplace_image = {
+  name      = "constellation"
+  product   = "constellation"
+  publisher = "edgelesssys"
+  version   = "2.13.0"
+}
 `
 	got := vars.String()
 	assert.Equal(t, want, got)

diff --git a/dev-docs/workflows/marketplace-images.md b/dev-docs/workflows/marketplace-images.md
@@ -0,0 +1,27 @@
+# Using Marketplace Images in Constellation
+
+This document explains the steps a user needs to take to run Constellation with dynamic billing via the cloud marketplaces.
+
+## AWS
+
+Marketplace Images on AWS are not available yet.
+
+## Azure
+
+On Azure, to use a marketplace image, ensure that the subscription has accepted the agreement to use marketplace images:
+
+```bash
+az vm image terms accept --publisher edgelesssystems --offer constellation --plan constellation
+```
+
+Then, set the VMs to use the marketplace image in the `constellation-conf.yaml` file:
+
+```bash
+yq eval -i ".provider.azure.useMarketplaceImage = true" constellation-conf.yaml
+```
+
+And ensure that the cluster uses a release image (i.e. `.image=vX.Y.Z` in the `constellation-conf.yaml` file). Afterwards, proceed with the cluster creation as usual.
+
+## GCP
+
+Marketplace Images on GCP are not available yet.
@@ -21,3 +21,7 @@ You are free to use the Constellation binaries provided by Edgeless Systems to c
 Enterprise Licenses don't have the above limitations and come with support and additional features. Find out more at the [product website](https://www.edgeless.systems/products/constellation/).
 
 Once you have received your Enterprise License file, place it in your [Constellation workspace](../architecture/orchestration.md#workspaces) in a file named `constellation.license`.
+
+### Azure Marketplace
+
+Constellation is available through the Azure Marketplace. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your Azure account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/).