8000 Improve `!terraform.output` Atmos YAML function. Implement `static` remote state backend for `!terraform.output` and `atmos.Component` functions by aknysh · Pull Request #863 · cloudposse/atmos · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Improve !terraform.output Atmos YAML function. Implement static remote state backend for !terraform.output and atmos.Component functions #863

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Dec 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/quick-start-advanced/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ ARG GEODESIC_OS=debian
# https://atmos.tools/
# https://github.com/cloudposse/atmos
# https://github.com/cloudposse/atmos/releases
ARG ATMOS_VERSION=1.127.0
ARG ATMOS_VERSION=1.129.0

# Terraform: https://github.com/hashicorp/terraform/releases
ARG TF_VERSION=1.5.7
Expand Down
10000
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,17 @@ components:
test_30: !exec atmos terraform output template-functions-test -s {{ .stack }} --skip-init -- -json test_label_id
test_31: !exec atmos terraform output template-functions-test -s {{ .stack }} --skip-init -- -json test_map
test_32: !exec atmos terraform output template-functions-test -s {{ .stack }} --skip-init -- -json test_list
# Call the `!terraform.output` function with two parameters
test_40: !terraform.output template-functions-test test_label_id
test_41: !terraform.output template-functions-test test_list
test_42: !terraform.output template-functions-test test_map
# Component `template-functions-test3` is configured with the remote state backend of type `static`
test_50: !terraform.output template-functions-test3 val1
test_51: !terraform.output template-functions-test3 {{ .stack }} val1
test_52: !terraform.output template-functions-test3 val2
test_53: !terraform.output template-functions-test3 val3
test_54: !terraform.output template-functions-test3 val4
test_55: !terraform.output template-functions-test3 val5
test_56: !terraform.output template-functions-test3 val6
# test_57: !terraform.output does_not_exist val6
# test_57: !terraform.output template-functions-test3 invalid-val
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# yaml-language-server: $schema=https://atmos.tools/schemas/atmos/atmos-manifest/1.0/atmos-manifest.json

components:
terraform:
template-functions-test3:
remote_state_backend_type: static
remote_state_backend:
static:
val1: true
val2: "2"
val3: 3
val4: null
val5:
- item1
- item2
- item3
val6:
i1: 1
i2: 2
i3: 3
5 changes: 3 additions & 2 deletions examples/tests/stacks/orgs/cp/tenant1/prod/us-east-2.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,9 @@ import:
- catalog/terraform/spacelift/infrastructure-tenant1

# Configurations to test `atmos.Component` template function
# - catalog/terraform/template-functions-test/defaults
# - catalog/terraform/template-functions-test2/defaults
# - catalog/terraform/template-functions-test/defaults
# - catalog/terraform/template-functions-test2/defaults
# - catalog/terraform/template-functions-test3/defaults

components:
terraform:
Expand Down
4 changes: 2 additions & 2 deletions internal/exec/describe_stacks.go
Original file line number Diff line number Diff line change
Expand Up @@ -362,7 +362,7 @@ func ExecuteDescribeStacks(
u.LogErrorAndExit(cliConfig, err)
}

componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted)
componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted, stackName)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -547,7 +547,7 @@ func ExecuteDescribeStacks(
u.LogErrorAndExit(cliConfig, err)
}

componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted)
componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted, stackName)
if err != nil {
return nil, err
}
Expand Down
27 changes: 26 additions & 1 deletion internal/exec/stack_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@ func BuildTerraformWorkspace(cliConfig schema.CliConfiguration, configAndStacksI
return strings.Replace(workspace, "/", "-", -1), nil
}

// ProcessComponentMetadata processes component metadata and returns a base component (if any) and whether the component is real or abstract and whether the component is disabled or not
// ProcessComponentMetadata processes component metadata and returns a base component (if any) and whether
// the component is real or abstract and whether the component is disabled or not
func ProcessComponentMetadata(
component string,
componentSection map[string]any,
Expand Down Expand Up @@ -196,3 +197,27 @@ func IsComponentEnabled(varsSection map[string]any) bool {
}
return true
}

// GetComponentRemoteStateBackendStaticType returns the `remote_state_backend` section for a component in a stack
// if the `remote_state_backend_type` is `static`
func GetComponentRemoteStateBackendStaticType(
sections map[string]any,
) (map[string]any, error) {
var remoteStateBackend map[string]any
var remoteStateBackendType string
var ok bool

if remoteStateBackendType, ok = sections[cfg.RemoteStateBackendTypeSectionName].(string); !ok {
return nil, nil
}

if remoteStateBackendType != "static" {
return nil, nil
}

if remoteStateBackend, ok = sections[cfg.RemoteStateBackendSectionName].(map[string]any); ok {
return remoteStateBackend, nil
}

return nil, nil
}
20 changes: 17 additions & 3 deletions internal/exec/template_funcs_component.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,27 @@ func componentFunc(cliConfig schema.CliConfiguration, component string, stack st
return nil, err
}

outputProcessed, err := execTerraformOutput(cliConfig, component, stack, sections)
var terraformOutputs map[string]any

// Check if the component in the stack is configured with the 'static' remote state backend,
// in which case get the `output` from the static remote state instead of executing `terraform output`
remoteStateBackendStaticTypeOutputs, err := GetComponentRemoteStateBackendStaticType(sections)
if err != nil {
return nil, err
}

if remoteStateBackendStaticTypeOutputs != nil {
terraformOutputs = remoteStateBackendStaticTypeOutputs
} else {
// Execute `terraform output`
terraformOutputs, err = execTerraformOutput(cliConfig, component, stack, sections)
if err != nil {
return nil, err
}
}

outputs := map[string]any{
"outputs": outputProcessed,
"outputs": terraformOutputs,
}

sections = lo.Assign(sections, outputs)
Expand All @@ -60,7 +74,7 @@ func componentFunc(cliConfig schema.CliConfiguration, component string, stack st

if cliConfig.Logs.Level == u.LogLevelTrace {
u.LogTrace(cliConfig, fmt.Sprintf("Executed template function 'atmos.Component(%s, %s)'\n\n'outputs' section:", component, stack))
y, err2 := u.ConvertToYAML(outputProcessed)
y, err2 := u.ConvertToYAML(terraformOutputs)
if err2 != nil {
u.LogError(cliConfig, err2)
} else {
Expand Down
2 changes: 1 addition & 1 deletion internal/exec/terraform_generate_backends.go
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ func ExecuteTerraformGenerateBackends(
u.LogErrorAndExit(cliConfig, err)
}

componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted)
componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted, stackName)
if err != nil {
return err
}
Expand Down
2 changes: 1 addition & 1 deletion internal/exec/terraform_generate_varfiles.go
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ func ExecuteTerraformGenerateVarfiles(
u.LogErrorAndExit(cliConfig, err)
}

componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted)
componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted, stackName)
if err != nil {
return err
}
Expand Down
10 changes: 4 additions & 6 deletions internal/exec/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -398,14 +398,12 @@ func ProcessStacks(
}
}

if foundStackCount == 0 {
if foundStackCount == 0 && !checkStack {
// Allow proceeding without error if checkStack is false (e.g., for operations that don't require a stack)
if !checkStack {
return configAndStacksInfo, nil
}
return configAndStacksInfo, nil
}

if foundStackCount == 0 && configAndStacksInfo.ComponentIsEnabled {
if foundStackCount == 0 {
cliConfigYaml := ""

if cliConfig.Logs.Level == u.LogLevelTrace {
Expand Down Expand Up @@ -521,7 +519,7 @@ func ProcessStacks(
u.LogErrorAndExit(cliConfig, err)
}

componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted)
componentSectionFinal, err := ProcessCustomYamlTags(cliConfig, componentSectionConverted, configAndStacksInfo.Stack)
if err != nil {
return configAndStacksInfo, err
}
Expand Down
6 changes: 5 additions & 1 deletion internal/exec/yaml_func_exec.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@ import (
u "github.com/cloudposse/atmos/pkg/utils"
)

func processTagExec(cliConfig schema.CliConfiguration, input string) any {
func processTagExec(
cliConfig schema.CliConfiguration,
input string,
currentStack string,
) any {
u.LogTrace(cliConfig, fmt.Sprintf("Executing Atmos YAML function: %s", input))

str, err := getStringAfterTag(cliConfig, input, config.AtmosYamlFuncExec)
Expand Down
6 changes: 5 additions & 1 deletion internal/exec/yaml_func_template.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,11 @@ import (
u "github.com/cloudposse/atmos/pkg/utils"
)

func processTagTemplate(cliConfig schema.CliConfiguration, input string) any {
func processTagTemplate(
cliConfig schema.CliConfiguration,
input string,
currentStack string,
) any {
u.LogTrace(cliConfig, fmt.Sprintf("Executing Atmos YAML function: %s", input))

str, err := getStringAfterTag(cliConfig, input, config.AtmosYamlFuncTemplate)
Expand Down
83 changes: 67 additions & 16 deletions internal/exec/yaml_func_terraform_output.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package exec

import (
"errors"
"fmt"
"strings"
"sync"
Expand All @@ -15,26 +14,42 @@ var (
terraformOutputFuncSyncMap = sync.Map{}
)

func processTagTerraformOutput(cliConfig schema.CliConfiguration, input string) any {
func processTagTerraformOutput(
cliConfig schema.CliConfiguration,
input string,
currentStack string,
) any {
u.LogTrace(cliConfig, fmt.Sprintf("Executing Atmos YAML function: %s", input))

str, err := getStringAfterTag(cliConfig, input, config.AtmosYamlFuncTerraformOutput)

if err != nil {
u.LogErrorAndExit(cliConfig, err)
}

parts := strings.Split(str, " ")

if len(parts) != 3 {
err := errors.New(fmt.Sprintf("invalid Atmos YAML function: %s\nthree parameters are required: component, stack, output", input))
var component string
var stack string
var output string

// Split the string into slices based on any whitespace (one or more spaces, tabs, or newlines),
// while also ignoring leading and trailing whitespace
parts := strings.Fields(str)
partsLen := len(parts)

if partsLen == 3 {
component = strings.TrimSpace(parts[0])
stack = strings.TrimSpace(parts[1])
output = strings.TrimSpace(parts[2])
} else if partsLen == 2 {
component = strings.TrimSpace(parts[0])
stack = currentStack
output = strings.TrimSpace(parts[1])
u.LogTrace(cliConfig, fmt.Sprintf("Atmos YAML function `%s` is called with two parameters 'component' and 'output'. "+
"Using the current stack '%s' as the 'stack' parameter", input, currentStack))
} else {
err := fmt.Errorf("invalid number of arguments in the Atmos YAML function: %s", input)
u.LogErrorAndExit(cliConfig, err)
}

component := strings.TrimSpace(parts[0])
stack := strings.TrimSpace(parts[1])
output := strings.TrimSpace(parts[2])

stackSlug := fmt.Sprintf("%s-%s", stack, component)

// If the result for the component in the stack already exists in the cache, return it
Expand All @@ -49,15 +64,28 @@ func processTagTerraformOutput(cliConfig schema.CliConfiguration, input string)
u.LogErrorAndExit(cliConfig, err)
}

outputProcessed, err := execTerraformOutput(cliConfig, component, stack, sections)
// Check if the component in the stack is configured with the 'static' remote state backend,
// in which case get the `output` from the static remote state instead of executing `terraform output`
remoteStateBackendStaticTypeOutputs, err := GetComponentRemoteStateBackendStaticType(sections)
if err != nil {
u.LogErrorAndExit(cliConfig, err)
}

// Cache the result
terraformOutputFuncSyncMap.Store(stackSlug, outputProcessed)

return getTerraformOutput(cliConfig, input, component, stack, outputProcessed, output)
if remoteStateBackendStaticTypeOutputs != nil {
// Cache the result
terraformOutputFuncSyncMap.Store(stackSlug, remoteStateBackendStaticTypeOutputs)
return getStaticRemoteStateOutput(cliConfig, input, component, stack, remoteStateBackendStaticTypeOutputs, output)
} else {
// Execute `terraform output`
terraformOutputs, err := execTerraformOutput(cliConfig, component, stack, sections)
if err != nil {
u.LogErrorAndExit(cliConfig, err)
}

// Cache the result
terraformOutputFuncSyncMap.Store(stackSlug, terraformOutputs)
return getTerraformOutput(cliConfig, input, component, stack, terraformOutputs, output)
}
}

func getTerraformOutput(
Expand All @@ -81,3 +109,26 @@ func getTerraformOutput(

return nil
}

func getStaticRemoteStateOutput(
cliConfig schema.CliConfiguration,
funcDef string,
component string,
stack string,
remoteStateSection map[string]any,
output string,
) any {
if u.MapKeyExists(remoteStateSection, output) {
return remoteStateSection[output]
}

u.LogErrorAndExit(cliConfig, fmt.Errorf("invalid Atmos YAML function: %s\nthe component '%s' in the stack '%s' "+
"is configured with the 'static' remote state backend, but the remote state backend does not have the output '%s'",
funcDef,
component,
stack,
output,
))

return nil
}
Loading
Loading
0