blackstork-io · traut · Mar 15, 2025 · Jan 12, 2025 · Jan 22, 2025 · Mar 13, 2025
diff --git a/cmd/data.go b/cmd/data.go
@@ -15,14 +15,17 @@ import (
 func init() {
 	rootCmd.AddCommand(dataCmd)
 	dataCmd.SetUsageTemplate(UsageTemplate(
-		[2]string{"TARGET", "a path to the data block to be executed. Data block must be inside of a document, so the path would look lile 'document.<doc-name>.data.<plugin-name>.<data-name>'"},
+		[2]string{
+			"PATH",
+			"a path to data blocks to be executed. The path format is 'document.<doc-name>.data[.<plugin-name>[.<data-name>]]'.",
+		},
 	))
 }
 
 var dataCmd = &cobra.Command{
 	Use:   "data TARGET",
-	Short: "Execute a single data block",
-	Long:  `Execute the data block and print out prettified JSON to stdout`,
+	Short: "Execute the data blocks that match the path",
+	Long:  `Execute the data blocks that match the path and print out prettified JSON to stdout`,
 	Args:  cobra.ExactArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) (err error) {
 		ctx := cmd.Context()

diff --git a/docs/plugins/plugins.json b/docs/plugins/plugins.json
@@ -130,14 +130,14 @@
       },
       {
         "name": "sleep",
-        "type": "data-source",
+        "type": "content-provider",
         "arguments": [
           "duration"
         ]
       },
       {
         "name": "sleep",
-        "type": "content-provider",
+        "type": "data-source",
         "arguments": [
           "duration"
         ]

diff --git a/engine/engine.go b/engine/engine.go
@@ -9,7 +9,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"slices"
 	"strings"
 
 	"github.com/hashicorp/hcl/v2"
@@ -195,7 +194,14 @@ func (e *Engine) LoadPluginResolver(ctx context.Context, includeRemote bool) (di
 		resolver.NewLocal(pluginDir, e.logger, e.tracer),
 	}
 
-	e.logger.DebugContext(ctx, "Loading plugin resolver", "include_remote", includeRemote, "plugins_dir", string(pluginDir))
+	e.logger.DebugContext(
+		ctx,
+		"Loading plugin resolver",
+		"include_remote",
+		includeRemote,
+		"plugins_dir",
+		string(pluginDir),
+	)
 
 	if e.config.PluginRegistry != nil {
 		if e.config.PluginRegistry.MirrorDir != "" {
@@ -204,7 +210,10 @@ func (e *Engine) LoadPluginResolver(ctx context.Context, includeRemote bool) (di
 				return diagnostics.Diag{{
 					Severity: hcl.DiagError,
 					Summary:  "Can't find a mirror directory",
-					Detail:   fmt.Sprintf("Can't find a directory specified as a mirror: %s", e.config.PluginRegistry.MirrorDir),
+					Detail: fmt.Sprintf(
+						"Can't find a directory specified as a mirror: %s",
+						e.config.PluginRegistry.MirrorDir,
+					),
 				}}
 			}
 			sources = append(sources, resolver.NewLocal(e.config.PluginRegistry.MirrorDir, e.logger, e.tracer))
@@ -258,12 +267,15 @@ func (e *Engine) PrintDiagnostics(output io.Writer, diags diagnostics.Diag, colo
 	diagnostics.PrintDiags(output, diags, e.fileMap, colorize)
 }
 
-func (e *Engine) loadGlobalData(ctx context.Context, source, name string) (_ *eval.PluginDataAction, diags diagnostics.Diag) {
+func (e *Engine) loadGlobalData(
+	ctx context.Context,
+	source, name string,
+) (_ *eval.PluginDataAction, diags diagnostics.Diag) {
 	ctx, span := e.tracer.Start(ctx, "Engine.loadGlobalData", trace.WithAttributes(
-		attribute.String("datasource", source),
+		attribute.String("data_source", source),
 		attribute.String("name", name),
 	))
-	e.logger.InfoContext(ctx, "Loading global data", "datasource", source, "name", name)
+	e.logger.InfoContext(ctx, "Loading global data", "data_source", source, "name", name)
 	defer func() {
 		if diags.HasErrors() {
 			span.RecordError(diags)
@@ -308,13 +320,18 @@ func (e *Engine) loadGlobalData(ctx context.Context, source, name string) (_ *ev
 	return loadedData, diags
 }
 
-func (e *Engine) loadDocumentData(ctx context.Context, doc, source, name string) (_ *eval.PluginDataAction, diags diagnostics.Diag) {
+func (e *Engine) loadDocumentData(
+	ctx context.Context,
+	doc string,
+	path []string,
+) (_ plugindata.Data, diags diagnostics.Diag) {
+	pathStr := strings.Join(path, ".")
+
 	ctx, span := e.tracer.Start(ctx, "Engine.loadDocumentData", trace.WithAttributes(
 		attribute.String("document", doc),
-		attribute.String("datasource", source),
-		attribute.String("name", name),
+		attribute.String("data_path", pathStr),
 	))
-	e.logger.InfoContext(ctx, "Loading document data", "document", doc, "datasource", source, "name", name)
+	e.logger.InfoContext(ctx, "Loading document data", "document", doc, "data_path", path)
 	defer func() {
 		if diags.HasErrors() {
 			span.RecordError(
8000
diags)
@@ -326,14 +343,14 @@ func (e *Engine) loadDocumentData(ctx context.Context, doc, source, name string)
 		return nil, diagnostics.Diag{{
 			Severity: hcl.DiagError,
 			Summary:  "No files parsed",
-			Detail:   "Parse files before selecting",
+			Detail:   "Parse files before selecting a path",
 		}}
 	}
 	if e.runner == nil {
 		return nil, diagnostics.Diag{{
 			Severity: hcl.DiagError,
 			Summary:  "Plugin runner is not loaded",
-			Detail:   "Load plugin runner before evaluating",
+			Detail:   "Load plugin runner before evaluating the template",
 		}}
 	}
 	docBlock, ok := e.blocks.Documents[doc]
@@ -344,25 +361,22 @@ func (e *Engine) loadDocumentData(ctx context.Context, doc, source, name string)
 			Detail:   fmt.Sprintf("Definition for document named '%s' not found", doc),
 		}}
 	}
+	e.logger.DebugContext(ctx, "Parsing a document template")
 	docParsed, diag := e.blocks.ParseDocument(ctx, docBlock)
 	if diags.Extend(diag) {
 		return nil, diags
 	}
-	idx := slices.IndexFunc(docParsed.Data, func(p *definitions.ParsedPlugin) bool {
-		return p.PluginName == source && p.BlockName == name
-	})
-	if idx < 0 {
-		return nil, diagnostics.Diag{{
-			Severity: hcl.DiagError,
-			Summary:  "Data source not found",
-			Detail:   fmt.Sprintf("Data source named '%s' not found", name),
-		}}
+
+	document, diag := eval.LoadDocument(ctx, e.runner, docParsed)
+	if diags.Extend(diag) {
+		return nil, diags
 	}
-	loadedData, diag := eval.LoadDataAction(ctx, e.runner, docParsed.Data[idx])
+
+	data, diag := document.FetchDataWithPath(ctx, path)
 	if diags.Extend(diag) {
 		return nil, diags
 	}
-	return loadedData, diags
+	return data, diags
 }
 
 var ErrInvalidDataTarget = diagnostics.Diag{{
@@ -371,11 +385,11 @@ var ErrInvalidDataTarget = diagnostics.Diag{{
 	Detail:   "Target must be in the format 'document.<doc-name>.data.<plugin-name>.<block-name>' or 'data.<plugin-name>.<block-name>'",
 }}
 
-func (e *Engine) FetchData(ctx context.Context, target string) (_ plugindata.Data, diags diagnostics.Diag) {
+func (e *Engine) FetchData(ctx context.Context, target string) (result plugindata.Data, diags diagnostics.Diag) {
 	ctx, span := e.tracer.Start(ctx, "Engine.FetchData", trace.WithAttributes(
 		attribute.String("target", target),
 	))
-	e.logger.InfoContext(ctx, "Fetching data", "target", target)
+	e.logger.InfoContext(ctx, "Fetching the data", "target", target)
 	defer func() {
 		if diags.HasErrors() {
 			span.RecordError(diags)
@@ -391,27 +405,43 @@ func (e *Engine) FetchData(ctx context.Context, target string) (_ plugindata.Dat
 	var diag diagnostics.Diag
 	switch head {
 	case "document":
+		// Possible options:
+		// - `<document-name>.data`
+		// - `<document-name>.data.<data-source>`
+		// - `<document-name>.data.<data-source>.<block-name>`
 		parts := strings.Split(base, ".")
-		if len(parts) != 4 {
+		// At the minimum, `<document-name>.data`
+		if len(parts) < 2 {
 			return nil, ErrInvalidDataTarget
 		}
 		if parts[1] != "data" {
 			return nil, ErrInvalidDataTarget
 		}
-		loadedData, diag = e.loadDocumentData(ctx, parts[0], parts[2], parts[3])
+		docName := parts[0]
+		path := parts[2:]
+
+		result, diag = e.loadDocumentData(ctx, docName, path)
+		// Wrap the result, to have `data` root key
+		result = plugindata.Map{"data": result}
+
 	case "data":
 		parts := strings.Split(base, ".")
 		if len(parts) != 2 {
 			return nil, ErrInvalidDataTarget
 		}
 		loadedData, diag = e.loadGlobalData(ctx, parts[0], parts[1])
+		if diags.Extend(diag) {
+			return nil, diags
+		}
+		result, diag = loadedData.FetchData(ctx)
 	default:
 		return nil, ErrInvalidDataTarget
 	}
 	if diags.Extend(diag) {
 		return nil, diags
 	}
-	return loadedData.FetchData(ctx)
+
+	return result, nil
 }
 
 func (e *Engine) loadEnv(ctx context.Context) (envMap plugindata.Map, diags diagnostics.Diag) {
@@ -443,7 +473,11 @@ func (e *Engine) initialDataCtx(ctx context.Context) (data plugindata.Map, diags
 	return
 }
 
-func (e *Engine) RenderContent(ctx context.Context, target string, requiredTags []string) (doc *eval.Document, content *plugin.ContentSection, data plugindata.Data, diags diagnostics.Diag) {
+func (e *Engine) RenderContent(
+	ctx context.Context,
+	target string,
+	requiredTags []string,
+) (doc *eval.Document, content *plugin.ContentSection, data plugindata.Data, diags diagnostics.Diag) {
 	ctx, span := e.tracer.Start(ctx, "Engine.RenderContent", trace.WithAttributes(
 		attribute.String("target", target),
 	))
@@ -479,7 +513,13 @@ func (e *Engine) RenderContent(ctx context.Context, target string, requiredTags
 	return doc, content, data, diags
 }
 
-func (e *Engine) PublishContent(ctx context.Context, target string, doc *eval.Document, content *plugin.ContentSection, dataCtx plugindata.Data) (diags diagnostics.Diag) {
+func (e *Engine) PublishContent(
+	ctx context.Context,
+	target string,
+	doc *eval.Document,
+	content *plugin.ContentSection,
+	dataCtx plugindata.Data,
+) (diags diagnostics.Diag) {
 	ctx, span := e.tracer.Start(ctx, "Engine.Publish", trace.WithAttributes(
 		attribute.String("target", target),
 	))

diff --git a/engine/engine_test.go b/engine/engine_test.go
@@ -23,15 +23,59 @@ func TestEngineFetchData(t *testing.T) {
 					path = "testdata/a.json"
 				}
 
+				data json "test2" {
+					path = "testdata/a.json"
+				}
+
 				content text {
-					text = "hello"
+					value = "hello"
 				}
 			}
 			`,
 		},
 		"document.hello.data.json.test",
 		plugindata.Map{
-			"property_for": plugindata.String("a.json"),
+			"data": plugindata.Map{
+				"json": plugindata.Map{
+					"test": plugindata.Map{
+						"property_for": plugindata.String("a.json"),
+					},
+				},
+			},
+		},
+		[][]diagtest.Assert{},
+	)
+	fetchDataTest(
+		t, "Basic",
+		[]string{
+			`
+			document "hello" {
+				data json "test" {
+					path = "testdata/a.json"
+				}
+
+				data json "test2" {
+					path = "testdata/a.json"
+				}
+
+				content text {
+					value = "hello"
+				}
+			}
+			`,
+		},
+		"document.hello.data.json",
+		plugindata.Map{
+			"data": plugindata.Map{
+				"json": plugindata.Map{
+					"test": plugindata.Map{
+						"property_for": plugindata.String("a.json"),
+					},
+					"test2": plugindata.Map{
+						"property_for": plugindata.String("a.json"),
+					},
+				},
+			},
 		},
 		[][]diagtest.Assert{},
 	)
@@ -44,7 +88,7 @@ func TestEngineFetchData(t *testing.T) {
 			}
 			document "hello" {
 				content text {
-					text = "hello"
+					value = "hello"
 				}
 			}
 			`,

diff --git a/eval/document.go b/eval/document.go
@@ -28,6 +28,11 @@ func (doc *Document) FetchData(ctx context.Context) (plugindata.Data, diagnostic
 	return evaluator.Execute()
 }
 
+func (doc *Document) FetchDataWithPath(ctx context.Context, path []string) (plugindata.Data, diagnostics.Diag) {
+	evaluator := makeAsyncDataEvaluatorWithPath(ctx, doc, path, slog.Default())
+	return evaluator.Execute()
+}
+
 func filterChildrenByTags(children []*Content, requiredTags []string) []*Content {
 	return slices.DeleteFunc(children, func(child *Content) bool {
 		switch {