8000 Support more processors in logs pipeline by tt810 · Pull Request #283 · zorkian/go-datadog-api · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

Support more processors in logs pipeline #283

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 8, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 124 additions & 0 deletions datadog-accessors.go
Original file line number Diff line number Diff line change
Expand Up @@ -6679,6 +6679,37 @@ func (f *FreeTextDefinition) SetType(v string) {
f.Type = &v
}

// GetTarget returns the Target field if non-nil, zero value otherwise.
func (g *GeoIPParser) GetTarget() string {
if g == nil || g.Target == nil {
return ""
}
return *g.Target
}

// GetTargetOk returns a tuple with the Target field if it's non-nil, zero value otherwise
// and a boolean to check if the value has been set.
func (g *GeoIPParser) GetTargetOk() (string, bool) {
if g == nil || g.Target == nil {
return "", false
}
return *g.Target, true
}

// HasTarget returns a boolean if a field has been set.
func (g *GeoIPParser) HasTarget() bool {
if g != nil && g.Target != nil {
return true
}

return false
}

// SetTarget allocates a new g.Target and returns the pointer to it.
func (g *GeoIPParser) SetTarget(v string) {
g.Target = &v
}

// GetDefinition returns the Definition field if non-nil, zero value otherwise.
func (g *Graph) GetDefinition() GraphDefinition {
if g == nil || g.Definition == nil {
Expand Down Expand Up @@ -17808,6 +17839,99 @@ func (s *ServicePDRequest) SetServiceName(v string) {
s.ServiceName = &v
}

// GetIsReplaceMissing returns the IsReplaceMissing field if non-nil, zero value otherwise.
func (s *StringBuilderProcessor) GetIsReplaceMissing() bool {
if s == nil || s.IsReplaceMissing == nil {
return false
}
return *s.IsReplaceMissing
}

// GetIsReplaceMissingOk returns a tuple with the IsReplaceMissing field if it's non-nil, zero value otherwise
// and a boolean to check if the value has been set.
func (s *StringBuilderProcessor) GetIsReplaceMissingOk() (bool, bool) {
if s == nil || s.IsReplaceMissing == nil {
return false, false
}
return *s.IsReplaceMissing, true
}

// HasIsReplaceMissing returns a boolean if a field has been set.
func (s *StringBuilderProcessor) HasIsReplaceMissing() bool {
if s != nil && s.IsReplaceMissing != nil {
return true
}

return false
}

// SetIsReplaceMissing allocates a new s.IsReplaceMissing and returns the pointer to it.
func (s *StringBuilderProcessor) SetIsReplaceMissing(v bool) {
s.IsReplaceMissing = &v
}

// GetTarget returns the Target field if non-nil, zero value otherwise.
func (s *StringBuilderProcessor) GetTarget() string {
if s == nil || s.Target == nil {
return ""
}
return *s.Target
}

// GetTargetOk returns a tuple with the Target field if it's non-nil, zero value otherwise
// and a boolean to check if the value has been set.
func (s *StringBuilderProcessor) GetTargetOk() (string, bool) {
if s == nil || s.Target == nil {
return "", false
}
return *s.Target, true
}

// HasTarget returns a boolean if a field has been set.
func (s *StringBuilderProcessor) HasTarget() bool {
if s != nil && s.Target != nil {
return true
}

return false
}

// SetTarget allocates a new s.Target and returns the pointer to it.
func (s *StringBuilderProcessor) SetTarget(v string) {
s.Target = &v
}

// GetTemplate returns the Template field if non-nil, zero value otherwise.
func (s *StringBuilderProcessor) GetTemplate() string {
if s == nil || s.Template == nil {
return ""
}
return *s.Template
}

// GetTemplateOk returns a tuple with the Template field if it's non-nil, zero value otherwise
// and a boolean to check if the value has been set.
func (s *StringBuilderProcessor) GetTemplateOk() (string, bool) {
if s == nil || s.Template == nil {
return "", false
}
return *s.Template, true
}

// HasTemplate returns a boolean if a field has been set.
func (s *StringBuilderProcessor) HasTemplate() bool {
if s != nil && s.Template != nil {
return true
}

return false
}

// SetTemplate allocates a new s.Template and returns the pointer to it.
func (s *StringBuilderProcessor) SetTemplate(v string) {
s.Template = &v
}

// GetFillMax returns the FillMax field if non-nil, zero value otherwise.
func (s *Style) GetFillMax() json.Number {
if s == nil || s.FillMax == nil {
Expand Down
17 changes: 17 additions & 0 deletions integration/logs_pipelines_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,23 @@ func TestLogsPipelineCrud(t *testing.T) {
},
},
},
}, {
Name: datadog.String("test string builder processor"),
IsEnabled: datadog.Bool(true),
Type: datadog.String("string-builder-processor"),
Definition: datadog.StringBuilderProcessor{
Template: datadog.String("hello %{user.name}"),
IsReplaceMissing: datadog.Bool(false),
Target: datadog.String("target"),
},
}, {
Name: datadog.String("geo ip parser test"),
IsEnabled: datadog.Bool(false),
Type: datadog.String("geo-ip-parser"),
Definition: datadog.GeoIPParser{
Sources: []string{"source1", "source2"},
Target: datadog.String("target"),
},
},
},
})
Expand Down
20 changes: 19 additions & 1 deletion logs_pipelines_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ var expectedPipeline = &LogsPipeline{
IsEnabled: Bool(true),
Type: String("grok-parser"),
Definition: GrokParser{
Source: String("text"),
Source: String("text"),
Samples: []string{"sample1", "sample2"},
GrokRule: &GrokRule{
SupportRules: String("date_parser %{date(\"yyyy-MM-dd HH:mm:ss,SSS\"):timestamp}"),
MatchRules: String("rule %{date(\"yyyy-MM-dd HH:mm:ss,SSS\"):timestamp}"),
Expand Down Expand Up @@ -157,6 +158,23 @@ var expectedPipeline = &LogsPipeline{
},
},
},
}, {
Name: String("test string builder processor"),
IsEnabled: Bool(true),
Type: String("string-builder-processor"),
Definition: StringBuilderProcessor{
Template: String("hello %{user.name}"),
IsReplaceMissing: Bool(false),
Target: String("target"),
},
}, {
Name: String("geo ip parser test"),
IsEnabled: Bool(false),
Type: String("geo-ip-parser"),
Definition: GeoIPParser{
Sources: []string{"source1", "source2"},
Target: String("target"),
},
},
},
}
51 changes: 39 additions & 12 deletions logs_processors.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,20 @@ import (
)

const (
ArithmeticProcessorType = "arithmetic-processor"
AttributeRemapperType = "attribute-remapper"
CategoryProcessorType = "category-processor"
DateRemapperType = "date-remapper"
GrokParserType = "grok-parser"
MessageRemapperType = "message-remapper"
NestedPipelineType = "pipeline"
ServiceRemapperType = "service-remapper"
StatusRemapperType = "status-remapper"
TraceIdRemapperType = "trace-id-remapper"
UrlParserType = "url-parser"
UserAgentParserType = "user-agent-parser"
ArithmeticProcessorType = "arithmetic-processor"
AttributeRemapperType = "attribute-remapper"
CategoryProcessorType = "category-processor"
DateRemapperType = "date-remapper"
GeoIPParserType = "geo-ip-parser"
GrokParserType = "grok-parser"
MessageRemapperType = "message-remapper"
NestedPipelineType = "pipeline"
ServiceRemapperType = "service-remapper"
StatusRemapperType = "status-remapper"
StringBuilderProcessorType = "string-builder-processor"
TraceIdRemapperType = "trace-id-remapper"
UrlParserType = "url-parser"
UserAgentParserType = "user-agent-parser"
)

// LogsProcessor struct represents the processor object from Config API.
Expand Down Expand Up @@ -66,9 +68,22 @@ type SourceRemapper struct {
Sources []string `json:"sources"`
}

// GeoIPParser represents geoIpParser object from config API.
type GeoIPParser struct {
Sources []string `json:"sources"`
Target *string `json:"target"`
}

type StringBuilderProcessor struct {
Template *string `json:"template"`
Target *string `json:"target"`
IsReplaceMissing *bool `json:"is_replace_missing"`
}

// GrokParser represents the grok parser processor object from config API.
type GrokParser struct {
Source *string `json:"source"`
Samples []string `json:"samples"`
GrokRule *GrokRule `json:"grok"`
}

Expand Down Expand Up @@ -173,6 +188,12 @@ func (processor *LogsProcessor) UnmarshalJSON(data []byte) error {
return err
}
processor.Definition = sourceRemapper
case GeoIPParserType:
var geoIPParser GeoIPParser
if err := json.Unmarshal(data, &geoIPParser); err != nil {
return err
}
processor.Definition = geoIPParser
case GrokParserType:
var grokParser GrokParser
if err := json.Unmarshal(data, &grokParser); err != nil {
Expand All @@ -185,6 +206,12 @@ func (processor *LogsProcessor) UnmarshalJSON(data []byte) error {
return err
}
processor.Definition = nestedPipeline
case StringBuilderProcessorType:
var stringBuilder StringBuilderProcessor
if err := json.Unmarshal(data, &stringBuilder); err != nil {
return err
}
processor.Definition = stringBuilder
case UrlParserType:
var urlParser UrlParser
if err := json.Unmarshal(data, &urlParser); err != nil {
Expand Down
16 changes: 16 additions & 0 deletions tests/fixtures/logs/pipeline_response.json
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
"name": "test grok parser",
"is_enabled": true,
"source": "text",
"samples": ["sample1", "sample2"],
"grok": {
"support_rules": "date_parser %{date(\"yyyy-MM-dd HH:mm:ss,SSS\"):timestamp}",
"match_rules": "rule %{date(\"yyyy-MM-dd HH:mm:ss,SSS\"):timestamp}"
Expand Down Expand Up @@ -133,6 +134,21 @@
],
"target": "test_category",
"type": "category-processor"
},
{
"name": "test string builder processor",
"is_enabled": true,
"template": "hello %{user.name}",
"is_replace_missing": false,
"target": "target",
"type": "string-builder-processor"
},
{
"name": "geo ip parser test",
"is_enabled": false,
"sources": ["source1", "source2"],
"target": "target",
"type": "geo-ip-parser"
}
]
}
0