diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..8788be99 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,6 @@ +# Raising a PR against the agent spec folder will automatically request reviews from all agent teams +# If the PR is not ready for review, create a draft PR instead +# See also /.github/pull_request_template.md +/specs/agents @elastic/ingest-otel-data @elastic/apm-agent-java @elastic/apm-agent-net @elastic/apm-agent-node-js @elastic/apm-agent-php @elastic/apm-agent-python @elastic/apm-agent-ruby @elastic/apm-agent-rum @elastic/apm-pm @elastic/apm-agent-ios @elastic/apm-agent-android +/specs/agents/mobile @elastic/apm-agent-ios @elastic/apm-agent-android +/.github/pull_request_template.md @elastic/ingest-otel-data @elastic/apm-agent-java @elastic/apm-agent-net @elastic/apm-agent-node-js @elastic/apm-agent-php @elastic/apm-agent-python @elastic/apm-agent-ruby @elastic/apm-agent-rum @elastic/apm-pm diff --git a/.github/ISSUE_TEMPLATE/agents-discussion.md b/.github/ISSUE_TEMPLATE/agents-discussion.md new file mode 100644 index 00000000..9d6ee3c5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/agents-discussion.md @@ -0,0 +1,12 @@ +--- +name: Agents discussion +about: \ + Open a draft PR to change the specification to initiate a discussion. + If discussion is required before a spec change proposal can even be assembled, create an Agent discussion issue first. +labels: agents, discussion +--- + + \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/agents-poll.md b/.github/ISSUE_TEMPLATE/agents-poll.md deleted file mode 100644 index 253a01f4..00000000 --- a/.github/ISSUE_TEMPLATE/agents-poll.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -name: Agents poll -about: Creates a poll for agent owners to vote on -labels: agents, poll ---- - -## Description of the issue - -## What we are voting on - -## Vote - -| Agent | Yes | No | Indifferent | N/A | Link to agent issue -| --------|:----:|:---:|:-----------:|:----:|:-------------------:| -| .NET ||||| -| Go ||||| -| Java ||||| -| Node.js ||||| -| Python ||||| -| Ruby ||||| -| RUM ||||| diff --git a/.github/ISSUE_TEMPLATE/apm-agents-meta.md b/.github/ISSUE_TEMPLATE/apm-agents-meta.md new file mode 100644 index 00000000..495f1a9f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/apm-agents-meta.md @@ -0,0 +1,37 @@ +--- +name: Meta issue for APM agents. +about: Template for a meta issue for APM agents that triggers creation of related agents sub-issues. +labels: meta, apm-agents +--- + +# Description +Put the issue description here ... + + + + +# Issues creation + +## Create Spec issue? +- [x] Spec issue + +## Create APM Agent issues? +- [x] elastic-otel-java +- [x] elastic-otel-dotnet +- [x] elastic-otel-node +- [x] elastic-otel-python +- [x] elastic-otel-php +- [ ] apm-agent-java +- [ ] apm-agent-dotnet +- [ ] apm-agent-nodejs +- [ ] apm-agent-python +- [ ] apm-agent-go +- [ ] apm-agent-php +- [ ] apm-agent-ruby + + diff --git a/.github/ISSUE_TEMPLATE/design_issue.md b/.github/ISSUE_TEMPLATE/design_issue.md index 5a6a714a..ff2c3185 100644 --- a/.github/ISSUE_TEMPLATE/design_issue.md +++ b/.github/ISSUE_TEMPLATE/design_issue.md @@ -5,20 +5,12 @@ about: Request UI/UX help for this project --- **Summary of the problem** (If there are multiple problems or use cases, prioritize them) -For example: as I user I want to see if one of my sites is down. Which one is down? Since when is it down? +For example: as I user I want to quickly identify if one of my sites is down -**Ideal solution** (optional) -For example: the ideal solution would provide an availability summary and detailed status information of my site. +**User stories** +For example: as an admin, I can create teams and invite new members **List known (technical) restrictions and requirements** For example: has to be scalable from 0-15k containers -**Are there any pages or actions that relate to this feature?** -For example: link to other solutions for further investigation - -- Refer to any related/depending issues -- If this is already scheduled add the appropriate version label -- Make sure the `design` label is added - - If in doubt, don’t hesitate to reach out to the `#observability-design` Slack channel. diff --git a/.github/ISSUE_TEMPLATE/new-fields-issue.md b/.github/ISSUE_TEMPLATE/new-fields-issue.md new file mode 100644 index 00000000..b83ea732 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/new-fields-issue.md @@ -0,0 +1,46 @@ +--- +name: New Metadata/Context Fields +about: Template for proposing new metadata/context fields +labels: agents, poll +--- + + + +## Proposed fields + +Add optional fields to +- [ ] _span context_ +- [ ] _transaction context_ + +as always, this should also be added to the _error context_. + +| Intake API field | Elasticsearch field | Elasticsearch Type | +| -----------------|---------------------|-----------------------| +| | | | + +## JSON Schema + +``` + +``` + +## Vote + + + +| Agent | Yes | No | Indifferent | N/A | Link to issue +| --------|:----:|:---:|:-----------:|:----:|:-------------------:| +| UI ||||| +| Server ||||| +| .NET ||||| +| Go ||||| +| Java ||||| +| Node.js ||||| +| Python ||||| +| Ruby ||||| +| RUM ||||| diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..b9a9834d --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +--- +version: 2 +updates: + - package-ecosystem: "github-actions" + directories: + - "/" + - "/.github/actions/*" + schedule: + interval: "weekly" + day: "sunday" + time: "22:00" + groups: + github-actions: + patterns: + - "*" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..da8051d4 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,47 @@ + + + + +- [ ] Create PR as draft +- [ ] Approval by at least one other agent +- [ ] Mark as Ready for Review (automatically requests reviews from all agents and PM via [`CODEOWNERS`](https://github.com/elastic/apm/tree/main/.github/CODEOWNERS)) + - Remove PM from reviewers if impact on product is negligible + - Remove agents from reviewers if the change is not relevant for them +- [ ] Merge after 2 business days passed without objections \ + To auto-merge the PR, add /`schedule YYYY-MM-DD` to the PR description. + + + +- May the instrumentation collect sensitive information, such as secrets or PII (ex. in headers)? + - [ ] Yes + - [ ] Add a section to the spec how agents should apply sanitization (such as `sanitize_field_names`) + - [ ] No + - [ ] Why? + - [ ] n/a +- [ ] Create PR as draft +- [ ] Approval by at least one other agent +- [ ] Mark as Ready for Review (automatically requests reviews from all agents and PM via [`CODEOWNERS`](https://github.com/elastic/apm/tree/main/.github/CODEOWNERS)) + - Remove PM from reviewers if impact on product is negligible + - Remove agents from reviewers if the change is not relevant for them +- [ ] Approved by at least 2 agents + PM (if relevant) +- [ ] Merge after 7 days passed without objections \ + To auto-merge the PR, add /`schedule YYYY-MM-DD` to the PR description. +- [ ] [Create implementation issues through the meta issue template](https://github.com/elastic/apm/issues/new?assignees=&labels=meta%2C+apm-agents&template=apm-agents-meta.md) (this will automate issue creation for individual agents) +- [ ] If this spec adds a new dynamic config option, [add it to central config](https://github.com/elastic/apm/blob/main/specs/agents/configuration.md#adding-a-new-configuration-option). diff --git a/.github/workflows/apm-agent-meta-issue-action.yml b/.github/workflows/apm-agent-meta-issue-action.yml new file mode 100644 index 00000000..b557bf2d --- /dev/null +++ b/.github/workflows/apm-agent-meta-issue-action.yml @@ -0,0 +1,42 @@ +name: "APM Agents meta issue handler" +on: + issues: + types: [opened] + +permissions: + contents: read + +jobs: + meta-issue-handler: + runs-on: ubuntu-latest + steps: + - name: Get token + id: get_token + uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0 + with: + app_id: ${{ secrets.OBS_AUTOMATION_APP_ID }} + private_key: ${{ secrets.OBS_AUTOMATION_APP_PEM }} + permissions: >- + { + "issues": "write", + "members": "read" + } + - name: Check team membership for user + uses: elastic/get-user-teams-membership@1.1.0 + id: checkUserMember + with: + username: ${{ github.actor }} + team: 'observability' + usernamesToExclude: | + apmmachine + GITHUB_TOKEN: ${{ steps.get_token.outputs.token }} + - name: Create sub issues + if: steps.checkUserMember.outputs.isTeamMember == 'true' && contains(github.event.issue.labels.*.name, 'meta') && contains(github.event.issue.labels.*.name, 'apm-agents') + uses: elastic/gh-action-meta-subissues-creator@1.0.2 + id: create_sub_issues + with: + token: ${{ steps.get_token.outputs.token }} + metaIssue: "${{ toJSON(github.event.issue) }}" + bodyRegex: "(.*)(.*)(.*)" + labelsToExclude: "meta,apm-agents" + specLabels: "spec-poc,apm-agents" diff --git a/.github/workflows/generate-plantuml.yml b/.github/workflows/generate-plantuml.yml new file mode 100644 index 00000000..67f805ef --- /dev/null +++ b/.github/workflows/generate-plantuml.yml @@ -0,0 +1,31 @@ +name: Generate PlantUML Diagrams +on: + push: + paths: + - "**.puml" + +permissions: + contents: write + +jobs: + plantuml: + runs-on: ubuntu-latest + steps: + - name: Checkout Source + uses: actions/checkout@v4 + + - name: Get changed UML files + id: getfile + run: | + echo "::set-output name=files::$(git diff --name-only HEAD^1 HEAD | grep .puml | xargs)" + + - name: Generate SVG Diagrams + uses: holowinski/plantuml-github-action@5ef932f0db4aa76e232fbf19f440248dd102b1d3 + with: + args: -v -tsvg ${{steps.getfile.outputs.files}} + + - name: Push Local Changes + uses: stefanzweifel/git-auto-commit-action@v5.0.1 + with: + commit_message: "Generate SVG files for PlantUML diagrams" + branch: ${{ github.head_ref }} diff --git a/.github/workflows/merge-schedule.yml b/.github/workflows/merge-schedule.yml new file mode 100644 index 00000000..ff7501b8 --- /dev/null +++ b/.github/workflows/merge-schedule.yml @@ -0,0 +1,27 @@ +name: Merge Schedule +on: + pull_request_target: + types: + - opened + - edited + - synchronize + schedule: + # At 6pm EOB + - cron: 0 18 * * * + +permissions: + contents: write + +jobs: + merge_schedule: + runs-on: ubuntu-latest + steps: + - uses: gr2m/merge-schedule-action@v2 + with: + # Merge method to use. Possible values are merge, squash or + # rebase. Default is merge. + merge_method: squash + # Time zone to use. Default is UTC. + time_zone: "America/Los_Angeles" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..e43b0f98 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.DS_Store diff --git a/README.md b/README.md index d6aaf6c0..214a871f 100644 --- a/README.md +++ b/README.md @@ -1,54 +1,71 @@ -

Elastic APM Elastic APM

+

Elastic APM Elastic APM

This project includes resources and general issue tracking for Elastic APM.

+Help us make Elastic APM better by sharing your experience with Elastic Product Management. [Schedule a 45-minute session](https://calendly.com/elasticapm-chris) with Elastic Product Management and share your feedback. + ## What we do… | Project | Repo | Docs | Contrib | :- | :- | :- | :- | APM Server | [apm-server][] | [πŸ“˜ elastic.co][apm-server-docs] | [πŸ“‚ contrib][apm-server-contrib] | APM UI | [apm-ui][] | [πŸ“˜ elastic.co][apm-ui-docs] | [πŸ“‚ contrib][apm-ui-contrib] +| Android Agent (🚧 In Development) | [apm-agent-android][] | [πŸ“˜ elastic.co][apm-agent-android-docs] | [πŸ“‚ contrib][apm-agent-android-contrib] | Go Agent | [apm-agent-go][] | [πŸ“˜ elastic.co][apm-agent-go-docs] | [πŸ“‚ contrib][apm-agent-go-contrib] +| iOS Agent (🚧 In Development) | [apm-agent-ios][] | [πŸ“˜ elastic.co][apm-agent-ios-docs] | [πŸ“‚ contrib][apm-agent-ios-contrib] | Java Agent | [apm-agent-java][] | [πŸ“˜ elastic.co][apm-agent-java-docs] | [πŸ“‚ contrib][apm-agent-java-contrib] | JavaScript RUM Agent | [apm-agent-rum-js][] | [πŸ“˜ elastic.co][apm-agent-js-base-docs] | [πŸ“‚ contrib][apm-agent-rum-js-contrib] | Node.js Agent | [apm-agent-nodejs][] | [πŸ“˜ elastic.co][apm-agent-nodejs-docs] | [πŸ“‚ contrib][apm-agent-nodejs-contrib] +| PHP Agent | [apm-agent-php][] | [πŸ“˜ elastic.co][apm-agent-php-docs] | [πŸ“‚ contrib][apm-agent-php-contrib] | Python Agent | [apm-agent-python][] | [πŸ“˜ elastic.co][apm-agent-python-docs] | [πŸ“‚ contrib][apm-agent-python-contrib] | Ruby Agent | [apm-agent-ruby][] | [πŸ“˜ elastic.co][apm-agent-ruby-docs] | [πŸ“‚ contrib][apm-agent-ruby-contrib] | .NET Agent | [apm-agent-dotnet][] | [πŸ“˜ elastic.co][apm-agent-dotnet-docs] | [πŸ“‚ contrib][apm-agent-dotnet-contrib] -[apm-server-contrib]: https://github.com/elastic/apm-contrib/tree/master/apm-server -[apm-server-docs]: https://www.elastic.co/guide/en/apm/server/current/index.html +[apm-server-contrib]: https://github.com/elastic/apm-contrib/tree/main/apm-server +[apm-server-docs]: https://www.elastic.co/guide/en/apm/guide/current/index.html [apm-server]: https://github.com/elastic/apm-server -[apm-ui-contrib]: https://github.com/elastic/apm-contrib/tree/master/apm-ui +[apm-ui-contrib]: https://github.com/elastic/apm-contrib/tree/main/apm-ui [apm-ui-docs]: https://www.elastic.co/guide/en/kibana/current/xpack-apm.html -[apm-ui]: https://github.com/elastic/kibana/tree/master/x-pack/legacy/plugins/apm +[apm-ui]: https://github.com/elastic/kibana/tree/main/x-pack/plugins/apm + +[apm-agent-android-contrib]: https://github.com/elastic/apm-contrib/tree/main/apm-agent-android +[apm-agent-android-docs]: https://www.elastic.co/guide/en/apm/agent/android/current/index.html +[apm-agent-android]: https://github.com/elastic/apm-agent-android -[apm-agent-go-contrib]: https://github.com/elastic/apm-contrib/tree/master/apm-agent-go +[apm-agent-go-contrib]: https://github.com/elastic/apm-contrib/tree/main/apm-agent-go [apm-agent-go-docs]: https://www.elastic.co/guide/en/apm/agent/go/current/index.html [apm-agent-go]: https://github.com/elastic/apm-agent-go -[apm-agent-java-contrib]: https://github.com/elastic/apm-contrib/tree/master/apm-agent-java +[apm-agent-ios-contrib]: https://github.com/elastic/apm-contrib/tree/main/apm-agent-ios +[apm-agent-ios-docs]: https://www.elastic.co/guide/en/apm/agent/swift/current/index.html +[apm-agent-ios]: https://github.com/elastic/apm-agent-ios + +[apm-agent-java-contrib]: https://github.com/elastic/apm-contrib/tree/main/apm-agent-java [apm-agent-java-docs]: https://www.elastic.co/guide/en/apm/agent/java/current/index.html [apm-agent-java]: https://github.com/elastic/apm-agent-java -[apm-agent-rum-js-contrib]: https://github.com/elastic/apm-contrib/tree/master/apm-agent-rum-js +[apm-agent-rum-js-contrib]: https://github.com/elastic/apm-contrib/tree/main/apm-agent-rum-js [apm-agent-js-base-docs]: https://www.elastic.co/guide/en/apm/agent/js-base/current/index.html [apm-agent-rum-js]: https://github.com/elastic/apm-agent-rum-js -[apm-agent-nodejs-contrib]: https://github.com/elastic/apm-contrib/tree/master/apm-agent-nodejs +[apm-agent-nodejs-contrib]: https://github.com/elastic/apm-contrib/tree/main/apm-agent-nodejs [apm-agent-nodejs-docs]: https://www.elastic.co/guide/en/apm/agent/nodejs/current/index.html [apm-agent-nodejs]: https://github.com/elastic/apm-agent-nodejs -[apm-agent-python-contrib]: https://github.com/elastic/apm-contrib/tree/master/apm-agent-python +[apm-agent-python-contrib]: https://github.com/elastic/apm-contrib/tree/main/apm-agent-python [apm-agent-python-docs]: https://www.elastic.co/guide/en/apm/agent/python/current/index.html [apm-agent-python]: https://github.com/elastic/apm-agent-python -[apm-agent-ruby-contrib]: https://github.com/elastic/apm-contrib/tree/master/apm-agent-ruby +[apm-agent-ruby-contrib]: https://github.com/elastic/apm-contrib/tree/main/apm-agent-ruby [apm-agent-ruby-docs]: https://www.elastic.co/guide/en/apm/agent/ruby/current/index.html [apm-agent-ruby]: https://github.com/elastic/apm-agent-ruby -[apm-agent-dotnet-contrib]: https://github.com/elastic/apm-contrib/tree/master/apm-agent-dotnet +[apm-agent-dotnet-contrib]: https://github.com/elastic/apm-contrib/tree/main/apm-agent-dotnet [apm-agent-dotnet-docs]: https://www.elastic.co/guide/en/apm/agent/dotnet/current/index.html [apm-agent-dotnet]: https://github.com/elastic/apm-agent-dotnet + +[apm-agent-php-contrib]: https://github.com/elastic/apm-contrib/tree/main/apm-agent-php +[apm-agent-php-docs]: https://www.elastic.co/guide/en/apm/agent/php/current/index.html +[apm-agent-php]: https://github.com/elastic/apm-agent-php diff --git a/docs/agent-development.md b/docs/agent-development.md deleted file mode 100644 index 77a5b673..00000000 --- a/docs/agent-development.md +++ /dev/null @@ -1,451 +0,0 @@ -# Building an agent - -So you want to build an agent for Elastic APM? That's great, here's what you need to know. - -**Note:** This is a living document. If you come across something weird or find something missing, please add it or ask open an issue. - ---- - -# Introduction - -The [Getting started with APM](https://www.elastic.co/guide/en/apm/get-started/current/overview.html) provides an overview to understand the big picture architecture. - -Your agent will be talking to the APM Server using HTTP, sending data to it as JSON or ND-JSON. There are multiple categories of data that each agent captures and sends to the APM Server: - - - Trace data: transactions and spans (distributed tracing) - - Errors/exceptions (i.e. for error tracking) - - Metrics (host process-level metrics, and language/runtime-specific metrics) - -You can find details about each of these in the [APM Data Model](https://www.elastic.co/guide/en/apm/get-started/current/apm-data-model.html) documentation. The [Intake API](https://www.elastic.co/guide/en/apm/server/current/intake-api.html) documentation describes the wire format expected by APM Server. APM Server converts the data into Elasticsearch documents, and then the APM UI in Kibana provides visualisations over that data, as well as enabling you to dig into the data in an interactive way. - -# Guiding Philosophy - -1. Agents try to be as good citizens as possible in the programming language they are written for. Even though every language ends up reporting to the same server API with the same JSON format, the agents should try to make as much sense in the context of the relevant language as possible. We want to both streamline the agents to work the same in every context **but** also make them feel like they were built specifically for each language. It's up to you to figure out how this looks in the language you are writing your agent for. - -2. Agents should be as close to zero configuration as possible. - - - Use sensible defaults, aligning across agents unless there is a compelling reason to have a language-specific default. - - Agents should typically come with out-of-the-box instrumentation for the most popular frameworks or libraries of their relevant language. - - Users should be able to disable specific instrumentation modules to reduce overhead, or where details are not interesting to them. - -3. The overhead of agents must be kept to a minimum, and must not affect application behaviour. - -# Features to implement - - - -- [Transport](#Transport) - - [Background sending](#Background-sending) - - [Batching/streaming data](#Batchingstreaming-data) - - [Transport errors](#Transport-errors) - - [Compression](#Compression) -- [Metadata](#Metadata) - - [System metadata](#System-metadata) - - [Container/Kubernetes metadata](#ContainerKubernetes-metadata) - - [Process metadata](#Process-metadata) - - [Service metadata](#Service-metadata) - - [Global labels](#Global-labels) -- [Tracing](#Tracing) - - [Transactions](#Transactions) - - [HTTP Transactions](#HTTP-Transactions) - - [Transaction sampling](#Transaction-sampling) - - [Spans](#Spans) - - [Span stack traces](#Span-stack-traces) - - [Span count](#Span-count) - - [HTTP client spans](#HTTP-client-spans) - - [Database spans](#Database-spans) - - [Database span names](#Database-span-names) - - [Database span type/subtype](#Database-span-typesubtype) - - [Manual APIs](#Manual-APIs) - - [Distributed Tracing](#Distributed-Tracing) -- [Error/exception tracking](#Errorexception-tracking) -- [Metrics](#Metrics) - - [System/process CPU/Heap](#Systemprocess-CPUHeap) - - [Runtime](#Runtime) - - [Transaction and span breakdown](#Transaction-and-span-breakdown) -- [Logging Correlation](#Logging-Correlation) -- [Agent Configuration](#Agent-Configuration) - - [APM Agent Configuration via Kibana](#APM-Agent-Configuration-via-Kibana) - - [Interaction with local config](#Interaction-with-local-config) - - [Caching](#Caching) - - [Dealing with errors](#Dealing-with-errors) - - - -## Transport - -Agents send data to the APM Server as JSON (application/json) or ND-JSON (application/x-ndjson) over HTTP. We describe here various details to guide transport implementation. - -### Background sending - -In order to avoid impacting application performance and behaviour, agents should (where possible) send data in a non-blocking manner, e.g. via a background thread/goroutine/process/what-have-you, or using asynchronous I/O. - -If data is sent in the background process, then there must be some kind of queuing between that background process and the application code. The queue should be limited in size to avoid memory exhaustion. In the event that the queue fills up, agents must drop events: either drop old events or simply stop recording new events. - -### Batching/streaming data - -With the exception of the RUM agent (which does not maintain long-lived connections to the APM Server), agents should use the ND-JSON format. The ND-JSON format enables agents to stream data to the server as it is being collected, with one event being encoded per line. This format is supported since APM Server 6.5.0. - -Agents should implement one of two methods for sending events to the server: - - - batch events together and send a complete request after a given size is reached, or amount of time has elapsed - - start streaming events immediately to the server using a chunked-encoding request, and end the request after a given amount of data has been sent, or amount of time has elapsed - -The streaming approach is preferred. There are two configuration options that agents should implement to control when data is sent: - - - [ELASTIC_APM_API_REQUEST_TIME](https://www.elastic.co/guide/en/apm/agent/python/current/configuration.html#config-api-request-time) - - [ELASTIC_APM_API_REQUEST_SIZE](https://www.elastic.co/guide/en/apm/agent/python/current/configuration.html#config-api-request-size) - -All events can be streamed as described in the [Intake API](https://www.elastic.co/guide/en/apm/server/current/intake-api.html) documentation. Each line encodes a single event, with the first line in a stream encoding the special metadata "event" which is folded into all following events. This metadata "event" is used to describe static properties of the system, process, agent, etc. - -When the batching approach is employed, unhandled exceptions/unexpected errors should typically be sent immediately to ensure timely error visibility, and to avoid data loss due to process termination. Even when using streaming there may be circumstances in which the agent should block the application until events are sent, but this should be both rare and configurable, to avoid interrupting normal program operation. For example, an application may terminate itself after logging a message at "fatal" level. In such a scenario, it may be useful for the agent to optionally block until enqueued events are sent prior to process termination. - -### Transport errors - -If the HTTP response status code isn’t 2xx or if a request is prematurely closed (either on the TCP or HTTP level) the request MUST be considered failed. - -When a request fails, the agent has no way of knowing exactly what data was successfully processed by the APM Server. And since the agent doesn’t keep a copy of the data that was sent, there’s no way for the agent to re-send any data. Furthermore, as the data waiting to be sent is already compressed, it’s impractical to recover any of it in a way so that it can be sent over a new HTTP request. - -The agent should therefore drop the entire compressed buffer: both the internal zlib buffer, and potentially the already compressed data if such data is also buffered. Data subsequently written to the compression library can be directed to a new HTTP request. - -The new HTTP request should not necessarily be started immediately after the previous HTTP request fails, as the reason for the failure might not have been resolved up-stream. Instead an incremental back-off algorithm SHOULD be used to delay new requests. The grace period should be calculated in seconds using the algorithm `min(reconnectCount++, 6) ** 2 Β± 10%`, where `reconnectCount` starts at zero. So the delay after the first error is 0 seconds, then circa 1, 4, 9, 16, 25 and finally 36 seconds. We add Β±10% jitter to the calculated grace period in case multiple agents entered the grace period simultaneously. This way they will not all try to reconnect at the same time. - -Agents should support specifying multiple server URLs. When a transport error occurs, the agent should switch to another server URL at the same time as backing off. - -While the grace period is in effect, the agent may buffer the data that was supposed to be sent if the grace period wasn’t in effect. If buffering, the agent must ensure the memory used to buffer data data does not grow indefinitely. - -### Compression - -The APM Server accepts both uncompressed and compressed HTTP requests. The following compression formats are supported: - -- zlib data format (`Content-Encoding: deflate`) -- gzip data format (`Content-Encoding: gzip`) - -Agents should compress the HTTP payload by default, optimising for speed over compactness (typically known as the "best speed" level). - -## Metadata - -As mentioned above, the first "event" in each ND-JSON stream contains metadata to fold into subsequent events. The metadata that agents should collect includes are described in the following sub-sections. - - - service metadata - - global labels (requires APM Server 7.2 or greater) - -### System metadata - -System metadata relates to the host/container in which the service being monitored is running: - - - hostname - - architecture - - operating system - - container ID - - kubernetes - - namespace - - node name - - pod name - - pod UID - -#### Container/Kubernetes metadata - -On Linux, the container ID and some of the Kubernetes metadata can be extracted by parsing `/proc/self/cgroup`. For each line in the file, we split the line according to the format "hierarchy-ID:controller-list:cgroup-path", extracting the "cgroup-path" part. We then attempt to extract information according to the following algorithm: - - 1. Split the path into dirname/basename (i.e. on the final slash) - - 2. If the basename ends with ".scope", check for a hyphen and remove everything up to and including that. This allows us to match `.../docker-.scope` as well as `.../`. - - 3. Attempt to extract the Kubernetes pod UID from the dirname by matching one of the following regular expressions: - - - `(?:^/kubepods/[^/]+/pod([^/]+)/$)` - - `(?:^/kubepods\.slice/kubepods-[^/]+\.slice/kubepods-[^/]+-pod([^/]+)\.slice/$)` - - The capturing group in either case is the pod UID. In the latter case, which occurs when using the systemd cgroup driver, we must unescape underscores (`_`) to hyphens (`-`) in the pod UID. - If we match a pod UID then we record the hostname as the pod name since, by default, Kubernetes will set the hostname to the pod name. Finally, we record the basename as the container ID without any further checks. - - 4. If we did not match a Kubernetes pod UID above, then we check if the basename matches one of the following regular expressions: - - - `^[[:xdigit:]]{64}$` - - `^[[:xdigit:]]{8}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4,}$` - - If we match, then the basename is assumed to be a container ID. - -If the Kubernetes pod name is not the hostname, it can be overridden by the `KUBERNETES_POD_NAME` environment variable, using the [Downward API](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/). In a similar manner, you can inform the agent of the node name and namespace, using the environment variables `KUBERNETES_NODE_NAME` and `KUBERNETES_NAMESPACE`. - -### Process metadata - -Process level metadata relates to the process running the service being monitored: - - - process ID - - parent process ID - - process arguments - - process title (e.g. "node /app/node_") - -### Service metadata - -Service metadata relates to the service/application being monitored: - - - service name and version - - environment name ("production", "development", etc.) - - agent name (e.g. "ruby") and version (e.g. "2.8.1") - - language name (e.g. "ruby") and version (e.g. "2.5.3") - - runtime name (e.g. "jruby") and version (e.g. "9.2.6.0") - - framework name (e.g. "flask") and version (e.g. "1.0.2") - -For official Elastic agents, the agent name should just be the name of the language for which the agent is written, in lower case. - -### Global labels - -Events sent by the agents can have labels associated, which may be useful for custom aggregations, or document-level access control. It is possible to add "global labels" to the metadata, which are labels that will be applied to all events sent by an agent. These are only understood by APM Server 7.2 or greater. - -Global labels can be specified via the environment variable `ELASTIC_APM_GLOBAL_LABELS`, formatted as a comma-separated list of `key=value` pairs. - -## Tracing - -### Transactions - -#### HTTP Transactions - -Agents should instrument HTTP request routers/handlers, starting a new transaction for each incoming HTTP request. When the request ends, the transaction should be ended, recording its duration. - -- The transaction `type` should be `request`. -- The transaction `result` should be `HTTP Nxx`, where N is the first digit of the status code (e.g. `HTTP 4xx` for a 404) -- The transaction `name` should be aggregatable, such as the route or handler name. Examples: - - - `GET /users/{id}` - - `UsersController#index` - -It's up to you to pick a naming scheme that is the most natural for the language or web framework you are instrumenting. - -In case a name cannot be automatically determined, and a custom name has not been provided by other means, the transaction should be named ` unknown route`, e.g. `POST unknown route`. This would normally also apply to requests to unknown endpoints, e.g. the transaction for the request `GET /this/path/does/not/exist` would be named `GET unknown route`, whereas the transaction for the request `GET /users/123` would still be named `GET /users/{id}` even if the id `123` did not match any known user and the request resulted in a 404. - -In addition to the above properties, HTTP-specific properties should be recorded in the transaction `context`, for sampled transactions only. Refer to the [Intake API Transaction](https://www.elastic.co/guide/en/apm/server/current/transaction-api.html) documentation for a description of the various context fields. - -By default request bodies are not captured. It should be possible to configure agents to enable their capture using the config variable `ELASTIC_APM_CAPTURE_BODY`. By default agents will capture request headers, but it should be possible to disable their capture using the config variable `ELASTIC_APM_CAPTURE_HEADERS`. - -Request and response headers, cookies, and form bodies should be sanitised (i.e. secrets removed). Each agent should define a default list of keys to sanitise, which should include at least the following (using wildcard matching): - - - `password` - - `passwd` - - `pwd` - - `secret` - - `*key` - - `*token*` - - `*session*` - - `*credit*` - - `*card*` - - `authorization` - - `set-cookie` - -Agents may may include additional patterns if there are common conventions specific to language frameworks. - -#### Transaction sampling - -To reduce processing and storage overhead, transactions may be "sampled". Currently sampling has the effect of limiting the amount of data we capture for transactions: for non-sampled transactions instrumentation should not record context, nor should any spans be captured. The default graphs in the APM UI will utilise the transaction properties available for both sampled and non-sampled transactions. - -By default all transactions will be sampled. Agents can be configured to sample probabilistically, by specifying a sampling -probability in the range \[0,1\] using the configuration `ELASTIC_APM_TRANSACTION_SAMPLE_RATE`. For example: - - - `ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0` means no transactions will be sampled - - `ELASTIC_APM_TRANSACTION_SAMPLE_RATE=1` means all transactions will be sampled (the default) - - `ELASTIC_APM_TRANSACTION_SAMPLE_RATE=0.5` means approximately 50% of transactions will be sampled - -If a transaction is not sampled, you should set the `sampled: false` property and omit collecting `spans` and `context`. - -### Spans - -The agent should also have a sense of the most common libraries for these and instrument them without any further setup from the app developers. - -#### Span stack traces - -Spans may have an associated stack trace, in order to locate the associated source code that caused the span to occur. If there are many spans being collected this can cause a significant amount of overhead in the application, due to the capture, rendering, and transmission of potentially large stack traces. It is possible to limit the recording of span stack traces to only spans that are slower than a specified duration, using the config variable `ELASTIC_APM_SPAN_FRAMES_MIN_DURATION`. - -#### Span count - -When a span is started a counter should be incremented on its transaction, in order to later identify the _expected_ number of spans. In this way we can identify data loss, e.g. because events have been dropped, or because of instrumentation errors. - -To handle edge cases where many spans are captured within a single transaction, the agent should enable the user to start dropping spans when the associated transaction exeeds a configurable number of spans. When a span is dropped, it is not reported to the APM Server, but instead another counter is incremented to track the number of spans dropped. In this case the above mentioned counter for started spans is not incremented. - -```json -"span_count": { - "started": 500, - "dropped": 42 -} -``` - -Here's how the limit can be configured for [Node.js](https://www.elastic.co/guide/en/apm/agent/nodejs/current/agent-api.html#transaction-max-spans) and [Python](https://www.elastic.co/guide/en/apm/agent/python/current/configuration.html#config-transaction-max-spans). - -#### HTTP client spans - -We capture spans for outbound HTTP requests. These should have a type of `external`, and subtype of `http`. The span name should have the format ` `. - -For outbound HTTP request spans we capture the following http-specific span context: - -- `http.url` (the target URL) -- `http.status_code` (the response status code) - -The captured URL should have the userinfo (username and password), if any, redacted. - -#### Database spans - -We capture spans for various types of database/data-stores operations, such as SQL queries, Elasticsearch queries, Redis commands, etc. We follow some of the same conventions defined by OpenTracing for capturing database-specific span context, including: - - - `db.instance`: database instance name, e.g. "customers" - - `db.statement`: statement/query, e.g. "SELECT * FROM foo" - - `db.user`: username used for database access, e.g. "readonly_user" - - `db.type`: database type/category, which should be "sql" for SQL databases, and the lower-cased database name otherwise. - -The full database statement should be stored in `db.statement`, which may be useful for debugging performance issues. We store up to 10000 Unicode characters per database statement. - -For SQL databases this will be the full SQL statement. - -For MongoDB, this can be set to the command encoded as MongoDB Extended JSON. - -For Elasticsearch search-type queries, the request body may be recorded. Alternatively, if a query is specified in HTTP query parameters, that may be used instead. If the body is gzip-encoded, the body should be decoded first. - -##### Database span names - -For SQL operations we perform a limited parsing the statement, and extract the operation name and outer-most table involved (if any). See more details here: https://docs.google.com/document/d/1sblkAP1NHqk4MtloUta7tXjDuI_l64sT2ZQ_UFHuytA. - -For Redis, the the span name can simply be set to the command name, e.g. `GET` or `LRANGE`. - -For MongoDB, the span name should be the command name in the context of its collection/database, e.g. `users.find`. - -For Elasticsearch, the span name should be `Elasticsearch: `, e.g. -`Elasticsearch: GET /index/_search`. - -##### Database span type/subtype - -For database spans, the type should be `db` and subtype should be the database name. Agents should standardise on the following span subtypes: - -- `postgresql` (PostgreSQL) -- `mysql` (MySQL) - -### Manual APIs - -All agents must provide an API to enable developers to instrument their applications manually, in addition to any automatic instrumentation. Agents document their APIs in the elastic.co docs: - -- [Node.js Agent](https://www.elastic.co/guide/en/apm/agent/nodejs/current/api.html) -- [Go Agent](https://www.elastic.co/guide/en/apm/agent/go/current/api.html) -- [Java Agent](https://www.elastic.co/guide/en/apm/agent/java/current/public-api.html) -- [.NET Agent](https://www.elastic.co/guide/en/apm/agent/dotnet/current/public-api.html) -- [Python Agent](https://www.elastic.co/guide/en/apm/agent/python/current/api.html) -- [Ruby Agent](https://www.elastic.co/guide/en/apm/agent/ruby/current/api.html) -- [RUM JS Agent](https://www.elastic.co/guide/en/apm/agent/js-base/current/api.html) - -In addition to each agent having a "native" API for instrumentation, they also implement the [OpenTracing APIs](https://opentracing.io). Agents should align implementations according to https://github.com/elastic/apm/issues/32. - -### Distributed Tracing - -We implement the [W3C Trace Context](https://w3c.github.io/trace-context/) standard, with the exceptions that we currently use the header name "elastic-apm-traceparent" and we do not currently support "tracestate". Both of these exceptions will be removed as Trace Context nears finalisation; we are tracking this in https://github.com/elastic/apm/issues/71. - -## Error/exception tracking - -The agent support reporting exceptions/errors. Errors may come in one of two forms: - - - unhandled (or handled and explicitly reported) exceptions/errors - - log records - -Agents should include exception handling in the instrumentation they provide, such that exceptions are reported to the APM Server automatically, without intervention. In addition, hooks into logging libraries may be provided such that logged errors are also sent to the APM Server. - -Errors may or may not occur within the context of a transaction or span. If they do, then they will be associated with them by recording the trace ID and transaction or span ID. This enables the APM UI to annotate traces with errors. - -## Metrics - -Agents periodically collect and report various metrics, described below. - -### System/process CPU/Heap - -All agents (excluding JavaScript RUM) should record the following basic system/process metrics: - - - `system.cpu.total.norm.pct`: system CPU usage since the last report, in the range `[0,1]` (0-100%) - - `system.process.cpu.total.norm.pct`: process CPU usage since the last report, in the range `[0,1]` (0-100%) - - `system.memory.total`: total usable (but not necessarily available) memory on the system, in bytes - - `system.memory.actual.free`: total available memory on the system, in bytes - - `system.process.memory.size`: process virtual memory size, in bytes - - `system.process.memory.rss.bytes`: process resident set size, in bytes - -### Runtime - -Agent should record runtime-specific metrics, such as garbage collection pauses. Due to their runtime-specific nature, these will differ for each agent. - -When capturing runtime metrics, keep in mind the end use-case: how will they be used? Is the format in which they are recorded appropriate for visualisation in Kibana? Do not record metrics just because it is easy; record them because they are useful. - -### Transaction and span breakdown - -Agents should record "breakdown metrics", which is a summarisation of how much time is spent per span type/subtype in each transaction group. This is described in detail in the [Breakdown Graphs](https://docs.google.com/document/d/1-_LuC9zhmva0VvLgtI0KcHuLzNztPHbcM0ZdlcPUl64#heading=h.ondan294nbpt) document, so we do not repeat it here. - -## Logging Correlation - -Agents should provide instrumentation/hooks for popular logging libraries in order to decorate structured log records with trace context. In particular, logging that occurs within the context of a transaction should add the fields `trace.id` and `transaction.id`; logging that occurs within a span should add the fields `trace.id`, `span.id`, and optionally `transaction.id`. - -By adding trace context to log records, users will be able to move between the APM UI and Logs UI. - -## Agent Configuration - -Even though the agents should _just work_ with as little configuration and setup as possible we provide a wealth of ways to configure them to users' needs. - -Generally we try to make these the same for every agent. Some agents might differ in nature like the JavaScript RUM agent but mostly these should fit. Still, languages are different so some of them might not make sense for your particular agent. That's ok! - -Here's a list of the config options across all agents, their types, default values etc. Please align with these whenever possible: - -- [APM Backend Agent Config Comparison](https://docs.google.com/spreadsheets/d/1JJjZotapacA3FkHc2sv_0wiChILi3uKnkwLTjtBmxwU/edit) - -They are provided as environment variables but depending on the language there might be several feasible ways to let the user tweak them. For example besides the environment variable `ELASTIC_APM_SERVER_URL`, the Node.js Agent might also allow the user to configure the server URL via a config option named `serverUrl`, while the Python Agent might also allow the user to configure it via a config option named `server_url`. - -### APM Agent Configuration via Kibana - -Also known as "central configuration". Agents can query the APM Server for configuration updates; the server proxies and caches requests to Kibana. - -Agents should poll the APM Server for config periodically by sending an HTTP request to the `/config/v1/agents` endpoint. Agents must specify their service name, and optionally environment. The server will use these to filter the configuration down to the relevant service and environment. There are two methods for sending these parameters: - -1. Using the `GET` method, pass them as query parameters: `http://localhost:8200/config/v1/agents?service.name=opbeans&service.environment=production` -2. Using the `POST` method, encode the parameters as a JSON object in the body, e.g. `{"service": {"name": "opbeans", "environment": "production"}}` - -The server will respond with a JSON object, where each key maps a config attribute to a string value. The string value should be interpreted the same as if it were passed in via an environment variable. Upon receiving these config changes, the agent will update its configuration dynamically, overriding any config previously specified. That is, config via Kibana takes highest precedence. - -To minimise the amount of work required by users, agents should aim to enable this feature by default. This excludes RUM, where there is a performance penalty. - -#### Interaction with local config - -When an instrumented application starts, the agent should first load locally-defined configuration via environment variables, config files, etc. Once this has completed, the agent will begin asynchronously polling the server for configuration. Once available, this configuration will override the locally-defined configuration. This means that there will be a short time window at application startup in which locally-defined configuration will apply. - -If a user defines and then later deletes configuration via Kibana, the agent should ideally fall back to the locally-defined configuration. As an example of how to achieve this: the Java agent defines a hierarchy of configuration sources, with configuration via Kibana having the highest precedence. When configuration is not available at one level, the agent obtains it via the next highest level, and so on. - -#### Caching - -As mentioned above, the server will cache config for each unique `service.name`, `service.environment` pair. The server will respond to config requests with two related response headers: [Etag](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag) and [Cache-Control](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control). - -Agents should keep a record of the `Etag` value returned by the most recent successful config request, and then present it to future requests via the [If-None-Match](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match) header. If the config has not changed, the server will respond with 304 (Not Modified). - -The `Cache-Control` header should contain a `max-age` directive, specifying the amount of time (in seconds) the response should be considered "fresh". Agents should use this to decide how long to wait before requesting config again. The server will respond with a `Cache-Control` header even if the request fails. - -#### Dealing with errors - -Agents must deal with various error scenarios, including: - - - 7.3 servers where the Kibana connection is not enabled (server responds with 403) - - 7.3 servers where the Kibana connection is enabled, but unavailable (server responds with 503) - - pre-7.3 servers that don't support the config endpoint (server responds with 404) - - any other error (server responds with 5xx) - -If the server responds with any 5xx, agents should log at error level. If the server responds with 4xx, agents are not required to log the response, but may choose to log it at debug level; either the central config feature is not available, or is not enabled. In either case, there is no expectation that the agent should take any action, so logging is not necessary. - -In any case, a 7.3+ server _should_ respond with a Cache-Control header, as described in the section above, and agents should retry after the specified interval. For older servers, or for whatever reason a 7.3+ server does not respond with that header (or it is invalid), agents should retry after 5 minutes. We include this behaviour for older servers so that the agent will start polling after server upgrade without restarting the application. - -If the agent does not recognise a config attribute, or does not support dynamically updating it, then it should log a warning such as: - -``` -Central config failure. Unsupported config names: unknown_option, disable_metrics, capture_headers -``` - -Note that in the initial implementation of this feature, not all config attributes will be supported by the APM UI or APM Server. Agents may choose to support only the attributes supported by the UI/server, or they may choose to accept additional attributes. The latter will enable them to work without change once additional config attributes are supported by the UI/server. - -If the agent receives a known but invalid config attribute, it should log a warning such as: - -``` -Central config failure. Invalid value for transactionSampleRate: 1.2 (out of range [0,1.0]) -``` - -Failure to process one config attribute should not affect processing of others. - -#### Feature flag - -Agents should implement a [configuration option](https://docs.google.com/spreadsheets/d/1JJjZotapacA3FkHc2sv_0wiChILi3uKnkwLTjtBmxwU), (`CENTRAL_CONFIG`) which lets users disable the central configuration polling. diff --git a/specs/agents/README.md b/specs/agents/README.md new file mode 100644 index 00000000..11f0dd42 --- /dev/null +++ b/specs/agents/README.md @@ -0,0 +1,71 @@ +# Building an agent + +So you want to build an agent for Elastic APM? That's great, here's what you need to know. + +**Note:** This is a living document. +If you come across something weird or find something missing, please add it or ask open an issue. + +--- + +# Introduction + +The [Getting started with APM](https://www.elastic.co/guide/en/observability/current/apm.html) provides an overview to understand the big picture architecture. + +Your agent will be talking to the APM Server using HTTP, sending data to it as JSON or ND-JSON. There are multiple categories of data that each agent captures and sends to the APM Server: + + - Trace data: transactions and spans (distributed tracing) + - Errors/exceptions (i.e. for error tracking) + - Metrics (host process-level metrics, and language/runtime-specific metrics) + +You can find details about each of these in the [APM Data Model](https://www.elastic.co/guide/en/observability/current/apm-data-model.html) documentation. The [Intake API](https://www.elastic.co/guide/en/observability/current/apm-api-events.html) documentation describes the wire format expected by APM Server. APM Server converts the data into Elasticsearch documents, and then the APM UI in Kibana provides visualisations over that data, as well as enabling you to dig into the data in an interactive way. + +# Guiding Philosophy + +1. Agents try to be as good citizens as possible in the programming language they are written for. Even though every language ends up reporting to the same server API with the same JSON format, the agents should try to make as much sense in the context of the relevant language as possible. We want to both streamline the agents to work the same in every context **but** also make them feel like they were built specifically for each language. It's up to you to figure out how this looks in the language you are writing your agent for. + +2. Agents should be as close to zero configuration as possible. + + - Use sensible defaults, aligning across agents unless there is a compelling reason to have a language-specific default. + - Agents should typically come with out-of-the-box instrumentation for the most popular frameworks or libraries of their relevant language. + - Users should be able to disable specific instrumentation modules to reduce overhead, or where details are not interesting to them. + +3. The overhead of agents must be kept to a minimum, and must not affect application behaviour. + + +# Features to implement + +- [Transport](transport.md) +- [Metadata](metadata.md) +- Tracing + - [Transactions](tracing-transactions.md) + - [Transaction Grouping](tracing-transaction-grouping.md) + - [Spans](tracing-spans.md) + - [Span destination](tracing-spans-destination.md) + - [Handling huge traces](handling-huge-traces/) + - [Hard limit on number of spans to collect](handling-huge-traces/tracing-spans-limit.md) + - [Collecting statistics about dropped spans](handling-huge-traces/tracing-spans-dropped-stats.md) + - [Dropping fast exit spans](handling-huge-traces/tracing-spans-drop-fast-exit.md) + - [Compressing spans](handling-huge-traces/tracing-spans-compress.md) + - [Sampling](tracing-sampling.md) + - [Distributed tracing](tracing-distributed-tracing.md) + - [Tracer API](tracing-api.md) + - Instrumentation + - [AWS](tracing-instrumentation-aws.md) + - [Databases](tracing-instrumentation-db.md) + - [HTTP](tracing-instrumentation-http.md) + - [Messaging systems](tracing-instrumentation-messaging.md) + - [gRPC](tracing-instrumentation-grpc.md) + - [GraphQL](tracing-instrumentation-graphql.md) + - [OpenTelemetry API Bridge](tracing-api-otel.md) +- [Error/exception tracking](error-tracking.md) +- [Metrics](metrics.md) +- [Logging Correlation](log-correlation.md) +- [Agent Configuration](configuration.md) +- [Agent logging](logging.md) +- [Data sanitization](sanitization.md) +- [Field limits](field-limits.md) + +# Processes + +- [Proposing changes to the specification](../../.github/pull_request_template.md) +- [Proposing new fields to the intake API](process-new-fields.md) diff --git a/specs/agents/breaking-changes.md b/specs/agents/breaking-changes.md new file mode 100644 index 00000000..e2afd4c9 --- /dev/null +++ b/specs/agents/breaking-changes.md @@ -0,0 +1,43 @@ +# What is a breaking change in a version of an APM agent? +A change is defined as breaking if it causes an application using an APM agent to break or if the APM product is no longer usable in a way that it previously was. + +Taken strictly, this definition could lead to treating every change in runtime behavior as a breaking change. At the same time, we need to be able to keep driving improvements to existing features of our APM product. This document gives some guidelines to help distinguish changes of implementation details from breaking changes. + +## Types of breaking changes +### Instrumentation versions +Each agent instruments a number of libraries that are used in their language ecosystem. These libraries themselves may introduce new versions, breaking changes, and deprecate older versions. The APM agents therefore will occasionally introduce changes in their instrumentation of external libraries. The changes that we consider breaking are ones that remove support for older versions. Agents can also continue supporting the instrumentation of a particular older library version but drop its testing of it because of some conflicts in installing test suite dependencies, for example. This change would not be considered breaking as long as it’s properly documented. + +### Language and runtime support +Similar to library version instrumentation, APM agents will typically support multiple versions of its language. Sometimes it is necessary to drop support for older versions of the languages as they themselves are EOL’ed. + +It is typically considered a breaking change when an APM agent drops support for a particular language version, but this may vary according to the conventions of the language ecosystem. +For example, it is common practice for Go libraries to follow the Go project's release policy of only supporting the two most recent releases of Go. + +### Configuration changes +All agents support a set of configuration options and default values. Changes to the configuration offering can be categorized into three types: + +__Change in default configuration value__: Each APM agent configuration option has a default value. Sometimes we change what that default configuration value is. We should consider the _effect_ of changing the value when we evaluate whether the change is breaking. For example, the default configuration value could enrich the data and provide an enhanced experience to the user. In this case, we wouldn’t consider the change to be breaking. On the other hand, if a default value is changed, and as a consequence, removes some information the user was previously able to see, we would consider that a breaking change. + +__Removal of a configuration option__: It is a breaking change to remove a configuration option. For example, APM agents may have removed the option `active` in favor of a new option, `enabled`. + +__Change in configuration value behavior__: If the semantics of a configuration value are altered, the change is considered breaking. For example, the configuration option `span_frames_min_duration` can be set to an integer millisecond value, 0, or -1. At the time this document was written, setting this value to 0 means to collect no stack traces and -1 means to collect all stack traces. If there is a change in what the special values 0 and -1 mean, the change is a breaking one. + +### Public API changes +Each APM agent has a Public API that is marked as such and documented. Agents may make a change to their Public API in order to support new features, support new integrations, resolve inconsistencies between agents, or for other reasons. + +__Public API__: When the name of a Public API component is changed or if a component is removed, this change is considered breaking. Applications may depend on the APM agent’s Public API so the agent would ideally issue a deprecation warning and clearly document the upcoming change for users before the version including the change is released. For example, changing the Public API for setting the `service.destination.resource` value to setting two new fields instead (`service.target.name`, `service.target.type`) is considered to be a breaking change. + +__Public API behavior__: If the effects of using a part of the Public API or the semantics of that API are changed to enhance a user experience or enrich the data in some way, we don’t consider it a breaking change. A Public API behavior change that removes or alters some information that was there before is considered breaking. + +### APM server support +__Support for APM server versions__: If an APM agent removes support for an older APM server version, the change is considered breaking. + +__Support for APM server protocols__: Similarly, if the APM agent removes support for an APM server protocol, the change is breaking. + + +## What is not a Breaking change +In general, we don’t consider changes in the data we collect to be breaking, unless they have security or privacy implications. Some examples of these changes are: +- Span metadata, such as the span name or the structured `db.statement`, or destination granularity +- Span compression (multiple similar or exact consecutive spans collapsed into one) +- Trace structure (e.g. span links + handling of messaging) + diff --git a/specs/agents/configuration.md b/specs/agents/configuration.md new file mode 100644 index 00000000..e25c3a1b --- /dev/null +++ b/specs/agents/configuration.md @@ -0,0 +1,198 @@ +## Agent Configuration + +Even though the agents should _just work_ with as little configuration and setup as possible we provide a wealth of ways to configure them to users' needs. + +Generally we try to make these the same for every agent. Some agents might differ in nature like the JavaScript RUM agent but mostly these should fit. Still, languages are different so some of them might not make sense for your particular agent. That's ok! + +Here's a list of the config options across all agents, their types, default values etc. Please align with these whenever possible: + +- [APM Backend Agent Config Comparison](https://docs.google.com/spreadsheets/d/1JJjZotapacA3FkHc2sv_0wiChILi3uKnkwLTjtBmxwU/edit) + +They are provided as environment variables but depending on the language there might be several feasible ways to let the user tweak them. For example besides the environment variable `ELASTIC_APM_SERVER_URL`, the Node.js Agent might also allow the user to configure the server URL via a config option named `serverUrl`, while the Python Agent might also allow the user to configure it via a config option named `server_url`. + +### Configuration Source Precedence + +Configuration can be provided via a number of sources. Values from central +configuration MUST have the highest precedence, and default values MUST have +the lowest precedence. Otherwise, agents MAY adopt the following config +source precedence. Sources higher on this list will override values provided +by sources lower on this list: + + - Central configuration + - Environment variables + - Inline configuration in code + - Config files + - Default value + +### Invalid Configuration Values + +If an invalid value for a configuration option is provided (for example: +`breakdown_metrics="yes"` or `apiRequestTime="1h"`) then the agent MUST ignore +the value (falling back to a config source with lower precedence) and SHOULD +emit a log warning about the ignored value. + +### Configuration Value Types + +The following list enumerates the available configuration types across the agents: + +- `String` +- `Integer` +- `Float` +- `Boolean`: Encoded as a lower-case boolean string: `"false"`, `"true"`. +- `List`: Encoded as a comma-separated string (whitespace surrounding items should be stripped): `"foo,bar,baz"`. +- `Mapping`: Encoded as a string, with `"key=value"` pairs separated by commas (whitespace surrounding items should be stripped): `"foo=bar,baz=foo"`. +- `Duration`: Case-sensitive string with duration encoded using unit suffixes (`ms` for millisecond, `s` for second, `m` for minute). Validating regex: `^(-)?(\d+)(ms|s|m)$`. +- `GranularDuration`: Case-sensitive string with duration encoded using unit suffixes which supports down to a microsecond duration (`us`). Validating regex: `^(-)?(\d+)(us|ms|s|m)$`. +- `Size`: Case-insensitive string with a positive size encoded using unit suffixes (`b` for bytes, `kb` for kilobytes, `mb` for megabytes, `gb` for gigabytes, with a 1024 multiplier between each unit). Validating regex: `^(\d+)(b|kb|mb|gb)$`. + + +### Configuration snapshotting + +To ensure consistent behavior within one transaction, +certain settings SHOULD be read at the beginning of the transaction. +The snapshot of these configuration values applies for the lifetime of the +transaction. + +Snapshotted configuration options: + + * `transaction_max_spans` + * `span_compression_enabled` + * `span_compression_exact_match_max_duration` + * `span_compression_same_kind_max_duration` + * `exit_span_min_duration` + + +#### Duration/Size Config Legacy Considerations + +For duration/size-formatted config options, some agents allow users to omit the unit +suffix for backwards compatibility reasons. Going forward, all +duration/size-formatted config options should require the unit suffix, falling back +to the default value if an invalid value is provided. Existing +duration/size-formatted config options should be changed to require the unit suffix +at the next major version. + +### APM Agent Configuration via Kibana + +Also known as "central configuration". Agents can query the APM Server for configuration updates; the server proxies and caches requests to Kibana. + +Agents should poll the APM Server for config periodically by sending an HTTP request to the `/config/v1/agents` endpoint. Agents must specify their service name, and optionally environment. The server will use these to filter the configuration down to the relevant service and environment. There are two methods for sending these parameters: + +1. Using the `GET` method, pass them as query parameters: `http://127.0.0.1:8200/config/v1/agents?service.name=opbeans&service.environment=production` +2. Using the `POST` method, encode the parameters as a JSON object in the body, e.g. `{"service": {"name": "opbeans", "environment": "production"}}` + +The server will respond with a JSON object, where each key maps a config attribute to a string value. The string value should be interpreted the same as if it were passed in via an environment variable. Upon receiving these config changes, the agent will update its configuration dynamically, overriding any config previously specified. That is, config via Kibana takes highest precedence. + +To minimise the amount of work required by users, agents should aim to enable this feature by default. This excludes RUM, where there is a performance penalty. + +#### Interaction with local config + +When an instrumented application starts, the agent should first load locally-defined configuration via environment variables, config files, etc. Once this has completed, the agent will begin asynchronously polling the server for configuration. Once available, this configuration will override the locally-defined configuration. This means that there will be a short time window at application startup in which locally-defined configuration will apply. + +If a user defines and then later deletes configuration via Kibana, the agent should ideally fall back to the locally-defined configuration. As an example of how to achieve this: the Java agent defines a hierarchy of configuration sources, with configuration via Kibana having the highest precedence. When configuration is not available at one level, the agent obtains it via the next highest level, and so on. + +#### Caching + +As mentioned above, the server will cache config for each unique `service.name`, `service.environment` pair. The server will respond to config requests with two related response headers: [Etag](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/ETag) and [Cache-Control](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control). + +Agents should keep a record of the `Etag` value returned by the most recent successful config request, and then present it to future requests via the [If-None-Match](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/If-None-Match) header. If the config has not changed, the server will respond with 304 (Not Modified). + +The `Cache-Control` header should contain a `max-age` directive, specifying the amount of time (in seconds) the response should be considered "fresh". Agents should use this to decide how long to wait before requesting config again. The server will respond with a `Cache-Control` header even if the request fails. + +#### Dealing with errors + +Agents must deal with various error scenarios, including: + + - 7.3 servers where the Kibana connection is not enabled (server responds with 403) + - 7.3 servers where the Kibana connection is enabled, but unavailable (server responds with 503) + - pre-7.3 servers that don't support the config endpoint (server responds with 404) + - any other error (server responds with 5xx) + +If the server responds with any 5xx, agents should log at error level. If the server responds with 4xx, agents are not required to log the response, but may choose to log it at debug level; either the central config feature is not available, or is not enabled. In either case, there is no expectation that the agent should take any action, so logging is not necessary. + +In any case, a 7.3+ server _should_ respond with a Cache-Control header, as described in the section above, and agents should retry after the specified interval. For older servers, or for whatever reason a 7.3+ server does not respond with that header (or it is invalid), agents should retry after 5 minutes. We include this behaviour for older servers so that the agent will start polling after server upgrade without restarting the application. + +If the agent does not recognise a config attribute, or does not support dynamically updating it, then it should log a warning such as: + +``` +Central config failure. Unsupported config names: unknown_option, disable_metrics, capture_headers +``` + +Note that in the initial implementation of this feature, not all config attributes will be supported by the APM UI or APM Server. Agents may choose to support only the attributes supported by the UI/server, or they may choose to accept additional attributes. The latter will enable them to work without change once additional config attributes are supported by the UI/server. + +If the agent receives a known but invalid config attribute, it should log a warning such as: + +``` +Central config failure. Invalid value for transactionSampleRate: 1.2 (out of range [0,1.0]) +``` + +Failure to process one config attribute should not affect processing of others. + +To ensure graceful recovery if the APM Server is offline or there is a network partition, agents should only retry at a maximum every 5 seconds, regardless of Cache-Control headers being less than that value. If the Cache-Control header is zero (or less than zero), it should be treated as missing (i.e. use default retry time). + +#### Feature flag + +Agents should implement a [configuration option](https://docs.google.com/spreadsheets/d/1JJjZotapacA3FkHc2sv_0wiChILi3uKnkwLTjtBmxwU), (`CENTRAL_CONFIG`) which lets users disable the central configuration polling. + +### Zero-configuration support + +To decrease onboarding friction, +APM agents MUST not require any configuration to send data to a local APM Server. +After onboarding, users can customize settings for which the defaults aren't appropriate. + +By default, agents MUST send data to the APM Server at `http://127.0.0.1:8200/`. +If possible, agents SHOULD detect sensible defaults for `service.name` and `service.version`. +In any case agents MUST include `service.name` - if discovering it is not possible then the default value: `unknown-${service.agent.name}-service` MUST be used. +This naming pattern allows the UI to display inline help on how to manually configure the name. + +### Adding a new configuration option + +#### Configuration option spec template + +When adding a new configuration option to the spec, please use the following template: + +```markdown +### configuration + +The description of the option. +Try to phrase it in a way that lets agents easily copy/paste the description into their configuration documentation page. +This should also match the description in the [central configuration](https://github.com/elastic/kibana/blob/main/x-pack/plugins/apm/common/agent_configuration/setting_definitions/general_settings.ts) in the APM UI. + +| | | +|----------------|-------------------------------------------| +| Type | See the Configuration Value Types section | +| Default | `` | +| Central config | `` | + +Additional description of the option that should not be part of the documentation. +For example, whether agents SHOULD or MUST implement the option. + +``` + +#### Adding to central configuration + +One of the things to determine when proposing to add a new configuration option is whether it should be added to central configuration. +In general, we should try to make the majority of options available in central configuration. +However, sometimes it might not be feasible if agents need to read the value at startup and changing the value at runtime is very difficult to achieve. + +When adding a centrally configurable option, +the spec owner is expected to create a pull request in the Kibana repository to add the option to central configuration. + +To make sure the tests are passing and to update some generated files, you need a local clone of Kibana and execute the build. +Follow [Kibana's contribution guide](https://github.com/elastic/kibana/blob/main/CONTRIBUTING.md) to get a local environment running. + +Add the configuration option(s) using this [example PR](https://github.com/elastic/kibana/pull/143668) as a reference. +By the time you're adding the configuration option, chances are that no agent has implemented the option, yet. +Therefore, set `includeAgents: []` in the option declaration. +When an agent implements the option, they just need to add the agent name to this section, without having to test the changes. +Please manually add a checkbox to all the implementation issues to make sure agents don't forget that step. + +Before creating a PR, execute the [jest tests and update the generated snapshot files](https://github.com/elastic/kibana/blob/main/x-pack/plugins/apm/dev_docs/testing.md#unit-tests-jest). +For testing purposes, comment out the `includeAgents` declaration and remember add it back in after testing. +This ensures that you can just select `all` services and `all` environments in the central configuration UI. +To test the changes, run Elasticsearch and Kibana as described in the contribution guide. +Just try out whether the option is rendered as expected and triple-check that the option name matches the one in the spec. + +When creating the PR, add the labels `release_note:enhancement`, `Team:APM`, and `v`. + +If everything works smoothly, request a review from the [apm-ui](https://github.com/orgs/elastic/teams/apm-ui) team. +If you need help, drop a message in the #apm-dev channel so that agent devs that did this before or UI devs can chime in. diff --git a/specs/agents/error-tracking.md b/specs/agents/error-tracking.md new file mode 100644 index 00000000..deaf59b6 --- /dev/null +++ b/specs/agents/error-tracking.md @@ -0,0 +1,30 @@ +## Error/exception tracking + +The agent support reporting exceptions/errors. Errors may come in one of two forms: + + - unhandled (or handled and explicitly reported) exceptions/errors + - log records + +Agents should include exception handling in the instrumentation they provide, such that exceptions are reported to the APM Server automatically, without intervention. In addition, hooks into logging libraries may be provided such that logged errors are also sent to the APM Server. + +Error properties +* `id` (which in the case of errors is 128 bits, encoded as 32 hexadecimal digits) + +Additional properties that agents SHOULD collect when the error happens within the context of a transaction +* `trace_id` +* `transaction_id` +* `parent_id` (which is the `id` of the transaction or span that caused the error). +* `transaction.sampled` +* `transaction.name`† +* `transaction.type`† + +† These properties may change during the lifetime of a transaction, for example if a user explicitly sets the transaction name after an error has been captured. +It is a known and accepted limitation that these properties are not always consistent with the transaction. +Agents MUST NOT buffer errors to ensure consistency as this comes at the expense of increased memory overhead. + +### Impact on the `outcome` + +Tracking an error that's related to a transaction does not impact its `outcome`. +A transaction might have multiple errors associated to it but still return with a 2xx status code. +Hence, the status code is a more reliable signal for the outcome of the transaction. +This, in turn, means that the `outcome` is always specific to the protocol. diff --git a/specs/agents/field-limits.md b/specs/agents/field-limits.md new file mode 100644 index 00000000..f9c8876a --- /dev/null +++ b/specs/agents/field-limits.md @@ -0,0 +1,46 @@ +## Field limits + +The maximum length of metadata, transaction, span, et al fields are determined +by the [APM Server Events Intake API schema](https://www.elastic.co/guide/en/apm/server/current/events-api.html). +Except for special cases, fields are typically limited to 1024 unicode characters. +Unless listed below as knowm "long fields", agents SHOULD truncate filed values to 1024 characters, as specified [below](#truncating-field-values). + +### Long fields + +Some APM event fields are not limited in the APM server intake API schema. +Such fields are considered "long fields". + +Agents SHOULD treat the following fields as long fields: + +- `transaction.context.request.body`, `error.context.request.body` +- `transaction.context.message.body`, `error.context.message.body` +- `span.context.db.statement` + +In addition, agents MAY treat the following fields as long fields: + +- `error.exception.message` +- `error.log.message` + +Agents SHOULD limit the maximum length of long fields by [truncating](#truncating-field-values) them to 10,000 unicode characters, +or based on user configuration for long field length, as specified [below](#long_field_max_length-configuration). + +### `long_field_max_length` configuration + +Agents MAY support the `long_field_max_length` configuration option to allow +the user to configure this maximum length. This option defines a maximum number +of unicode characters for each field. + +| | | +|----------------|-----------| +| Type | `Integer` | +| Default | `10000` | +| Dynamic | `false` | +| Central config | `false` | + +Ultimately the maximum length of any field is limited by the [`max_event_size`](https://www.elastic.co/guide/en/apm/server/current/configuration-process.html#max_event_size) +configured for the receiving APM server. + +### Truncating field values + +When field values exceed the maximum allowed number of unicode characters, agents SHOULD truncate the values to fit the maximum allowed length, +replacing the last character of the eventual value with the ellipsis character (unicode character `U+2026`: "…"). diff --git a/specs/agents/handling-huge-traces/README.md b/specs/agents/handling-huge-traces/README.md new file mode 100644 index 00000000..297e18bc --- /dev/null +++ b/specs/agents/handling-huge-traces/README.md @@ -0,0 +1,41 @@ +# Handling huge traces + +Instrumenting applications that make lots of requests (such as 10k+) to backends like caches or databases can lead to several issues: +- A significant performance impact in the target application. + For example due to high allocation rate, network traffic, garbage collection, additional CPU cycles for serializing, compressing and sending spans, etc. +- Dropping of events in agents or APM Server due to exhausted queues. +- High load on the APM Server. +- High storage costs. +- Decreased performance of the Elastic APM UI due to slow searches and rendering of huge traces. +- Loss of clarity and overview (--> decreased user experience) in the UI when analyzing the traces. + +Agents can implement several strategies to mitigate these issues. +These strategies are designed to capture significant information about relevant spans while at the same time limiting the trace to a manageable size. +Applying any of these strategies inevitably leads to a loss of information. +However, they aim to provide a better tradeoff between cost and insight by not capturing or summarizing less relevant data. + +- [Hard limit on number of spans to collect](tracing-spans-limit.md) \ + Even after applying the most advanced strategies, there must always be a hard limit on the number of spans we collect. + This is the last line of defense that comes with the highest amount of data loss. +- [Collecting statistics about dropped spans](tracing-spans-dropped-stats.md) \ + Makes sure even if dropping spans, we at least have stats about them. +- [Dropping fast exit spans](tracing-spans-drop-fast-exit.md) \ + If a span was blazingly fast, it's probably not worth the cost to send and store it. +- [Compressing spans](tracing-spans-compress.md) \ + If there are a bunch of very similar spans, we can represent them in a single document - a composite span. + +In a nutshell, this is how the different settings work in combination: + +```java +if (span.transaction.spanCount > transaction_max_spans) { + // drop span + // collect statistics for dropped spans +} else if (compression possible) { + // apply compression +} else if (span.duration < exit_span_min_duration) { + // drop span + // collect statistics for dropped spans +} else { + // report span +} +``` diff --git a/specs/agents/handling-huge-traces/tracing-spans-compress.md b/specs/agents/handling-huge-traces/tracing-spans-compress.md new file mode 100644 index 00000000..1583aca0 --- /dev/null +++ b/specs/agents/handling-huge-traces/tracing-spans-compress.md @@ -0,0 +1,305 @@ +# Compressing spans + +To mitigate the potential flood of spans to a backend, +agents SHOULD implement the strategies laid out in this section to avoid sending almost identical and very similar spans. + +While compressing multiple similar spans into a single composite span can't fully eliminate the collection overhead, +it can significantly reduce the impact on the following areas, +with very little loss of information: +- Agent reporter queue utilization +- Capturing stack traces, serialization, compression, and sending events to APM Server +- Potential to re-use span objects, significantly reducing allocations +- Downstream effects like reducing impact on APM Server, ES storage, and UI performance + +### Configuration option `span_compression_enabled` + +Setting this option to true will enable span compression feature. +Span compression reduces the collection, processing, and storage overhead, and removes clutter from the UI. +The tradeoff is that some information such as DB statements of all the compressed spans will not be collected. + +| | | +|----------------|----------| +| Type | `boolean`| +| Default | `true` | +| Dynamic | `true` | + + +## Consecutive-Exact-Match compression strategy + +One of the biggest sources of excessive data collection are n+1 type queries and repetitive requests to a cache server. +This strategy detects consecutive spans that hold the same information (except for the duration) +and creates a single [composite span](#composite-span). + +``` +[ ] +GET /users + [] [] [] [] [] [] [] [] [] [] + 10x SELECT FROM users +``` + +Two spans are considered to be an exact match if they are of the [same kind](#consecutive-same-kind-compression-strategy) and if their span names are equal: +- `type` +- `subtype` +- `destination.service.resource` (DEPRECATED, replaced by `service.target.{type,name}` ) +- `service.target.type` +- `service.target.name` +- `name` + +### Configuration option `span_compression_exact_match_max_duration` + +Consecutive spans that are exact match and that are under this threshold will be compressed into a single composite span. +This option does not apply to [composite spans](#composite-span). +This reduces the collection, processing, and storage overhead, and removes clutter from the UI. +The tradeoff is that the DB statements of all the compressed spans will not be collected. + +| | | +|----------------|----------| +| Type | `duration`| +| Default | `50ms` | +| Dynamic | `true` | + +## Consecutive-Same-Kind compression strategy + +Another pattern that often occurs is a high amount of alternating queries to the same backend. +Especially if the individual spans are quite fast, recording every single query is likely to not be worth the overhead. + +``` +[ ] +GET /users + [] [] [] [] [] [] [] [] [] [] + 10x Calls to mysql +``` + +Two spans are considered to be of the same type if the following properties are equal: +- `type` +- `subtype` +- `destination.service.resource` (DEPRECATED, replaced by `service.target.{type,name}` ) +- `service.target.type` +- `service.target.name` + + +```java +boolean isSameKind(Span other) { + return type == other.type + && subtype == other.subtype + && service.target.type == other.service.target.type + && service.target.name == other.service.target.name +} +``` + +When applying this compression strategy, the `span.name` is set to `Calls to <...>` determined from `service.target.*` as follows: + +```js +function getCompositeSpanName(serviceTarget) { + const prefix = 'Calls to ' + if (!serviceTarget.type) { + if (!serviceTarget.name) { + return prefix + 'unknown' + } else { + return prefix + serviceTarget.name + } + } else if (!serviceTarget.name) { + return prefix + serviceTarget.type + } else { + return prefix + serviceTarget.type + '/' + serviceTarget.name + } +} +``` + +The rest of the context, such as the `db.statement` will be determined by the first compressed span, which is turned into a composite span. + +### Configuration option `span_compression_same_kind_max_duration` + +Consecutive spans to the same destination that are under this threshold will be compressed into a single composite span. +This option does not apply to [composite spans](#composite-span). +This reduces the collection, processing, and storage overhead, and removes clutter from the UI. +The tradeoff is that the DB statements of all the compressed spans will not be collected. + +| | | +|----------------|----------| +| Type | `duration`| +| Default | `0ms` | +| Dynamic | `true` | + +## Composite span + +Compressed spans don't have a physical span document. +Instead, multiple compressed spans are represented by a composite span. + +### Data model + +The `timestamp` and `duration` have slightly similar semantics, +and they define properties under the `composite` context. + +- `timestamp`: The start timestamp of the first span. +- `duration`: gross duration (i.e., __ - __). +- `composite` + - `count`: The number of compressed spans this composite span represents. + The minimum count is 2 as a composite span represents at least two spans. + - `sum`: sum of durations of all compressed spans this composite span represents in milliseconds. + Thus `sum` is the net duration of all the compressed spans while `duration` is the gross duration (including "whitespace" between the spans). + - `compression_strategy`: A string value indicating which compression strategy was used. The valid values are: + - `exact_match` - [Consecutive-Exact-Match compression strategy](tracing-spans-compress.md#consecutive-exact-match-compression-strategy) + - `same_kind` - [Consecutive-Same-Kind compression strategy](tracing-spans-compress.md#consecutive-same-kind-compression-strategy) + +### Effects on metric processing + +As laid out in the [span destination spec](../tracing-spans-destination.md#contextdestinationserviceresource), +APM Server tracks span destination metrics. +To avoid compressed spans to skew latency metrics and cause throughput metrics to be under-counted, +APM Server will take `composite.count` into account when tracking span destination metrics. + +### Effects on [span count](https://github.com/elastic/apm/blob/main/specs/agents/handling-huge-traces/tracing-spans-limit.md#span-count) + +When a span is compressed into a composite, +`span_count.reported` should ONLY count the compressed composite as a single +span. +Spans that have been compressed into the composite should not be counted. + +## Compression algorithm + +### Eligibility for compression + +A span is eligible for compression if all the following conditions are met +1. It's an [exit span](../tracing-spans.md#exit-spans) +2. The trace context of this span has not been propagated to a downstream service +3. If the span has `outcome` (i.e., `outcome` is present and it's not `null`) then it should be `success`. + It means spans with outcome indicating an issue of potential interest should not be compressed. + +The second condition is important so that we don't remove (compress) a span that may be the parent of a downstream service. +This would orphan the sub-graph started by the downstream service and cause it to not appear in the waterfall view. + +```java +boolean isCompressionEligible() { + return exit && !context.hasPropagated && (outcome == null || outcome == "success") +} +``` + +### Span buffering + +When a span ends, if it is not compression-eligible or if its parent has already +ended, it may be reported immediately. Otherwise, it does not immediately get +reported. +Instead, the span is buffered within its parent. +A span/transaction can buffer at most one child span. + +Span buffering allows to "look back" one span when determining whether a given span should be compressed. + +A buffered span gets reported when +1. its parent ends +2. a non-compressible sibling ends + +```java +void onEnd() { + if (buffered != null) { + report(buffered) + } +} + +void onChildEnd(Span child) { + if (ended || !child.isCompressionEligible()) { + if (buffered != null) { + report(buffered) + buffered = null + } + report(child) + return + } + + if (buffered == null) { + buffered = child + return + } + + if (!buffered.tryToCompress(child)) { + report(buffered) + buffered = child + } +} +``` + +### Turning compressed spans into a composite span + +Spans have `tryToCompress` method that is called on a span buffered by its parent. +On the first call the span checks if it can be compressed with the given sibling and it selects the best compression strategy. +Note that the compression strategy selected only once based on the first two spans of the sequence. +The compression strategy cannot be changed by the rest the spans in the sequence. +So when the current sibling span cannot be added to the ongoing sequence under the selected compression strategy +then the ongoing is terminated, it is sent out as a composite span and the current sibling span is buffered. + +If the spans are of the same kind, and have the same name and both spans `duration` <= `span_compression_exact_match_max_duration`, +we apply the [Consecutive-Exact-Match compression strategy](tracing-spans-compress.md#consecutive-exact-match-compression-strategy). +Note that if the spans are _exact match_ +but duration threshold requirement is not satisfied we just stop compression sequence. +In particular it means that the implementation should not proceed to try _same kind_ strategy. +Otherwise user would have to lower both `span_compression_exact_match_max_duration` and `span_compression_same_kind_max_duration` +to prevent longer _exact match_ spans from being compressed. + +If the spans are of the same kind but have different span names and both spans `duration` <= `span_compression_same_kind_max_duration`, +we compress them using the [Consecutive-Same-Kind compression strategy](tracing-spans-compress.md#consecutive-same-kind-compression-strategy). + +```java +bool tryToCompress(Span sibling) { + isAlreadyComposite = composite != null + canBeCompressed = isAlreadyComposite ? tryToCompressComposite(sibling) : tryToCompressRegular(sibling) + if (!canBeCompressed) { + return false + } + + if (!isAlreadyComposite) { + composite.count = 1 + composite.sum = duration + } + + ++composite.count + composite.sum += other.duration + return true +} + +bool tryToCompressRegular(Span sibling) { + if (!isSameKind(sibling)) { + return false + } + + if (name == sibling.name) { + if (duration <= span_compression_exact_match_max_duration && sibling.duration <= span_compression_exact_match_max_duration) { + composite.compressionStrategy = "exact_match" + return true + } + return false + } + + if (duration <= span_compression_same_kind_max_duration && sibling.duration <= span_compression_same_kind_max_duration) { + composite.compressionStrategy = "same_kind" + name = "Calls to " + destination.service.resource + return true + } + + return false +} + +bool tryToCompressComposite(Span sibling) { + switch (composite.compressionStrategy) { + case "exact_match": + return isSameKind(sibling) && name == sibling.name && sibling.duration <= span_compression_exact_match_max_duration + + case "same_kind": + return isSameKind(sibling) && sibling.duration <= span_compression_same_kind_max_duration + } +} +``` + +### Concurrency + +The pseudo-code in this spec is intentionally not written in a thread-safe manner to make it more concise. +Also, thread safety is highly platform/runtime dependent, and some don't support parallelism or concurrency. + +However, if there can be a situation where multiple spans may end concurrently, agents MUST guard against race conditions. +To do that, agents should prefer [lock-free algorithms](https://en.wikipedia.org/wiki/Non-blocking_algorithm) +paired with retry loops over blocking algorithms that use mutexes or locks. + +In particular, operations that work with the buffer require special attention: +- Setting a span into the buffer must be handled atomically. +- Retrieving a span from the buffer must be handled atomically. + Retrieving includes atomically getting and clearing the buffer. + This makes sure that only one thread can compare span properties and call mutating methods, such as `compress` at a time. diff --git a/specs/agents/handling-huge-traces/tracing-spans-drop-fast-exit.md b/specs/agents/handling-huge-traces/tracing-spans-drop-fast-exit.md new file mode 100644 index 00000000..f55ec92d --- /dev/null +++ b/specs/agents/handling-huge-traces/tracing-spans-drop-fast-exit.md @@ -0,0 +1,77 @@ +# Dropping fast exit spans + +If an exit span was really fast, chances are that it's not relevant for analyzing latency issues. +Therefore, agents SHOULD implement the strategy laid out in this section to let users choose the level of detail/cost tradeoff that makes sense for them. +If an agent implements this strategy, it MUST also implement [Collecting statistics about dropped spans](tracing-spans-dropped-stats.md). + +## `exit_span_min_duration` configuration + +Sets the minimum duration of exit spans. +Exit spans with a duration lesser than this threshold are attempted to be discarded. +If the exit span is equal or greater the threshold, it should be kept. + +In some cases exit spans cannot be discarded. +For example, spans that propagate the trace context to downstream services, +such as outgoing HTTP requests, +can't be discarded. +However, external calls that don't propagate context, +such as calls to a database, can be discarded using this threshold. + +Additionally, spans that lead to an error can't be discarded. + +| | | +|----------------|------------| +| Type | [`GranularDuration`](../configuration.md#configuration-value-types) | +| Default | `0ms` | +| Central config | `true` | + +## Interplay with span compression + +If an agent implements [span compression](tracing-spans-compress.md), +the limit applies to the [composite span](tracing-spans-compress.md#composite-span). + +For example, if 10 Redis calls are compressed into a single composite span whose total duration is lower than `exit_span_min_duration`, +it will be dropped. +If, on the other hand, the individual Redis calls are below the threshold, +but the sum of their durations is above it, the composite span will not be dropped. + +## Limitations + +The limitations are based on the premise that the `parent_id` of each span and transaction that's stored in Elasticsearch +should point to another valid transaction or span that's present in the Elasticsearch index. + +A span that refers to a missing span via is `parent_id` is also known as an "orphaned span". + +### Spans that propagate context to downstream services can't be discarded + +We only know whether to discard after the call has ended. +At that point, +the trace has already continued on the downstream service. +Discarding the span for the external request would orphan the transaction of the downstream call. + +Propagating the trace context to downstream services is also known as out-of-process context propagation. + +## Implementation + +### `discardable` flag + +Spans store an additional `discardable` flag in order to determine whether a span can be discarded. +The default value is `true` for [exit spans](../tracing-spans.md#exit-spans) and `false` for any other span. + +According to the [limitations](#Limitations), +there are certain situations where the `discardable` flag of a span is set to `false`: +- the span's `outcome` field is set to anything other than `success`. + So spans with outcome indicating an issue of potential interest are not discardable +- On out-of-process context propagation + +### Determining whether to report a span + +If the span's duration is less than `exit_span_min_duration` and the span is discardable (`discardable=true`), +the `span_count.dropped` count is incremented, and the span will not be reported. +We're deliberately using the same dropped counter we also use when dropping spans due to [`transaction_max_spans`](tracing-spans-limit.md#configuration-option-transaction_max_spans). +This ensures that a dropped fast span doesn't consume from the max spans limit. + +### Metric collection + +To reduce the data loss, agents [collect statistics about dropped spans](tracing-spans-dropped-stats.md). +Dropped spans contribute to [breakdown metrics](https://docs.google.com/document/d/1-_LuC9zhmva0VvLgtI0KcHuLzNztPHbcM0ZdlcPUl64#heading=h.ondan294nbpt) the same way as non-discarded spans. diff --git a/specs/agents/handling-huge-traces/tracing-spans-dropped-stats.md b/specs/agents/handling-huge-traces/tracing-spans-dropped-stats.md new file mode 100644 index 00000000..bff9db49 --- /dev/null +++ b/specs/agents/handling-huge-traces/tracing-spans-dropped-stats.md @@ -0,0 +1,66 @@ +# Collecting statistics about dropped spans + +To still retain some information about dropped spans (for example due to [`transaction_max_spans`](tracing-spans-limit.md) or [`exit_span_min_duration`](tracing-spans-drop-fast-exit.md)), +agents SHOULD collect statistics on the corresponding transaction about dropped spans. +These statistics MUST only be sent for sampled transactions. + +Agents SHOULD only collect these statistics for exit spans that have a non-empty `service.target.type` (and `service.target.name`), +or a non-empty `destination.service.resource` if they donΒ΄t use [Service Target fields](../tracing-spans-service-target.md) + +This feature used to rely on the deprecated `destination.service.resource` field, which is replaced by `service.target.type` +and `service.target.name`. +However, in order to preserve compatibility, we still need to provide its value in dropped spans metrics. + +## Use cases + +This allows APM Server to consider these metrics for the service destination metrics. +In practice, +this means that the service map, the dependencies table, +and the backend details view can show accurate throughput statistics for backends like Redis, +even if most of the spans are dropped. + +## Data model + +This is an example of the statistics that are added to the `transaction` events sent via the intake v2 protocol. + +```json +{ + "dropped_spans_stats": [ + { + "destination_service_resource": "example.com:443", + "service_target_type": "http", + "service_target_name": "example.com:443", + "outcome": "failure", + "duration.count": 28, + "duration.sum.us": 123456 + }, + { + "destination_service_resource": "mysql", + "service_target_type": "mysql", + "outcome": "success", + "duration.count": 81, + "duration.sum.us": 9876543 + } + ] +} +``` + +### Compatibility + +When the `service_target_*` fields are provided, APM server has to use those fields to identify the destination. + +When the `service_target_*` fields are not provided, APM server has to infer equivalent values using the algorigthm +described in [Service Target Fields](../tracing-spans-service-target.md). + +## Limits + +To avoid the structures from growing without bounds (which is only expected in pathological cases), +agents MUST limit the size of the `dropped_spans_stats` to 128 entries per transaction. +Any entries that would exceed the limit are silently dropped. + +## Effects on destination service metrics + +As laid out in the [span destination spec](tracing-spans-destination.md#contextdestinationserviceresource), +APM Server tracks span destination metrics. +To avoid dropped spans to skew latency metrics and cause throughput metrics to be under-counted, +APM Server will take `dropped_spans_stats` into account when tracking span destination metrics. diff --git a/specs/agents/handling-huge-traces/tracing-spans-limit.md b/specs/agents/handling-huge-traces/tracing-spans-limit.md new file mode 100644 index 00000000..04ca90cd --- /dev/null +++ b/specs/agents/handling-huge-traces/tracing-spans-limit.md @@ -0,0 +1,92 @@ +# Hard limit on number of spans to collect + +This is the last line of defense that comes with the highest amount of data loss. +This strategy MUST be implemented by all agents. +Ideally, the other mechanisms limit the amount of spans enough so that the hard limit does not kick in. + +Agents SHOULD also [collect statistics about dropped spans](tracing-spans-dropped-stats.md) when implementing this spec. + +## Configuration option `transaction_max_spans` + +Limits the amount of spans that are recorded per transaction. + +This is helpful in cases where a transaction creates a very high amount of spans (e.g. thousands of SQL queries). + +Setting an upper limit will prevent overloading the agent and the APM server with too much work for such edge cases. + +| | | +|----------------|----------| +| Type | `integer`| +| Default | `500` | +| Dynamic | `true` | + +## Implementation + +### Span count + +When a span is put in the agent's reporter queue, a counter should be incremented on its transaction, in order to later identify the _expected_ number of spans. +In this way we can identify data loss, e.g. because events have been dropped. + +This counter SHOULD internally be named `reported` and MUST be mapped to `span_count.started` in the intake API. +The word `started` is a misnomer but needs to be used for backward compatibility. +The rest of the spec will refer to this field as `span_count.reported`. + +When a span is dropped, it is not reported to the APM Server, +instead another counter is incremented to track the number of spans dropped. +In this case the above mentioned counter for `reported` spans is not incremented. + +```json +"span_count": { + "started": 500, + "dropped": 42 +} +``` + +The total number of spans that an agent created within a transaction is equal to `span_count.started + span_count.dropped`. +Note that this might be an under count, because spans that end *after* their +transaction has been reported (typically when the transaction ends) will not be +counted. + +### Checking the limit + +Before creating a span, +agents must determine whether that span would exceed the span limit. +The limit is reached when the number of reported spans is greater or equal to the max number of spans. +In other words, the limit is reached if this condition is true: + + atomic_get(transaction.span_count.eligible_for_reporting) >= transaction_max_spans + +On span end, agents that support the concurrent creation of spans need to check the condition again. +That is because any number of spans may be started before any of them end. + +```java +if (atomic_get(transaction.span_count.eligible_for_reporting) <= transaction_max_spans // optional optimization + && atomic_get_and_increment(transaction.span_count.eligible_for_reporting) <= transaction_max_spans ) { + should_be_reported = true + atomic_increment(transaction.span_count.reported) +} else { + should_be_reported = false + atomic_increment(transaction.span_count.dropped) + transaction.track_dropped_stats(this) +} +``` + +`eligible_for_reporting` is another counter in the span_count object, but it's not reported to APM Server. +It's similar to `reported` but the value may be higher. + +### Configuration snapshot + +To ensure consistent behavior within one transaction, +the `transaction_max_spans` option should be read once on transaction start. +Even if the option is changed via remote config during the lifetime of a transaction, +the value that has been read at the start of the transaction should be used. + +### Metric collection + +Even though we can determine whether to drop a span before starting it, it's not legal to return a `null` or noop span in that case. +That's because we're [collecting statistics about dropped spans](tracing-spans-dropped-stats.md) as well as +[breakdown metrics](https://docs.google.com/document/d/1-_LuC9zhmva0VvLgtI0KcHuLzNztPHbcM0ZdlcPUl64#heading=h.ondan294nbpt) +even for spans that exceed `transaction_max_spans`. + +For spans that are known to be dropped upfront, Agents SHOULD NOT collect information that is expensive to get and not needed for metrics collection. +This includes capturing headers, request bodies, and summarizing SQL statements, for example. diff --git a/specs/agents/log-correlation.md b/specs/agents/log-correlation.md new file mode 100644 index 00000000..eb1c182a --- /dev/null +++ b/specs/agents/log-correlation.md @@ -0,0 +1,87 @@ +## Log correlation + +Agents should provide instrumentation/hooks for popular logging libraries in order to decorate structured log records with trace context. +In particular, logging that occurs within the context of a transaction should add the fields `trace.id` and `transaction.id`; +logging that occurs within a span should add the fields `trace.id` and optionally `transaction.id`. + +By adding trace context to log records, users will be able to move between the APM UI and Logs UI. + +Logging frameworks and libraries may provide a way to inject key-value pairs in log messages, +this allows to reuse those fields in log message formats (for example in plain text). + +Log correlation relies on two sets of fields: +- [metadata fields](#service-correlation-fields) + - They allow to build the per-service logs view in UI. + - They are implicitly provided when using log-sending by the agent metadata. + - When using ECS logging, they might be set by the application. +- [per-log-event fields](#trace-correlation-fields): `trace.id`, `transaction.id` and `error.id` + - They allow to build the per-trace/transaction/error logs view in UI. + - They are added to the log event + - They must be written in each log event document + +The values for those fields can be set in two places: +- when using [ecs-logging](https://github.com/elastic/ecs-logging) directly in the application +- when the agent reformats a log event + +The values set at the application level have higher priority than the values set by agents. +Agents must provide fallback values if they are not explicitly set by the application. + +In case the values set in the application and agent configuration differ, the resulting log +messages won't correlate to the expected service in UI. In order to prevent such inconsistencies +agents may issue a warning when there is a mis-configuration. + +### Service correlation fields + +They allow to build the per-service logs view in UI. +They are implicitly provided when using log-sending by the agent metadata. +When using ECS logging, they might be set by the application in ECS logging configuration. + +- `service.name`: + - used to filter/link log messages to a given service. + - must be provided even if there is no active transaction + - Configuration source (in order of precedence): + - Configured value + - `ELASTIC_APM_SERVICE_NAME` + - `OTEL_SERVICE_NAME` + - `OTEL_RESOURCE_ATTRIBUTES` value for `service.name` + - Default from Elastic Agent (if available) +- `service.version`: + - only used for service metadata correlation + - must be provided even if there is no active transaction + - Configuration source (in order of precedence): + - Configured value + - `ELASTIC_APM_SERVICE_VERSION` + - `OTEL_RESOURCE_ATTRIBUTES` value for `service.version` + - Default from Elastic Agent (if available) +- `service.environment`: + - allows to filter/link log messages to a given service/environment. + - must be provided even if there is no active transaction + - Configuration source (in order of precedence): + - Configured value + - `ELASTIC_APM_ENVIRONMENT` + - `OTEL_RESOURCE_ATTRIBUTES` value for `deployment.environment` + - Default from Elastic Agent (if available) +- `service.node.name`: + - must be provided even if there is no active transaction + - Configuration source (in order of precedence): + - Configured value + - `ELASTIC_APM_SERVICE_NODE_NAME` + - `OTEL_RESOURCE_ATTRIBUTES` value for `service.instance.id` + - Default from Elastic Agent (if available) + + +The `container.id` field can also be used as a fallback to provide service-level correlation in UI, however agents ARE NOT expected to set it: + +- log collector (filebeat) is expected to do that when ingesting logs. +- all data sent through agent intake implicitly provides `container.id` through metadata, which also includes the log events that may be sent to apm-server. + +### Trace correlation fields + +They allow to build the per-trace/transaction/error logs view in UI. +They allow to navigate from the log event to the trace/transaction/error in UI. +They should be added to the log event. +They must be written in each log event document they relate to, either reformatted or sent by the agent. + +- `trace.id` +- `transaction.id` +- `error.id` diff --git a/specs/agents/log-reformatting.md b/specs/agents/log-reformatting.md new file mode 100644 index 00000000..4a94d1f4 --- /dev/null +++ b/specs/agents/log-reformatting.md @@ -0,0 +1,107 @@ +# Log reformatting + +The Agents will be a critical part of log collection onboarding for their +application logs. This is primarily accomplished via the `log_ecs_reformatting` +configuration option, described below. + +In future iterations, the shipping of ECS logs will become more automated by auto-parsing ECS-JSON logs in Filebeat +and [automatically shipping log files](https://github.com/elastic/apm/issues/374) that got reformatted via +`log_ecs_reformatting`. + +## `log_ecs_reformatting` configuration + +Configures the agent to automatically format application logs as ECS-compatible JSON +(if possible). + +The configuration option must be marked experimental for now to allow for breaking changes we may need to introduce. +Once the end-to-end process for seamless log onboarding with Elastic Agent works, we'll remove the experimental flag. + +As the implementation of this configuration option will be specific for each supported log library, +the supported technologies documentation should list the supported frameworks (including supported version ranges) +and the agent version that introduced support for each logging library. + +| | | +|----------------|---| +| Valid options | `override`, `off` (case insensitive) | +| Default | `off` | +| Dynamic | `false` | +| Central config | `false` | + +Not all agents will be able to automatically format logs in this way. Those +agents should not implement this configuration option. + +For some agents, additional options makes sense. For example, the Java agent +also accepts the values `shade` and `replace`, where ECS-reformatted logs are written to a dedicated `.ecs.json` +file in addition to (`shade`) or instead of (`replace`) the original log stream. + +When this option is set to `override`, the agent should format all logs from the +app as ECS-compatible json, as shown in the +[spec](https://github.com/elastic/ecs-logging/blob/main/spec/spec.json). + +For all options other than `off`, the [log correlation](log-correlation.md) should be implicitly enabled. + +## `log_ecs_formatter_allow_list` configuration + +Only formatters that match an item on this list will be automatically reformatted to ECS when `log_ecs_reformatting` is +set to any option other than `off`. A "formatter" is a generic name used to describe the logging-framework-specific entity +that is responsible for the formatting of log events. Currently this option is only implemented in the Java agent, where +formatters are subtypes of `Layout` or `Encoder`, depending on the logging framework. + +| | | +|----------------|---| +| Type | `List<`[`WildcardMatcher`](../../tests/agents/json-specs/wildcard_matcher_tests.json)`>` | +| Default | agent specific | +| Dynamic | `false` | +| Central config | `false` | + +## Required fields + +The following fields are required: + +* `@timestamp` +* `log.level` +* `message` +* `ecs.version` + +## Recommended fields + +### `service.name` + +See [Log correlation](log-correlation.md) + +### `service.version` + +See [Log correlation](log-correlation.md) + +### `service.environment` + +See [Log correlation](log-correlation.md) + +### `event.dataset` + +The `event.dataset` field is used to power the [log anomaly chart in the logs UI](https://www.elastic.co/guide/en/observability/current/inspect-log-anomalies.html#anomalies-chart). +The dataset can also be useful to filter for different log streams from the same pod, for example. +This field should be a step more granular than +`service.name` where possible. However, the cardinality of this field should be +limited, so per-class or per-file logger names are not appropriate for this +field. + +A good example is in the Java agent, where `event.dataset` is set to +`${service.name}.${appender.name}`, where `appender.name` is the name of the +log appender. + +If an agent doesn't have reasonable options for this field, it should be set +to `${service.name}`. + +Some examples: +- opbeans +- opbeans.checkout +- opbeans.login +- opbeans.audit + +## Testing + +Due to differences in the possible Agent implementations of this feature, no +Gherkin spec is provided. Testing will primarily be accomplished via Opbeans. +Each Agent team should update their Opbeans app so that it only relies on this +configuration option to format ECS logs that will be picked up by Filebeat. diff --git a/specs/agents/log-sending.md b/specs/agents/log-sending.md new file mode 100644 index 00000000..c5d2f039 --- /dev/null +++ b/specs/agents/log-sending.md @@ -0,0 +1,61 @@ +# Log sending + +### `log_sending` configuration + +**Warning**: experimental feature, may be subject to change until GA. Also, only a small subset of agents will provide it before GA. + +Controls the ability to send logs directly from the agent to APM server. + +| | | +|----------------|-----------------| +| Valid options | `true`, `false` | +| Default | `false` | +| Dynamic | `true` | +| Central config | `true` | + +When set to `true`, the agent will send log events to apm-server. +Original log events are unaltered and written to their usual destinations (file, stdout, ...). + +The APM server only supports log events as of version 8.6+, thus trying to use this with an older version should +issue a warning/error in the agent logs. + +### Log event format + +On the agent side, there are two ways to get an ECS-formatted log message from a log event: +- The application already uses [ecs-logging](https://github.com/elastic/ecs-logging) +- The agent embeds a copy of [ecs-logging](https://github.com/elastic/ecs-logging), which might also be used for [log reformatting](./log-reformatting.md). + +In both cases, the output of [ecs-logging](https://github.com/elastic/ecs-logging) can be reused as follows: + +``` +{"log":}\n` +``` + +The ECS logging event `` must not include an `EOL` character in order to preserve the ND-JSON +format where each event is written to a single line. + +### Log event fields + +The ECS logging fields are the same as the ones defined in log reformatting: +- [required fields](./log-reformatting.md#required-fields) +- [recommended fields](./log-reformatting.md#recommended-fields) + +However, the values of `service.name` and `service.version` can be omitted as they are redundant to the values that are +already sent in the [ND-JSON metadata](metadata.md). In the case where the formatted ECS log event already contains +them, the agent may send the event as-is, rather than rewriting the event in order to reduce overhead. + +### Agent log + +When `log_sending` option is enabled, agents may also send their own logs to APM server. + +Agents usually have internal debug/trace logging statements that allow to diagnose communication issues and serialized data +sent to APM server. Special care must be taken to ensure that sending APM agent logs do not trigger an exponential loop +of log events or excessively large log event. +For APM agent logs, ignoring those log statements is an acceptable compromise -- diagnosis of agent errors in serializing or communicating with APM server may rely on local logging. + +When the agent starts, agent log events might require some limited buffering until the agent initialization is complete. +This allows to capture the early log messages when the agent initializes which often provide details about the agent +setup and configuration which are required for support. + +For the `event.dataset` field, the `${service.name}.apm-agent` value should be used to allow keeping application logs +and agent logs separate if needed. diff --git a/specs/agents/logging.md b/specs/agents/logging.md new file mode 100644 index 00000000..df342e01 --- /dev/null +++ b/specs/agents/logging.md @@ -0,0 +1,177 @@ +# Agent logging + +## `log_level` configuration + +Sets the logging level for the agent. + +This option is case-insensitive. + +| | | +|----------------|---| +| Valid options | `trace`, `debug`, `info`, `warning`, `error`, `critical`, `off` | +| Default | `info` (soft default) | +| Dynamic | `true` | +| Central config | `true` | + +Note that this default is not enforced among all agents. +If an agent development team thinks that a different default should be used +(such as `warning`), that is acceptable. + +## Mapping to native log levels + +Not all logging frameworks used by the different agents can natively work with these levels. +Thus, agents will need to translate them, using their best judgment for the mapping. + +Some examples: +If the logging framework used by an agent doesn't have `trace`, +it would map it to the same level as `debug`. +If the underlying logging framework doesn't support `critical`, +agents can treat that as a synonym for `error` or `fatal`. + +The `off` level is a switch to completely turn off logging. + +## Backwards compatibility + +Most agents have already implemented `log_level`, +accepting a different set of levels. +Those agents should still accept their "native" log levels to preserve backwards compatibility. +However, in central config, +there will only be a dropdown with the levels that are consistent across agents. +Also, the documentation should not mention the old log levels going forward. + +## Logging + +Agents may provide the following log-related features: + +- [Log correlation](log-correlation.md): inject service metadata, trace IDs and error IDs in log events. +- [Log reformatting](log-reformatting.md): reformat plain-text logs to ECS, equivalent to using [ecs logging](https://github.com/elastic/ecs-logging) +without modifying the application nor its dependencies. +- [Log sending](log-sending.md): send logs directly to APM server. + +## Logging Preamble + +The intention of this logging preamble is to ensure agent supportability. Relevant +data about an agent (e.g. version) and the environment it is running in (e.g. host, +operating system) should be provided in it. + +All agents MUST print this preamble on startup using the `info` logging level unless +a different level is explicitly mentioned. + +The agent logging preamble consists of 3 blocks: + +* **Agent**: This block is mandatory and contains basic version and build date information. +* **Environment**: This block is optional but for supportability reasons it should be provided. +* **Configuration**: This block is mandatory and contains a minimum set of relevant configuration values. + +**Note** that this specification does not prescribe a specific format to be used for creating +the log messages. It is up to the implementing agent to chose a format (e.g. ecs-logging format). + +### Agent + +On startup, all APM agents MUST log basic information regarding their technology (language, runtime), +and version information. +This log message MUST provide sufficient data to uniquely identify the agent build that generated the +log message. Hence, if e.g. the version information is not sufficient, agents +MUST include further information (e.g. build timestamp, git hash) that uniquely identifies an agent build. + +This SHOULD be the very first log message that is created by an agent. + +Example: + +```text +Elastic APM .NET Agent, version: 1.19.1-preview, build date: 2022-10-27 10:55:42 UTC +``` + +Agents SHOULD also detect when they are running in a non-final version (e.g. a debug +or pre-release build) and report that fact using the `warning` logging level. + +Example: + +```text +This is a pre-release version and not intended for use in production environments! +``` + +### Environment + +Additionally, agents SHOULD report information about their environment (e.g. host, process, runtime). + +| Item | Description | Example | +| - | - | - | +| Process ID | The Process ID in decimal format. | `83606` | +| Process Name | The executable image name or the full path to it. | `w3wp.exe`, `/usr/local/share/dotnet/dotnet` | +| Command Line | The full command line used to launch this process as available to the runtime. [1] | `/Users/acme/some_app/bin/Debug/net7.0/some_app.dll foo=bar` | +| Operating System | OS name and version in a human-readable format. | `macOS Version 12.6.1 (build 21G217)` | +| CPU architecture | See table below. | `arm64` | +| Host | The (optionally fully-qualified) host name. | `MacBook-Pro.localdomain` | +| Time zone | The local time zone in UTC-offset notation. | `UTC+0200` | +| Runtime | Name and version of the executing runtime. | `.NET Framework 4.8.4250.0`| +| Framework | Name and version of the instrumented framework. | `Django 4.1.3`, `ASP.NET 4.8.4494.0`| + +[1]: Due to privacy concerns in the past (see e.g. [here](https://github.com/elastic/apm-agent-nodejs/issues/1916)), +agents may decide to not log this information. + +**CPU Architecture:** + +This table provides an exemplary list of well-known values for reporting the CPU architecture. +An agent can decide to use different values that might be readily available to their language/runtime +ecosystem (e.g. Node.js' `os.arch()`). + +| Value | Description | +| - | - | +| `amd64` | AMD64 | +| `arm32` |ARM32 | +| `arm64` |ARM64 | +| `ia64` | Itanium | +| `ppc32` | 32-bit PowerPC | +| `ppc64` | 64-bit PowerPC | +| `s390x` | IBM z/Architecture | +| `x86` | 32-bit x86 | + +### Configuration + +The start of the configuration block MUST be denoted as such (e.g. `Agent Configuration:`). + +If configuration files are used in the configuration process, their fully-qualified paths +SHOULD be logged. + +Configuration item names SHOULD be provided in normalized (lower-case, snake_case) notation. +Configuration value strings MUST be printed in quotes (so accidental leading or trailing whitespace can be spotted). + +Agents SHOULD log all configuration items that do not have default values. +At the very minimum, agents MUST provide information about following essential configuration items. +Items denoted as *"Log always"* MUST be logged in any case (i.e. having a default value or a custom one). + +| Item | Needs masking | Log Always | Example | +| - | - | - | - | +| `server_url` | no | yes | `http://localhost:8200` [2] | +| `service_name` | no | yes | `foo` | +| `service_version` | no | yes | `42` | +| `log_level` | no | yes | `warning` | +| `secret_token` | yes | no | `[REDACTED]` | +| `api_key` | yes | no | `[REDACTED]` | + +[2]: Agents MAY decide to mask potential sensitive data (e.g. basic authentication information) +that could be part of this URL. + +For each configuration option its **source** SHOULD be reported. These sources can be: + +* `default` +* `environment`: Environment variable +* `file`: Configuration file +* `central`: Central Configuration + * **Note:** Agents MAY print their configuration block again on changes in the central configuration. + +Example: + +```text +Agent Configuration: +- configuration files used: + - '/path/to/some/config.json' + - '/path/to/some/other/config.xml' +- server_url: 'http://localhost:8200' (default) +- secret_token: [REDACTED] (environment) +- api_key: [REDACTED] (default) +- service_name: `unknown-dotnet-service` (default) +- log_level: info (file) +- disable_metrics: '*' (file) +``` diff --git a/specs/agents/metadata.md b/specs/agents/metadata.md new file mode 100644 index 00000000..8ebea3e2 --- /dev/null +++ b/specs/agents/metadata.md @@ -0,0 +1,382 @@ +## Metadata + +As mentioned above, the first "event" in each ND-JSON stream contains metadata to fold into subsequent events. The metadata that agents should collect includes is are described in the following sub-sections. + + - service metadata + - global labels (requires APM Server 7.2 or greater) + +The process for proposing new metadata fields is detailed +[here](process-new-fields.md). + +### System metadata + +System metadata relates to the host/container in which the service being monitored is running: + + - hostname + - host.id + - architecture + - operating system + - container ID + - kubernetes + - namespace + - node name + - pod name + - pod UID + +#### Hostname + +The hostname value(s) reported by the agent are mapped by APM Server to the ECS +[`host.hostname`](https://www.elastic.co/guide/en/ecs/current/ecs-host.html#field-host-hostname) and +[`host.name`](https://www.elastic.co/guide/en/ecs/current/ecs-host.html#field-host-name) fields. + +Agents SHOULD return the lower-cased FQDN whenever possible, which might require a DNS query. + +Agents SHOULD implement this hostname discovery algorithm wherever possible: +``` +var hostname; +if os == windows + // https://stackoverflow.com/questions/12268885/powershell-get-fqdn-hostname + // https://learn.microsoft.com/en-us/dotnet/api/system.net.dns.gethostentry + hostname = exec "powershell.exe -NoLogo -NonInteractive -NoProfile -ExecutionPolicy Bypass -Command [System.Net.Dns]::GetHostEntry($env:computerName).HostName" // or any equivalent * + if (hostname == null || hostname.length == 0) + hostname = exec "cmd.exe /c hostname" // or any equivalent * + if (hostname == null || hostname.length == 0) + hostname = env.get("COMPUTERNAME") +else + hostname = exec "hostname -f" // or any equivalent * + if (hostname == null || hostname.length == 0) + hostname = env.get("HOSTNAME") + if (hostname == null || hostname.length == 0) + hostname = env.get("HOST") + if (hostname == null || hostname.length == 0) + hostname = readfile("/etc/hostname") // or any equivalent * + +if hostname != null + hostname = hostname.toLowerCase().trim() // see details below ** +``` +`*` this algorithm is using external commands in order to be OS-specific and language-independent, however these +may be replaced with language-specific APIs that provide the equivalent result. + +`**` in this case, `trim()` refers to the removal of all leading and trailing characters of which codepoint is less-than +or equal to `U+0020` (space), the `toLowerCase()` refers to the replacement of characters in `A-Z` with their `a-z` equivalents. + +In addition to auto-discovery of the hostname, agents SHOULD also expose the `ELASTIC_APM_HOSTNAME` config option that +can be used as a manual fallback. + +Up to APM Server 7.4, only the `system.hostname` field was used for this purpose. Agents communicating with +APM Server of these versions MUST set `system.hostname` with the value of `ELASTIC_APM_HOSTNAME`, if such is manually +configured. Otherwise, agents MUST set it with the automatically-discovered hostname. + +Since APM Server 7.4, `system.hostname` field is deprecated in favour of two newer fields: +- `system.configured_hostname` - it should only be sent when configured by the user through the `ELASTIC_APM_HOSTNAME` +config option. If provided, it is used by the APM Server as the event's hostname. +- `system.detected_hostname` - the hostname automatically detected by the APM agent. It will be used as the event's +hostname if `configured_hostname` is not provided. + +Agents that are APM-Server-version-aware, or that are compatible only with versions >= 7.4, should +use the new fields wherever applicable. + +#### Host.id + +APM agents MAY collect the `host.id` as an unique identifier for the host. +If they collect it, it MUST be conformant to the [OpenTelemetry SemConv for `host.id`](https://opentelemetry.io/docs/specs/semconv/attributes-registry/host/). + +If the APM agent performs correlation of its spans/transactions with universal profiling data, it MUST send the `host.id` (see the [profiling integration spec](universal-profiling-integration.md#profiler-registration-message)) as part of the metadata. The APM agent MAY solely rely on the `host.id` provided by the profiling host agent in that case. + +#### Container/Kubernetes metadata + +On Linux, the container ID and some of the Kubernetes metadata can be extracted by parsing `/proc/self/cgroup`. For each line in the file, we split the line according to the format "hierarchy-ID:controller-list:cgroup-path", extracting the "cgroup-path" part. We then attempt to extract information according to the following algorithm: + + 1. Split the path into `dirname` and `basename`: + - split based on the last occurrence of the colon character, if such exists, in order to support paths of containers + created by [containerd-cri](https://github.com/containerd/cri), where the path part takes the form: + `:cri-containerd:` + - if colon char is not found within the path, the split is done based on the last occurrence of the slash character + + 2. If the `basename` ends with ".scope", check for a hyphen and remove everything up to and including that. This allows + us to match `.../docker-.scope` as well as `.../`. + + 3. Attempt to extract the Kubernetes pod UID from the `dirname` by matching one of the following regular expressions: + - `(?:^/kubepods[\\S]*/pod([^/]+)$)` + - `(?:kubepods[^/]*-pod([^/]+)\.slice)` + + If there is a match to either expression, the capturing group contains the pod ID. We then unescape underscores + (`_`) to hyphens (`-`) in the pod UID. + If we match a pod UID then we set the pod name to the hostname, as that's the default in Kubernetes. + Finally, we record the basename as the container ID without any further checks. + + 4. If we did not match a Kubernetes pod UID above, then we check if the basename matches one of the following regular + expressions: + + - `^[[:xdigit:]]{64}$` + - `^[[:xdigit:]]{8}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4,}$` + - `^[[:xdigit:]]{32}-[[:digit:]]{1,10}$` (AWS ECS/Fargate environments) + + If we match, then the basename is assumed to be a container ID. + +Sometimes the `KUBERNETES_POD_NAME` is set using the [Downward API](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/), +so we set the pod name to its value if it exists. +In a similar manner, you can inform the agent of the node name, namespace, and pod UID, using the environment variables `KUBERNETES_NODE_NAME`, `KUBERNETES_NAMESPACE`, and `KUBERNETES_POD_UID`. + +With cgroups v2, the `/proc/self/cgroup` contains only `0::/` and does not contain the container ID and we have to parse the `/proc/self/mountinfo` with the following algorithm as a fallback. + + 1. filter the line containing `/etc/hostname` to retrieve the file mount that provides the host name to the container. + + 2. split the line on spaces and take the 3rd element containing the host path. + + 3. extract the container ID from file path by using a regular expression matching a 64 character hexadecimal ID. + +*Note:* [container_metadata_discovery.json](../../tests/agents/json-specs/container_metadata_discovery.json) provides test cases for parsing `/self/proc/*` files. + +### Process metadata + +Process level metadata relates to the process running the service being monitored: + + - process ID + - parent process ID + - process arguments + - process title (e.g. "node /app/node_") + +### Service metadata + +Service metadata relates to the service/application being monitored: + + - service name and version + - environment name ("production", "development", etc.) + - agent name (e.g. "ruby") and version (e.g. "2.8.1") + - language name (e.g. "ruby") and version (e.g. "2.5.3") + - runtime name (e.g. "jruby") and version (e.g. "9.2.6.0") + - framework name (e.g. "flask") and version (e.g. "1.0.2") + +For official Elastic agents, the agent name should just be the name of the language for which the agent is written, in lower case. + +Services running on AWS Lambda [require specific values](tracing-instrumentation-aws-lambda.md) for some of the above mentioned fields. + +#### Activation method + +Most of the APM Agents can be activated in several ways. Agents SHOULD collect information about the used activation method and send it in the `service.agent.activation_method` field within the metadata. +This field MUST be omitted in version `8.7.0` due to a bug in APM server (preventing properly capturing metrics). +This field SHOULD be included when the APM server version is unknown or at least `8.7.1`. + +The intention of this field is to drive telemetry so there is a way to know which activation methods are commonly used. This field MUST produce data with very low cardinality, therefore agents SHOULD use one of the values defined below. + +If the agent is unable to infer the activation method, it SHOULD send `unknown`. + +There are some well-known activation methods which can be used by multiple agents. In those cases, agents SHOULD send the following values in `service.agent.activation_method`: + +- `aws-lambda-layer`: when the agent was installed as a Lambda layer. +- `k8s-attach`: when the agent is attached via [the K8s webhook](https://github.com/elastic/apm-k8s-attacher). +- `env-attach`: when the agent is activated by setting some environment variables. Only use this if there is a single way to activate the agent via an environment variable. If the given runtime offers multiple environment variables to activate the agent, use more specific values to avoid ambiguity. +- `fleet`: when the agent is activated via fleet. + +Cross agent activation methods defined above have higher priority than agent specific values below. +If none of the above matches the activation method, agents define specific values for specific scenarios. + +Node.js: +- `require`: when the agent is started via CommonJS `require('elastic-apm-node').start()` or `require('elastic-apm-node/start')`. +- `import`: when the agent is started via ESM, e.g. `import 'elastic-apm-node/start.js'`. +- `preload`: when the agent is started via the Node.js `--require` flag, e.g. `node -r elastic-apm-node/start ...`, without using `NODE_OPTIONS`. + +Java: +- `javaagent-flag`: when the agent is attached via the `-javaagent` JVM flag. +- `apm-agent-attach-cli`: when the agent is attached via the `apm-agent-attach-cli` tool. +- `programmatic-self-attach`: when the agent is attached by manually calling the `ElasticApmAttacher` API in user code. + +.NET: +- `nuget`: when the agent was installed via a NuGet package. +- `profiler`: when the agent was installed via the CLR Profiler. +- `startup-hook`: when the agent relies on the `DOTNET_STARTUP_HOOKS` mechanism to install the agent. + +Python: +- `wrapper`: when the agent was invoked with the wrapper script, `elasticapm-run` + +### Cloud Provider Metadata + +[Cloud provider metadata](https://github.com/elastic/apm-server/blob/main/docs/spec/v2/metadata.json) +is collected from local cloud provider metadata services: + +- availability_zone +- account + - id + - name +- instance + - id + - name +- machine.type +- project + - id + - name +- provider (**required**) +- region + +This metadata collection is controlled by a configuration value, +`CLOUD_PROVIDER`. The default is `auto`, which automatically detects the cloud +provider. If set to `none`, no cloud metadata will be generated. If set to +any of `aws`, `gcp`, or `azure`, metadata will only be generated from the +chosen provider. + +Any intake API requests to the APM server should be delayed until this +metadata is available. + +A sample implementation of this metadata collection is available in +[the Python agent](https://github.com/elastic/apm-agent-python/blob/main/elasticapm/utils/cloud.py). + +Fetching of cloud metadata for services running as AWS Lambda functions follow a [different approach defined in the tracing-instrumentation-aws-lambda spec](tracing-instrumentation-aws-lambda.md). + +#### AWS metadata + +[Metadata about an EC2 instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html) can be retrieved from the internal metadata endpoint, `http://169.254.169.254`. + +In the case where a proxy is configured on the application, the agents SHOULD attempt to make +the calls to the metadata endpoint directly, without using the proxy. +This is recommended as those HTTP calls could be caller-sensitive and have to be made directly + by the virtual machine where the APM agent executes, also, the `169.254.x.x` IP address range +is reserved for "link-local" addresses that are not routed. + +As an example with curl, first, an API token must be created + +```sh +TOKEN=`curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 300"` +``` + +Then, metadata can be retrieved, passing the API token + +```sh +curl -H "X-aws-ec2-metadata-token: $TOKEN" -v http://169.254.169.254/latest/meta-data +``` + +From the returned metadata, the following fields are useful + +| Cloud metadata field | AWS Metadata field | +| -------------------- | ------------------- | +| `account.id` | `accountId` | +| `instance.id` | `instanceId` | +| `availability_zone` | `availabilityZone` | +| `machine.type` | `instanceType` | +| `provider` | aws | +| `region` | `region` | + +#### GCP metadata + +Metadata about a GCP machine instance can be retrieved from the +metadata service, [documented here](https://cloud.google.com/compute/docs/metadata/default-metadata-values). + +In the case where a proxy is configured on the application, the agents SHOULD attempt to make +the calls to the metadata endpoint directly, without using the proxy. +This is recommended as those HTTP calls could be caller-sensitive and have to be made directly + by the virtual machine where the APM agent executes, also, the `169.254.x.x` IP address range +is reserved for "link-local" addresses that are not routed. + +An example with curl + +```sh +curl -X GET "http://metadata.google.internal/computeMetadata/v1/?recursive=true" -H "Metadata-Flavor: Google" +``` + +From the returned metadata, the following fields are useful + +| Cloud metadata field | GCP Metadata field | +| -------------------- | ------------------- | +| `instance.id` | `instance.id` as a string [1] | +| `instance.name` | `instance.name` | +| `project.id` | `project.projectId` [2] | +| `availability_zone` | last part of `instance.zone`, split by `/` | +| `machine.type` | last part of `instance.machineType`, split by `/` | +| `provider` | gcp | +| `region` | last part of `instance.zone` split by '/', then remove the last '-'-delimited part (e.g., `us-west1` from `projects/123456789012/zones/us-west1-b`)| + +[1]: Beware JSON parsing the `instance.id` field from the HTTP response body, because it is formatted as an integer that is larger [JavaScript's `Number.MAX_SAFE_INTEGER`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER). It may require native support for or explicit usage of BigInt types. +[2]: Google cloud project identifiers are [described here](https://cloud.google.com/resource-manager/docs/creating-managing-projects#before_you_begin). + +(For comparison and consistency, here is the equivalent collection code for +[beats](https://github.com/elastic/beats/blob/aa5da2847957fa62687e0b4e7777c86fe7c05f6c/libbeat/processors/add_cloud_metadata/provider_google_gce.go#L45-L153).) + +#### Azure metadata + +##### Azure VMs + +Metadata about an Azure VM can be retrieved from the internal metadata +endpoint, `http://169.254.169.254`. + +In the case where a proxy is configured on the application, the agents SHOULD attempt to make +the calls to the metadata endpoint directly, without using the proxy. +This is recommended as those HTTP calls could be caller-sensitive and have to be made directly + by the virtual machine where the APM agent executes, also, the `169.254.x.x` IP address range +is reserved for "link-local" addresses that are not routed. + +An example with curl + +```sh +curl -X GET "http://169.254.169.254/metadata/instance/compute?api-version=2019-08-15" -H "Metadata: true" +``` + +From the returned metadata, the following fields are useful + +| Cloud metadata field | Azure Metadata field| +| -------------------- | ------------------- | +| `account.id` | `subscriptionId` | +| `instance.id` | `vmId` | +| `instance.name` | `name` | +| `project.name` | `resourceGroupName` | +| `availability_zone` | `zone` | +| `machine.type` | `vmSize` | +| `provider` | azure | +| `region` | `location` | + +##### Azure App Services _(Optional)_ + +Azure App Services are a PaaS offering within Azure which does not +have access to the internal metadata endpoint. Metadata about +an App Service can however be retrieved from environment variables + + +| Cloud metadata field | Environment variable | +| -------------------- | ------------------- | +| `account.id` | first part of `WEBSITE_OWNER_NAME`, split by `+` | +| `instance.id` | `WEBSITE_INSTANCE_ID` | +| `instance.name` | `WEBSITE_SITE_NAME` | +| `project.name` | `WEBSITE_RESOURCE_GROUP` | +| `provider` | azure | +| `region` | last part of `WEBSITE_OWNER_NAME`, split by `-`, trim end `"webspace"` and anything following | + +The environment variable `WEBSITE_OWNER_NAME` has the form + +``` +{subscription id}+{app service plan resource group}-{region}webspace{.*} +``` + +an example of which is `f5940f10-2e30-3e4d-a259-63451ba6dae4+elastic-apm-AustraliaEastwebspace` + +Cloud metadata for Azure App Services is optional; it is up +to each agent to determine whether it is useful to implement +for their language ecosystem. See [azure_app_service_metadata specs](../../tests/agents/gherkin-specs/azure_app_service_metadata.feature) +for scenarios and expected outcomes. + +##### Azure Functions + +Azure Functions running within a consumption/premium plan (see [Azure Functions hosting options](https://learn.microsoft.com/en-us/azure/azure-functions/functions-scale)) are a FaaS offering within Azure that do not have access +to the internal [Azure metadata endpoint](#azure-vms). Metadata about an Azure Function can however be +retrieved from environment variables. +**Note:** These environment variables slightly differ from those available to [Azure App Services](#azure_app_service-optional). + +| Cloud metadata field | Environment variable | +| -------------------- | ------------------- | +| `account.id` | Token `{subscription id}` from `WEBSITE_OWNER_NAME` | +| `instance.name` | `WEBSITE_SITE_NAME` | +| `project.name` | `WEBSITE_RESOURCE_GROUP` (fallback: `{resource group}` from `WEBSITE_OWNER_NAME`) | +| `provider` | azure | +| `region` | `REGION_NAME` (fallback: `{region}` from `WEBSITE_OWNER_NAME`) | +| `service.name` | `functions` see the [ECS fields doc](https://www.elastic.co/guide/en/ecs/current/ecs-cloud.html#field-cloud-service-name). | + +The environment variable `WEBSITE_OWNER_NAME` has the following form: + +`{subscription id}+{resource group}-{region}webspace{.*}` + +Example: `d2cd53b3-acdc-4964-9563-3f5201556a81+wolfgangfaas_group-CentralUSwebspace-Linux` + +### Global labels + +Events sent by the agents can have labels associated, which may be useful for custom aggregations, or document-level access control. It is possible to add "global labels" to the metadata, which are labels that will be applied to all events sent by an agent. These are only understood by APM Server 7.2 or greater. + +Global labels can be specified via the environment variable `ELASTIC_APM_GLOBAL_LABELS`, formatted as a comma-separated +list of `key=value` pairs. diff --git a/specs/agents/metrics-health.md b/specs/agents/metrics-health.md new file mode 100644 index 00000000..c94c5108 --- /dev/null +++ b/specs/agents/metrics-health.md @@ -0,0 +1,133 @@ +## Agent Health Metrics + +Agents SHOULD expose metrics which allow the analysis of the internal state of the agent, especially including the [reporting/transport mechanism](transport.md). + +Many of these metrics are `COUNTER`s: a `COUNTER` is a monotonically increasing metric. +It is periodically reported by sending the difference of the counter-value now vs its value at the time of the previous report. +In order to save elasticsearch disk space, zero-increases are not sent. + +Example: +| **Time** | **Counter** | **Reported Metric Value** | +|----------|-------------|---------------------------| +| 0s | 7 | 7 | +| 30s | 10 | 3 | +| 1m | 10 | 0 (not sent) | +| 1m30s | 12 | 2 | + + +### Event Count Metrics + +Agents send their data to the Intake API of the APM-server in the form of a stream of events (e.g. transactions, spans, metricsets...). + +The following metrics counting these events SHOULD be exposed: + - `agent.events.total`: `COUNTER` of events either submitted to the internal reporting queue or dropped upon submit attempt + - `agent.events.dropped`: `COUNTER` of events which failed to be transmitted to the APM server and therefore were dropped + +If an agent supports these metrics, the metrics MUST be captured for all events which would reach the APM-Server in an ideal, failure-free world. +Therefore an event MUST be taken into account for these metrics if an attempt is made to submit it to the internal reporting queue, regardless of the success. +Events dropped before this attempt (e.g. due to [sampling](tracing-sampling.md) or [transaction_max_spans](handling-huge-traces/tracing-spans-limit.md)) are NOT counted. + +The `agent.events.total` metric MUST have the label `event_type`. The possible values for `event_type` are the lowercase names of the events according to the [spec](https://github.com/elastic/apm-server/tree/main/docs/spec/v2) (e.g. `transaction`, `metricset`) or the value `undefined`. +Agents SHOULD attempt to assign the appropriate label value based on the counted event. If this would impose significant implementation overhead, the value `undefined` MUST be used instead. + +The `agent.events.dropped` metric MUST have a value for the `reason` label. The following values MUST be used: +* `queue` MUST be used for events where the attempt to add it to the reporting queue failed (e.g. due to a full queue) +* `error` MUST be used for events dropped after they previously have been successfully added to the queue. + +The value of `agent.events.dropped` with `reason=error` is an upper bound for the actual number of dropped events after entering the queue. +For example, when a request to the Intake API fails without a response, all events within this request MUST be considered as failed. +In reality, it is possible that e.g. half of the data was actually successfully received and forwarded by the APM server. +When the APM server responds with an HTTP error code, the number of dropped events SHOULD be computed by subtracting the `accepted` field from the response body from the total number of inflight events of this request. + +It MUST be possible to disable all event count metrics via the `disable_metrics` configuration option. + +### Event Queue Metrics + +The following metrics SHOULD be recorded by agents regarding the state of the internal event reporting queue: + + - `agent.events.queue.min_size.pct`: The smallest queue usage in percent (range [0-1]) observed since this metric was reported the last time + - `agent.events.queue.max_size.pct`: The biggest queue usage in percent (range [0-1]) observed since this metric was reported the last time + +We capture both `min_size` and `max_size` to allow to distinguish between reasons for why the queue is becoming full: + * There is too much data captured / network errors: Both `min_size` and `max_size` are at `100%` + * The data is collected in bursts for which the queue is not correctly sized: `max_size` is at `100%`, but `min_size` is low. + +The queue usage can be computed based on how the agent defines the queue capacity. +E.g. if the queue capacity is a fixed number of events, the usage can be computed based on the current number of events. +If the queue capacity is in bytes, the usage can be computed based on the number of bytes currently occupied in the queue. + +The `agent.events.queue.*` metrics MUST have a `queue_name` label. If agents use multiple queues, the `agent.events.queue.*` SHOULD be exposed for each queue with a implementation-defined value for `queue_name` per queue. +If agents use just a single queue or have a shared primary queue, the value `generic` SHOULD be used as value for `queue_name` for this queue. + +It MUST be possible to disable all event queue metrics via the `disable_metrics` configuration option. + +#### Possible implementation for agents + +In order to track these metrics, most likely a custom implementation is required on top of the underlying queue. + +One approach can be to have two variables `min_size` and `max_size`, which are atomically updated on changes to the queue: + * When an item is added to the queue, update `max_size` with the current queue size if it is greater than `max_size` + * When an item is removed from the queue, update `min_size` with the current queue size if it is less than `min_size` + * When an item cannot be added to the queue (dropped), set `max_size` to `1` + +After the metrics are exported, `min_size` and `max_size` need to be reset to the current queue size. + +An atomic min/max can be implemented using the following algorithm: +```java +AtomicLong maxSize = new AtomicLong(); + +public void updateMaxSize(long currentSize) { + //alternatively abort after like 10 iterations for constant latency + while (true) { + long current = maxSize.get(); + if (current >= currentSize) { + return; + } + boolean casSuccess = maxSize.compareAndSet(current, currentSize); + if (casSuccess) { + return; + } + } +} + +``` + +### Event Request Metrics + +Agents SHOULD expose the following metrics regarding Intake API networking: + + - `agent.events.requests.count`: `COUNTER` of the number of requests made or attempted to the Intake API of the APM server + - `agent.events.requests.bytes`: `COUNTER` of the approximate number of bytes on the wire sent to the Intake API of the APM + +The `agent.events.requests.*` metrics MUST have the label `success`, which can have the values `true` or `false`. A request is counted with `success=true` iff the APM Server responded with `2xx`. + +The metric `agent.events.requests.bytes` does not need to represent the exact network usage. +Instead, the number of compressed bytes within the request body can be used as approximation. + +It MUST be possible to disable all event request metrics via the `disable_metrics` configuration option. + +### Agent Overhead Metrics + +If possible with a reasonable amount of runtime and implementation overhead, agents MAY expose the following metrics for approximating their own resource usage: + + - `agent.background.cpu.overhead.pct`: The fraction of the process cpu usage caused by the agent in a range [0-1] within the last reporting interval. + - `agent.background.cpu.total.pct`: The approximate CPU usage caused by the agent. Derived by multiplying `agent.background.cpu.overhead.pct` with `system.process.cpu.total.norm.pct`. + - `agent.background.memory.allocation.bytes`: `COUNTER` of the number of bytes allocated in the main memory by the agent. + - `agent.background.threads.count`: Number of currently running agent background threads at the time of reporting. + +The `agent.background.threads.count` only applies to agents in whose language the concept of `threads` is present and used by the agent for background tasks. + +Agents SHOULD add the label `task` to all these metrics. The values for `task` can be freely chosen by each agent implementation. The label values should be chosen in a way which allows the resource usages to be assigned to specific parts of the agent implementation (e.g. thread names as `task`s). + +In most cases, it will not be feasible to capture the overhead of every action the agent does due to the overhead of the measurement itself. E.g. for the Java agent it is not feasible to measure the overhead of the code added to user application methods via instrumentation. Instead, only the overhead of agent-owned threads is measured. + +The `agent.background.cpu.overhead.pct` can be computed by measuring the increase of thread CPU time of a given task ([this method](https://docs.oracle.com/javase/7/docs/api/java/lang/management/ThreadMXBean.html#getCurrentThreadCpuTime()) for Java) and dividing it by the increase of the total process CPU time ([this method](https://docs.oracle.com/javase/7/docs/jre/api/management/extension/com/sun/management/OperatingSystemMXBean.html#getProcessCpuTime()) for Java) within the same period. + +All agent overhead metrics are enabled/disabled via the `agent_background_overhead_metrics` configuration: + +| | | +|----------------|---| +| Type | `boolean` | +| Default | `false` | +| Dynamic | `false` | +| Central config | `false` | diff --git a/specs/agents/metrics-otel.md b/specs/agents/metrics-otel.md new file mode 100644 index 00000000..e205f2cc --- /dev/null +++ b/specs/agents/metrics-otel.md @@ -0,0 +1,181 @@ +# OpenTelemetry Metrics API Support + +Agents SHOULD support collecting custom metrics via the OpenTelemetry Metrics API. +This SHOULD be done by supplying an OpenTelemetry Metrics SDK compatible [MetricExporter](https://opentelemetry.io/docs/reference/specification/metrics/sdk/#metricexporter). If a pull based approach is preferred, a [MetricReader](https://opentelemetry.io/docs/reference/specification/metrics/sdk/#metricreader) can be used instead. This `MetricExporter/MetricReader` SHOULD transform the received metrics data to IntakeV2 metricsets and send them via the existing reporting mechanism. + +## Metric naming + +Agents MUST NOT alter/sanitize OpenTelemetry metric names. For example, agents MUST NOT de-dot metric names. +Unfortunately, this can lead to mapping problems in elasticsearch (state: January 2023) if for example metrics with the names `foo.bar` and `foo` are both ingested. +Due to the nested object representation within metricsets, `foo` would need to be mapped as both an object and a number at the same time, which is not possible. + +We plan on solving these conflicts on the APM-server side instead by adjusting the mapping of metricsets (Again, state of January 2023). + +If agents offer the capability to disable selected metrics (e.g. the `disable_metrics` configuration in case of java), it MUST be respected for OpenTelemetry metrics. + +## Aggregation Temporality + +In OpenTelemetry the [AggregationTemporality](https://opentelemetry.io/docs/reference/specification/metrics/data-model/#temporality) defines whether metrics report the total, cumulative observed values or the delta since the last export. +The temporality is fully controlled by exporters / MetricReaders, therefore we can decide which temporality to use. + +For now, Agents MUST use the delta-preferred temporality: + +| **Instrument Type** | **Temporality** | +|-----------------------------|----------------------------| +| Counter | Delta | +| Asynchronous Counter | Delta | +| UpDownCounter | Cumulative | +| Asynchronous UpDownCounter | Cumulative | +| Histogram | Delta | + +The reason is that monotonic counter metrics are currently more difficult to handle and visualize in kibana and elasticsearch. +As soon as the support gets better, we will revisit this spec and allow users to switch between cumulative and delta temporality via configuration. + +For all instrument types with delta temporality, agents MUST filter out zero values before exporting. +E.g. if a counter does not change since the last export, it must not be exported. + +## Aggregation + +The OpenTelemetry SDK [Aggregations](https://opentelemetry.io/docs/reference/specification/metrics/sdk/#aggregation) define how the observations made on a given Instrument are aggregated into a single value for export. For example, counters are aggregated by summing all values, whereas for gauges the last value is used. + +Aggregations are configurable in OpenTelemetry. Currently, this mainly makes sense for Histogram-Instruments. A histogram instrument is usually used when the distribution of the observed values is of interest, e.g. for latency. At the moment the SDK offers [two options](https://opentelemetry.io/docs/reference/specification/metrics/sdk/#histogram-aggregations) for histogram aggregations: + * A bucket based histogram with explicitly provided bucket boundaries + * An exponential histogram with predefined, exponentially increasing bucket boundaries + +In the SDK, the default aggregation to use for each instrument type (counter, gauge, histogram, etc) is defined by the Exporter / MetricsReader. However, users can explicitly override the aggregation per metric by using [Views](https://opentelemetry.io/docs/reference/specification/metrics/sdk/#view). Note that views are an SDK concept and are not part of the OpenTelemetry API. + +Agents MUST use the OpenTelemetry SDK [default aggregations per instrument type](https://opentelemetry.io/docs/reference/specification/metrics/sdk/#default-aggregation) as default aggregations for their exporter: + +| **Instrument Type** | **Aggregation** | +|-----------------------------|----------------------------| +| Counter | Sum | +| Asynchronous Counter | Sum | +| UpDownCounter | Sum | +| Asynchronous UpDownCounter | Sum | +| Asynchronous Gauge | Last Value | +| Histogram | Explicit Bucket Histogram | + +### Histogram Aggregation + +The OpenTelemetry SDK uses the following prometheus-inspired bucket boundaries for histograms by default: +``` +[0, 5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10000] +``` + +Because IntakeV2 histogram serialization skips empty buckets, we are free to use histogram boundaries which are more accurate and cover a greater range. Therefore, agents SHOULD use the following histogram boundaries by default instead: + +``` +[0.00390625, 0.00552427, 0.0078125, 0.0110485, 0.015625, 0.0220971, 0.03125, 0.0441942, 0.0625, 0.0883883, 0.125, 0.176777, 0.25, 0.353553, 0.5, 0.707107, 1, 1.41421, 2, 2.82843, 4, 5.65685, 8, 11.3137, 16, 22.6274, 32, 45.2548, 64, 90.5097, 128, 181.019, 256, 362.039, 512, 724.077, 1024, 1448.15, 2048, 2896.31, 4096, 5792.62, 8192, 11585.2, 16384, 23170.5, 32768, 46341.0, 65536, 92681.9, 131072] +``` + +These boundaries are an exponential scale base `sqrt(2)` rounded to six significant figures. + +Agents MUST allow users to configure the default histogram boundaries via the `custom_metrics_histogram_boundaries` configuration option (described in the section below). + +In the future, we might replace the default aggregation with one better suited for elasticsearch, e.g. HDRHistogram or T-Digest. + +Because users would need to take extra steps to prevent our exporter from using the preferred explicit bucket aggregation (by configuring SDK views), agents are not required to support any other histogram aggregation types. In other words, agents MAY ignore [ExponentialHistograms](https://opentelemetry.io/docs/reference/specification/metrics/data-model/#exponentialhistogram) and [Summaries](https://opentelemetry.io/docs/reference/specification/metrics/data-model/#summary-legacy). If a metric is ignored due to its aggregation, agents SHOULD emit a warning with a message including the metric name. + +However, agents MUST support explicit bucket histogram aggregations with custom boundaries provided by users via views. + +Before export, the OpenTelemetry histogram bucket data needs to be converted to IntakeV2 histogram data. Because the OpenTelemetry specification allows negative values in histograms, the data needs to be converted to T-Digests: For T-Digests, the IntakeV2 metricset `samples.values` properties holds the **midpoints** of the histogram buckets instead of the upper/lower bounds. + +Therefore, the following algorithm needs to be used to convert [OpenTelemetry explicit bucket histogram data](https://opentelemetry.io/docs/reference/specification/metrics/data-model/#histogram) to IntakeV2 histograms: + +``` +IntakeV2Histo convertBucketBoundaries(OtelHisto input) { + List intakeCounts = new ArrayList<>(); + List intakeValues = new ArrayList<>(); + + List otelCounts = input.getCounts(); + List otelBoundaries = input.getBoundaries(); + + int bucketCount = otelCounts.size(); + // otelBoundaries has a size of bucketCount-1 + // the first bucket has the boundaries ( -inf, otelBoundaries[0] ] + // the second bucket has the boundaries ( otelBoundaries[0], otelBoundaries[1] ] + // .. + // the last bucket has the boundaries (otelBoundaries[bucketCount-2], inf) + + for (int i = 0; i < bucketCount; i++) { + if (otelCounts.get(i) != 0) { //ignore empty buckets + intakeCounts.add(otelCounts.get(i)); + if (i == 0) { //first bucket + double bounds = otelBoundaries.get(0); + if(bounds > 0) { + bounds /= 2; + } + intakeValues.add(bounds); + } else if (i == bucketCount - 1) { //last bucket + intakeValues.add(otelBoundaries.get(bucketCount - 2)); + } else { //in between + double lower = otelBoundaries.get(i - 1); + double upper = otelBoundaries.get(i); + intakeValues.add(lower + (upper - lower) / 2); + } + } + } + + return new IntakeV2Histo(intakeCounts, intakeValues); +} +``` + +The same algorithm is used by the APM server to convert OTLP histograms. +The `sum`, `count`, `min` and `max` within the OpenTelemetry histogram data are discarded for now until we support them in IntakeV2 histograms. + +#### custom_metrics_histogram_boundaries configuration + +Defines the default bucket boundaries to use for OpenTelemetry histograms. + +| | | +|----------------|------------------------------------------| +| Type | `double list` | +| Default | | +| Dynamic | `false` | +| Central config | `false` | + +## Labels + +Conceptually, elastic metric labels keys correspond to [OpenTelemetry Attributes](https://opentelemetry.io/docs/reference/specification/common/#attribute). +When exporting, the agents MUST convert the attributes to labels as described in this section and MUST group metrics with the same attributes and OpenTelemetry instrumentation scope together into a single metricset. + +Attribute keys MUST NOT be modified. Potential sanitization will happen within the APM server, if any. +APM servers pre version 7.11 will drop metricsets if the label keys contain any of the characters `.`, `*` or `"`, which are however allowed characters in OpenTelemetry. +Therefore, agents SHOULD document that OpenTelemetry metrics might be dropped when using an APM Server pre version 7.11. + +The attribute values MUST NOT be modified and their type MUST be preserved. E.g. `strings` must remain `string`-labels, `booleans` must remain `boolean`-labels. +Metricsets currently do not support array-valued labels, whereas OpenTelemetry attribute values can be arrays. For this reason, array-valued attributes MUST be ignored. Agents SHOULD emit a warning with a message containing the metric name when attributes are dropped because they are array-valued. + +To summarize and to give an example, given the following set of attributes + +| **Attribute Key** | **Attribute Value** | **Type** | +|-------------------|---------------------|----------------| +| `foo.bar` | `baz` | `string` | +| `my_array` | `[first, second]` | `string array` | +| `testnum*` | `42.42` | `number` | + +the following labels must be used for the resulting metricset: + +| **Label Key** | **Label Value** | **Type** | +|---------------|-----------------|----------------| +| `foo_bar` | `baz` | `string` | +| `testnum_` | `42.42` | `number` | + +The OpenTelemetry specification allows the definition of metrics with the same name as long as they reside within a different instrumentation scope ([spec link](https://opentelemetry.io/docs/reference/specification/metrics/api/#get-a-meter)). Agents MUST report metrics from different instrumentation scopes in separate metricsets to avoid naming conflicts at collection time. This separation MUST be done based on the instrumentation scope name and version. In the future, we might add dedicated intake fields to metricsets for differentiation them based on the instrumentation scope identifiers. + +## Exporter Installation + +The onboarding should be as easy as possible for users. Ideally, the users should be able to browse their OpenTelemetry metrics by just following the standard agent installation process without requiring additional configuration or code changes. + +We focus on two ways of how OpenTelemetry may be used by users: + +1. The user application has a OpenTelemetry Metrics SDK Instance setup and configured. E.g. the user already has a prometheus exporter running and did some customizations on their metrics via [Views](https://opentelemetry.io/docs/reference/specification/metrics/sdk/#view). +2. The user application does not setup a OpenTelemetry Metrics SDK Instance, but uses the OpenTelemetry Metrics API to define metrics. This is usually the case when the user application does not care / know about OpenTelemetry metrics, but a library used by the application implemented observability via OpenTelemetry. + +For **1.** we want to make sure that the existing user configuration is respected by our apm agents. Ideally, agents SHOULD just register an additional exporter to the existing OpenTelemetry Metrics SDK instance(s). If the agent and language capabilities allow it, the exporter registration SHOULD be done automatically without requiring code changes by the user. + +For **2.** agents MAY automatically register an agent-provided SDK instance to bind the user provided OpenTelemetry API to, if this is possible in their language and does not cause too much overhead of any kind (e.g. implementation or agent package size). +Agents MUST NOT override a user-provided global OpenTelemetry metrics SDK with their own SDK or prevent the user from providing his own SDK instance in any means. +For example the Java agent MUST NOT install an OpenTelemetry Metrics SDK instance in the [GlobalOpenTelemetry](https://www.javadoc.io/static/io.opentelemetry/opentelemetry-api/1.20.0/io/opentelemetry/api/GlobalOpenTelemetry.html) if it detects that another Metrics SDK has already been registered there. + +Agents SHOULD ensure that remaining metric data is properly flushed when the user application shuts down. for **1.**, the user is responsible for shutting down the SDK instance before their application terminates. This shutdown initiates a metrics flush, therefore no special actions needs to be taken by agents in this case. For **2.**, agents SHOULD initiate a proper shutdown of the agent provided SDK instance when the user application terminates, which automatically causes a flush. \ No newline at end of file diff --git a/specs/agents/metrics.md b/specs/agents/metrics.md new file mode 100644 index 00000000..374a6aa2 --- /dev/null +++ b/specs/agents/metrics.md @@ -0,0 +1,70 @@ +## Metrics + +Agents periodically collect and report various metrics, described below. + +### System/process CPU/Heap + +All agents (excluding JavaScript RUM) should record the following basic system/process metrics: + + - `system.cpu.total.norm.pct`: system CPU usage since the last report, in the range `[0,1]` (0-100%) + - `system.process.cpu.total.norm.pct`: process CPU usage since the last report, in the range `[0,1]` (0-100%) + - `system.memory.total`: total usable (but not necessarily available) memory on the system, in bytes + - `system.memory.actual.free`: total available memory on the system, in bytes + - `system.process.memory.size`: process virtual memory size, in bytes + - `system.process.memory.rss.bytes`: process resident set size, in bytes + +### cgroup metrics + +Where applicable, all agents (excluding JavaScript RUM) should record the following cgroup metrics: + + - `system.process.cgroup.memory.mem.limit.bytes` + - `system.process.cgroup.memory.mem.usage.bytes` + +#### Metrics source + +##### [cgroup-v1](https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt) + - `system.process.cgroup.memory.mem.limit.bytes` - based on the `memory.limit_in_bytes` file + - `system.process.cgroup.memory.mem.usage.bytes` - based on the `memory.usage_in_bytes` file + +##### [cgroup-v2](https://www.kernel.org/doc/Documentation/cgroup-v2.txt) + - `system.process.cgroup.memory.mem.limit.bytes` - based on the `memory.max` file + - `system.process.cgroup.memory.mem.usage.bytes` - based on the `memory.current` file + +#### Discovery of the memory files + +All files mentioned above are located at the same directory. Ideally, we can discover this dir by parsing the `/proc/self/mountinfo` file, looking for the memory mount line and extracting the path from within it. An example of such line is: +``` +436 431 0:33 /docker/5042cfbb4ab36fcef9ca5f1eda54f40265c6ef3fe0694dfe34b9b474e70f8df5 /sys/fs/cgroup/memory ro,nosuid,nodev,noexec,relatime master:22 - cgroup memory rw,memory +``` +The regex `^\d+? \d+? .+? .+? (.*?) .*cgroup.*memory.*` works in the cgroup-v1 systems tested so far, where the first and only group should be the directory path. However, it will probably take a few iterations and tests on different container runtimes and OSs to get it right. +There is no regex currently suggested for cgroup-v2. Look in other agent PRs to get ideas. + +Whenever agents fail to discover the memory mount path, they should default to `/sys/fs/cgroup/memory`. + +#### Special values for unlimited memory quota + +Special values are used to indicate that the cgroup is not configured with a memory limit. In cgroup v1, this value is numeric - `0x7ffffffffffff000` and in cgroup v2 it is represented by the string `max`. +Agents should not send the `system.process.cgroup.memory.mem.limit.bytes` metric whenever these special values are set. + +### Runtime + +Agent should record runtime-specific metrics, such as garbage collection pauses. Due to their runtime-specific nature, these will differ for each agent. + +When capturing runtime metrics, keep in mind the end use-case: how will they be used? Is the format in which they are recorded appropriate for visualization in Kibana? Do not record metrics just because it is easy; record them because they are useful. + +### Transaction and span breakdown + +Agents should record "breakdown metrics", which is a summarization of how much time is spent per span type/subtype in each transaction group. This is described in detail in the [Breakdown Graphs](https://docs.google.com/document/d/1-_LuC9zhmva0VvLgtI0KcHuLzNztPHbcM0ZdlcPUl64#heading=h.ondan294nbpt) document, so we do not repeat it here. + +### Agent Health and Overhead Metrics + +Agents SHOULD record metrics which give insight into the agent's health state. This is explained in detail in [this spec](metrics-health.md). + +### OpenTelemetry Metrics + +OpenTelemetry provides an API for collecting user defined metrics. Agents SHOULD allow custom metric collection via this API, this is described in detail in [this spec](metrics-otel.md). + +## Shutdown behavior + +Agents should make an effort to flush any metrics before shutting down. +If this cannot be achieved with shutdown hooks provided by the language/runtime, the agent should provide a public API that the user can call to flush any remaining data. \ No newline at end of file diff --git a/specs/agents/mobile/README.md b/specs/agents/mobile/README.md new file mode 100644 index 00000000..7fd53a94 --- /dev/null +++ b/specs/agents/mobile/README.md @@ -0,0 +1,87 @@ +# Introduction + +General documentation for the mobile apm agents can be found in [Getting started with APM](https://www.elastic.co/guide/en/apm/get-started/current/overview.html) docs. + +The mobile specific docs can be found: + +* [iOS Agent](https://www.elastic.co/guide/en/apm/agent/swift/0.x/intro.html) +* [Android Agent](https://www.elastic.co/guide/en/apm/agent/android/current/index.html) + +## OpenTelemetry + +The mobile agents are the first agents at Elastic to be built on top of Open-Telementry. +A large portion of the mobile agents' functionality can be attributed to the [opentelementry-swift](https://github.com/open-telemetry/opentelemetry-swift) and [opentelementry-java](https://github.com/open-telemtry/opentelemetry-java) packages. + +The Open-Telemetry libraries adhere to the semantic conventions outlined in [opentelementry-specifications](https://github.com/open-telemetry/opentelemetry-specification). +However, the Elastic mobile agents don't set every attribute defined (many only apply to server type monitoring). Additionally, these Open Telementry attributes will be remapped to Elastic specific terms. + +## Semantic Conventions and APM Server Mappings + +### Resource Attributes +Here is a list of resource attributes that are relevant for our mobile agents: + +| OTel Convention | Elastic Convention | Example | Required | Comment | +|------------------------|-----------------------------------|----------------------------------------| ---------| -----------------------------------| +| [`service.name`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/#service) | `service.name` | `opbeans-swift` | :white_check_mark: yes | | +| [`service.version`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/#service) | `service.version` | `5.2.0` | :white_check_mark: yes | | +| [`telemetry.sdk.name`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/#telemetry-sdk) | `agent.name` | `iOS`, `android` | :white_check_mark: yes | Elastic's `agent.name` is mapped from: `telemetry.sdk.name`/`telemetry.sdk.language` | +| [`telemetry.sdk.version`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/#telemetry-sdk)| `agent.version` | `1.0.1` | :white_check_mark: yes | | +| [`telemetry.sdk.language`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/#telemetry-sdk)| `service.language.name` | `swift`, `java` | :white_check_mark: yes | Elastic's `agent.name` is mapped from: `telemetry.sdk.name`/`telemetry.sdk.language` | +| [`os.description`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/os/) | `host.os.full` | `iOS Version 15.5 (Build 19F70)` | :x: no | | +| [`os.type`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/os/) | `host.os.platform` | `darwin` | :white_check_mark: yes | | +| [`os.version`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/os/) | `host.os.version` | `15.5.0` | :x: no | | +| [`os.name`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/os/) | `host.os.name` and `host.os.type` | `iOS`, `Android` | :white_check_mark: yes | | +| [`deployment.environment`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/deployment_environment/) | `service.environment` | `production`, `dev` | :x: no | | +| [`device.id`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/device/) | `device.id` | `E733F41E-DF47-4BB4-AAF0-FD784FD95653` | :x: no | [Follow this description.](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/device/) (Device ID should be fix and unique for a device but should not carry PII) | +| [`device.model.identifier`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/device/) | `device.model.identifier` | `iPhone4`,`SM-G920F` | :x: no | | +| [`device.model.name`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/device/) | `device.model.name` | `Samsung Galaxy S6` | :x: no | This information is potentially not directly available on the device and needs to be derived / mapped from `device.model.identifier`. In this case, the APM server should do the mapping. | +| [`device.manufacturer`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/device/) | `device.manufacturer` | `Apple`, `Samsung` | :x: no | This information is potentially not directly available on the device and needs to be derived / mapped from `device.model.identifier`. In this case, the APM server should do the mapping. | +| [`process.runtime.name`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/process/#process-runtimes) | `service.runtime.name` | `Android Runtime` | :x: no | Use `Android Runtime` for Android. For iOS use `iOS`. | +| [`process.runtime.version`](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/process/#process-runtimes) | `service.runtime.version` | `2.0.1` | :x: no | Use the Dalvik version for Android (`System.getProperty("java.vm.version")`). For iOS use the version of iOS. | + +### Common Span attributes +| OTel Convention | Elastic Convention | Example | Required | Comment | +|-----------------------------------------|----------------------------|----------------------------------| ---------| ---------------------------------| +| [Instrumentation name / tracer name](https://opentelemetry.io/docs/reference/specification/trace/api/#get-a-tracer) |`service.framework.name`| `SwiftUI`, `UIKit` | :x: no | Use the name of the library that is instrumented.| +| [Instrumentation version / tracer version](https://opentelemetry.io/docs/reference/specification/trace/api/#get-a-tracer) |`service.framework.version`| `1.2.3`| :x: no | Use the version of the library that is instrumented. | +| [`net.host.connection.type`](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/span-general/) | `network.connection.type` | `cell`, `wifi` | :x: no|| +| [`net.host.connection.subtype`](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/span-general/) | `network.connection.subtype` | `lte`, `edge` | :x: no|| +| [`net.host.carrier.name`](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/span-general/) | `network.carrier.name` | [see OTEL](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/span-general/) | :x: no|| +| [`net.host.carrier.mcc`](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/span-general/) | `network.carrier.mcc` | [see OTEL](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/span-general/) | :x: no|| +| [`net.host.carrier.mnc`](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/span-general/) | `network.carrier.mnc` | [see OTEL](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/span-general/) | :x: no|| +| [`net.host.carrier.icc`](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/span-general/) | `network.carrier.icc` | [see OTEL](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/span-general/) | :x: no|| +| [`http.client_ip`](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/http/#http-server-semantic-conventions) | `client.ip` | `10.121.13.12` | :x: no | This field is used to derive the geo location of the IP. | + + +### Additional Span attributes + +The following attributes do not have an OpenTelemetry semantic convention: + +| Attribute name | Elastic Convention | Example | Required | Comment | +|------------------------------------------|-----------------------------------------|-------------------------|-----------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `telemetry.sdk.elastic_export_timestamp` | N/A: only relevant for APM server. | `1658149487000000000` | :no_entry: deprecated | This is required to deal with the time skew on mobile devices. Set this to the timestamp (in nanoseconds) when the span is exported in the OpenTelemetry span processer. | +| `type` | `transaction.type` | `mobile` :interrobang: | :white_check_mark: yes | :heavy_exclamation_mark: Need to define new values for transactions resulting from mobile interactions. | +| `visibility` | :heavy_exclamation_mark: not mapped yet | `foreground/background` | :x: yes | This will be needed to tell whether the signal happened when the app was visible or in the background. | +| `service.build` | N/A | `555` | For Android: `yes`, for iOS: `no` | This is the build number (or the versionCode for Android builds). | + +### Attributes on outgoing HTTP spans + +The span name should have the format: ` `. + +| OTel Convention | Elastic convention | Example | Required | Comment | +|--------------------------|-----------------------|----------------| ---------| -----------------------------------| +| [`http.method`](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/http/) | `http.request.method` | `GET`, `POST` | :white_check_mark: yes | | +| [`http.status_code`](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/http/#common-attributes) | `http.response.status_code` | `200`, `500` | :white_check_mark: yes | Also used to derive the `event.outcome` | +| [`http.url`](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/http/) | `url.original` and other HTTP-related fields. | `http://localhost:3000/images/products/OP-DRC-C1.jpg` | :x: no| | +| [`http.target`](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/http/) | --- | `/images/products/OP-DRC-C1.jpg` | :x: no | Fallback field to derive HTTP-related fields if `http.url` field is not provided. | +| [`http.scheme`](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/http/) | --- | `http` | :x: no| Fallback field to derive HTTP-related fields if `http.url` field is not provided.| +| [`net.peer.name`](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/http/) | --- | `localhost` | :x: no| Fallback field to derive HTTP-related fields if `http.url` field is not provided.| +| [`net.peer.port`](https://opentelemetry.io/docs/reference/specification/trace/semantic_conventions/http/) | --- | `3000` | :x: no| Fallback field to derive HTTP-related fields if `http.url` field is not provided. | + +### Other Attributes + +The following OTel attributes do not fall under the category span or resource attributes. + +| OTel Convention | Elastic Convention | Example | Required | Comment | +|------------------------|-----------------------------------|----------------------------------------| ---------|--------------------------------------| +| `session.id` | `session.id` | `A73DC41E-DF18-4BB4-ABC0-F0000FD95653` | :x: no | [Elastic session spec](./session.md) | diff --git a/specs/agents/mobile/configuration.md b/specs/agents/mobile/configuration.md new file mode 100644 index 00000000..48cec73e --- /dev/null +++ b/specs/agents/mobile/configuration.md @@ -0,0 +1,20 @@ +## Mobile Configuration + +This document describes the configurable parameters used in mobile agents. The ones supported +by [central configuration](../configuration.md) can be set through Kibana's APM Settings. + +### `recording` configuration + +A boolean specifying if the agent should be recording or not. When recording, the agent instruments incoming HTTP +requests, tracks errors and collects and sends metrics. When not recording, the agent works as a noop, not collecting +data and not communicating with the APM sever, except for polling the central configuration endpoint. As this is a +reversible switch, agent threads are not being killed when inactivated, but they will be mostly idle in this state, so +the overhead should be negligible. + +You can use this setting to dynamically disable Elastic APM at runtime. + +| | | +|----------------|-----------| +| Type | `Boolean` | +| Default | `true` | +| Central config | `true` | diff --git a/specs/agents/mobile/events.md b/specs/agents/mobile/events.md new file mode 100644 index 00000000..e8019f82 --- /dev/null +++ b/specs/agents/mobile/events.md @@ -0,0 +1,673 @@ +## Mobile Events + +Status: [Experimental](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/document-status.md) + +This document describes event used by the Mobile SDKs using the [OpenTelementry Event Api](https://github.com/open-telemetry/opentelemetry-specification/blob/0a4c6656d1ac1261cfe426b964fd63b1c302877d/specification/logs/event-api.md). +All events collected by the mobile agents should set the `event.domain` to `device`. +Event names will be recording using the `event.name`. + +Due to the dependency on OpenTelemetry's events API this document's contents are subject to change and considered [experimental](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/document-status.md). +### Application Lifecycle events +These event represents the occurrence of app entering the foreground or background or other application lifecycle states. +The precise names of these events are still to be determined. They may mirror the lifecycle events their respective mobile platforms. + +| Name | Type | Values | Description | +|-------------------|--------|--------------------|--------------------------------------------| +| `event.name` | String | `lifecycle` | The name of the event. | +| `event.domain` | String | `device` | The domain of the event. | +| `lifecycle.state` | String | platform specific | The state entered at the time of the event | + +#### `lifecycle.state` values + +##### iOS +| Name | `lifecycle.state` Value | description | +|--------------------------------|-------------------------|--------------------------------------------| +| applicationDidBecomeActive | `active` | The app has become "active" | +| applicationWillResignActive | `inactive` | The app is about to become "inactive". | +| applicationDidEnterBackground | `background` | The app is now in the background. | +| applicationWillEnterForeground | `foreground` | The app is about to enter the foreground. | +| applicationWillTerminate | `terminate` | The app is about to terminate. | + +##### Android + +| Name | `lifecycle.state` Value | description | +|----------------------|-------------------------|-------------------------------------------------| +| App process onCreate | `created` | The app's process has been launched. | +| App process onStart | `started` | The app is about to be shown in the foreground. | +| App process onResume | `resumed` | The app is in the foreground. | +| App process onPause | `paused` | The app is about to go into the background. | +| App process onStop | `stopped` | The app is in the background. | + +### Crashes + +This event represent a crash event + +#### Attributes + +| OTel Convention | Elastic Convention | Type | Example | Required | Comment | +|------------------------|----------------------------|---------|--------------------|------------------------|------------------------| +| `event.name` | `error.type` | String | `crash` | :white_check_mark: YES | The event name. | +| `event.domain` | `event.category` | String | `device` | :white_check_mark: YES | the event domain. | +| `exception.message` | `error.exception.messasge` | String | `Division by zero` | :x: NO | The exception message. | +| `exception.stacktrace` | `error.stack_trace` | String | *see below | :white_check_mark: YES | A Stacktrace. | +| `exception.type` | `error.exception.type` | String | `OSError` | :white_check_mark: YES | The exception type. | + + +##### Stack Trace +###### Android + +###### iOS +``` +Incident Identifier: 4EEF9635-3A5D-4F24-886F-76CE909D7E8F +Hardware Model: x86_64 +Process: opbeans-swift [61567] +Path: /Users/brycebuchanan/Library/Developer/CoreSimulator/Devices/B6EAFC89-4B9F-4A92-923A-8F0D33ADD3B6/data/Containers/Bundle/Application/9CA73473-5BA1-4EB0-A46C-8C52995A049B/opbeans-swift.app/opbeans-swift +Identifier: co.elastic.opbeans-swift +Version: 2.2 (2.1) +Code Type: X86-64 +Parent Process: launchd_sim [47405] + +Date/Time: 2022-11-16 20:38:48 +0000 +OS Version: Mac OS X 16.1 (21G217) +Report Version: 104 + +Exception Type: SIGILL +Exception Codes: ILL_NOOP at 0x0 +Crashed Thread: 0 + +Thread 0 Crashed: +0 libswiftCore.dylib 0x00007ff80d700e3e 0x7ff80d6d4000 + 183870 +1 opbeans-swift 0x000000010132465b 0x1012f0000 + 214619 +2 SwiftUI 0x0000000109f81a14 0x109979000 + 6326804 +3 SwiftUI 0x000000010a450270 0x109979000 + 11367024 +4 SwiftUI 0x000000010a9df0b6 0x109979000 + 17195190 +5 SwiftUI 0x000000010a9dc77a 0x109979000 + 17184634 +6 SwiftUI 0x000000010ad4dd2c 0x109979000 + 20794668 +7 SwiftUI 0x000000010ad4dd72 0x109979000 + 20794738 +8 UIKitCore 0x0000000106ec8433 0x105a5f000 + 21402675 +9 QuartzCore 0x00007ff80893c0fc 0x7ff808761000 + 1945852 +10 QuartzCore 0x00007ff8089475d5 0x7ff808761000 + 1992149 +11 QuartzCore 0x00007ff808859431 0x7ff808761000 + 1016881 +12 QuartzCore 0x00007ff808890d56 0x7ff808761000 + 1244502 +13 UIKitCore 0x00000001068cee0f 0x105a5f000 + 15138319 +14 CoreFoundation 0x00007ff8003860a5 0x7ff800302000 + 540837 +15 CoreFoundation 0x00007ff800380a12 0x7ff800302000 + 518674 +16 CoreFoundation 0x00007ff800380f5d 0x7ff800302000 + 520029 +17 CoreFoundation 0x00007ff8003806f7 0x7ff800302000 + 517879 +18 GraphicsServices 0x00007ff809c5c28a 0x7ff809c59000 + 12938 +19 UIKitCore 0x000000010689a62b 0x105a5f000 + 14923307 +20 UIKitCore 0x000000010689f547 0x105a5f000 + 14943559 +21 SwiftUI 0x000000010aa5ecfb 0x109979000 + 17718523 +22 SwiftUI 0x000000010aa5eba8 0x109979000 + 17718184 +23 SwiftUI 0x000000010a114b7d 0x109979000 + 7977853 +24 opbeans-swift 0x00000001013271be 0x1012f0000 + 225726 +25 opbeans-swift 0x0000000101327299 0x1012f0000 + 225945 +26 ??? 0x0000000103eaa2bf 0x0 + 0 +27 ??? 0x00000001059ac52e 0x0 + 0 + +Thread 1: +0 libsystem_kernel.dylib 0x00007ff834be105a 0x7ff834bde000 + 12378 +1 libsystem_pthread.dylib 0x00007ff834c3a034 0x7ff834c37000 + 12340 +2 libsystem_pthread.dylib 0x00007ff834c38f57 0x7ff834c37000 + 8023 + +Thread 2: +0 libsystem_kernel.dylib 0x00007ff834be434e 0x7ff834bde000 + 25422 +1 opbeans-swift 0x000000010193c5d9 0x1012f0000 + 6604249 +2 opbeans-swift 0x00000001019431af 0x1012f0000 + 6631855 +3 opbeans-swift 0x0000000101943674 0x1012f0000 + 6633076 +4 opbeans-swift 0x000000010193d23b 0x1012f0000 + 6607419 +5 opbeans-swift 0x00000001019430a8 0x1012f0000 + 6631592 +6 opbeans-swift 0x0000000101925746 0x1012f0000 + 6510406 +7 opbeans-swift 0x000000010192678f 0x1012f0000 + 6514575 +8 opbeans-swift 0x0000000101917723 0x1012f0000 + 6453027 +9 opbeans-swift 0x0000000101923f63 0x1012f0000 + 6504291 +10 opbeans-swift 0x000000010191d974 0x1012f0000 + 6478196 +11 opbeans-swift 0x0000000101912669 0x1012f0000 + 6432361 +12 opbeans-swift 0x0000000101916024 0x1012f0000 + 6447140 +13 opbeans-swift 0x000000010190bea2 0x1012f0000 + 6405794 +14 opbeans-swift 0x000000010190beec 0x1012f0000 + 6405868 +15 libswiftObjectiveC.dylib 0x00007ff8250f7c4c 0x7ff8250f6000 + 7244 +16 opbeans-swift 0x000000010190be49 0x1012f0000 + 6405705 +17 opbeans-swift 0x00000001019114aa 0x1012f0000 + 6427818 +18 opbeans-swift 0x00000001018dd789 0x1012f0000 + 6215561 +19 opbeans-swift 0x00000001018dde71 0x1012f0000 + 6217329 +20 opbeans-swift 0x00000001018e398c 0x1012f0000 + 6240652 +21 opbeans-swift 0x0000000101943a0f 0x1012f0000 + 6633999 +22 opbeans-swift 0x0000000101949698 0x1012f0000 + 6657688 +23 opbeans-swift 0x0000000101949749 0x1012f0000 + 6657865 +24 libsystem_pthread.dylib 0x00007ff834c3d4e1 0x7ff834c37000 + 25825 +25 libsystem_pthread.dylib 0x00007ff834c38f6b 0x7ff834c37000 + 8043 + +Thread 3: +0 libsystem_kernel.dylib 0x00007ff834be105a 0x7ff834bde000 + 12378 +1 libsystem_pthread.dylib 0x00007ff834c3a034 0x7ff834c37000 + 12340 +2 libsystem_pthread.dylib 0x00007ff834c38f57 0x7ff834c37000 + 8023 + +Thread 4: +0 libsystem_kernel.dylib 0x00007ff834be23ea 0x7ff834bde000 + 17386 +1 libsystem_pthread.dylib 0x00007ff834c3daa4 0x7ff834c37000 + 27300 +2 Foundation 0x00007ff800bfad9e 0x7ff8006fd000 + 5234078 +3 opbeans-swift 0x0000000101475b51 0x1012f0000 + 1596241 +4 opbeans-swift 0x0000000101477960 0x1012f0000 + 1603936 +5 libswiftObjectiveC.dylib 0x00007ff8250f7c4c 0x7ff8250f6000 + 7244 +6 opbeans-swift 0x0000000101475969 0x1012f0000 + 1595753 +7 opbeans-swift 0x0000000101475d3c 0x1012f0000 + 1596732 +8 Foundation 0x00007ff800c7f1c3 0x7ff8006fd000 + 5775811 +9 libsystem_pthread.dylib 0x00007ff834c3d4e1 0x7ff834c37000 + 25825 +10 libsystem_pthread.dylib 0x00007ff834c38f6b 0x7ff834c37000 + 8043 + +Thread 5: +0 libsystem_kernel.dylib 0x00007ff834be23ea 0x7ff834bde000 + 17386 +1 libsystem_pthread.dylib 0x00007ff834c3daa4 0x7ff834c37000 + 27300 +2 Foundation 0x00007ff800bfad9e 0x7ff8006fd000 + 5234078 +3 opbeans-swift 0x00000001013f360e 0x1012f0000 + 1062414 +4 opbeans-swift 0x00000001013f7350 0x1012f0000 + 1078096 +5 libswiftObjectiveC.dylib 0x00007ff8250f7c4c 0x7ff8250f6000 + 7244 +6 opbeans-swift 0x00000001013f3429 0x1012f0000 + 1061929 +7 opbeans-swift 0x00000001013f37ec 0x1012f0000 + 1062892 +8 Foundation 0x00007ff800c7f1c3 0x7ff8006fd000 + 5775811 +9 libsystem_pthread.dylib 0x00007ff834c3d4e1 0x7ff834c37000 + 25825 +10 libsystem_pthread.dylib 0x00007ff834c38f6b 0x7ff834c37000 + 8043 + +Thread 6: +0 libsystem_kernel.dylib 0x00007ff834be105a 0x7ff834bde000 + 12378 +1 libsystem_pthread.dylib 0x00007ff834c3a034 0x7ff834c37000 + 12340 +2 libsystem_pthread.dylib 0x00007ff834c38f57 0x7ff834c37000 + 8023 + +Thread 7: +0 libsystem_kernel.dylib 0x00007ff834be105a 0x7ff834bde000 + 12378 +1 libsystem_pthread.dylib 0x00007ff834c3a034 0x7ff834c37000 + 12340 +2 libsystem_pthread.dylib 0x00007ff834c38f57 0x7ff834c37000 + 8023 + +Thread 8: +0 libsystem_kernel.dylib 0x00007ff834be105a 0x7ff834bde000 + 12378 +1 libsystem_pthread.dylib 0x00007ff834c3a034 0x7ff834c37000 + 12340 +2 libsystem_pthread.dylib 0x00007ff834c38f57 0x7ff834c37000 + 8023 + +Thread 9: +0 libsystem_kernel.dylib 0x00007ff834be105a 0x7ff834bde000 + 12378 +1 libsystem_pthread.dylib 0x00007ff834c3a034 0x7ff834c37000 + 12340 +2 libsystem_pthread.dylib 0x00007ff834c38f57 0x7ff834c37000 + 8023 + +Thread 10: +0 libsystem_kernel.dylib 0x00007ff834be105a 0x7ff834bde000 + 12378 +1 libsystem_pthread.dylib 0x00007ff834c3a034 0x7ff834c37000 + 12340 +2 libsystem_pthread.dylib 0x00007ff834c38f57 0x7ff834c37000 + 8023 + +Thread 11: +0 libsystem_kernel.dylib 0x00007ff834be105a 0x7ff834bde000 + 12378 +1 libsystem_pthread.dylib 0x00007ff834c3a034 0x7ff834c37000 + 12340 +2 libsystem_pthread.dylib 0x00007ff834c38f57 0x7ff834c37000 + 8023 + +Thread 12: +0 libsystem_kernel.dylib 0x00007ff834be105a 0x7ff834bde000 + 12378 +1 libsystem_pthread.dylib 0x00007ff834c3a034 0x7ff834c37000 + 12340 +2 libsystem_pthread.dylib 0x00007ff834c38f57 0x7ff834c37000 + 8023 + +Thread 13: +0 libsystem_kernel.dylib 0x00007ff834be105a 0x7ff834bde000 + 12378 +1 libsystem_pthread.dylib 0x00007ff834c3a034 0x7ff834c37000 + 12340 +2 libsystem_pthread.dylib 0x00007ff834c38f57 0x7ff834c37000 + 8023 + +Thread 14: +0 libsystem_kernel.dylib 0x00007ff834be105a 0x7ff834bde000 + 12378 +1 libsystem_pthread.dylib 0x00007ff834c3a034 0x7ff834c37000 + 12340 +2 libsystem_pthread.dylib 0x00007ff834c38f57 0x7ff834c37000 + 8023 + +Thread 15: +0 libsystem_kernel.dylib 0x00007ff834be105a 0x7ff834bde000 + 12378 +1 libsystem_pthread.dylib 0x00007ff834c3a034 0x7ff834c37000 + 12340 +2 libsystem_pthread.dylib 0x00007ff834c38f57 0x7ff834c37000 + 8023 + +Thread 16: +0 libsystem_kernel.dylib 0x00007ff834be105a 0x7ff834bde000 + 12378 +1 libsystem_pthread.dylib 0x00007ff834c3a034 0x7ff834c37000 + 12340 +2 libsystem_pthread.dylib 0x00007ff834c38f57 0x7ff834c37000 + 8023 + +Thread 17: +0 libsystem_kernel.dylib 0x00007ff834be105a 0x7ff834bde000 + 12378 +1 libsystem_pthread.dylib 0x00007ff834c3a034 0x7ff834c37000 + 12340 +2 libsystem_pthread.dylib 0x00007ff834c38f57 0x7ff834c37000 + 8023 + +Thread 18: +0 libsystem_kernel.dylib 0x00007ff834be105a 0x7ff834bde000 + 12378 +1 libsystem_pthread.dylib 0x00007ff834c3a034 0x7ff834c37000 + 12340 +2 libsystem_pthread.dylib 0x00007ff834c38f57 0x7ff834c37000 + 8023 + +Thread 19: +0 opbeans-swift 0x0000000101fd5f07 0x1012f0000 + 13524743 +1 opbeans-swift 0x0000000101fcac8e 0x1012f0000 + 13479054 +2 libsystem_pthread.dylib 0x00007ff834c3d4e1 0x7ff834c37000 + 25825 +3 libsystem_pthread.dylib 0x00007ff834c38f6b 0x7ff834c37000 + 8043 + +Thread 20: +0 libsystem_kernel.dylib 0x00007ff834bdf97a 0x7ff834bde000 + 6522 +1 libsystem_kernel.dylib 0x00007ff834bdfce8 0x7ff834bde000 + 7400 +2 CoreFoundation 0x00007ff8003868de 0x7ff800302000 + 542942 +3 CoreFoundation 0x00007ff80038102f 0x7ff800302000 + 520239 +4 CoreFoundation 0x00007ff8003806f7 0x7ff800302000 + 517879 +5 Foundation 0x00007ff800c5595c 0x7ff8006fd000 + 5605724 +6 Foundation 0x00007ff800c55bd5 0x7ff8006fd000 + 5606357 +7 UIKitCore 0x000000010696e886 0x105a5f000 + 15792262 +8 Foundation 0x00007ff800c7f1c3 0x7ff8006fd000 + 5775811 +9 libsystem_pthread.dylib 0x00007ff834c3d4e1 0x7ff834c37000 + 25825 +10 libsystem_pthread.dylib 0x00007ff834c38f6b 0x7ff834c37000 + 8043 + +Thread 21: +0 ??? 0x0000000000000000 0x0 + 0 + +Thread 22: +0 libsystem_kernel.dylib 0x00007ff834bdf97a 0x7ff834bde000 + 6522 +1 libsystem_kernel.dylib 0x00007ff834bdfce8 0x7ff834bde000 + 7400 +2 CoreFoundation 0x00007ff8003868de 0x7ff800302000 + 542942 +3 CoreFoundation 0x00007ff80038102f 0x7ff800302000 + 520239 +4 CoreFoundation 0x00007ff8003806f7 0x7ff800302000 + 517879 +5 Foundation 0x00007ff800c5595c 0x7ff8006fd000 + 5605724 +6 Foundation 0x00007ff800c55b7a 0x7ff8006fd000 + 5606266 +7 SwiftUI 0x0000000109eec31d 0x109979000 + 5714717 +8 SwiftUI 0x0000000109eec438 0x109979000 + 5715000 +9 Foundation 0x00007ff800c7f1c3 0x7ff8006fd000 + 5775811 +10 libsystem_pthread.dylib 0x00007ff834c3d4e1 0x7ff834c37000 + 25825 +11 libsystem_pthread.dylib 0x00007ff834c38f6b 0x7ff834c37000 + 8043 + +Thread 0 crashed with X86-64 Thread State: + rip: 0x00007ff80d700e3e rbp: 0x00007ff7bec0a3b0 rsp: 0x00007ff7bec0a360 rax: 0x00000000cbb4814f + rbx: 0x0000000101ff5000 rcx: 0x0000000000000000 rdx: 0x00000000cbd488da rdi: 0xe000000000000000 + rsi: 0x000000000000014f r8: 0x0000000000000400 r9: 0x00000000000003ff r10: 0x0000000000000d90 + r11: 0x0000600001cdf270 r12: 0x0000000000000000 r13: 0x0000000000000000 r14: 0xe000000000000000 + r15: 0x000000000000000b rflags: 0x0000000000010286 cs: 0x000000000000002b fs: 0x0000000000000000 + gs: 0x0000000000000000 + +Binary Images: + 0x1012f0000 - 0x102133fff +opbeans-swift x86_64 /Users/brycebuchanan/Library/Developer/CoreSimulator/Devices/B6EAFC89-4B9F-4A92-923A-8F0D33ADD3B6/data/Containers/Bundle/Application/9CA73473-5BA1-4EB0-A46C-8C52995A049B/opbeans-swift.app/opbeans-swift + 0x103dd5000 - 0x103dd5fff UIKit x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/UIKit.framework/UIKit + 0x103ddc000 - 0x103e1cfff WebGPU x86_64 <23483bf4b296392d96e6481a08c70d34> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Cryptexes/OS/System/Library/PrivateFrameworks/WebGPU.framework/WebGPU + 0x103e52000 - 0x103e64fff RecapPerformanceTesting x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/RecapPerformanceTesting.framework/RecapPerformanceTesting + 0x103f40000 - 0x103fddfff PrintKitUI x86_64 <1a3cbb5d5ba339e9a0923491665bb130> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/PrintKitUI.framework/PrintKitUI + 0x10406d000 - 0x104079fff libobjc-trampolines.dylib x86_64 <0e8ef289dd1f3934853b814b0e036ae9> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libobjc-trampolines.dylib + 0x10408d000 - 0x1040d9fff DocumentManager x86_64 <6752ee8146923c26818746dcd4bf1211> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/DocumentManager.framework/DocumentManager + 0x1041d5000 - 0x104240fff libswiftUIKit.dylib x86_64 <63c2daff5b683cae849bc2fc12211def> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftUIKit.dylib + 0x104325000 - 0x1044bdfff WebKitLegacy x86_64 <502989767ea63634b258422f259f45dc> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Cryptexes/OS/System/Library/PrivateFrameworks/WebKitLegacy.framework/WebKitLegacy + 0x104782000 - 0x104888fff ShareSheet x86_64 <789ea4e9318630a1b97ca066fc85c8ca> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/ShareSheet.framework/ShareSheet + 0x10499c000 - 0x1052b7fff libwebrtc.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Cryptexes/OS/System/Library/PrivateFrameworks/WebCore.framework/Frameworks/libwebrtc.dylib + 0x105a5f000 - 0x10751bfff UIKitCore x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/UIKitCore.framework/UIKitCore + 0x108c5e000 - 0x108fd0fff libANGLE-shared.dylib x86_64 <01520fb4055130d4b8586feb8d020534> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Cryptexes/OS/System/Library/PrivateFrameworks/WebCore.framework/Frameworks/libANGLE-shared.dylib + 0x109979000 - 0x10b051fff SwiftUI x86_64 <1622c1620a3a354c94ea62c5dd369612> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/SwiftUI.framework/SwiftUI + 0x10eca2000 - 0x1102befff JavaScriptCore x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Cryptexes/OS/System/Library/Frameworks/JavaScriptCore.framework/JavaScriptCore + 0x117abd000 - 0x11a22efff WebCore x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Cryptexes/OS/System/Library/PrivateFrameworks/WebCore.framework/WebCore + 0x7ff80002c000 - 0x7ff80005ffe9 libobjc.A.dylib x86_64 <6d6cbe478a163a78ba9076b48d34921a> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libobjc.A.dylib + 0x7ff800060000 - 0x7ff800077ffe libsystem_trace.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_trace.dylib + 0x7ff800078000 - 0x7ff8000b0ff8 libxpc.dylib x86_64 <0a4f2baaa11032c8bba85835332e4178> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libxpc.dylib + 0x7ff8000b1000 - 0x7ff8000b4ff2 libsystem_blocks.dylib x86_64 <26fd5f84f03c3699a7edce263a8aa8a6> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_blocks.dylib + 0x7ff8000b5000 - 0x7ff800139ff7 libsystem_c.dylib x86_64 <0d67a685513e32efbe4b9ed886be9acb> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_c.dylib + 0x7ff80013a000 - 0x7ff800185ff3 libdispatch.dylib x86_64 <1b19169b60d3328c91c1b35f3aa8a173> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libdispatch.dylib + 0x7ff800186000 - 0x7ff8001b0fff libsystem_malloc.dylib x86_64 <4d681c2122493cb7a26a3155dfe85f53> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_malloc.dylib + 0x7ff8001b1000 - 0x7ff800246ff7 libcorecrypto.dylib x86_64 <9b20e776f8b431668438b4b5b84fa1ea> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libcorecrypto.dylib + 0x7ff800247000 - 0x7ff80025cffb libc++abi.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libc++abi.dylib + 0x7ff80025d000 - 0x7ff80027dff7 libdyld.dylib x86_64 <25f541a137b23f7e8b8b3d8703f4e0de> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libdyld.dylib + 0x7ff80027e000 - 0x7ff800286ff7 libsystem_darwin.dylib x86_64 <40e5cae4013930deac9c4f962a35275f> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_darwin.dylib + 0x7ff800287000 - 0x7ff8002dcff7 libc++.1.dylib x86_64 <272d2f718a2a3634a5e65d8619f3a786> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libc++.1.dylib + 0x7ff8002dd000 - 0x7ff800301ff7 libsystem_info.dylib x86_64 <96e4e636e0143a3ca441cd1bb9d8e6de> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_info.dylib + 0x7ff800302000 - 0x7ff80068affc CoreFoundation x86_64 <55edff37af143fedb932031049d0a665> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation + 0x7ff80068b000 - 0x7ff8006fcffb SystemConfiguration x86_64 <5da3ae3a7bb535fab666f1bf3f45cc01> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/SystemConfiguration.framework/SystemConfiguration + 0x7ff8006fd000 - 0x7ff80102dffc Foundation x86_64 <353e6739fc3a363689f2194adba7203b> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Foundation.framework/Foundation + 0x7ff80102e000 - 0x7ff80105cff3 libCRFSuite.dylib x86_64 <5a6611bfefba303fa1b7ce73aa3dc30e> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libCRFSuite.dylib + 0x7ff80105d000 - 0x7ff80125bff8 CoreServices x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CoreServices.framework/CoreServices + 0x7ff80125c000 - 0x7ff8012d1ff7 libSparse.dylib x86_64 <8863bb3a39f63b6582df097b10b35aa7> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Accelerate.framework/Frameworks/vecLib.framework/libSparse.dylib + 0x7ff8012d2000 - 0x7ff801789ff3 ImageIO x86_64 <3df3e3db829c37a48838cac96ffcc4a5> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/ImageIO.framework/ImageIO + 0x7ff80178d000 - 0x7ff801962ff1 CoreText x86_64 <76bd911e6f6d31a4aa9d46ac56eaf74e> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CoreText.framework/CoreText + 0x7ff801963000 - 0x7ff801ae3ffe Security x86_64 <445c823d97af3377ba89cb65c727a9e4> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Security.framework/Security + 0x7ff801ae4000 - 0x7ff801b79ff8 IOKit x86_64 <2bad9090d588385cb36954499e5cdbd8> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit + 0x7ff801b7a000 - 0x7ff801babff1 libMobileGestalt.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libMobileGestalt.dylib + 0x7ff801c18000 - 0x7ff801ea4ff3 libicucore.A.dylib x86_64 <047c3ae11bea3160b91b1d565e36e14f> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libicucore.A.dylib + 0x7ff801ea5000 - 0x7ff801ed5fff CoreServicesInternal x86_64 <11dbd84231673debae3c1ca06da94930> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CoreServicesInternal.framework/CoreServicesInternal + 0x7ff801f49000 - 0x7ff8023bbff7 CoreAudio x86_64 <7159cbeeb00c38b497e52db84003b525> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CoreAudio.framework/CoreAudio + 0x7ff8023bc000 - 0x7ff802735ff7 CoreImage x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CoreImage.framework/CoreImage + 0x7ff802736000 - 0x7ff802888ff3 LanguageModeling x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/LanguageModeling.framework/LanguageModeling + 0x7ff802889000 - 0x7ff8028d1ff3 Lexicon x86_64 <978c65c5ec8a3b62bdcfcb3c0ac82ff0> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/Lexicon.framework/Lexicon + 0x7ff8028d2000 - 0x7ff802a83ff7 libsqlite3.dylib x86_64 <3a9edb3de32d3947aad404a58071dfb2> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libsqlite3.dylib + 0x7ff802a84000 - 0x7ff802a8eff7 libsystem_notify.dylib x86_64 <2cacafe644da378b8df159839f32e4bb> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_notify.dylib + 0x7ff802cc3000 - 0x7ff802efdff0 Montreal x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/Montreal.framework/Montreal + 0x7ff803019000 - 0x7ff803059ff7 AppSupport x86_64 <060146fc2aec338b9804614342719968> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AppSupport.framework/AppSupport + 0x7ff80305a000 - 0x7ff80305afff libnetwork.dylib x86_64 <73c6a4056890314698bcdb308742d0f5> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libnetwork.dylib + 0x7ff80305b000 - 0x7ff80318bff4 ManagedConfiguration x86_64 <2ef265171ab23daa8e4fd0db8486eb9e> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/ManagedConfiguration.framework/ManagedConfiguration + 0x7ff80318c000 - 0x7ff8031c4ff7 CoreServicesStore x86_64 <01f3969d84a73ced958c0bab0899a280> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CoreServicesStore.framework/CoreServicesStore + 0x7ff8031c5000 - 0x7ff8031ecfff UserManagement x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/UserManagement.framework/UserManagement + 0x7ff8031ed000 - 0x7ff8035a1ff8 CoreML x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CoreML.framework/CoreML + 0x7ff8035a2000 - 0x7ff8035baffc ProtocolBuffer x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/ProtocolBuffer.framework/ProtocolBuffer + 0x7ff8035bb000 - 0x7ff8035d2fff CommonUtilities x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CommonUtilities.framework/CommonUtilities + 0x7ff8035d3000 - 0x7ff803629ff2 RunningBoardServices x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/RunningBoardServices.framework/RunningBoardServices + 0x7ff80362a000 - 0x7ff8036e0ff2 BaseBoard x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/BaseBoard.framework/BaseBoard + 0x7ff8036e1000 - 0x7ff80403ffe3 SiriTTS x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SiriTTS.framework/SiriTTS + 0x7ff804040000 - 0x7ff804127ff4 CoreLocation x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CoreLocation.framework/CoreLocation + 0x7ff804131000 - 0x7ff8041a9ffb Accounts x86_64 <9f617442c07b37df96c4fb5ba4aa2470> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Accounts.framework/Accounts + 0x7ff8041cd000 - 0x7ff804567ff7 CFNetwork x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CFNetwork.framework/CFNetwork + 0x7ff804568000 - 0x7ff8046b6fff UIFoundation x86_64 <2d04f31f720a37bc8e58feac03792da4> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/UIFoundation.framework/UIFoundation + 0x7ff8046b7000 - 0x7ff8046c3ff8 AssertionServices x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AssertionServices.framework/AssertionServices + 0x7ff8046c4000 - 0x7ff8047e0ffe CoreTelephony x86_64 <46b3d5ec96853228b4c1502abfec2205> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CoreTelephony.framework/CoreTelephony + 0x7ff8047e1000 - 0x7ff8047e1ffc AggregateDictionary x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AggregateDictionary.framework/AggregateDictionary + 0x7ff8047e2000 - 0x7ff8047f8ff3 libsystem_asl.dylib x86_64 <0b52af584a5a3e499eed687c3425fa14> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_asl.dylib + 0x7ff8048a6000 - 0x7ff804c8dff0 CoreData x86_64 <6f605ede51d332fb93358777416e69fd> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CoreData.framework/CoreData + 0x7ff804c8e000 - 0x7ff804fe2ffd Vision x86_64 <1ec3ef4e9dbb31e68d0290de810e5228> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Vision.framework/Vision + 0x7ff804fee000 - 0x7ff80504bffd BoardServices x86_64 <9a227228a114377fab0d1f4fec4ff994> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/BoardServices.framework/BoardServices + 0x7ff80504c000 - 0x7ff805117ff3 libboringssl.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libboringssl.dylib + 0x7ff805118000 - 0x7ff805140ffb CoreAnalytics x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CoreAnalytics.framework/CoreAnalytics + 0x7ff805141000 - 0x7ff8053f2ff9 CloudKit x86_64 <1a60123722b53490bed1f4882fbf680e> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CloudKit.framework/CloudKit + 0x7ff8053f3000 - 0x7ff805473ff0 SpringBoardServices x86_64 <493c8b56c218324ba28adaaf3f1fa1c2> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SpringBoardServices.framework/SpringBoardServices + 0x7ff805474000 - 0x7ff805515ff4 FrontBoardServices x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/FrontBoardServices.framework/FrontBoardServices + 0x7ff805516000 - 0x7ff806024ffd Network x86_64 <1d39ba4ff59d3b0aa34d6a3e0e1a58c5> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Network.framework/Network + 0x7ff806025000 - 0x7ff806089ff6 libusrtcp.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libusrtcp.dylib + 0x7ff80608a000 - 0x7ff807cbafe2 GeoServices x86_64 <09c376fc9f2d3ec9942299f1efaca5d7> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/GeoServices.framework/GeoServices + 0x7ff807cbb000 - 0x7ff807cccff5 TCC x86_64 <02ee17943de13bb8827a68d5f2b4a91b> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/TCC.framework/TCC + 0x7ff807ccd000 - 0x7ff807d34fff IMFoundation x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/IMFoundation.framework/IMFoundation + 0x7ff807d35000 - 0x7ff807eabff8 CoreUtils x86_64 <875537c6796c389d85743c526991b0a4> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CoreUtils.framework/CoreUtils + 0x7ff807fd7000 - 0x7ff808003ff7 libsystem_containermanager.dylib x86_64 <9d7404d588153f4fb0637ea581bb9d09> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_containermanager.dylib + 0x7ff808004000 - 0x7ff80816cfff AppleAccount x86_64 <57f4f30a1591337e812975913d40bc83> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AppleAccount.framework/AppleAccount + 0x7ff80816d000 - 0x7ff808197ffc ApplePushService x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/ApplePushService.framework/ApplePushService + 0x7ff808198000 - 0x7ff808327ffc IDS x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/IDS.framework/IDS + 0x7ff808328000 - 0x7ff808553ff6 IDSFoundation x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/IDSFoundation.framework/IDSFoundation + 0x7ff808554000 - 0x7ff808555ffb libCTGreenTeaLogger.dylib x86_64 <76d8976c3ce33bb7bda91e45bed6392d> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libCTGreenTeaLogger.dylib + 0x7ff8085ca000 - 0x7ff8086b1ffc CoreMedia x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CoreMedia.framework/CoreMedia + 0x7ff8086b2000 - 0x7ff8086daff6 UIKitServices x86_64 <1602efd471c937fcad344f68336a0685> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/UIKitServices.framework/UIKitServices + 0x7ff8086db000 - 0x7ff808760ffa BackBoardServices x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/BackBoardServices.framework/BackBoardServices + 0x7ff808761000 - 0x7ff808a6aff0 QuartzCore x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/QuartzCore.framework/QuartzCore + 0x7ff808a6b000 - 0x7ff809253fef CoreGraphics x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics + 0x7ff809423000 - 0x7ff809466ffd UserNotifications x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/UserNotifications.framework/UserNotifications + 0x7ff809467000 - 0x7ff809498ffb LocationSupport x86_64 <56b10ba22b6230c9b1d46f819f8c179c> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/LocationSupport.framework/LocationSupport + 0x7ff809499000 - 0x7ff80962fff1 Sharing x86_64 <56b038ae481f33e9b187089f744bfb05> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/Sharing.framework/Sharing + 0x7ff809630000 - 0x7ff80965dff4 libAccessibility.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libAccessibility.dylib + 0x7ff80965e000 - 0x7ff80967cff4 AXCoreUtilities x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AXCoreUtilities.framework/AXCoreUtilities + 0x7ff80971f000 - 0x7ff80972cff3 IOSurface x86_64 <0b1dff4ee18d3de28667573ee58f8283> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/IOSurface.framework/IOSurface + 0x7ff80972d000 - 0x7ff809c58ffe MediaToolbox x86_64 <5ec9d3942614367f9503417609e7f1df> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/MediaToolbox.framework/MediaToolbox + 0x7ff809c59000 - 0x7ff809c60ff2 GraphicsServices x86_64 <5dad91c5e70d3f9a88f22d1ed7c8dd24> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/GraphicsServices.framework/GraphicsServices + 0x7ff809e08000 - 0x7ff809e08fff AVFoundation x86_64 <7fe239a5b99e3fed8d1f3f50f622d397> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/AVFoundation.framework/AVFoundation + 0x7ff809e09000 - 0x7ff809e7cff3 MobileAsset x86_64 <5789221412de3b34ad125c8b59c727b6> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/MobileAsset.framework/MobileAsset + 0x7ff809e7d000 - 0x7ff809e86ff5 libGSFont.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/FontServices.framework/libGSFont.dylib + 0x7ff809e87000 - 0x7ff809e95ff0 FontServices x86_64 <9b3c0d16400033b395cab5b90a3299bc> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/FontServices.framework/FontServices + 0x7ff809e96000 - 0x7ff809fe6ff6 libFontParser.dylib x86_64 <08fe2b10c65d3fa99c2b683c2ad64c69> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/FontServices.framework/libFontParser.dylib + 0x7ff809fe7000 - 0x7ff80a083ff9 AXRuntime x86_64 <95cf5570752c3a83b0b4010bfd1caa36> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AXRuntime.framework/AXRuntime + 0x7ff80a084000 - 0x7ff80a087ff3 libAXSafeCategoryBundle.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libAXSafeCategoryBundle.dylib + 0x7ff80ab43000 - 0x7ff80b758ffb vImage x86_64 <13146ca9e1253fba8b3e0bffe90b4594> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Accelerate.framework/Frameworks/vImage.framework/vImage + 0x7ff80b759000 - 0x7ff80b87bff8 AudioToolbox x86_64 <69517cebfe423a9e813b9cb152335390> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/AudioToolbox.framework/AudioToolbox + 0x7ff80b87c000 - 0x7ff80b8adfff libAudioToolboxUtility.dylib x86_64 <6f3286a1f6533a2e9bc4f0a04f980f86> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libAudioToolboxUtility.dylib + 0x7ff80becc000 - 0x7ff80bf88fff AuthKit x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AuthKit.framework/AuthKit + 0x7ff80c5ac000 - 0x7ff80c5c8fff libCGInterfaces.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Accelerate.framework/Frameworks/vImage.framework/Libraries/libCGInterfaces.dylib + 0x7ff80c5c9000 - 0x7ff80c67dffe TextInput x86_64 <27761bad08f439d4a9d77ca33917b229> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/TextInput.framework/TextInput + 0x7ff80c787000 - 0x7ff80c78bff1 XCTTargetBootstrap x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/XCTTargetBootstrap.framework/XCTTargetBootstrap + 0x7ff80c78c000 - 0x7ff80cb94ff2 AppleMediaServices x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AppleMediaServices.framework/AppleMediaServices + 0x7ff80ce73000 - 0x7ff80d0eeff3 RawCamera x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/CoreServices/RawCamera.bundle/RawCamera + 0x7ff80d0ef000 - 0x7ff80d1e4ff2 CoreUI x86_64 <464b53d8a8d039c5bfa8cdf1bbed3a90> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CoreUI.framework/CoreUI + 0x7ff80d1e5000 - 0x7ff80d217ffd CoreVideo x86_64 <47067e803d413019b089d88aca219201> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CoreVideo.framework/CoreVideo + 0x7ff80d218000 - 0x7ff80d4a1ff1 AudioToolboxCore x86_64 <41948778fa3d3415bd02c30f7cff299a> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AudioToolboxCore.framework/AudioToolboxCore + 0x7ff80d4f5000 - 0x7ff80d530ff5 SetupAssistant x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SetupAssistant.framework/SetupAssistant + 0x7ff80d69c000 - 0x7ff80d6d3ff1 PlugInKit x86_64 <2e85181ed0283259aeaeb56715b43bd4> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/PlugInKit.framework/PlugInKit + 0x7ff80d6d4000 - 0x7ff80db90ff4 libswiftCore.dylib x86_64 <1d23cd508b48349b9163f7990e0f95bd> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftCore.dylib + 0x7ff80db91000 - 0x7ff80ddc2ff1 AssistantServices x86_64 <0b2d8184bef2347491791798f86fd943> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AssistantServices.framework/AssistantServices + 0x7ff80ddc3000 - 0x7ff80de46ffb ProactiveSupport x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/ProactiveSupport.framework/ProactiveSupport + 0x7ff80de47000 - 0x7ff80de6affc PrototypeTools x86_64 <9175f569590839b8bee2185565219056> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/PrototypeTools.framework/PrototypeTools + 0x7ff80de6b000 - 0x7ff80de70ffe MediaExperience x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/MediaExperience.framework/MediaExperience + 0x7ff80de71000 - 0x7ff80de73ff4 Celestial x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/Celestial.framework/Celestial + 0x7ff80de74000 - 0x7ff80df04ff9 CallKit x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CallKit.framework/CallKit + 0x7ff80eca9000 - 0x7ff80ecabffb libapp_launch_measurement.dylib x86_64 <88dc60b663fc31cdbcd769b0667c3562> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libapp_launch_measurement.dylib + 0x7ff80ee1a000 - 0x7ff80ee55ff7 MobileBluetooth x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/MobileBluetooth.framework/MobileBluetooth + 0x7ff80ee56000 - 0x7ff80ee58ff7 libsystem_configuration.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_configuration.dylib + 0x7ff80ee59000 - 0x7ff80ef28ff9 CoreNLP x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CoreNLP.framework/CoreNLP + 0x7ff80ef29000 - 0x7ff80ef2afff liblangid.dylib x86_64 <70c9e7da365534fc8051f484eb855c35> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/liblangid.dylib + 0x7ff80ef2b000 - 0x7ff80ef6eff4 libTelephonyUtilDynamic.dylib x86_64 <16009df9c94d3c03ae6da0368335d465> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libTelephonyUtilDynamic.dylib + 0x7ff80f196000 - 0x7ff80f1b7ff7 CoreMaterial x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CoreMaterial.framework/CoreMaterial + 0x7ff80f1b8000 - 0x7ff80f297fff libxml2.2.dylib x86_64 <4061324df7e53ce48e26308467a29b4e> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libxml2.2.dylib + 0x7ff80fdf6000 - 0x7ff80fe11ff1 FMCoreLite x86_64 <92c16da7940438239ab4438a22eb8763> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/FMCoreLite.framework/FMCoreLite + 0x7ff80fe12000 - 0x7ff81007eff1 NetworkExtension x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/NetworkExtension.framework/NetworkExtension + 0x7ff81007f000 - 0x7ff8100b9ff4 DataDetectorsCore x86_64 <36ca039b127f36a390386167482e2530> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/DataDetectorsCore.framework/DataDetectorsCore + 0x7ff8102ed000 - 0x7ff810347ffb MediaServices x86_64 <3b9df4e003a330e69e83e61820f5bb7c> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/MediaServices.framework/MediaServices + 0x7ff810356000 - 0x7ff81037dff4 PersistentConnection x86_64 <349bba0397753f6688d0826602d794f2> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/PersistentConnection.framework/PersistentConnection + 0x7ff810516000 - 0x7ff810850ff4 MediaRemote x86_64 <7cda1227fbf4355388b6d73f84fcc852> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/MediaRemote.framework/MediaRemote + 0x7ff810851000 - 0x7ff810859ff8 CorePhoneNumbers x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CorePhoneNumbers.framework/CorePhoneNumbers + 0x7ff81086e000 - 0x7ff8108a1ffb CoreSVG x86_64 <95e5d284eab939beaedbde8f3af2a7f9> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CoreSVG.framework/CoreSVG + 0x7ff8118f9000 - 0x7ff81190cfff BluetoothManager x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/BluetoothManager.framework/BluetoothManager + 0x7ff81190d000 - 0x7ff8119baff7 CoreBluetooth x86_64 <5781d3b18680371c96171f35ad480e19> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CoreBluetooth.framework/CoreBluetooth + 0x7ff8119bb000 - 0x7ff8119beffb libsystem_sandbox.dylib x86_64 <8759af487717312fb7725dc1ff6259f4> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_sandbox.dylib + 0x7ff811a1d000 - 0x7ff811aaeffe Rapport x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/Rapport.framework/Rapport + 0x7ff811afc000 - 0x7ff811b34ff2 MobileInstallation x86_64 <3e602d1aec7d33e0bb283f6028a15cb5> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/MobileInstallation.framework/MobileInstallation + 0x7ff811b35000 - 0x7ff811cc0ff0 Metal x86_64 <2e79c828417931688ed151f759e6b717> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Metal.framework/Metal + 0x7ff811cc1000 - 0x7ff811ccffff MediaAccessibility x86_64 <4ce47c810e033aafa8fb1b38e2fc0d02> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/MediaAccessibility.framework/MediaAccessibility + 0x7ff811cd0000 - 0x7ff811cf2ff8 FindMyDevice x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/FindMyDevice.framework/FindMyDevice + 0x7ff811cf3000 - 0x7ff811cfbfff libsystem_dnssd.dylib x86_64 <7c419714fc473ccd8145dd592606ec62> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_dnssd.dylib + 0x7ff811cfc000 - 0x7ff811d04ff6 PushKit x86_64 <9fe644509cd63388926c40c6d1ac1b76> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/PushKit.framework/PushKit + 0x7ff811d05000 - 0x7ff811e8cffa FileProvider x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/FileProvider.framework/FileProvider + 0x7ff811e8d000 - 0x7ff811e90ff8 LinguisticData x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/LinguisticData.framework/LinguisticData + 0x7ff811f1c000 - 0x7ff8123c9ff3 VideoToolbox x86_64 <341b1dec4c6d30e4b818c34b13faee79> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/VideoToolbox.framework/VideoToolbox + 0x7ff81261d000 - 0x7ff812633ff3 libcoretls.dylib x86_64 <4161c5e0f4ca37afa7f9034642bec02a> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libcoretls.dylib + 0x7ff812634000 - 0x7ff81275afcc libate.dylib x86_64 <19cba9e5164d3870be321b15c31b80ff> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libate.dylib + 0x7ff8139a9000 - 0x7ff8139deff1 Pasteboard x86_64 <85f28b17368e344983d933c473439680> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/Pasteboard.framework/Pasteboard + 0x7ff8139df000 - 0x7ff813a2eff4 FTServices x86_64 <40ec9062cded366ca1f10747d607cf77> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/FTServices.framework/FTServices + 0x7ff813b13000 - 0x7ff813bceffa SAObjects x86_64 <9ffd89a6a6e936d2bc1749a3ab737351> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SAObjects.framework/SAObjects + 0x7ff813cae000 - 0x7ff813cbafff DataMigration x86_64 <16b0bef5f219359d84d4e9e6370706c1> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/DataMigration.framework/DataMigration + 0x7ff813d6b000 - 0x7ff813dc0ffd IconServices x86_64 <8187e2efdd77343dbd5d5e225ddfa84c> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/IconServices.framework/IconServices + 0x7ff81428f000 - 0x7ff8142a2ffe libprequelite.dylib x86_64 <38a1a6e4938d3b91aab30991cdb68969> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libprequelite.dylib + 0x7ff814327000 - 0x7ff814338ffd CoreEmoji x86_64 <15f8a5d86e073044a72e4e6090a8e9aa> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CoreEmoji.framework/CoreEmoji + 0x7ff814509000 - 0x7ff81452cff7 RTCReporting x86_64 <8fcfeb6a6371333892959eb1998d716f> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/RTCReporting.framework/RTCReporting + 0x7ff81463b000 - 0x7ff814642ff6 CoreTime x86_64 <875b13b91625368b9f1b0b8e8898eff5> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CoreTime.framework/CoreTime + 0x7ff814d5e000 - 0x7ff814d86ff1 AppConduit x86_64 <2aa96b923bd2328f88166574054f3f4f> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AppConduit.framework/AppConduit + 0x7ff814d87000 - 0x7ff814da9ff9 IntlPreferences x86_64 <961356eb1e0e3a639a94f993818020cd> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/IntlPreferences.framework/IntlPreferences + 0x7ff814eb8000 - 0x7ff8155ebfe3 libBNNS.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Accelerate.framework/Frameworks/vecLib.framework/libBNNS.dylib + 0x7ff8166bf000 - 0x7ff8166f3ff5 LocalAuthentication x86_64 <6198f83a8e9634bfb2cc84c07a6082fe> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/LocalAuthentication.framework/LocalAuthentication + 0x7ff816722000 - 0x7ff816723ff1 CaptiveNetwork x86_64 <303f9f76b331351fba0eb28ab3273901> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CaptiveNetwork.framework/CaptiveNetwork + 0x7ff816766000 - 0x7ff8168effd3 libBLAS.dylib x86_64 <290acb325ee23033b4eb226bacffdea0> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Accelerate.framework/Frameworks/vecLib.framework/libBLAS.dylib + 0x7ff816b03000 - 0x7ff816b0fff4 MobileIcons x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/MobileIcons.framework/MobileIcons + 0x7ff816dbc000 - 0x7ff816e85ff7 CoreSymbolication x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CoreSymbolication.framework/CoreSymbolication + 0x7ff816e86000 - 0x7ff816e8dff8 IdleTimerServices x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/IdleTimerServices.framework/IdleTimerServices + 0x7ff816f41000 - 0x7ff816fb0ffc LoggingSupport x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/LoggingSupport.framework/LoggingSupport + 0x7ff817177000 - 0x7ff8171ccff4 ProtectedCloudStorage x86_64 <9e080c4fbe43379d99136e058bb44217> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/ProtectedCloudStorage.framework/ProtectedCloudStorage + 0x7ff8172ba000 - 0x7ff8172c6ff4 OpenGLES x86_64 <280919f29f453f8c8a9a3aeab3fcf55d> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/OpenGLES.framework/OpenGLES + 0x7ff81744b000 - 0x7ff817453ffb libGFXShared.dylib x86_64 <9a10c9f6a6763267a3196bace924fd9a> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/OpenGLES.framework/libGFXShared.dylib + 0x7ff817454000 - 0x7ff817476ff2 SharedUtils x86_64 <8f5de073fdc43653930e10bc73664501> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/LocalAuthentication.framework/Support/SharedUtils.framework/SharedUtils + 0x7ff8185d9000 - 0x7ff81861bffd StreamingZip x86_64 <884df7ebadf932fa97a6b0245cf00d40> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/StreamingZip.framework/StreamingZip + 0x7ff8189e4000 - 0x7ff8189e6ff7 InternationalTextSearch x86_64 <92de6cd58dcb38c18f989384d906b3c9> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/InternationalTextSearch.framework/InternationalTextSearch + 0x7ff819a4d000 - 0x7ff819a54ffa OSAServicesClient x86_64 <0c3a980754f43630a3f2a6fa314cb0c7> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/OSAServicesClient.framework/OSAServicesClient + 0x7ff819a55000 - 0x7ff819a57fff libgermantok.dylib x86_64 <33e737774c6837bc99c481123db9662a> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libgermantok.dylib + 0x7ff819a58000 - 0x7ff819ab0ff7 libmecab.dylib x86_64 <203afd326b5b3a8085076584ac5e610b> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libmecab.dylib + 0x7ff819b53000 - 0x7ff819bb7ffe VoiceServices x86_64 <14fb84b6ddeb3617abd90ee798b2d04b> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/VoiceServices.framework/VoiceServices + 0x7ff819f82000 - 0x7ff819f82ff7 FTAWD x86_64 <1a83cc243e2a3dc286ee56450e8d3ebd> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/FTAWD.framework/FTAWD + 0x7ff81a0a5000 - 0x7ff81a12dff3 libarchive.2.dylib x86_64 <48039f9fd6693af99442fa34914db801> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libarchive.2.dylib + 0x7ff81a12e000 - 0x7ff81a173ff0 C2 x86_64 <35c806522e7a3286a0637d52e45ffdef> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/C2.framework/C2 + 0x7ff81a174000 - 0x7ff81a1c8ff1 NaturalLanguage x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/NaturalLanguage.framework/NaturalLanguage + 0x7ff81a2fa000 - 0x7ff81a2fbff7 libsystem_coreservices.dylib x86_64 <8f391c581a9130cc88c218cbb9648b4c> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_coreservices.dylib + 0x7ff81a533000 - 0x7ff81a53cff3 libcopyfile.dylib x86_64 <9a4dcaa57e093e58aa1b168568835c7f> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libcopyfile.dylib + 0x7ff81a60a000 - 0x7ff81a614ff6 AppleIDSSOAuthentication x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AppleIDSSOAuthentication.framework/AppleIDSSOAuthentication + 0x7ff81adc6000 - 0x7ff81adcdffd URLFormatting x86_64 <2764e7ce5b7c3cb594aa4bf09b078245> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/URLFormatting.framework/URLFormatting + 0x7ff81b240000 - 0x7ff81b25eff0 SecurityFoundation x86_64 <61f7e7cfa5443d32a11fcfdbf2d46918> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SecurityFoundation.framework/SecurityFoundation + 0x7ff81b32b000 - 0x7ff81b3afff7 CoreLocationProtobuf x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CoreLocationProtobuf.framework/CoreLocationProtobuf + 0x7ff81b3dd000 - 0x7ff81b4afff6 Quagga x86_64 <685f92ca57f0357ba0f69d4f09d6c41e> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/Quagga.framework/Quagga + 0x7ff81b92d000 - 0x7ff81b93bff4 libperfcheck.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libperfcheck.dylib + 0x7ff81b93c000 - 0x7ff81b949ff4 libAudioStatistics.dylib x86_64 <02ea7fd57cfb319c8cca238cea679339> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libAudioStatistics.dylib + 0x7ff81b9f9000 - 0x7ff81ba1eff7 caulk x86_64 <316ce6303ad33d45a0c0236fbfdd5c13> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/caulk.framework/caulk + 0x7ff81ba63000 - 0x7ff81ba66fff MobileSystemServices x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/MobileSystemServices.framework/MobileSystemServices + 0x7ff81bb7e000 - 0x7ff81bc10ffb AirPlaySync x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AirPlaySync.framework/AirPlaySync + 0x7ff81c0ae000 - 0x7ff81c0ebffb libGLImage.dylib x86_64 <0da8ca7214a832a28e97a5a7357cc09b> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/OpenGLES.framework/libGLImage.dylib + 0x7ff81c41a000 - 0x7ff81c42cff7 libSparseBLAS.dylib x86_64 <0eb1c976f77338d8a847dffa3fceb719> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Accelerate.framework/Frameworks/vecLib.framework/libSparseBLAS.dylib + 0x7ff81c42d000 - 0x7ff81c443ff5 Engram x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/Engram.framework/Engram + 0x7ff81d421000 - 0x7ff81d44affc RemoteTextInput x86_64 <9b7d20273ed0342386a396b19096b90d> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/RemoteTextInput.framework/RemoteTextInput + 0x7ff81d4a1000 - 0x7ff81d4b6fff libLinearAlgebra.dylib x86_64 <86807e3a4bc23656a68b3a7eb050918c> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Accelerate.framework/Frameworks/vecLib.framework/libLinearAlgebra.dylib + 0x7ff81d757000 - 0x7ff81d7a9ffc PhysicsKit x86_64 <97ca96a5990438179369853909d0874a> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/PhysicsKit.framework/PhysicsKit + 0x7ff81da53000 - 0x7ff81da73fff GenerationalStorage x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/GenerationalStorage.framework/GenerationalStorage + 0x7ff81decf000 - 0x7ff81e09cfcf libvDSP.dylib x86_64 <391f50332ba836899e6472b958c8ef27> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Accelerate.framework/Frameworks/vecLib.framework/libvDSP.dylib + 0x7ff81e60a000 - 0x7ff81e60afff Accelerate x86_64 <1b9fb26005ba3d91ae751491798f73cb> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Accelerate.framework/Accelerate + 0x7ff81e60b000 - 0x7ff81e9c1ffb libLAPACK.dylib x86_64 <83dadf625a57359898e604e31c7514f7> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Accelerate.framework/Frameworks/vecLib.framework/libLAPACK.dylib + 0x7ff81e9c2000 - 0x7ff81e9c7ff3 libQuadrature.dylib x86_64 <336a7b1142283df38692b3d92d755002> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Accelerate.framework/Frameworks/vecLib.framework/libQuadrature.dylib + 0x7ff81e9c8000 - 0x7ff81eb3aff7 libvMisc.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Accelerate.framework/Frameworks/vecLib.framework/libvMisc.dylib + 0x7ff81eb3b000 - 0x7ff81eb3bfff vecLib x86_64 <24063448a6f0321c809039d669666f6f> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Accelerate.framework/Frameworks/vecLib.framework/vecLib + 0x7ff81eb43000 - 0x7ff81ec40ff7 Combine x86_64 <61af2cf0a7173887a1320e316443010a> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Combine.framework/Combine + 0x7ff81ec41000 - 0x7ff81ecadff4 CoreMIDI x86_64 <26f6c3818be23c2796a9bc3ce5f6a10b> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CoreMIDI.framework/CoreMIDI + 0x7ff81ece7000 - 0x7ff81ed39ff0 CryptoKit x86_64 <9d15df27daef32caae62c405d9876d01> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CryptoKit.framework/CryptoKit + 0x7ff81ed6c000 - 0x7ff81eddcff4 MPSCore x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/MetalPerformanceShaders.framework/Frameworks/MPSCore.framework/MPSCore + 0x7ff81eddd000 - 0x7ff81ee71ff2 MPSImage x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/MetalPerformanceShaders.framework/Frameworks/MPSImage.framework/MPSImage + 0x7ff81ee72000 - 0x7ff81eea1ff3 MPSMatrix x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/MetalPerformanceShaders.framework/Frameworks/MPSMatrix.framework/MPSMatrix + 0x7ff81eea2000 - 0x7ff81ef6cffe MPSNDArray x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/MetalPerformanceShaders.framework/Frameworks/MPSNDArray.framework/MPSNDArray + 0x7ff81ef6d000 - 0x7ff81f191ff7 MPSNeuralNetwork x86_64 <180adf5df0a037d0b942b7b120c8cdaa> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/MetalPerformanceShaders.framework/Frameworks/MPSNeuralNetwork.framework/MPSNeuralNetwork + 0x7ff81f192000 - 0x7ff81f1e9ffc MPSRayIntersector x86_64 <9bac029b3380316dad6ff686e7260fcd> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/MetalPerformanceShaders.framework/Frameworks/MPSRayIntersector.framework/MPSRayIntersector + 0x7ff81f1ea000 - 0x7ff81f1eaff8 MetalPerformanceShaders x86_64 <8b352b60f4913b5a9c5feef60a3ad90f> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/MetalPerformanceShaders.framework/MetalPerformanceShaders + 0x7ff81f1eb000 - 0x7ff81f202fff MetricKit x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/MetricKit.framework/MetricKit + 0x7ff81f203000 - 0x7ff81f203ff7 MobileCoreServices x86_64 <9a7c15228f3d3a33a94ac09c4349d9aa> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/MobileCoreServices.framework/MobileCoreServices + 0x7ff81f204000 - 0x7ff81f206fff libCVMSPluginSupport.dylib x86_64 <309ed6c0763d36bba0d854903c6d2711> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/OpenGLES.framework/libCVMSPluginSupport.dylib + 0x7ff81f207000 - 0x7ff81f20dff3 libCoreFSCache.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/OpenGLES.framework/libCoreFSCache.dylib + 0x7ff81f20e000 - 0x7ff81f216ff3 libCoreVMClient.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/OpenGLES.framework/libCoreVMClient.dylib + 0x7ff81f46e000 - 0x7ff81f479ff1 AppleIDAuthSupport x86_64 <64163be6ecf33fa4acec2ac3d9f9ba75> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AppleIDAuthSupport.framework/AppleIDAuthSupport + 0x7ff81f47a000 - 0x7ff81f4c4fff AppleJPEG x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AppleJPEG.framework/AppleJPEG + 0x7ff81f4d8000 - 0x7ff81f4faff7 AppleSauce x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AppleSauce.framework/AppleSauce + 0x7ff81f502000 - 0x7ff81f53effa AttributeGraph x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AttributeGraph.framework/AttributeGraph + 0x7ff81f53f000 - 0x7ff81f5a1fff Bom x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/Bom.framework/Bom + 0x7ff82048e000 - 0x7ff820492ffc CryptoKitCBridging x86_64 <9c357fc951da345d9206628702052c05> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CryptoKitCBridging.framework/CryptoKitCBridging + 0x7ff8205bd000 - 0x7ff8205eaff1 DocumentManagerCore x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/DocumentManagerCore.framework/DocumentManagerCore + 0x7ff82060d000 - 0x7ff820fa7ff3 Espresso x86_64 <0daf0b77685733699bf78ca3f8225756> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/Espresso.framework/Espresso + 0x7ff8217cf000 - 0x7ff8217e2ffd FeatureFlagsSupport x86_64 <3bce9acc94ed3d0589bb9ce0e1491ea8> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/FeatureFlagsSupport.framework/FeatureFlagsSupport + 0x7ff821815000 - 0x7ff821854ffb libGSFontCache.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/FontServices.framework/libGSFontCache.dylib + 0x7ff8218b8000 - 0x7ff8218c2ff7 libhvf.dylib x86_64 <0a3a7de3e9733c8492bec2df024c27fd> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/FontServices.framework/libhvf.dylib + 0x7ff8218da000 - 0x7ff8218f2ffc Futhark x86_64 <4a736ed4a6493b7da0d34d859e1721e6> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/Futhark.framework/Futhark + 0x7ff822be5000 - 0x7ff822bf3ffc GraphVisualizer x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/GraphVisualizer.framework/GraphVisualizer + 0x7ff822cf9000 - 0x7ff822d6dfff InertiaCam x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/InertiaCam.framework/InertiaCam + 0x7ff822dbc000 - 0x7ff822dc6ff1 InternationalSupport x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/InternationalSupport.framework/InternationalSupport + 0x7ff822fab000 - 0x7ff822fabff5 Marco x86_64 <17cd817630073e53b75bbf73d388d0f5> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/Marco.framework/Marco + 0x7ff822fbf000 - 0x7ff822fe9ff4 MessageSecurity x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/MessageSecurity.framework/MessageSecurity + 0x7ff823403000 - 0x7ff823447ff8 OTSVG x86_64 <6ee23239dd903c49aadb988ee3811881> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/OTSVG.framework/OTSVG + 0x7ff823448000 - 0x7ff823460ff9 Osprey x86_64 <7ea61fe9aa363ed2a5821a5740cf70ea> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/Osprey.framework/Osprey + 0x7ff82346d000 - 0x7ff82346dff2 PhoneNumbers x86_64 <24162fdd57e03d6eb59338fceea00af6> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/PhoneNumbers.framework/PhoneNumbers + 0x7ff823778000 - 0x7ff823834ff4 RenderBox x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/RenderBox.framework/RenderBox + 0x7ff8238e4000 - 0x7ff8238f8ffe SetupAssistantSupport x86_64 <12dba9be1f8836f1851eb7497bb6c967> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SetupAssistantSupport.framework/SetupAssistantSupport + 0x7ff8238fb000 - 0x7ff8238fbffa SignpostMetrics x86_64 <02cf820b11f83c6989a5642aeef0632b> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SignpostMetrics.framework/SignpostMetrics + 0x7ff8238fd000 - 0x7ff823f1dff1 SiriInstrumentation x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SiriInstrumentation.framework/SiriInstrumentation + 0x7ff823f32000 - 0x7ff8240bdffa TextRecognition x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/TextRecognition.framework/TextRecognition + 0x7ff8240be000 - 0x7ff824191ff6 TextureIO x86_64 <3a5e9c2dce60321c99684d807297d0fd> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/TextureIO.framework/TextureIO + 0x7ff824192000 - 0x7ff824203ff9 Trial x86_64 <17b813b94c9e33719c3a2f3f73943e54> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/Trial.framework/Trial + 0x7ff824204000 - 0x7ff824279ffd TrialProto x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/TrialProto.framework/TrialProto + 0x7ff8245a7000 - 0x7ff8245b3ff2 perfdata x86_64 <16faf57efb3137619add22bc8867460f> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/perfdata.framework/perfdata + 0x7ff8248ca000 - 0x7ff824933fe7 libParallelCompression.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libParallelCompression.dylib + 0x7ff824934000 - 0x7ff824935ff7 libSystem.B.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libSystem.B.dylib + 0x7ff824936000 - 0x7ff824937fff libThaiTokenizer.dylib x86_64 <4a07589aa4ff3bdfbe9f622b52645f9a> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libThaiTokenizer.dylib + 0x7ff824938000 - 0x7ff82494ffff libapple_nghttp2.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libapple_nghttp2.dylib + 0x7ff824950000 - 0x7ff82495ffff libbsm.0.dylib x86_64 <4f118f53560d3dc5a5a16e47231df7a3> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libbsm.0.dylib + 0x7ff824960000 - 0x7ff82496ffff libbz2.1.0.dylib x86_64 <8c85521550853b7d9eb9d433f3613dd5> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libbz2.1.0.dylib + 0x7ff824970000 - 0x7ff824970fff libcharset.1.dylib x86_64 <7092406a3dc73117978c0c79bad9039f> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libcharset.1.dylib + 0x7ff824971000 - 0x7ff824981ff3 libcmph.dylib x86_64 <90eb7b4cc4f935a2b56485276b027d4b> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libcmph.dylib + 0x7ff824982000 - 0x7ff824a58fc7 libcompression.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libcompression.dylib + 0x7ff824a59000 - 0x7ff824a5affb libcoretls_cfhelpers.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libcoretls_cfhelpers.dylib + 0x7ff824a5b000 - 0x7ff824a60ff3 libcupolicy.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libcupolicy.dylib + 0x7ff824a61000 - 0x7ff824a7eff3 libedit.3.dylib x86_64 <4fdee1f414463ed391a9b375c24256dd> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libedit.3.dylib + 0x7ff824a9a000 - 0x7ff824a9eff3 libheimdal-asn1.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libheimdal-asn1.dylib + 0x7ff824a9f000 - 0x7ff824b8fff3 libiconv.2.dylib x86_64 <92232a0779493f2a8457d34a2efbee7d> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libiconv.2.dylib + 0x7ff824b90000 - 0x7ff824ba8ff7 liblzma.5.dylib x86_64 <0520737db3793900b5448b9021986897> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/liblzma.5.dylib + 0x7ff824e09000 - 0x7ff824e38ff3 libncurses.5.4.dylib x86_64 <72c03d8ca9773d608825fedd08e58a68> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libncurses.5.4.dylib + 0x7ff824e39000 - 0x7ff824e5eff3 libnetworkextension.dylib x86_64 <56b61b563f1c301b83c944caa8d5c5e1> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libnetworkextension.dylib + 0x7ff824e5f000 - 0x7ff824e77fff libresolv.9.dylib x86_64 <8e9fbd585c00370fad337bebdc899cdd> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libresolv.9.dylib + 0x7ff824ea9000 - 0x7ff824eacff3 libutil.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libutil.dylib + 0x7ff824ed9000 - 0x7ff824eebffb libz.1.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libz.1.dylib + 0x7ff824efb000 - 0x7ff824efdffc liblog_network.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/log/liblog_network.dylib + 0x7ff824f01000 - 0x7ff824f1dff6 libswiftAVFoundation.dylib x86_64 <97474cc506f93e34a339fca88e4692da> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftAVFoundation.dylib + 0x7ff824f1e000 - 0x7ff824fddfff libswiftAccelerate.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftAccelerate.dylib + 0x7ff824fe3000 - 0x7ff824fe8ff7 libswiftCompression.dylib x86_64 <2344f27d5c7b3b2ca9fe03ab273ea44f> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftCompression.dylib + 0x7ff824fea000 - 0x7ff824ff1fff libswiftCoreAudio.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftCoreAudio.dylib + 0x7ff824ff3000 - 0x7ff824ff9fff libswiftCoreFoundation.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftCoreFoundation.dylib + 0x7ff824ffa000 - 0x7ff825000fff libswiftCoreGraphics.dylib x86_64 <8447d75747cd3456a8de93149091d980> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftCoreGraphics.dylib + 0x7ff825001000 - 0x7ff825001fff libswiftCoreImage.dylib x86_64 <496db2df16213aa982ffa628952cb068> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftCoreImage.dylib + 0x7ff825002000 - 0x7ff825004ffc libswiftCoreLocation.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftCoreLocation.dylib + 0x7ff825005000 - 0x7ff82500efff libswiftCoreMIDI.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftCoreMIDI.dylib + 0x7ff82500f000 - 0x7ff825042ffa libswiftCoreMedia.dylib x86_64 <3bab87571d313036b093a567d01e3c31> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftCoreMedia.dylib + 0x7ff825043000 - 0x7ff825049fff libswiftDarwin.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftDarwin.dylib + 0x7ff82504a000 - 0x7ff82505eff7 libswiftDispatch.dylib x86_64 <000e21da5b95380c9126cd6e3bc997ce> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftDispatch.dylib + 0x7ff82505f000 - 0x7ff82505fff5 libswiftFoundation.dylib x86_64 <5c301a39f7a3380e85bc4db57ebdc4df> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftFoundation.dylib + 0x7ff82506a000 - 0x7ff82506effe libswiftMetal.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftMetal.dylib + 0x7ff82506f000 - 0x7ff82506ffff libswiftMetricKit.dylib x86_64 <8eb57bc99b3a3902aa4b3b847ff9bc0d> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftMetricKit.dylib + 0x7ff82507b000 - 0x7ff8250f5ffe libswiftNetwork.dylib x86_64 <7cdda0d4a1e33af09bcc25f5e4d4db3b> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftNetwork.dylib + 0x7ff8250f6000 - 0x7ff8250f9ff5 libswiftObjectiveC.dylib x86_64 <59363047b9a938ed93b9678eadd02bce> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftObjectiveC.dylib + 0x7ff8250fe000 - 0x7ff8250ffff7 libswiftQuartzCore.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftQuartzCore.dylib + 0x7ff825124000 - 0x7ff825125fff libswiftVision.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftVision.dylib + 0x7ff825126000 - 0x7ff82513dffa libswiftos.dylib x86_64 <1f512e693ede30fb8816c8e5d705eb81> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftos.dylib + 0x7ff82513e000 - 0x7ff82515dff3 libswiftsimd.dylib x86_64 <001908c6db0a331d9f6c0ed2bdb72c61> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftsimd.dylib + 0x7ff82515e000 - 0x7ff825163ff7 libcache.dylib x86_64 <044254cfb3b938f2be46faf5ec23edbf> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libcache.dylib + 0x7ff825164000 - 0x7ff82516fff7 libcommonCrypto.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libcommonCrypto.dylib + 0x7ff825170000 - 0x7ff825177ff7 libcompiler_rt.dylib x86_64 <83fa81b154a4312e933727a0a2b82f80> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libcompiler_rt.dylib + 0x7ff825299000 - 0x7ff82529dfff libmacho.dylib x86_64 <0923cddcbd163c6b9aeda268638ba02b> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libmacho.dylib + 0x7ff82529e000 - 0x7ff82529ffff libremovefile.dylib x86_64 <3616c5afc3d6385d8fc33409a9734a59> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libremovefile.dylib + 0x7ff8252a0000 - 0x7ff8252a2ff3 libsystem_featureflags.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_featureflags.dylib + 0x7ff8252a3000 - 0x7ff8252f6fdf libsystem_m.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_m.dylib + 0x7ff8252f7000 - 0x7ff8252fefff libunwind.dylib x86_64 <8c4c085c15d73a1fbca91d6555c36f71> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libunwind.dylib + 0x7ff825318000 - 0x7ff825387ffc NanoRegistry x86_64 <1b7764b149123e2ba7e796f82614f794> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/NanoRegistry.framework/NanoRegistry + 0x7ff825388000 - 0x7ff825398ff7 NanoPreferencesSync x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/NanoPreferencesSync.framework/NanoPreferencesSync + 0x7ff825464000 - 0x7ff8255a0ffd AVFAudio x86_64 <2f518e22e4b93050ba7b040cb6175aed> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/AVFAudio.framework/AVFAudio + 0x7ff8255a9000 - 0x7ff8255bfffa Accessibility x86_64 <7cfd819882413e07b3bade164d7630cb> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Accessibility.framework/Accessibility + 0x7ff82585c000 - 0x7ff825927ffb ColorSync x86_64 <51659aae56983da7b7cbbd5c00bf99a3> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/ColorSync.framework/ColorSync + 0x7ff825928000 - 0x7ff825952ff4 CoreTransferable x86_64 <029ecf01fea936e69ce2c4e232fa8c04> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CoreTransferable.framework/CoreTransferable + 0x7ff825953000 - 0x7ff8259a0ff4 CryptoTokenKit x86_64 <931feae3b1f8310687d94b21278e62bf> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/CryptoTokenKit.framework/CryptoTokenKit + 0x7ff8259a1000 - 0x7ff8259a3ff3 DataDetection x86_64 <12b0b70ced083bc8b013e9f9a15cd0af> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/DataDetection.framework/DataDetection + 0x7ff825a61000 - 0x7ff825ac5ffc ExtensionFoundation x86_64 <4289064a4f583d0da5565f96f3bfcb5e> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/ExtensionFoundation.framework/ExtensionFoundation + 0x7ff8267cb000 - 0x7ff8267ddff6 MPSFunctions x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/MetalPerformanceShaders.framework/Frameworks/MPSFunctions.framework/MPSFunctions + 0x7ff827ad7000 - 0x7ff827ae5ff0 OSLog x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/OSLog.framework/OSLog + 0x7ff827b0f000 - 0x7ff8288d0f0b libLLVMContainer.dylib x86_64 <6ff25dbf106639ef977a57fa451c81fe> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/OpenGLES.framework/libLLVMContainer.dylib + 0x7ff828ca1000 - 0x7ff828cb4ff3 UniformTypeIdentifiers x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/UniformTypeIdentifiers.framework/UniformTypeIdentifiers + 0x7ff828cb5000 - 0x7ff8293cdffc libfaceCore.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/Frameworks/Vision.framework/libfaceCore.dylib + 0x7ff829590000 - 0x7ff82959fff6 AAAFoundation x86_64 <669ece6f4edb334b866c87e6234b3935> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AAAFoundation.framework/AAAFoundation + 0x7ff82965f000 - 0x7ff82975eff0 AVFCapture x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AVFCapture.framework/AVFCapture + 0x7ff82975f000 - 0x7ff829934ff2 AVFCore x86_64 <3ff6c780536f339eadec569b17213ed4> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AVFCore.framework/AVFCore + 0x7ff829bcd000 - 0x7ff829c0dffc AudioSession x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AudioSession.framework/AudioSession + 0x7ff829c0e000 - 0x7ff829c3dffa libSessionUtility.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/AudioSession.framework/libSessionUtility.dylib + 0x7ff829c84000 - 0x7ff829cb8ff0 BacklightServices x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/BacklightServices.framework/BacklightServices + 0x7ff82a0a9000 - 0x7ff82a102ff0 CMCapture x86_64 <47b1c467fb35381d8ca83b1271cb471d> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CMCapture.framework/CMCapture + 0x7ff82a103000 - 0x7ff82a10efff CMCaptureCore x86_64 <1e22df7cc4493dc6a083e16ad362c075> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CMCaptureCore.framework/CMCaptureCore + 0x7ff82a12b000 - 0x7ff82a1ceff9 CMPhoto x86_64 <2c6083c01dae3e3283f0522d8aacce5f> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CMPhoto.framework/CMPhoto + 0x7ff82a1eb000 - 0x7ff82a2d3ff3 CVNLP x86_64 <3a7e5da416e1373a83d46bcde973d84b> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CVNLP.framework/CVNLP + 0x7ff82a544000 - 0x7ff82a681fff CloudKitDistributedSync x86_64 <31fc964eeb793701897f90353b373bec> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CloudKitDistributedSync.framework/CloudKitDistributedSync + 0x7ff82a6f5000 - 0x7ff82a707ff5 CollectionViewCore x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CollectionViewCore.framework/CollectionViewCore + 0x7ff82a8a4000 - 0x7ff82a8eaff4 CoreAutoLayout x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CoreAutoLayout.framework/CoreAutoLayout + 0x7ff82a948000 - 0x7ff82a96dffb CoreGlyphs x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CoreGlyphs.framework/CoreGlyphs + 0x7ff82ac20000 - 0x7ff82ac3bfff CoreUtilsExtras x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/CoreUtilsExtras.framework/CoreUtilsExtras + 0x7ff82ac64000 - 0x7ff82ac6cff3 DEPClientLibrary x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/DEPClientLibrary.framework/DEPClientLibrary + 0x7ff82ac8b000 - 0x7ff82acc0ff6 DMCUtilities x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/DMCUtilities.framework/DMCUtilities + 0x7ff82b6a0000 - 0x7ff82b6a0ffb FeatureFlags x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/FeatureFlags.framework/FeatureFlags + 0x7ff82b6a9000 - 0x7ff82b6bcff0 FeedbackLogger x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/FeedbackLogger.framework/FeedbackLogger + 0x7ff82ba01000 - 0x7ff82ba02fff libGPUCompilerUtils.dylib x86_64 <29a1685b511e30de866b03c33a4e6da4> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/GPUCompiler.framework/Libraries/libGPUCompilerUtils.dylib + 0x7ff82bc65000 - 0x7ff82bc68ff3 GeoServicesCore x86_64 <8829c8537ecf3b029a125f1124e162c1> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/GeoServicesCore.framework/GeoServicesCore + 0x7ff82c1c3000 - 0x7ff82c1ddff1 IconFoundation x86_64 <98349e741088300d8857794f497a60c9> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/IconFoundation.framework/IconFoundation + 0x7ff82c1ec000 - 0x7ff82c26aff0 InstalledContentLibrary x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/InstalledContentLibrary.framework/InstalledContentLibrary + 0x7ff82cd77000 - 0x7ff82cd95fff MDMClientLibrary x86_64 <94154af87f2c32fabbfd4d198c138b51> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/MDMClientLibrary.framework/MDMClientLibrary + 0x7ff82cd96000 - 0x7ff82d072ff3 MIL x86_64 <73206bd1bb3d341288042229a178aeb8> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/MIL.framework/MIL + 0x7ff82d073000 - 0x7ff82d18dff1 MLAssetIO x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/MLAssetIO.framework/MLAssetIO + 0x7ff82e2e9000 - 0x7ff82e30bff0 OctagonTrust x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/OctagonTrust.framework/OctagonTrust + 0x7ff82edf7000 - 0x7ff82ee07ff2 PointerUIServices x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/PointerUIServices.framework/PointerUIServices + 0x7ff830250000 - 0x7ff830256ff3 SimFramebufferClient x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SimFramebufferClient.framework/SimFramebufferClient + 0x7ff830257000 - 0x7ff83025aff8 SimulatorClient x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SimulatorClient.framework/SimulatorClient + 0x7ff830262000 - 0x7ff8302aeff5 SiriAnalytics x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SiriAnalytics.framework/SiriAnalytics + 0x7ff830fd9000 - 0x7ff830fe1ffc SiriPowerInstrumentation x86_64 <3051bb30366c3fdaa8dddbe2a5701d4a> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SiriPowerInstrumentation.framework/SiriPowerInstrumentation + 0x7ff8312e2000 - 0x7ff83142effb SiriTTSService x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SiriTTSService.framework/SiriTTSService + 0x7ff8319d6000 - 0x7ff8319d6ff7 SoftLinking x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SoftLinking.framework/SoftLinking + 0x7ff8319d7000 - 0x7ff8319eaff0 SoftwareUpdateCoreConnect x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SoftwareUpdateCoreConnect.framework/SoftwareUpdateCoreConnect + 0x7ff8319eb000 - 0x7ff831a3fffc SoftwareUpdateCoreSupport x86_64 <9a9e52ca7bed3076a772bac983d054f8> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SoftwareUpdateCoreSupport.framework/SoftwareUpdateCoreSupport + 0x7ff831d16000 - 0x7ff831d24ffb SystemWake x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/System/Library/PrivateFrameworks/SystemWake.framework/SystemWake + 0x7ff8344c0000 - 0x7ff8344c1ffd libBASupport.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libBASupport.dylib + 0x7ff8344c2000 - 0x7ff8344caffc libCoreEntitlements.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libCoreEntitlements.dylib + 0x7ff8344da000 - 0x7ff834504ffb libcmark-gfm.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libcmark-gfm.dylib + 0x7ff834505000 - 0x7ff83451fffb libexpat.1.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libexpat.1.dylib + 0x7ff834520000 - 0x7ff834557ffb libllvm-flatbuffers.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libllvm-flatbuffers.dylib + 0x7ff8346ec000 - 0x7ff834802ffc libquic.dylib x86_64 <3044a53b93023867ae0074281227035b> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/libquic.dylib + 0x7ff83485e000 - 0x7ff834882ffc libswiftCoreML.dylib x86_64 <189dc875f5e433298117ca8c0546e21d> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftCoreML.dylib + 0x7ff83488e000 - 0x7ff83488fff6 libswiftCryptoTokenKit.dylib x86_64 <50209966d34b398f8fba46313fc04866> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftCryptoTokenKit.dylib + 0x7ff834890000 - 0x7ff834890fff libswiftDataDetection.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftDataDetection.dylib + 0x7ff8348ef000 - 0x7ff8348feff9 libswiftExtensionFoundation.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftExtensionFoundation.dylib + 0x7ff8348ff000 - 0x7ff8348ffff4 libswiftFileProvider.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftFileProvider.dylib + 0x7ff834973000 - 0x7ff834974ff3 libswiftOSLog.dylib x86_64 <1251168b7a52396687eaf692d041fde8> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftOSLog.dylib + 0x7ff8349fc000 - 0x7ff834a08ffd libswiftUniformTypeIdentifiers.dylib x86_64 <7ea619c59b77303c9829259893f5dde7> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftUniformTypeIdentifiers.dylib + 0x7ff834a09000 - 0x7ff834a11ff3 libswiftXPC.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswiftXPC.dylib + 0x7ff834a12000 - 0x7ff834a57ff7 libswift_Concurrency.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswift_Concurrency.dylib + 0x7ff834a91000 - 0x7ff834b48fff libswift_RegexParser.dylib x86_64 <4445f7740b883b5d98a47ec1b8b00026> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswift_RegexParser.dylib + 0x7ff834b49000 - 0x7ff834bd8fff libswift_StringProcessing.dylib x86_64 <69d49859fddc3dd5bc04496c938e8197> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/swift/libswift_StringProcessing.dylib + 0x7ff834bd9000 - 0x7ff834bddff7 libsystem_collections.dylib x86_64 <6aeb2d672c273e56ac585eec93acfea3> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_collections.dylib + 0x7ff834bde000 - 0x7ff834c15fff libsystem_kernel.dylib x86_64 <0ea0d8acc27b3a71a59bec3a6f116acf> /usr/lib/system/libsystem_kernel.dylib + 0x7ff834c16000 - 0x7ff834c2cffb libsystem_networkextension.dylib x86_64 /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_networkextension.dylib + 0x7ff834c2d000 - 0x7ff834c36fef libsystem_platform.dylib x86_64 /usr/lib/system/libsystem_platform.dylib + 0x7ff834c37000 - 0x7ff834c42ff7 libsystem_pthread.dylib x86_64 /usr/lib/system/libsystem_pthread.dylib + 0x7ff834c43000 - 0x7ff834c44ff3 libsystem_sim_kernel.dylib x86_64 <799d6c868a153f12a7a2cbe5a6c3d602> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_sim_kernel.dylib + 0x7ff834c45000 - 0x7ff834c45fff libsystem_sim_kernel_host.dylib x86_64 <49b843979b363232b3dc197e62157eb2> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_sim_kernel_host.dylib + 0x7ff834c46000 - 0x7ff834c49fef libsystem_sim_platform.dylib x86_64 <9e91310595f03782abf22bbf8a9d99f8> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_sim_platform.dylib + 0x7ff834c4a000 - 0x7ff834c4afff libsystem_sim_platform_host.dylib x86_64 <40d93d0dc7cf3570b6451cd09bd9f3f5> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_sim_platform_host.dylib + 0x7ff834c4b000 - 0x7ff834c4bff3 libsystem_sim_pthread.dylib x86_64 <285cfc09fc20394fa7c8951aac2657a6> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_sim_pthread.dylib + 0x7ff834c4c000 - 0x7ff834c4cfff libsystem_sim_pthread_host.dylib x86_64 <03a60a04835c3fb7a38d34e08fd27da5> /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime/Contents/Resources/RuntimeRoot/usr/lib/system/libsystem_sim_pthread_host.dylib +``` \ No newline at end of file diff --git a/specs/agents/mobile/metrics.md b/specs/agents/mobile/metrics.md new file mode 100644 index 00000000..0e5ff1da --- /dev/null +++ b/specs/agents/mobile/metrics.md @@ -0,0 +1,43 @@ +## Mobile Metrics + +### CPU metrics +| Name | Type | Units | Description | +|--------------------|------------------|------------|---------------------------------| +| `system.cpu.usage` | Gauge | percentage | A percentage value of cpu usage | + +### Memory Metrics +| Name | Type | Units | Description | +|------------------------|------------------|-------|-----------------------------------------| +| `system.memory.usage` | Gauge | bytes | The application's memory usage in bytes | + + +### Application Metrics +#### load times +| Name | Type | Units | Description | +|--------------------------------------|--------------------------------|---------|-----------------------------------------------------------------------| +| `application.launch.time` | histogram(iOS), gauge(Android) | milliseconds | The amount of time spent launching the app | + +| Labels | Values | Description | +|--------|-------------------------------------------------|-----------------------------------------------------| +| `type` | `first draw`, `first draw (optimized)`, `resume`| The type of application launch that is being timed. | + +#### responsiveness +| Name | Type | Units | Description | +|----------------------------------------|-----------|---------|-------------------------------------------------------------| +| `application.responsiveness.hangtime` | histogram | millisseconds | The amount of time the applications has spent unresponsive. | + +### Application exit +Traces application exit counts in both healthy and unhealthy (crashes) states + +| Name | Type | Units | Description | +|---------------------|-------|-------|-------------------------------| +| `application.exits` | count | unit | A count of application exits. | + + +| Labels | Values | Description | +|------------|------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------| +| `appState` | `background`, `foreground` | This denotes whether the application exited in the background or foreground | +| `type` | `memoryResourceLimit`, `AppWatchDog`, `BadAccess`, `Abnormal`, `IllegalInstruction`, `Normal` | The cause of the application exit. All but normal could be considered a crash. | + + + diff --git a/specs/agents/mobile/session.md b/specs/agents/mobile/session.md new file mode 100644 index 00000000..e69dba86 --- /dev/null +++ b/specs/agents/mobile/session.md @@ -0,0 +1,23 @@ +# Session + +Status: [Experimental](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/document-status.md) + +### Overview +A `session` is a collection of `logs`, `events`, `transactions` and `spans` (`LETS`) associated with a specific device within a specific period of time. +A `session` is represented by a unique identify that is attached to `LETS` as an attribute. + +The primary purpose of `sessions` are to provide insight into the series of user actions or events that lead up to a critical error or crash. Sessions also provide a means to quantify application usage. + +This document depends on the Open Telemetry semantic convention for [session](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/general/session.md). Due to the dependency on Open Telemetry's events API this document's contents are subject to change and considered [experimental](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/document-status.md). + +### How a session operates +- All `LETS` will have a `session` identifier attached as an attribute using the name `session.id`. +- After a period of timeout, the `session` identifier will be refreshed. +- The timeout period will be restarted when any `LETS` is recorded. + + +#### The session timeout period can be customized. +Default session timeout should be 30 minutes. This should only be done when the agent is configured, and shouldn't be updated in the middle of a session. + +#### Max session length +Sessions will be limited to a maximum length of four (4) hours. This limitation will be implemented in the mobile agent. diff --git a/specs/agents/otel-distribution.md b/specs/agents/otel-distribution.md new file mode 100644 index 00000000..352bf7da --- /dev/null +++ b/specs/agents/otel-distribution.md @@ -0,0 +1,127 @@ + +## Terminology + +**Vanilla OpenTelemetry Distribution**: this is the "upstream" OpenTelemetry distribution that is maintained by the OpenTelemetry community. +Implementation differs per platform, but it usually consists of an API/SDK and can also provide automatic instrumentation. + +**Elastic OpenTelemetry Distribution**: this is an OpenTelemetry distribution provided by Elastic that is derived from the +_Vanilla OpenTelemetry Distribution_. + +## General guidelines + +These statements are guiding principles of the Elastic OpenTelemetry distributions, they should be considered more as advice than strict rules. + +Elastic OpenTelemetry distribution SHOULD ideally: +- behave as drop-in replacements of their upstream counterparts +- provide a simple setup and favor onboarding experience (aka "things should work by default"). +- avoid capturing potentially confusing data (see [system metrics](#system-metrics) example below). + +## Configuration + +Elastic OpenTelemetry distributions MAY override the default configuration. +When doing so, user-configuration should remain consistent with vanilla distribution: +- explicit user configuration SHOULD remain effective +- overriden default configuration MUST have the ability to be restored to upstream default + +Elastic specific configuration items MUST be prefixed with `ELASTIC_OTEL_`. +For example, the [universal profiling integration](#universal-profiling-integration) can be enabled with `ELASTIC_OTEL_UNIVERSAL_PROFILING_INTEGRATION_ENABLED`. + +Elastic and platform specific configuration items must be prefixed with `ELASTIC_OTEL_${platform}_` to be consistent with +the upstream `OTEL_${platform}_` prefix. + +When introducing new features, the decision between starting with platform-specific or general namespace is made on a feature by feature case: +- feature can be aligned cross-platform even if implemented only in only one: use `ELASTIC_OTEL_` prefix, for example [System metrics](#system-metrics). +- feature that we know will be platform-specific: use `ELASTIC_OTEL_${platform}_` prefix. + +For simplicity the configuration in this specification will use the "environment variable" syntax, some platforms like Java +might also support other ways to configure. + +## Identification + +### User Agent headers + +Per the [OpenTelemetry Specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md#user-agent), OpenTelemetry SDKs are expected to send a `User-Agent` header when exporting data to a backend. At a minimum, this header SHOULD identify the exporter, the language of its implementation, and the version of the exporter. + +Elastic distributions SHOULD configure a customized `User-Agent` header when possible[^1]. +This allows data exported from a vanilla SDK and an Elastic distribution to be easily distinguished. + +[^1]: Some OpenTelemetry SDKs (e.g. .NET) do not provide a mechanism to modify the `User-Agent` header. In this case, we accept their default. + +To conform with [RFC7231](https://datatracker.ietf.org/doc/html/rfc7231#section-5.5.3), the existing SDK `User-Agent` should be preceded by a product identifier and version for the Elastic distribution. + +``` +/ +``` + +For example, in the .NET distribution, the `User-Agent` header would be configured as follows: + +``` +elastic-otel-dotnet/1.0.0 OTel-OTLP-Exporter-Dotnet/1.6.0 +``` + +### Telemetry resource attributes + +Per the [semantic conventions](https://opentelemetry.io/docs/specs/semconv/resource/#telemetry-sdk), SDKs are expected to include the following resource attributes on captured signals. These are used to identify the SDK where the data was captured and should not be modified. + +- `telemetry.sdk.name` +- `telemetry.sdk.version` +- `telemetry.sdk.language` + +In the above attributes, the name and version should be the OTel SDK name and version. +The language should be the primary language that the SDK is intended for. +It is expected that the OpenTelemetry SDK sets these values. +Our distros should set them, only if the SDK code does not do so automatically. + +Intake currently reads these attributes and uses them to populate the `agent.Name` and `agent.Version` fields. + +The semantic conventions also [define two experimental attributes](https://opentelemetry.io/docs/specs/semconv/resource/#telemetry-sdk-experimental) to identify the distribution: + +- `telemetry.distro.name`: must be set to `elastic` +- `telemetry.distro.version`: must reflect the distribution version + +Distributions SHOULD set these attributes with appropriate values. + +## Features + +### Inferred spans + +Supported platforms: [Java](https://github.com/elastic/elastic-otel-java/tree/main/inferred-spans) + +Configuration to enable: `ELASTIC_OTEL_INFERRED_SPANS_ENABLED` + +Note: While the implementation is Java-only for now, it should probably have been using `ELASTIC_OTEL_JAVA_INFERRED_SPANS_ENABLED` +instead, but we plan to fix this inconsistency once it has been contributed upstream. + +### System metrics + +Supported platforms: Python + +These metrics are usually captured using the collector running locally but in case where no collector is present, or a centralized +collector is used then the user might opt in to also collect those. + +These metrics are not captured by default in order to prevent duplicated metrics when they are also captured by a collector. + +Configuration to enable: `ELASTIC_OTEL_SYSTEM_METRICS_ENABLED` + +### Cloud resource attributes + +Supported platforms: Java + +The cloud resource attributes ([semconv](https://opentelemetry.io/docs/specs/semconv/resource/cloud/)) is a subset of +the [resource attributes](https://opentelemetry.io/docs/specs/semconv/resource/) providing equivalent attributes to the +[cloud provider metadata](metadata.md#cloud-provider-metadata). +Those attributes are usually provided through a metadata HTTP(s) endpoint accessible from the application. + +Elastic OpenTelemetry distributions SHOULD capture those by default for a better onboarding experience. +Users MUST be able to disable this default to minimize application startup overhead or if those attributes are provided through the collector. + +Elastic distribution MUST allow to opt out of this behavior through explicit configuration. +Implementation is currently platform specific: +- Java: `OTEL_RESOURCE_PROVIDERS_${provider}_ENABLED=false` +- NodeJS: `OTEL_NODE_RESOURCE_DETECTORS` ([doc](https://github.com/open-telemetry/opentelemetry-js-contrib/tree/main/metapackages/auto-instrumentations-node/#usage-auto-instrumentation)) + +### Universal profiling integration + +Supported platforms: [Java](https://github.com/elastic/elastic-otel-java/tree/main/universal-profiling-integration) + +For the configuration options see [this section](universal-profiling-integration.md#configuration-options). diff --git a/specs/agents/process-new-fields.md b/specs/agents/process-new-fields.md new file mode 100644 index 00000000..97b1ffaf --- /dev/null +++ b/specs/agents/process-new-fields.md @@ -0,0 +1,91 @@ +# Process for adding new fields + +If an agent dev wants to show new data they write a proposal for how it should be stored in the context. +They should not add it to tags or custom. +We can’t have agents just add data as they see fit because then it won’t be aligned, +it will move around if they change their mind etc., +that would break peoples assumptions about where it is and if they add to tags, +it would create new fields in the index and then stop using it when we standardize + +* The proposal should specify how the data fits into a top level key under `context` in the Intake API and how it fits in the Elasticsearch events that get written by APM Server. +For example `context.elasticsearch.url` in the intake API becomes `elasticsearch.url` in Elasticsearch, `context.elasticsearch.error_reason` becomes `elasticsearch.error_reason` etc. +* The proposal needs to specify which fields should be indexed. +An APM Server person might need toΒ assist here to determine the right data type for the indexed fields. +* The proposal should include the suggested [JSON Schema](https://github.com/elastic/apm-server/tree/main/docs/spec/v2) changes for all new fields. +This forces alignment on the exact field names, JSON data type, length restrictions etc. +* Make sure to check if [ECS](https://github.com/elastic/ecs) has defined appropriate fields for what you're proposing. +* Agents should agree to the changes in a voting format (checkboxes), +once they agree an issue should be created on the agent, +apm-server and/or kibana repos to track the implementation. +Once we have issues for all the implementations, the original one can be closed. +* As soon as the JSON Schema changes have been merged into APM Server, +agents can implement and test their implementation against the new schema. +It is typically only a matter of a few hours to implement new fields in APM Server once the details are agreed upon. +* Agent devs can release new versions that send the new fields as soon the changes are merged into APM Server. +APM Server will not reject arbitrary fields in `context`, but fields that are not defined in APM Server are not stored, indexed or validated. +When users upgrade their stack, the new fields will start to appear. +* The UI will show every field under the existing top-level fields. E.g. everything under `request` shows up in the UI automatically. If we add a new top level fields, the UI also needs to get updated. +* When we add data to the span context or transaction context, +this data should also be allowed in the error context and arbitrary data in the error context should be shown, just like for spans and transactions. +That way, when an error happens, we can supply the context in the error context directly. +We have previously decided that we need an error context and that it's not enough to just link errors to their parent span. +* Errors that are captured in instrumentations should include/copy all the contextual data that would go on that span into the error context + + +Example: + +1. We have built an Elasticsearch instrumentation that gets some useful context: `elasticsearch.url`, `elasticsearch.response_code`, `elasticsearch.error_reason`. +2. Agent dev opens a proposal that looks like this: + +**Proposal:** + +Add optional fields to +- [x] _span context_ +- [ ] _transaction context_ + +as always, this should also be added to the _error context_. + +| Intake API field | Elasticsearch field | Elasticsearch Type | +| -----------------|-------------------------|---------------------| +| `context.elasticsearch.url` | `elasticsearch.url` | not indexed | +| `context.elasticsearch.response_code` | `elasticsearch.response_code` | indexed as keyword | +| `context.elasticsearch.error_reason` | `elasticsearch.error_reason` | not indexed | +| `context.elasticsearch.cluster_name` | `elasticsearch.cluster_name` | not indexed | + + +JSON Schema: +```json +{ + "url": { + "type": ["string"] + }, + "response_code": { + "type": ["string"], + "maxLength": 1024 + }, + "error_reason": { + "type": ["string"], + "maxLength": 10000 + }, + "cluster_name": { + "type": ["string"], + } +} +``` +Not all agents will send `context.elasticsearch.cluster_name`. This is _fine_. We should still align on the ones we can. + +Note: As this is a new top level field, the UI needs an update. + +Agents OK with this change: + +- [ ] @elastic/apm-ui (if this is a new top level field) +- [ ] RUM +- [ ] Node.js +- [ ] Java +- [ ] ... + +1. When agent devs and APM Server agree, APM Server implements the changes necessary +1. When merged into `main`, agent devs can implement the fields immediately. +The agent tests against APM Server `main` now tests the integration with the new fields in the JSON Schema. +1. Agents can release when their test are green. Next APM Server release will include the changes, +which might include indexing some new fields. diff --git a/specs/agents/sanitization.md b/specs/agents/sanitization.md new file mode 100644 index 00000000..0dffcb15 --- /dev/null +++ b/specs/agents/sanitization.md @@ -0,0 +1,56 @@ +The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", +"SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to +be interpreted as described in +[RFC 2119](https://www.ietf.org/rfc/rfc2119.txt). + +## Data sanitization + +### `sanitize_field_names` configuration + +Sometimes it is necessary to sanitize, i.e., remove, +sensitive data sent to Elastic APM. + +This config accepts a list of wildcard patterns of field names which control +how an agent will sanitize data. + +| | | +|----------------|---| +| Type | `List<`[`WildcardMatcher`](../../tests/agents/json-specs/wildcard_matcher_tests.json)`>` | +| Default | `password, passwd, pwd, secret, *key, *token*, *session*, *credit*, *card*, *auth*, set-cookie, *principal*` | +| Dynamic | `true` | +| Central config | `true` | + +#### Configuration + +Agents MUST provide a minimum default configuration of + + [ 'password', 'passwd', 'pwd', 'secret', '*key', '*token*', '*session*', + '*credit*','*card*', '*auth*', 'set-cookie', '*principal*' ] + +for the `sanitize_field_names` configuration value. Agent's MAY include the +following extra fields in their default configuration to avoid breaking changes + + ['pw','pass','connect.sid'] + +## Sanitizing Values + +If a payload field's name (a header key, a form key) matches a configured +wildcard, that field's _value_ MUST be redacted and the key itself +MUST still be reported in the agent payload. Agents MAY choose the string +they use to replace the value so long as it's consistent and does not reveal +the value it has replaced. The replacement string SHOULD be `[REDACTED]`. + +Fields that MUST be sanitized are: +- HTTP Request and Response headers (except [HTTP/2 pseudo-headers](https://datatracker.ietf.org/doc/html/rfc7540#section-8.1.2.3) which SHOULD NOT be redacted), +- form fields in an `application/x-www-form-urlencoded` request body, and +- HTTP Request cookies. + +Additionally, if cookie headers are parsed into name/value pairs and reported +to APM Server via the agent (for example, `transaction.context.request.cookies`), the +values of these pairs MUST be sanitized and the cookie header removed or redacted. + + +The query string and other captured request bodies (such as `application/json`) +SHOULD NOT be sanitized. + +Agents SHOULD NOT sanitize fields based on the _value_ of a particular field. diff --git a/specs/agents/span-links.md b/specs/agents/span-links.md new file mode 100644 index 00000000..0690c8a2 --- /dev/null +++ b/specs/agents/span-links.md @@ -0,0 +1,39 @@ +## Span Links + +A Span or Transaction MAY link to zero or more other Spans/Transactions that are causally related. + +Example use-cases for Span Links: + +1. When a single transaction represents the batch processing of several messages, the agent is able to link back to the traces that have produced the messages. +2. When the agent receives a `traceparent` header from outside a trust boundary, it [can restart the trace](trace-continuation.md) (creating a different trace id with its own sampling decision) and link to the originating trace. +3. Close gap for the OTLP intake - [OTel's specification of span links](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/overview.md#links-between-spans) + +Spans and Transactions MUST collect links in the `links` array with the following fields on each item: +- `trace_id`: the id of the linked trace. +- `span_id`: the id of the linked span or transaction. + +Example: + +``` +"links": [ + {"trace_id": "traceId1", "span_id": "spanId1"}, + {"trace_id": "traceId2", "span_id": "spanId2"}, +] +``` + +### API + +Agents MAY provide a public API to add span links at span/transaction creation. +A use-case for user's manually adding span links is for [batch message processing](tracing-instrumentation-messaging.md#batch-message-processing) +that the APM agent does not or cannot instrument. (For some agents it would be +a burden to internally support span links and *not* expose the API publicly.) + +If provided, the API SHOULD be written such that user code is not broken if/when +support for span link *attributes* is added in the future. + +If provided, the API and semantics SHOULD be compatible with the +[OpenTelemetry specification on specifying span links](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#specifying-links). A compatible API will facilitate +[OpenTelemetry bridge](trace-api-otel.md) support. OpenTelemetry requirements: + +- The public API SHOULD allow adding span links *after* span creation. +- Links SHOULD preserve the order in which they are set. diff --git a/specs/agents/trace-continuation.md b/specs/agents/trace-continuation.md new file mode 100644 index 00000000..b4eea311 --- /dev/null +++ b/specs/agents/trace-continuation.md @@ -0,0 +1,24 @@ +## Trace Continuation + +### `trace_continuation_strategy` configuration + +| | | +|----------------|---| +| Valid options | `continue`, `restart`, `restart_external` | +| Default | `continue` | +| Dynamic | `true` | +| Central config | `true` | + +The `traceparent` header of requests that are traced with our agents might have been added by a 3rd party component. + +This situation becomes more and more common as the w3c trace context gets adopted. In such cases we can end up with traces where part of the trace is outside of our system. + +In order to handle this properly, the agent SHOULD offer several trace continuation strategies. + +The agent SHOULD offer a configuration called `trace_continuation_strategy` with the following values and behavior: + +- `continue`: The agent takes the `traceparent` header as it is and applies it to the new transaction. +- `restart`: The agent always creates a new trace with a new trace id. In this case the agent MUST create a [span link](span-links.md) in the new transaction pointing to the original traceparent. +- `restart_external`: The agent first checks the `tracestate` header. If the header contains the `es` vendor flag, it's treated as internal, otherwise (including the case when the `tracestate` header is not present) it's treated as external. In case of external calls the agent MUST create a new trace with a new trace id and MUST create a link in the new transaction pointing to the original trace. + +In the case of internal calls, the agent MUST use the `continue` strategy above. \ No newline at end of file diff --git a/specs/agents/tracing-api-otel.md b/specs/agents/tracing-api-otel.md new file mode 100644 index 00000000..6ccec1d5 --- /dev/null +++ b/specs/agents/tracing-api-otel.md @@ -0,0 +1,278 @@ +## OpenTelemetry API (Tracing) + +[OpenTelemetry](https://opentelemetry.io) (OTel in short) provides a vendor-neutral API that allows to capture tracing, logs and metrics data. + +Agents MAY provide a bridge implementation of OpenTelemetry Tracing API following this specification. +When available, implementation MUST be configurable and should be disabled by default when marked as `experimental`. + +Agents MAY add an explicit agent configuration to enable/disable the bridge. If an agent implements such a configuration, the configuration name MUST be `opentelemetry_bridge_enabled`. + +The bridge implementation relies on APM Server version 7.16 or later. Agents SHOULD recommend this minimum version to users in bridge documentation. + +Bridging here means that for each OTel span created with the API, a native span/transaction will be created and sent to APM server. + +### User experience + +On a high-level, from the perspective of the application code, using the OTel bridge should not differ from using the +OTel API for tracing. See [limitations](#limitations) below for details on the currently unsupported OTel features. +For tracing the support should include: +- creating spans with attributes +- context propagation +- capturing errors + +The aim of the bridge is to allow any application/library that is instrumented with OTel API to capture OTel spans to +seamlessly delegate to Elastic APM span/transactions. Also, it provides a vendor-neutral alternative to any existing +manual agent API with similar features. + +One major difference though is that since the implementation of OTel API will be delegated to Elastic APM agent, the +whole OTel configuration that might be present in the application code (OTel processor pipeline) or deployment +(env. variables) will be ignored. + +### Limitations + +The OTel API/specification goes beyond tracing, as a result, the following OTel features are not supported: +- metrics +- logs +- span events +- span link *attributes* + +### Spans and Transactions + +OTel only defines Spans, whereas Elastic APM relies on both Spans and Transactions. +OTel allows users to provide the _remote context_ when creating a span, which is equivalent to providing a parent to a transaction or span, +it also allows to provide a (local) parent span. + +As a result, when creating Spans through OTel API with a bridge, agents must implement the following algorithm: + +```javascript +// otel_span contains the properties set through the OTel API +span_or_transaction = null; +if (otel_span.remote_context != null) { + span_or_transaction = createTransactionWithParent(otel_span.remote_context); +} else if (otel_span.parent == null) { + span_or_transaction = createRootTransaction(); +} else { + span_or_transaction = createSpanWithParent(otel_span.parent); +} +``` + +### Span Kind + +OTel spans have an `SpanKind` property ([specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#spankind)) which is close but not strictly equivalent to our definition of spans and transactions. + +For both transactions and spans, an optional `otel.span_kind` property will be provided by agents when set through +the OTel API. +This value should be stored into Elasticsearch documents to preserve OTel semantics and help future OTel integration. + +Possible values are `CLIENT`, `SERVER`, `PRODUCER`, `CONSUMER` and `INTERNAL`, refer to [specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#spankind) for details on semantics. + +By default, OTel spans have their `SpanKind` set to `INTERNAL` by OTel API implementation, so it is assumed to always be provided when using the bridge. + +For existing agents without OTel bridge or for data captured without the bridge, the APM server has to infer the value of `otel.span_kind` with the following algorithm: + +```javascript +span_kind = null; +if (isTransaction(item)) { + if (item.type == "messaging") { + span_kind = "CONSUMER"; + } else if (item.type == "request") { + span_kind = "SERVER"; + } +} else { + // span + if (item.type == "external" || item.type == "storage" || item.type == "db") { + span_kind = "CLIENT"; + } +} + +if (span_kind == null) { + span_kind = "INTERNAL"; +} + +``` + +While being optional, inferring the value of `otel.span_kind` helps to keep the data model closer to the OTel specification, even if the original data was sent using the native agent protocol. + +### Span status + +OTel spans have a [Status](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#set-status) +field to indicate the status of the underlying task they represent. + +When the [Set Status](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#set-status) on OTel API is used, we can map it directly to `span.outcome`: +- OK => Success +- Error => Failure +- Unset (default) => Unknown + +However, when not provided explicitly agents can infer the outcome from the presence of a reported error. +This behavior is not expected with OTel API with status, thus bridged spans/transactions should NOT have their outcome +altered by reporting (or lack of reporting) of an error. Here the behavior should be identical to when the end-user provides +the outcome explicitly and thus have higher priority over the inferred value. + +For OTel spans that are mapped to Elastic Transactions, agent should ensure that `transaction.result` is not sent and let +the server infer the value from `otel.attributes`. This allows to have consistent values for `transaction.result` between +agent intake and native OTLP intake. + +### Attributes mapping + +OTel relies on key-value pairs for span attributes. +Keys and values are protocol-specific and are defined in [semantic convention](https://github.com/open-telemetry/opentelemetry-specification/tree/main/specification/trace/semantic_conventions) specification. + +In order to minimize the mapping complexity in agents, most of the mapping between OTel attributes and agent protocol will be delegated to APM server: +- All OTel span attributes should be captured as-is and written to agent protocol. +- APM server will handle the mapping between OTel attributes and their native transaction/spans equivalents +- Some native span/transaction attributes will still require mapping within agents for [compatibility with existing features](#compatibility-mapping) + +OpenTelemetry attributes should be stored in `otel.attributes` as a flat key-value pair mapping added to `span` and `transaction` objects: +```json +{ + // [...] other span/transaction attributes + "otel": { + "span_kind": "CLIENT", + "attributes": { + "db.system": "mysql", + "db.statement": "SELECT * from table_1" + } + } +} +``` + +Starting from version 7.16 onwards, APM server must provide a mapping that is equivalent to the native OpenTelemetry Protocol (OTLP) intake for the +fields provided in `otel.attributes`. + +When sending data to APM server version before 7.16, agents MAY use span and transaction labels as fallback to store OTel attributes to avoid dropping information. + +### Compatibility mapping + +Agents should ensure compatibility with the following features: +- breakdown metrics +- [dropped spans statistics](handling-huge-traces/tracing-spans-dropped-stats.md) +- [compressed spans](handling-huge-traces/tracing-spans-compress.md) + +As a consequence, agents must provide values for the following attributes: +- `transaction.name` or `span.name` : value directly provided by OTel API +- `transaction.type` : see inference algorithm below +- `span.type` and `span.subtype` : see inference algorithm below +- `span.destination.service.resource` : see inference algorithm below + +#### Transaction type + +```javascript +a = transation.otel.attributes; +span_kind = transaction.otel_span_kind; +isRpc = a['rpc.system'] !== undefined; +isHttp = a['http.url'] !== undefined || a['http.scheme'] !== undefined; +isMessaging = a['messaging.system'] !== undefined; +if (span_kind == 'SERVER' && (isRpc || isHttp)) { + type = 'request'; +} else if (span_kind == 'CONSUMER' && isMessaging) { + type = 'messaging'; +} else { + type = 'unknown'; +} +``` + +#### Span type, sub-type and service target + +```javascript +a = span.otel.attributes; +type = undefined; +subtype = undefined; +serviceTargetType = null; +serviceTargetName = null; + +httpPortFromScheme = function (scheme) { + if ('http' == scheme) { + return 80; + } else if ('https' == scheme) { + return 443; + } + return -1; +} + +// extracts 'host' or 'host:port' from URL +parseNetName = function (url) { + var u = new URL(url); // https://developer.mozilla.org/en-US/docs/Web/API/URL + if (u.port != '') { + return u.host; // host:port already in URL + } else { + var port = httpPortFromScheme(u.protocol.substring(0, u.protocol.length - 1)); + return port > 0 ? u.hostname + ':'+ port : u.hostname; + } +} + +netPort = a['net.peer.port'] || -1; +netPeer = a['net.peer.name'] || a['net.peer.ip']; +netName = netPeer; // netName includes port, if provided +if (netName && netPort > 0) { + netName += ':' + netPort; +} + +if (a['db.system']) { + type = 'db' + subtype = a['db.system']; + serviceTargetType = subtype; + serviceTargetName = a['db.name'] || null; + +} else if (a['messaging.system']) { + type = 'messaging'; + subtype = a['messaging.system']; + + serviceTargetType = subtype; + serviceTargetName = a['messaging.destination'] || null; + +} else if (a['rpc.system']) { + type = 'external'; + subtype = a['rpc.system']; + serviceTargetType = subtype; + serviceTargetName = netName || a['rpc.service'] || null; + +} else if (a['http.url'] || a['http.scheme']) { + type = 'external'; + subtype = 'http'; + serviceTargetType = subtype; + + httpHost = a['http.host'] || netPeer; + if (httpHost) { + if (netPort < 0) { + netPort = httpPortFromScheme(a['http.scheme']); + } + serviceTargetName = netPort < 0 ? httpHost : httpHost + ':' + netPort; + } else if (a['http.url']) { + serviceTargetName = parseNetName(a['http.url']); + } +} + +if (type === undefined) { + if (span.otel.span_kind == 'INTERNAL') { + type = 'app'; + subtype = 'internal'; + } else { + type = 'unknown'; + } +} +span.type = type; +span.subtype = subtype; +if (serviceTargetType || serviceTargetName) { + span.setServiceTarget(serviceTargetType, serviceTargetName); +} +``` + +### Active Spans and Context + +When possible, bridge implementation MUST ensure proper interoperability between Elastic transactions/spans and OTel spans when +used from their respective APIs: +- After activating an Elastic span via the agent's API, the [`Context`] returned via the [get current context API] should contain that Elastic span +- When an OTel context is [attached] (aka activated), the [get current context API] should return the same [`Context`] instance. +- Starting an OTel span in the scope of an active Elastic span should make the OTel span a child of the Elastic span. +- Starting an Elastic span in the scope of an active OTel span should make the Elastic span a child of the OTel span. + +[`Context`]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/context/context.md +[attached]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/context/context.md#attach-context +[get current context API]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/context/context.md#get-current-context + +Both OTel and our agents have their own definition of what "active context" is, for example: +- Java Agent: Elastic active context is implemented as a thread-local stack +- Java OTel API: active context is implemented as a key-value map propagated through thread local + +In order to avoid potentially complex and tedious synchronization issues between OTel and our existing agent +implementations, the bridge implementation SHOULD provide an abstraction to have a single "active context" storage. diff --git a/specs/agents/tracing-api.md b/specs/agents/tracing-api.md new file mode 100644 index 00000000..961236b3 --- /dev/null +++ b/specs/agents/tracing-api.md @@ -0,0 +1,16 @@ +## Tracer APIs + +All agents must provide a native API to enable developers to instrument their applications manually, in addition to any +automatic instrumentation. + +Agents document their APIs in the elastic.co docs: + +- [Node.js Agent](https://www.elastic.co/guide/en/apm/agent/nodejs/current/api.html) +- [Go Agent](https://www.elastic.co/guide/en/apm/agent/go/current/api.html) +- [Java Agent](https://www.elastic.co/guide/en/apm/agent/java/current/public-api.html) +- [.NET Agent](https://www.elastic.co/guide/en/apm/agent/dotnet/current/public-api.html) +- [Python Agent](https://www.elastic.co/guide/en/apm/agent/python/current/api.html) +- [Ruby Agent](https://www.elastic.co/guide/en/apm/agent/ruby/current/api.html) +- [RUM JS Agent](https://www.elastic.co/guide/en/apm/agent/js-base/current/api.html) + +In addition, each agent may provide "bridge" implementations of vendor-neutral [OpenTelemetry API](tracing-api-otel.md). \ No newline at end of file diff --git a/specs/agents/tracing-distributed-tracing.md b/specs/agents/tracing-distributed-tracing.md new file mode 100644 index 00000000..53b38cc0 --- /dev/null +++ b/specs/agents/tracing-distributed-tracing.md @@ -0,0 +1,240 @@ +## Distributed Tracing + +We implement the [W3C standards](https://www.w3.org/TR/trace-context-1/) for +`traceparent` and `tracestate`, both for HTTP headers and binary fields. + + +### `trace_id`, `parent_id`, and `traceparent` + +Our `trace_id`, `parent_id`, and the combined `traceparent` HTTP header follow +the standard established by the +[W3C Trace-Context Spec](https://github.com/w3c/trace-context/blob/main/spec/20-http_request_header_format.md#traceparent-header). + +The `traceparent` header is composed of four parts: + + * `version` + * `trace-id` + * `parent-id` + * `trace-flags` + +Example: + +``` +traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01 + () (______________________________) (______________) () + v v v v + Version Trace-Id Parent-Id Flags +``` + + +#### Version + +The `version` is 1 byte (2 hexadecimal digits) representing an 8-bit unsigned +integer. Currently, the `version` will always be `00`. + +#### Trace ID + +A Trace ID is globally unique, and consists of 128 random bits (like a UUID). +Its string representation is 32 hexadecimal digits. This is the ID for the +whole distributed trace and stays constant throughout a given trace. + +#### Transaction/Span ID, and Parent ID + +Each transaction and span object will store the global `trace_id`. If the transaction +is started without an incoming `traceparent` header, then the `trace_id` +should be generated. + +Each transaction and span object will have an `id`. This is generated for each +transaction and span, and is 64 random bits (with a string representation of +16 hexadecimal digits). + +Each transaction and span object will have a `parent_id`, except for the very +first transaction in the distributed trace. Some agents allow the user to +ensure a parent ID is present on a transaction via an API call. In this case, +if the transaction doesn't have a `parent_id` the agent will generate a new ID +and set it as the `parent_id` for the transaction. + +The `parent_id` will be the `id` of the parent transaction/span. For new +transactions with an incoming `traceparent` header, the `parent-id` piece of +the `traceparent` should be used as the `parent_id`. + +In addition to the above rules, spans will also have a `transaction_id`, +which is the `id` of the current transaction. While not necessary for +distributed tracing, this inclusion allows for simpler and more performant UI +queries. + +Error objects will also include the `trace_id` (optional), an `id` (which in +the case of errors is 128 bits, encoded as 32 hexadecimal digits), a +`transaction_id`, and a `parent_id` (which is the `id` of the transaction or +span that caused the error). If an error occurs outside of the context of a +transaction or span, these fields may be missing. + + +#### Flags + +The W3C traceparent header specifies 8 bits for flags. Currently, only a single +flag (`sampled`) is defined, with the rest reserved for later use. These flags +are recommendations given by the by the caller rather than strict rules to +follow. + +##### Sampled + +The `sampled` flag is the least significant bit (right-most) and denotes that +the caller may have recorded trace data. If this flag is unset (`0` in the +least significant bit), the agent should not sample the transaction. If this +flag is set (`1` in the least significant bit), the agent should sample the +transaction. The agent may ignore this flag if sampling a transaction would +conflict with another config option, e.g. rate limit.See the +[sampling](tracing-sampling.md) specification for more details. + + +### `tracestate` + +For our own `es` `tracestate` entry we will introduce a `key:value` formatted list of attributes. +This is used to propagate the sampling rate downstream, for example. +See the [sampling](tracing-sampling.md) specification for more details. + +The general `tracestate` format is: + + tracestate: es=key:value;key:value...,othervendor= + +For example: + + tracestate: es=s:0.1,othervendor= + + +#### Validation and length limits + +The [`tracestate`](https://www.w3.org/TR/trace-context/#tracestate-header) +specification lists a number of validation rules. +In addition to that, +there are specific rules for the attributes under the `es` entry. + +Agents MUST implement these validation rules when mutating `tracestate`: + +- Vendor keys (such as `es`) have a maximum size of 256 chars. +- Vendor keys MUST begin with a lowercase letter or a digit, + and can only contain lowercase letters (`a-z`), + digits (`0-9`), underscores (`_`), dashes (`-`), asterisks (`*`), + and forward slashes (`/`). +- Vendor values have a maximum size of 256 chars. +- Vendor values may only contain ASCII RFC0020 characters (i.e., the range `0x20` to `0x7E`) except comma `,` and `=`. +- In addition to the above limitations, the keys and values used in the `es` entry must not contain the characters `:` and `;`. +- If adding another key/value pair to the `es` entry would exceed the limit of + 256 chars (including separator characters `:` and `;`), that key/value pair + MUST be ignored by agents. + +Note that we will currently only ever populate an `es` `tracestate` entry at the trace root. +It is not strictly necessary to validate `tracestate` in its entirety when received downstream. +Instead, downstream agents may opt to only parse the `es` entry and skip validation of other vendors' entries. +This means that the vendor key validations are only relevant if an agent adds +its own non-`es` keys to tracestate + +In addition, we do not enforce the 32-entry limit for vendor entries in +`tracestate`. Doing so would cripple our ability to use `tracestate` for our +own purposes, arbitrarily. Removing other entries to make way for our own +would also cause unexpected behavior. In any case, this situation should be +rare and we feel comfortable ignoring the validation rules in this case. + + +### HTTP Headers + +Every outgoing request should be intercepted and modified to include both the +`traceparent` and `tracestate` headers, described above. + +If an incoming request contains either of the `traceparent` or `tracestate` +headers, they should be propagated throughout the transaction and mutated as +specified above before being set on outgoing requests. + +There is an edge case where `tracestate` is present but blank `""` - in this +case, we should not propagate the tracestate header. Propagating the header in +this case can cause downstream receivers of requests from the monitored application +to error if they are excessively sensitive to blank values (blank values are within +spec, but still unexpected) + +The `span-id` part of the `traceparent` header should be the `id` of the span +representing the outgoing request. If (and only if) that span is not sampled, +the `span-id` may instead be the `id` of the current transaction. + +HTTP/text format should be used for headers wherever possible. Only in cases +where binary fields are necessary (such as in Kafka record headers) should +binary fields be used. (See below for binary fields) + + +### Binary Fields + +Binary fields should only be used where strings are not allowed, such as in +Kafka record headers. + +Until the revision of this spec as of January 2023, our implementation relied on the [W3C Binary Trace +Context](https://w3c.github.io/trace-context-binary/) standard. Hereby, we relied on the specification described in +[this commit](https://github.com/w3c/trace-context-binary/blob/571cafae56360d99c1f233e7df7d0009b44201fe/spec/20-binary-format.md). +We used the proprietary `elasticapmtraceparent` field name for the binary `traceparent`, `tracestate` was ignored. + +The available OpenTelemetry instrumentations however went a different route: + * The binary header spec has been removed and the [issue](https://github.com/open-telemetry/opentelemetry-specification/issues/437) to re-add it has not been touched for a long time. + * Instead, the OpenTelemetry instrumentations for e.g. Kafka use the textual `traceparent` and `tracestate` formats and encode the values via UTF8 to binary. + +To maximize compatibility and not break traces, our agents need to support the textual `traceparent` and `tracestate` via UTF8 encoding as well. +So, the following rules should be used to decode/encode context propagation headers: + +Encoding: + * Add a `elasticapmtraceparent` header with the binary specification above for backwards compatibility + * Add the textual `traceparent` and `tracestate` headers, encode their values via UTF8 + +Decoding: + * If a `traceparent` header is present, use the `traceparent` and `tracestate` headers as textual with UTF8 decoding of the values + * If no `traceparent` header is present, use the `elasticapmtraceparent` with the binary specification above. `tracestate` is ignored. + +Implementation note: The `traceparent` and `tracestate` specifications only allow ASCII characters. Therefore we don't need to use fully fledged UTF8 decoding, instead each byte can directly be interpreted as ASCII character. + +This implementation guarantees backwards compatibility with our older agents and compatibility with current OpenTelemetry instrumentations. +We will eventually remove the `elasticapmtraceparent` entirely as soon as we can safely assume that most users have already upgraded all their agents. + +### Baggage + +To propagate distributed context, we implement the [W3C Baggage](https://www.w3.org/TR/baggage/) specification. + +Agents MUST parse, validate, and attach the `baggage` header according [to the W3C specification](https://www.w3.org/TR/baggage/). + +In addition to the W3C specification, agents also SHOULD offer users 2 ways to use the values propagated via baggage: +- Offer a Baggage API to manually manipulate and read baggage values - this is preferable the OpenTelemetry API. +- Offer baggage related configuration to automatically store baggage values on events. + +#### Baggage API + +Agents SHOULD integrate with the existing OpenTelemetry API. Users should be able to use the OpenTelemetry API to interact with the baggage maintained by the agent. The primary way to interact with baggage SHOULD be the OpenTelemetry API. + +In case using the OpenTelemetry API isn't feasible in a given language, the agent SHOULD extend the proprietary agent API to interact with baggage. + +This API MUST offer the following functionalities: +- Adding a new baggage item +- Changing an existing baggage item +- Removing an existing baggage item +- Reading a specific baggage item +- Reading all baggage items + +#### Baggage related configuration + +The following configuration enables users to automatically store baggage items on a given event without any code change. Baggage items with matching keys are stored in `otel.attributes`, except on errors, since there is no `otel.attributes` on errors. Keys of baggage items lifted by the agent from baggage into attributes/labels MUST be prefixed with `baggage.`. + +`baggage_to_attach` configuration + +A list of baggage keys which are automatically attached to the current event (transaction, span, or error). When the event is created, the agent iterates through all baggage items currently available and stores the ones with keys that match one of the items from the configured wildcard matcher list on the newly created event with a prefix `baggage.`. In case of transactions and spans, the agent MUST send this data in `otel.attributes`, in case of errors the agent MUST send the data in labels. + +| | | +|----------------|---| +| Type | `List<`[`WildcardMatcher`](../../tests/agents/json-specs/wildcard_matcher_tests.json)`>` | +| Default | `*` | +| Dynamic | `true` | +| Central config | `true` | + +### Legacy HTTP Header Name + +Some agents support the legacy header name `elastic-apm-traceparent`. This name +was used while the W3C standard was being finalized, to avoid any +backwards-compatibility issues. New agents do not need to support this legacy +name. Because `tracestate` was not implemented until the standard was +finalized, no legacy names exist for this field. + +Agents that do support the legacy header MUST give precedence to the `traceparent` header if both are present. diff --git a/specs/agents/tracing-instrumentation-aws-lambda.md b/specs/agents/tracing-instrumentation-aws-lambda.md new file mode 100644 index 00000000..c2c46ef0 --- /dev/null +++ b/specs/agents/tracing-instrumentation-aws-lambda.md @@ -0,0 +1,439 @@ +# AWS Lambda instrumentation +This spec specifies the instrumentation of applications / services running on AWS Lambda. + +An AWS Lambda application needs to implement a **handler method** that is called whenever that Lambda function is invoked. A handler method receives (at least) two objects with useful information: +- `event`: Depending on the trigger type of the Lambda function this object contains different, trigger-specific data. For some trigger types this object may be empty/null. +- `context`: This object provides generic (trigger agnostic) meta information about the Lambda function. + +In our instrumentation we use these objects to derive useful meta and context information. + +## Generic Lambda Instrumentation +In general, to instrument Lambda functions, we create transactions that wrap the execution of that handler method. In cases where we cannot assume any type or information in the `event` object (e.g. if the trigger type is undefined), we ignore trigger-specific information and simply wrap the handler method with a transaction, while using the `context` object to derive some necessary fields: + +Field | Value | Description | Source +--- | --- | --- | --- +`name` | e.g. `MyFunctionName` | The transaction name. Use function name if trigger type is `other`. | `context.functionName` +`type` | e.g. `request`, `messaging` | The transaction type. | Use `request` if trigger type is undefined. +`outcome` | `success` / `failure` | Set to `failure` if there was a [function error](https://docs.aws.amazon.com/lambda/latest/dg/invocation-retries.html). | - +`result` | `success` / `failure` / `HTTP Xxx` | See API Gateway below. For other triggers, set to `failure` if there was a function error, otherwise `success`. | Trigger specific. +`faas.name` | e.g. `my-function` | The lambda function name. | Use the value of `context.functionName` or the `AWS_LAMBDA_FUNCTION_NAME` environment variable. +`faas.version` | e.g. `123` | The lambda function version. | Use the value of `context.functionVersion` or the `AWS_LAMBDA_FUNCTION_VERSION` environment variable. +`faas.id` | e.g. `arn:aws:lambda:us-west-2:123456789012:function:my-function` | Use the ARN of the function **without the alias suffix**. | `context.invokedFunctionArn`, remove the 8th ARN segment if the ARN contains an alias suffix. `arn:aws:lambda:us-west-2:123456789012:function:my-function:someAlias` will become `arn:aws:lambda:us-west-2:123456789012:function:my-function`. +`faas.trigger.type` | `other` | The trigger type. Use `other` if trigger type is unknown / cannot be specified. | More concrete triggers are `http`, `pubsub`, `datasource`, `timer` (see specific triggers below). +`faas.execution` | `af9aa4-a6...` | The AWS request ID of the function invocation | `context.awsRequestId` +`faas.coldstart` | `true` / `false` | Boolean value indicating whether a Lambda function invocation was a cold start or not. | [see section below](deriving-cold-starts) +`faas.trigger.request_id` | - | Do not set this field if trigger type is `other`. | Trigger specific. +`context.cloud.origin.provider` | `aws` | Constant value for the origin cloud provider. | - +`context.cloud.origin.*` | - | Do not set these fields if trigger type is `other`. | Trigger specific. +`context.service.origin.*` | - | Do not set these fields if trigger type is `other`. | Trigger specific. + +Note that `faas.*` fields *are not* nested under the context property [in the intake api](https://github.com/elastic/apm-server/blob/main/docs/spec/v2/transaction.json)! `faas` is a top-level key on the transaction. + +### Overwriting Metadata +Automatically capturing cloud metadata doesn't work reliably from a Lambda environment. Moreover, retrieving cloud metadata through an additional HTTP request may slowdown the lambda function / increase cold start behaviour. Therefore, the generic cloud metadata fetching should be disabled when the agent is running in a lambda context (for instance through checking for the existence of the `AWS_LAMBDA_FUNCTION_NAME` environment variable). +Where possible, metadata should be overwritten at Lambda runtime startup corresponding to the field specifications in this spec. + +Some metadata fields are not available at startup (e.g. `invokedFunctionArn` which is needed for `cloud.account.id` and `faas.id`). Therefore, retrieval of metadata fields in a lambda context needs to be delayed until the first execution of the lambda function, so that information provided in the `context` object can used to set metadata fields properly. + +The following metadata fields are relevant for lambda functions: + +Field | Value | Description | Source +--- | --- | --- | --- +`service.name`| e.g. `MyFunctionName` | If the service name is *explicitly* specified through the `service_name` agent config option, use the configured name. Otherwise, use the name of the Lambda function. | If the service name is not explicitly configured, use the Lambda function name: `AWS_LAMBDA_FUNCTION_NAME` or `context.functionName` +`service.version` | e.g. `$LATEST` | If the service version is *explicitly* specified through the `service_version` agent config option, use the configured version. Otherwise, use the lambda function version. | If the service version is not explicitly configured, use the Lambda function version: `AWS_LAMBDA_FUNCTION_VERSION` or `context.functionVersion` +`service.framework.name` | `AWS Lambda` | Constant value for the framework name. | - +`service.runtime.name`| e.g. `AWS_Lambda_java8` |Β The lambda runtime. | `AWS_EXECUTION_ENV` +`service.node.configured_name` | e.g. `2019/06/07/[$LATEST]e6f...` | The log stream name uniquely identifying a function instance. | `AWS_LAMBDA_LOG_STREAM_NAME` or `context.logStreamName` +`cloud.provider` | `aws` | Constant value for the cloud provider. | - +`cloud.region` | e.g. `us-east-1` | The cloud region. | `AWS_REGION` +`cloud.service.name` | `lambda` | Constant value for the AWS service. +`cloud.account.id` | e.g. `123456789012` | The cloud account id of the lambda function. | 5th fragment in `context.invokedFunctionArn`. + +### Deriving cold starts +A cold start occurs if AWS needs first to initialize the Lambda runtime (including the Lambda process, such as JVM, Node.js process, etc.) in order to handle a request. This happens for the first request and after long function idle times. A Lambda function instance only executes one event at a time (there is no concurrency). Thus, detecting a cold start is as simple as detecting whether the invocation of a __handler method__ is the **first since process startup** or not. This can be achieved with a global / process-scoped flag that is flipped at the first execution of the handler method. + +### Disabled functionalities +The following agent functionalities need to be disabled when tracing AWS Lambda functions until decided otherwise: +- **Metrics collection:** this includes all kind of metrics: system, process and breakdown metrics and is equivalent to +setting `ELASTIC_APM_METRICS_INTERVAL = 0` +- **Remote configuration:** equivalent to setting `ELASTIC_APM_CENTRAL_CONFIG = false` +- **Cloud metadata discovery:** equivalent to setting `ELASTIC_APM_CLOUD_PROVIDER = none` +- **System metadata discovery:** in some agents, this may be a relatively heavy task. For example, the Java agent +executes extenal commands in order to discover the hostname, which is not required for AWS Lambda metadata. All other +agents read and parse files to extract container and k8s metadata, which is not required as well. + +There are two main approaches for agents to disable the above functionalities: +* Agents that will be always deployed as part of an additional APM Agent Lambda layer (e.g. Java agent) may disable this +through configuration options (e.g. environment variables) built in the lambda wrapper script(`AWS_LAMBDA_EXEC_WRAPPER`) +that is provided with the APM Agent Lambda layer. +* Agents that will be used with AWS lambda without the need for an additional Lambda layer must detect that they are +running in an AWS Lambda environment (for instance through checking for the existence of the `AWS_LAMBDA_FUNCTION_NAME` +environment variable). Such agents should disable the aforementioned functionalities programmatically to achieve the +same behaviour that would be achieved through the corresponding configuration options. + +## Trigger-specific Instrumentation +Lambda functions can be triggered in many different ways. A generic transaction for a Lambda invocation can be created independently of the actual trigger. However, depending on the trigger type, different information might be available that can be used to capture additional transaction data or that allows additional, valuable spans to be derived. The most common triggers that we want dedicated instrumentation support for are the following: + +- API Gateway V1 +- API Gateway V2 +- SQS +- SNS +- S3 + +If none of the above apply, the fallback should be a generic instrumentation (as described above) that can deal with any type of trigger (thus capturing only the minimal available information). + +### API Gateway / Lambda URLS +There are two different API Gateway versions (V1 & V2) that differ slightly in the information (`event` object) that is passed to the Lambda handler function. + +With both versions, the `event` object contains information about the http request. +Usually API Gateway-based Lambda functions return an object that contains the HTTP response information. +The agent should use the information in the request and response objects to fill the HTTP context (`context.request` and `context.response`) fields in the same way it is done for HTTP transactions. + +In particular, agents must use the `event.headers` to retrieve the `traceparent` and the `tracestate` and use them to start the transaction for the lambda function execution. + +In addition the following fields should be set for API Gateway-based Lambda functions: + +Field | Value | Description | Source +--- | --- | --- | --- +`type` | `request`| Transaction type: constant value for API gateway. | - +`name` | e.g. `GET /prod/proxy/{proxy+}` | Transaction name: Http method followed by a whitespace and the (resource) path. See section below. | - +`transaction.result` | `HTTP Xxx` / `success` | `HTTP 5xx` if there was a function error (see [Lambda error handling doc](https://docs.aws.amazon.com/lambda/latest/dg/services-apigateway.html#services-apigateway-errors). If the [invocation response has a "statusCode" field](https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html#http-api-develop-integrations-lambda.response), then set to `HTTP Xxx` based on the status code, otherwise `success`. | Error or `response.statusCode`. +`faas.trigger.type` | `http` | Constant value for API gateway. | - +`faas.trigger.request_id` | e.g. `afa4-a6...` | ID of the API gateway request. | `event.requestContext.requestId` +`context.service.origin.name` | e.g. `gdnrpwmtsb...amazonaws.com` | The full domain name of the API Gateway. | `event.requestContext.domainName` +`context.service.origin.id` | e.g. `gy415nu...` | `event.requestContext.apiId` | +`context.service.origin.version` | e.g. `1.0` | `1.0` for API Gateway V1, `2.0` for API Gateway V2. | `event.version` (or `1.0` if that field is not present) +`context.cloud.origin.service.name` | `api gateway` or `lambda url` | Constant value. | Detect lambda URLs by searching for `.lambda-url.` in the `event.requestContext.domainName`. Otherwise assume API Gateway. +`context.cloud.origin.account.id` | e.g. `12345678912` | Account ID of the API gateway. | `event.requestContext.accountId` +`context.cloud.origin.provider` | `aws` | Use `aws` as constant value. | - + +**Set `transaction.name` for the API Gateway trigger** + +There are different ways to setup an API Gateway in AWS resulting in different payload format versions: +* ["HTTP" proxy integrations](https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html) can be configured to use payload format 1.0 or 2.0. +* The older ["REST" proxy integrations](https://docs.aws.amazon.com/apigateway/latest/developerguide/set-up-lambda-proxy-integrations.html#api-gateway-simple-proxy-for-lambda-input-format) use payload format version 1.0. + +For both payload format versions (1.0 and 2.0), the general pattern is `$method $resourcePath` (unless the `use_path_as_transaction_name` agent config option is used). Some examples are: +* `GET /prod/some/resource/path` (specific resource path) +* `GET /prod/proxy/{proxy+}` (proxy in v1.0 with dynamic path) +* `POST /prod/$default` (proxy in v2.0 with dynamic path) + +*Payload format version 1.0:* + +For payload format version 1.0, use `${event.requestContext.httpMethod} /${event.requestContext.stage}${event.requestContext.resourcePath}`. + +If `use_path_as_transaction_name` is applicable and set to `true`, use `${event.requestContext.httpMethod} ${event.requestContext.path}` as the transaction name. + +*Payload format version 2.0:* + +For [payload format version](https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html) 2.0 (which can be identified as having `${event.requestContext.http}`), use `${event.requestContext.http.method} /${event.requestContext.stage}${_routeKey_}`. + +In version 2.0, the `${event.requestContext.routeKey}` can have the format `GET /some/path`, `ANY /some/path` or `$default`. For the `_routeKey_` part, extract the path (after the space) in the `${event.requestContext.routeKey}` or use `/$default`, in case of `$default` value in `${event.requestContext.routeKey}`. + +If `use_path_as_transaction_name` is applicable and set to `true`, use `${event.requestContext.http.method} ${event.requestContext.http.path}` as the transaction name. + +### Elastic Load Balancer (ELB) + +An Application Load Balancer (ALB) -- a type of ELB -- can be targeted directly +to lambda. + +The agent should use the information in the request and response objects to +fill the HTTP context (`context.request` and `context.response`) fields in the +same way it is done for HTTP transactions. +[Request/Response Docs](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/lambda-functions.html#receive-event-from-load-balancer) + +In particular, agents must use the `event.headers` to retrieve the +`traceparent` and the `tracestate` and use them to start the transaction for +the lambda function execution. + +In addition the following fields should be set for ELB-based Lambda functions: + +Field | Value | Description | Source +--- | --- | --- | --- +`type` | `request`| Transaction type: constant value for ELB. | - +`name` | e.g. `GET unknown route` | `{event.httpMethod} unknown route` An ALB acts as a gateway for any URL path, so to avoid high cardinality issues, "unknown route" should be used as described in [the HTTP Transactions spec](./tracing-instrumentation-http.md). The `use_path_as_transaction_name` config option should be honored. If a web framework is used in the Lambda function, it may provide a route name. | - +`result` | `HTTP Xxx` / `success` | `HTTP 5xx` if there was a function error. If the [invocation response has a "statusCode" field](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/lambda-functions.html#respond-to-load-balancer) and it is an integer, then set to `HTTP Xxx` based on the status code, otherwise `HTTP 5xx`. ELB defaults to a "502 Bad Gateway" response if the function returns no or an invalid "statusCode" | Error or `response.statusCode`. +`outcome` | `success` / `failure` | Set to `failure` if there was a function error, the response object does not contain an integer "statusCode", or the "statusCode" is `>= 500`. | - +`faas.trigger.type` | `http` | Constant value for ELB. | - +`context.service.origin.name` | e.g. `targetgroup` | ELB target group | `event.requestContext.elb.targetGroupArn` is formed as `arn:aws:elasticloadbalancing:region-code:account-id:targetgroup/target-group-name/target-group-id`, so use `targetGroupArn.split(':')[5].split('/')[1]` to get the `target-group-name`. +`context.service.origin.id` | e.g. `arn:aws:elasticlo...65c45c6791a` | ELB target group ARN | `event.requestContext.elb.targetGroupArn` | +`context.cloud.origin.service.name` | `elb` | Constant value for ELB. | - +`context.cloud.origin.account.id` | e.g. `123456789012` | Account ID for the ELB. | Derived from the 5th segment of `event.requestContext.elb.targetGroupArn` +`context.cloud.origin.region` | e.g. `us-east-2` | Cloud region. | Derived from the 4th segment of `event.requestContext.elb.targetGroupArn` +`context.cloud.origin.provider` | `aws` | Use `aws` as constant value. | - + +Note that the `context.service.origin.version` is omitted for ELB requests. + +An example ELB event: + +``` +{ + "requestContext": { + "elb": { + "targetGroupArn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/lambda-279XGJDqGZ5rsrHC2Fjr/49e9d65c45c6791a" + } + }, + "httpMethod": "POST", + "path": "/toolz/api/v2.0/downloadPDF/PDF_2020-09-11_11-06-01.pdf", + "queryStringParameters": { + "test%40key": "test%40value", + "language": "en-DE" + }, + "headers": { + "accept-encoding": "gzip,deflate", + "connection": "Keep-Alive", + "host": "blabla.com", + "user-agent": "Apache-HttpClient/4.5.13 (Java/11.0.15)", + "x-amzn-trace-id": "Root=1-5bdb40ca-556d8b0c50dc66f0511bf520", + "x-forwarded-for": "199.99.99.999", + "x-forwarded-port": "443", + "x-forwarded-proto": "https" + }, + "body": "blablablabody", + "isBase64Encoded": false +} +``` + +### SQS / SNS + +Lambda functions that are triggered by SQS (or SNS) accept an `event` input that may contain one or more SQS / SNS messages in the `event.records` array. All message-related context information (including the `traceparent`) is encoded in the individual message attributes (if at all). + +#### SQS + +Agents SHOULD check each record, [up to a maximum of 1000](tracing-instrumentation-messaging.md#receiving-trace-context), +for a `traceparent` message attribute, and create a [span link](span-links.md) +on the transaction for each message with trace-context. + +In addition to [the generic Lambda transaction fields](#generic-lambda-instrumentation) +the following fields SHOULD be set. The use of `records[0]` below depends on the +understanding from [AWS Lambda SQS docs](https://docs.aws.amazon.com/lambda/latest/dg/with-sqs.html) +that a trigger invocation can only include messages from *one* queue. + +Field | Value | Description | Source +--- | --- | --- | --- +`type` | `messaging`| Transaction type: constant value for SQS. | - +`name` | e.g. `RECEIVE SomeQueue` | Transaction name: Follow the [messaging spec](./tracing-instrumentation-messaging.md) for transaction naming. | Simple queue name can be derived from the 6th segment of `records[0].eventSourceArn`. +`faas.trigger.type` | `pubsub` | Constant value for message based triggers | - +`context.service.origin.name` | e.g. `my-queue` | SQS queue name | Simple queue name can be derived from the 6th segment of `records[0].eventSourceArn`. +`context.service.origin.id` | e.g. `arn:aws:sqs:us-east-2:123456789012:my-queue` | SQS queue ARN. |Β `records[0].eventSourceArn` +`context.cloud.origin.service.name` | `sqs` | Constant value for SQS. | - +`context.cloud.origin.region` | e.g. `us-east-1` | SQS queue region. | `records[0].awsRegion` +`context.cloud.origin.account.id` | e.g. `12345678912` | Account ID of the SQS queue. | Parse account segment (5th) from `records[0].eventSourceArn`. +`context.cloud.origin.provider` | `aws` | Use `aws` as constant value. | - + +An example SQS event: + +``` +{ + "Records": [ + { + "messageId": "94f54f6b-0b5d-4071-b752-103f481796d9", + "receiptHandle": "A...w==", + "body": "this is my body", + "attributes": { + "ApproximateReceiveCount": "1", + "SentTimestamp": "1646785527296", + "SenderId": "A...W", + "ApproximateFirstReceiveTimestamp": "1646785527302" + }, + "messageAttributes": { + "Greeting": { + "binaryValue": "SGVsbG8sIFdvcmxkIQ==", + "stringListValues": [], + "binaryListValues": [], + "dataType": "Binary" + }, + "Population": { + "stringValue": "1250800", + "stringListValues": [], + "binaryListValues": [], + "dataType": "Number" + }, + "City": { + "stringValue": "Any City", + "stringListValues": [], + "binaryListValues": [], + "dataType": "String" + } + }, + "md5OfBody": "567762fc32b60cd7fc4abbe9cf1fcfbe", + "md5OfMessageAttributes": "28eb0e573cf8e8a77e349a2f968eac4a", + "eventSource": "aws:sqs", + "eventSourceARN": "arn:aws:sqs:us-west-2:627286350134:my-queue", + "awsRegion": "us-west-2" + } + ] +} +``` + +#### SNS + +Agents SHOULD check each record, [up to a maximum of 1000](tracing-instrumentation-messaging.md#receiving-trace-context), +for a `traceparent` message attribute (`Records.*.Sns.MessageAttributes`), and +create a [span link](span-links.md) on the transaction for each message with +trace-context. + +In addition to [the generic Lambda transaction fields](#generic-lambda-instrumentation) +the following fields should be set. The use of `records[0]` is based on the +understanding, from ["all notification messages will contain a single published +message"](https://aws.amazon.com/sns/faqs/#Reliability), that an SNS trigger +will only ever have a single record. + +Field | Value | Description | Source +--- | --- | --- | --- +`type` | `messaging`| Transaction type: constant value for SNS. | - +`name` | e.g. `RECEIVE SomeTopic` | Transaction name: Follow the [messaging spec](./tracing-instrumentation-messaging.md) for transaction naming. | Simple topic name can be derived from the 6th segment of `records[0].sns.topicArn`. +`faas.trigger.type` | `pubsub` | Constant value for message based triggers | - +`context.service.origin.name` | e.g. `my-topic` | SNS topic name | Simple topic name can be derived from the 6th segment of `records[0].sns.topicArn`. +`context.service.origin.id` | e.g. `arn:aws:sns:us-east-2:123456789012:my-topic` | SNS topic ARN. |Β `records[0].sns.topicArn` +`context.cloud.origin.service.name` | `sns` | Constant value for SNS. | - +`context.cloud.origin.region` | e.g. `us-east-1` | SNS topic region. | Parse region segment (4th) from `records[0].sns.topicArn`. +`context.cloud.origin.account.id` | e.g. `12345678912` | Account ID of the SNS topic. | Parse account segment (5th) from `records[0].sns.topicArn`. +`context.cloud.origin.provider` | `aws` | Use `aws` as constant value. | - + +An example SNS event: + +``` +{ + "Records": [ + { + "EventSource": "aws:sns", + "EventVersion": "1.0", + "EventSubscriptionArn": "arn:aws:sns:us-west-2:123456789012:my-topic1:761195b9-8bb2-4dc7-bab2-8fb74214bb8b", + "Sns": { + "Type": "Notification", + "MessageId": "d68d14fb-1178-51b7-99ae-4e5ae7c39b7f", + "TopicArn": "arn:aws:sns:us-west-2:123456789012:my-topic", + "Subject": "this is my subject", + "Message": "this is my message", + "Timestamp": "2022-03-09T00:27:39.304Z", + "SignatureVersion": "1", + "Signature": "W...Q==", + "SigningCertUrl": "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-7ff5318490ec183fbaddaa2a969abfda.pem", + "UnsubscribeUrl": "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&...", + "MessageAttributes": { + "Greeting": { + "Type": "Binary", + "Value": "SGVsbG8sIFdvcmxkIQ==" + }, + "Population": { + "Type": "String", + "Value": "1250800" + }, + "City": { + "Type": "String", + "Value": "Any City" + } + } + } + } + ] +} +``` + +### S3 +Lambda functions that are triggered by S3 accept an `event` input that may contain one ore more `S3 event notification records` in the `event.records` array. We cannot (automatically) wrap the processing of the individual records that are sent as a batch of S3 event notification records with a single `event`. + +Thus, in case that an S3 `event` contains **exactly one** `S3 event notification record`, the agents must apply the following, S3-specific retrieval of information. Otherwise, the agents should apply the [Generic Lambda Instrumentation](generic-lambda-instrumentation) as desribed above. + +In addition the following fields should be set for Lambda functions triggered by S3: + +Field | Value | Description | Source +--- | --- | --- | --- +`type` | `request`| Transaction type: constant value for S3. | - +`name` | e.g. `ObjectCreated:Put mybucket` | Transaction name: Use event name and bucket name. | `${record.eventName} ${record.s3.bucket.name}` +`faas.trigger.type` | `datasource` | Constant value. | - +`faas.trigger.reuqest_id` | e.g. `C3D13FE58DE4C810`| S3 event request ID. | `record.responseElements.xAmzRequestId` +`context.service.origin.name` | e.g. `mybucket` | S3 bucket name. | `record.s3.bucket.name` +`context.service.origin.id` | e.g. `arn:aws:s3:::mybucket` | S3 bucket ARN. |Β `record.s3.bucket.arn` +`context.service.origin.version` | e.g. `2.1` | S3 event version. | `record.eventVersion` +`context.cloud.origin.service.name` | `s3` | Constant value for S3. | - +`context.cloud.origin.region` | e.g. `us-east-1` | S3 bucket region. | `record.awsRegion` +`context.cloud.origin.provider` | `aws` | Use `aws` as constant value. | - +__**otel.attributes._**__ |
|
|
+`_["aws.s3.bucket"]`| `mybucket` | S3 bucket name, if available. See [OTel Semantic Conventions](https://github.com/open-telemetry/opentelemetry-specification/blob/v1.20.0/semantic_conventions/trace/instrumentation/aws-sdk.yml#L435). Note: this must be a single dotted string key in the `otel.attributes` mapping -- for example `{"otel": {"attributes": {"aws.s3.bucket": "mybucket"}}}` -- and *not* a nested object. | `record.s3.bucket.name` +`_["aws.s3.key"]`| `my/key/path` | S3 object key, if applicable. See [OTel Semantic Conventions](https://github.com/open-telemetry/opentelemetry-specification/blob/v1.20.0/semantic_conventions/trace/instrumentation/aws-sdk.yml#L435). Note: this must be a single dotted string key in the `otel.attributes` mapping and *not* a nested object. | `record.s3.object.key` + +An example S3 event: + +``` +{ + "Records": [ + { + "eventVersion": "2.1", + "eventSource": "aws:s3", + "awsRegion": "us-east-2", + "eventTime": "2019-09-03T19:37:27.192Z", + "eventName": "ObjectCreated:Put", + "userIdentity": { + "principalId": "AWS:AIDAINPONIXQXHT3IKHL2" + }, + "requestParameters": { + "sourceIPAddress": "205.255.255.255" + }, + "responseElements": { + "x-amz-request-id": "D82B88E5F771F645", + "x-amz-id-2": "vlR7PnpV2Ce81l0PRw6jlUpck7Jo5ZsQjryTjKlc5aLWGVHPZLj5NeC6qMa0emYBDXOo6QBU0Wo=" + }, + "s3": { + "s3SchemaVersion": "1.0", + "configurationId": "828aa6fc-f7b5-4305-8584-487c791949c1", + "bucket": { + "name": "DOC-EXAMPLE-BUCKET", + "ownerIdentity": { + "principalId": "A3I5XTEXAMAI3E" + }, + "arn": "arn:aws:s3:::lambda-artifacts-deafc19498e3f2df" + }, + "object": { + "key": "b21b84d653bb07b05b1e6b33684dc11b", + "size": 1305107, + "eTag": "b21b84d653bb07b05b1e6b33684dc11b", + "sequencer": "0C0F6F405D6ED209E1" + } + } + } + ] +} +``` + +## Data Flushing +Lambda functions are immediately frozen as soon as the handler method ends. +Because APM data is sent in an asynchronous way, data can get lost if not sent +before the lambda function ends. + +Therefore, the agents MUST ensure that data is flushed synchronously before the +execution of the handler function ends. + +Instrumented lambda functions should use the custom-built [Lambda extension][1] +that allows the agent to send its data to `localhost`. The extension +asynchronously forwards the data it receives from the agent to the APM server +so the Lambda function can return its result with minimal delay. In order for +the extension to know when it can flush its data, it must receive a signal +indicating that the lambda function has completed. There are two possible +signals: one is via a subscription to the AWS Lambda Logs API and the other is +an agent intake request with the query param `flushed=true`. A signal from the +agent is preferable because there is an inherent delay with the sending of the +Logs API signal. Therefore, the agent must send its final intake request at +the end of the function invocation with the query param `flushed=true`. In case +there is no more data to send at the end of the function invocation, the agent +must send an empty intake request with this query param. + +[1]: https://github.com/elastic/apm-aws-lambda + +If the `ELASTIC_APM_LAMBDA_APM_SERVER` environment variable is defined, +instrumented lambda functions should register a "partial transaction" with the +lambda extension as soon as the transaction is started. This allows the lambda +extension to create and send a proxy transaction document even if the lambda +function times out and is unable to send the transaction document itself. The +process is as follows: + +At the start of the transaction, the agent will encode and immediately send the +newly-created transaction to the extension at the `/register/transaction` +endpoint via a `POST` request. The first line of the ndjson payload should be +the metadata. The second line should be the transaction json blob. The request +MUST have the header `x-elastic-aws-request-id` (the lambda invocation's +request ID) and the Content-Type MUST be +`application/vnd.elastic.apm.transaction+ndjson`. If the extension returns +anything other than a 200 HTTP code, the agent MUST stop sending partial +transaction registrations for future invocations. diff --git a/specs/agents/tracing-instrumentation-aws.md b/specs/agents/tracing-instrumentation-aws.md new file mode 100644 index 00000000..fe639705 --- /dev/null +++ b/specs/agents/tracing-instrumentation-aws.md @@ -0,0 +1,55 @@ +## AWS services spans + +We describe how to instrument some of AWS' services in this document. +Some of the services can use existing specs. When there are differences or additions, they have been noted below. +The spec for [instrumenting AWS Lambda](tracing-instrumentation-aws-lambda.md) is in a separate document. + +### S3 (Simple Storage Service) + +AWS Simple Storage Service offers object storage via a REST API. The objects are organized into buckets, which are +themselves organized into regions. + +Field semantics and values for S3 are defined in the [S3 table within the database spec](tracing-instrumentation-db.md#aws-s3). + +### DynamoDB + +AWS DynamoDB is a document database so instrumenting it will follow the [db spec](tracing-instrumentation-db.md). +DynamoDB-specific specifications that supercede generic db field semantics are defined in the [DynamoDB table within the database spec](tracing-instrumentation-db.md#aws-dynamodb). + +### SQS (Simple Queue Service) + +AWS Simple Queue Service is a message queuing service. The [messaging spec](tracing-instrumentation-messaging.md) can +be used for instrumenting SQS, but the following specifications supersede those of the messaging spec. + +For a batch send message operation, the span name is `SQS SEND_BATCH to MyQueue`. The `span.action` is `send_batch`. + +The SQS API also includes delete message and batch delete message operations. These should be instrumented in addition +to the operations described in the messaging spec. For a delete message operation, the span name is +`SQS DELETE from MyQueue`. +For a batch delete message operation, the span name is `SQS DELETE_BATCH from MyQueue`. +The `span.action` is `delete_batch`. + +- **`context.destination.cloud.region`**: mandatory. The AWS region where the queue is. + +#### Distributed Tracing + +For distributed tracing, the SQS API has "message attributes" that can be used in lieu of headers. + +Agents should use an attribute name of `traceparent` when sending the trace parent header value via the SQS message attributes. Agents should use an attribute name of `tracestate` if sending trace state header value in an SQS message attribute. Agents should not prefix these names with an `elastic-` namespace. + +SQS has a documented limit of ten message attributes per message. Agents _should not_ add `traceparent` or `tracestate` headers to the message attributes if adding those fields would put an individual message over this limit. Agents _should_ log a message if they omit either `traceparent` or `tracestate` due to these length limits. + +### SNS (AWS Simple Notification Service) + +The AWS Simple Notification Service can be instrumented using the [messaging spec](tracing-instrumentation-messaging.md), +but the only action that is instrumented is [Publish](https://docs.aws.amazon.com/sns/latest/api/API_Publish.html). These specifications supersede those of the messaging spec: + +- `span.name`: + - For a publish action including a `TopicArn`, the span name MUST be `SNS PUBLISH to `. For example, for a TopicArn of `arn:aws:sns:us-east-2:123456789012:My-Topic` the topic-name is `My-Topic`. (Implementation note: this can extracted with the equivalent of this Python expression: `topicArn.split(':').pop()`.) + - For a publish action including a `TargetArn` (an endpoint ARN created via [CreatePlatformEndpoint](https://docs.aws.amazon.com/sns/latest/api/API_CreatePlatformEndpoint.html)), the span name MUST be `SNS PUBLISH to `. For example, for a TargetArn of `arn:aws:sns:us-west-2:123456789012:endpoint/GCM/gcmpushapp/5e3e9847-3183-3f18-a7e8-671c3a57d4b3` the application-name is `endpoint/GCM/gcmpushapp`. The endpoint UUID represents a device and mobile app. For manageable cardinality, the UUID must be excluded from the span name. (Implementation note: this can be extracted with the equivalent of this Python expression: `targetArn.split(':').pop().rsplit('/', 1)[0]`) + - For a publish action including a `PhoneNumber`, the span name MUST be `SNS PUBLISH to [PHONENUMBER]`. The actual phone number MUST NOT be included because it is [PII](https://en.wikipedia.org/wiki/Personal_data) and cardinality is too high. +- `span.action`: 'publish' + +- **`context.destination.cloud.region`**: mandatory. The AWS region where the topic is. + +For distributed tracing, the SNS API has "message attributes" that can be used in lieu of headers. diff --git a/specs/agents/tracing-instrumentation-azure-functions.md b/specs/agents/tracing-instrumentation-azure-functions.md new file mode 100644 index 00000000..6d737acf --- /dev/null +++ b/specs/agents/tracing-instrumentation-azure-functions.md @@ -0,0 +1,113 @@ +# Tracing Azure Functions + +An Azure Function Application can implement one or more handler functions that are executed via different trigger mechanisms (e.g. HTTP). +Depending on these trigger mechanisms and the actual [Azure Functions runtime language and version](https://learn.microsoft.com/en-us/azure/azure-functions/supported-languages), +the signature of these handler functions will vary, of course. + +In general however, there is always a *generic context* object available (e.g. [`FunctionContext`](https://learn.microsoft.com/en-us/dotnet/api/microsoft.azure.functions.worker.functioncontext)) that provides metadata and context for an instrumentation. + +Depending on the actual kind of invocation (trigger type) there might be an additional object to retrieve metadata and context from. +In case of an HTTP invocation/trigger there is e.g. [`HttpRequestData`](https://learn.microsoft.com/en-us/dotnet/api/microsoft.identitymodel.protocols.httprequestdata). + +## Generic Instrumentation + +In general, to instrument an Azure Functions application, we create transactions that wrap the execution of its handler methods. In cases where we cannot assume any trigger type or extract trigger-specific data (e.g. if the trigger type is unsupported), +we wrap the handler method with a transaction, while using the generic context object to derive necessary fields. + +**Note:** This table represents generic values that agents SHOULD always provide regardless of the +actual `faas.trigger.type`. For trigger types that are specifically supported (e.g HTTP) these values +can vary on trigger-specific specifications apply. + +| Field | Value | Description | Source | +| - | - | - | - | +| `name` | e.g. `MySampleTrigger` | The transaction name. Use function name if trigger type is `other`. | *generic context* | +| `type` | e.g. `request`, `messaging` | The transaction type. | Use `request` if trigger type is undefined. | +| `outcome` | `success`/`failure` | Set to `failure` if a function error can be detected otherwise `success`. | | +| `result` | `success`/`failure` |Set to `failure` if a function error can be detected, otherwise `success`. | Trigger specific. | +| `faas.name` | e.g. `MyFunctionApp/MySampleTrigger` | The function app name and the function name, using this format: `/FUNCTION_NAME>`. | *generic context* | +| `faas.id` | e.g. `/subscriptions/d2ba53be-0815-4...` | The [fully qualified resource ID](https://learn.microsoft.com/en-us/rest/api/resources/resources/get-by-id) of the Azure Function, which has this format: `/subscriptions//resourceGroups//providers/Microsoft.Web/sites//functions/` | *generic context*, environment | +| `faas.trigger.type` | `other` | The trigger type. Use `other` if the trigger type is unknown or cannot be specified. | More concrete triggers are `http`, `pubsub`, `datasource`, `timer` (see specific triggers below). | +| `faas.execution` | `203621a2-62f...` | The unique invocation id of the function. | *generic context* | +| `faas.coldstart` | `true`/`false` | A boolean value indicating whether this function invocation was a cold start or not. See the [Deriving cold starts](#deriving-cold-starts) section below. | + +### Metadata + +Generic cloud metadata fetching SHOULD be disabled when the agent is running in an Azure Functions app. It will not +yield any useful data and only incur additional cold start overhead due to the non-existent HTTP metadata endpoints. + +Agents SHOULD rather: + +* Check for the existence of the `FUNCTIONS_WORKER_RUNTIME` environment variable that indicates that the agent is running +in an Azure Function application. +* Perform specific cloud metadata detection for Azure Functions (see [Metadata](./metadata.md)). + +In addition to those described in [Metadata](./metadata.md), following metadata fields are relevant for Azure Functions: + +| Field | Value | Description | Source | +| - | - | - | - | +| `service.name` | e.g. `MyFunctionApp` | If the service name is *explicitly* specified through the `service_name` agent config option, use that value. Otherwise, use the name of the Function App. | If `service_name` is not specified, use `WEBSITE_SITE_NAME`. | +| `service.framework.name` | `Azure Functions` | Constant value for the framework name. | | +| `service.framework.version` | e.g. `~4` | Version of the Azure Functions runtime. | `FUNCTIONS_EXTENSION_VERSION` | +| `service.runtime.name`| e.g. `dotnet-isolated` |Β The language worker runtime (see [here](https://learn.microsoft.com/en-us/azure/azure-functions/functions-app-settings#functions_worker_runtime)). | `FUNCTIONS_WORKER_RUNTIME` | +| `service.node.configured_name` | e.g. `25d4009bce1d ...` | Unique ID of the VM instance. | `WEBSITE_INSTANCE_ID` ([Azure docs](https://learn.microsoft.com/en-us/azure/app-service/reference-app-settings#scaling)) **Note:** If the `service_node_name` agent configuration setting is provided (which normally will set `service.node.configured_name`) it MUST be ignored in favor of `WEBSITE_INSTANCE_ID`, and a log warning MUST be emitted saying it was ignored. | + +### Deriving cold starts + +A cold start occurs if the Azure Functions runtime needs to be initialized in order to handle the first function execution. +Since an Azure Function app can provide multiple function entry points, those may run concurrently. +Hence, cold start detection must happen in a thread-safe manner. + +The first function invocation MUST be reported as `faas.coldstart=true` and all subsequent invocations +to this function or other functions in the same function app MUST be reported as `faas.coldstart=false`. + +**Note:** The Azure Functions [Premium plan](https://learn.microsoft.com/en-us/azure/azure-functions/functions-scale) +guarantees pre-warmed workers (no cold starts). Nonetheless, reporting the first function execution as such might still make +sense since it could come with additional processing due to function-specific initialization overhead or JIT costs. +Alternatively, agents could try to detect their hosting plan and not report cold starts for Premium plans at all. + +### Disabled Functionality + +Agents SHOULD detect whether they are running in an Azure Functions environment by testing +for the existence of the `FUNCTIONS_WORKER_RUNTIME` environment variable. + +The following agent functionalities SHOULD to be turned off when tracing Azure Functions until decided otherwise: + +* **Metrics collection:** this includes all kind of metrics: system, process and breakdown metrics and is equivalent to +setting `ELASTIC_APM_METRICS_INTERVAL = 0` +* **Remote configuration:** equivalent to setting `ELASTIC_APM_CENTRAL_CONFIG = false` +* **Cloud metadata discovery:** + * Agents SHOULD disable their generic cloud metadata discovery mechanism to avoid pointless + HTTP requests to non-existent metadata endpoints (as described [here](https://github.com/elastic/apm/blob/main/specs/agents/metadata.md#cloud-provider-metadata)). + * Instead an Azure Functions specific metadata detection MUST be performed (see [Metadata](./metadata.md)). +* **System metadata discovery:** in some agents, this may be a relatively heavy task. For example, the Java agent +executes external commands in order to discover the hostname, which is not required for tracing Azure Functions. All other +agents read and parse files to extract container and k8s metadata, which is not required as well. + +## Trigger-Specific Instrumentation + +Azure Functions can be triggered in [many different ways](https://learn.microsoft.com/en-us/azure/azure-functions/functions-triggers-bindings). +A generic transaction for an Azure Functions invocation can be created independently of the actual trigger +based on the [generic instrumentation](#generic-instrumentation) fields described above. + +However, depending on the trigger type, different information might be available that can be used +to capture additional transaction data or that allows additional, valuable spans to be derived. + +### HTTP Trigger + +HTTP invocations typically provide *some* variants of an HTTP *request object* (e.g. [`HttpRequestData`](https://learn.microsoft.com/en-us/dotnet/api/microsoft.identitymodel.protocols.httprequestdata)) +and an HTTP *response object* (e.g. [`HttpResponseData`](https://learn.microsoft.com/en-us/dotnet/api/microsoft.azure.functions.worker.http.httpresponsedata)). + +Agents SHOULD use the information in the *request* and *response objects* to +fill the HTTP context (`context.request` and `context.response`) fields in the same way it is done for HTTP transactions. + +In particular, agents MUST use HTTP headers to retrieve the `traceparent` and the `tracestate` +and use those to start the transaction for the tracing the function execution. + +In addition the following fields should be set for HTTP trigger invocations: + +| Field | Value | Description | Source | +| - | - | - | - | +| `type` | `request`| Transaction type. Constant value for HTTP trigger invocations. | | +| `name` | e.g. `GET /api/MyFuncName`, `GET /api/products/{category:alpha}/{id:int?}` | ` //` | `` from the request object. `` from "extensions.http.routePrefix" in host.json, defaults to `api`. `` from "[``](https://learn.microsoft.com/en-us/azure/azure-functions/functions-referencefolder-structure)/function.json", defaults to the function name. The route must be normalized to have exactly one leading slash. [Azure docs link](https://learn.microsoft.com/en-us/azure/azure-functions/functions-bindings-http-webhook-trigger#customize-the-http-endpoint). | +| `transaction.result` | `HTTP Xxx` / `success` | `HTTP Xxx` based on the *response object* status code, otherwise `success`. | *response object* | +| `faas.trigger.type` | `http` | Constant value for HTTP trigger invocations. | | diff --git a/specs/agents/tracing-instrumentation-azure.md b/specs/agents/tracing-instrumentation-azure.md new file mode 100644 index 00000000..90c0545a --- /dev/null +++ b/specs/agents/tracing-instrumentation-azure.md @@ -0,0 +1,413 @@ +# Azure services spans + +We describe how to instrument some of Azure's services in this document. +Some of the services can use existing specs. When there are differences or additions, they have been noted below. + + +---- + +**NOTE**: +Azure Storage can be run as part of [Azure Stack](https://azure.microsoft.com/en-au/overview/azure-stack/). Azure services cannot be inferred from Azure Stack HTTP service endpoints. + +---- + +## Azure Storage + +Azure Storage is a collection of storage services, accessed through a +Storage account. The services include + +- Blob storage +- Queue storage +- File share storage +- Table storage + +A storage account is created in an Azure location (_region_), but the +location is not discoverable through REST API calls. + +### Blob storage + +Object storage for binary and text data via a REST API. There are three types of blobs: block blobs, page blobs and append blobs. + +Blobs are organized into containers, where a container can contain an +unlimited number of blobs, and a storage account can contain an unlimited +number of containers. Although blob storage is a flat storage scheme +(only one level of containers), blob names can include path segments (`/`), +providing a _virtual_ hierarchy. + +`` is the full name of the container and blob. For example, Given container `foo` and blob `bar/baz`, the resource name +is `foo/bar/baz`. + +`` is the leftmost subdomain of the domain of an Azure storage account endpoint. + +| APM field | Required? | Format | Notes | Example | +| --------- | --------- | ------ | ----- | ------- | +| `span.name` | yes | `AzureBlob ` | Pascal case Operation name | `AzureBlob Upload foo/bar/baz` | +| `span.type` | yes | `storage` | | | +| `span.subtype` | yes | `azureblob` | | | +| `span.action` | yes | `` | Pascal case | `Upload` | + +#### Span context fields + +| APM field | Required? | Format | Notes | Example | +| --------- | --------- | ------ | ----- | ------- | +| `context.destination.address` | yes | URL host | | `accountname.blob.core.windows.net` | +| `context.destination.service.resource` | yes | `azureblob/` | | `azureblob/accountname` | + + +#### Determining operations + +There are [_many_ blob storage operations](https://docs.microsoft.com/en-us/rest/api/storageservices/operations-on-blobs). + +Azure service endpoints for blob storage have one of the following host names + +| Cloud | Azure Service Endpoint | +| ----- | ---------------------- | +| Azure Global | `.blob.core.windows.net` | +| [Azure Government](https://docs.microsoft.com/en-us/azure/azure-government/documentation-government-developer-guide) | `.blob.core.usgovcloudapi.net` | +| [Azure China](https://docs.microsoft.com/en-us/azure/china/resources-developer-guide) |`.blob.core.chinacloudapi.cn` | +| [Azure Germany](https://docs.microsoft.com/en-us/azure/germany/germany-developer-guide) | `.blob.core.cloudapi.de` | + +where `` is the name of the storage account. New Azure service endpoints may be introduced by Azure later. + +Rules derived from the [Blob service REST API reference](https://docs.microsoft.com/en-us/rest/api/storageservices/blob-service-rest-api). The rules determine the operation name based +on the presence of data in the HTTP request + +| HTTP verb | HTTP headers | HTTP query string | Resulting Operation Name | +| --------- | ---------- | ------------------- | ------------------------ | +| DELETE | | | Delete | +| GET | | `restype=container` | GetProperties | +| GET | | `comp=metadata` | GetMetadata | +| GET | | `restype=container` and `comp=acl` | GetAcl | +| GET | | `restype=container` and `comp=list` | ListBlobs | +| GET | | `comp=list` | ListContainers | +| GET | | `comp=tags` | GetTags | +| GET | | `comp=tags` and `where=` | FindTags | +| GET | | `comp=blocklist` | GetBlockList | +| GET | | | Download | +| GET | | `comp=pagelist` | GetPageRanges | +| GET | | `comp=stats` | Stats | +| GET | | `comp=blobs` | FilterBlobs | +| HEAD | | | GetProperties | +| HEAD | | `restype=container` and `comp=metadata` | GetMetadata | +| HEAD | | `restype=container` and `comp=acl` | GetAcl | +| POST | | `comp=batch` | Batch | +| POST | | `comp=query` | Query | +| POST | | `comp=userdelegationkey` | GetUserDelegationKey | +| PUT | `x-ms-copy-source` | | Copy | +| PUT | `x-ms-copy-source` | `comp=block` | Copy | +| PUT | `x-ms-copy-source` | `comp=page` | Copy | +| PUT | `x-ms-copy-source` | `comp=incrementalcopy` | Copy | +| PUT | `x-ms-copy-source` | `comp=appendblock` | Copy | +| PUT | | `comp=copy` | Abort | +| PUT | `x-ms-blob-type` | | Upload | +| PUT | | `comp=block` | Upload | +| PUT | | `comp=blocklist` | Upload | +| PUT | | `comp=page` | Upload | +| PUT | | `comp=appendblock` | Upload | +| PUT | | | Create | +| PUT | | `comp=metadata` | SetMetadata | +| PUT | | `restype=container` and `comp=acl` | SetAcl | +| PUT | | `comp=properties` | SetProperties | +| PUT | | `comp=lease` | Lease | +| PUT | | `comp=snapshot` | Snapshot | +| PUT | | `comp=undelete` | Undelete | +| PUT | | `comp=tags` | SetTags | +| PUT | | `comp=tier` | SetTier | +| PUT | | `comp=expiry` | SetExpiry | +| PUT | | `comp=seal` | Seal | +| PUT | | `comp=rename` | Rename | +| PUT | `x-ms-page-write` | `comp=page` | Clear | + +### Queue storage + +Queue storage allows sending and receiving messages that may be read by any +client who has access to the storage account. Messages are sent to and received from queues. + +The [messaging spec](tracing-instrumentation-messaging.md) can +be used for instrumenting Queue storage, with the following additions superseding the messaging spec. + +A new span is created when there is a current transaction, and when a message is sent to a queue + +| APM field | Required? | Format | Notes | Example | +| --------- | --------- | ------ | ----- | ------- | +| `span.name` | yes | `AzureQueue to ` | Upper case Operation name | `AzureQueue SEND to queuename` | +| `span.type` | yes | `messaging` | | | +| `span.subtype` | yes | `azurequeue` | | | +| `span.action` | yes | `` | lower case | `send` | + +#### Span context fields + +| APM field | Required? | Format | Notes | Example | +| --------- | --------- | ------ | ----- | ------- | +| `context.destination.address` | yes | URL host | | `accountname.queue.core.windows.net` | +| `context.destination.service.resource` | yes | `azurequeue/` | | `azurequeue/queuename` | +| `context.service.target.type` | yes | `azurequeue` | | | +| `context.service.target.name` | yes | `` | | `queuename` | + +---- + +A new transaction is created when one or more messages are received from a queue + +| APM field | Required? | Format | Notes | Example | +| --------- | --------- | ------ | ----- | ------- | +| `transaction.name` | yes | `AzureQueue from ` | Upper case Operation name | `AzureQueue RECEIVE from queuename` | +| `transaction.type` | yes | `messaging` | | | + + +#### Transaction context fields + +| APM field | Required? | Format | Notes | Example | +| --------- | --------- | ------ | ----- | ------- | +| `context.service.framework` | yes | `AzureQueue` | | | + +#### Determining operations + +Azure service endpoints for queue storage have one of the following host names + +| Cloud | Azure Service Endpoint | +| ----- | ---------------------- | +| Azure Global | `.queue.core.windows.net` | +| [Azure Government](https://docs.microsoft.com/en-us/azure/azure-government/documentation-government-developer-guide) | `.queue.core.usgovcloudapi.net` | +| [Azure China](https://docs.microsoft.com/en-us/azure/china/resources-developer-guide) |`.queue.core.chinacloudapi.cn` | +| [Azure Germany](https://docs.microsoft.com/en-us/azure/germany/germany-developer-guide) | `.queue.core.cloudapi.de` | + +where `` is the name of the storage account. New Azure service endpoints may be introduced by Azure later. + +Rules derived from the [Queue service REST API reference](https://docs.microsoft.com/en-us/rest/api/storageservices/queue-service-rest-api) + +| URL | HTTP verb | HTTP headers | HTTP query string | Resulting Operation Name | +| --- | --------- | ---------- | ------------------- | ------------------------ | +| | DELETE | | | DELETE | +| ends with /messages | DELETE | | no `popreceipt` | CLEAR | +| | DELETE | | `popreceipt=` | DELETE | +| | GET | | `comp=list` | LISTQUEUES | +| | GET | | `comp=properties` | GETPROPERTIES | +| | GET | | `comp=stats` | STATS | +| | GET | | `comp=metadata` | GETMETADATA | +| | GET | | `comp=acl` | GETACL | +| | GET | | | RECEIVE | +| | GET | | `peekonly=true` | PEEK | +| | HEAD | | `comp=metadata` | GETMETADATA | +| | HEAD | | `comp=acl` | GETACL | +| | OPTIONS | | | PREFLIGHT | +| | POST | | | SEND | +| | PUT | | `comp=acl` | SETACL | +| | PUT | | `comp=properties` | SETPROPERTIES | +| | PUT | | | CREATE | +| | PUT | | `comp=metadata` | SETMETADATA | +| | PUT | | `popreceipt=` | UPDATE | + +### Table storage + +The Table service offers non-relational schema-less structured storage in the +form of tables. It is a popular service due to its cost-to-capability, though +messaging in recent years from Microsoft has deprecated Table storage and encouraged +the use of CosmosDB and its Table storage compatible API. + +Tables store data as a collection of entities, where an entity is a collection of properties. +Entities are similar to rows and properties are similar to columns. + +`` is the name of the resource in the form of the table, or the table and partition key and row key of an entity. For example, + +- tablename +- tablename(PartitionKey='partitionkey', RowKey='rowkey') + +`` is the leftmost subdomain of the domain of an Azure storage account endpoint. + +| APM field | Required? | Format | Notes | Example | +| --------- | --------- | ------ | ----- | ------- | +| `span.name` | yes | `AzureTable ` | Pascal case Operation name | `AzureTable Insert tablename` | +| `span.type` | yes | `storage` | | | +| `span.subtype` | yes | `azuretable` | | | +| `span.action` | yes | `` | Pascal case | `Insert` | + +#### Span context fields + +| APM field | Required? | Format | Notes | Example | +| --------- | --------- | ------ | ----- | ------- | +| `context.destination.address` | yes | URL host | | `accountname.table.core.windows.net` | +| `context.destination.service.resource` | yes | `azuretable/` | | `azuretable/accountname` | + +#### Determining operations + +Azure service endpoints for table storage have one of the following host names + +| Cloud | Azure Service Endpoint | +| ----- | ---------------------- | +| Azure Global | `.table.core.windows.net` | +| [Azure Government](https://docs.microsoft.com/en-us/azure/azure-government/documentation-government-developer-guide) | `.table.core.usgovcloudapi.net` | +| [Azure China](https://docs.microsoft.com/en-us/azure/china/resources-developer-guide) |`.table.core.chinacloudapi.cn` | +| [Azure Germany](https://docs.microsoft.com/en-us/azure/germany/germany-developer-guide) | `.table.core.cloudapi.de` | + +where `` is the name of the storage account. New Azure service endpoints may be introduced by Azure later. + +Rules derived from the [Table service REST API reference](https://docs.microsoft.com/en-us/rest/api/storageservices/table-service-rest-api) + +| URL | HTTP verb | HTTP headers | HTTP query string | Resulting Operation Name | Notes | +| --- | --------- | ---------- | ------------------- | ------------------------ | ----- | +| | PUT | | `comp=properties` | SetProperties | | +| | GET | | `comp=properties` | GetProperties | | +| | GET | | `comp=stats` | Stats | | +| ends with /Tables | GET | | | Query | | +| ends with /Tables | POST | | | Create | `` is in `request_body["TableName"]`| +| ends with /Tables('``') | DELETE | | | Delete | | +| ends with /`
` | OPTIONS | | | Preflight | | +| | HEAD | | `comp=acl` | GetAcl | | +| | GET | | `comp=acl` | GetAcl | | +| | PUT | | `comp=acl` | SetAcl | | +| ends with /`
`() or /`
`(PartitionKey='``',RowKey='``')` | GET | | | Query | | +| ends with /`
` | POST | | | Insert | | +| ends with /`
`(PartitionKey='``',RowKey='``')` | PUT | | | Update | | +| ends with /`
`(PartitionKey='``',RowKey='``')` | MERGE | | | Merge | | +| ends with /`
`(PartitionKey='``',RowKey='``')` | DELETE | | | Delete | | + +### File share storage + +Azure file service (known also as AFS or file share storage) is a managed file share service +that uses Server Message Block (SMB) protocol or Network File System (NFS) protocol +to provide general purpose file shares. File shares can be mounted concurrently by +machines in the cloud or on-premises. + +The `` is determined from the path of the URL. + +`` is the leftmost subdomain of the domain of an Azure storage account endpoint. + +| APM field | Required? | Format | Notes | Example | +| --------- | --------- | ------ | ----- | ------- | +| `span.name` | yes | `AzureFile ` | Pascal case Operation name | `AzureFile Create directoryname` | +| `span.type` | yes | `storage` | | | +| `span.subtype` | yes | `azurefile` | | | +| `span.action` | yes | `` | Pascal case | `Insert` | + +#### Span context fields + +| APM field | Required? | Format | Notes | Example | +| --------- | --------- | ------ | ----- | ------- | +| `context.destination.address` | yes | URL host | | `accountname.file.core.windows.net` | +| `context.destination.service.resource` | yes | `azurefile/` | | `azurefile/accountname` | + +#### Determining operations + +Azure service endpoints for file share storage have one of the following host names + +| Cloud | Azure Service Endpoint | +| ----- | ---------------------- | +| Azure Global | `.file.core.windows.net` | +| [Azure Government](https://docs.microsoft.com/en-us/azure/azure-government/documentation-government-developer-guide) | `.file.core.usgovcloudapi.net` | +| [Azure China](https://docs.microsoft.com/en-us/azure/china/resources-developer-guide) |`.file.core.chinacloudapi.cn` | +| [Azure Germany](https://docs.microsoft.com/en-us/azure/germany/germany-developer-guide) | `.table.file.cloudapi.de` | + +where `` is the name of the storage account. New Azure service endpoints may be introduced by Azure later. + +Rules derived from the [File service REST API reference](https://docs.microsoft.com/en-us/rest/api/storageservices/file-service-rest-api). + +| URL | HTTP verb | HTTP headers | HTTP query string | Resulting Operation Name | +| --- | --------- | ---------- | -------------------- | ------------------------- | +| | GET | | `comp=list` | List | +| | PUT | | `comp=properties` | SetProperties | +| | GET | | `comp=properties` | GetProperties | +| ends with /`` | OPTIONS | | | Preflight | +| | PUT | | | Create | +| | PUT | | `comp=snapshot` | Snapshot | +| | PUT | | `restype=share` and `comp=properties` | SetProperties | +| | GET | | `restype=share` | GetProperties | +| | HEAD | | `restype=share` | GetProperties | +| | GET | | `comp=metadata` | GetMetadata | +| | HEAD | | `comp=metadata` | GetMetadata | +| | PUT | | `comp=metadata` | SetMetadata | +| | DELETE | | | Delete | +| | PUT | | `comp=undelete` | Undelete | +| | HEAD | | `comp=acl` | GetAcl | +| | GET | | `comp=acl` | GetAcl | +| | PUT | | `comp=acl` | SetAcl | +| | GET | | `comp=stats` | Stats | +| | GET | | `comp=filepermission` | GetPermission | +| | PUT | | `comp=filepermission` |SetPermission | +| | PUT | | `restype=directory` | Create | +| | GET | | `comp=listhandles` | ListHandles | +| | PUT | | `comp=forceclosehandles` | CloseHandles | +| | GET | | | Download | +| | HEAD | | | GetProperties | +| | PUT | | `comp=range` | Upload | +| | PUT | `x-ms-copy-source` | | Copy | +| | PUT | `x-ms-copy-action:abort` | `comp=copy` | Abort | +| | GET | | `comp=rangelist` | ListRanges | +| | PUT | | `comp=lease` | Lease | + + +## Azure Service Bus + +Azure Service Bus is a message broker service. The [messaging spec](tracing-instrumentation-messaging.md) can +be used for instrumenting Azure Service Bus, but the follow specifications supersede those of the messaging spec. + +Azure Service Bus can use the following protocols + +- Advanced Message Queuing Protocol 1.0 (AMQP) +- Hypertext Transfer Protocol 1.1 with TLS (HTTPS) + +The offical Azure SDKs _generally_ use AMQP for sending and receiving messages. + +### Typing + +A new span is created when there is a current transaction, and when a message is sent or scheduled to a queue or topic + +| APM field | Required? | Format | Notes | Example | +| --------- | --------- | ------ | ----- | ------- | +| `span.name` | yes | `AzureServiceBus to ` | Upper case Operation name | `AzureServiceBus SEND to queuename` | +| `span.type` | yes | `messaging` | | | +| `span.subtype` | yes | `azureservicebus` | | | +| `span.action` | yes | `` | lower case | `send` | + +#### Span context fields + +| APM field | Required? | Format | Notes | Example | +| --------- | --------- | ------ | ----- | ------- | +| `context.destination.address` | yes | URL host | | `namespace.servicebus.windows.net` | +| `context.destination.service.resource` | yes | azureservicebus/``\|`` | | `azurequeue/myqueue`, `azureservicebus/mytopic` | +| `context.service.target.type` | yes | `azureservicebus` | | | +| `context.service.target.name` | yes | ``\|`` | | `mytopic` | + +---- + +A new transaction is created when one or more messages are received or receive deferred from a queue or topic subscription + +| APM field | Required? | Format | Notes | Example | +| --------- | --------- | ------ | ----- | ------- | +| `transaction.name` | yes | AzureServiceBus `` from ``\|`` | Upper case Operation name | `AzureServiceBus RECEIVE from queuename` | +| `transaction.type` | yes | `messaging` | | | + + +#### Transaction context fields + +| APM field | Required? | Format | Notes | Example | +| --------- | --------- | ------ | ----- | ------- | +| `context.service.framework` | yes | `AzureServiceBus` | | | + + +### Additional actions + +Azure Service Bus supports actions that should be traced in addition to `SEND` and `RECEIVE`: + +- `SCHEDULE` + + Message published to the bus, but not visible until some later point. + +- `RECEIVEDEFERRED` + + Message received from the bus, but marked as deferred on the bus, then later retrieved with receive deferred. + +### Naming + +Messages can be sent to queues and topics, and can be received from queues and topic subscriptions. + +Transaction and span names *should* follow these patterns: + +For send and schedule, + +- AzureServiceBus SEND|SCHEDULE to `` +- AzureServiceBus SEND|SCHEDULE to `` + +For receive and receive deferred, + +- AzureServiceBus RECEIVE|RECEIVEDEFERRED from `` +- AzureServiceBus RECEIVE|RECEIVEDEFERRED from ``/Subscriptions/`` diff --git a/specs/agents/tracing-instrumentation-db.md b/specs/agents/tracing-instrumentation-db.md new file mode 100644 index 00000000..837aa832 --- /dev/null +++ b/specs/agents/tracing-instrumentation-db.md @@ -0,0 +1,275 @@ + +## Table of Contents +* [Database and Datastore spans](#database-and-datastore-spans) +* [Specific Databases](#specific-databases) + * [AWS DynamoDb](#aws-dynamodb) + * [AWS S3](#aws-s3) + * [Cassandra](#cassandra) + * [Elasticsearch](#elasticsearch) + * [MongoDB](#mongodb) + * [Redis](#redis) + * [SQL Databases](#sql-databases) + +## Database and Datastore spans + +We capture spans for various types of database/data-stores operations, such as SQL queries, Elasticsearch queries, is commands, etc. +Database and datastore spans **must not have child spans that have a different `type` or `subtype`** within the same transaction (see [span-spec](tracing-spans.md)). + +The following fields are relevant for database and datastore spans. Where possible, agents should provide information for as many as possible of these fields. The semantics of and concrete values for these fields may vary between different technologies. See sections below for details on specific technologies. + +| Field | Description | Mandatory | +|-------|-------------|:---------:| +|`name`| The name of the exit database span. **The span name must have a low cardinality as it is used as a dimension for derived metrics!** Therefore, for SQL operations we perform a limited parsing of the statement, and extract the operation name and outer-most table involved. Other databases and storages may have different strategies for the span name (see specific databases and stores in the sections below).| :white_check_mark: | +|`type`|For database spans, the type should be `db`.| :white_check_mark:| +|`subtype`|For database spans, the subtype should be the database vendor name. See details below for specific databases.| :x: | +|`action`|The database action, e.g. `query`| +|
|
|
| +|`context.db.instance`| Database instance name, e.g. "customers". For DynamoDB, this is the region.| :x: | +|`context.db.statement`| Statement/query, e.g. `SELECT * FROM foo WHERE ...`. The full database statement should be stored in db.statement, which may be useful for debugging performance issues. We store up to 10000 Unicode characters per database statement. For Non-SQL data stores see details below.| :x: | +|`context.db.type`| Database type/category, which should be "sql" for SQL databases, and the lower-cased database name otherwise.| :x: | +|`context.db.user`| Username used for database access, e.g. `readonly_user`| :x: | +|`context.db.link`| Some SQL databases (e.g. Oracle) provide a feature for linking multiple databases to form a single logical database. The DB link differentiates single DBs of a logical database. See https://github.com/elastic/apm/issues/107 for more details. | :x: | +|`context.db.rows_affected`| The number of rows / entities affected by the corresponding db statement / query.| :x: | +|
|
|
| +|`context.destination.address`|The hostname / address of the database.| :x: | +|`context.destination.port`|The port under which the database is accessible.| :x: | +|`context.destination.service.resource`| DEPRECATED, replaced by `service.target.{type,name}` fields | :white_check_mark: mandatory when APM server ignores `service.target.{type,name}`, optional otherwise | +|`context.destination.service.type`| DEPRECATED, replaced by `service.target.{type,name}` fields | :x: | +|`context.destination.service.name`| DEPRECATED, replaced by `service.target.{type,name}` fields | :x: | +|`context.destination.cloud.region`| The cloud region in case the datastore is hosted in a public cloud or is a managed datasatore / database. E.g. AWS regions, such as `us-east-1` | :x: | +|
|
|
| +|`service.target.type`| Defines destination service type, the (`type`,`name`) pair replaces deprecated `context.destination.service.resource` | :white_check_mark:| +|`service.target.name`| Defines destination service name, the (`type`,`name`) pair replaces deprecated `context.destination.service.resource`, for relational databases, the value is equal to `context.db.instance` | :x: | + + +## Specific Databases + +### AWS DynamoDb + +| Field | Value / Examples | Comments | +|-------|:---------------:|----------| +|`name`| e.g. `DynamoDB UpdateItem my_table`| The span name should capture the operation name (as used by AWS for the action name) and the table name, if available. The format should be `DynamoDB `. TableName MAY be omitted from the name for operations (`batchWriteItem`, `batchGetItem`, PartiQL-related methods like `executeStatement` etc.) that are acting on more than a single table. If `TableName` is not available, agents SHOULD also check the `TableArn` or `SourceTableArn` query params for a table name and extract the table name from the AWS ARN value.| +|`type`|`db`| +|`subtype`|`dynamodb`| +|`action`| `query` | +| __**context.db._**__ |
|
| +|`_.instance`| e.g. `us-east-1` | The AWS region where the table is. | +|`_.statement`| e.g. `ForumName = :name and Subject = :sub` | For a DynamoDB Query operation, capture the KeyConditionExpression in this field. In order to avoid a high cardinality of collected values, agents SHOULD NOT include the full SQL statment for PartiQL-related methods like `executeStatment.| +|`_.type`|`dynamodb`| +|`_.user`| :heavy_minus_sign: | +|`_.link`| :heavy_minus_sign: | +|`_.rows_affected`| :heavy_minus_sign: | +| __**context.destination._**__ |
|
| +|`_.address`|e.g. `dynamodb.us-west-2.amazonaws.com`| +|`_.port`|e.g. `5432`| +|`_.service.name`| `dynamodb` | DEPRECATED +|`_.service.type`|`db`| DEPRECATED +|`_.service.resource`| `dynamodb`, `dynamodb/us-east-1`| DEPRECATED +|`_.cloud.region`| e.g. `us-east-1` | The AWS region where the table is, if available. | +| __**service.target._**__ |
|
| +|`_.type`| `dynamodb` || +|`_.name`| e.g. `us-east-1` | Use same value as `context.db.instance` | + +### AWS S3 + +| Field | Value / Examples | Comments | +|-------|:---------------:|----------| +|`name`| e.g. `S3 GetObject my-bucket`| The span name should follow this pattern: `S3 `. Note that the operation name is in PascalCase. | +|`type`|`storage`| +|`subtype`|`s3`| +|`action`| e.g. `GetObject` | The operation name in PascalCase. | +| __**context.destination._**__ |
|
| +|`_.address`|e.g. `s3.amazonaws.com`| Not available in some cases. Only set if the actual connection is available. | +|`_.port`|e.g. `443`| Not available in some cases. Only set if the actual connection is available. | +|`_.service.name`| `s3` | DEPRECATED, use `service.target.{type,name}` +|`_.service.type`|`storage`| DEPRECATED, use `service.target.{type,name}` +|`_.service.resource`| e.g. `s3/my-bucket`, `s3/accesspoint/myendpointslashes`, or `s3/accesspoint:myendpointcolons`| DEPRECATED, use `service.target.{type,name}` | +|`_.cloud.region`| e.g. `us-east-1` | The AWS region where the bucket is. | +| __**service.target._**__ |
|
| +|`_.type`| `s3` || +|`_.name`| e.g. `my-bucket`, `accesspoint/myendpointslashes`, or `accesspoint:myendpointcolons` | The bucket name, if available. The s3 API allows either the bucket name or an Access Point to be provided when referring to a bucket. Access Points can use either slashes or colons. When an Access Point is provided, the access point name preceded by accesspoint/ or accesspoint: should be extracted. For example, given an Access Point such as `arn:aws:s3:us-west-2:123456789012:accesspoint/myendpointslashes`, the agent extracts `accesspoint/myendpointslashes`. Given an Access Point such as `arn:aws:s3:us-west-2:123456789012:accesspoint:myendpointcolons`, the agent extracts `accesspoint:myendpointcolons`. | +| __**otel.attributes._**__ |
|
| +|`_["aws.s3.bucket"]`| `my-bucket` | The bucket name, if available. See [OTel Semantic Conventions](https://github.com/open-telemetry/opentelemetry-specification/blob/v1.20.0/semantic_conventions/trace/instrumentation/aws-sdk.yml#L435). Note: this must be a single dotted string key in the `otel.attributes` mapping -- for example `{"otel": {"attributes": {"aws.s3.bucket": "my-bucket"}}}` -- and *not* a nested object. | +|`_["aws.s3.key"]`| `my/key/path` | The S3 object key, if applicable. See [OTel Semantic Conventions](https://github.com/open-telemetry/opentelemetry-specification/blob/v1.20.0/semantic_conventions/trace/instrumentation/aws-sdk.yml#L435). Note: this must be a single dotted string key in the `otel.attributes` mapping and *not* a nested object. | + +### Cassandra + +| Field | Value / Examples | Comments | +|----------------------------------------|:-----------------------------------|---------------| +| `type` | `db` | | +| `subtype` | `cassandra` | | +| `context.db.instance` | e.g. `customers` | Keyspace name | +| `context.destination.service.resource` | `cassandra`, `cassandra/customers` | DEPRECATED | +| `service.target.type` | `cassandra` | | +| `service.target.name` | e.g. `customers` | Keyspace name | + +### Elasticsearch + +| Span Field | Value / Examples | Comments | +|------------|:----------------:|----------| +|`name`| e.g. `Elasticsearch: GET /index/_search` | The span name should be `Elasticsearch: ` | +|`type`|`db`| +|`subtype`|`elasticsearch`| +|`action`| `request` | +| __**context.db._**__ |
|
| +|`_.instance`| e.g. `my-cluster` | [Cluster name](#cluster-name), if available. +|`_.statement`| e.g.
{"query": {"match": {"user.id": "kimchy"}}}
| For Elasticsearch search-type queries, the request body SHOULD be recorded according to the `elasticsearch_capture_body_urls` option (see below). If the body is gzip-encoded, the body MUST be decoded first.| +|`_.type`|`elasticsearch`| +|`_.user`| :heavy_minus_sign: | +|`_.link`| :heavy_minus_sign: | +|`_.rows_affected`| :heavy_minus_sign: | +| __**context.http._**__ |
|
| +|`_.status_code`|`200`| +|`_.method`|`GET`| +|`_.url`|`https://localhost:9200/index/_search?q=user.id:kimchy`| +| __**context.destination._**__ |
|
| +|`_.address`|e.g. `localhost`| +|`_.port`|e.g. `5432`| +|`_.service.name`| `elasticsearch` | DEPRECATED, use `service.target.{type,name}` | +|`_.service.type`|`db`| DEPRECATED, use `service.target.{type,name}` | +|`_.service.resource`| `elasticsearch`, `elasticsearch/my-cluster` | DEPRECATED, use `service.target.{type,name}` | +| __**context.service.target._**__ |
|
| +|`_.type`| `elasticsearch` | | +|`_.name`| e.g. `my-cluster` | [Cluster name](#cluster-name), if available. + +In addition to [the usual error capture specification](./error-tracking.md), the following apply to errors captured for Elasticsearch client auto-instrumentation. + +| Error Field | Value / Examples | Comments | +|-------------|:----------------:|----------| +| `exception.type` | `$exceptionType` or `${exceptionType} (${esResponseBody.error.type})`, e.g. `ResponseError (index_not_found_exception)` | Some Elasticsearch client errors include an error [response body from Elasticsearch with a `error.type`](https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#common-options-error-options). For these errors, the APM agent MAY capture that type as in the given example. This helps with error grouping in the APM app. | + +#### Cluster name + +The Elasticsearch cluster name is not available in Elasticsearch clients. +When using an Elastic Cloud deployment, the name of the Elasticsearch cluster is provided by the `x-found-handling-cluster` HTTP response header. + +#### `elasticsearch_capture_body_urls` configuration + +The URL patterns for which the agent is capturing the request body for Elasticsearch clients. + +Agents MAY offer this configuration option. +If they don't, they MUST use a hard-coded list of URLs that correspond with the default value of this option. + +| | | +|----------------|------------| +| Type | `List<`[`WildcardMatcher`](../../tests/agents/json-specs/wildcard_matcher_tests.json)`>` | +| Default | `*/_search, */_search/template, */_msearch, */_msearch/template, */_async_search, */_count, */_sql, */_eql/search`| +| Central config | `false` + +### MongoDB + +| Field | Value / Examples | Comments | +|-------|:---------------:|----------| +|`name`| e.g. `users.find` | The name for MongoDB spans should be the command name in the context of its collection/database. | +|`type`|`db`| +|`subtype`|`mongodb`| +|`action`|e.g. `find` , `insert`, etc.| The MongoDB command executed with this action. | +| __**context.db._**__ |
|
| +|`_.instance`| e.g. `customers` | Database name, if available | +|`_.statement`| e.g.
find({status: {$in: ["A","D"]}})
| The MongoDB command encoded as MongoDB Extended JSON, if the command name matches `mongodb_capture_statement_commands`. | +|`_.type`|`mongodb`| +|`_.user`| :heavy_minus_sign: | +|`_.link`| :heavy_minus_sign: | +|`_.rows_affected`| :heavy_minus_sign: | +| __**context.destination._**__ |
|
| +|`_.address`|e.g. `localhost`| +|`_.port`|e.g. `5432`| +|`_.service.name`| `mongodb` | DEPRECATED, use `service.target.{type,name}` | +|`_.service.type`|`db`| DEPRECATED, use `service.target.{type,name}` | +|`_.service.resource`| `mongodb/customers` | DEPRECATED, use `service.target.{type,name}` | +| __**service.target._**__ |
|
| +|`_.type`| `mongodb` | | +|`_.name`| e.g. `customers` | Database name, same as `db.instance` if available | + +#### `mongodb_capture_statement_commands` configuration + +This config var specifies the MongoDB command names for which the agent will capture the statement. +Agents that support capturing MongoDB statements MUST implement this option. + +| | | +|----------------|--| +| Type | `List<`[`WildcardMatcher`](../../tests/agents/json-specs/wildcard_matcher_tests.json)`>` | +| Default | `"find,aggregate,count,distinct,mapReduce"` | +| Central config | `false` | + +Examples: +- capture for all commands: `*` (This should be discouraged, because it can lead to capturing sensitive information in `insert` and `update` commands.) +- capture common read commands: `find,aggregate,count,distinct,mapReduce` +- capture no statement : `""` (empty) + +### Redis + +| Field | Value / Examples | Comments | +|-------|:---------------:|----------| +|`name`| e.g. `GET` or `LRANGE` | The name for Redis spans can simply be set to the command name. | +|`type`|`db`| +|`subtype`|`redis`| +|`action`| `query` | +| __**context.db._**__ |
|
| +|`_.instance`| :heavy_minus_sign: | +|`_.statement`| :heavy_minus_sign: | +|`_.type`|`redis`| +|`_.user`| :heavy_minus_sign: | +|`_.link`| :heavy_minus_sign: | +|`_.rows_affected`| :heavy_minus_sign: | +| __**context.destination._**__ |
|
| +|`_.address`|e.g. `localhost`| +|`_.port`|e.g. `5432`| +|`_.service.name`| `redis` |DEPRECATED, use `service.target.{type,name}` | +|`_.service.type`|`db`|DEPRECATED, use `service.target.{type,name}` | +|`_.service.resource`| `redis` |DEPRECATED, use `service.target.{type,name}` | +| __**service.target._**__ |
|
| +|`_.type`| `redis` | | +|`_.name`| :heavy_minus_sign: | | + +### SQL Databases + +| Field | Common values / patterns for all SQL DBs | Comments | +|-------|:---------------:|---------------| +|`name`| e.g. `SELECT FROM products` | For SQL operations we perform a limited parsing the statement, and extract the operation name and outer-most table involved (if any). See more details [here](https://docs.google.com/document/d/1sblkAP1NHqk4MtloUta7tXjDuI_l64sT2ZQ_UFHuytA). | +|`type`|`db`| +|`subtype`| e.g. `oracle`, `mysql` | [see below](#database-subtype) | +|`action`| e.g. `query`, `connect`, `ping`, `prepare`, `exec` | +| __**context.db._**__ |
|
| +|`_.instance`| e.g. `instance-name`| [see below](#database-instance) | +|`_.statement`| e.g. `SELECT * FROM products WHERE ...`| The full SQL statement. We store up to 10000 Unicode characters per database statement. | +|`_.type`|`sql`| +|`_.user`| e.g. `readonly_user`| +|`_.rows_affected`| e.g. `123`| +| __**context.destination._**__ |
|
| +|`_.address`|e.g. `localhost`| +|`_.port`|e.g. `5432`| +|`_.service.type`|`db`| DEPRECATED, use `service.target.{type,name}` | +|`_.service.resource`| |DEPRECATED, use `service.target.{type,name}` | +| __**service.target._**__ |
|
| +|`_.type`| e.g. `mysql` | Same value as `subtype` | +|`_.name`| | Database name, same as `db.instance` if available | + +#### Database subtype + +| Database name | `subtype` | +|:---------------------|:-----------:| +| MySQL | `mysql` | +| MariaDB | `mariadb` | +| PostgreSQL | `postgresql`| +| Microsoft SQL server | `mssql` | +| Oracle | `oracle` | +| IBM Db2 | `db2` | + +#### Database instance + +For most relational databases, the value of `db.instance` should map to the concept of "current database". +When no database selected, for example when creating a database, this field should be omitted. + +While the semantics may vary across vendors, the goal here is to have a single string that can be used for correlation, +it is thus important to be able to get the same value across all agents. + +There are multiple ways to capture it, agents SHOULD attempt to capture it with the following priorities: +1. Parsing the database connection string: parsing can be complex, no runtime impact, +2. Querying connection metadata at runtime: acceptable as fallback, might trigger extra SQL queries, require caching to minimize overhead + +For most databases, the `database` parameter of the connection string should be available. For those that implement the [`INFORMATION_SCHEMA`](https://en.wikipedia.org/wiki/Information_schema) standard, it should be included in the values returned by `SELECT schema_name FROM information_schema.schemata`; + +**Oracle** : Use instance as defined in [Oracle DB instances](https://docs.oracle.com/cd/E11882_01/server.112/e40540/startup.htm#CNCPT005), the instance name should be the same as retrieved through `SELECT sys_context('USERENV','INSTANCE_NAME') AS Instance`. When multiple identifiers are available, the following priotity should be applied (first available wins): `INSTANCE_NAME`, `SERVICE_NAME`, `SID`. + +**MS SQL** : Use instance as defined in [MS SQL instances](https://docs.microsoft.com/en-us/sql/database-engine/configure-windows/database-engine-instances-sql-server?view=sql-server-ver15) diff --git a/specs/agents/tracing-instrumentation-graphql.md b/specs/agents/tracing-instrumentation-graphql.md new file mode 100644 index 00000000..01d494d0 --- /dev/null +++ b/specs/agents/tracing-instrumentation-graphql.md @@ -0,0 +1,153 @@ +## GraphQL transactions and spans + +**NB:** This document is not guaranteed to be final. + +### Problems with our current approach + +Grouping transactions by HTTP method and path fits perfectly for REST-style APIs and the like. + +With a GraphQL API, the client always requests the same endpoint. +This means queries and mutations with very different costs and consequences all end up in the same transaction group. +Spans are likewise impossible to tell apart. + +This document describes and explains a common approach to better support GraphQL. + +Example GraphQL query: + +```graphql +{ + user(id: "99") { + name + comments { + body + } + } +} +``` + +Turns into an HTTP POST request like so: + +```plain +POST /graphql +Content-Type: application/json + +{ + "query": "query ($id: ID!) { + user(id: $id) { + name + comments { + body + } + } + }", + "variables": { "id": "99" } +} +``` + +**Sidenote:** The Node.js agent already supports GraphQL. This document is written with that in mind but not necessarily with its implementation as a target result. + +### Transactions + +#### Prefix + +To distinguish GraphQL transactions from others we prefix them with `GraphQL:`. + +#### Operation Name + +It is common (and [recommended](https://graphql.org/learn/queries/#operation-name)) to provide an _Operation Name_ for queries. Here for example `UserWithComments`: + +```graphql +query UserWithComments { + user { + id + name + comments { + body + } + } +} +``` + +The point of these are to provide an easy way for the developers, when things go wrong, to pinpoint where exactly they did so. + +This name is available on the server too and serves as a great distinguishing key. + +Transaction name examples: +- `GraphQL: UserWithComments` +- `GraphQL: UpdateUser` + +##### Sidenote: Multiple endpoints + +The Node.js implementation adds the request path to the GraphQL span names. + +We do not find serving multiple endpoints and using them with the same Operation Names likely enough to add it to this document. + +#### Anonymous queries + +An Operation Name isn't required. When one isn't provided it's hard for us to tell apart the queries. + +We considered hashing queries to tell them apart, but decided against it. +Instead we will just consider all unnamed queries _unnamed_ and consequently put them in the same bucket. + +
+Rationale: + +1. Some clients generate `id`s from hashing the contents of the query (see [apollo-tooling](https://github.com/apollographql/apollo-tooling/blob/1dfd737eaf85b89b2cfb13913342e091e3c03d18/packages/apollo-codegen-core/src/compiler/visitors/generateOperationId.ts#L5)). This would split the anonymous queries into separate buckets. + + A problem with this approach is that a user of the APM UI has no way to recognise queries in the transactions list before clicking through. + + Using just the `id` will not reveal the true culprit since there can be variables associated with the query. Different values for the variables can lead to very different workloads and response times. + +2. Another approach is to simply label them `[unnamed]`. + + A problem with _that_ approach is that the contents and thereby the relevant db queries and other sub-span actions that the server might do while resolving these queries may be wildly different making it hard to provide a _true_ sample waterfall. + + These two examples for example will look the same for the top-level GraphQL spans but will represent significantly different workloads. + + ``` + [- anonymous graphql span --------------] + [- 1,000x SELECT * ---------------] + [- 1,000 more SELECT * -] + + [- anonymous graphql span --------------] + [- SELECT id FROM users WHERE id=? -] + ``` + +No one of these are perfect. Because the benefits of using `id`s in the worst case could be misleading anyway, we're going with option 2. +
+ +To further help and nudge developers to use Operation Names for their queries, a tooltip will be shown in the Kibana UI, suggesting to use Operation Names. + +Transaction name examples: +- `GraphQL: [unnamed]` + +#### Batching/Multiplexing queries + +Some clients and servers allow batching/multiplexing queries (see for example [apollo-link-batch-http](https://www.apollographql.com/docs/link/links/batch-http/#gatsby-focus-wrapper) or [dataloader](https://github.com/graphql/dataloader#batching)) allowing multiple queries to be run from the same HTTP request. + +If multiple queries are run from the same request, we join their operation names in the transaction name with a `+`. + +Transaction name examples: +- `GraphQL: UserWithComments+PostWithSiblings+MoreThings+[unnamed]` + +To avoid very long transaction names, if a request has more than five queries, we abbreviate it to `[more-than-five-queries]`. + +Transaction name examples: +- `GraphQL: [more-than-five-queries]` + +### Spans + +The life cycle of responding to a GraphQL query is mostly split in two parts. First, the quer(y/ies) are read, parsed and analyzed. Second, they are executed. + +![Example GraphQL waterfall](https://user-images.githubusercontent.com/2163464/90007930-dfbd7b00-dc9b-11ea-8729-5d0237a0d6a4.png) + +This example shows the server responding to one request, with two named queries, `SKUs` and `Names`. + +As each language's server implementation can vary slightly in phases and their names, these might be named differently between agents. + +GraphQL spans have the following parameters: + +- `name: graphql.[action]` +- `type: "app"` +- `subtype: "graphql"` +- `action: [action]` diff --git a/specs/agents/tracing-instrumentation-grpc.md b/specs/agents/tracing-instrumentation-grpc.md new file mode 100644 index 00000000..410aacd8 --- /dev/null +++ b/specs/agents/tracing-instrumentation-grpc.md @@ -0,0 +1,92 @@ +## gRPC support in agents + +### Header + +#### Value format +The value format of the header is text, as other vendors use text and there's no advantage to using a binary encoding. See technical details about the gRPC header [here](https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests). + +#### Key Names +The header key names are `elastic-apm-traceparent` (for backwards compatibility with older agents) and `traceparent`. + +### Instrumented calls +Server and Client Unary request/response calls are instrumented. Support for other calls may be added later (i.e. client/server streaming, bidirectional streaming). + +### Transaction/Span context schemas + +#### Transaction context + +* **name**: \, ex: `/helloworld.Greeter/SayHello` +* **type**: `request` +* **trace_context**: \ +* **result**: [\](https://github.com/grpc/grpc/blob/master/doc/statuscodes.md#status-codes-and-their-use-in-grpc), ex: `OK` +* **outcome**: See [Outcome](#outcome) + +#### Span context + +Note that the destination fields are optional as some gRPC libraries don't expose host and port information. +See [apm#180](https://github.com/elastic/apm/issues/180) and [apm#115](https://github.com/elastic/apm/issues/115) for details on `destination` fields. + +* **name**: \, ex: `/helloworld.Greeter/SayHello` +* **type**: `external` +* **subtype**: `grpc` +* **outcome**: See [Outcome](#outcome) +* **destination**: + * **address**: Either an IP (v4 or v6) or a host/domain name. + * **port**: A port number; Should report default ports. + * **service**: + * **resource**: Capture host, and port. + * **name**: Capture the scheme, host, and non-default port. + * **type**: Same as `span.type` + +#### Outcome + +With gRPC, transaction and span outcome is set from gRPC response status. + +If such status is not available, then we default to the following: + +- `failure` if an error is reported +- `success` otherwise + +According to the [gRPC status codes reference spec](https://github.com/grpc/grpc/blob/master/doc/statuscodes.md), some +statuses are not used by gRPC client & server, thus some of them should be considered as client-side errors. + +The gRPC `UNKNOWN` status refers to an error that is not known, thus we should treat it as a `failure` and NOT map it to +an `unknown` outcome. + +For gRPC spans (from the client): + +- `OK` : `success` +- anything else: `failure` + +For gRPC transactions (from the server): + +This mapping can be quite subjective, as we know that some statuses are not used by the gRPC server & client +implementations and thus their meaning would be application specific. However, we attempt to report as `failure` +outcomes errors that might require attention from the server point of view and report as `success` all the statuses +that are only relevant on the client-side. + +| status | outcome | justification | +| ------------------------- | --------- | ------------------------------------------------ | +| `OK` | `success` | | +| `CANCELLED` | `success` | Operation cancelled by client | +| `UNKNOWN` | `failure` | Error of an unknown type, but still an error | +| `INVALID_ARGUMENT` (*) | `success` | Client-side error | +| `DEADLINE_EXCEEDED` | `failure` | | +| `NOT_FOUND` (*) | `success` | Client-side error (similar to HTTP 404) | +| `ALREADY_EXISTS` (*) | `success` | Client-side error (similar to HTTP 409) | +| `PERMISSION_DENIED` (*) | `success` | Client authentication (similar to HTTP 403) | +| `RESOURCE_EXHAUSTED` (*) | `failure` | Likely used for server out of resources | +| `FAILED_PRECONDITION` (*) | `failure` | Similar to UNAVAILABLE | +| `ABORTED` (*) | `failure` | Similar to UNAVAILABLE | +| `OUT_OF_RANGE` (*) | `success` | Client-side error (similar to HTTP 416) | +| `UNIMPLEMENTED` | `success` | Client called a non-implemented feature | +| `INTERNAL` | `failure` | Internal error (similar to HTTP 500) | +| `UNAVAILABLE` | `failure` | Transient error, client may retry with backoff | +| `DATA_LOSS` (*) | `failure` | Lost data should always be reported | +| `UNAUTHENTICATED` (*) | `success` | Client-side authentication (similar to HTTP 401) | + +The statuses marked with (*) are not used by gRPC libraries and thus their actual meaning is contextual to the +application. + +Also, the gRPC status code for a given transaction should be reported in the `transaction.result` field, thus we still have the +capability to detect an abnormal rate of a given status, in a similar way as we do with HTTP 4xx and 5xx errors. diff --git a/specs/agents/tracing-instrumentation-http.md b/specs/agents/tracing-instrumentation-http.md new file mode 100644 index 00000000..ff1713f3 --- /dev/null +++ b/specs/agents/tracing-instrumentation-http.md @@ -0,0 +1,90 @@ +## HTTP Transactions + +Agents should instrument HTTP request routers/handlers, starting a new transaction for each incoming HTTP request. When the request ends, the transaction should be ended, recording its duration. + +- The transaction `type` should be `request`. +- The transaction `result` should be `HTTP Nxx`, where N is the first digit of the status code (e.g. `HTTP 4xx` for a 404) +- The transaction `outcome` is set from response status code (see [Outcome](#outcome)) + + As there's no browser API to get the status code of a page load, the RUM agent always reports `"unknown"` for those transactions. +- The transaction `name` should be aggregatable, such as the route or handler name. Examples: + - `GET /users/{id}` + - `UsersController#index` + +It's up to you to pick a naming scheme that is the most natural for the language or web framework you are instrumenting. + +In case a name cannot be automatically determined, and a custom name has not been provided by other means, the transaction should be named ` unknown route`, e.g. `POST unknown route`. This would normally also apply to requests to unknown endpoints, e.g. the transaction for the request `GET /this/path/does/not/exist` would be named `GET unknown route`, whereas the transaction for the request `GET /users/123` would still be named `GET /users/{id}` even if the id `123` did not match any known user and the request resulted in a 404. + +In addition to the above properties, HTTP-specific properties should be recorded in the transaction `context`, for sampled transactions only. Refer to the [Intake API Transaction](https://www.elastic.co/guide/en/apm/server/current/transaction-api.html) documentation for a description of the various context fields. + +By default request bodies are not captured. It should be possible to configure agents to enable their capture using the config variable `ELASTIC_APM_CAPTURE_BODY`. By default agents will capture request headers, but it should be possible to disable their capture using the config variable `ELASTIC_APM_CAPTURE_HEADERS`. + +Captured request and response headers, cookies, and form bodies MUST be sanitised (i.e. secrets removed) according to [data sanitization rules](sanitization.md#data-sanitization). + + +### `transaction_ignore_urls` configuration + +Used to restrict requests to certain URLs from being instrumented. + +This property should be set to a list containing one or more strings. +When an incoming HTTP request is detected, +its request [`path`](https://tools.ietf.org/html/rfc3986#section-3.3) +will be tested against each element in this list. +For example, adding `/home/index` to this list would match and remove instrumentation from the following URLs: + +``` +https://www.mycoolsite.com/home/index +http://localhost/home/index +http://whatever.com/home/index?value1=123 +``` + +| | | +|----------------|---| +| Type | `List<`[`WildcardMatcher`](../../tests/agents/json-specs/wildcard_matcher_tests.json)`>` | +| Default | agent specific | +| Dynamic | `true` | +| Central config | `true` | + +### `transaction_ignore_user_agents` configuration + +Used to restrict requests made by certain User-Agents from being instrumented. + +This property should be set to a list containing one or more strings. +When an incoming HTTP request is detected, the `User-Agent` request headers will be tested against each element in this list and if a match is found, no trace will be captured for this request. + +| | | +|----------------|---| +| Type | `List<`[`WildcardMatcher`](../../tests/agents/json-specs/wildcard_matcher_tests.json)`>` | +| Default | `` | +| Dynamic | `true` | +| Central config | `true` | + +## HTTP client spans + +We capture spans for outbound HTTP requests. These should have a type of `external`, and subtype of `http`. The span name should have the format ` `. + +For outbound HTTP request spans we capture the following http-specific span context: + +- `http.url` (the target URL) \ + The captured URL should have the userinfo (username and password), if any, redacted. +- `http.status_code` (the response status code) +- `outcome` is set from response status code (see [Outcome](#outcome) for details) + +## Outcome + +For HTTP transactions (from the server perspective), status codes in the 4xx range (client errors) are not considered +a `failure` as the failure has not been caused by the application itself but by the caller. + +For HTTP spans (from the client perspective), the span's `outcome` should be set to `"success"` if the status code is +lower than 400 and to `"failure"` otherwise. + +For both transactions and spans, if there is no HTTP status we set `outcome` from the reported error: + +- `failure` if an error is reported +- `success` otherwise + +## Destination + +- `context.destination.address`: `url.host` +- `context.destination.port`: `url.port` +- `context.destination.service.*`: See [destination spec](tracing-spans-destination.md) diff --git a/specs/agents/tracing-instrumentation-messaging.md b/specs/agents/tracing-instrumentation-messaging.md new file mode 100644 index 00000000..4f4923a0 --- /dev/null +++ b/specs/agents/tracing-instrumentation-messaging.md @@ -0,0 +1,225 @@ +# Messaging Systems + +The instrumentation of messaging systems includes: + +- Transaction creation for received message processing, either as a child +of the message sending span if the sending operation is traced (meaning +distributed tracing support), or as root +- Span creation for message sending operation +- Span creation for message *receiving operation that occurs within a traced transaction* + +## Message sending/publishing + +A Message send/publish SHOULD be captured as a `messaging` span only if +occurring within a traced transaction. + +![publish](uml/publish.svg) + +### Sending Trace Context + +If the messaging system exposes a mechanism for sending additional [message metadata](#message-metadata), it +SHOULD be used to propagate the [Trace Context](https://www.w3.org/TR/trace-context/) of the +`messaging` span, to continue the [distributed trace](tracing-distributed-tracing.md). + +## Message reception/consumption + +Message reception can be divided into two types, **passive** and **active**. + +### Passive message reception + +Passive message reception typically involves a listener or callback subscribed +to a queue/topic/subscription that is called once a message or batch of messages +is available, and passed the message or batch of messages for processing. + +Passive reception SHOULD be captured as a `messaging` transaction that starts when +a message or batch of messages is received for processing, and ends when the +processing of the message or batch of messages finishes. + +### Active message reception + +Active message reception typically involves initiating the reception of a +message or batch of messages from a queue/topic, on demand. This might be a +blocking or non-blocking operation, and for the purposes of this spec, is +collectively referred to as polling. + +When message polling is performed within a traced transaction, it SHOULD be +captured in a `messaging` span with a name indicating a poll action. A +span SHOULD be created irrespective of whether the operation returns any messages. + +If message polling is performed when there is no active transaction, it +can be the initiating event for a message processing flow. Our goal in this +scenario is to capture the message processing flow as a `messaging` transaction +that SHOULD start after the polling operation exits **and** MUST only be started +if the polling operation returned a message or batch of messages. Capturing +the message processing flow in a transaction in this manner may be unfeasible if +the processing flow is not implemented within a well defined API. + +### Batch message processing + +When active or passive message reception results in receiving a batch of messages, +a `messaging` transaction SHOULD be started and ended for the processing +of each message in the batch, if possible. For example, the Java agent +instruments the Kafka consumer batch iterator so that a transaction is started whenever +a message is retrieved from the batch, and ended either when the next message +is retrieved, or when the iterator is depleted i.e. `iterator.hasNext()` returns +`false`. + +If creating a transaction for the processing of each message in a batch is not possible, +the agent SHOULD create a single `messaging` transaction for the processing of the batch +of messages. + +### Receiving Trace Context + +This section applies to messaging systems that support [message metadata](#message-metadata). +The instrumentation of message reception SHOULD check message metadata for the +presence of [Trace Context](https://www.w3.org/TR/trace-context/). + +When single message reception is captured as a `messaging` transaction, +and a Trace Context is present, it SHOULD be used as the parent of the `messaging` transaction +to continue the [distributed trace](tracing-distributed-tracing.md). + +Otherwise (a single message being captured as a `messaging` span, or a batch +of messages is processed in a single `messaging` transaction or span), a +[span link](span-links.md) SHOULD be added for each message with Trace Context. +This includes the case where the size of the batch of received messages is one. + +The number of events processed for trace context SHOULD be limited to a maximum +of 1000, as a guard on agent overhead for extremely large batches of events. +(For example, SQS's maximum batch size is 10000 messages. The maximum number of +span links that could be sent for a single transaction/span to APM server with +the default configuration is approximately 4000: 307200 bytes +[APM server `max_event_size` default](https://www.elastic.co/guide/en/apm/server/current/configuration-process.html#max_event_size) +/ 77 bytes per serialized span link.) + +### Examples + +#### Kafka consumer + +A Kafka consumer consumes data in a loop from one or more topics, as part of a +consumer group. A `Consume()` method retrieves records one-by-one for processing. +Upon entering a consume loop, an APM agent SHOULD check if there is an active +`messaging` transaction for consumption, and end it. Following a `Consume()` +method returning a message, an APM agent SHOULD start a `messaging` transaction +to capture the message handling flow. Upon exiting a consume loop, an APM agent SHOULD check if there is an active `messaging` transaction for consumption, and end it. + +![Kafka consume](uml/kafka_consume.svg) + + +### Typing + +- Transactions: + - `transaction.type`: `messaging` +- Spans: + - `span.type`: `messaging` + - `span.subtype`: depends on service/provider, see table below + - `span.action`: `send`, `receive` or `poll` + +| `subtype` | Description | +| ------------------- | --------------------------------- | +| `azurequeue` | Azure Queue | +| `azureservicebus` | Azure Service Bus | +| `jms` | Java Messaging Service | +| `kafka` | Apache Kafka | +| `rabbitmq` | RabbitMQ | +| `sns` | AWS Simple Notification Service | +| `sqs` | AWS Simple Queue Service | + +### Naming + +Transaction and span names *should* follow this pattern: ` SEND/RECEIVE/POLL to/from `. +Examples: +- `JMS SEND to MyQueue` +- `RabbitMQ RECEIVE from MyQueue`** +- `RabbitMQ POLL from MyExchange`** + +Agents may deviate from this pattern, as long as they ensure a proper cardinality is maintained, that is- neither too low nor too high. +For example, agents may choose to name all transactions/spans reading-from/sending-to temporary queues equally. +On the other end, agents may choose to append a cardinality-increasing factor to the name, like the routing key in RabbitMQ. + +\* At least up to version 1.19.0, the Java agent's instrumentation for Kafka does not follow this pattern. + +#### \** RabbitMQ naming specifics + +In RabbitMQ, queues are only relevant in the receiver side, so the exchange name is used instead for sender spans. +When the default exchange is used (denoted with an empty string), it should be replaced with ``. + +Agents may add an opt-in config to append the routing key to the name as well, for example: `RabbitMQ RECEIVE from MyExchange/58D7EA987`. + +For RabbitMQ transaction and polling spans, the queue name is used, whenever available (i.e. when the polling yields a message). +RabbitMQ broker can generate a unique queue name on behalf of an application, which conforms to a naming convention starting with +`amq.gen-`. A generated queue name MUST be replaced with `amq.gen-*` when used in a span name, to reduce cardinality. + +### Context fields + +- **`context.message.queue.name`**: optional for `messaging` spans and transactions. Indexed as keyword. Wherever the broker terminology +uses "topic", this field will contain the topic name. In RabbitMQ, whenever the queue name is not available, use exchange name instead. +- **`context.message.age.ms`**: optional for message/record receivers only (transactions or spans). +A numeric field indicating the message's age in milliseconds. Relevant for transactions and +`receive` spans that receive valid messages. There is no accurate definition as to how this is calculated. If the messaging framework +provides a timestamp for the message, agents may use it (subtract the message/record timestamp from the read timestamp). +If a timestamp is not available, agents should omit this field or find an alternative and document it in this spec. For example, the +sending agent can add a timestamp to the message's metadata to be retrieved by the receiving agent. +Clock skews between agents are ignored, unless the calculated age (receive-timestamp minus send-timestamp) is negative, in which case the +agent should report 0 for this field. +- **`context.message.routing_key`**: optional. Use only where relevant. Currently only RabbitMQ. + +#### Transaction context fields + +- **`context.message.body`**: similar to HTTP requests' `context.request.body`- only fill in messaging-related **transactions** (ie +incoming messages creating a transaction) and not for outgoing messaging spans. + - Capture only when `ELASTIC_APM_CAPTURE_BODY` config is set to `'all'` or `'transactions'`. + - Only capture UTF-8 encoded message bodies. + - Limit size to 10000 characters. If longer than this size, trim to 9999 and append with ellipsis +- **`context.message.headers`**: similar to HTTP requests' `context.request.headers`- only fill in messaging-related **transactions**. + - Capture only when `ELASTIC_APM_CAPTURE_HEADERS` config is set to `true`. + - Sanitize headers with keys configured through `ELASTIC_APM_SANITIZE_FIELD_NAMES`. + - Intake: key-value pairs, same like `context.request.headers`. +- **`context.service.framework.name`**: same as `span.subtype`, but not in lowercase, e.g. `Kafka`, `RabbitMQ` + +#### Span context fields + +- **`context.destination.address`**: optional. Not available in some cases. Only set if the actual connection is available. +- **`context.destination.port`**: optional. Not available in some cases. Only set if the actual connection is available. +- **`context.destination.service.resource*`**: mandatory. Value should be either `${span.subtype}/${context.mesage.queue.name}` or `${span.subtype}`. +- **`context.destination.service.type*`**: deprecated but mandatory until 7.14.0. See [destination spec](tracing-spans-destination.md) for details. +- **`context.destination.service.name`**: deprecated but mandatory until 7.14.0. See [destination spec](tracing-spans-destination.md) for details. +- **`context.service.target.type`**: mandatory. Same value as `span.subtype`. See [Service Target](tracing-spans-service-target.md) for details. +- **`context.service.target.name`**: optional. same value as `context.mesage.queue.name` if available. See [Service Target](tracing-spans-service-target.md) for details. + +### `ignore_message_queues` configuration + +Used to filter out specific messaging queues/topics/exchanges from being traced. + +This property should be set to a list containing one or more wildcard matcher strings. When set, sends-to and receives-from the specified +queues/topics/exchanges will be ignored. + +| | | +|----------------|---| +| Type | `List<`[`WildcardMatcher`](../../tests/agents/json-specs/wildcard_matcher_tests.json)`>` | +| Default | empty | +| Dynamic | `true` | +| Central config | `true` | + + +### Message metadata + +To support distributed tracing with automatic instrumentation, the messaging +system must provide a mechanism to add metadata/properties/attributes to +individual messages, akin to HTTP headers. If an APM agent supports +trace-context for a given messaging system, it MUST use the following mechanisms +so that cross-language tracing works: + +| Messaging system | Mechanism | +| ---------------------- | --------- | +| Azure Queue | No mechanism | +| Azure Service Bus | `Diagnostic-Id` [application property](https://docs.microsoft.com/en-us/dotnet/api/azure.messaging.servicebus.servicebusmessage.applicationproperties). See [this doc](https://docs.microsoft.com/en-us/azure/service-bus-messaging/service-bus-end-to-end-tracing). | +| Java Messaging Service | [Message Properties](https://docs.oracle.com/javaee/7/api/javax/jms/Message.html) | +| Apache Kafka | [Kafka Record headers](https://cwiki.apache.org/confluence/display/KAFKA/KIP-82%2B-%2BAdd%2BRecord%2BHeaders) using [binary trace context fields](tracing-distributed-tracing.md#binary-fields) | +| RabbitMQ | [Message Attributes](https://www.rabbitmq.com/tutorials/amqp-concepts.html#messages) (a.k.a. `AMQP.BasicProperties` in [Java API](https://www.rabbitmq.com/api-guide.html)) | +| AWS SQS | [SQS message attributes](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html), if within message attribute limits. See [AWS instrumentation](tracing-instrumentation-aws.md). | +| AWS SNS | [SNS message attributes](https://docs.aws.amazon.com/sns/latest/dg/sns-message-attributes.html), if within message attribute limits. See [AWS instrumentation](tracing-instrumentation-aws.md). | + + +### AWS messaging systems + +The instrumentation of [SQS](tracing-instrumentation-aws.md#sqs-simple-queue-service) and [SNS](tracing-instrumentation-aws.md#sns-aws-simple-notification-service) services generally follow this spec, with some nuances specified in the linked specs. diff --git a/specs/agents/tracing-sampling.md b/specs/agents/tracing-sampling.md new file mode 100644 index 00000000..e38a636f --- /dev/null +++ b/specs/agents/tracing-sampling.md @@ -0,0 +1,124 @@ +## Transaction sampling + +To reduce processing and storage overhead, transactions may be sampled by agents. +Sampling here refers to "head-based sampling". + +Head-based sampling is where a sampling decision is made at the root of the distributed trace, +before the details or outcome of the trace are known, +and propagated throughout the trace. + +### `transaction_sample_rate` configuration + +By default, all transactions will be sampled. +Agents can be configured to sample probabilistically, +by specifying a sampling probability in the range \[0,1\]. +e.g. + + - `1` means all transactions will be sampled (the default) + - `0` means no transactions will be sampled + - `0.5` means approximately 50% of transactions will be sampled + +The maximum precision of the sampling rate is `0.0001` (0.01%). +The sampling rate should be rounded half away from zero to 4 decimal places. +Values greater than `0` but less than `0.0001` should be rounded to 0.0001. + +e.g. + + 0.00001 -> 0.0001 + 0.55554 -> 0.5555 + 0.55555 -> 0.5556 + 0.55556 -> 0.5556 + +The implementation will look something like `math.Round(sampleRate*10000)/10000`. +It is recommended to do that calculation once rather than every time the sampling rate is queried. +This is to ensure we are consistent when [propagating](#propagation) the sampling rate through `tracestate`. + +| | | +|----------------|---------| +| Valid options | \[0,1\] | +| Type | `float` | +| Default | `1` | +| Dynamic | `true` | +| Central config | `true` | + +### Effect on metrics + +At the time of making a sampling decision, +the sampling rate must be recorded so that it can be associated with every transaction and span in the trace. +The sampling rate will be used by the server for scaling transaction and span metrics. + +Transaction metrics will be used by the UI to display transaction distributions and throughput, +from the perspective of the transaction's service (grouped by `service.name` and `transaction.name`). + +Span metrics will be used by the UI for inter-service request rates on service maps, +from the perspective of the requesting service (grouped by `service.name` and `destination.service.resource`). +These are also referred as edge metrics. + +The server will calculate metrics by measuring only events from sampled traces, +and scaling the metrics based on the sampling rate associated with each one. +For example if the sampling rate is 0.5, +then each sampled transaction and span would be counted twice in metrics aggregations. + +Metrics will be more accurate when the sampling rate is high. +With lower sampling rates the server is able to calculate representative, but less accurate, metrics. +If the sampling rate is 0 then no metrics will be calculated at all. + +When the sampling rate is available Agents MUST record the sampling rate on transactions and spans as `sample_rate`, e.g. + + {"transaction":{"name":"GET /","sample_rate":0.1,...}} + {"span":{"name":"SELECT FROM table","sample_rate":0.1,...}} + +See [Propagation section of this document](#propagation) for details about the case when the sampling rate is not available. + +For non-sampled transactions the `sample_rate` field MUST be set to 0, +to ensure non-sampled transactions are not counted in transaction metrics. +This is important to avoid double-counting, +as non-sampled transactions will be represented in metrics calculated from sampled transactions. + +When calculating transaction metrics, +if the `sample_rate` transaction field is missing, +the server will count each transaction (sampled and unsampled) as single events. +This is required for backwards compatibility with agents that do not send a sampling rate. + +The server will only calculate span metrics for newer agents that include `sample_rate` in spans, +as otherwise the representative counts will be incorrect for sampling rates less than 1. + +### Non-sampled transactions + +In the case where the server version is not known, agents MUST assume that the server version is pre 8.0 and keep +sending non-sampled transactions. + +Given the health-check call to APM server might not have returned or even fail when unsampled +transactions are started, this conservative behavior avoids discarding relevant data. + +#### Pre 8.0 +When connected to an APM Server < 8.0, both sampled and non-sampled transactions MUST be captured by Elastic APM agents. +Sampling controls how much data is captured for transactions: +sampled transactions have complete context recorded and include spans; +non-sampled transactions have limited context and no spans. + +For non-sampled transactions set the transaction attributes `sampled: false` and `sample_rate: 0`, and omit `context`. +No spans should be captured. + +#### Post 8.0 +When connected to an APM Server 8.0+, agents SHOULD NOT send non-sampled transactions, or capture spans for these transactions. +Sampled transactions MUST be captured by Elastic APM agents. + +### Propagation + +As mentioned above, the sampling decision must be propagated throughout the trace. +We adhere to the W3C Trace-Context spec for this, propagating the decision through trace-flags: https://www.w3.org/TR/trace-context/#sampled-flag + +In addition to propagating the sampling decision (boolean), agents MUST also propagate the sampling rate to ensure it is consistently attached to to all events in the trace. +This is achieved by adding an `s` attribute to our [`es` `tracestate` key](tracing-distributed-tracing.md#tracestate) with the value of the sampling rate. +e.g. + + tracestate: es=s:0.1,othervendor= + +As `tracestate` has modest size limits we must keep the size down. +This is ensured as the `transaction_sample_rate` configuration option has a maximum precision of 4 decimal places. + +For non-root transactions the agent MUST parse incoming `tracestate` headers to identify the `es` entry and extract the `s` attribute. +The `s` attribute value should be used to populate the `sample_rate` field of transactions and spans. +If there is no `tracestate` or no valid `es` entry with an `s` attribute, +then the agent MUST omit `sample_rate` from non-root transactions and their spans. diff --git a/specs/agents/tracing-spans-destination.md b/specs/agents/tracing-spans-destination.md new file mode 100644 index 00000000..ac4cf2d0 --- /dev/null +++ b/specs/agents/tracing-spans-destination.md @@ -0,0 +1,142 @@ +## Span destination + +The span destination information is relevant for exit spans and helps to identify the downstream service. +This information is used for the [service map](https://www.elastic.co/guide/en/kibana/current/service-maps.html), +the [dependencies table](https://www.elastic.co/guide/en/kibana/current/service-overview.html#service-span-duration) in the service overview, +and the [APM SIEM integration](https://www.elastic.co/blog/elastic-apm-7-6-0-released). + +### Destination service fields + +In `context.destination.service`, `_.name` and `_.type` fields are deprecated and replaced by `context.service.target.*` fields. +See [related specification](tracing-spans-service-target.md) for more details. + +The only field still required is `context.destination.service.resource` until APM server is able to infer it. + +#### Deprecated fields + +- `context.destination.service.name` : deprecated but still required in protocol, thus value should be an empty string `""`. +- `context.destination.service.type` : deprecated but still required in protocol, thus value should be an empty string `""`. + +Agents MUST NOT manually set these fields. +Agents MUST NOT offer non-deprecated public APIs to set them. + +The intake JSON spec until 7.14.0 requires the deprecated fields to be present if `context.destination.service.resource` is set. +Future versions of APM Server will remove the fields from the intake API and drop it if sent by agents. + +Agents MAY omit the deprecated fields when sending spans to an APM Server version >= 7.14.0 . +Otherwise, the field MUST be serialized as an empty string if `context.destination.service.resource` is set. +Both options result in the fields being omitted from the Elasticsearch document. + +#### Destination resource + +- `context.destination.service.resource` : + - ES field: `span.destination.service.resource` + - Identifies unique destinations for each service. + - value should be inferred from `context.service.target.*` fields + - required for compatibility with existing features (Service Map, Dependencies) that rely on it + - might become optional in the future once APM server is able to infer the value from `context.service.target.*` fields. + +Spans representing an external call MUST have `context.destination.service` information. +If the span represents a call to an in-memory database, the information SHOULD still be set. + +Agents SHOULD have a generic component used in all tests that validates that the destination information is present for exit spans. +Rather than opting into the validation, the testing should provide an opt-out if, +for whatever reason, the destination information can't or shouldn't be collected for a particular exit span. + +**Usage** + +Each unique resource will result in a node on the [service map](https://www.elastic.co/guide/en/kibana/current/service-maps.html). +Also, APM Server will roll up metrics based on the resource. +These metrics are currently used for the [dependencies table](https://www.elastic.co/guide/en/kibana/current/service-overview.html#service-span-duration) +on the service overview page. +There are plans to use the service destination metrics in the service map, too. + +The metrics are calculated based on the (head-based) sampled span documents that are sent to APM Server. +That's why agents have to send the [`sample_rate`](tracing-sampling.md#effect-on-metrics) +attribute for transactions and spans: +It is used by APM Server to extrapolate the service destination metrics based on the (head-based) sampled spans. + +**Cardinality** + +To avoid a huge impact on storage requirements for metrics, +and to not "spam" the service map with lots of fine-grained nodes, +the cardinality has to be kept low. +However, the cardinality should not be too low, either, +so that different clusters, instances, and queues can be displayed separately in the service map. + +The cardinality should be the same or higher as `span.destination.service.name`. +Higher, if there are individual sub-resources for a service, such as individual queues for a message broker. +Same cardinality otherwise. + +**API** + +Agents SHOULD offer a public API to set this field so that users can customize the value if the generic mapping is not +sufficient. If set to `null` or an empty value, agents MUST omit the `span.destination.service` field altogether, thus +providing a way to manually disable the automatic setting/inference of this field (e.g. in order to remove a node +from a service map or an external service from the dependencies table). +A user-supplied value MUST have the highest precedence, regardless if it was set before or after the automatic setting is invoked. + +**Value** + +For all [exit spans](tracing-spans.md#exit-spans), unless the `context.destination.service.resource` field was set by the user to `null` or an empty +string through API, agents MUST infer the value of this field based on properties that are set on the span. + +If no value is set to the `context.destination.service.resource` field, the logic for automatically inferring +it MUST be the following: + +```groovy +if (!span.context.service.target.name) + span.context.service.target.type +else if (!span.context.service.target.type) + span.context.service.target.name +else if (span.type == 'external') + // Special case for HTTP, gRPC, and other rpc.system spans: skip the + // "${service.target.type}/" prefix. + span.context.service.target.name +else + "${span.context.service.target.type}/${span.context.service.target.name}" +``` + +If an agent API was used to set the `context.destination.service.resource` to `null` or an empty string, agents MUST +omit the `context.destination.service` field from the reported span event. + +The inference of `context.destination.service.resource` SHOULD be implemented in a central place within the agent, +such as an on-span-end-callback or the setter of a dependant property, +rather than being implemented for each individual library integration/instrumentation. + +For specific technologies, the field MAY be set non-centrally. +However, updating the generic inference logic SHOULD be preferred, if feasible. +Setting the value within a specific library integration/instrumentation is perfectly fine if there's only one canonical library for it. +Examples: gRPC and cloud-provider specific backends. + +### Destination fields + +These fields are used within the APM/SIEM integration. +They don't play a role for service maps. + +Spans representing an external call SHOULD have `context.destination` information if it is easy to gather. + +Examples when the effort of capturing the address and port is not justified: +* When the underlying protocol-layer code is not readily available in the instrumented code. +* When the instrumentation captures the exit event, + but the actual client is not bound to a specific connection (e.g. a client that does load balancing). + +#### `context.destination.address` + +ES field: [`destination.address`](https://www.elastic.co/guide/en/ecs/current/ecs-destination.html#_destination_field_details) + +Address is the destination network address: hostname (e.g. `localhost`), FQDN (e.g. `elastic.co`), IPv4 (e.g. `127.0.0.1`) IPv6 (e.g. `::1`) + +Agents MAY offer a public API to set this field so that users can override the automatically discovered one. +This includes the ability to set `null` or empty value in order to unset the automatically-set value. +A user-supplied value MUST have the highest precedence, regardless of whether it was set before or after the automatic setting is invoked. + +#### `context.destination.port` + +ES field: [`destination.port`](https://www.elastic.co/guide/en/ecs/current/ecs-destination.html#_destination_field_details) + +Port is the destination network port (e.g. 443) + +Agents MAY offer a public API to set this field so that users can override the automnatically discovered one. +This includes the ability to set a non-positive value in order to unset the automatically-set value. +A user-supplied value MUST have the highest precedence, regardless of whether it was set before or after the automatic setting is invoked. diff --git a/specs/agents/tracing-spans-service-target.md b/specs/agents/tracing-spans-service-target.md new file mode 100644 index 00000000..4f49d787 --- /dev/null +++ b/specs/agents/tracing-spans-service-target.md @@ -0,0 +1,248 @@ +## Span service target + +The span service target fields replace the `span.destination.service.*` fields that are deprecated. + +However, it does not replace [destination fields](https://www.elastic.co/guide/en/ecs/current/ecs-destination.html) +which are used for APM/SIEM integration and focus on network-level attributes. + +- [span.context.destination.address](https://www.elastic.co/guide/en/ecs/current/ecs-destination.html#field-destination-address) +- [span.context.destination.port](https://www.elastic.co/guide/en/ecs/current/ecs-destination.html#field-destination-port) + +APM Agents should now use fields described below in `span.context.service.target` for exit spans, those are a subset +of [ECS Service fields](https://www.elastic.co/guide/en/ecs/current/ecs-service.html). + +- `span.context.service.target.type` : [ECS service.type](https://www.elastic.co/guide/en/ecs/current/ecs-service.html#field-service-type) + , optional, might be empty. +- `span.context.service.target.name` : [ECS service name](https://www.elastic.co/guide/en/ecs/current/ecs-service.html#field-service-name) + , optional. +- at least one of those two fields is required to be provided and not empty + +Alignment to ECS provides the following benefits: + +- Easier correlation with other data sources like logs and metrics if they also rely on those service fields. +- Posible future extension with other + ECS [service fields](https://www.elastic.co/guide/en/ecs/current/ecs-service.html) to provide higher-granularity +- Bring APM Agents intake and data stored in ES closer to ECS + +On agents side, it splits the values that were previously written into `span.destination.service.resource` in distinct +fields. It provides a generic way to provide higher-granularity for service map and service dependencies. + +#### APM Agents + +There are a few features in APM Agents relying on `span.destination.service.resource`: + +- Dropped spans metrics +- Compressed spans +- Agent API might provide a way to manually set the `span.destination.service.resource`, if such API exists we should provide an equivalent to enable setting `span.service.target.type` and `span.service.target.name`. + +#### APM Server + +The following features rely on `span.destination.service.resource` field: + +- APM Map +- APM Services Dependencies +- Dropped spans metrics (indirect dependency) + +As a result, we need to make APM server handle the compatibility with those for both existing and new agents. + +### Implementation phases + +Because there are a lots of moving pieces, implementation will be split into multiple phases: + +- **Phase 1** : APM server ingest + compatibility for older agents + - Spans intake: `span.context.service.target.*`, store them as-is in ES Span documents + - Transactions intake: add `service_target_type` and `service_target_name` next to `destination_service_resource` in `transaction.dropped_spans_stats` array, the related metrics documents should include `span.service.target.type` and `span.service.target.name` fields. + - On the server, service destination metrics and dropped spans metrics should be updated to include new dimensions: `span.service.target.type` and `span.service.target.name` next to the existing + `span.destination.service.resource`, `span.destination.service.response_time.*` fields and their aggregation remain untouched for now. + - compatibility: fields `span.context.service.target.*` are inferred from `span.destination.service.resource` + - compatibility: dropped spans and destination metrics still able to use provided `span.destination.service.resource`. +- **Phase 2** : modify one or more agents to: + - Add and capture values for `span.context.service.target.type` and `span.context.service.target.name` for exit spans. + - Infer from those new fields the value of `span.destination.service.resource` and keep sending it. + - Add `service_target_*` fields to dropped spans metrics (as described in Phase 1) + - Handle span compression with new fields (stop relying on `resource` internally) +- **Phase 3** : modify the agents not covered in Phase 2 + - Add `span.context.service.target.type` and `span.context.service.target.name` + - Handle dropped spans metrics with only the new fields + - Handle span compression with new fields +- **Phase 4** : modify the UI to display and query new fields (to be further clarified) + - service dependencies + - service maps + - display fallback on `resource` field when `span.context.service.target.type` is empty + +### Examples + +1. Database call to a `mysql` server without database instance name +2. Database call to a `mysql` server on the `my-db` database +3. Send message on `rabbitmq` server without queue +4. Send message on `rabbitmq` server on the `my-queue` queue +5. HTTP request to `host:80` server + +| Span field | #1 | #2 | #3 | #4 | #5 | +|-------------------------------------------------|---------|---------------|-------------|---------------------|---------------| +| `span.type` | `db` | `db` | `messaging` | `messaging` | `external` | +| `span.subtype` | `mysql` | `mysql` | `rabbitmq` | `rabbitmq` | `http` | +| `span.context.service.target.type` | `mysql` | `mysql` | `rabbitmq` | `rabbitmq` | `http` | +| `span.context.service.target.name` (1) | | `my-db` | | `my-queue` | `host:80` | +| `span.context.destination.service.resource` (2) | `mysql` | `mysql/my-db` | `rabbitmq` | `rabbitmq/my-queue` | `host:80` (3) | + +(1) Value depends on the instrumented backend, see [below](#field-values) for details. + +(2) Value is always sent by APM agents for compatibility, but they SHOULD NOT rely on it internally. + +(3) HTTP spans (and a few other spans) can't have their `resource` value inferred on APM server without relying on a +brittle mapping on span `type` and `subtype` and breaking the breakdown metrics where `type` and `subtype` are not available. + +## Implementation details + +This specification assumes that values for `span.type` and `span.subtype` fit the [span_types.json](../../tests/agents/json-specs/span_types.json) specification. + +### Field values + +- `span.context.service.target.*` fields should be omitted for non-exit spans. +- Values set by user through the agent API should have priority over inferred values. +- `span.context.service.target.type` should have the same value as `span.subtype` and fallback to `span.type`. +- `span.context.service.target.name` depends on the span context attributes + +On agents, the following algorithm should be used to infer the values for `span.context.service.target.*` fields. + +```javascript +// span created on agent +span = {}; + +if (span.isExit) { + service_target = span.context.service.target; + + if (!('type' in service_target)) { // If not manually specified, infer type from span type & subtype. + service_target.type = span.subtype || span.type; + } + + if (!('name' in service_target)) { // If not manually specified, infer name from span attributes. + + if (span.context.db) { // database spans + if (span.context.db.instance) { + service_target.name = span.context.db.instance; + } + + } else if (span.context.message) { // messaging spans + if (span.context.message.queue?.name) { + service_target.name = span.context.message.queue?.name + } + + } else if (context.http?.url) { // http spans + service_target.name = getHostFromUrl(context.http.url); + // + // We always expect a valid port number here (80/443 default). + // + port = getPortFromUrl(context.http.url); + service_target.name += ":" + port; + } + } +} else { + // non-exit spans should not have service.target.* fields + span.context.service.target = undefined; +} +``` + +The values for `span.context.db.instance` are described in [SQL Databases](./tracing-instrumentation-db.md#sql-databases). + +The values for `span.context.message.queue.name` are described in [Messaging context fields](./tracing-instrumentation-messaging.md#context-fields) + +### User API + +Agents SHOULD provide an API entrypoint to set the value of `span.context.destination.service.resource`, +setting an empty or `null` value allows the user to discard the inferred value. +This API entrypoint should be marked as deprecated and replaced by the following: + +Agents SHOULD provide an API entrypoint to set the value of `span.context.service.target.type` and `span.context.service.target.name`, +setting an empty or `null` value on both of those fields allows the user to discard the inferred values. + +When a user-provided value is set, it should take precedence over inferred values from the span `_.type` `_.subtype` or any `_.context` attribute. + +In order to provide compatibility with existing agent API usage, when user calls the deprecated method to set `_.resource` = `""`, +agents MAY set `_.type` = `""` (empty string) and `_.name` = `""`, which replicates the behavior on APM server described below. + +### Phase 1 - server-side compatibility + +In Phase 1 the `span.service.target.{type,name}` fields are inferred on APM Server with the following algorithm and internal +usage of `resource` field in apm-server can be replaced with `span.service.target.{type,name}` fields. + +When this phase is implemented, the stored spans can be summarized as follows: + +| `span.context._` | `_.destination.service.resource` | `_.service.target.type` | `_.service.target.name` | +|------------------------------------------------------------|-------------------------------------------------|---------------------------------------------------|------------------------------------------| +| Non-exit span | - | - | - | +| Exit span captured before server 8.3 | `mysql`, `mysql/myDb` | - | - | +| Exit span captured with server 8.3 or later + legacy agent | `mysql`
`mysql/myDb`
`localhost:8080` | `mysql`
`mysql`
`""` (empty string) (1) | -
`myDb`
`localhost:8080` | +| Exit span captured with server 8.3 + latest agent | `mysql`
`mysql/myDb`
`localhost:8080` | `mysql`
`mysql`
`http` or `grpc` (2) | -
`myDB`
`localhost:8080` (2) | + +(1) : APM Server can't infer the value of the equivalent `service.target.type`, so we use the empty string `""` to allow UI to fallback on using the `_.resource` or `_.service.target.name` for display and compatibility. + +(2) : in this case the values are provided by the agent and not inferred by APM server. + +```javascript + +// Infer new fields values from an existing 'resource' value +// Empty type value (but not null) that can be used on UI to use the existing resource for display. +// For internal aggregation on (type,name) and usage this will be equivalent to relying on 'resource' value. +inferFromResource = function (r) { + + singleSlashRegex = new RegExp('^([a-z0-9]+)/(\w+)$').exec(r); + typeOnlyRegex = new RegExp(('^[a-z0-9]+$')).exec(r); + + if (singleSlashRegex != null) { + // Type + breakdown + // e.g. 'mysql/mydatabase', 'rabbitmq/myQueue' + return { + type: singleSlashRegex[1], + name: singleSlashRegex[2] + } + + } else if (typeOnlyRegex != null) { + // Type only + // e.g. 'mysql' + return { + type: r, + }; + + } else { + // Other cases, should rely on default, UI will have to display resource as fallback + // e.g. 'localhost:8080' + + return { + type: '', + name: r + } + } +} + +// usage with span from agent intake +span = {}; + +if (!span.service.target.type && span.destination.service.resource) { + // try to infer new fields from provided resource + + inferred = inferFromResource(span.destination.service.resource); + span.service.target.type = inferred.type; + span.service.target.name = inferred.name; +} + +``` + +#### OTel and bridge compatibility + +APM server already infers the `span.destination.service.resource` value from OTel span attributes, this algorithm needs +to be updated in order to also infer the values of `span.context.service.target.*` fields. +- `span.context.service.target.type` should be set from the inferred value of `span.subtype` with fallback to `span.type` + - For database spans: use value of `db.system` attribute + - For HTTP client spans: use `http` + - For messaging spans: use value of `messaging.system` attribute + - For RPC spans: use value of `rpc.system` attribute +- `span.context.service.target.name` should be set from OTel attributes if they are present + - For database spans: use value of `db.name` attribute + - For HTTP client spans: create `:` string from `http.host`, `net.peer.port` attributes or equivalent + - For messaging spans: use value of `messaging.destination` attribute if `messaging.temp_destination` is `false` or absent to limit cardinality + - For RPC spans: use value of `rpc.service` + +When OTel bridge data is sent in `_.otel.attributes` for spans and transactions captured through agent OTel bridges, +the inferred values on OTel attributes should take precedence over the equivalent attributes in regular agent protocol. diff --git a/specs/agents/tracing-spans.md b/specs/agents/tracing-spans.md new file mode 100644 index 00000000..a41a33d7 --- /dev/null +++ b/specs/agents/tracing-spans.md @@ -0,0 +1,199 @@ +## Spans + +The agent should also have a sense of the most common libraries for these and instrument them without any further setup from the app developers. + +### Span ID fields + +Each span object will have an `id`. This is generated for each transaction and +span, and is 64 random bits (with a string representation of 16 hexadecimal +digits). + +Each span will have a `parent_id`, which is the ID of its parent transaction or +span. + +Spans will also have a `transaction_id`, which is the `id` of the current +transaction. While not necessary for distributed tracing, this inclusion allows +for simpler and more performant UI queries. + +### Transaction and Span type and subtype fields + +Each transaction has a `type` field, each span has both `type` and `subtype` fields. +The values for each of those fields is protocol-specific and defined in the respective instrumentation specification +for each protocol. +If no `transaction.type` or `span.type` is provided or the value is an empty string, the agent needs to set a default value `custom`. + +For spans, the type/subtype must fit the [span type specification in JSON format](../../tests/agents/json-specs/span_types.json). +In order to help align all agents on this specification, changing `type` and `subtype` field values is not considered +to be a _breaking change_, but rather a _potentially breaking change_ if for example existing users rely on values to +build visualizations. As a consequence, modification of those values is not limited to major versions. + +### Span Name + +Each span will have a `name`, which is a descriptive, low-cardinality string. + +If a span is created without a valid `name`, the string `"unnamed"` SHOULD be used. + +### Span `sync` + +Span execution within a transaction or span can be synchronous (the caller waits for completion), or asynchronous (the caller does not wait +for completion). + +In UI: + +- when `sync` field is not present or `null`, we assume it's the platform default and no badge is shown. +- when `sync` field is set to `true`, a `blocking` badge is shown in traces where the platform default is `async`: `nodejs`, `rum` and `javascript` +- when `sync` field is set to `false`, an `async` badge is shown in traces where the platform default is `blocking`: other agents + +### Span outcome + +The `outcome` property denotes whether the span represents a success or failure, it is used to compute error rates +to calling external services (exit spans) from the monitored application. It supports the same values as `transaction.outcome`. + +This property is optional to preserve backwards compatibility, thus it is allowed to omit it or use a `null` value. + +If an agent does not report the `outcome` property (or use a `null` value), then the outcome will be set according to HTTP +response status if available, or `unknown` if not available. This allows a server-side fallback for existing +agents that might not report `outcome`. + +While the transaction outcome lets you reason about the error rate from the service's point of view, +other services might have a different perspective on that. +For example, if there's a network error so that service A can't call service B, +the error rate of service B is 100% from service A's perspective. +However, as service B doesn't receive any requests, the error rate is 0% from service B's perspective. +The `span.outcome` also allows reasoning about error rates of external services. + +The following protocols get their outcome from protocol-level attributes: + +- [gRPC](tracing-instrumentation-grpc.md#outcome) +- [HTTP](tracing-instrumentation-http.md#outcome) + +For other protocols, we can default to the following behavior: + +- `failure` when an error is reported +- `success` otherwise + +Also, while we encourage most instrumentations to create spans that have a deterministic outcomes, there are a few +examples for which we might still have to report `unknown` outcomes to prevent reporting any misleading information: +- Inferred spans created through a sampling profiler: those are not exit spans, we can't know if those could be reported +as either `failure` or `outcome` due to inability to capture any errors. +- External process execution, we can't know the `outcome` until the process has exited with an exit code. + +### Outcome API + +Agents should expose an API to manually override the outcome. +This value must always take precedence over the automatically determined value. +The documentation should clarify that spans with `unknown` outcomes are ignored in the error rate calculation. + +### Span stack traces + +Spans may have an associated stack trace, in order to locate the associated +source code that caused the span to occur. If there are many spans being +collected this can cause a significant amount of overhead in the application, +due to the capture, rendering, and transmission of potentially large stack +traces. It is possible to limit the recording of span stack traces to only +spans that are slower than a specified duration, using the config variable +`span_stack_trace_min_duration`. (Previously +`span_frames_min_duration`.) + +Agents based on OpenTelemetry should capture this using the `code.stacktrace` semantic conventions attribute [added in 1.24.0]( +https://github.com/open-telemetry/semantic-conventions/pull/435). + +#### `span_stack_trace_min_duration` configuration + +Sets the minimum duration of a span for which stack frames/traces will be +captured. + +This values for this option are case-sensitive. + +| | | +|----------------|---| +| Valid options | [duration](configuration.md#configuration-value-types) | +| Default | `5ms` (soft default, agents may modify as needed) | +| Dynamic | `true` | +| Central config | `true` | + +A negative value will result in never capturing the stack traces. + +A value of `0` (regardless of unit suffix) will result in always capturing the +stack traces. + +A non-default value for this configuration option should override any value +set for the deprecated `span_frames_min_duration`. + +### Exit spans + +Exit spans are spans that describe a call to an external service, +such as an outgoing HTTP request or a call to a database. + +A span is considered an exit span if it has explicitly been marked as such; a +span's status should not be inferred. + +#### Child spans of exit spans + +Exit spans MUST not have child spans that have a different `type` or `subtype`. +For example, when capturing a span representing a query to Elasticsearch, +there should not be an HTTP span for the same operation. +Doing that would make [breakdown metrics](https://github.com/elastic/apm/blob/main/specs/agents/metrics.md#transaction-and-span-breakdown) +less meaningful, +as most of the time would be attributed to `http` instead of `elasticsearch`. + +Agents MAY add information from the lower level transport to the exit span, though. +For example, the HTTP `context.http.status_code` may be added to an `elasticsearch` span. + +Exit spans MAY have child spans that have the same `type` and `subtype`. +For example, an HTTP exit span may have child spans with the `action` `request`, `response`, `connect`, `dns`. +These spans MUST NOT have any destination context, so that there's no effect on destination metrics. + +Most agents would want to treat exit spans as leaf spans, though. +This brings the benefit of being able to [compress](handling-huge-traces/tracing-spans-compress.md) repetitive exit spans, +as span compression is only applicable to leaf spans. + +Agents MAY implement mechanisms to prevent the creation of child spans of exit spans. +For example, agents MAY implement internal (or even public) APIs to mark a span as an exit or leaf span. +Agents can then prevent the creation of a child span of a leaf/exit span. +This can help to drop nested HTTP spans for instrumented calls that use HTTP as the transport layer (for example Elasticsearch). + +#### Context propagation + +When tracing an exit span, agents SHOULD propagate the trace context via the underlying protocol wherever possible. + +Example: for Elasticsearch requests, which use HTTP as the transport, agents SHOULD add `traceparent` headers to the outgoing HTTP request. + +This means that such spans cannot be [compressed](handling-huge-traces/tracing-spans-compress.md) if the context has +been propagated, because the `parent.id` of the downstream transaction may refer to a span that's not available. +For now, the implication would be the inability to compress HTTP spans. Should we decide to enable that in the future, +following are two options how to do that: +- Add a denylist of span `type` and/or `subtype` to identify exit spans of which underlying protocol supports context propagation by default. +For example, such list could contain `type == storage, subtype == s3`, preventing context propagation at S3 queries, even though those rely on HTTP/S. +- Add a list of child IDs to compressed exit spans that can be used when looking up `parent.id` of downstream transactions. + +### Span lifetime + +In the common case we expect spans to start and end within the lifetime of their +parent and their transaction. However, agents SHOULD support spans starting +and/or ending *after* their parent has ended and after their transaction has +ended. + +This may result in [transaction `span_count` values](handling-huge-traces/tracing-spans-limit.md#span-count) +being low. Agents do not need to wait for children to end before reporting a +parent. + +### Inferred Spans + +Agents MAY support so-called inferred spans: Inferred spans are spans which are not created using instrumentation, but derived from profiling data. +E.g. when a transaction is active on a thread, the agent will periodically fetch stacktraces for the given thread. +Based on these stacktraces, the agent tries to infer spans in addition to the ones captured via normal instrumentation. + +Inferred spans can be parents of normal spans. Given the following example: + * Transaction `A` has a child span `C` + * The span `B` is inferred to be a child of `A`, and `B` is the new parent of `C` + * Resulting trace: `A` β†’ `B` β†’ `C` + +Agents MAY perform the span inference after the transaction or child spans were ended. As a result, the spans `A` and `C` might have already been sent at the time `B` is created. +The problem in this case is that `C` is sent with `A` as parent, whereas the actual parent will be `B`. + +For this reason, inferred spans can use the following mechanism to override the parent-child relationship for spans which have already been sent: + * When reporting via IntakeV2, the `child_ids` attribute can be used (`B.child_ids=[spanIdOf(C)]`) + * When reporting via OTLP, inferred spans should add span-links to their children for which they want to override the relationship. These links must have the `elastic.is_child` attribute set to `true`. + +The UI will then correct the parent-child relationships when displaying the trace. diff --git a/specs/agents/tracing-transaction-grouping.md b/specs/agents/tracing-transaction-grouping.md new file mode 100644 index 00000000..9972917c --- /dev/null +++ b/specs/agents/tracing-transaction-grouping.md @@ -0,0 +1,39 @@ +## Transaction Grouping + +Even though agents should choose a transaction name that has a reasonable cardinality, +they can't always guarantee that. +For example, +when the auto-instrumentation of a job scheduling framework sets the transaction name to the name of the instrumented job, +the agent has no control over the job name itself. +While usually the job name is expected to have low cardinality, +users might set dynamic parts as part of the job name, such as a UUID. + +In order to give users an option to group transactions whose name contain dynamic parts that don't require code changes, +agents MAY implement the following configuration option: + +### `transaction_name_groups` configuration + +With this option, +you can group transaction names that contain dynamic parts with a wildcard expression. +For example, +the pattern `GET /user/*/cart` would consolidate transactions, +such as `GET /users/42/cart` and `GET /users/73/cart` into a single transaction name `GET /users/*/cart`, hence reducing the transaction name cardinality. +The first matching expression wins, so make sure to place more specific expressions before more generic ones, for example: `GET /users/*/cart, GET /users/*`. + +| | | +|----------------|------------------------------------------------------------------------------------------| +| Type | `List<`[`WildcardMatcher`](../../tests/agents/json-specs/wildcard_matcher_tests.json)`>` | +| Default | `` | +| Dynamic | `true` | +| Central config | `true` | + +The `url_groups` option that the Java and PHP agent offered is deprecated in favor of `transaction_name_groups`. +### When to apply the grouping + +The grouping can be applied either every time the transaction name is set, or lazily, when the transaction name is read. + +It's not sufficient to only apply the grouping when the transaction ends. +That's because when an error is tracked, the transaction name is copied to the error object. +See also [the error spec](error-tracking.md) + +Agents MUST also ensure that the grouping is applied before breakdown metrics are reported. diff --git a/specs/agents/tracing-transactions.md b/specs/agents/tracing-transactions.md new file mode 100644 index 00000000..bb54fa45 --- /dev/null +++ b/specs/agents/tracing-transactions.md @@ -0,0 +1,57 @@ +## Transactions + +Transactions are a special kind of span. +They represent the entry into a service. +They are sometimes also referred to as local roots or entry spans. + +Transactions are created either by the built-in auto-instrumentation or an agent or the [tracer API](tracing-api.md). + +### Transaction outcome + +The `outcome` property denotes whether the transaction represents a success or a failure from the perspective of the entity that produced the event. +The APM Server converts this to the [`event.outcome`](https://www.elastic.co/guide/en/ecs/current/ecs-allowed-values-event-outcome.html) field. +This property is optional to preserve backwards compatibility. +If an agent doesn't report the `outcome` (or reports `null`), the APM Server will set it based on `context.http.response.status_code`. If the status code is not available, then it will be set to `"unknown"`. + +- `"failure"`: Indicates that this transaction describes a failed result. \ + Note that client errors (such as HTTP 4xx) don't fall into this category as they are not an error from the perspective of the server. +- `"success"`: Indicates that this transaction describes a successful result. +- `"unknown"`: Indicates that there's no information about the outcome. + This is the default value that applies when an outcome has not been set explicitly. + This may be the case when a user tracks a custom transaction without explicitly setting an outcome. + For existing auto-instrumentations, agents should set the outcome either to `"failure"` or `"success"`. + +What counts as a failed or successful request depends on the protocol. + +The following protocols get their outcome from protocol-level attributes: + +- [gRPC](tracing-instrumentation-grpc.md#outcome) +- [HTTP](tracing-instrumentation-http.md#outcome) + +For other protocols, we can default to the following behavior: + +- `failure` when an error is reported +- `success` otherwise + +#### Error rate + +The error rate of a transaction group is based on the `outcome` of its transactions. + + error_rate = failure / (failure + success) + +Note that when calculating the error rate, +transactions with an `unknown` or non-existent outcome are not considered. + +The calculation just looks at the subset of transactions where the result is known and extrapolates the error rate for the total population. +This avoids that `unknown` or non-existant outcomes reduce the error rate, +which would happen when looking at a mix of old and new agents, +or when looking at RUM data (as page load transactions have an `unknown` outcome). + +Also note that this only reflects the error rate as perceived from the application itself. +The error rate perceived from its clients is greater or equal to that. + +#### Outcome API + +Agents should expose an API to manually override the outcome. +This value must always take precedence over the automatically determined value. +The documentation should clarify that transactions with `unknown` outcomes are ignored in the error rate calculation. diff --git a/specs/agents/transport.md b/specs/agents/transport.md new file mode 100644 index 00000000..5486e9bc --- /dev/null +++ b/specs/agents/transport.md @@ -0,0 +1,112 @@ +## Transport + +Agents send data to the APM Server as JSON (application/json) or ND-JSON (application/x-ndjson) over HTTP. We describe here various details to guide transport implementation. + +### User-Agent + +In order to help debugging and gathering usage statistics, agents should use one of the following values for the `User-Agent` HTTP header: + +- Header value should start with agent github repository as prefix and version `apm-agent-${language}/${agent.version}`. +- If both `service.name` and `service.version` are set, append ` (${service.name} ${service.version})` +- If only `service.name` is set, append `(${service.name})` + +An executable gherkin specification is also provided in [user_agent.feature](../../tests/agents/gherkin-specs/user_agent.feature). + +Examples: +- `apm-agent-java/v1.25.0` +- `apm-agent-ruby/4.4.0 (myservice)` +- `apm-agent-python/6.4.0 (myservice v42.7)` + +### Background sending + +In order to avoid impacting application performance and behaviour, agents should (where possible) send data in a non-blocking manner, e.g. via a background thread/goroutine/process/what-have-you, or using asynchronous I/O. + +If data is sent in the background process, then there must be some kind of queuing between that background process and the application code. The queue should be limited in size to avoid memory exhaustion. In the event that the queue fills up, agents must drop events: either drop old events or simply stop recording new events. + +### Batching/streaming data + +With the exception of the RUM agent (which does not maintain long-lived connections to the APM Server), agents should use the ND-JSON format. The ND-JSON format enables agents to stream data to the server as it is being collected, with one event being encoded per line. This format is supported since APM Server 6.5.0. + +Agents should implement one of two methods for sending events to the server: + + - batch events together and send a complete request after a given size is reached, or amount of time has elapsed + - start streaming events immediately to the server using a chunked-encoding request, and end the request after a given amount of data has been sent, or amount of time has elapsed + +The streaming approach is preferred. There are two configuration options that agents should implement to control when data is sent: + + - [ELASTIC_APM_API_REQUEST_TIME](https://www.elastic.co/guide/en/apm/agent/python/current/configuration.html#config-api-request-time) + - [ELASTIC_APM_API_REQUEST_SIZE](https://www.elastic.co/guide/en/apm/agent/python/current/configuration.html#config-api-request-size) + +All events can be streamed as described in the [Intake API](https://www.elastic.co/guide/en/apm/server/current/intake-api.html) documentation. Each line encodes a single event, with the first line in a stream encoding the special metadata "event" which is folded into all following events. This metadata "event" is used to describe static properties of the system, process, agent, etc. + +When the batching approach is employed, unhandled exceptions/unexpected errors should typically be sent immediately to ensure timely error visibility, and to avoid data loss due to process termination. Even when using streaming there may be circumstances in which the agent should block the application until events are sent, but this should be both rare and configurable, to avoid interrupting normal program operation. For example, an application may terminate itself after logging a message at "fatal" level. In such a scenario, it may be useful for the agent to optionally block until enqueued events are sent prior to process termination. + +### Transport errors + +If the HTTP response status code isn’t 2xx or if a request is prematurely closed (either on the TCP or HTTP level) the request MUST be considered failed. + +When a request fails, the agent has no way of knowing exactly what data was successfully processed by the APM Server. And since the agent doesn’t keep a copy of the data that was sent, there’s no way for the agent to re-send any data. Furthermore, as the data waiting to be sent is already compressed, it’s impractical to recover any of it in a way so that it can be sent over a new HTTP request. + +The agent should therefore drop the entire compressed buffer: both the internal zlib buffer, and potentially the already compressed data if such data is also buffered. Data subsequently written to the compression library can be directed to a new HTTP request. + +The new HTTP request should not necessarily be started immediately after the previous HTTP request fails, as the reason for the failure might not have been resolved up-stream. Instead an incremental back-off algorithm SHOULD be used to delay new requests. The grace period should be calculated in seconds using the algorithm `min(reconnectCount++, 6) ** 2 Β± 10%`, where `reconnectCount` starts at zero. So the delay after the first error is 0 seconds, then circa 1, 4, 9, 16, 25 and finally 36 seconds. We add Β±10% jitter to the calculated grace period in case multiple agents entered the grace period simultaneously. This way they will not all try to reconnect at the same time. + +Agents should support specifying multiple server URLs. When a transport error occurs, the agent should switch to another server URL at the same time as backing off. + +While the grace period is in effect, the agent may buffer the data that was supposed to be sent if the grace period wasn’t in effect. If buffering, the agent must ensure the memory used to buffer data data does not grow indefinitely. + +### Compression + +The APM Server accepts both uncompressed and compressed HTTP requests. The following compression formats are supported: + +- zlib data format (`Content-Encoding: deflate`) +- gzip data format (`Content-Encoding: gzip`) + +Agents MUST compress the HTTP payload and SHOULD optimize for speed over compactness (typically known as the "best speed" level). + +If the host part of the APM Server URL is either `localhost`, `127.0.0.1`, `::1`, or `0:0:0:0:0:0:0:1`, agents SHOULD disable compression. +Agents MUST NOT use the compression level `NO_COMPRESSION` to disable compression. +That's because the [Lambda extension](https://github.com/elastic/apm-aws-lambda/tree/main/apm-lambda-extension) +would otherwise consider the data as being compressed (due to the `Content-Encoding` header) and send data to APM Server that's actually uncompressed. + +### `context_propagation_only` configuration + +| | | +|----------------|---| +| Type | `boolean` | +| Default | `false` | +| Dynamic | `true` | +| Central config | `true` | + +Agents MAY implement this configuration option. +`context_propagation_only` is a boolean configuration option to have an APM +agent perform trace-context propagation and log correlation *only*; and to +explicitly *not* send event data to APM server. This allows an application to +get automatic context propagation and log correlation, **without** having +deployed an APM server for event collection. + +Agents that implement this configuration option: + +- MUST continue to propagate trace headers (`traceparent`, `tracestate`, etc.) + per normal; +- MUST start a trace-id if no `traceparent` header is present where they would normally start a transaction and propagate it. +- MUST continue to support [log correlation](./log-correlation.md); +- MUST NOT send event data to the APM server +- SHOULD attempt to reduce runtime overhead where possible. For example, + because events will be dropped there is no need to collect stack traces, + collect metrics, calculate breakdown metrics, or to create spans (other than + the top-level transaction required for context propagation, similarly to non-sampled traces). + + +### `disable_send` configuration + +Agents MAY implement this configuration option. +`disable_send` is a boolean configuration option to have an APM agent be fully +functioning, but not communicate with an APM server. Use case for this include +testing and continuous integration (CI) systems. + +Agents that implement this configuration option: + +- MUST NOT attempt to communicate with APM server. This includes central configuration. +- MUST NOT log warnings/errors related to failures to communicate with APM server. +- SHOULD otherwise perform all functions. diff --git a/specs/agents/uml/kafka_consume.puml b/specs/agents/uml/kafka_consume.puml new file mode 100644 index 00000000..bc5405c7 --- /dev/null +++ b/specs/agents/uml/kafka_consume.puml @@ -0,0 +1,31 @@ +@startuml kafka_consume +hide footbox +participant "Application (Consumer)" as app +participant "APM agent" as apm +queue Kafka as queue + +activate app +loop while true + activate apm #00BFB3 + apm -> apm: transaction.End() + deactivate apm + + app -> queue: **consumer.Consume()** + deactivate app + activate queue + + group Message processing flow + ... ~~blocking operation~~ ... + queue --> app: message + + deactivate queue + activate app + + apm -> apm: transaction.Start() + activate apm #00BFB3 + end + + deactivate app + deactivate apm +end +@enduml \ No newline at end of file diff --git a/specs/agents/uml/kafka_consume.svg b/specs/agents/uml/kafka_consume.svg new file mode 100644 index 00000000..3371a29c --- /dev/null +++ b/specs/agents/uml/kafka_consume.svg @@ -0,0 +1,41 @@ +Application (Consumer)APM agentKafkaloop[while true]transaction.End()consumer.Consume()Message processing flowblocking operationmessagetransaction.Start() \ No newline at end of file diff --git a/specs/agents/uml/publish.puml b/specs/agents/uml/publish.puml new file mode 100644 index 00000000..05b01d4e --- /dev/null +++ b/specs/agents/uml/publish.puml @@ -0,0 +1,25 @@ +@startuml publish +hide footbox +participant "Application (Publisher)" as app +participant "APM agent" as apm +queue "Messaging system" as queue + + activate app + activate apm #00BFB3 + app -> queue: **publish message** + deactivate app + activate queue + activate apm #1BA9F5 + apm -> apm: span.Start() + note left of apm + **Capture new span** + + There is an active transaction + end note + apm -> apm: span.End() + queue --> app + deactivate queue + activate app + deactivate apm + +@enduml \ No newline at end of file diff --git a/specs/agents/uml/publish.svg b/specs/agents/uml/publish.svg new file mode 100644 index 00000000..2c0358b4 --- /dev/null +++ b/specs/agents/uml/publish.svg @@ -0,0 +1,35 @@ +Application (Publisher)APM agentMessaging systempublish messagespan.Start()Capture new spanThere is an active transactionspan.End() \ No newline at end of file diff --git a/specs/agents/universal-profiling-integration.md b/specs/agents/universal-profiling-integration.md new file mode 100644 index 00000000..519dc09b --- /dev/null +++ b/specs/agents/universal-profiling-integration.md @@ -0,0 +1,265 @@ +This specification document describes the integration and communication between the universal profiling host agent and apm-agents. +The goal is to enrich both profiling and APM data with correlation information to allow linking and filtering based of concepts from both worlds. + +# General Approach + +The profiling host agent runs as a standalone process with root rights. The apm-agents in contrast run within the user application process, which is likely to have limited permissions and to also be containerized. + +Due to this mismatch in permissions all communication is actually initiated from the host-agent: + * The APM agent exposes the information it wants to send to the profiler host agent in native memory at "well known" (defined below) locations. Using its root permissions, the host agent reads this memory. + * The APM agent creates a `DGRAM` socket for receiving messages. The profiler host agent connects to it (again, using its root permissions) and sends messages to the APM agent. + +# APM Agent exposed memory + +The APM agent "sends" information to the profiling host agent by exposing this information in native memory. +The pointers to this native memory MUST be stored in native global variables (exported symbols) with the following names: +``` +thread_local void* elastic_apm_profiling_correlation_tls_v1 = nullptr; + +void* elastic_apm_profiling_correlation_process_storage_v1 = nullptr; +``` + +The profiling host agent is capable of detected these global variables in processes and can read the memory they point to. +Note that we use two variables because we expose two types of storage: + * thread-local information (e.g. trace-id, span-id) + * process-level information (e.g. service-name) + +The variables are suffixed with `_v1`. If we require breaking changes in the future, this allows us to introduce a `_v2`. +For backwards compatibility with older host agent versions agents can then expose both `_v1` and `_v2` variables at the same time. + +The `thread_local` variables must be compiled with the `TLSDESC` model in order to be readable by the profiler host agent. This means that corresponding GCC arguments must be used when compiling the native library: + * ARM64: `-ftls-model=global-dynamic -mtls-dialect=desc` + * x86_64: `-ftls-model=global-dynamic -mtls-dialect=gnu2` + +The library file name MUST match the following regular expression to be picked up by the profiler: + +```regexp +.*/elastic-jvmti-linux-([\w-]*)\.so +``` + +## General Memory Layout + +The shared memory always uses the native endianess of the current platform for multi-byte numbers. +Strings are always UTF-8 length encoded: +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ length : uint32 β”‚ utf8-buf : uint8[length] β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` +`utf8-buf` is not null-terminated, the length is already defined by the `length` field. +In the remainder of the document the type `utf8-str` is used to denote the encoding shown above. + +## Process Storage Layout + +This section explains the layout of the memory to which the `elastic_apm_profiling_correlation_process_storage_v1` variable points. +Note that `elastic_apm_profiling_correlation_process_storage_v1` MUST only point to a non-zero memory location when that memory has been fully initialized with the layout described in this section! + +Name | Data type +--------------------- | ------------- +layout-minor-version | uint16 +service-name | utf8-str +service-environment | utf8-str +socket-file-path | utf8-str + +* *layout-minor-version*: Always `1` for now. The minor version will be incremented when new fields are added to the storage (non breaking changes). +* *service-name*: The APM service name of the process +* *service-environment*: The configured environment name for this process (e.g. production, testing, dev) +* *socket-file-path*: The APM Agent opens a UNIX domain socket for receiving messages from running profiler host agents (See [this section](#messaging-socket). This field contains the path to the socket file as seen from the process running the APM Agent (e.g. within the container). + +## Thread Local Storage Layout + +This section explains the memory layout of the memory to which the `elastic_apm_profiling_correlation_tls_v1` thread local variable points. +Note that `elastic_apm_profiling_correlation_tls_v1` MUST only point to a non-zero memory location when that memory has been fully initialized for that thread with the layout described in this section! Multiple threads must not share their memory area. + +Name | Data type +--------------------- | ------------- +layout-minor-version | uint16 +valid | uint8 +trace-present | uint8 +trace-flags | uint8 +trace-id | uint8[16] +span-id | uint8[8] +transaction-id | uint8[8] + +* *layout-minor-version*: Always `1` for now. The minor version will be incremented when new fields are added to the storage (non breaking changes). +* *valid*: Set to `0` by the APM-agent if it currently is in the process of updating this thread local storage and to non-zero after the update is done. The profiling host agent MUST ignore any data read if this value is `0`. +* *trace-present*: `1` if there currently is a trace active on this thread, `0` otherwise. If the value is `0`, the profiler host agent must ignore the `trace-flags`, `trace-id`, `span-id` and `transaction-id` fields +* *trace-flags*: Corresponds to the [W3C trace flags](https://www.w3.org/TR/trace-context/#trace-flags) of the currently active trace +* *trace-id*: The W3C trace id of the currently active trace +* *span-id*: The W3C trace id of the currently active span +* *transaction-id*: The W3C span id of the currently active transaction (=the local root span) + +APM-agents MAY start populating the thread-local storage only after receiving a host agent [registration message](#profiler-registration-message) + +### Concurrency-safe Updates + +The profiler might interrupt a thread and take a profiling sample while that thread is in the process of updating the contents of the shared thread local storage. Fortunately, we have the following guarantees about this interruption: + * While the profiler is taking a sample, the thread being sampled is guaranteed to be "paused" (no parallelism involved, only concurrency) + * The profiler will read the thread local storage on the same CPU core as the thread writing it, therefore we won't have any problems with memory / cache visibility + +Based on these guarantees we can safely perform updates to the thread local storage the following way: + * Before updating anything else, the APM-agent sets the `valid`-byte to `0` + * The APM agent updates the thread local storage content (e.g. `span-id`) + * Finally, the APM-agent sets the `valid`-byte to `1` + +This way, the profiler is able to detect and discard incomplete data by inspecting the `valid` byte. + +Note that APM-agents must make sure that compilers do not reorder the steps listed above. +For Java for example this can be achieved by inserting volatile writes after setting the `valid` byte to `0` and before setting it back to `1`. + +# Messaging Socket + +In order to receive profiling data from the profiling host agent, APM agents MUST create a Datagram Unix Domain Socket. We use that kind of socket because: + * The socket is reliable and ensures "packet boundaries". This means we don't have to deal with buffering and reassembling partial messages + * If multiple senders send to the same socket (e.g. multiple profiler host agents are active), their packets stay separated + +Here is a code example on how to open a corresponding non-blocking socket at a given filepath: +```c +void createProfilerSocket(const char* filepath) { + int fd = socket(PF_UNIX, SOCK_DGRAM, 0); + if (fd == -1) { + //handle error + } + int flags = fcntl(fd, F_GETFL, 0); + if (flags == -1) { + close(fd); + //handle error + } + if(fcntl(fd, F_SETFL, flags | O_NONBLOCK) != 0){ + close(fd); + //handle error + } + + sockaddr_un addr = { .sun_family = AF_UNIX }; + strncpy(addr.sun_path, filepath, sizeof(addr.sun_path) - 1); + + if (bind(fd, (sockaddr*)&addr, sizeof(addr) ) < 0) { + close(fd); + //handle error + } +} +``` + +And here how to read a messages in a non-blocking way: + +```c +size_t readProfilerSocketMessages(uint8_t* outputBuffer, size_t bufferSize) { + int n = recv(profilerSocket, outputBuffer, bufferSize, 0); + if (n == -1) { + if(errno == EAGAIN || errno == EWOULDBLOCK) { + return -1; //no message to read + } else { + //something went wrong, handle error + } + } + if(n == bufferSize) { + //handle error, message might have been truncated + } + return n; //n is the size of the received message +} +``` + +Note that in certain edge cases (e.g. a full buffer on the sender side) messages might have been truncated! APM-agents must protected against this by detecting messages that are shorter than expected from the message type and version, discarding the message. + +The agent may create the file for the socket binding at any path and with any filename it likes. +The path to the opened socket must be exposed in the process storage field `socket-file-path`. +The process storage MUST NOT be initialized and exposed before the socket has been created. + +## Message Format + +All messages have the following layout: +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ message-type : uint16 β”‚ minor-version : uint16 β”‚ payload : * β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +* *message-type* : An ID uniquely identifying the type (and therefore payload structure) of the message. +* *minor-version* : The version number for the given *message-type*. This value is incremented when new fields are added to the payload while preserving the *message-type* (non breaking changes). For breaking changes a new *message-type* must be used. + +## Profiler Registration Message + +Whenever the profiling agent starts communicating for the first time with a process running an APM Agent, it MUST send this message. +This message is used to let the APM-agent know that a profiler is actually active on the current host. Note that an APM-agent may receive this message zero, one or several times: this may happen if no profiling agent is active, if one is active or if a profiling agent is restarted during the lifetime of the APM-agent respectively. + +The *message-type* is `2` and the current *minor-version* is `1`. + +The payload layout is as follows: +Name | Data type +--------------------- | ------------- +samples-delay-ms | uint32 +host-id | utf8-str + +* *samples-delay-ms*: A sane upper bound for the time taken in milliseconds by the profiling agent between the collection of a stacktrace and it being written to the APM agent via the [messaging socket](#cpu-profiler-trace-correlation-message). The APM agent will assume that all profiling data related to a span has been written to the socket if a span ended at least the provided duration ago. Note that this value doesn't need to be a hard guarantee, but it should be the 99% case so that profiling data isn't distorted in the expected case. +* *host-id*: The [`host.id` resource attribute](https://opentelemetry.io/docs/specs/semconv/attributes-registry/host/) is an optional argument (the string may have a length of zero) used to correlate profiling data by the profiling agent. If an APM-agent is already sending a `host.id` it MUST print a warning if the `host.id` is different and otherwise ignore the value received by the profiling agent. A mismatch will lead to certain correlation features (e.g. cost and CO2 consumption) not working. If an APM-agent does not collect the `host.id` by itself, it MUST start sending the `host.id` after receiving a non-empty `host.id` from the profiling agent to ensure aforementioned correlation features work correctly. + + +## CPU Profiler Trace Correlation Message + +Whenever the profiler is able to correlate a taken CPU stacktrace sample with an APM trace (see [this section](#thread-local-storage-layout)). It sends the ID of the stacktrace back to the APM agent. + +The *message-type* is `1` and the current *minor-version* is `1`. + +The payload layout is as follows: +Name | Data type +--------------------- | ------------- +trace-id | uint8[16] +transaction-id | uint8[8] +stack-trace-id | uint8[16] +count | uint16 + +* *trace-id*: The APM W3C trace id of the trace which was active for the given profiling samples +* *transaction-id*: The APM W3C span id of the transaction which was active for the given profiling samples +* *stack-trace-id*: The unique ID for the stacktrace captured assigned by the profiler. This ID is stored in elasticsearch in base64 URL safe encoding by the universal profiling solution. +* The number of samples observed since the last report for the (*trace-id*, *transaction-id*, *stack-trace-id*) combination. + + +# APM Agent output of correlation data + +## Correlation Attribute + +APM Agents will receive the IDs of stacktraces which occurred during transactions via [correlation messages](#cpu-profiler-trace-correlation-message). +If the correlation feature is enabled, agents SHOULD store the received IDs of stacktraces as `elastic.profiler_stack_trace_ids` OpenTelemetry span attribute on the transaction: + + * The type of the `elastic.profiler_stack_trace_ids` MUST be string-array + * The stacktrace-IDs MUST be provided as base64 URL-safe encoded strings without padding + * The order of elements in the array is not relevant + * The counts of stacktrace-IDs must be preserved: If a stacktrace occurred *n* times for a given transaction, its ID must appear exactly *n* times in `elastic.profiler_stack_trace_ids` of that transaction + +The APM intake will store `elastic.profiler_stack_trace_ids` as `transaction.profiler_stack_trace_ids` on transaction documents with the special `counted_keyword` mapping type, ensuring duplicates are preserved. + +For example, if for a single transaction the following correlation messages are received + +* (stack-trace-ID: 0x60b420bb3851d9d47acb933dbe70399b, count: 2) +* (stack-trace-ID: 0x4c9326bb9805fa8f85882c12eae724ce, count: 1) +* (stack-trace-ID: 0x60b420bb3851d9d47acb933dbe70399b, count: 1) + +the resulting transaction MUST have the OpenTelemetry attribute `elastic.profiler_stack_trace_ids` with a value of (elements in any order) `[YLQguzhR2dR6y5M9vnA5mw, YLQguzhR2dR6y5M9vnA5mw, TJMmu5gF-o-FiCwS6uckzg, YLQguzhR2dR6y5M9vnA5mw]`. + +Note that the [correlation messages](#cpu-profiler-trace-correlation-message) will arrive delayed relative to when they were sampled due to the processing delay of the profiling agent and the transfer over the domain socket. APM agents therefore MUST defer sending ended transactions until they are relatively confident that all correlation messages for the transaction have arrived. + + * When a [profiler registration message](#profiler-registration-message) has been received, APM agents SHOULD use the duration from that message as delay for transactions + * If no [profiler registration message](#profiler-registration-message) has been received yet, APM agents SHOULD use a default of one second as reasonable default delay. + * If the correlation feature is not enabled, APM agents MUST NOT defer sending ended transactions + * Non-Transaction spans (non local-roots) and unsampled transactions MUST NOT be deferred + +Typically, this deferral would be implemented by putting transactions with a timestamp into a fixed-size FIFO queue when they end. +The head of the queue is removed, once the delay has elapsed and the corresponding transaction is only then forwarded to the exporter. +If a transaction cannot be buffered because the queue is full, it MUST be forwarded to the exporter immediately instead of being dropped. +In this case, agents SHOULD print a warning about profiling correlation data potentially being inaccurate/incomplete. + +## Configuration Options + +OpenTelemetry based agents SHOULD use the following configuration options: + + * `ELASTIC_OTEL_UNIVERSAL_PROFILING_INTEGRATION_ENABLED`: `true`, `false`, `auto` (optional) + + Defines whether the correlation feature is enabled or disabled. APM agents MAY optionally implement the `auto` mode: Hereby, the APM agent will open the correlation socket, but will not perform any correlation and won't buffer spans until a [profiler registration message](#profiler-registration-message) has been received. If `auto` is supported, APM agents SHOULD use it as default, as it provides a zero-configuration experience to end users. Otherwise the default SHOULD be `false`. + + * `ELASTIC_OTEL_UNIVERSAL_PROFILING_INTEGRATION_SOCKET_DIR` + + Defines the directory in which the socket-file for communication with the profiler will be created. Should have a reasonable default (e.g. a temp dir). + + * `ELASTIC_OTEL_UNIVERSAL_PROFILING_INTEGRATION_BUFFER_SIZE` + + The size of the FIFO queue [used to buffer transactions](#correlation-attribute) until all correlation data has arrived. Should have a reasonable default to sustain typical transaction per second rates while not occupying too much memory in edge cases (e.g. 8096). diff --git a/specs/integrations/synthetics.md b/specs/integrations/synthetics.md new file mode 100644 index 00000000..76f67fe4 --- /dev/null +++ b/specs/integrations/synthetics.md @@ -0,0 +1,112 @@ +## Synthetics Integration + +Synthetic monitors play a crucial role in periodically checking the status of your services and applications on a global scale. General documentation about synthetic monitors can be found in +[Synthetics getting started page](https://www.elastic.co/guide/en/observability/current/synthetics-get-started.html). + +This integration goes in to more detail about how the sythetics monitors would +be correlated with the APM traces. Synthetics traces can be categorized in to two +main types + 1. HTTP checks - These have one-one mapping with APM transactions + 2. Browser checks - These have a one-to-many mapping with APM transactions + +### Correlation + +The Synthetics agent (including Heartbeat) takes the responsibility of creating the +[`traceparent`](../agents/tracing-distributed-tracing.md#trace_id-parent_id-and-traceparent) +header for each outgoing network request associated with a test during every +monitor execution. + +- `trace.id` and `parent.id` + - outgoing requests that are being explicity traced by the synthetics agent + will have the `parent.id` and `trace.id` as part of the trace context. + - must be unique for each step for a browser monitor + - must be unique for a http monitor +- `sampled` Flag + - used to control the sampling decision for all the downstream services. + - 100% sampling when tracing is enabled + +#### Browser checks + +When executing a Synthetics journey with tracing enabled for all outgoing requests `**/*` or for specific URLs with the --apm_tracing_urls flag, the Synthetics agent takes the following actions: + +1. Adds the traceparent header to each matching outgoing request. +2. Includes trace.id and parent.id in all the Step Elasticsearch (ES) documents for the journey. + +```ts +// run journey +npx @elastic/synthetics --apm_tracing_urls "elastic.co/*" + +// example.journey.ts +journey("elastic e2e", ({ page }) => { + step("home page", async() => { + await page.goto("https://www.elastic.co") + }) + step("blog page", async() => { + await page.goto("https://www.elastic.co/blog") + }) +}) +``` + +Example of the tracing information added to the ES documents for two steps in the journey: + +```json +// Step - homepage +{"type":"step/end","journey":{"name":"elastic e2e"},"step":{"name":"home page","index":1,"status":"failed","duration":{"us":17382122}}, "trace.id": "t1"} +{"type":"journey/network_info","journey":{"name":"elastic e2e"},"step":{"name":"home page","index":1},"http":{"request":{"url":"http://www.elastic.co/","method":"GET"}},"trace.id": "t1", "span.id": "s1"} + + +// Step - blog page +{"type":"step/end","journey":{"name":"elastic e2e"},"step":{"name":"blog page","index":2,"status":"failed","duration":{"us":17382122}}, "trace.id": "t2"} +{"type":"journey/network_info","journey":{"name":"elastic e2e"},"step":{"name":"blog page","index":2},"http":{"request":{"url":"http://www.elastic.co/blog","method":"GET"}},"trace.id": "t2", "span.id": "s2"} +``` + +With this tracing information available in the ES documents for each step's network requests, the Synthetics UI can link back to the individual backend transactions in APM. + +#### HTTP Checks + +For the below HTTP monitor + +```yml +# heartbeat.yml +heartbeat.monitors: +- type: http + id: test-http + urls: ["https://www.example.com"] + apm: + enabled: true +``` + +Heartbeat would add the `traceparent` header to the monitored URL and add the +other tracing related information to the ES documents. + +```json +{"event":{"action":"monitor.run"},"monitor":{"id":"test-http","type":"http","status":"up","duration":{"ms":112}}, "trace.id": "t1", "span.id": "s1"} +``` + +It's important to note that there is no dedicated waterfall information for the HTTP checks in the Synthetics UI. Consequently, the linking here will directly take you to the transaction if the backend is also traced by Elastic APM or OTel (OpenTelemetry)-based agents. This works similar to the Browser checks where the network request is directly linked to the transaction. + +**NOTE: The correlation remain applicable even if downstream services are traced by OpenTelemetry (OTel)-based agents. This ensures a consistent and seamless tracing experience regardless of the underlying tracing infrastructure.** + +### Identifying Synthetics trace + +When tracing is enabled on the Synthetics monitors, the agent appends the `Elastic/Synthetics` to the HTTP `User-Agent` header for all outgoing requests. Tracing UI can use this information to identify the traces that are originated from +Synthetics using the following approaches. + +- Elastic APM agents + - The information is stored in `http.headers.user-agent` +- OTel agents + - The information is stored in `user_agent.original` + +UI will check both of these fields to identify the Synthetics traces and will +prefer `user_agent.original` if both are present. + +There is a limitation with this approach +- users can override the `User-Agent` header in the monitor configuration which + might lead to users seeing only partial traces on APM UI. + +When a trace is confirmed to be originated from Synthetics-based monitors, the +Trace Explorer view can be linked back to the Synthetics waterfall. + +- `/app/synthetics/link-to/:span.id` + - links back to the explicit browser waterfall step on the Synthetics UI, and + it follows the format `/monitor/:monitorId/test-run/:runId/step/:stepIndex#:spanId`. \ No newline at end of file diff --git a/docs/terminology.md b/specs/terminology.md similarity index 95% rename from docs/terminology.md rename to specs/terminology.md index 7b86e754..378f0e12 100644 --- a/docs/terminology.md +++ b/specs/terminology.md @@ -62,7 +62,10 @@ A trace is a grouping of spans and transactions that all share the same `trace.i ## Sampling -To reduce processing and storage overhead, transactions may be "sampled". Sampling limits the amount of data that is captured for transactions: non-sampled transactions will not record context, and related spans will not be captured. +To reduce processing and storage overhead, transactions may be "sampled". +Non-sampled transactions and spans will not be stored in Elasticsearch. + +In versions prior to 8.0, non-sampled transactions were stored, but no context was recorded and no related spans were captured. #### Adaptive sampling TODO diff --git a/tests/README.md b/tests/agents/README.md similarity index 74% rename from tests/README.md rename to tests/agents/README.md index b2470fc4..88fafea0 100644 --- a/tests/README.md +++ b/tests/agents/README.md @@ -6,9 +6,18 @@ Files provided here may be used by agents to ensure matching results across lang SQL-based data stores' span names are abbreviated versions of their queries, eg. `SELECT * FROM users WHERE id=1` becomes `SELECT FROM users`. +### For precision + +- `tests/agents/sql_token_examples.json` +- `tests/agents/sql_signature_examples.json` + To get similar results across agents a set of `input -> expected output` examples are provided here as JSON files. Using or complying to these isn't a requirement. - Reference issue: [elastic/apm#12](https://github.com/elastic/apm/issues/12). - Reference doc: [RFC: SQL parsing](https://docs.google.com/document/d/1sblkAP1NHqk4MtloUta7tXjDuI_l64sT2ZQ_UFHuytA/) + +### For performance + +See [this distribution](https://github.com/johnthebrave/nlidb-datasets) for example queries. diff --git a/tests/agents/gherkin-specs/api_key.feature b/tests/agents/gherkin-specs/api_key.feature new file mode 100644 index 00000000..ae7ea9ab --- /dev/null +++ b/tests/agents/gherkin-specs/api_key.feature @@ -0,0 +1,24 @@ +Feature: APM server authentication with API key and secret token + + Scenario: A configured API key is sent in the Authorization header + Given an agent configured with + | setting | value | + | api_key | RTNxMjlXNEJt | + When the agent sends a request to APM server + Then the Authorization header of the request is 'ApiKey RTNxMjlXNEJt' + + Scenario: A configured secret token is sent in the Authorization header + Given an agent configured with + | setting | value | + | secret_token | secr3tT0ken | + When the agent sends a request to APM server + Then the Authorization header of the request is 'Bearer secr3tT0ken' + + Scenario: A configured API key takes precedence over a secret token + Given an agent configured with + | setting | value | + | api_key | MjlXNEJasdfDt | + | secret_token | secr3tT0ken | + When the agent sends a request to APM server + Then the Authorization header of the request is 'ApiKey MjlXNEJasdfDt' + diff --git a/tests/agents/gherkin-specs/azure_app_service_metadata.feature b/tests/agents/gherkin-specs/azure_app_service_metadata.feature new file mode 100644 index 00000000..38575a7e --- /dev/null +++ b/tests/agents/gherkin-specs/azure_app_service_metadata.feature @@ -0,0 +1,73 @@ +Feature: Extracting Metadata for Azure App Service + + Background: + Given an agent configured with + | setting | value | + | cloud_provider | azure | + + Scenario Outline: Azure App Service with all environment variables present in expected format + Given the following environment variables are present + | name | value | + | WEBSITE_OWNER_NAME | | + | WEBSITE_RESOURCE_GROUP | resource_group | + | WEBSITE_SITE_NAME | site_name | + | WEBSITE_INSTANCE_ID | instance_id | + When cloud metadata is collected + Then cloud metadata is not null + And cloud metadata 'account.id' is 'f5940f10-2e30-3e4d-a259-63451ba6dae4' + And cloud metadata 'provider' is 'azure' + And cloud metadata 'instance.id' is 'instance_id' + And cloud metadata 'instance.name' is 'site_name' + And cloud metadata 'project.name' is 'resource_group' + And cloud metadata 'region' is 'AustraliaEast' + Examples: + | WEBSITE_OWNER_NAME | + | f5940f10-2e30-3e4d-a259-63451ba6dae4+elastic-apm-AustraliaEastwebspace | + | f5940f10-2e30-3e4d-a259-63451ba6dae4+appsvc_linux_australiaeast-AustraliaEastwebspace-Linux | + + # WEBSITE_OWNER_NAME is expected to include a + character + Scenario: WEBSITE_OWNER_NAME environment variable not expected format + Given the following environment variables are present + | name | value | + | WEBSITE_OWNER_NAME | f5940f10-2e30-3e4d-a259-63451ba6dae4-elastic-apm-AustraliaEastwebspace | + | WEBSITE_RESOURCE_GROUP | resource_group | + | WEBSITE_SITE_NAME | site_name | + | WEBSITE_INSTANCE_ID | instance_id | + When cloud metadata is collected + Then cloud metadata is null + + Scenario: Missing WEBSITE_OWNER_NAME environment variable + Given the following environment variables are present + | name | value | + | WEBSITE_RESOURCE_GROUP | resource_group | + | WEBSITE_SITE_NAME | site_name | + | WEBSITE_INSTANCE_ID | instance_id | + When cloud metadata is collected + Then cloud metadata is null + + Scenario: Missing WEBSITE_RESOURCE_GROUP environment variable + Given the following environment variables are present + | name | value | + | WEBSITE_OWNER_NAME | f5940f10-2e30-3e4d-a259-63451ba6dae4+elastic-apm-AustraliaEastwebspace | + | WEBSITE_SITE_NAME | site_name | + | WEBSITE_INSTANCE_ID | instance_id | + When cloud metadata is collected + Then cloud metadata is null + + Scenario: Missing WEBSITE_SITE_NAME environment variable + Given the following environment variables are present + | name | value | + | WEBSITE_OWNER_NAME | f5940f10-2e30-3e4d-a259-63451ba6dae4+elastic-apm-AustraliaEastwebspace | + | WEBSITE_RESOURCE_GROUP | resource_group | + | WEBSITE_INSTANCE_ID | instance_id | + When cloud metadata is collected + Then cloud metadata is null + + Scenario: Missing WEBSITE_INSTANCE_ID environment variable + Given the following environment variables are present + | name | value | + | WEBSITE_OWNER_NAME | f5940f10-2e30-3e4d-a259-63451ba6dae4+elastic-apm-AustraliaEastwebspace | + | WEBSITE_RESOURCE_GROUP | resource_group | + | WEBSITE_SITE_NAME | site_name | + When cloud metadata is collected + Then cloud metadata is null \ No newline at end of file diff --git a/tests/agents/gherkin-specs/azure_functions_metadata.feature b/tests/agents/gherkin-specs/azure_functions_metadata.feature new file mode 100644 index 00000000..aabb10d6 --- /dev/null +++ b/tests/agents/gherkin-specs/azure_functions_metadata.feature @@ -0,0 +1,70 @@ +Feature: Extracting Metadata for Azure Function Apps + + Background: + Given an agent configured with + | setting | value | + | cloud_provider | azure | + + Scenario Outline: Azure Function App with minimum set of environment variables present in expected format + Given the following environment variables are present + | name | value | + | FUNCTIONS_EXTENSION_VERSION | version | + | WEBSITE_OWNER_NAME | d2cd53b3-acdc-4964-9563-3f5201556a81+faas_group-CentralUSwebspace-Linux | + | WEBSITE_SITE_NAME | site_name | + When cloud metadata is collected + Then cloud metadata is not null + And cloud metadata 'account.id' is 'd2cd53b3-acdc-4964-9563-3f5201556a81' + And cloud metadata 'provider' is 'azure' + And cloud metadata 'service.name' is 'functions' + And cloud metadata 'instance.name' is 'site_name' + And cloud metadata 'project.name' is 'faas_group' + And cloud metadata 'region' is 'CentralUS' + + Scenario Outline: Azure Function App with typical set of environment variables present in expected format + Given the following environment variables are present + | name | value | + | FUNCTIONS_EXTENSION_VERSION | version | + | WEBSITE_OWNER_NAME | d2cd53b3-acdc-4964-9563-3f5201556a81+faas_group-CentralUSwebspace-Linux | + | WEBSITE_SITE_NAME | site_name | + | REGION_NAME | Central US | + | WEBSITE_RESOURCE_GROUP | faas_group_from_env | + When cloud metadata is collected + Then cloud metadata is not null + And cloud metadata 'account.id' is 'd2cd53b3-acdc-4964-9563-3f5201556a81' + And cloud metadata 'provider' is 'azure' + And cloud metadata 'service.name' is 'functions' + And cloud metadata 'instance.name' is 'site_name' + And cloud metadata 'project.name' is 'faas_group_from_env' + And cloud metadata 'region' is 'Central US' + + Scenario: WEBSITE_OWNER_NAME environment variable not expected format + Given the following environment variables are present + | name | value | + | WEBSITE_OWNER_NAME | d2cd53b3-acdc-4964-9563-3f5201556a81-faas_group-CentralUSwebspace-Linux | + | WEBSITE_SITE_NAME | site_name | + When cloud metadata is collected + Then cloud metadata is null + + Scenario: Missing FUNCTIONS_EXTENSION_VERSION environment variable + Given the following environment variables are present + | name | value | + | WEBSITE_OWNER_NAME | d2cd53b3-acdc-4964-9563-3f5201556a81+faas_group-CentralUSwebspace-Linux | + | WEBSITE_SITE_NAME | site_name | + When cloud metadata is collected + Then cloud metadata is null + + Scenario: Missing WEBSITE_OWNER_NAME environment variable + Given the following environment variables are present + | name | value | + | FUNCTIONS_EXTENSION_VERSION | version | + | WEBSITE_SITE_NAME | site_name | + When cloud metadata is collected + Then cloud metadata is null + + Scenario: Missing WEBSITE_SITE_NAME environment variable + Given the following environment variables are present + | name | value | + | FUNCTIONS_EXTENSION_VERSION | version | + | WEBSITE_OWNER_NAME | d2cd53b3-acdc-4964-9563-3f5201556a81+faas_group-CentralUSwebspace-Linux | + When cloud metadata is collected + Then cloud metadata is null diff --git a/tests/agents/gherkin-specs/otel_bridge.feature b/tests/agents/gherkin-specs/otel_bridge.feature new file mode 100644 index 00000000..901d9315 --- /dev/null +++ b/tests/agents/gherkin-specs/otel_bridge.feature @@ -0,0 +1,249 @@ +@opentelemetry-bridge +Feature: OpenTelemetry bridge + + # --- Creating Elastic span or transaction from OTel span + + Scenario: Create transaction from OTel span with remote context + Given an agent + And OTel span is created with remote context as parent + Then Elastic bridged object is a transaction + Then Elastic bridged transaction has remote context as parent + + Scenario: Create root transaction from OTel span without parent + Given an agent + And OTel span is created without parent + And OTel span ends + Then Elastic bridged object is a transaction + Then Elastic bridged transaction is a root transaction + # outcome should not be inferred from the lack/presence of errors + Then Elastic bridged transaction outcome is "unknown" + + Scenario: Create span from OTel span + Given an agent + And OTel span is created with local context as parent + And OTel span ends + Then Elastic bridged object is a span + Then Elastic bridged span has local context as parent + # outcome should not be inferred from the lack/presence of errors + Then Elastic bridged span outcome is "unknown" + + # --- OTel span kind mapping for spans & transactions + + Scenario Outline: OTel span kind for spans & default span type & subtype + Given an agent + And an active transaction + And OTel span is created with kind "" + And OTel span ends + Then Elastic bridged object is a span + Then Elastic bridged span OTel kind is "" + Then Elastic bridged span type is "" + Then Elastic bridged span subtype is "" + Examples: + | kind | default_type | default_subtype | + | INTERNAL | app | internal | + | SERVER | unknown | | + | CLIENT | unknown | | + | PRODUCER | unknown | | + | CONSUMER | unknown | | + + Scenario Outline: OTel span kind for transactions & default transaction type + Given an agent + And OTel span is created with kind "" + And OTel span ends + Then Elastic bridged object is a transaction + Then Elastic bridged transaction OTel kind is "" + Then Elastic bridged transaction type is 'unknown' + Examples: + | kind | + | INTERNAL | + | SERVER | + | CLIENT | + | PRODUCER | + | CONSUMER | + + # OTel span status mapping for spans & transactions + + Scenario Outline: OTel span mapping with status for transactions + Given an agent + And OTel span is created with kind 'SERVER' + And OTel span status set to "" + And OTel span ends + Then Elastic bridged object is a transaction + Then Elastic bridged transaction outcome is "" + Then Elastic bridged transaction result is not set + Examples: + | status | outcome | + | unset | unknown | + | ok | success | + | error | failure | + + Scenario Outline: OTel span mapping with status for spans + Given an agent + Given an active transaction + And OTel span is created with kind 'INTERNAL' + And OTel span status set to "" + And OTel span ends + Then Elastic bridged object is a span + Then Elastic bridged span outcome is "" + Examples: + | status | outcome | + | unset | unknown | + | ok | success | + | error | failure | + + # --- span type, subtype and action inference from OTel attributes + + # --- HTTP server + # https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md#http-server + Scenario Outline: HTTP server [ ] + Given an agent + And OTel span is created with kind 'SERVER' + And OTel span has following attributes + | http.url | | + | http.scheme | | + And OTel span ends + Then Elastic bridged object is a transaction + Then Elastic bridged transaction type is "request" + Examples: + | http.url | http.scheme | + | http://testing.invalid/ | | + | | http | + + # --- HTTP client + # https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/http.md#http-client + Scenario Outline: HTTP client [ ] + Given an agent + And an active transaction + And OTel span is created with kind 'CLIENT' + And OTel span has following attributes + | http.url | | + | http.scheme | | + | http.host | | + | net.peer.ip | | + | net.peer.name | | + | net.peer.port | | + And OTel span ends + Then Elastic bridged span type is 'external' + Then Elastic bridged span subtype is 'http' + Then Elastic bridged span OTel attributes are copied as-is + Then Elastic bridged span destination resource is set to "" + Then Elastic bridged span service target type is 'http' and name is "" + Examples: + | http.url | http.scheme | http.host | net.peer.ip | net.peer.name | net.peer.port | target_service_name | + | https://testing.invalid:8443/ | | | | | | testing.invalid:8443 | + | https://[::1]/ | | | | | | [::1]:443 | + | http://testing.invalid/ | | | | | | testing.invalid:80 | + | | http | testing.invalid | | | | testing.invalid:80 | + | | https | testing.invalid | 127.0.0.1 | | | testing.invalid:443 | + | | http | | 127.0.0.1 | | 81 | 127.0.0.1:81 | + | | https | | 127.0.0.1 | | 445 | 127.0.0.1:445 | + | | http | | 127.0.0.1 | host1 | 445 | host1:445 | + | | https | | 127.0.0.1 | host2 | 445 | host2:445 | + + # --- DB client + # https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/database.md + Scenario Outline: DB client [ ] + Given an agent + And an active transaction + And OTel span is created with kind 'CLIENT' + And OTel span has following attributes + | db.system | | + | db.name | | + | net.peer.ip | | + | net.peer.name | | + | net.peer.port | | + And OTel span ends + Then Elastic bridged span type is 'db' + Then Elastic bridged span subtype is "" + Then Elastic bridged span OTel attributes are copied as-is + Then Elastic bridged span destination resource is set to "" + Then Elastic bridged span service target type is "" and name is "" + Examples: + | db.system | db.name | net.peer.ip | net.peer.name | net.peer.port | resource | target_service_name | + | mysql | | | | | mysql | | + | oracle | | | oracledb | | oracle | | + | oracle | | 127.0.0.1 | | | oracle | | + | mysql | | 127.0.0.1 | dbserver | 3307 | mysql | | + | mysql | myDb | | | | mysql/myDb | myDb | + | oracle | myDb | | oracledb | | oracle/myDb | myDb | + | oracle | myDb | 127.0.0.1 | | | oracle/myDb | myDb | + | mysql | myDb | 127.0.0.1 | dbserver | 3307 | mysql/myDb | myDb | + + # --- Messaging consumer (transaction consuming/receiving a message) + # https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/messaging.md + Scenario: Messaging consumer + Given an agent + And OTel span is created with kind 'CONSUMER' + And OTel span has following attributes + | messaging.system | anything | + And OTel span ends + Then Elastic bridged transaction type is 'messaging' + + # --- Messaging producer (client span emitting a message) + # https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/messaging.md + Scenario Outline: Messaging producer [ ] + Given an agent + And an active transaction + And OTel span is created with kind 'PRODUCER' + And OTel span has following attributes + | messaging.system | | + | messaging.destination | | + | messaging.url | | + | net.peer.ip | | + | net.peer.name | | + | net.peer.port | | + And OTel span ends + Then Elastic bridged span type is 'messaging' + Then Elastic bridged span subtype is "" + Then Elastic bridged span OTel attributes are copied as-is + Then Elastic bridged span destination resource is set to "" + Then Elastic bridged span service target type is "" and name is "" + Examples: + | messaging.system | messaging.destination | messaging.url | net.peer.ip | net.peer.name | net.peer.port | resource | target_service_name | + | rabbitmq | | amqp://carrot:4444/q1 | | | | rabbitmq | | + | rabbitmq | | | 127.0.0.1 | carrot-server | 7777 | rabbitmq | | + | rabbitmq | | | | carrot-server | | rabbitmq | | + | rabbitmq | | | 127.0.0.1 | | | rabbitmq | | + | rabbitmq | myQueue | amqp://carrot:4444/q1 | | | | rabbitmq/myQueue | myQueue | + | rabbitmq | myQueue | | 127.0.0.1 | carrot-server | 7777 | rabbitmq/myQueue | myQueue | + | rabbitmq | myQueue | | | carrot-server | | rabbitmq/myQueue | myQueue | + | rabbitmq | myQueue | | 127.0.0.1 | | | rabbitmq/myQueue | myQueue | + + # --- RPC client + # https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/rpc.md + Scenario Outline: RPC client [ ] + Given an agent + And an active transaction + And OTel span is created with kind 'CLIENT' + And OTel span has following attributes + | rpc.system | | + | rpc.service | | + | net.peer.ip | | + | net.peer.name | | + | net.peer.port | | + And OTel span ends + Then Elastic bridged span type is 'external' + Then Elastic bridged span subtype is "" + Then Elastic bridged span OTel attributes are copied as-is + Then Elastic bridged span destination resource is set to "" + Then Elastic bridged span service target type is "" and name is "" + Examples: + | rpc.system | rpc.service | net.peer.ip | net.peer.name | net.peer.port | resource | target_service_name | + | grpc | | | | | grpc | | + | grpc | myService | | | | myService | myService | + | grpc | myService | | rpc-server | | rpc-server | rpc-server | + | grpc | myService | 127.0.0.1 | rpc-server | | rpc-server | rpc-server | + | grpc | | 127.0.0.1 | rpc-server | 7777 | rpc-server:7777 | rpc-server:7777 | + | grpc | myService | 127.0.0.1 | rpc-server | 7777 | rpc-server:7777 | rpc-server:7777 | + | grpc | myService | 127.0.0.1 | | 7777 | 127.0.0.1:7777 | 127.0.0.1:7777 | + + # --- RPC server + # https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/rpc.md + Scenario: RPC server + Given an agent + And OTel span is created with kind 'SERVER' + And OTel span has following attributes + | rpc.system | grpc | + And OTel span ends + Then Elastic bridged transaction type is 'request' + diff --git a/tests/agents/gherkin-specs/outcome.feature b/tests/agents/gherkin-specs/outcome.feature new file mode 100644 index 00000000..074a5333 --- /dev/null +++ b/tests/agents/gherkin-specs/outcome.feature @@ -0,0 +1,104 @@ +Feature: Outcome + + Background: An agent with default configuration + Given an agent + + # ---- user set outcome + + Scenario: User set outcome on span has priority over instrumentation + Given an active span + And the agent sets the span outcome to 'success' + And a user sets the span outcome to 'failure' + When the span ends + Then the span outcome is 'failure' + + Scenario: User set outcome on transaction has priority over instrumentation + Given an active transaction + And the agent sets the transaction outcome to 'failure' + And a user sets the transaction outcome to 'unknown' + When the transaction ends + Then the transaction outcome is 'unknown' + + # ---- span & transaction outcome from reported errors + + Scenario: span with error + Given an active span + And an error is reported to the span + When the span ends + Then the span outcome is 'failure' + + Scenario: span without error + Given an active span + When the span ends + Then the span outcome is 'success' + + Scenario: transaction with error + Given an active transaction + And an error is reported to the transaction + When the transaction ends + Then the transaction outcome is 'failure' + + Scenario: transaction without error + Given an active transaction + When the transaction ends + Then the transaction outcome is 'success' + + # ---- HTTP + + @http + Scenario Outline: HTTP transaction and span outcome + Given an active transaction + And a HTTP call is received that returns + When the transaction ends + Then the transaction outcome is '' + Given an active span + And a HTTP call is made that returns + When the span ends + Then the span outcome is '' + Examples: + | status | client | server | + | 100 | success | success | + | 200 | success | success | + | 300 | success | success | + | 400 | failure | success | + | 404 | failure | success | + | 500 | failure | failure | + | -1 | failure | failure | + # last row with negative status represents the case where the status is not available + # for example when an exception/error is thrown without status (IO error, redirect loop, ...) + + # ---- gRPC + + # reference spec : https://github.com/grpc/grpc/blob/master/doc/statuscodes.md + + @grpc + Scenario Outline: gRPC transaction and span outcome + Given an active transaction + And a gRPC call is received that returns '' + When the transaction ends + Then the transaction outcome is '' + Given an active span + And a gRPC call is made that returns '' + When the span ends + Then the span outcome is '' + Examples: + | status | client | server | + | OK | success | success | + | CANCELLED | failure | success | + | UNKNOWN | failure | failure | + | INVALID_ARGUMENT | failure | success | + | DEADLINE_EXCEEDED | failure | failure | + | NOT_FOUND | failure | success | + | ALREADY_EXISTS | failure | success | + | PERMISSION_DENIED | failure | success | + | RESOURCE_EXHAUSTED | failure | failure | + | FAILED_PRECONDITION | failure | failure | + | ABORTED | failure | failure | + | OUT_OF_RANGE | failure | success | + | UNIMPLEMENTED | failure | success | + | INTERNAL | failure | failure | + | UNAVAILABLE | failure | failure | + | DATA_LOSS | failure | failure | + | UNAUTHENTICATED | failure | success | + | n/a | failure | failure | + # last row with 'n/a' status represents the case where status is not available diff --git a/tests/agents/gherkin-specs/user_agent.feature b/tests/agents/gherkin-specs/user_agent.feature new file mode 100644 index 00000000..af7c168b --- /dev/null +++ b/tests/agents/gherkin-specs/user_agent.feature @@ -0,0 +1,32 @@ +Feature: Agent Transport User agent Header + + Scenario: Default user-agent + Given an agent + When the agent sends a request to APM server + Then the User-Agent header of the request matches regex '^apm-agent-[a-z]+/[^ ]* \(.*\)' + + Scenario: Default user-agent when setting invalid service + Given an agent configured with + | setting | value | + | service_name | myService/()<>@ | + When the agent sends a request to APM server + Then the User-Agent header of the request matches regex '^apm-agent-[a-z]+/[^ ]* \(.*\)' + + Scenario: User-agent with service name only + Given an agent configured with + | setting | value | + | service_name | myService | + When the agent sends a request to APM server + Then the User-Agent header of the request matches regex '^apm-agent-[a-z]+/[^ ]* \(myService\)' + + Scenario Outline: User-agent with service name and service version + Given an agent configured with + | setting | value | + | service_name | | + | service_version | | + When the agent sends a request to APM server + Then the User-Agent header of the request matches regex '^apm-agent-[a-z]+/[^ ]* \( \)' + Examples: + | SERVICE_NAME | ESCAPED_SERVICE_NAME | SERVICE_VERSION | ESCAPED_SERVICE_VERSION | + | myService | myService | v42 | v42 | + | myService | myService | 123(:\;)456 | 123_:_;_456 | diff --git a/tests/agents/json-specs/container_metadata_discovery.json b/tests/agents/json-specs/container_metadata_discovery.json new file mode 100644 index 00000000..d1797ca0 --- /dev/null +++ b/tests/agents/json-specs/container_metadata_discovery.json @@ -0,0 +1,100 @@ +{ + "cgroup_v1_underscores": { + "files": { + "/proc/self/cgroup": [ + "1:name=systemd:/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod90d81341_92de_11e7_8cf2_507b9d4141fa.slice/crio-2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63.scope" + ] + }, + "containerId": "2227daf62df6694645fee5df53c1f91271546a9560e8600a525690ae252b7f63", + "podId": "90d81341-92de-11e7-8cf2-507b9d4141fa" + }, + "cgroup_v1_openshift": { + "files": { + "/proc/self/cgroup": [ + "9:freezer:/kubepods.slice/kubepods-pod22949dce_fd8b_11ea_8ede_98f2b32c645c.slice/docker-b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f.scope" + ] + }, + "containerId": "b15a5bdedd2e7645c3be271364324321b908314e4c77857bbfd32a041148c07f", + "podId": "22949dce-fd8b-11ea-8ede-98f2b32c645c" + }, + "cgroup_v1_ubuntu": { + "files": { + "/proc/self/cgroup": [ + "1:name=systemd:/user.slice/user-1000.slice/user@1000.service/apps.slice/apps-org.gnome.Terminal.slice/vte-spawn-75bc72bd-6642-4cf5-b62c-0674e11bfc84.scope" + ] + }, + "containerId": null, + "podId": null + }, + "cgroup_v1_awsEcs": { + "files": { + "/proc/self/cgroup": [ + "1:name=systemd:/ecs/03752a671e744971a862edcee6195646/03752a671e744971a862edcee6195646-4015103728" + ] + }, + "containerId": "03752a671e744971a862edcee6195646-4015103728", + "podId": null + }, + "cgroup_v2": { + "files": { + "/proc/self/cgroup": [ + "0::/" + ], + "/proc/self/mountinfo": [ + "3984 3905 0:73 / / rw,relatime shared:1863 master:1733 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/KEX7CWLHQCXQY2RHPGTXJ3C26N:/var/lib/docker/overlay2/l/2PVS7JRTRSTVZS4KSUAFML3BIV:/var/lib/docker/overlay2/l/52M7ARM4JDVHCJAYUI6JIKBO4B,upperdir=/var/lib/docker/overlay2/267f825fb89e584605bf161177451879c0ba8b15f7df9b51fb7843c7beb9ed25/diff,workdir=/var/lib/docker/overlay2/267f825fb89e584605bf161177451879c0ba8b15f7df9b51fb7843c7beb9ed25/work", + "3985 3984 0:77 / /proc rw,nosuid,nodev,noexec,relatime shared:1864 - proc proc rw", + "3986 3984 0:78 / /dev rw,nosuid shared:1865 - tmpfs tmpfs rw,size=65536k,mode=755,inode64", + "3987 3986 0:79 / /dev/pts rw,nosuid,noexec,relatime shared:1866 - devpts devpts rw,gid=5,mode=620,ptmxmode=666", + "3988 3984 0:80 / /sys ro,nosuid,nodev,noexec,relatime shared:1870 - sysfs sysfs ro", + "3989 3988 0:30 / /sys/fs/cgroup rw,nosuid,nodev,noexec,relatime shared:1871 - cgroup2 cgroup rw", + "3990 3986 0:76 / /dev/mqueue rw,nosuid,nodev,noexec,relatime shared:1867 - mqueue mqueue rw", + "3991 3986 0:81 / /dev/shm rw,nosuid,nodev,noexec,relatime shared:1868 - tmpfs shm rw,size=65536k,inode64", + "3992 3984 253:1 /var/lib/docker/volumes/9d18ce5b36572d85358fa936afe5a4bf95cca5c822b04941aa08c6118f6e0d33/_data /var rw,relatime shared:1872 master:1 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", + "3993 3984 0:82 / /run rw,nosuid,nodev,noexec,relatime shared:1873 - tmpfs tmpfs rw,inode64", + "3994 3984 0:83 / /tmp rw,nosuid,nodev,noexec,relatime shared:1874 - tmpfs tmpfs rw,inode64", + "3995 3984 253:1 /usr/lib/modules /usr/lib/modules ro,relatime shared:1875 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", + "3996 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/resolv.conf /etc/resolv.conf rw,relatime shared:1876 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", + "3997 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/hostname /etc/hostname rw,relatime shared:1877 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro", + "3998 3984 253:1 /var/lib/docker/containers/6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6/hosts /etc/hosts rw,relatime shared:1878 - ext4 /dev/mapper/vgubuntu-root rw,errors=remount-ro" + ] + }, + "containerId": "6548c6863fb748e72d1e2a4f824fde92f720952d062dede1318c2d6219a672d6", + "podId": null + }, + "gardener": { + "files": { + "/proc/self/mountinfo": [ + "10112 5519 0:864 / / ro,relatime master:1972 - overlay overlay rw,lowerdir=/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/35235/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/27346/fs:/var/lib/containerd/io.containerd.snapshotter.v1.overlayfs/snapshots/27345/fs:/var/lib/containerd/io.containerd.snapsh", + "10113 10112 0:884 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw", + "10301 10112 0:926 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755,inode64", + "10302 10301 0:930 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666", + "10519 10301 0:820 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw", + "10520 10112 0:839 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro", + "10716 10520 0:26 /kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod121157b5_c67d_4c3e_9052_cb27bbb711fb.slice/cri-containerd-1cd3449e930b8a28c7595240fa32ba20c84f36d059e5fbe63104ad40057992d1.scope /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw", + "10736 10112 8:3 /var/lib/kubelet/pods/121157b5-c67d-4c3e-9052-cb27bbb711fb/volumes/kubernetes.io~empty-dir/tmpdir /tmp rw,relatime - ext4 /dev/sda3 rw,discard,prjquota,errors=remount-ro", + "10737 10112 0:786 / /vault/tls ro,relatime - tmpfs tmpfs rw,size=4194304k,inode64", + "10738 10112 8:3 /var/lib/kubelet/pods/121157b5-c67d-4c3e-9052-cb27bbb711fb/etc-hosts /etc/hosts rw,relatime - ext4 /dev/sda3 rw,discard,prjquota,errors=remount-ro", + "10739 10301 8:3 /var/lib/kubelet/pods/121157b5-c67d-4c3e-9052-cb27bbb711fb/containers/application-search-indexer/9bf2b38c /dev/termination-log rw,relatime - ext4 /dev/sda3 rw,discard,prjquota,errors=remount-ro", + "10740 10112 8:3 /var/lib/containerd/io.containerd.grpc.v1.cri/sandboxes/26a006f558da58874bc37863efe9d2b5d715afc54453d95b22a7809a4e65566c/hostname /etc/hostname ro,relatime - ext4 /dev/sda3 rw,discard,prjquota,errors=remount-ro", + "10741 10112 8:3 /var/lib/containerd/io.containerd.grpc.v1.cri/sandboxes/26a006f558da58874bc37863efe9d2b5d715afc54453d95b22a7809a4e65566c/resolv.conf /etc/resolv.conf ro,relatime - ext4 /dev/sda3 rw,discard,prjquota,errors=remount-ro", + "10761 10301 0:788 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k,inode64", + "10762 10112 0:787 / /var/run/secrets/kubernetes.io/serviceaccount ro,relatime - tmpfs tmpfs rw,size=4194304k,inode64", + "5630 10113 0:884 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw", + "5631 10113 0:884 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw", + "5632 10113 0:884 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw", + "5633 10113 0:884 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw", + "5634 10113 0:931 / /proc/acpi ro,relatime - tmpfs tmpfs ro,inode64", + "5635 10113 0:926 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755,inode64", + "5636 10113 0:926 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755,inode64", + "5637 10113 0:926 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755,inode64", + "5639 10520 0:932 / /sys/firmware ro,relatime - tmpfs tmpfs ro,inode64" + ], + "/proc/self/cgroup": [ + "0::/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod121157b5_c67d_4c3e_9052_cb27bbb711fb.slice/cri-containerd-1cd3449e930b8a28c7595240fa32ba20c84f36d059e5fbe63104ad40057992d1.scope" + ] + }, + "containerId": "1cd3449e930b8a28c7595240fa32ba20c84f36d059e5fbe63104ad40057992d1", + "podId": "121157b5-c67d-4c3e-9052-cb27bbb711fb" + } +} + diff --git a/tests/agents/json-specs/service_resource_inference.json b/tests/agents/json-specs/service_resource_inference.json new file mode 100644 index 00000000..12ef6b13 --- /dev/null +++ b/tests/agents/json-specs/service_resource_inference.json @@ -0,0 +1,275 @@ +[ + { + "span": { + "exit": "true", + "type": "custom", + "subtype": "test-subtype" + }, + "expected_resource": "test-subtype", + "expected_service_target": { + "type": "test-subtype" + }, + "failure_message": "In the absence of specific context fields, subtype should used" + }, + { + "span": { + "exit": "true", + "type": "custom", + "subtype": "test-subtype", + "context": { + "service": { + "target": { + "type": "custom-service-type", + "name": "custom-service-name" + } + } + } + }, + "expected_resource": "custom-service-type/custom-service-name", + "expected_service_target": { + "type": "custom-service-type", + "name": "custom-service-name" + }, + "failure_message": "If the `service target type or name` is already set, the inference mechanism should not override it" + }, + { + "span": { + "exit": "true", + "type": "custom" + }, + "expected_resource": "custom", + "expected_service_target": { + "type": "custom" + }, + "failure_message": "In the absence of specific context fields and absence of subtype, the type should be used" + }, + { + "span": { + "exit": "false", + "type": "custom", + "subtype": "test-subtype" + }, + "expected_resource": null, + "expected_service_target": null, + "failure_message": "The output for non-exit spans should be `null`" + }, + { + "span": { + "exit": "false", + "type": "custom", + "subtype": "proprietary-db", + "context": { + "db": { + "instance": "myInstance" + } + } + }, + "expected_resource": null, + "expected_service_target": null, + "failure_message": "The output for non-exit spans should be `null` even if exit-related context data is set" + }, + { + "span": { + "exit": "true", + "type": "db", + "subtype": "mysql", + "context": { + "db": { + "instance": "myInstance" + } + } + }, + "expected_resource": "mysql/myInstance", + "expected_service_target": { + "type": "mysql", + "name": "myInstance" + }, + "failure_message": "If `context.db.instance` exists, the output should be: `${subtype}/${context.db.instance}`" + }, + { + "span": { + "exit": "true", + "type": "db", + "subtype": "mysql", + "context": { + "db": { + "type": "sql" + } + } + }, + "expected_resource": "mysql", + "expected_service_target": { + "type": "mysql" + }, + "failure_message": "If `context.db` exists without `context.db.instance`, the subtype should be used" + }, + { + "span": { + "exit": "true", + "type": "db", + "context": { + "db": { + "instance": "myInstance" + } + } + }, + "expected_resource": "db/myInstance", + "expected_service_target": { + "type": "db", + "name": "myInstance" + }, + "failure_message": "If `context.db.instance` exists and subtype is `null`, the output should be: `${type}/${context.db.instance}`" + }, + { + "span": { + "exit": "true", + "type": "db", + "subtype": "elasticsearch", + "context": { + "db": { + "type": "elasticsearch" + }, + "http": { + "url": "https://my-cluster.com:9200" + } + } + }, + "expected_resource": "elasticsearch", + "expected_service_target": { + "type": "elasticsearch" + }, + "failure_message": "If `context.db` exists without `context.db.instance`, the subtype should be used, even if `context.http` exists" + }, + { + "span": { + "exit": "true", + "type": "messaging", + "subtype": "msg-http-client", + "context": { + "message": { + "body": "Text message" + }, + "http": { + "url": "https://my-broker.com:8888" + } + } + }, + "expected_resource": "msg-http-client", + "expected_service_target": { + "type": "msg-http-client" + }, + "failure_message": "If `context.message` exists without `context.message.queue.name`, the subtype should be used, even if `context.http` exists" + }, + { + "span": { + "exit": "true", + "type": "external", + "subtype": "http", + "context": { + "http": { + "url": "http://my-cluster.com:9200" + } + } + }, + "expected_resource": "my-cluster.com:9200", + "expected_service_target": { + "type": "http", + "name": "my-cluster.com:9200" + }, + "failure_message": "If `context.http.url` exists, output should be `${context.http.url}`" + }, + { + "span": { + "exit": "true", + "type": "external", + "subtype": "http", + "context": { + "http": { + "url": "https://my-cluster.com" + } + } + }, + "expected_resource": "my-cluster.com:443", + "expected_service_target": { + "type": "http", + "name": "my-cluster.com:443" + }, + "failure_message": "`context.http.url` without an explicit default HTTPS port, output should be reported as `${context.http.url}:443`" + }, + { + "span": { + "exit": "true", + "type": "external", + "subtype": "http", + "context": { + "http": { + "url": "http://my-cluster.com" + } + } + }, + "expected_resource": "my-cluster.com:80", + "expected_service_target": { + "type": "http", + "name": "my-cluster.com:80" + }, + "failure_message": "`context.http.url` without an explicit default HTTP port, output should be reported as `${context.http.url}:80`" + }, + { + "span": { + "exit": "true", + "type": "messaging", + "context": { + "message": { + "body": "Text message", + "queue": { + "name": "myQueue" + } + } + } + }, + "expected_resource": "messaging/myQueue", + "expected_service_target": { + "type": "messaging", + "name": "myQueue" + }, + "failure_message": "If `context.message` exists, and subtype is `null`, output should be `${type}/${context.message.queue.name}" + }, + { + "span": { + "exit": "true", + "type": "messaging", + "subtype": "kafka", + "context": { + "message": { + "body": "Text message", + "queue": { + "name": "myQueue" + } + } + } + }, + "expected_resource": "kafka/myQueue", + "expected_service_target": { + "type": "kafka", + "name": "myQueue" + }, + "failure_message": "If `context.message` exists, output should be `${subtype}/${context.message.queue.name}" + }, + { + "span": { + "exit": "true", + "type": "messaging", + "subtype": "kafka", + "context": { + "message": { + "body": "Text message" + } + } + }, + "expected_resource": "kafka", + "expected_service_target": { + "type": "kafka" + }, + "failure_message": "If `context.message` exists without `context.message.queue.name`, output should be `${subtype}`" + } +] diff --git a/tests/agents/json-specs/span_types.json b/tests/agents/json-specs/span_types.json new file mode 100644 index 00000000..aa6e47f5 --- /dev/null +++ b/tests/agents/json-specs/span_types.json @@ -0,0 +1,338 @@ +{ + "__description": { + "": "root element for type identified by ''", + ".__description": "description for '' (optional)", + ".__used_by": "list of agents that use '' to help document alignment (optional)", + ".allow_null_subtype": "true to allow null subtype, false by default if omitted", + ".allow_unlisted_subtype": "true to allow unlisted subtypes, false by default if omitted", + ".subtypes": "root element for sub-types of type '', if omitted or empty subtype must be null, unless 'allow_unlisted_subtype' is set to true", + ".subtypes.": "sub-type element for ", + ".subtypes..__description": "description of subtype (optional)", + ".subtypes..__used_by": "list of agents that use to help document alignment (optional)" + }, + "app": { + "__description": "Spans within application (usually not calling an external system)", + "allow_null_subtype": true, + "subtypes": { + "inferred": { + "__description": "Sampling profiler inferred spans", + "__used_by": [ + "java" + ] + }, + "internal": { + "__description": "Application generic internal span for controller/handler/processing delegation", + "__used_by": [ + ] + }, + "controller": { + "__description": "Deprecated: use app.internal instead", + "__used_by": [ + "ruby" + ] + }, + "graphql": { + "__description": "Deprecated: use app.internal instead", + "__used_by": [ + "ruby" + ] + }, + "mailer": { + "__description": "Deprecated: use app.internal instead", + "__used_by": [ + "ruby" + ] + }, + "resource": { + "__description": "Deprecated: use app.internal instead", + "__used_by": [ + "ruby" + ] + }, + "handler": { + "__description": "Deprecated: use app.internal instead", + "__used_by": [ + "java" + ] + } + } + }, + "custom": { + "__description": "API custom instrumentation", + "__used_by": [ + "java", + "ruby" + ], + "allow_null_subtype": true + }, + "db": { + "__description": "database span", + "subtypes": { + "cassandra": { + "__description": "Cassandra", + "__used_by": [ + "java" + ] + }, + "cosmosdb": { + "__description": "Azure CosmosDB" + }, + "db2": { + "__description": "IBM DB2", + "__used_by": [ + "java" + ] + }, + "derby": { + "__description": "Apache Derby", + "__used_by": [ + "java" + ] + }, + "dynamodb": { + "__description": "AWS DynamoDB", + "__used_by": [ + "ruby" + ] + }, + "elasticsearch": { + "__description": "Elasticsearch", + "__used_by": [ + "java", + "ruby" + ] + }, + "graphql": { + "__description": "GraphQL", + "__used_by": [ + "nodejs" + ] + }, + "h2": { + "__description": "H2", + "__used_by": [ + "java" + ] + }, + "hsqldb": { + "__description": "HSQLDB", + "__used_by": [ + "java" + ] + }, + "ingres": { + "__description": "Ingres" + }, + "mariadb": { + "__description": "MariaDB", + "__used_by": [ + "java", + "ruby" + ] + }, + "memcached": { + "__description": "Memcached", + "__used_by": [ + "nodejs" + ] + }, + "mongodb": { + "__description": "MongoDB", + "__used_by": [ + "java", + "ruby" + ] + }, + "mssql": { + "__description": "Microsoft SQL Server", + "__used_by": [ + "nodejs", + "java" + ] + }, + "mysql": { + "__description": "MySQL", + "__used_by": [ + "java", + "ruby" + ] + }, + "oracle": { + "__description": "Oracle Database", + "__used_by": [ + "java" + ] + }, + "postgresql": { + "__description": "PostgreSQL", + "__used_by": [ + "ruby" + ] + }, + "redis": { + "__description": "Redis", + "__used_by": [ + "java", + "ruby" + ] + }, + "sqlite": { + "__description": "SQLite", + "__used_by": [ + "ruby" + ] + }, + "sqlite3": { + "__description": "Deprecated: use db/sqlite", + "__used_by": [ + "ruby" + ] + }, + "sqlserver": { + "__description": "Deprecated: use db/mssql", + "__used_by": [ + "java" + ] + }, + "unknown": { + "__description": "Unknown database", + "__used_by": [ + "java", + "ruby" + ] + } + } + }, + "external": { + "__description": "Request to external service, usually in request/response pattern", + "subtypes": { + "dubbo": { + "__description": "Apache Dubbo", + "__used_by": [ + "java" + ] + }, + "grpc": { + "__description": "gRPC", + "__used_by": [ + "ruby", + "java" + ] + }, + "http": { + "__description": "HTTP client", + "__used_by": [ + "ruby", + "java" + ] + }, + "ldap": { + "__description": "LDAP client", + "__used_by": [ + "java" + ] + } + } + }, + "json": { + "__description": "Deprecated: use app.internal instead", + "subtypes": { + "parse": { + "__description": "JSON parsing" + }, + "generate": { + "__description": "JSON generation" + } + }, + "__used_by": [ + "ruby" + ] + }, + "messaging": { + "__description": "Messaging", + "subtypes": { + "azurequeue": { + "__description": "Azure Queue" + }, + "azureservicebus": { + "__description": "Azure Service Bus" + }, + "jms": { + "__description": "Java Messaging Service", + "__used_by": [ + "java" + ] + }, + "kafka": { + "__description": "Apache Kafka", + "__used_by": [ + "java" + ] + }, + "rabbitmq": { + "__description": "RabbitMQ", + "__used_by": [ + "java" + ] + }, + "sns": { + "__description": "AWS Simple Notification Service", + "__used_by": [ + "ruby" + ] + }, + "sqs": { + "__description": "AWS Simple Queue Service", + "__used_by": [ + "ruby" + ] + } + } + }, + "process": { + "__description": "External process", + "__used_by": [ + "java" + ] + }, + "storage": { + "subtypes": { + "azureblob": { + "__description": "Azure Blob Storage" + }, + "azurefile": { + "__description": "Azure Files" + }, + "azuretable": { + "__description": "Azure Storage Table", + "__used_by": [ + "ruby" + ] + }, + "s3": { + "__description": "AWS S3", + "__used_by": [ + "ruby" + ] + } + } + }, + "template": { + "__description": "Template engines (no sub-type for now as really platform-specific)", + "__used_by": [ + "java", + "ruby" + ], + "allow_unlisted_subtype": true + }, + "websocket": { + "__description": "Websockets", + "subtypes": { + "send": { + "__used_by": [ + "nodejs" + ] + } + } + } +} diff --git a/tests/sql_signature_examples.json b/tests/agents/json-specs/sql_signature_examples.json similarity index 100% rename from tests/sql_signature_examples.json rename to tests/agents/json-specs/sql_signature_examples.json diff --git a/tests/sql_token_examples.json b/tests/agents/json-specs/sql_token_examples.json similarity index 94% rename from tests/sql_token_examples.json rename to tests/agents/json-specs/sql_token_examples.json index 4c89fee8..b443df7d 100644 --- a/tests/sql_token_examples.json +++ b/tests/agents/json-specs/sql_token_examples.json @@ -116,6 +116,20 @@ } ] }, + { + "name": "CQL line comment", + "input": "/* /*nested*/ */ // SELECT /*", + "tokens": [ + { + "kind": "COMMENT", + "text": "/* /*nested*/ */" + }, + { + "kind": "COMMENT", + "text": "// SELECT /*" + } + ] + }, { "name": "string-literal", "input": "'abc '' def\\''", diff --git a/tests/agents/json-specs/w3c_distributed_tracing.json b/tests/agents/json-specs/w3c_distributed_tracing.json new file mode 100644 index 00000000..30a3b040 --- /dev/null +++ b/tests/agents/json-specs/w3c_distributed_tracing.json @@ -0,0 +1,92 @@ +// +// from https://github.com/w3c/distributed-tracing/blob/main/test/test_data.json +// +// NOTE: This file is manually copied from the above link and +// there is currently NO automation keeping it in sync with the upstream version. +// +// Latest commit 98f210e on Sep 24, 2019 +// +[ + {"headers": [["traceparent", "00-00000000000000000000000000000000-1234567890123456-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-01"]], "is_traceparent_valid": true}, + {"headers": [], "is_traceparent_valid": false}, + {"headers": [], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789011-1234567890123456-01"], ["traceparent", "00-12345678901234567890123456789012-1234567890123456-01"]], "is_traceparent_valid": false}, + {"headers": [], "is_traceparent_valid": false}, + {"headers": [], "is_traceparent_valid": false}, + {"headers": [["TraceParent", "00-12345678901234567890123456789012-1234567890123456-01"]], "is_traceparent_valid": true}, + {"headers": [["TrAcEpArEnT", "00-12345678901234567890123456789012-1234567890123456-01"]], "is_traceparent_valid": true}, + {"headers": [["TRACEPARENT", "00-12345678901234567890123456789012-1234567890123456-01"]], "is_traceparent_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-01"]], "is_traceparent_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-01"]], "is_traceparent_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-01"]], "is_traceparent_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-01 "]], "is_traceparent_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-01\t"]], "is_traceparent_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-01 \t"]], "is_traceparent_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-0000000000000000-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-.234567890123456-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-123456789012345.-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-12345678901234567-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-123456789012345-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-.0"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-0."]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-001"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-1"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-00000000000000000000000000000000-1234567890123456-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-.2345678901234567890123456789012-1234567890123456-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-1234567890123456789012345678901.-1234567890123456-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-123456789012345678901234567890123-1234567890123456-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-1234567890123456789012345678901-1234567890123456-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-01."]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-01-what-the-future-will-be-like"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "cc-12345678901234567890123456789012-1234567890123456-01"]], "is_traceparent_valid": true}, + {"headers": [["traceparent", "cc-12345678901234567890123456789012-1234567890123456-01-what-the-future-will-be-like"]], "is_traceparent_valid": true}, + {"headers": [["traceparent", "cc-12345678901234567890123456789012-1234567890123456-01.what-the-future-will-be-like"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "ff-12345678901234567890123456789012-1234567890123456-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", ".0-12345678901234567890123456789012-1234567890123456-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "0.-12345678901234567890123456789012-1234567890123456-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "000-12345678901234567890123456789012-1234567890123456-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "0000-12345678901234567890123456789012-1234567890123456-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "0-12345678901234567890123456789012-1234567890123456-01"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "abcdefghijklmnopqrstuvwxyz0123456789_-*/= !\"#$%&'()*+-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "abcdefghijklmnopqrstuvwxyz0123456789_-*/@a-z0-9_-*/= !\"#$%&'()*+-./0123456789:;<>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1,foo=1"]], "is_traceparent_valid": true, "is_tracestate_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1,foo=2"]], "is_traceparent_valid": true, "is_tracestate_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1"], ["tracestate", "foo=1"]], "is_traceparent_valid": true, "is_tracestate_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1"], ["tracestate", "foo=2"]], "is_traceparent_valid": true, "is_tracestate_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", ""]], "is_traceparent_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1"], ["tracestate", ""]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", ""], ["tracestate", "foo=1"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"]], "is_traceparent_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"]], "is_traceparent_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["TraceState", "foo=1"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["TrAcEsTaTe", "foo=1"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["TRACESTATE", "foo=1"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1,bar=2"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["tracestate", "foo=1"]], "is_traceparent_valid": false}, + {"headers": [["tracestate", "foo=1,bar=2"]], "is_traceparent_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo =1"]], "is_traceparent_valid": true, "is_tracestate_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "FOO=1"]], "is_traceparent_valid": true, "is_tracestate_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo.bar=1"]], "is_traceparent_valid": true, "is_tracestate_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo@=1,bar=2"]], "is_traceparent_valid": true, "is_tracestate_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "@foo=1,bar=2"]], "is_traceparent_valid": true, "is_tracestate_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo@@bar=1,bar=2"]], "is_traceparent_valid": true, "is_tracestate_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo@bar@baz=1,bar=2"]], "is_traceparent_valid": true, "is_tracestate_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1"], ["tracestate", "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz=1"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1"], ["tracestate", "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz=1"]], "is_traceparent_valid": true, "is_tracestate_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1"], ["tracestate", "ttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt@vvvvvvvvvvvvvv=1"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1"], ["tracestate", "tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt@v=1"]], "is_traceparent_valid": true, "is_tracestate_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1"], ["tracestate", "t@vvvvvvvvvvvvvvv=1"]], "is_traceparent_valid": true, "is_tracestate_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "bar01=01,bar02=02,bar03=03,bar04=04,bar05=05,bar06=06,bar07=07,bar08=08,bar09=09,bar10=10"], ["tracestate", "bar11=11,bar12=12,bar13=13,bar14=14,bar15=15,bar16=16,bar17=17,bar18=18,bar19=19,bar20=20"], ["tracestate", "bar21=21,bar22=22,bar23=23,bar24=24,bar25=25,bar26=26,bar27=27,bar28=28,bar29=29,bar30=30"], ["tracestate", "bar31=31,bar32=32"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "bar01=01,bar02=02,bar03=03,bar04=04,bar05=05,bar06=06,bar07=07,bar08=08,bar09=09,bar10=10"], ["tracestate", "bar11=11,bar12=12,bar13=13,bar14=14,bar15=15,bar16=16,bar17=17,bar18=18,bar19=19,bar20=20"], ["tracestate", "bar21=21,bar22=22,bar23=23,bar24=24,bar25=25,bar26=26,bar27=27,bar28=28,bar29=29,bar30=30"], ["tracestate", "bar31=31,bar32=32,bar33=33"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1,bar=2"], ["tracestate", "rojo=1,congo=2"], ["tracestate", "baz=3"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1 \t , \t bar=2, \t baz=3"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1\t \t,\t \tbar=2,\t \tbaz=3"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1 "]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1\t"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=1 \t"]], "is_traceparent_valid": true, "is_tracestate_valid": true}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=bar=baz"]], "is_traceparent_valid": true, "is_tracestate_valid": false}, + {"headers": [["traceparent", "00-12345678901234567890123456789012-1234567890123456-00"], ["tracestate", "foo=,bar=3"]], "is_traceparent_valid": true, "is_tracestate_valid": false} +] diff --git a/tests/agents/json-specs/wildcard_matcher_tests.json b/tests/agents/json-specs/wildcard_matcher_tests.json new file mode 100644 index 00000000..2b071470 --- /dev/null +++ b/tests/agents/json-specs/wildcard_matcher_tests.json @@ -0,0 +1,194 @@ +{ + "testMatchesStartsWith": { + "foo*": { + "foo": true, + "foobar": true, + "bar": false, + "barfoo": false, + "rfoo": false + } + }, + "testWildcardInTheMiddle": { + "/foo/*/baz": { + "/foo/bar/baz": true, + "/foo/bar": false + } + }, + "testCompoundWildcardMatcher": { + "*foo*foo*": { + "foofoo": true, + "foo/bar/foo": true, + "/foo/bar/foo/bar": true, + "foo": false + } + }, + "testCompoundWildcardMatcher3": { + "*foo*oo*": { + "foooo": true, + "foofoo": true, + "foo/bar/foo": true, + "/foo/bar/foo/bar": true, + "foo": false, + "fooo": false + } + }, + "testCompoundWildcardMatcher2": { + "*foo*bar*": { + "foobar": true, + "foo/bar/foo/baz": true, + "/foo/bar/baz": true, + "bar/foo": false, + "barfoo": false + } + }, + "testCompoundWildcardMatcher4": { + "*foo*far*": { + "foofar": true, + "foo/far/foo/baz": true, + "/foo/far/baz": true, + "/far/foo": false, + "farfoo": false + } + }, + "testMatchBetween": { + "*foo*foo*": { + "foofoo": true, + "foo/foo/foo/baz": true, + "/foo/foo/baz": true, + "/foo/foo": true, + "foobar": false + } + }, + "testComplexExpressions": { + "/foo/*/baz*": { + "/foo/a/bar/b/baz": true + }, + "/foo/*/bar/*/baz": { + "/foo/a/bar/b/baz": true + } + }, + "testInfixEmptyMatcher": { + "**": { + "": true, + "foo": true + } + }, + "testMatchesEndsWith": { + "*foo": { + "foo": true, + "foobar": false, + "bar": false, + "barfoo": true, + "foor": false + } + }, + "testMatchesEquals": { + "foo": { + "foo": true, + "foobar": false, + "bar": false, + "barfoo": false + } + }, + "testMatchesInfix": { + "*foo*": { + "foo": true, + "foobar": true, + "bar": false, + "barfoo": true, + "barfoobaz": true + } + }, + "testMatchesNoWildcard": { + "foo": { + "foo": true, + "foobar": false + } + }, + "testMatchesStartsWith_ignoreCase": { + "foo*": { + "foo": true, + "foobar": true, + "bar": false, + "barfoo": false + } + }, + "testInfixEmptyMatcher_ignoreCase": { + "**": { + "": true, + "foo": true + } + }, + "testMatchesEndsWith_ignoreCase": { + "*foo": { + "fOo": true, + "foobar": false, + "bar": false, + "baRFoo": true + } + }, + "testMatchesEquals_ignoreCase": { + "foo": { + "fOo": true, + "foOBar": false, + "BAR": false, + "barfoo": false + } + }, + "testMatchesInfix_ignoreCase": { + "*foo*": { + "FOO": true, + "foOBar": true, + "BAR": false, + "baRFOo": true, + "BARFOOBAZ": true + } + }, + "testMatchesInfix_caseSensitive": { + "(?-i)*foo*": { + "foo": true, + "FOO": false + } + }, + "testMatchesNoWildcard_ignoreCase": { + "foo": { + "FOO": true, + "foobar": false + } + }, + "testNeedleLongerThanHaystack": { + "*foo": { + "baz": false + }, + "*foob": { + "baz": false + }, + "*fooba": { + "baz": false + }, + "*foobar": { + "baz": false + }, + "foo*": { + "baz": false + }, + "foob*": { + "baz": false + }, + "fooba*": { + "baz": false + }, + "foobar*": { + "baz": false + }, + "*foobar*": { + "baz": false + } + }, + "testSingleCharacterWildcardNotSupported": { + "fo?": { + "foo": false, + "fo?": true + } + } +}