From d94e93e09a35dede2fb7309bfca63b131259b0fb Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Sat, 5 Oct 2024 18:43:31 -0700 Subject: [PATCH 1/3] Add API docs for custom attribute context manager. --- ...ecordllmfeedbackevent-python-agent-api.mdx | 2 +- ...thllmcustomattributes-python-agent-api.mdx | 92 +++++++++++++++++++ .../python-agent-100100.mdx | 87 ++++++++++++++++++ 3 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 src/content/docs/apm/agents/python-agent/python-agent-api/withllmcustomattributes-python-agent-api.mdx create mode 100644 src/content/docs/release-notes/agent-release-notes/python-release-notes/python-agent-100100.mdx diff --git a/src/content/docs/apm/agents/python-agent/python-agent-api/recordllmfeedbackevent-python-agent-api.mdx b/src/content/docs/apm/agents/python-agent/python-agent-api/recordllmfeedbackevent-python-agent-api.mdx index 2cc98957fcb..a8293fab7fb 100644 --- a/src/content/docs/apm/agents/python-agent/python-agent-api/recordllmfeedbackevent-python-agent-api.mdx +++ b/src/content/docs/apm/agents/python-agent/python-agent-api/recordllmfeedbackevent-python-agent-api.mdx @@ -112,7 +112,7 @@ In many cases, the endpoint for AI messages are recorded in different places fro -## Return values [#return-valuess] +## Return values [#return-values] None. diff --git a/src/content/docs/apm/agents/python-agent/python-agent-api/withllmcustomattributes-python-agent-api.mdx b/src/content/docs/apm/agents/python-agent/python-agent-api/withllmcustomattributes-python-agent-api.mdx new file mode 100644 index 00000000000..de168013b17 --- /dev/null +++ b/src/content/docs/apm/agents/python-agent/python-agent-api/withllmcustomattributes-python-agent-api.mdx @@ -0,0 +1,92 @@ +--- +title: WithLlmCustomAttributes (Python agent API) +type: apiDoc +shortDescription: Add custom attributes to LLM events +tags: + - Agents + - Python agent + - Python agent API +metaDescription: 'Python API: This API adds custom attribuets to a Large Language Model (LLM) events generated in AI applications.' +redirects: + - /docs/agents/python-agent/python-agent-api/withllmcustomattributes-python-agent-api + - /docs/agents/python-agent/python-agent-api/withllmcustomattributes +freshnessValidatedDate: never +--- + +## Syntax [#syntax] + +```py + with newrelic.agent.WithLlmCustomAttributes(custom_attribute_map): +``` + +Context manager API that adds user-specified attributes to Large Language Model (LLM) events generated by LLM calls in application code. + +## Requirements [#requirements] + +Python agent version 10.1.0 or higher. + +## Description [#description] + +This context manager API adds custom, user-specified attributes to each LLM event generated within its context based on calls made to LLMs. +The agent will automatically add an `llm.` prefix to each custom attribute key name specified in the passed in dictionary argument. + +These custom attributes can be viewed on LLM events and queried for in the New Relic UI. For more information on AI Monitoring, see our [AI docs](https://docs.newrelic.com/docs/ai-monitoring/intro-to-ai-monitoring/). + +This API does not support embedding events. + + +## Parameters [#parameters] + + + + + + + + + + + + + + + + + +
+ Parameter + + Description +
+ `custom_attribute_map` + + _dictionary_ + + Required. A non-empty dictionary where each key-value pair indicates the custom attribute name and its respective value. +
+ +## Return values [#return-values] + +None. + +## Examples [#examples] + +### Add custom attributes to an OpenAI chat completion call + + +````py + import newrelic.agent + + from openai import OpenAI + + client = OpenAI() + + with newrelic.agent.WithLlmCustomAttributes({"custom": "attr", "custom1": "attr1"}): + response = client.chat.completions.create( + messages=[{ + "role": "user", + "content": "Say this is a test", + }], + model="gpt-4o-mini", + ) +```` diff --git a/src/content/docs/release-notes/agent-release-notes/python-release-notes/python-agent-100100.mdx b/src/content/docs/release-notes/agent-release-notes/python-release-notes/python-agent-100100.mdx new file mode 100644 index 00000000000..190a83edccd --- /dev/null +++ b/src/content/docs/release-notes/agent-release-notes/python-release-notes/python-agent-100100.mdx @@ -0,0 +1,87 @@ +--- +subject: 'Python agent' +releaseDate: '2024-10-7' +version: 10.1.0 +downloadLink: 'https://pypi.python.org/pypi/newrelic' +features: ['Remove Python 2.7 support', 'Add Kafka server metric', 'Add capturing of memcache host and port', 'Add support for kafka-python-ng', 'Add support for uvicorn_workers', 'Automatically detect function signature in LangChain similarity search', 'Add environment variable for garbage collector runtime metrics'] +bugs: ['Fix bug in gRPC entity name detection', 'Fix issue in LangChain where a thread is started and the transaction context is lost', 'Fix parsing of request and response when running OpenAI via LangChain', 'Instrument new path to OpenAI chat class', 'Fix pyscopg v3 API incompatibilities', 'Remove versioned logic in loguru instrumentation'] +security: [] +--- + +## Notes + +This release of the Python agent drops support for Python 2.7 and adds the following: + + * a Kafka server metric + * host and port attributes for memcache + * support for [kafka-python-ng](https://pypi.org/project/kafka-python-ng/) + * support for [uvicorn_worker](https://pypi.org/project/uvicorn-worker/) + * an environment variable for garbage collector (GC) runtime metrics + * automatic detection for the function signature in [LangChain](https://pypi.org/project/langchain/)'s `similarity_search` + * updated support for [OpenAI](https://pypi.org/project/openai/)'s chat class + * a fix for a bug in gRPC's entity name detection + * a fix for transaction context propagation loss in [LangChain](https://pypi.org/project/langchain/) in the case of a new thread + * parsing of the request and response when running [OpenAI](https://pypi.org/project/openai/) via [LangChain](https://pypi.org/project/langchain/) + * a fix for [pyscopg](https://pypi.org/project/psycopg/) v3 API incompatibilities + * a fix that removes versioned logic in [loguru](https://pypi.org/project/loguru/) instrumentation + +Install the agent using `easy_install/pip/distribute` via the [Python Package Index](https://pypi.python.org/pypi/newrelic) or download it directly from the [New Relic download site](https://download.newrelic.com/python_agent/release). + + +## New features + +* Add Kafka server metric + + * Adds a metric to capture the host and port information for Kafka entities on both consumers and producers. This metric allows the UI to link AWS information with AWS MSK entities in the service map. + +* Add capturing of memcache `host` and `port` + + * Adds capturing of `host` and `port` info on the following memcache libraries: [bmemcached](https://pypi.org/project/python-binary-memcached/), [aiomcache](https://pypi.org/project/aiomcache/), and [pymemcache](https://pypi.org/project/pymemcache/). This allows the UI to link AWS information with AWS memcache entities in the service map. + +* Add support for [kafka-python-ng](https://pypi.org/project/kafka-python-ng/) + + * [kafka-python](https://pypi.org/project/kafka-python/) has been released under a new name: [kafka-python-ng](https://pypi.org/project/kafka-python-ng/). This continues to support kafka-python under the new package name. + +* Add support for [uvicorn_worker](https://pypi.org/project/uvicorn-worker/) + + * `uvicorn.workers` has been moved to a separate package called `uvicorn_worker`. This checks for both names of the module when reporting dispatcher information. + +* Automatically detect function signature in [LangChain](https://pypi.org/project/langchain/) `similarity_search` + + * Automatically detect the function signature when wrapping `similarity_search`. + +* Add environment variable for garbage collector runtime metrics + + * Garbage collector runtime metrics can now be enabled/disabled via the following environment variable: `NEW_RELIC_GC_RUNTIME_METRICS_ENABLED`. + +## Bug fixes + +* Fix bug in gRPC entity name detection + + * Previously, the gRPC channel entity name was missing the first character(s) in specific cases. This bug has been fixed. + +* Fix issue in [LangChain](https://pypi.org/project/langchain/) where a thread is started and the transaction context is lost + + * Previously, when LangChain called certain chains, such as retrieval chains, LangChain started a thread that caused the transaction to be lost, resulting in broken instrumentation. This has been fixed so the transaction context is passed across threads and instrumentation still works for retrieval chains. + +* Fix parsing of request and response when running [OpenAI](https://pypi.org/project/openai/) via [LangChain](https://pypi.org/project/langchain/) + + * Previously, when running [OpenAI](https://pypi.org/project/openai/) via [LangChain](https://pypi.org/project/langchain/) the request and response were a different format and were not being recorded. This issue has been fixed. + +* Instrument new path to [OpenAI](https://pypi.org/project/openai/) chat completions class + + * [OpenAI](https://pypi.org/project/openai/) moved the `ChatCompletions` class to a different path which prevented the instrumentation from being applied. This has been fixed. + +* Fix [pyscopg](https://pypi.org/project/psycopg/) v3 API incompatibilities + + * Expand DBAPI2 wrappers to allow arbitrary `kwargs` on `executemany()`. Upgrade [psycopg](https://pypi.org/project/psycopg/) v3 instrumentation to allow arbitrary `kwargs` on `executemany()`. Rename existing psycopg v3 wrapper arguments to match the upstream library's names for compatibility. + +* Remove versioned logic in loguru instrumentation + + * Removed versioned logic in [loguru](https://pypi.org/project/loguru/) instrumentation to fix a bug that occurred when the version was undeterminable. + +## Support statement + +We recommend updating to the latest agent version as soon as it's available. If you can't upgrade to the latest version, update your agents to a version no more than 90 days old. Read [more](/docs/new-relic-solutions/new-relic-one/install-configure/update-new-relic-agent/) about keeping agents up to date. + +See the New Relic Python agent [EOL policy](/docs/apm/agents/python-agent/getting-started/python-agent-eol-policy/) for information about agent releases and support dates. From dbb23a7413eaaa89c9aa506aa9e7a9e53a377d40 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 10 Oct 2024 13:07:13 -0700 Subject: [PATCH 2/3] Add v10.1.0 release notes. --- ...thllmcustomattributes-python-agent-api.mdx | 2 - .../python-agent-100100.mdx | 71 ++++--------------- 2 files changed, 13 insertions(+), 60 deletions(-) diff --git a/src/content/docs/apm/agents/python-agent/python-agent-api/withllmcustomattributes-python-agent-api.mdx b/src/content/docs/apm/agents/python-agent/python-agent-api/withllmcustomattributes-python-agent-api.mdx index de168013b17..f1737fa7d99 100644 --- a/src/content/docs/apm/agents/python-agent/python-agent-api/withllmcustomattributes-python-agent-api.mdx +++ b/src/content/docs/apm/agents/python-agent/python-agent-api/withllmcustomattributes-python-agent-api.mdx @@ -32,8 +32,6 @@ The agent will automatically add an `llm.` prefix to each custom attribute key n These custom attributes can be viewed on LLM events and queried for in the New Relic UI. For more information on AI Monitoring, see our [AI docs](https://docs.newrelic.com/docs/ai-monitoring/intro-to-ai-monitoring/). -This API does not support embedding events. - ## Parameters [#parameters] diff --git a/src/content/docs/release-notes/agent-release-notes/python-release-notes/python-agent-100100.mdx b/src/content/docs/release-notes/agent-release-notes/python-release-notes/python-agent-100100.mdx index 190a83edccd..51905d5f7c0 100644 --- a/src/content/docs/release-notes/agent-release-notes/python-release-notes/python-agent-100100.mdx +++ b/src/content/docs/release-notes/agent-release-notes/python-release-notes/python-agent-100100.mdx @@ -1,84 +1,39 @@ --- subject: 'Python agent' -releaseDate: '2024-10-7' +releaseDate: '2024-10-10' version: 10.1.0 downloadLink: 'https://pypi.python.org/pypi/newrelic' -features: ['Remove Python 2.7 support', 'Add Kafka server metric', 'Add capturing of memcache host and port', 'Add support for kafka-python-ng', 'Add support for uvicorn_workers', 'Automatically detect function signature in LangChain similarity search', 'Add environment variable for garbage collector runtime metrics'] -bugs: ['Fix bug in gRPC entity name detection', 'Fix issue in LangChain where a thread is started and the transaction context is lost', 'Fix parsing of request and response when running OpenAI via LangChain', 'Instrument new path to OpenAI chat class', 'Fix pyscopg v3 API incompatibilities', 'Remove versioned logic in loguru instrumentation'] +features: ['Add support for Python 3.13', 'Add new Large LLM custom attribute context manager API', 'Add support for reporting of Amazon ECS Fargate Docker IDs', 'Add instrumentation for `SQLiteVec` vectorstore in Langchain'] +bugs: [] security: [] --- ## Notes -This release of the Python agent drops support for Python 2.7 and adds the following: - - * a Kafka server metric - * host and port attributes for memcache - * support for [kafka-python-ng](https://pypi.org/project/kafka-python-ng/) - * support for [uvicorn_worker](https://pypi.org/project/uvicorn-worker/) - * an environment variable for garbage collector (GC) runtime metrics - * automatic detection for the function signature in [LangChain](https://pypi.org/project/langchain/)'s `similarity_search` - * updated support for [OpenAI](https://pypi.org/project/openai/)'s chat class - * a fix for a bug in gRPC's entity name detection - * a fix for transaction context propagation loss in [LangChain](https://pypi.org/project/langchain/) in the case of a new thread - * parsing of the request and response when running [OpenAI](https://pypi.org/project/openai/) via [LangChain](https://pypi.org/project/langchain/) - * a fix for [pyscopg](https://pypi.org/project/psycopg/) v3 API incompatibilities - * a fix that removes versioned logic in [loguru](https://pypi.org/project/loguru/) instrumentation +This release of the Python agent adds support for Python 3.13, adds a new Large Language Model (LLM) API, supports Docker ID parsing in Amazon ECS environments, and instruments a new Langchain vectorstore. + Install the agent using `easy_install/pip/distribute` via the [Python Package Index](https://pypi.python.org/pypi/newrelic) or download it directly from the [New Relic download site](https://download.newrelic.com/python_agent/release). ## New features -* Add Kafka server metric - - * Adds a metric to capture the host and port information for Kafka entities on both consumers and producers. This metric allows the UI to link AWS information with AWS MSK entities in the service map. - -* Add capturing of memcache `host` and `port` - - * Adds capturing of `host` and `port` info on the following memcache libraries: [bmemcached](https://pypi.org/project/python-binary-memcached/), [aiomcache](https://pypi.org/project/aiomcache/), and [pymemcache](https://pypi.org/project/pymemcache/). This allows the UI to link AWS information with AWS memcache entities in the service map. - -* Add support for [kafka-python-ng](https://pypi.org/project/kafka-python-ng/) - - * [kafka-python](https://pypi.org/project/kafka-python/) has been released under a new name: [kafka-python-ng](https://pypi.org/project/kafka-python-ng/). This continues to support kafka-python under the new package name. - -* Add support for [uvicorn_worker](https://pypi.org/project/uvicorn-worker/) - - * `uvicorn.workers` has been moved to a separate package called `uvicorn_worker`. This checks for both names of the module when reporting dispatcher information. - -* Automatically detect function signature in [LangChain](https://pypi.org/project/langchain/) `similarity_search` - - * Automatically detect the function signature when wrapping `similarity_search`. - -* Add environment variable for garbage collector runtime metrics - - * Garbage collector runtime metrics can now be enabled/disabled via the following environment variable: `NEW_RELIC_GC_RUNTIME_METRICS_ENABLED`. - -## Bug fixes - -* Fix bug in gRPC entity name detection - - * Previously, the gRPC channel entity name was missing the first character(s) in specific cases. This bug has been fixed. - -* Fix issue in [LangChain](https://pypi.org/project/langchain/) where a thread is started and the transaction context is lost - - * Previously, when LangChain called certain chains, such as retrieval chains, LangChain started a thread that caused the transaction to be lost, resulting in broken instrumentation. This has been fixed so the transaction context is passed across threads and instrumentation still works for retrieval chains. +* Add support for Python 3.13 -* Fix parsing of request and response when running [OpenAI](https://pypi.org/project/openai/) via [LangChain](https://pypi.org/project/langchain/) + * The agent now supports applications running in Python 3.13. - * Previously, when running [OpenAI](https://pypi.org/project/openai/) via [LangChain](https://pypi.org/project/langchain/) the request and response were a different format and were not being recorded. This issue has been fixed. +* Add new Large LLM custom attribute context manager API -* Instrument new path to [OpenAI](https://pypi.org/project/openai/) chat completions class + * The agent now includes a new context manager API that adds custom attributes to LLM events generated from calls to LLMs in application code. For more information on usage, please see our [API documentation](/docs/apm/agents/python-agent/python-agent-api/withllmcustomattributes-python-agent-api). - * [OpenAI](https://pypi.org/project/openai/) moved the `ChatCompletions` class to a different path which prevented the instrumentation from being applied. This has been fixed. +* Add support for reporting of [Amazon ECS Fargate](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/AWS_Fargate.html) Docker IDs -* Fix [pyscopg](https://pypi.org/project/psycopg/) v3 API incompatibilities + * The agent now reports Docker IDs for containers running in ECS Fargate environments. - * Expand DBAPI2 wrappers to allow arbitrary `kwargs` on `executemany()`. Upgrade [psycopg](https://pypi.org/project/psycopg/) v3 instrumentation to allow arbitrary `kwargs` on `executemany()`. Rename existing psycopg v3 wrapper arguments to match the upstream library's names for compatibility. +* Add instrumentation for `SQLiteVec` vectorstore in Langchain -* Remove versioned logic in loguru instrumentation + * The agent now instruments `similarity_search` for the `SQLiteVec` vectorstore. - * Removed versioned logic in [loguru](https://pypi.org/project/loguru/) instrumentation to fix a bug that occurred when the version was undeterminable. ## Support statement From b811689f7668174a45b98d40e885297e99f00518 Mon Sep 17 00:00:00 2001 From: Uma Annamalai Date: Thu, 10 Oct 2024 14:13:32 -0700 Subject: [PATCH 3/3] Address review feedback. --- .../withllmcustomattributes-python-agent-api.mdx | 5 +++-- .../python-release-notes/python-agent-100100.mdx | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/content/docs/apm/agents/python-agent/python-agent-api/withllmcustomattributes-python-agent-api.mdx b/src/content/docs/apm/agents/python-agent/python-agent-api/withllmcustomattributes-python-agent-api.mdx index f1737fa7d99..e421fbd4a56 100644 --- a/src/content/docs/apm/agents/python-agent/python-agent-api/withllmcustomattributes-python-agent-api.mdx +++ b/src/content/docs/apm/agents/python-agent/python-agent-api/withllmcustomattributes-python-agent-api.mdx @@ -6,7 +6,7 @@ tags: - Agents - Python agent - Python agent API -metaDescription: 'Python API: This API adds custom attribuets to a Large Language Model (LLM) events generated in AI applications.' +metaDescription: 'Python API: This API adds custom attributes to a Large Language Model (LLM) events generated in AI applications.' redirects: - /docs/agents/python-agent/python-agent-api/withllmcustomattributes-python-agent-api - /docs/agents/python-agent/python-agent-api/withllmcustomattributes @@ -27,8 +27,9 @@ Python agent version 10.1.0 or higher. ## Description [#description] -This context manager API adds custom, user-specified attributes to each LLM event generated within its context based on calls made to LLMs. +This context manager API adds custom user-specified attributes to each LLM event generated within its context based on calls made to LLMs. The agent will automatically add an `llm.` prefix to each custom attribute key name specified in the passed in dictionary argument. +This API must be called within the context of an active transaction. These custom attributes can be viewed on LLM events and queried for in the New Relic UI. For more information on AI Monitoring, see our [AI docs](https://docs.newrelic.com/docs/ai-monitoring/intro-to-ai-monitoring/). diff --git a/src/content/docs/release-notes/agent-release-notes/python-release-notes/python-agent-100100.mdx b/src/content/docs/release-notes/agent-release-notes/python-release-notes/python-agent-100100.mdx index 51905d5f7c0..8d08d8ce186 100644 --- a/src/content/docs/release-notes/agent-release-notes/python-release-notes/python-agent-100100.mdx +++ b/src/content/docs/release-notes/agent-release-notes/python-release-notes/python-agent-100100.mdx @@ -3,14 +3,14 @@ subject: 'Python agent' releaseDate: '2024-10-10' version: 10.1.0 downloadLink: 'https://pypi.python.org/pypi/newrelic' -features: ['Add support for Python 3.13', 'Add new Large LLM custom attribute context manager API', 'Add support for reporting of Amazon ECS Fargate Docker IDs', 'Add instrumentation for `SQLiteVec` vectorstore in Langchain'] +features: ['Add support for Python 3.13', 'Add new LLM custom attribute context manager API', 'Add support for reporting of Amazon ECS Fargate Docker IDs', 'Add instrumentation for `SQLiteVec` vectorstore in Langchain'] bugs: [] security: [] --- ## Notes -This release of the Python agent adds support for Python 3.13, adds a new Large Language Model (LLM) API, supports Docker ID parsing in Amazon ECS environments, and instruments a new Langchain vectorstore. +This release of the Python agent adds support for Python 3.13, adds a new Large Language Model (LLM) API, supports Docker ID parsing in Amazon ECS Fargate environments, and instruments a new Langchain vectorstore. Install the agent using `easy_install/pip/distribute` via the [Python Package Index](https://pypi.python.org/pypi/newrelic) or download it directly from the [New Relic download site](https://download.newrelic.com/python_agent/release). @@ -22,7 +22,7 @@ Install the agent using `easy_install/pip/distribute` via the [Python Package In * The agent now supports applications running in Python 3.13. -* Add new Large LLM custom attribute context manager API +* Add new LLM custom attribute context manager API * The agent now includes a new context manager API that adds custom attributes to LLM events generated from calls to LLMs in application code. For more information on usage, please see our [API documentation](/docs/apm/agents/python-agent/python-agent-api/withllmcustomattributes-python-agent-api).