diff --git a/.circleci/config.templ.yml b/.circleci/config.templ.yml index 73994eab222..29cb1d886db 100644 --- a/.circleci/config.templ.yml +++ b/.circleci/config.templ.yml @@ -404,25 +404,12 @@ jobs: paths: - "." - appsec_iast_packages: - <<: *machine_executor - parallelism: 5 - steps: - - when: - condition: - matches: { pattern: "main", value: << pipeline.git.branch >> } - steps: - - run_test: - pattern: 'appsec_iast_packages' - snapshot: true - - run: echo "This test is skipped outside of main branch" - - appsec_integrations: + appsec_integrations_pygoat: <<: *machine_executor parallelism: 13 steps: - run_test: - pattern: 'appsec_integrations' + pattern: 'appsec_integrations_pygoat' snapshot: true run_agent_checks: false docker_services: "pygoat" diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d281fe80148..9d76c78f9c3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -110,6 +110,7 @@ benchmarks/base/aspects_benchmarks_generate.py @DataDog/asm-python ddtrace/appsec/ @DataDog/asm-python ddtrace/settings/asm.py @DataDog/asm-python ddtrace/contrib/subprocess/ @DataDog/asm-python +ddtrace/contrib/internal/subprocess/ @DataDog/asm-python ddtrace/contrib/flask_login/ @DataDog/asm-python ddtrace/contrib/webbrowser @DataDog/asm-python ddtrace/contrib/urllib @DataDog/asm-python @@ -118,8 +119,6 @@ tests/appsec/ @DataDog/asm-python tests/contrib/dbapi/test_dbapi_appsec.py @DataDog/asm-python tests/contrib/subprocess @DataDog/asm-python tests/contrib/flask/test_flask_appsec.py @DataDog/asm-python -tests/contrib/django/django_app/appsec_urls.py @DataDog/asm-python -tests/contrib/django/test_django_appsec.py @DataDog/asm-python tests/snapshots/tests*appsec*.json @DataDog/asm-python tests/contrib/*/test*appsec*.py @DataDog/asm-python scripts/iast/* @DataDog/asm-python diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index ccf6c6501d9..3f9d9308c83 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -47,6 +47,7 @@ jobs: - weblog-variant: fastapi # runs django-poc for 3.12 - weblog-variant: python3.12 + - weblog-variant: django-py3.13 fail-fast: false env: TEST_LIBRARY: python @@ -96,7 +97,7 @@ jobs: needs: [system-tests-build-agent, system-tests-build-weblog] strategy: matrix: - weblog-variant: [flask-poc, uwsgi-poc , django-poc, fastapi, python3.12] + weblog-variant: [flask-poc, uwsgi-poc , django-poc, fastapi, python3.12, django-py3.13] scenario: [remote-config, appsec, appsec-1, other, debugger-1, debugger-2] fail-fast: false diff --git a/.github/workflows/test_frameworks.yml b/.github/workflows/test_frameworks.yml index 809dee38234..2e1502b4d3d 100644 --- a/.github/workflows/test_frameworks.yml +++ b/.github/workflows/test_frameworks.yml @@ -111,68 +111,6 @@ jobs: if: needs.needs-run.outputs.outcome == 'success' run: cat debugger-expl.txt - sanic-testsuite: - strategy: - matrix: - include: - # TODO: profiling fails with a timeout error - #- suffix: Profiling - # profiling: 1 - # iast: 0 - # appsec: 0 - - suffix: IAST - profiling: 0 - iast: 1 - appsec: 0 - - suffix: APPSEC - profiling: 0 - iast: 0 - appsec: 1 - - suffix: Tracer only - profiling: 0 - iast: 0 - appsec: 0 - name: Sanic 24.6 (with ${{ matrix.suffix }}) - runs-on: ubuntu-20.04 - needs: needs-run - timeout-minutes: 15 - env: - DD_PROFILING_ENABLED: ${{ matrix.profiling }} - DD_IAST_ENABLED: ${{ matrix.iast }} - DD_APPSEC_ENABLED: ${{ matrix.appsec }} - DD_TESTING_RAISE: true - CMAKE_BUILD_PARALLEL_LEVEL: 12 - DD_DEBUGGER_EXPL_OUTPUT_FILE: debugger-expl.txt - defaults: - run: - working-directory: sanic - steps: - - uses: actions/checkout@v4 - if: needs.needs-run.outputs.outcome == 'success' - with: - persist-credentials: false - path: ddtrace - - uses: actions/checkout@v4 - if: needs.needs-run.outputs.outcome == 'success' - with: - persist-credentials: false - repository: sanic-org/sanic - ref: v24.6.0 - path: sanic - - uses: actions/setup-python@v5 - if: needs.needs-run.outputs.outcome == 'success' - with: - python-version: "3.11" - - name: Install sanic and dependencies required to run tests - if: needs.needs-run.outputs.outcome == 'success' - run: pip3 install '.[test]' aioquic - - name: Install ddtrace - if: needs.needs-run.outputs.outcome == 'success' - run: pip3 install ../ddtrace - - name: Run tests - if: needs.needs-run.outputs.outcome == 'success' - run: ddtrace-run pytest -k "not test_reloader and not test_reload_listeners and not test_no_exceptions_when_cancel_pending_request and not test_add_signal and not test_ode_removes and not test_skip_touchup and not test_dispatch_signal_triggers and not test_keep_alive_connection_context and not test_redirect_with_params and not test_keep_alive_client_timeout and not test_logger_vhosts and not test_ssl_in_multiprocess_mode" - django-testsuite: strategy: matrix: @@ -963,58 +901,3 @@ jobs: - name: Debugger exploration results if: needs.needs-run.outputs.outcome == 'success' run: cat debugger-expl.txt - - beautifulsoup-testsuite-4_12_3: - strategy: - matrix: - include: - # TODO: profiling is disabled due to a bug in the profiler paths - # - suffix: Profiling - # profiling: 1 - # iast: 0 - # appsec: 0 - - suffix: IAST - profiling: 0 - iast: 1 - appsec: 0 - - suffix: APPSEC - profiling: 0 - iast: 0 - appsec: 1 - - suffix: Tracer only - profiling: 0 - iast: 0 - appsec: 0 - name: Beautifulsoup 4.12.3 (with ${{ matrix.suffix }}) - runs-on: "ubuntu-latest" - needs: needs-run - env: - DD_TESTING_RAISE: true - DD_PROFILING_ENABLED: ${{ matrix.profiling }} - DD_IAST_ENABLED: ${{ matrix.iast }} - DD_APPSEC_ENABLED: ${{ matrix.appsec }} - CMAKE_BUILD_PARALLEL_LEVEL: 12 - DD_DEBUGGER_EXPL_OUTPUT_FILE: debugger-expl.txt - steps: - - uses: actions/setup-python@v5 - if: needs.needs-run.outputs.outcome == 'success' - with: - python-version: '3.9' - - uses: actions/checkout@v4 - if: needs.needs-run.outputs.outcome == 'success' - with: - persist-credentials: false - path: ddtrace - - name: Checkout beautifulsoup - if: needs.needs-run.outputs.outcome == 'success' - run: | - git clone -b 4.12.3 https://git.launchpad.net/beautifulsoup - - name: Install ddtrace - if: needs.needs-run.outputs.outcome == 'success' - run: pip3 install ./ddtrace - - name: Pytest fix - if: needs.needs-run.outputs.outcome == 'success' - run: pip install pytest==8.2.1 - - name: Run tests - if: needs.needs-run.outputs.outcome == 'success' - run: cd beautifulsoup && ddtrace-run pytest diff --git a/.riot/requirements/1147cef.txt b/.riot/requirements/1147cef.txt deleted file mode 100644 index a760b2a10c4..00000000000 --- a/.riot/requirements/1147cef.txt +++ /dev/null @@ -1,62 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --no-annotate --resolver=backtracking .riot/requirements/1147cef.in -# -aiohttp==3.9.1 -aiosignal==1.3.1 -annotated-types==0.6.0 -anyio==4.2.0 -async-timeout==4.0.3 -attrs==23.2.0 -blinker==1.7.0 -certifi==2023.11.17 -charset-normalizer==3.3.2 -click==8.1.7 -coverage[toml]==7.4.0 -dataclasses-json==0.6.3 -exceptiongroup==1.2.0 -flask==3.0.0 -frozenlist==1.4.1 -greenlet==3.0.3 -gunicorn==21.2.0 -hypothesis==6.45.0 -idna==3.6 -iniconfig==2.0.0 -itsdangerous==2.1.2 -jinja2==3.1.2 -jsonpatch==1.33 -jsonpointer==2.4 -langchain==0.0.354 -langchain-community==0.0.8 -langchain-core==0.1.5 -langchain-experimental==0.0.47 -langsmith==0.0.77 -markupsafe==2.1.3 -marshmallow==3.20.1 -mock==5.1.0 -multidict==6.0.4 -mypy-extensions==1.0.0 -numpy==1.26.3 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -psycopg2-binary==2.9.9 -pydantic==2.5.3 -pydantic-core==2.14.6 -pytest==7.4.4 -pytest-cov==4.1.0 -pytest-mock==3.12.0 -pyyaml==6.0.1 -requests==2.31.0 -sniffio==1.3.0 -sortedcontainers==2.4.0 -sqlalchemy==2.0.25 -tenacity==8.2.3 -tomli==2.0.1 -typing-extensions==4.9.0 -typing-inspect==0.9.0 -urllib3==2.1.0 -werkzeug==3.0.1 -yarl==1.9.4 diff --git a/.riot/requirements/1221a04.txt b/.riot/requirements/1221a04.txt deleted file mode 100644 index b07a317c8ce..00000000000 --- a/.riot/requirements/1221a04.txt +++ /dev/null @@ -1,36 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.7 -# by the following command: -# -# pip-compile --no-annotate --resolver=backtracking .riot/requirements/1221a04.in -# -attrs==23.1.0 -certifi==2023.7.22 -charset-normalizer==3.3.2 -click==8.1.7 -coverage[toml]==7.2.7 -exceptiongroup==1.1.3 -flask==2.2.5 -gunicorn==21.2.0 -hypothesis==6.45.0 -idna==3.4 -importlib-metadata==6.7.0 -iniconfig==2.0.0 -itsdangerous==2.1.2 -jinja2==3.1.2 -markupsafe==2.1.3 -mock==5.1.0 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.2.0 -psycopg2-binary==2.9.9 -pytest==7.4.3 -pytest-cov==4.1.0 -pytest-mock==3.11.1 -requests==2.31.0 -sortedcontainers==2.4.0 -tomli==2.0.1 -typing-extensions==4.7.1 -urllib3==2.0.7 -werkzeug==2.2.3 -zipp==3.15.0 diff --git a/.riot/requirements/127eabf.txt b/.riot/requirements/127eabf.txt new file mode 100644 index 00000000000..698f32885be --- /dev/null +++ b/.riot/requirements/127eabf.txt @@ -0,0 +1,82 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --no-annotate .riot/requirements/127eabf.in +# +annotated-types==0.7.0 +attrs==24.3.0 +aws-sam-translator==1.94.0 +aws-xray-sdk==2.14.0 +boto3==1.34.49 +botocore==1.34.49 +certifi==2024.12.14 +cffi==1.17.1 +cfn-lint==1.22.2 +charset-normalizer==3.4.0 +coverage[toml]==7.6.9 +cryptography==44.0.0 +docker==7.1.0 +ecdsa==0.19.0 +graphql-core==3.2.5 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +jinja2==3.1.4 +jmespath==1.0.1 +jsondiff==2.2.1 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema==4.23.0 +jsonschema-path==0.3.3 +jsonschema-specifications==2023.12.1 +lazy-object-proxy==1.10.0 +markupsafe==3.0.2 +mock==5.1.0 +moto[all]==4.2.14 +mpmath==1.3.0 +multidict==6.1.0 +multipart==1.2.1 +networkx==3.4.2 +openapi-schema-validator==0.6.2 +openapi-spec-validator==0.7.1 +opentracing==2.4.0 +packaging==24.2 +pathable==0.4.3 +pluggy==1.5.0 +propcache==0.2.1 +py-partiql-parser==0.5.0 +pyasn1==0.6.1 +pycparser==2.22 +pydantic==2.10.4 +pydantic-core==2.27.2 +pyparsing==3.2.0 +pytest==8.3.4 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.16.0 +python-dateutil==2.9.0.post0 +python-jose[cryptography]==3.3.0 +pyyaml==6.0.2 +referencing==0.35.1 +regex==2024.11.6 +requests==2.32.3 +responses==0.25.3 +rfc3339-validator==0.1.4 +rpds-py==0.22.3 +rsa==4.9 +s3transfer==0.10.4 +six==1.17.0 +sortedcontainers==2.4.0 +sshpubkeys==3.3.1 +sympy==1.13.3 +typing-extensions==4.12.2 +urllib3==2.0.7 +vcrpy==6.0.1 +werkzeug==3.1.3 +wrapt==1.17.0 +xmltodict==0.14.2 +yarl==1.18.3 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/.riot/requirements/131666a.txt b/.riot/requirements/131666a.txt deleted file mode 100644 index e2346f500df..00000000000 --- a/.riot/requirements/131666a.txt +++ /dev/null @@ -1,36 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# pip-compile --no-annotate --resolver=backtracking .riot/requirements/131666a.in -# -attrs==23.1.0 -blinker==1.7.0 -certifi==2023.7.22 -charset-normalizer==3.3.2 -click==8.1.7 -coverage[toml]==7.3.2 -exceptiongroup==1.1.3 -flask==2.3.3 -gunicorn==21.2.0 -hypothesis==6.45.0 -idna==3.4 -importlib-metadata==6.8.0 -iniconfig==2.0.0 -itsdangerous==2.1.2 -jinja2==3.1.2 -markupsafe==2.1.3 -mock==5.1.0 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -psycopg2-binary==2.9.9 -pytest==7.4.3 -pytest-cov==4.1.0 -pytest-mock==3.12.0 -requests==2.31.0 -sortedcontainers==2.4.0 -tomli==2.0.1 -urllib3==2.1.0 -werkzeug==3.0.1 -zipp==3.17.0 diff --git a/.riot/requirements/1522dd0.txt b/.riot/requirements/1522dd0.txt new file mode 100644 index 00000000000..09ed34718bd --- /dev/null +++ b/.riot/requirements/1522dd0.txt @@ -0,0 +1,37 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --no-annotate .riot/requirements/1522dd0.in +# +aiobotocore==2.16.0 +aiohappyeyeballs==2.4.4 +aiohttp==3.11.11 +aioitertools==0.12.0 +aiosignal==1.3.2 +async-generator==1.10 +attrs==24.3.0 +botocore==1.35.81 +coverage[toml]==7.6.9 +frozenlist==1.5.0 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +jmespath==1.0.1 +mock==5.1.0 +multidict==6.1.0 +opentracing==2.4.0 +packaging==24.2 +pluggy==1.5.0 +propcache==0.2.1 +pytest==8.3.4 +pytest-asyncio==0.21.1 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.16.0 +python-dateutil==2.9.0.post0 +six==1.17.0 +sortedcontainers==2.4.0 +urllib3==2.2.3 +wrapt==1.17.0 +yarl==1.18.3 diff --git a/.riot/requirements/2d19e52.txt b/.riot/requirements/16562eb.txt similarity index 63% rename from .riot/requirements/2d19e52.txt rename to .riot/requirements/16562eb.txt index 8de360e7316..e2aac88c146 100644 --- a/.riot/requirements/2d19e52.txt +++ b/.riot/requirements/16562eb.txt @@ -2,31 +2,31 @@ # This file is autogenerated by pip-compile with Python 3.7 # by the following command: # -# pip-compile --allow-unsafe --config=pyproject.toml --no-annotate --resolver=backtracking .riot/requirements/2d19e52.in +# pip-compile --allow-unsafe --config=pyproject.toml --no-annotate --resolver=backtracking .riot/requirements/16562eb.in # attrs==24.2.0 coverage[toml]==7.2.7 exceptiongroup==1.2.2 -gevent==22.10.2 -greenlet==3.1.1 hypothesis==6.45.0 +idna==3.10 importlib-metadata==6.7.0 iniconfig==2.0.0 mock==5.1.0 -msgpack==1.0.5 +multidict==6.0.5 opentracing==2.4.0 packaging==24.0 pluggy==1.2.0 pytest==7.4.4 +pytest-asyncio==0.21.1 pytest-cov==4.1.0 pytest-mock==3.11.1 -pytest-randomly==3.12.0 +pyyaml==6.0.1 +six==1.17.0 sortedcontainers==2.4.0 tomli==2.0.1 typing-extensions==4.7.1 +urllib3==1.26.20 +vcrpy==4.4.0 +wrapt==1.16.0 +yarl==1.9.4 zipp==3.15.0 -zope-event==5.0 -zope-interface==6.4.post2 - -# The following packages are considered to be unsafe in a requirements file: -setuptools==68.0.0 diff --git a/.riot/requirements/16ed652.txt b/.riot/requirements/16ed652.txt deleted file mode 100644 index e4914dd4ab4..00000000000 --- a/.riot/requirements/16ed652.txt +++ /dev/null @@ -1,33 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# pip-compile --no-annotate .riot/requirements/16ed652.in -# -attrs==23.1.0 -certifi==2023.7.22 -charset-normalizer==3.3.2 -click==7.1.2 -coverage[toml]==7.3.2 -exceptiongroup==1.1.3 -flask==1.1.4 -gunicorn==21.2.0 -hypothesis==6.45.0 -idna==3.4 -iniconfig==2.0.0 -itsdangerous==1.1.0 -jinja2==2.11.3 -markupsafe==1.1.1 -mock==5.1.0 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -psycopg2-binary==2.9.9 -pytest==7.4.3 -pytest-cov==4.1.0 -pytest-mock==3.12.0 -requests==2.31.0 -sortedcontainers==2.4.0 -tomli==2.0.1 -urllib3==2.1.0 -werkzeug==1.0.1 diff --git a/.riot/requirements/173e759.txt b/.riot/requirements/173e759.txt deleted file mode 100644 index 305b7740f25..00000000000 --- a/.riot/requirements/173e759.txt +++ /dev/null @@ -1,64 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# pip-compile --no-annotate .riot/requirements/173e759.in -# -aiohttp==3.9.1 -aiosignal==1.3.1 -annotated-types==0.6.0 -anyio==4.2.0 -async-timeout==4.0.3 -attrs==23.2.0 -blinker==1.7.0 -certifi==2023.11.17 -charset-normalizer==3.3.2 -click==8.1.7 -coverage[toml]==7.4.0 -dataclasses-json==0.6.3 -exceptiongroup==1.2.0 -flask==3.0.0 -frozenlist==1.4.1 -greenlet==3.0.3 -gunicorn==21.2.0 -hypothesis==6.45.0 -idna==3.6 -importlib-metadata==7.0.1 -iniconfig==2.0.0 -itsdangerous==2.1.2 -jinja2==3.1.2 -jsonpatch==1.33 -jsonpointer==2.4 -langchain==0.0.354 -langchain-community==0.0.8 -langchain-core==0.1.5 -langchain-experimental==0.0.47 -langsmith==0.0.77 -markupsafe==2.1.3 -marshmallow==3.20.1 -mock==5.1.0 -multidict==6.0.4 -mypy-extensions==1.0.0 -numpy==1.24.4 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -psycopg2-binary==2.9.9 -pydantic==2.5.3 -pydantic-core==2.14.6 -pytest==7.4.4 -pytest-cov==4.1.0 -pytest-mock==3.12.0 -pyyaml==6.0.1 -requests==2.31.0 -sniffio==1.3.0 -sortedcontainers==2.4.0 -sqlalchemy==2.0.25 -tenacity==8.2.3 -tomli==2.0.1 -typing-extensions==4.9.0 -typing-inspect==0.9.0 -urllib3==2.1.0 -werkzeug==3.0.1 -yarl==1.9.4 -zipp==3.17.0 diff --git a/.riot/requirements/17b5eda.txt b/.riot/requirements/17b5eda.txt deleted file mode 100644 index 6ae81a00705..00000000000 --- a/.riot/requirements/17b5eda.txt +++ /dev/null @@ -1,32 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --no-annotate .riot/requirements/17b5eda.in -# -attrs==23.1.0 -blinker==1.7.0 -certifi==2023.7.22 -charset-normalizer==3.3.2 -click==8.1.7 -coverage[toml]==7.3.2 -flask==2.3.3 -gunicorn==21.2.0 -hypothesis==6.45.0 -idna==3.4 -iniconfig==2.0.0 -itsdangerous==2.1.2 -jinja2==3.1.2 -markupsafe==2.1.3 -mock==5.1.0 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -psycopg2-binary==2.9.9 -pytest==7.4.3 -pytest-cov==4.1.0 -pytest-mock==3.12.0 -requests==2.31.0 -sortedcontainers==2.4.0 -urllib3==2.1.0 -werkzeug==3.0.1 diff --git a/.riot/requirements/1a1ddb4.txt b/.riot/requirements/1a1ddb4.txt deleted file mode 100644 index 3e64e630c33..00000000000 --- a/.riot/requirements/1a1ddb4.txt +++ /dev/null @@ -1,36 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.7 -# by the following command: -# -# pip-compile --no-annotate --resolver=backtracking .riot/requirements/1a1ddb4.in -# -attrs==23.1.0 -certifi==2023.7.22 -charset-normalizer==3.3.2 -click==7.1.2 -coverage[toml]==7.2.7 -exceptiongroup==1.1.3 -flask==1.1.4 -gunicorn==21.2.0 -hypothesis==6.45.0 -idna==3.4 -importlib-metadata==6.7.0 -iniconfig==2.0.0 -itsdangerous==1.1.0 -jinja2==2.11.3 -markupsafe==1.1.1 -mock==5.1.0 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.2.0 -psycopg2-binary==2.9.9 -pytest==7.4.3 -pytest-cov==4.1.0 -pytest-mock==3.11.1 -requests==2.31.0 -sortedcontainers==2.4.0 -tomli==2.0.1 -typing-extensions==4.7.1 -urllib3==2.0.7 -werkzeug==1.0.1 -zipp==3.15.0 diff --git a/.riot/requirements/1b0d603.txt b/.riot/requirements/1b0d603.txt deleted file mode 100644 index c6dfa7f6fae..00000000000 --- a/.riot/requirements/1b0d603.txt +++ /dev/null @@ -1,64 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# pip-compile --no-annotate --resolver=backtracking .riot/requirements/1b0d603.in -# -aiohttp==3.9.1 -aiosignal==1.3.1 -annotated-types==0.6.0 -anyio==4.2.0 -async-timeout==4.0.3 -attrs==23.2.0 -blinker==1.7.0 -certifi==2023.11.17 -charset-normalizer==3.3.2 -click==8.1.7 -coverage[toml]==7.4.0 -dataclasses-json==0.6.3 -exceptiongroup==1.2.0 -flask==3.0.0 -frozenlist==1.4.1 -greenlet==3.0.3 -gunicorn==21.2.0 -hypothesis==6.45.0 -idna==3.6 -importlib-metadata==7.0.1 -iniconfig==2.0.0 -itsdangerous==2.1.2 -jinja2==3.1.2 -jsonpatch==1.33 -jsonpointer==2.4 -langchain==0.0.354 -langchain-community==0.0.8 -langchain-core==0.1.5 -langchain-experimental==0.0.47 -langsmith==0.0.77 -markupsafe==2.1.3 -marshmallow==3.20.1 -mock==5.1.0 -multidict==6.0.4 -mypy-extensions==1.0.0 -numpy==1.26.3 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -psycopg2-binary==2.9.9 -pydantic==2.5.3 -pydantic-core==2.14.6 -pytest==7.4.4 -pytest-cov==4.1.0 -pytest-mock==3.12.0 -pyyaml==6.0.1 -requests==2.31.0 -sniffio==1.3.0 -sortedcontainers==2.4.0 -sqlalchemy==2.0.25 -tenacity==8.2.3 -tomli==2.0.1 -typing-extensions==4.9.0 -typing-inspect==0.9.0 -urllib3==2.1.0 -werkzeug==3.0.1 -yarl==1.9.4 -zipp==3.17.0 diff --git a/.riot/requirements/1d4e95e.txt b/.riot/requirements/1d4e95e.txt new file mode 100644 index 00000000000..9d2871696ae --- /dev/null +++ b/.riot/requirements/1d4e95e.txt @@ -0,0 +1,27 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1d4e95e.in +# +attrs==24.3.0 +coverage[toml]==7.6.10 +gevent==24.11.1 +greenlet==3.1.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +msgpack==1.1.0 +opentracing==2.4.0 +packaging==24.2 +pluggy==1.5.0 +pytest==8.3.4 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.16.0 +sortedcontainers==2.4.0 +zope-event==5.0 +zope-interface==7.2 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.8.0 diff --git a/.riot/requirements/1e2d655.txt b/.riot/requirements/1e2d655.txt deleted file mode 100644 index 7f6b56e2776..00000000000 --- a/.riot/requirements/1e2d655.txt +++ /dev/null @@ -1,60 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --allow-unsafe --no-annotate .riot/requirements/1e2d655.in -# -aiohappyeyeballs==2.4.0 -aiohttp==3.10.5 -aiosignal==1.3.1 -annotated-types==0.7.0 -anyio==4.4.0 -attrs==24.2.0 -blinker==1.8.2 -certifi==2024.7.4 -charset-normalizer==3.3.2 -click==8.1.7 -coverage[toml]==7.6.1 -dataclasses-json==0.6.7 -flask==3.0.3 -frozenlist==1.4.1 -greenlet==3.0.3 -gunicorn==23.0.0 -hypothesis==6.45.0 -idna==3.8 -iniconfig==2.0.0 -itsdangerous==2.2.0 -jinja2==3.1.4 -jsonpatch==1.33 -jsonpointer==3.0.0 -langchain==0.0.354 -langchain-community==0.0.20 -langchain-core==0.1.23 -langchain-experimental==0.0.47 -langsmith==0.0.87 -markupsafe==2.1.5 -marshmallow==3.22.0 -mock==5.1.0 -multidict==6.0.5 -mypy-extensions==1.0.0 -numpy==1.26.4 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.5.0 -psycopg2-binary==2.9.9 -pydantic==2.8.2 -pydantic-core==2.20.1 -pytest==8.3.2 -pytest-cov==5.0.0 -pytest-mock==3.14.0 -pyyaml==6.0.2 -requests==2.32.3 -sniffio==1.3.1 -sortedcontainers==2.4.0 -sqlalchemy==2.0.32 -tenacity==8.5.0 -typing-extensions==4.12.2 -typing-inspect==0.9.0 -urllib3==2.2.2 -werkzeug==3.0.4 -yarl==1.9.4 diff --git a/.riot/requirements/1e81527.txt b/.riot/requirements/1e81527.txt deleted file mode 100644 index 3eae5fd518c..00000000000 --- a/.riot/requirements/1e81527.txt +++ /dev/null @@ -1,36 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.8 -# by the following command: -# -# pip-compile --no-annotate .riot/requirements/1e81527.in -# -attrs==23.1.0 -blinker==1.7.0 -certifi==2023.7.22 -charset-normalizer==3.3.2 -click==8.1.7 -coverage[toml]==7.3.2 -exceptiongroup==1.1.3 -flask==2.3.3 -gunicorn==21.2.0 -hypothesis==6.45.0 -idna==3.4 -importlib-metadata==6.8.0 -iniconfig==2.0.0 -itsdangerous==2.1.2 -jinja2==3.1.2 -markupsafe==2.1.3 -mock==5.1.0 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -psycopg2-binary==2.9.9 -pytest==7.4.3 -pytest-cov==4.1.0 -pytest-mock==3.12.0 -requests==2.31.0 -sortedcontainers==2.4.0 -tomli==2.0.1 -urllib3==2.1.0 -werkzeug==3.0.1 -zipp==3.17.0 diff --git a/.riot/requirements/1fb9968.txt b/.riot/requirements/1fb9968.txt deleted file mode 100644 index 57524a248f5..00000000000 --- a/.riot/requirements/1fb9968.txt +++ /dev/null @@ -1,59 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile --no-annotate .riot/requirements/1fb9968.in -# -aiohttp==3.9.1 -aiosignal==1.3.1 -annotated-types==0.6.0 -anyio==4.2.0 -attrs==23.2.0 -blinker==1.7.0 -certifi==2023.11.17 -charset-normalizer==3.3.2 -click==8.1.7 -coverage[toml]==7.4.0 -dataclasses-json==0.6.3 -flask==3.0.0 -frozenlist==1.4.1 -greenlet==3.0.3 -gunicorn==21.2.0 -hypothesis==6.45.0 -idna==3.6 -iniconfig==2.0.0 -itsdangerous==2.1.2 -jinja2==3.1.2 -jsonpatch==1.33 -jsonpointer==2.4 -langchain==0.0.354 -langchain-community==0.0.8 -langchain-core==0.1.5 -langchain-experimental==0.0.47 -langsmith==0.0.77 -markupsafe==2.1.3 -marshmallow==3.20.1 -mock==5.1.0 -multidict==6.0.4 -mypy-extensions==1.0.0 -numpy==1.26.3 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -psycopg2-binary==2.9.9 -pydantic==2.5.3 -pydantic-core==2.14.6 -pytest==7.4.4 -pytest-cov==4.1.0 -pytest-mock==3.12.0 -pyyaml==6.0.1 -requests==2.31.0 -sniffio==1.3.0 -sortedcontainers==2.4.0 -sqlalchemy==2.0.25 -tenacity==8.2.3 -typing-extensions==4.9.0 -typing-inspect==0.9.0 -urllib3==2.1.0 -werkzeug==3.0.1 -yarl==1.9.4 diff --git a/.riot/requirements/2b94418.txt b/.riot/requirements/2b94418.txt deleted file mode 100644 index a64003e98e9..00000000000 --- a/.riot/requirements/2b94418.txt +++ /dev/null @@ -1,34 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.10 -# by the following command: -# -# pip-compile --no-annotate --resolver=backtracking .riot/requirements/2b94418.in -# -attrs==23.1.0 -blinker==1.7.0 -certifi==2023.7.22 -charset-normalizer==3.3.2 -click==8.1.7 -coverage[toml]==7.3.2 -exceptiongroup==1.1.3 -flask==2.3.3 -gunicorn==21.2.0 -hypothesis==6.45.0 -idna==3.4 -iniconfig==2.0.0 -itsdangerous==2.1.2 -jinja2==3.1.2 -markupsafe==2.1.3 -mock==5.1.0 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -psycopg2-binary==2.9.9 -pytest==7.4.3 -pytest-cov==4.1.0 -pytest-mock==3.12.0 -requests==2.31.0 -sortedcontainers==2.4.0 -tomli==2.0.1 -urllib3==2.1.0 -werkzeug==3.0.1 diff --git a/.riot/requirements/5ccc957.txt b/.riot/requirements/5ccc957.txt new file mode 100644 index 00000000000..144bd78b08e --- /dev/null +++ b/.riot/requirements/5ccc957.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --no-annotate .riot/requirements/5ccc957.in +# +attrs==24.3.0 +coverage[toml]==7.6.9 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.2 +pluggy==1.5.0 +pytest==8.3.4 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.16.0 +sortedcontainers==2.4.0 +tornado==6.4.1 diff --git a/.riot/requirements/5e31227.txt b/.riot/requirements/5e31227.txt new file mode 100644 index 00000000000..a3815ab0d74 --- /dev/null +++ b/.riot/requirements/5e31227.txt @@ -0,0 +1,23 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/5e31227.in +# +attrs==24.3.0 +certifi==2024.12.14 +charset-normalizer==3.4.1 +coverage[toml]==7.6.10 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.2 +pluggy==1.5.0 +pytest==8.3.4 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +requests==2.32.3 +sortedcontainers==2.4.0 +urllib3==2.3.0 diff --git a/.riot/requirements/628e8fe.txt b/.riot/requirements/628e8fe.txt new file mode 100644 index 00000000000..163d0416c31 --- /dev/null +++ b/.riot/requirements/628e8fe.txt @@ -0,0 +1,23 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/628e8fe.in +# +attrs==24.3.0 +certifi==2024.12.14 +charset-normalizer==3.4.1 +coverage[toml]==7.6.10 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.2 +pluggy==1.5.0 +pytest==8.3.4 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +requests==2.32.3 +sortedcontainers==2.4.0 +urllib3==2.3.0 diff --git a/.riot/requirements/27d0ff3.txt b/.riot/requirements/6bec1ec.txt similarity index 95% rename from .riot/requirements/27d0ff3.txt rename to .riot/requirements/6bec1ec.txt index c03419edbdb..3e128a77c79 100644 --- a/.riot/requirements/27d0ff3.txt +++ b/.riot/requirements/6bec1ec.txt @@ -2,13 +2,13 @@ # This file is autogenerated by pip-compile with Python 3.8 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/27d0ff3.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/6bec1ec.in # attrs==24.3.0 coverage[toml]==7.6.1 exceptiongroup==1.2.2 gevent==24.2.1 -greenlet==3.1.1 +greenlet==3.1.0 hypothesis==6.45.0 importlib-metadata==8.5.0 iniconfig==2.0.0 diff --git a/.riot/requirements/8dd53b1.txt b/.riot/requirements/8dd53b1.txt new file mode 100644 index 00000000000..1dbf9b66b89 --- /dev/null +++ b/.riot/requirements/8dd53b1.txt @@ -0,0 +1,25 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/8dd53b1.in +# +attrs==24.3.0 +certifi==2024.12.14 +charset-normalizer==3.4.1 +coverage[toml]==7.6.10 +exceptiongroup==1.2.2 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.2 +pluggy==1.5.0 +pytest==8.3.4 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +requests==2.32.3 +sortedcontainers==2.4.0 +tomli==2.2.1 +urllib3==2.3.0 diff --git a/.riot/requirements/968fdc9.txt b/.riot/requirements/968fdc9.txt new file mode 100644 index 00000000000..6633b871d53 --- /dev/null +++ b/.riot/requirements/968fdc9.txt @@ -0,0 +1,23 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/968fdc9.in +# +attrs==24.3.0 +certifi==2024.12.14 +charset-normalizer==3.4.1 +coverage[toml]==7.6.10 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.2 +pluggy==1.5.0 +pytest==8.3.4 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +requests==2.32.3 +sortedcontainers==2.4.0 +urllib3==2.3.0 diff --git a/.riot/requirements/e53ccba.txt b/.riot/requirements/e53ccba.txt deleted file mode 100644 index 0da0b56ed64..00000000000 --- a/.riot/requirements/e53ccba.txt +++ /dev/null @@ -1,33 +0,0 @@ -# -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: -# -# pip-compile --no-annotate --resolver=backtracking .riot/requirements/e53ccba.in -# -attrs==23.1.0 -certifi==2023.7.22 -charset-normalizer==3.3.2 -click==7.1.2 -coverage[toml]==7.3.2 -exceptiongroup==1.1.3 -flask==1.1.4 -gunicorn==21.2.0 -hypothesis==6.45.0 -idna==3.4 -iniconfig==2.0.0 -itsdangerous==1.1.0 -jinja2==2.11.3 -markupsafe==1.1.1 -mock==5.1.0 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -psycopg2-binary==2.9.9 -pytest==7.4.3 -pytest-cov==4.1.0 -pytest-mock==3.12.0 -requests==2.31.0 -sortedcontainers==2.4.0 -tomli==2.0.1 -urllib3==2.1.0 -werkzeug==1.0.1 diff --git a/benchmarks/bm/utils.py b/benchmarks/bm/utils.py index ba6461336b5..dd7b4991c57 100644 --- a/benchmarks/bm/utils.py +++ b/benchmarks/bm/utils.py @@ -8,8 +8,8 @@ from ddtrace import __version__ as ddtrace_version from ddtrace._trace.span import Span -from ddtrace.filters import TraceFilter from ddtrace.internal import telemetry +from ddtrace.trace import TraceFilter _Span = Span diff --git a/ddtrace/__init__.py b/ddtrace/__init__.py index 1f2049cd0a5..b555d1117ca 100644 --- a/ddtrace/__init__.py +++ b/ddtrace/__init__.py @@ -1,4 +1,5 @@ import sys +import os import warnings @@ -26,7 +27,7 @@ from ._monkey import patch_all # noqa: E402 from .internal.compat import PYTHON_VERSION_INFO # noqa: E402 from .internal.utils.deprecations import DDTraceDeprecationWarning # noqa: E402 -from .pin import Pin # noqa: E402 +from ddtrace._trace.pin import Pin # noqa: E402 from ddtrace._trace.span import Span # noqa: E402 from ddtrace._trace.tracer import Tracer # noqa: E402 from ddtrace.vendor import debtcollector @@ -42,39 +43,42 @@ # initialization, which added this module to sys.modules. We catch deprecation # warnings as this is only to retain a side effect of the package # initialization. +# TODO: Remove this in v3.0 when the ddtrace/tracer.py module is removed with warnings.catch_warnings(): warnings.simplefilter("ignore") from .tracer import Tracer as _ - __version__ = get_version() -# a global tracer instance with integration settings -tracer = Tracer() +# TODO: Deprecate accessing tracer from ddtrace.__init__ module in v4.0 +if os.environ.get("_DD_GLOBAL_TRACER_INIT", "true").lower() in ("1", "true"): + from ddtrace.trace import tracer # noqa: F401 __all__ = [ "patch", "patch_all", "Pin", "Span", - "tracer", "Tracer", "config", "DDTraceDeprecationWarning", ] -_DEPRECATED_MODULE_ATTRIBUTES = [ +_DEPRECATED_TRACE_ATTRIBUTES = [ "Span", "Tracer", + "Pin", ] def __getattr__(name): - if name in _DEPRECATED_MODULE_ATTRIBUTES: + if name in _DEPRECATED_TRACE_ATTRIBUTES: debtcollector.deprecate( ("%s.%s is deprecated" % (__name__, name)), + message="Import from ddtrace.trace instead.", category=DDTraceDeprecationWarning, + removal_version="3.0.0", ) if name in globals(): diff --git a/ddtrace/_monkey.py b/ddtrace/_monkey.py index 488211e46b1..d2306ace9ce 100644 --- a/ddtrace/_monkey.py +++ b/ddtrace/_monkey.py @@ -6,6 +6,7 @@ from wrapt.importer import when_imported from ddtrace.appsec import load_common_appsec_modules +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE from .appsec._iast._utils import _is_iast_enabled from .internal import telemetry @@ -186,7 +187,10 @@ def on_import(hook): ) telemetry.telemetry_writer.add_integration(module, False, PATCH_MODULES.get(module) is True, str(e)) telemetry.telemetry_writer.add_count_metric( - "tracers", "integration_errors", 1, (("integration_name", module), ("error_type", type(e).__name__)) + TELEMETRY_NAMESPACE.TRACERS, + "integration_errors", + 1, + (("integration_name", module), ("error_type", type(e).__name__)), ) else: if hasattr(imported_module, "get_versions"): @@ -206,7 +210,7 @@ def on_import(hook): def patch_all(**patch_modules): # type: (bool) -> None - """Automatically patches all available modules. + """Enables ddtrace library instrumentation. In addition to ``patch_modules``, an override can be specified via an environment variable, ``DD_TRACE__ENABLED`` for each module. diff --git a/ddtrace/_trace/filters.py b/ddtrace/_trace/filters.py new file mode 100644 index 00000000000..a2e6884f05c --- /dev/null +++ b/ddtrace/_trace/filters.py @@ -0,0 +1,72 @@ +import abc +import re +from typing import TYPE_CHECKING # noqa:F401 +from typing import List # noqa:F401 +from typing import Optional # noqa:F401 +from typing import Union # noqa:F401 + +from ddtrace._trace.processor import TraceProcessor +from ddtrace.ext import http + + +if TYPE_CHECKING: # pragma: no cover + from ddtrace._trace.span import Span # noqa:F401 + + +class TraceFilter(TraceProcessor): + @abc.abstractmethod + def process_trace(self, trace): + # type: (List[Span]) -> Optional[List[Span]] + """Processes a trace. + + None can be returned to prevent the trace from being exported. + """ + pass + + +class FilterRequestsOnUrl(TraceFilter): + r"""Filter out traces from incoming http requests based on the request's url. + + This class takes as argument a list of regular expression patterns + representing the urls to be excluded from tracing. A trace will be excluded + if its root span contains a ``http.url`` tag and if this tag matches any of + the provided regular expression using the standard python regexp match + semantic (https://docs.python.org/3/library/re.html#re.match). + + :param list regexps: a list of regular expressions (or a single string) defining + the urls that should be filtered out. + + Examples: + To filter out http calls to domain api.example.com:: + + FilterRequestsOnUrl(r'http://api\\.example\\.com') + + To filter out http calls to all first level subdomains from example.com:: + + FilterRequestOnUrl(r'http://.*+\\.example\\.com') + + To filter out calls to both http://test.example.com and http://example.com/healthcheck:: + + FilterRequestOnUrl([r'http://test\\.example\\.com', r'http://example\\.com/healthcheck']) + """ + + def __init__(self, regexps: Union[str, List[str]]): + if isinstance(regexps, str): + regexps = [regexps] + self._regexps = [re.compile(regexp) for regexp in regexps] + + def process_trace(self, trace): + # type: (List[Span]) -> Optional[List[Span]] + """ + When the filter is registered in the tracer, process_trace is called by + on each trace before it is sent to the agent, the returned value will + be fed to the next filter in the list. If process_trace returns None, + the whole trace is discarded. + """ + for span in trace: + url = span.get_tag(http.URL) + if span.parent_id is None and url is not None: + for regexp in self._regexps: + if regexp.match(url): + return None + return trace diff --git a/ddtrace/_trace/pin.py b/ddtrace/_trace/pin.py new file mode 100644 index 00000000000..7dd83474749 --- /dev/null +++ b/ddtrace/_trace/pin.py @@ -0,0 +1,229 @@ +from typing import TYPE_CHECKING # noqa:F401 +from typing import Any # noqa:F401 +from typing import Dict # noqa:F401 +from typing import Optional # noqa:F401 + +import wrapt + +import ddtrace +from ddtrace.vendor.debtcollector import deprecate + +from ..internal.logger import get_logger + + +log = get_logger(__name__) + + +# To set attributes on wrapt proxy objects use this prefix: +# http://wrapt.readthedocs.io/en/latest/wrappers.html +_DD_PIN_NAME = "_datadog_pin" +_DD_PIN_PROXY_NAME = "_self_" + _DD_PIN_NAME + + +class Pin(object): + """Pin (a.k.a Patch INfo) is a small class which is used to + set tracing metadata on a particular traced connection. + This is useful if you wanted to, say, trace two different + database clusters. + + >>> conn = sqlite.connect('/tmp/user.db') + >>> # Override a pin for a specific connection + >>> pin = Pin.override(conn, service='user-db') + >>> conn = sqlite.connect('/tmp/image.db') + """ + + __slots__ = ["tags", "tracer", "_target", "_config", "_initialized"] + + def __init__( + self, + service=None, # type: Optional[str] + tags=None, # type: Optional[Dict[str, str]] + tracer=None, + _config=None, # type: Optional[Dict[str, Any]] + ): + # type: (...) -> None + if tracer is not None and tracer is not ddtrace.tracer: + deprecate( + "Initializing ddtrace.Pin with `tracer` argument is deprecated", + message="All Pin instances should use the global tracer instance", + removal_version="3.0.0", + ) + tracer = tracer or ddtrace.tracer + self.tags = tags + self.tracer = tracer + self._target = None # type: Optional[int] + # keep the configuration attribute internal because the + # public API to access it is not the Pin class + self._config = _config or {} # type: Dict[str, Any] + # [Backward compatibility]: service argument updates the `Pin` config + self._config["service_name"] = service + self._initialized = True + + @property + def service(self): + # type: () -> str + """Backward compatibility: accessing to `pin.service` returns the underlying + configuration value. + """ + return self._config["service_name"] + + def __setattr__(self, name, value): + if getattr(self, "_initialized", False) and name != "_target": + raise AttributeError("can't mutate a pin, use override() or clone() instead") + super(Pin, self).__setattr__(name, value) + + def __repr__(self): + return "Pin(service=%s, tags=%s, tracer=%s)" % (self.service, self.tags, self.tracer) + + @staticmethod + def _find(*objs): + # type: (Any) -> Optional[Pin] + """ + Return the first :class:`ddtrace.pin.Pin` found on any of the provided objects or `None` if none were found + + + >>> pin = Pin._find(wrapper, instance, conn) + + :param objs: The objects to search for a :class:`ddtrace.pin.Pin` on + :type objs: List of objects + :rtype: :class:`ddtrace.pin.Pin`, None + :returns: The first found :class:`ddtrace.pin.Pin` or `None` is none was found + """ + for obj in objs: + pin = Pin.get_from(obj) + if pin: + return pin + return None + + @staticmethod + def get_from(obj): + # type: (Any) -> Optional[Pin] + """Return the pin associated with the given object. If a pin is attached to + `obj` but the instance is not the owner of the pin, a new pin is cloned and + attached. This ensures that a pin inherited from a class is a copy for the new + instance, avoiding that a specific instance overrides other pins values. + + >>> pin = Pin.get_from(conn) + + :param obj: The object to look for a :class:`ddtrace.pin.Pin` on + :type obj: object + :rtype: :class:`ddtrace.pin.Pin`, None + :returns: :class:`ddtrace.pin.Pin` associated with the object, or None if none was found + """ + if hasattr(obj, "__getddpin__"): + return obj.__getddpin__() + + pin_name = _DD_PIN_PROXY_NAME if isinstance(obj, wrapt.ObjectProxy) else _DD_PIN_NAME + pin = getattr(obj, pin_name, None) + # detect if the PIN has been inherited from a class + if pin is not None and pin._target != id(obj): + pin = pin.clone() + pin.onto(obj) + return pin + + @classmethod + def override( + cls, + obj, # type: Any + service=None, # type: Optional[str] + tags=None, # type: Optional[Dict[str, str]] + tracer=None, + ): + # type: (...) -> None + """Override an object with the given attributes. + + That's the recommended way to customize an already instrumented client, without + losing existing attributes. + + >>> conn = sqlite.connect('/tmp/user.db') + >>> # Override a pin for a specific connection + >>> Pin.override(conn, service='user-db') + """ + if tracer is not None: + deprecate( + "Calling ddtrace.Pin.override(...) with the `tracer` argument is deprecated", + message="All Pin instances should use the global tracer instance", + removal_version="3.0.0", + ) + if not obj: + return + + pin = cls.get_from(obj) + if pin is None: + Pin(service=service, tags=tags, tracer=tracer).onto(obj) + else: + pin.clone(service=service, tags=tags, tracer=tracer).onto(obj) + + def enabled(self): + # type: () -> bool + """Return true if this pin's tracer is enabled.""" + # inline to avoid circular imports + from ddtrace.settings.asm import config as asm_config + + return bool(self.tracer) and (self.tracer.enabled or asm_config._apm_opt_out) + + def onto(self, obj, send=True): + # type: (Any, bool) -> None + """Patch this pin onto the given object. If send is true, it will also + queue the metadata to be sent to the server. + """ + # Actually patch it on the object. + try: + if hasattr(obj, "__setddpin__"): + return obj.__setddpin__(self) + + pin_name = _DD_PIN_PROXY_NAME if isinstance(obj, wrapt.ObjectProxy) else _DD_PIN_NAME + + # set the target reference; any get_from, clones and retarget the new PIN + self._target = id(obj) + if self.service: + ddtrace.config._add_extra_service(self.service) + return setattr(obj, pin_name, self) + except AttributeError: + log.debug("can't pin onto object. skipping", exc_info=True) + + def remove_from(self, obj): + # type: (Any) -> None + # Remove pin from the object. + try: + pin_name = _DD_PIN_PROXY_NAME if isinstance(obj, wrapt.ObjectProxy) else _DD_PIN_NAME + + pin = Pin.get_from(obj) + if pin is not None: + delattr(obj, pin_name) + except AttributeError: + log.debug("can't remove pin from object. skipping", exc_info=True) + + def clone( + self, + service=None, # type: Optional[str] + tags=None, # type: Optional[Dict[str, str]] + tracer=None, + ): + # type: (...) -> Pin + """Return a clone of the pin with the given attributes replaced.""" + # do a shallow copy of Pin dicts + if not tags and self.tags: + tags = self.tags.copy() + + if tracer is not None: + deprecate( + "Initializing ddtrace.Pin with `tracer` argument is deprecated", + message="All Pin instances should use the global tracer instance", + removal_version="3.0.0", + ) + + # we use a copy instead of a deepcopy because we expect configurations + # to have only a root level dictionary without nested objects. Using + # deepcopy introduces a big overhead: + # + # copy: 0.00654911994934082 + # deepcopy: 0.2787208557128906 + config = self._config.copy() + + return Pin( + service=service or self.service, + tags=tags, + tracer=tracer or self.tracer, # do not clone the Tracer + _config=config, + ) diff --git a/ddtrace/_trace/processor/__init__.py b/ddtrace/_trace/processor/__init__.py index 3657ee79c39..fc59a64828b 100644 --- a/ddtrace/_trace/processor/__init__.py +++ b/ddtrace/_trace/processor/__init__.py @@ -9,6 +9,7 @@ from typing import Union from ddtrace import config +from ddtrace._trace.sampler import BaseSampler from ddtrace._trace.span import Span from ddtrace._trace.span import _get_64_highest_order_bits_as_hex from ddtrace._trace.span import _is_top_level @@ -25,9 +26,8 @@ from ddtrace.internal.sampling import is_single_span_sampled from ddtrace.internal.service import ServiceStatusError from ddtrace.internal.telemetry.constants import TELEMETRY_LOG_LEVEL -from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE_TAG_TRACER +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE from ddtrace.internal.writer import TraceWriter -from ddtrace.sampler import BaseSampler try: @@ -392,6 +392,6 @@ def _queue_span_count_metrics(self, metric_name: str, tag_name: str, min_count: if config._telemetry_enabled and sum(self._span_metrics[metric_name].values()) >= min_count: for tag_value, count in self._span_metrics[metric_name].items(): telemetry.telemetry_writer.add_count_metric( - TELEMETRY_NAMESPACE_TAG_TRACER, metric_name, count, tags=((tag_name, tag_value),) + TELEMETRY_NAMESPACE.TRACERS, metric_name, count, tags=((tag_name, tag_value),) ) self._span_metrics[metric_name] = defaultdict(int) diff --git a/ddtrace/_trace/sampler.py b/ddtrace/_trace/sampler.py new file mode 100644 index 00000000000..6e2515ee152 --- /dev/null +++ b/ddtrace/_trace/sampler.py @@ -0,0 +1,368 @@ +"""Samplers manage the client-side trace sampling + +Any `sampled = False` trace won't be written, and can be ignored by the instrumentation. +""" + +import abc +import json +from typing import TYPE_CHECKING # noqa:F401 +from typing import Dict # noqa:F401 +from typing import List # noqa:F401 +from typing import Optional # noqa:F401 +from typing import Tuple # noqa:F401 + +from ddtrace import config +from ddtrace.constants import SAMPLING_LIMIT_DECISION + +from ..constants import ENV_KEY +from ..internal.constants import _PRIORITY_CATEGORY +from ..internal.constants import DEFAULT_SAMPLING_RATE_LIMIT +from ..internal.constants import MAX_UINT_64BITS as _MAX_UINT_64BITS +from ..internal.logger import get_logger +from ..internal.rate_limiter import RateLimiter +from ..internal.sampling import _get_highest_precedence_rule_matching +from ..internal.sampling import _set_sampling_tags +from ..settings import _config as ddconfig +from .sampling_rule import SamplingRule + + +PROVENANCE_ORDER = ["customer", "dynamic", "default"] + +try: + from json.decoder import JSONDecodeError +except ImportError: + # handling python 2.X import error + JSONDecodeError = ValueError # type: ignore + +if TYPE_CHECKING: # pragma: no cover + from ddtrace._trace.span import Span # noqa:F401 + + +log = get_logger(__name__) + +# All references to MAX_TRACE_ID were replaced with _MAX_UINT_64BITS. +# Now that ddtrace supports generating 128bit trace_ids, +# the max trace id should be 2**128 - 1 (not 2**64 -1) +# MAX_TRACE_ID is no longer used and should be removed. +MAX_TRACE_ID = _MAX_UINT_64BITS +# Has to be the same factor and key as the Agent to allow chained sampling +KNUTH_FACTOR = 1111111111111111111 + + +class SamplingError(Exception): + pass + + +class BaseSampler(metaclass=abc.ABCMeta): + __slots__ = () + + @abc.abstractmethod + def sample(self, span): + # type: (Span) -> bool + pass + + +class BasePrioritySampler(BaseSampler): + __slots__ = () + + @abc.abstractmethod + def update_rate_by_service_sample_rates(self, sample_rates): + pass + + +class AllSampler(BaseSampler): + """Sampler sampling all the traces""" + + def sample(self, span): + return True + + +class RateSampler(BaseSampler): + """Sampler based on a rate + + Keep (100 * `sample_rate`)% of the traces. + It samples randomly, its main purpose is to reduce the instrumentation footprint. + """ + + def __init__(self, sample_rate: float = 1.0) -> None: + """sample_rate is clamped between 0 and 1 inclusive""" + sample_rate = min(1, max(0, sample_rate)) + self.set_sample_rate(sample_rate) + log.debug("initialized RateSampler, sample %s%% of traces", 100 * sample_rate) + + def set_sample_rate(self, sample_rate: float) -> None: + self.sample_rate = float(sample_rate) + self.sampling_id_threshold = self.sample_rate * _MAX_UINT_64BITS + + def sample(self, span): + sampled = ((span._trace_id_64bits * KNUTH_FACTOR) % _MAX_UINT_64BITS) <= self.sampling_id_threshold + return sampled + + +class _AgentRateSampler(RateSampler): + pass + + +class RateByServiceSampler(BasePrioritySampler): + """Sampler based on a rate, by service + + Keep (100 * `sample_rate`)% of the traces. + The sample rate is kept independently for each service/env tuple. + """ + + __slots__ = ("sample_rate", "_by_service_samplers", "_default_sampler") + + _default_key = "service:,env:" + + @staticmethod + def _key( + service=None, # type: Optional[str] + env=None, # type: Optional[str] + ): + # type: (...) -> str + """Compute a key with the same format used by the Datadog agent API.""" + service = service or "" + env = env or "" + return "service:" + service + ",env:" + env + + def __init__(self, sample_rate=1.0): + # type: (float) -> None + self.sample_rate = sample_rate + self._default_sampler = RateSampler(self.sample_rate) + self._by_service_samplers = {} # type: Dict[str, RateSampler] + + def set_sample_rate( + self, + sample_rate, # type: float + service=None, # type: Optional[str] + env=None, # type: Optional[str] + ): + # type: (...) -> None + + # if we have a blank service, we need to match it to the config.service + if service is None: + service = config.service + if env is None: + env = config.env + + self._by_service_samplers[self._key(service, env)] = _AgentRateSampler(sample_rate) + + def sample(self, span): + sampled, sampler = self._make_sampling_decision(span) + _set_sampling_tags( + span, + sampled, + sampler.sample_rate, + self._choose_priority_category(sampler), + ) + return sampled + + def _choose_priority_category(self, sampler): + # type: (BaseSampler) -> str + if sampler is self._default_sampler: + return _PRIORITY_CATEGORY.DEFAULT + elif isinstance(sampler, _AgentRateSampler): + return _PRIORITY_CATEGORY.AUTO + else: + return _PRIORITY_CATEGORY.RULE_DEF + + def _make_sampling_decision(self, span): + # type: (Span) -> Tuple[bool, BaseSampler] + env = span.get_tag(ENV_KEY) + key = self._key(span.service, env) + sampler = self._by_service_samplers.get(key) or self._default_sampler + sampled = sampler.sample(span) + return sampled, sampler + + def update_rate_by_service_sample_rates(self, rate_by_service): + # type: (Dict[str, float]) -> None + samplers = {} # type: Dict[str, RateSampler] + for key, sample_rate in rate_by_service.items(): + samplers[key] = _AgentRateSampler(sample_rate) + + self._by_service_samplers = samplers + + +class DatadogSampler(RateByServiceSampler): + """ + By default, this sampler relies on dynamic sample rates provided by the trace agent + to determine which traces are kept or dropped. + + You can also configure a static sample rate via ``default_sample_rate`` to use for sampling. + When a ``default_sample_rate`` is configured, that is the only sample rate used, and the agent + provided rates are ignored. + + You may also supply a list of ``SamplingRule`` instances to set sample rates for specific + services. + + Example rules:: + + DatadogSampler(rules=[ + SamplingRule(sample_rate=1.0, service="my-svc"), + SamplingRule(sample_rate=0.0, service="less-important"), + ]) + + Rules are evaluated in the order they are provided, and the first rule that matches is used. + If no rule matches, then the agent sample rates are used. + + This sampler can be configured with a rate limit. This will ensure the max number of + sampled traces per second does not exceed the supplied limit. The default is 100 traces kept + per second. + """ + + __slots__ = ("limiter", "rules", "default_sample_rate", "_rate_limit_always_on") + + NO_RATE_LIMIT = -1 + # deprecate and remove the DEFAULT_RATE_LIMIT field from DatadogSampler + DEFAULT_RATE_LIMIT = DEFAULT_SAMPLING_RATE_LIMIT + + def __init__( + self, + rules=None, # type: Optional[List[SamplingRule]] + default_sample_rate=None, # type: Optional[float] + rate_limit=None, # type: Optional[int] + rate_limit_window=1e9, # type: float + rate_limit_always_on=False, # type: bool + ): + # type: (...) -> None + """ + Constructor for DatadogSampler sampler + + :param rules: List of :class:`SamplingRule` rules to apply to the root span of every trace, default no rules + :param default_sample_rate: The default sample rate to apply if no rules matched (default: ``None`` / + Use :class:`RateByServiceSampler` only) + :param rate_limit: Global rate limit (traces per second) to apply to all traces regardless of the rules + applied to them, (default: ``100``) + """ + # Use default sample rate of 1.0 + super(DatadogSampler, self).__init__() + self.default_sample_rate = default_sample_rate + effective_sample_rate = default_sample_rate + if default_sample_rate is None: + if ddconfig._get_source("_trace_sample_rate") != "default": + effective_sample_rate = float(ddconfig._trace_sample_rate) + + if rate_limit is None: + rate_limit = int(ddconfig._trace_rate_limit) + + self._rate_limit_always_on = rate_limit_always_on + + if rules is None: + env_sampling_rules = ddconfig._trace_sampling_rules + if env_sampling_rules: + rules = self._parse_rules_from_str(env_sampling_rules) + else: + rules = [] + self.rules = rules + else: + self.rules = [] + # Validate that rules is a list of SampleRules + for rule in rules: + if isinstance(rule, SamplingRule): + self.rules.append(rule) + elif config._raise: + raise TypeError("Rule {!r} must be a sub-class of type ddtrace.sampler.SamplingRules".format(rule)) + + # DEV: sampling rule must come last + if effective_sample_rate is not None: + self.rules.append(SamplingRule(sample_rate=effective_sample_rate)) + + # Configure rate limiter + self.limiter = RateLimiter(rate_limit, rate_limit_window) + + log.debug("initialized %r", self) + + def __str__(self): + rates = {key: sampler.sample_rate for key, sampler in self._by_service_samplers.items()} + return "{}(agent_rates={!r}, limiter={!r}, rules={!r})".format( + self.__class__.__name__, rates, self.limiter, self.rules + ) + + __repr__ = __str__ + + @staticmethod + def _parse_rules_from_str(rules): + # type: (str) -> List[SamplingRule] + sampling_rules = [] + try: + json_rules = json.loads(rules) + except JSONDecodeError: + if config._raise: + raise ValueError("Unable to parse DD_TRACE_SAMPLING_RULES={}".format(rules)) + for rule in json_rules: + if "sample_rate" not in rule: + if config._raise: + raise KeyError("No sample_rate provided for sampling rule: {}".format(json.dumps(rule))) + continue + sample_rate = float(rule["sample_rate"]) + service = rule.get("service", SamplingRule.NO_RULE) + name = rule.get("name", SamplingRule.NO_RULE) + resource = rule.get("resource", SamplingRule.NO_RULE) + tags = rule.get("tags", SamplingRule.NO_RULE) + provenance = rule.get("provenance", "default") + try: + sampling_rule = SamplingRule( + sample_rate=sample_rate, + service=service, + name=name, + resource=resource, + tags=tags, + provenance=provenance, + ) + except ValueError as e: + if config._raise: + raise ValueError("Error creating sampling rule {}: {}".format(json.dumps(rule), e)) + continue + sampling_rules.append(sampling_rule) + + # Sort the sampling_rules list using a lambda function as the key + sampling_rules = sorted(sampling_rules, key=lambda rule: PROVENANCE_ORDER.index(rule.provenance)) + return sampling_rules + + def sample(self, span): + span.context._update_tags(span) + + matched_rule = _get_highest_precedence_rule_matching(span, self.rules) + + sampler = self._default_sampler # type: BaseSampler + sample_rate = self.sample_rate + if matched_rule: + # Client based sampling + sampled = matched_rule.sample(span) + sample_rate = matched_rule.sample_rate + else: + # Agent based sampling + sampled, sampler = super(DatadogSampler, self)._make_sampling_decision(span) + if isinstance(sampler, RateSampler): + sample_rate = sampler.sample_rate + + if matched_rule or self._rate_limit_always_on: + # Avoid rate limiting when trace sample rules and/or sample rates are NOT provided + # by users. In this scenario tracing should default to agent based sampling. ASM + # uses DatadogSampler._rate_limit_always_on to override this functionality. + if sampled: + sampled = self.limiter.is_allowed() + span.set_metric(SAMPLING_LIMIT_DECISION, self.limiter.effective_rate) + _set_sampling_tags( + span, + sampled, + sample_rate, + self._choose_priority_category_with_rule(matched_rule, sampler), + ) + + return sampled + + def _choose_priority_category_with_rule(self, rule, sampler): + # type: (Optional[SamplingRule], BaseSampler) -> str + if rule: + provenance = rule.provenance + if provenance == "customer": + return _PRIORITY_CATEGORY.RULE_CUSTOMER + if provenance == "dynamic": + return _PRIORITY_CATEGORY.RULE_DYNAMIC + return _PRIORITY_CATEGORY.RULE_DEF + elif self._rate_limit_always_on: + # backwards compaitbiility for ASM, when the rate limit is always on (ASM standalone mode) + # we want spans to be set to a MANUAL priority to avoid agent based sampling + return _PRIORITY_CATEGORY.USER + return super(DatadogSampler, self)._choose_priority_category(sampler) diff --git a/ddtrace/_trace/sampling_rule.py b/ddtrace/_trace/sampling_rule.py new file mode 100644 index 00000000000..532a0b71f51 --- /dev/null +++ b/ddtrace/_trace/sampling_rule.py @@ -0,0 +1,244 @@ +from typing import TYPE_CHECKING # noqa:F401 +from typing import Any +from typing import Optional +from typing import Tuple + +from ddtrace.internal.compat import pattern_type +from ddtrace.internal.constants import MAX_UINT_64BITS as _MAX_UINT_64BITS +from ddtrace.internal.glob_matching import GlobMatcher +from ddtrace.internal.logger import get_logger +from ddtrace.internal.utils.cache import cachedmethod +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + + +if TYPE_CHECKING: # pragma: no cover + from ddtrace._trace.span import Span # noqa:F401 + +log = get_logger(__name__) +KNUTH_FACTOR = 1111111111111111111 + + +class SamplingRule(object): + """ + Definition of a sampling rule used by :class:`DatadogSampler` for applying a sample rate on a span + """ + + NO_RULE = object() + + def __init__( + self, + sample_rate: float, + service: Any = NO_RULE, + name: Any = NO_RULE, + resource: Any = NO_RULE, + tags: Any = NO_RULE, + provenance: str = "default", + ) -> None: + """ + Configure a new :class:`SamplingRule` + + .. code:: python + + DatadogSampler([ + # Sample 100% of any trace + SamplingRule(sample_rate=1.0), + + # Sample no healthcheck traces + SamplingRule(sample_rate=0, name='flask.request'), + + # Sample all services ending in `-db` based on a regular expression + SamplingRule(sample_rate=0.5, service=re.compile('-db$')), + + # Sample based on service name using custom function + SamplingRule(sample_rate=0.75, service=lambda service: 'my-app' in service), + ]) + + :param sample_rate: The sample rate to apply to any matching spans + :type sample_rate: :obj:`float` clamped between 0.0 and 1.0 inclusive + :param service: Rule to match the `span.service` on, default no rule defined + :type service: :obj:`object` to directly compare, :obj:`function` to evaluate, or :class:`re.Pattern` to match + :param name: Rule to match the `span.name` on, default no rule defined + :type name: :obj:`object` to directly compare, :obj:`function` to evaluate, or :class:`re.Pattern` to match + :param tags: A dictionary whose keys exactly match the names of tags expected to appear on spans, and whose + values are glob-matches with the expected span tag values. Glob matching supports "*" meaning any + number of characters, and "?" meaning any one character. If all tags specified in a SamplingRule are + matches with a given span, that span is considered to have matching tags with the rule. + """ + self.sample_rate = float(min(1, max(0, sample_rate))) + # since span.py converts None to 'None' for tags, and does not accept 'None' for metrics + # we can just create a GlobMatcher for 'None' and it will match properly + self._tag_value_matchers = ( + {k: GlobMatcher(str(v)) for k, v in tags.items()} if tags != SamplingRule.NO_RULE else {} + ) + self.tags = tags + self.service = self.choose_matcher(service) + self.name = self.choose_matcher(name) + self.resource = self.choose_matcher(resource) + self.provenance = provenance + + @property + def sample_rate(self) -> float: + return self._sample_rate + + @sample_rate.setter + def sample_rate(self, sample_rate: float) -> None: + self._sample_rate = sample_rate + self._sampling_id_threshold = sample_rate * _MAX_UINT_64BITS + + def _pattern_matches(self, prop, pattern): + # If the rule is not set, then assume it matches + # DEV: Having no rule and being `None` are different things + # e.g. ignoring `span.service` vs `span.service == None` + if pattern is self.NO_RULE: + return True + if isinstance(pattern, GlobMatcher): + return pattern.match(str(prop)) + + # If the pattern is callable (e.g. a function) then call it passing the prop + # The expected return value is a boolean so cast the response in case it isn't + if callable(pattern): + try: + return bool(pattern(prop)) + except Exception: + log.warning("%r pattern %r failed with %r", self, pattern, prop, exc_info=True) + # Their function failed to validate, assume it is a False + return False + + # The pattern is a regular expression and the prop is a string + if isinstance(pattern, pattern_type): + try: + return bool(pattern.match(str(prop))) + except (ValueError, TypeError): + # This is to guard us against the casting to a string (shouldn't happen, but still) + log.warning("%r pattern %r failed with %r", self, pattern, prop, exc_info=True) + return False + + # Exact match on the values + return prop == pattern + + @cachedmethod() + def _matches(self, key: Tuple[Optional[str], str, Optional[str]]) -> bool: + # self._matches exists to maintain legacy pattern values such as regex and functions + service, name, resource = key + for prop, pattern in [(service, self.service), (name, self.name), (resource, self.resource)]: + if not self._pattern_matches(prop, pattern): + return False + else: + return True + + def matches(self, span): + # type: (Span) -> bool + """ + Return if this span matches this rule + + :param span: The span to match against + :type span: :class:`ddtrace._trace.span.Span` + :returns: Whether this span matches or not + :rtype: :obj:`bool` + """ + tags_match = self.tags_match(span) + return tags_match and self._matches((span.service, span.name, span.resource)) + + def tags_match(self, span): + # type: (Span) -> bool + tag_match = True + if self._tag_value_matchers: + tag_match = self.check_tags(span.get_tags(), span.get_metrics()) + return tag_match + + def check_tags(self, meta, metrics): + if meta is None and metrics is None: + return False + + tag_match = False + for tag_key in self._tag_value_matchers.keys(): + value = meta.get(tag_key) + tag_match = self._tag_value_matchers[tag_key].match(str(value)) + # If the value doesn't match in meta, check the metrics + if tag_match is False: + value = metrics.get(tag_key) + # Floats: Matching floating point values with a non-zero decimal part is not supported. + # For floating point values with a non-zero decimal part, any all * pattern always returns true. + # Other patterns always return false. + if isinstance(value, float): + if not value.is_integer(): + if self._tag_value_matchers[tag_key].pattern == "*": + tag_match = True + else: + return False + continue + else: + value = int(value) + + tag_match = self._tag_value_matchers[tag_key].match(str(value)) + else: + continue + # if we don't match with all specified tags for a rule, it's not a match + if tag_match is False: + return False + + return tag_match + + def sample(self, span): + """ + Return if this rule chooses to sample the span + + :param span: The span to sample against + :type span: :class:`ddtrace._trace.span.Span` + :returns: Whether this span was sampled + :rtype: :obj:`bool` + """ + if self.sample_rate == 1: + return True + elif self.sample_rate == 0: + return False + + return ((span._trace_id_64bits * KNUTH_FACTOR) % _MAX_UINT_64BITS) <= self._sampling_id_threshold + + def _no_rule_or_self(self, val): + if val is self.NO_RULE: + return "NO_RULE" + elif val is None: + return "None" + elif type(val) == GlobMatcher: + return val.pattern + else: + return val + + def choose_matcher(self, prop): + # We currently support the ability to pass in a function, a regular expression, or a string + # If a string is passed in we create a GlobMatcher to handle the matching + if callable(prop) or isinstance(prop, pattern_type): + # deprecated: passing a function or a regular expression' + deprecate( + "Using methods or regular expressions for SamplingRule matching is deprecated. ", + message="Please move to passing in a string for Glob matching.", + removal_version="3.0.0", + category=DDTraceDeprecationWarning, + ) + return prop + # Name and Resource will never be None, but service can be, since we str() + # whatever we pass into the GlobMatcher, we can just use its matching + elif prop is None: + prop = "None" + else: + return GlobMatcher(prop) if prop != SamplingRule.NO_RULE else SamplingRule.NO_RULE + + def __repr__(self): + return "{}(sample_rate={!r}, service={!r}, name={!r}, resource={!r}, tags={!r}, provenance={!r})".format( + self.__class__.__name__, + self.sample_rate, + self._no_rule_or_self(self.service), + self._no_rule_or_self(self.name), + self._no_rule_or_self(self.resource), + self._no_rule_or_self(self.tags), + self.provenance, + ) + + __str__ = __repr__ + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, SamplingRule): + return False + return str(self) == str(other) diff --git a/ddtrace/_trace/telemetry.py b/ddtrace/_trace/telemetry.py index f9cd9ef79b9..929acd101ec 100644 --- a/ddtrace/_trace/telemetry.py +++ b/ddtrace/_trace/telemetry.py @@ -2,11 +2,12 @@ from typing import Tuple from ddtrace.internal.telemetry import telemetry_writer +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE def record_span_pointer_calculation(context: str, span_pointer_count: int) -> None: telemetry_writer.add_count_metric( - namespace="tracers", + namespace=TELEMETRY_NAMESPACE.TRACERS, name="span_pointer_calculation", value=1, tags=(("context", context), ("count", _span_pointer_count_to_tag(span_pointer_count))), @@ -45,7 +46,7 @@ def record_span_pointer_calculation_issue( tags += additional_tags telemetry_writer.add_count_metric( - namespace="tracers", + namespace=TELEMETRY_NAMESPACE.TRACERS, name="span_pointer_calculation.issue", value=1, tags=tags, diff --git a/ddtrace/_trace/tracer.py b/ddtrace/_trace/tracer.py index f815e0f184e..5bde36ef480 100644 --- a/ddtrace/_trace/tracer.py +++ b/ddtrace/_trace/tracer.py @@ -18,6 +18,7 @@ from ddtrace import _hooks from ddtrace import config from ddtrace._trace.context import Context +from ddtrace._trace.filters import TraceFilter from ddtrace._trace.processor import SpanAggregator from ddtrace._trace.processor import SpanProcessor from ddtrace._trace.processor import TopLevelSpanProcessor @@ -25,13 +26,15 @@ from ddtrace._trace.processor import TraceSamplingProcessor from ddtrace._trace.processor import TraceTagsProcessor from ddtrace._trace.provider import DefaultContextProvider +from ddtrace._trace.sampler import BasePrioritySampler +from ddtrace._trace.sampler import BaseSampler +from ddtrace._trace.sampler import DatadogSampler from ddtrace._trace.span import Span from ddtrace.appsec._constants import APPSEC from ddtrace.constants import ENV_KEY from ddtrace.constants import HOSTNAME_KEY from ddtrace.constants import PID from ddtrace.constants import VERSION_KEY -from ddtrace.filters import TraceFilter from ddtrace.internal import agent from ddtrace.internal import atexit from ddtrace.internal import compat @@ -41,6 +44,7 @@ from ddtrace.internal.atexit import register_on_exit_signal from ddtrace.internal.constants import SAMPLING_DECISION_TRACE_TAG_KEY from ddtrace.internal.constants import SPAN_API_DATADOG +from ddtrace.internal.core import dispatch from ddtrace.internal.dogstatsd import get_dogstatsd_client from ddtrace.internal.logger import get_logger from ddtrace.internal.peer_service.processor import PeerServiceProcessor @@ -62,9 +66,6 @@ from ddtrace.internal.writer import AgentWriter from ddtrace.internal.writer import LogWriter from ddtrace.internal.writer import TraceWriter -from ddtrace.sampler import BasePrioritySampler -from ddtrace.sampler import BaseSampler -from ddtrace.sampler import DatadogSampler from ddtrace.settings import Config from ddtrace.settings.asm import config as asm_config from ddtrace.settings.peer_service import _ps_config @@ -194,6 +195,7 @@ class Tracer(object): """ SHUTDOWN_TIMEOUT = 5 + _instance = None def __init__( self, @@ -208,7 +210,23 @@ def __init__( :param url: The Datadog agent URL. :param dogstatsd_url: The DogStatsD URL. """ - + # Do not set self._instance if this is a subclass of Tracer. Here we only want + # to reference the global instance. + if type(self) is Tracer: + if Tracer._instance is None: + Tracer._instance = self + else: + # ddtrace library does not support context propagation for multiple tracers. + # All instances of ddtrace ContextProviders share the same ContextVars. This means that + # if you create multiple instances of Tracer, spans will be shared between them creating a + # broken experience. + # TODO(mabdinur): Convert this warning to an ValueError in 3.0.0 + deprecate( + "Support for multiple Tracer instances is deprecated", + ". Use ddtrace.tracer instead.", + category=DDTraceDeprecationWarning, + removal_version="3.0.0", + ) self._filters: List[TraceFilter] = [] # globally set tags @@ -774,8 +792,7 @@ def _start_span( service = config.service_mapping.get(service, service) links = context._span_links if not parent else [] - - if trace_id: + if trace_id or links or context._baggage: # child_of a non-empty context, so either a local child span or from a remote context span = Span( name=name, @@ -849,7 +866,7 @@ def _start_span( for p in chain(self._span_processors, SpanProcessor.__processors__, self._deferred_processors): p.on_span_start(span) self._hooks.emit(self.__class__.start_span, span) - + dispatch("trace.span_start", (span,)) return span start_span = _start_span @@ -866,6 +883,8 @@ def _on_span_finish(self, span: Span) -> None: for p in chain(self._span_processors, SpanProcessor.__processors__, self._deferred_processors): p.on_span_finish(span) + dispatch("trace.span_finish", (span,)) + if log.isEnabledFor(logging.DEBUG): log.debug("finishing span %s (enabled:%s)", span._pprint(), self.enabled) @@ -940,18 +959,23 @@ def trace( ) def current_root_span(self) -> Optional[Span]: - """Returns the root span of the current execution. + """Returns the local root span of the current execution/process. + + Note: This cannot be used to access the true root span of the trace + in a distributed tracing setup if the actual root span occurred in + another execution/process. - This is useful for attaching information related to the trace as a - whole without needing to add to child spans. + This is useful for attaching information to the local root span + of the current execution/process, which is often also service + entry span. For example:: - # get the root span - root_span = tracer.current_root_span() + # get the local root span + local_root_span = tracer.current_root_span() # set the host just once on the root span - if root_span: - root_span.set_tag('host', '127.0.0.1') + if local_root_span: + local_root_span.set_tag('host', '127.0.0.1') """ span = self.current_span() if span is None: @@ -1168,11 +1192,11 @@ def _on_global_config_update(self, cfg: Config, items: List[str]) -> None: if "_logs_injection" in items: if config._logs_injection: - from ddtrace.contrib.logging import patch + from ddtrace.contrib.internal.logging.patch import patch patch() else: - from ddtrace.contrib.logging import unpatch + from ddtrace.contrib.internal.logging.patch import unpatch unpatch() diff --git a/ddtrace/appsec/_asm_request_context.py b/ddtrace/appsec/_asm_request_context.py index adb78a4447c..ff398d56b14 100644 --- a/ddtrace/appsec/_asm_request_context.py +++ b/ddtrace/appsec/_asm_request_context.py @@ -17,10 +17,8 @@ from ddtrace.appsec._constants import EXPLOIT_PREVENTION from ddtrace.appsec._constants import SPAN_DATA_NAMES from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled -from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_source from ddtrace.appsec._iast._taint_tracking import OriginType from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject -from ddtrace.appsec._iast._taint_utils import taint_structure from ddtrace.appsec._iast._utils import _is_iast_enabled from ddtrace.appsec._utils import add_context_log from ddtrace.appsec._utils import get_triggers @@ -508,27 +506,8 @@ def _on_wrapped_view(kwargs): return return_value -def _on_set_request_tags(request, span, flask_config): - from ddtrace.appsec._iast._utils import _is_iast_enabled - - if _is_iast_enabled(): - _set_metric_iast_instrumented_source(OriginType.COOKIE_NAME) - _set_metric_iast_instrumented_source(OriginType.COOKIE) - - if not is_iast_request_enabled(): - return - - request.cookies = taint_structure( - request.cookies, - OriginType.COOKIE_NAME, - OriginType.COOKIE, - override_pyobject_tainted=True, - ) - - def _on_pre_tracedrequest(ctx): current_span = ctx.span - _on_set_request_tags(ctx.get_item("flask_request"), current_span, ctx.get_item("flask_config")) block_request_callable = ctx.get_item("block_request_callable") if asm_config._asm_enabled: set_block_request_callable(functools.partial(block_request_callable, current_span)) @@ -599,7 +578,6 @@ def asm_listen(): core.on("django.after_request_headers", _get_headers_if_appsec, "headers") core.on("django.extract_body", _get_headers_if_appsec, "headers") core.on("django.after_request_headers.finalize", _set_headers_and_response) - core.on("flask.set_request_tags", _on_set_request_tags) core.on("asgi.start_request", _call_waf_first) core.on("asgi.start_response", _call_waf) diff --git a/ddtrace/appsec/_capabilities.py b/ddtrace/appsec/_capabilities.py index c173f2d6471..c999b61cb97 100644 --- a/ddtrace/appsec/_capabilities.py +++ b/ddtrace/appsec/_capabilities.py @@ -31,6 +31,7 @@ class Flags(enum.IntFlag): ASM_SESSION_FINGERPRINT = 1 << 33 ASM_NETWORK_FINGERPRINT = 1 << 34 ASM_HEADER_FINGERPRINT = 1 << 35 + ASM_RASP_CMDI = 1 << 37 _ALL_ASM_BLOCKING = ( @@ -49,7 +50,7 @@ class Flags(enum.IntFlag): | Flags.ASM_HEADER_FINGERPRINT ) -_ALL_RASP = Flags.ASM_RASP_SQLI | Flags.ASM_RASP_LFI | Flags.ASM_RASP_SSRF | Flags.ASM_RASP_SHI +_ALL_RASP = Flags.ASM_RASP_SQLI | Flags.ASM_RASP_LFI | Flags.ASM_RASP_SSRF | Flags.ASM_RASP_SHI | Flags.ASM_RASP_CMDI _FEATURE_REQUIRED = Flags.ASM_ACTIVATION | Flags.ASM_AUTO_USER diff --git a/ddtrace/appsec/_common_module_patches.py b/ddtrace/appsec/_common_module_patches.py index 215d8b05ee6..0b455dbba6b 100644 --- a/ddtrace/appsec/_common_module_patches.py +++ b/ddtrace/appsec/_common_module_patches.py @@ -7,16 +7,20 @@ from typing import Callable from typing import Dict from typing import Iterable +from typing import List +from typing import Union from wrapt import FunctionWrapper from wrapt import resolve_path import ddtrace from ddtrace.appsec._asm_request_context import get_blocked +from ddtrace.appsec._constants import EXPLOIT_PREVENTION from ddtrace.appsec._constants import WAF_ACTIONS from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink from ddtrace.appsec._iast.constants import VULN_PATH_TRAVERSAL +import ddtrace.contrib.internal.subprocess.patch as subprocess_patch from ddtrace.internal import core from ddtrace.internal._exceptions import BlockingException from ddtrace.internal._unpatched import _gc as gc @@ -30,6 +34,9 @@ _is_patched = False +_RASP_SYSTEM = "rasp_os.system" +_RASP_POPEN = "rasp_Popen" + def patch_common_modules(): global _is_patched @@ -39,7 +46,10 @@ def patch_common_modules(): try_wrap_function_wrapper("urllib.request", "OpenerDirector.open", wrapped_open_ED4CF71136E15EBF) try_wrap_function_wrapper("_io", "BytesIO.read", wrapped_read_F3E51D71B4EC16EF) try_wrap_function_wrapper("_io", "StringIO.read", wrapped_read_F3E51D71B4EC16EF) - try_wrap_function_wrapper("os", "system", wrapped_system_5542593D237084A7) + # ensure that the subprocess patch is applied even after one click activation + subprocess_patch.patch() + subprocess_patch.add_str_callback(_RASP_SYSTEM, wrapped_system_5542593D237084A7) + subprocess_patch.add_lst_callback(_RASP_POPEN, popen_FD233052260D8B4D) core.on("asm.block.dbapi.execute", execute_4C9BAC8E228EB347) if asm_config._iast_enabled: _set_metric_iast_instrumented_sink(VULN_PATH_TRAVERSAL) @@ -54,6 +64,8 @@ def unpatch_common_modules(): try_unwrap("urllib.request", "OpenerDirector.open") try_unwrap("_io", "BytesIO.read") try_unwrap("_io", "StringIO.read") + subprocess_patch.del_str_callback(_RASP_SYSTEM) + subprocess_patch.del_lst_callback(_RASP_POPEN) _is_patched = False @@ -106,7 +118,6 @@ def wrapped_open_CFDDB7ABBA9081B6(original_open_callable, instance, args, kwargs try: from ddtrace.appsec._asm_request_context import call_waf_callback from ddtrace.appsec._asm_request_context import in_asm_context - from ddtrace.appsec._constants import EXPLOIT_PREVENTION except ImportError: # open is used during module initialization # and shouldn't be changed at that time @@ -124,7 +135,9 @@ def wrapped_open_CFDDB7ABBA9081B6(original_open_callable, instance, args, kwargs rule_type=EXPLOIT_PREVENTION.TYPE.LFI, ) if res and _must_block(res.actions): - raise BlockingException(get_blocked(), "exploit_prevention", "lfi", filename) + raise BlockingException( + get_blocked(), EXPLOIT_PREVENTION.BLOCKING, EXPLOIT_PREVENTION.TYPE.LFI, filename + ) try: return original_open_callable(*args, **kwargs) except Exception as e: @@ -151,7 +164,6 @@ def wrapped_open_ED4CF71136E15EBF(original_open_callable, instance, args, kwargs try: from ddtrace.appsec._asm_request_context import call_waf_callback from ddtrace.appsec._asm_request_context import in_asm_context - from ddtrace.appsec._constants import EXPLOIT_PREVENTION except ImportError: # open is used during module initialization # and shouldn't be changed at that time @@ -168,7 +180,9 @@ def wrapped_open_ED4CF71136E15EBF(original_open_callable, instance, args, kwargs rule_type=EXPLOIT_PREVENTION.TYPE.SSRF, ) if res and _must_block(res.actions): - raise BlockingException(get_blocked(), "exploit_prevention", "ssrf", url) + raise BlockingException( + get_blocked(), EXPLOIT_PREVENTION.BLOCKING, EXPLOIT_PREVENTION.TYPE.SSRF, url + ) return original_open_callable(*args, **kwargs) @@ -191,7 +205,6 @@ def wrapped_request_D8CB81E472AF98A2(original_request_callable, instance, args, try: from ddtrace.appsec._asm_request_context import call_waf_callback from ddtrace.appsec._asm_request_context import in_asm_context - from ddtrace.appsec._constants import EXPLOIT_PREVENTION except ImportError: # open is used during module initialization # and shouldn't be changed at that time @@ -206,50 +219,67 @@ def wrapped_request_D8CB81E472AF98A2(original_request_callable, instance, args, rule_type=EXPLOIT_PREVENTION.TYPE.SSRF, ) if res and _must_block(res.actions): - raise BlockingException(get_blocked(), "exploit_prevention", "ssrf", url) + raise BlockingException( + get_blocked(), EXPLOIT_PREVENTION.BLOCKING, EXPLOIT_PREVENTION.TYPE.SSRF, url + ) return original_request_callable(*args, **kwargs) -def wrapped_system_5542593D237084A7(original_command_callable, instance, args, kwargs): +def wrapped_system_5542593D237084A7(command: str) -> None: """ wrapper for os.system function """ - command = args[0] if args else kwargs.get("command", None) - if command is not None: - if asm_config._iast_enabled and is_iast_request_enabled(): - from ddtrace.appsec._iast.taint_sinks.command_injection import _iast_report_cmdi - - _iast_report_cmdi(command) - - if ( - asm_config._asm_enabled - and asm_config._ep_enabled - and ddtrace.tracer._appsec_processor is not None - and ddtrace.tracer._appsec_processor.rasp_cmdi_enabled - ): - try: - from ddtrace.appsec._asm_request_context import call_waf_callback - from ddtrace.appsec._asm_request_context import in_asm_context - from ddtrace.appsec._constants import EXPLOIT_PREVENTION - except ImportError: - return original_command_callable(*args, **kwargs) - - if in_asm_context(): - res = call_waf_callback( - {EXPLOIT_PREVENTION.ADDRESS.CMDI: command}, - crop_trace="wrapped_system_5542593D237084A7", - rule_type=EXPLOIT_PREVENTION.TYPE.CMDI, + if ( + asm_config._asm_enabled + and asm_config._ep_enabled + and ddtrace.tracer._appsec_processor is not None + and ddtrace.tracer._appsec_processor.rasp_shi_enabled + ): + try: + from ddtrace.appsec._asm_request_context import call_waf_callback + from ddtrace.appsec._asm_request_context import in_asm_context + except ImportError: + return + + if in_asm_context(): + res = call_waf_callback( + {EXPLOIT_PREVENTION.ADDRESS.SHI: command}, + crop_trace="wrapped_system_5542593D237084A7", + rule_type=EXPLOIT_PREVENTION.TYPE.SHI, + ) + if res and _must_block(res.actions): + raise BlockingException( + get_blocked(), EXPLOIT_PREVENTION.BLOCKING, EXPLOIT_PREVENTION.TYPE.SHI, command + ) + + +def popen_FD233052260D8B4D(arg_list: Union[List[str], str]) -> None: + """ + listener for subprocess.Popen class + """ + if ( + asm_config._asm_enabled + and asm_config._ep_enabled + and ddtrace.tracer._appsec_processor is not None + and ddtrace.tracer._appsec_processor.rasp_cmdi_enabled + ): + try: + from ddtrace.appsec._asm_request_context import call_waf_callback + from ddtrace.appsec._asm_request_context import in_asm_context + except ImportError: + return + + if in_asm_context(): + res = call_waf_callback( + {EXPLOIT_PREVENTION.ADDRESS.CMDI: arg_list if isinstance(arg_list, list) else [arg_list]}, + crop_trace="popen_FD233052260D8B4D", + rule_type=EXPLOIT_PREVENTION.TYPE.CMDI, + ) + if res and _must_block(res.actions): + raise BlockingException( + get_blocked(), EXPLOIT_PREVENTION.BLOCKING, EXPLOIT_PREVENTION.TYPE.CMDI, arg_list ) - if res and _must_block(res.actions): - raise BlockingException(get_blocked(), "exploit_prevention", "cmdi", command) - try: - return original_command_callable(*args, **kwargs) - except Exception as e: - previous_frame = e.__traceback__.tb_frame.f_back - raise e.with_traceback( - e.__traceback__.__class__(None, previous_frame, previous_frame.f_lasti, previous_frame.f_lineno) - ) _DB_DIALECTS = { @@ -279,7 +309,6 @@ def execute_4C9BAC8E228EB347(instrument_self, query, args, kwargs) -> None: try: from ddtrace.appsec._asm_request_context import call_waf_callback from ddtrace.appsec._asm_request_context import in_asm_context - from ddtrace.appsec._constants import EXPLOIT_PREVENTION except ImportError: # execute is used during module initialization # and shouldn't be changed at that time @@ -296,7 +325,9 @@ def execute_4C9BAC8E228EB347(instrument_self, query, args, kwargs) -> None: rule_type=EXPLOIT_PREVENTION.TYPE.SQLI, ) if res and _must_block(res.actions): - raise BlockingException(get_blocked(), "exploit_prevention", "sqli", query) + raise BlockingException( + get_blocked(), EXPLOIT_PREVENTION.BLOCKING, EXPLOIT_PREVENTION.TYPE.SQLI, query + ) def try_unwrap(module, name): diff --git a/ddtrace/appsec/_constants.py b/ddtrace/appsec/_constants.py index 83cb53e78ff..92b9e239900 100644 --- a/ddtrace/appsec/_constants.py +++ b/ddtrace/appsec/_constants.py @@ -75,6 +75,8 @@ class APPSEC(metaclass=Constant_Class): CUSTOM_EVENT_PREFIX: Literal["appsec.events"] = "appsec.events" USER_LOGIN_EVENT_PREFIX: Literal["_dd.appsec.events.users.login"] = "_dd.appsec.events.users.login" USER_LOGIN_EVENT_PREFIX_PUBLIC: Literal["appsec.events.users.login"] = "appsec.events.users.login" + USER_LOGIN_USERID: Literal["_dd.appsec.usr.id"] = "_dd.appsec.usr.id" + USER_LOGIN_USERNAME: Literal["_dd.appsec.usr.login"] = "_dd.appsec.usr.login" USER_LOGIN_EVENT_SUCCESS_TRACK: Literal[ "appsec.events.users.login.success.track" ] = "appsec.events.users.login.success.track" @@ -180,6 +182,7 @@ class WAF_DATA_NAMES(metaclass=Constant_Class): REQUEST_COOKIES: Literal["server.request.cookies"] = "server.request.cookies" REQUEST_HTTP_IP: Literal["http.client_ip"] = "http.client_ip" REQUEST_USER_ID: Literal["usr.id"] = "usr.id" + REQUEST_USERNAME: Literal["usr.login"] = "usr.login" RESPONSE_STATUS: Literal["server.response.status"] = "server.response.status" RESPONSE_HEADERS_NO_COOKIES: Literal["server.response.headers.no_cookies"] = "server.response.headers.no_cookies" RESPONSE_BODY: Literal["server.response.body"] = "server.response.body" @@ -194,6 +197,7 @@ class WAF_DATA_NAMES(metaclass=Constant_Class): REQUEST_COOKIES, REQUEST_HTTP_IP, REQUEST_USER_ID, + REQUEST_USERNAME, RESPONSE_STATUS, RESPONSE_HEADERS_NO_COOKIES, RESPONSE_BODY, @@ -202,7 +206,8 @@ class WAF_DATA_NAMES(metaclass=Constant_Class): # EPHEMERAL ADDRESSES PROCESSOR_SETTINGS: Literal["waf.context.processor"] = "waf.context.processor" - CMDI_ADDRESS: Literal["server.sys.shell.cmd"] = "server.sys.shell.cmd" + CMDI_ADDRESS: Literal["server.sys.exec.cmd"] = "server.sys.exec.cmd" + SHI_ADDRESS: Literal["server.sys.shell.cmd"] = "server.sys.shell.cmd" LFI_ADDRESS: Literal["server.io.fs.file"] = "server.io.fs.file" SSRF_ADDRESS: Literal["server.io.net.url"] = "server.io.net.url" SQLI_ADDRESS: Literal["server.db.statement"] = "server.db.statement" @@ -328,6 +333,7 @@ class DEFAULT(metaclass=Constant_Class): class EXPLOIT_PREVENTION(metaclass=Constant_Class): + BLOCKING: Literal["exploit_prevention"] = "exploit_prevention" STACK_TRACE_ID: Literal["stack_id"] = "stack_id" EP_ENABLED: Literal["DD_APPSEC_RASP_ENABLED"] = "DD_APPSEC_RASP_ENABLED" STACK_TRACE_ENABLED: Literal["DD_APPSEC_STACK_TRACE_ENABLED"] = "DD_APPSEC_STACK_TRACE_ENABLED" @@ -339,6 +345,7 @@ class EXPLOIT_PREVENTION(metaclass=Constant_Class): class TYPE(metaclass=Constant_Class): CMDI: Literal["command_injection"] = "command_injection" + SHI: Literal["shell_injection"] = "shell_injection" LFI: Literal["lfi"] = "lfi" SSRF: Literal["ssrf"] = "ssrf" SQLI: Literal["sql_injection"] = "sql_injection" @@ -346,6 +353,7 @@ class TYPE(metaclass=Constant_Class): class ADDRESS(metaclass=Constant_Class): CMDI: Literal["CMDI_ADDRESS"] = "CMDI_ADDRESS" LFI: Literal["LFI_ADDRESS"] = "LFI_ADDRESS" + SHI: Literal["SHI_ADDRESS"] = "SHI_ADDRESS" SSRF: Literal["SSRF_ADDRESS"] = "SSRF_ADDRESS" SQLI: Literal["SQLI_ADDRESS"] = "SQLI_ADDRESS" SQLI_TYPE: Literal["SQLI_SYSTEM_ADDRESS"] = "SQLI_SYSTEM_ADDRESS" diff --git a/ddtrace/appsec/_iast/_ast/ast_patching.py b/ddtrace/appsec/_iast/_ast/ast_patching.py index 7e2258bd556..bb3a9c74d44 100644 --- a/ddtrace/appsec/_iast/_ast/ast_patching.py +++ b/ddtrace/appsec/_iast/_ast/ast_patching.py @@ -7,7 +7,9 @@ from sys import version_info import textwrap from types import ModuleType +from typing import Iterable from typing import Optional +from typing import Set from typing import Text from typing import Tuple @@ -25,16 +27,43 @@ _PREFIX = IAST.PATCH_ADDED_SYMBOL_PREFIX # Prefixes for modules where IAST patching is allowed -IAST_ALLOWLIST: Tuple[Text, ...] = ("tests.appsec.iast.",) +# Only packages that have the test_propagation=True in test_packages and are not in the denylist must be here +IAST_ALLOWLIST: Tuple[Text, ...] = ( + "attrs.", + "beautifulsoup4.", + "cachetools.", + "cryptography.", + "docutils.", + "idna.", + "iniconfig.", + "jinja2.", + "lxml.", + "multidict.", + "platformdirs", + "pygments.", + "pynacl.", + "pyparsing.", + "multipart", + "sqlalchemy.", + "tomli", + "yarl.", +) + +# NOTE: For testing reasons, don't add astunparse here, see test_ast_patching.py IAST_DENYLIST: Tuple[Text, ...] = ( - "flask.", - "werkzeug.", + "_psycopg.", # PostgreSQL adapter for Python (v3) + "_pytest.", "aiohttp._helpers.", "aiohttp._http_parser.", "aiohttp._http_writer.", "aiohttp._websocket.", "aiohttp.log.", "aiohttp.tcp_helpers.", + "aioquic.", + "altgraph.", + "anyio.", + "api_pb2.", # Patching crashes with these auto-generated modules, propagation is not needed + "api_pb2_grpc.", # Patching crashes with these auto-generated modules, propagation is not needed "asyncio.base_events.", "asyncio.base_futures.", "asyncio.base_subprocess.", @@ -57,11 +86,15 @@ "asyncio.transports.", "asyncio.trsock.", "asyncio.unix_events.", + "asyncpg.pgproto.", "attr._config.", "attr._next_gen.", "attr.filters.", "attr.setters.", + "autopep8.", "backports.", + "black.", + "blinker.", "boto3.docs.docstring.", "boto3.s3.", "botocore.docs.bcdoc.", @@ -69,6 +102,8 @@ "botocore.vendored.requests.", "brotli.", "brotlicffi.", + "bytecode.", + "cattrs.", "cchardet.", "certifi.", "cffi.", @@ -103,13 +138,23 @@ "colorama.", "concurrent.futures.", "configparser.", + "contourpy.", "coreschema.", "crispy_forms.", + "crypto.", # This module is patched by the IAST patch methods, propagation is not needed + "cx_logging.", + "cycler.", + "cython.", "dateutil.", + "dateutil.", + "ddsketch.", + "ddtrace.", "defusedxml.", + "deprecated.", "difflib.", "dill.info.", "dill.settings.", + "dipy.", "django.apps.config.", "django.apps.registry.", "django.conf.", @@ -255,76 +300,136 @@ "django_filters.rest_framework.filterset.", "django_filters.utils.", "django_filters.widgets.", - "crypto.", # This module is patched by the IAST patch methods, propagation is not needed - "deprecated.", - "api_pb2.", # Patching crashes with these auto-generated modules, propagation is not needed - "api_pb2_grpc.", # Patching crashes with these auto-generated modules, propagation is not needed - "asyncpg.pgproto.", - "blinker.", - "bytecode.", - "cattrs.", - "ddsketch.", - "ddtrace.", + "dnspython.", + "elasticdeform.", "envier.", "exceptiongroup.", + "flask.", + "fonttools.", "freezegun.", # Testing utilities for time manipulation + "google.auth.", + "googlecloudsdk.", + "gprof2dot.", + "h11.", + "h5py.", + "httpcore.", + "httptools.", + "httpx.", "hypothesis.", # Testing utilities + "imageio.", "importlib_metadata.", "inspect.", # this package is used to get the stack frames, propagation is not needed "itsdangerous.", + "kiwisolver.", + "matplotlib.", "moto.", # used for mocking AWS, propagation is not needed + "mypy.", + "mypy_extensions.", + "networkx.", + "nibabel.", + "nilearn.", + "numba.", + "numpy.", "opentelemetry-api.", "packaging.", + "pandas.", + "pdf2image.", + "pefile.", + "pil.", "pip.", "pkg_resources.", "pluggy.", "protobuf.", "psycopg.", # PostgreSQL adapter for Python (v3) - "_psycopg.", # PostgreSQL adapter for Python (v3) "psycopg2.", # PostgreSQL adapter for Python (v2) + "pycodestyle.", "pycparser.", # this package is called when a module is imported, propagation is not needed + "pydicom.", + "pyinstaller.", + "pynndescent.", + "pystray.", "pytest.", # Testing framework - "_pytest.", + "pytz.", + "rich.", + "sanic.", + "scipy.", "setuptools.", + "silk.", # django-silk package + "skbase.", "sklearn.", # Machine learning library + "sniffio.", "sqlalchemy.orm.interfaces.", # Performance optimization + "threadpoolctl.", + "tifffile.", + "tqdm.", + "trx.", "typing_extensions.", + "umap.", "unittest.mock.", - "uvloop.", "urlpatterns_reverse.tests.", # assertRaises eat exceptions in native code, so we don't call the original function - "wrapt.", - "zipp.", - # This is a workaround for Sanic failures: + "uvicorn.", + "uvloop.", + "wcwidth.", "websocket.", - "h11.", - "aioquic.", - "httptools.", - "sniffio.", - "sanic.", - "rich.", - "httpx.", "websockets.", - "uvicorn.", - "anyio.", - "httpcore.", - "google.auth.", - "googlecloudsdk.", - "umap.", - "pynndescent.", - "numba.", + "werkzeug.", + "win32ctypes.", + "wrapt.", + "xlib.", + "zipp.", ) +USER_ALLOWLIST = tuple(os.environ.get(IAST.PATCH_MODULES, "").split(IAST.SEP_MODULES)) +USER_DENYLIST = tuple(os.environ.get(IAST.DENY_MODULES, "").split(IAST.SEP_MODULES)) -if IAST.PATCH_MODULES in os.environ: - IAST_ALLOWLIST += tuple(os.environ[IAST.PATCH_MODULES].split(IAST.SEP_MODULES)) +ENCODING = "" -if IAST.DENY_MODULES in os.environ: - IAST_DENYLIST += tuple(os.environ[IAST.DENY_MODULES].split(IAST.SEP_MODULES)) +log = get_logger(__name__) -ENCODING = "" +class _TrieNode: + __slots__ = ("children", "is_end") -log = get_logger(__name__) + def __init__(self): + self.children = {} + self.is_end = False + + def __iter__(self): + if self.is_end: + yield ("", None) + else: + for k, v in self.children.items(): + yield (k, dict(v)) + + +def build_trie(words: Iterable[str]) -> _TrieNode: + root = _TrieNode() + for word in words: + node = root + for char in word: + if char not in node.children: + node.children[char] = _TrieNode() + node = node.children[char] + node.is_end = True + return root + + +_TRIE_ALLOWLIST = build_trie(IAST_ALLOWLIST) +_TRIE_DENYLIST = build_trie(IAST_DENYLIST) +_TRIE_USER_ALLOWLIST = build_trie(USER_ALLOWLIST) +_TRIE_USER_DENYLIST = build_trie(USER_DENYLIST) + + +def _trie_has_prefix_for(trie: _TrieNode, string: str) -> bool: + node = trie + for char in string: + node = node.children.get(char) + if not node: + return False + + if node.is_end: + return True + return node.is_end def get_encoding(module_path: Text) -> Text: @@ -341,11 +446,26 @@ def get_encoding(module_path: Text) -> Text: return ENCODING -_NOT_PATCH_MODULE_NAMES = _stdlib_for_python_version() | set(builtin_module_names) +_NOT_PATCH_MODULE_NAMES = {i.lower() for i in _stdlib_for_python_version() | set(builtin_module_names)} + +_IMPORTLIB_PACKAGES: Set[str] = set() def _in_python_stdlib(module_name: str) -> bool: - return module_name.split(".")[0].lower() in [x.lower() for x in _NOT_PATCH_MODULE_NAMES] + return module_name.split(".")[0].lower() in _NOT_PATCH_MODULE_NAMES + + +def _is_first_party(module_name: str): + global _IMPORTLIB_PACKAGES + if "vendor." in module_name or "vendored." in module_name: + return False + + if not _IMPORTLIB_PACKAGES: + from ddtrace.internal.packages import get_package_distributions + + _IMPORTLIB_PACKAGES = set(get_package_distributions()) + + return module_name.split(".")[0] not in _IMPORTLIB_PACKAGES def _should_iast_patch(module_name: Text) -> bool: @@ -358,17 +478,30 @@ def _should_iast_patch(module_name: Text) -> bool: # max_deny = max((len(prefix) for prefix in IAST_DENYLIST if module_name.startswith(prefix)), default=-1) # diff = max_allow - max_deny # return diff > 0 or (diff == 0 and not _in_python_stdlib_or_third_party(module_name)) - dotted_module_name = module_name.lower() + "." - if dotted_module_name.startswith(IAST_ALLOWLIST): - log.debug("IAST: allowing %s. it's in the IAST_ALLOWLIST", module_name) - return True - if dotted_module_name.startswith(IAST_DENYLIST): - log.debug("IAST: denying %s. it's in the IAST_DENYLIST", module_name) - return False if _in_python_stdlib(module_name): log.debug("IAST: denying %s. it's in the _in_python_stdlib", module_name) return False - return True + + if _is_first_party(module_name): + return True + + # else: third party. Check that is in the allow list and not in the deny list + dotted_module_name = module_name.lower() + "." + + # User allow or deny list set by env var have priority + if _trie_has_prefix_for(_TRIE_USER_ALLOWLIST, dotted_module_name): + return True + + if _trie_has_prefix_for(_TRIE_USER_DENYLIST, dotted_module_name): + return False + + if _trie_has_prefix_for(_TRIE_ALLOWLIST, dotted_module_name): + if _trie_has_prefix_for(_TRIE_DENYLIST, dotted_module_name): + return False + log.debug("IAST: allowing %s. it's in the IAST_ALLOWLIST", module_name) + return True + log.debug("IAST: denying %s. it's in the IAST_DENYLIST", module_name) + return False def visit_ast( diff --git a/ddtrace/appsec/_iast/_handlers.py b/ddtrace/appsec/_iast/_handlers.py index 2c681e548e9..e873d8a6a5a 100644 --- a/ddtrace/appsec/_iast/_handlers.py +++ b/ddtrace/appsec/_iast/_handlers.py @@ -171,8 +171,6 @@ def _on_django_func_wrapped(fn_args, fn_kwargs, first_arg_expected_type, *_): http_req = fn_args[0] http_req.COOKIES = taint_structure(http_req.COOKIES, OriginType.COOKIE_NAME, OriginType.COOKIE) - http_req.GET = taint_structure(http_req.GET, OriginType.PARAMETER_NAME, OriginType.PARAMETER) - http_req.POST = taint_structure(http_req.POST, OriginType.BODY, OriginType.BODY) if ( getattr(http_req, "_body", None) is not None and len(getattr(http_req, "_body", None)) > 0 @@ -202,6 +200,8 @@ def _on_django_func_wrapped(fn_args, fn_kwargs, first_arg_expected_type, *_): except AttributeError: log.debug("IAST can't set attribute http_req.body", exc_info=True) + http_req.GET = taint_structure(http_req.GET, OriginType.PARAMETER_NAME, OriginType.PARAMETER) + http_req.POST = taint_structure(http_req.POST, OriginType.PARAMETER_NAME, OriginType.BODY) http_req.headers = taint_structure(http_req.headers, OriginType.HEADER_NAME, OriginType.HEADER) http_req.path = taint_pyobject( http_req.path, source_name="path", source_value=http_req.path, source_origin=OriginType.PATH @@ -367,3 +367,39 @@ def _on_iast_fastapi_patch(): # Instrumented on _iast_starlette_scope_taint _set_metric_iast_instrumented_source(OriginType.PATH_PARAMETER) + + +def _on_pre_tracedrequest_iast(ctx): + current_span = ctx.span + _on_set_request_tags_iast(ctx.get_item("flask_request"), current_span, ctx.get_item("flask_config")) + + +def _on_set_request_tags_iast(request, span, flask_config): + if _is_iast_enabled(): + _set_metric_iast_instrumented_source(OriginType.COOKIE_NAME) + _set_metric_iast_instrumented_source(OriginType.COOKIE) + _set_metric_iast_instrumented_source(OriginType.PARAMETER_NAME) + + if not is_iast_request_enabled(): + return + + request.cookies = taint_structure( + request.cookies, + OriginType.COOKIE_NAME, + OriginType.COOKIE, + override_pyobject_tainted=True, + ) + + request.args = taint_structure( + request.args, + OriginType.PARAMETER_NAME, + OriginType.PARAMETER, + override_pyobject_tainted=True, + ) + + request.form = taint_structure( + request.form, + OriginType.PARAMETER_NAME, + OriginType.PARAMETER, + override_pyobject_tainted=True, + ) diff --git a/ddtrace/appsec/_iast/_listener.py b/ddtrace/appsec/_iast/_listener.py index 356199a3cad..953e22b1288 100644 --- a/ddtrace/appsec/_iast/_listener.py +++ b/ddtrace/appsec/_iast/_listener.py @@ -2,8 +2,10 @@ from ddtrace.appsec._iast._handlers import _on_django_patch from ddtrace.appsec._iast._handlers import _on_flask_patch from ddtrace.appsec._iast._handlers import _on_grpc_response +from ddtrace.appsec._iast._handlers import _on_pre_tracedrequest_iast from ddtrace.appsec._iast._handlers import _on_request_init from ddtrace.appsec._iast._handlers import _on_set_http_meta_iast +from ddtrace.appsec._iast._handlers import _on_set_request_tags_iast from ddtrace.appsec._iast._handlers import _on_wsgi_environ from ddtrace.appsec._iast._iast_request_context import _iast_end_request from ddtrace.internal import core @@ -19,6 +21,8 @@ def iast_listen(): core.on("django.func.wrapped", _on_django_func_wrapped) core.on("flask.patch", _on_flask_patch) core.on("flask.request_init", _on_request_init) + core.on("flask._patched_request", _on_pre_tracedrequest_iast) + core.on("flask.set_request_tags", _on_set_request_tags_iast) core.on("context.ended.wsgi.__call__", _iast_end_request) core.on("context.ended.asgi.__call__", _iast_end_request) diff --git a/ddtrace/appsec/_iast/_metrics.py b/ddtrace/appsec/_iast/_metrics.py index e9e0f604e69..35e2729565e 100644 --- a/ddtrace/appsec/_iast/_metrics.py +++ b/ddtrace/appsec/_iast/_metrics.py @@ -12,7 +12,7 @@ from ddtrace.internal import telemetry from ddtrace.internal.logger import get_logger from ddtrace.internal.telemetry.constants import TELEMETRY_LOG_LEVEL -from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE_TAG_IAST +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE from ddtrace.settings.asm import config as asm_config @@ -73,19 +73,19 @@ def _set_metric_iast_instrumented_source(source_type): from ._taint_tracking import origin_to_str telemetry.telemetry_writer.add_count_metric( - TELEMETRY_NAMESPACE_TAG_IAST, "instrumented.source", 1, (("source_type", origin_to_str(source_type)),) + TELEMETRY_NAMESPACE.IAST, "instrumented.source", 1, (("source_type", origin_to_str(source_type)),) ) @metric_verbosity(TELEMETRY_MANDATORY_VERBOSITY) def _set_metric_iast_instrumented_propagation(): - telemetry.telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE_TAG_IAST, "instrumented.propagation", 1) + telemetry.telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.IAST, "instrumented.propagation", 1) @metric_verbosity(TELEMETRY_MANDATORY_VERBOSITY) def _set_metric_iast_instrumented_sink(vulnerability_type, counter=1): telemetry.telemetry_writer.add_count_metric( - TELEMETRY_NAMESPACE_TAG_IAST, "instrumented.sink", counter, (("vulnerability_type", vulnerability_type),) + TELEMETRY_NAMESPACE.IAST, "instrumented.sink", counter, (("vulnerability_type", vulnerability_type),) ) @@ -94,14 +94,14 @@ def _set_metric_iast_executed_source(source_type): from ._taint_tracking import origin_to_str telemetry.telemetry_writer.add_count_metric( - TELEMETRY_NAMESPACE_TAG_IAST, "executed.source", 1, (("source_type", origin_to_str(source_type)),) + TELEMETRY_NAMESPACE.IAST, "executed.source", 1, (("source_type", origin_to_str(source_type)),) ) @metric_verbosity(TELEMETRY_INFORMATION_VERBOSITY) def _set_metric_iast_executed_sink(vulnerability_type): telemetry.telemetry_writer.add_count_metric( - TELEMETRY_NAMESPACE_TAG_IAST, "executed.sink", 1, (("vulnerability_type", vulnerability_type),) + TELEMETRY_NAMESPACE.IAST, "executed.sink", 1, (("vulnerability_type", vulnerability_type),) ) @@ -115,9 +115,7 @@ def _request_tainted(): def _set_metric_iast_request_tainted(): total_objects_tainted = _request_tainted() if total_objects_tainted > 0: - telemetry.telemetry_writer.add_count_metric( - TELEMETRY_NAMESPACE_TAG_IAST, "request.tainted", total_objects_tainted - ) + telemetry.telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.IAST, "request.tainted", total_objects_tainted) def _set_span_tag_iast_request_tainted(span): diff --git a/ddtrace/appsec/_iast/_overhead_control_engine.py b/ddtrace/appsec/_iast/_overhead_control_engine.py index b1f490b14ef..9c6b1716d0a 100644 --- a/ddtrace/appsec/_iast/_overhead_control_engine.py +++ b/ddtrace/appsec/_iast/_overhead_control_engine.py @@ -8,11 +8,11 @@ from typing import Tuple from typing import Type +from ddtrace._trace.sampler import RateSampler from ddtrace._trace.span import Span from ddtrace.appsec._iast._utils import _is_iast_debug_enabled from ddtrace.internal._unpatched import _threading as threading from ddtrace.internal.logger import get_logger -from ddtrace.sampler import RateSampler from ddtrace.settings.asm import config as asm_config diff --git a/ddtrace/appsec/_iast/_patch_modules.py b/ddtrace/appsec/_iast/_patch_modules.py index d93e7855029..7ae1b0986fb 100644 --- a/ddtrace/appsec/_iast/_patch_modules.py +++ b/ddtrace/appsec/_iast/_patch_modules.py @@ -2,6 +2,7 @@ IAST_PATCH = { + "code_injection": True, "command_injection": True, "header_injection": True, "weak_cipher": True, diff --git a/ddtrace/appsec/_iast/_pytest_plugin.py b/ddtrace/appsec/_iast/_pytest_plugin.py index 672acc4a031..82c23c53174 100644 --- a/ddtrace/appsec/_iast/_pytest_plugin.py +++ b/ddtrace/appsec/_iast/_pytest_plugin.py @@ -27,6 +27,8 @@ def ddtrace_iast(request, ddspan): Optionally output the test as failed if vulnerabilities are found. """ yield + if ddspan is None: + return data = ddspan.get_tag(IAST.JSON) if not data: return diff --git a/ddtrace/appsec/_iast/constants.py b/ddtrace/appsec/_iast/constants.py index 37abb340d71..55ec5a5e740 100644 --- a/ddtrace/appsec/_iast/constants.py +++ b/ddtrace/appsec/_iast/constants.py @@ -12,6 +12,7 @@ VULN_NO_SAMESITE_COOKIE = "NO_SAMESITE_COOKIE" VULN_CMDI = "COMMAND_INJECTION" VULN_HEADER_INJECTION = "HEADER_INJECTION" +VULN_CODE_INJECTION = "CODE_INJECTION" VULN_SSRF = "SSRF" VULNERABILITY_TOKEN_TYPE = Dict[int, Dict[str, Any]] diff --git a/ddtrace/appsec/_iast/taint_sinks/_base.py b/ddtrace/appsec/_iast/taint_sinks/_base.py index 16eaac2452c..543246581cd 100644 --- a/ddtrace/appsec/_iast/taint_sinks/_base.py +++ b/ddtrace/appsec/_iast/taint_sinks/_base.py @@ -7,7 +7,6 @@ from ddtrace import tracer from ddtrace.appsec._trace_utils import _asm_manual_keep from ddtrace.internal.logger import get_logger -from ddtrace.internal.utils.cache import LFUCache from ..._deduplications import deduplication from .._iast_request_context import get_iast_reporter @@ -47,12 +46,6 @@ def _check_positions_contained(needle, container): class VulnerabilityBase(Operation): vulnerability_type = "" - _redacted_report_cache = LFUCache() - - @classmethod - def _reset_cache_for_testing(cls): - """Reset the redacted reports and deduplication cache. For testing purposes only.""" - cls._redacted_report_cache.clear() @classmethod def wrap(cls, func: Callable) -> Callable: diff --git a/ddtrace/appsec/_iast/taint_sinks/code_injection.py b/ddtrace/appsec/_iast/taint_sinks/code_injection.py new file mode 100644 index 00000000000..3c1673de004 --- /dev/null +++ b/ddtrace/appsec/_iast/taint_sinks/code_injection.py @@ -0,0 +1,87 @@ +import inspect +from typing import Text + +from ddtrace.appsec._common_module_patches import try_unwrap +from ddtrace.appsec._constants import IAST_SPAN_TAGS +from ddtrace.appsec._iast import oce +from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled +from ddtrace.appsec._iast._metrics import _set_metric_iast_executed_sink +from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink +from ddtrace.appsec._iast._metrics import increment_iast_span_metric +from ddtrace.appsec._iast._patch import set_and_check_module_is_patched +from ddtrace.appsec._iast._patch import set_module_unpatched +from ddtrace.appsec._iast._patch import try_wrap_function_wrapper +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast.constants import VULN_CODE_INJECTION +from ddtrace.appsec._iast.taint_sinks._base import VulnerabilityBase +from ddtrace.internal.logger import get_logger +from ddtrace.settings.asm import config as asm_config + + +log = get_logger(__name__) + + +def get_version() -> Text: + return "" + + +def patch(): + if not asm_config._iast_enabled: + return + + if not set_and_check_module_is_patched("builtins", default_attr="_datadog_code_injection_patch"): + return + + try_wrap_function_wrapper("builtins", "eval", _iast_coi) + # TODO: wrap exec functions is very dangerous because it needs and modifies locals and globals from the original + # function + # try_wrap_function_wrapper("builtins", "exec", _iast_coi_exec) + + _set_metric_iast_instrumented_sink(VULN_CODE_INJECTION) + + +def unpatch(): + try_unwrap("builtins", "eval") + # try_unwrap("builtins", "exec") + + set_module_unpatched("builtins", default_attr="_datadog_code_injection_patch") + + +def _iast_coi(wrapped, instance, args, kwargs): + if asm_config._iast_enabled and len(args) >= 1: + _iast_report_code_injection(args[0]) + + caller_frame = None + if len(args) > 1: + func_globals = args[1] + elif kwargs.get("globals"): + func_globals = kwargs.get("globals") + else: + frames = inspect.currentframe() + caller_frame = frames.f_back + func_globals = caller_frame.f_globals + + if len(args) > 2: + func_locals = args[2] + elif kwargs.get("locals"): + func_locals = kwargs.get("locals") + else: + if caller_frame is None: + frames = inspect.currentframe() + caller_frame = frames.f_back + func_locals = caller_frame.f_locals + + return wrapped(args[0], func_globals, func_locals) + + +@oce.register +class CodeInjection(VulnerabilityBase): + vulnerability_type = VULN_CODE_INJECTION + + +def _iast_report_code_injection(code_string: Text): + increment_iast_span_metric(IAST_SPAN_TAGS.TELEMETRY_EXECUTED_SINK, CodeInjection.vulnerability_type) + _set_metric_iast_executed_sink(CodeInjection.vulnerability_type) + if is_iast_request_enabled(): + if is_pyobject_tainted(code_string): + CodeInjection.report(evidence_value=code_string) diff --git a/ddtrace/appsec/_iast/taint_sinks/command_injection.py b/ddtrace/appsec/_iast/taint_sinks/command_injection.py index ee22b294bfc..2607c6c9447 100644 --- a/ddtrace/appsec/_iast/taint_sinks/command_injection.py +++ b/ddtrace/appsec/_iast/taint_sinks/command_injection.py @@ -1,18 +1,15 @@ -import os -import subprocess # nosec from typing import List from typing import Union -from ddtrace.appsec._common_module_patches import try_unwrap from ddtrace.appsec._constants import IAST_SPAN_TAGS from ddtrace.appsec._iast import oce from ddtrace.appsec._iast._iast_request_context import is_iast_request_enabled from ddtrace.appsec._iast._metrics import _set_metric_iast_executed_sink from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink from ddtrace.appsec._iast._metrics import increment_iast_span_metric -from ddtrace.appsec._iast._patch import try_wrap_function_wrapper from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted from ddtrace.appsec._iast.constants import VULN_CMDI +import ddtrace.contrib.internal.subprocess.patch as subprocess_patch from ddtrace.internal.logger import get_logger from ddtrace.settings.asm import config as asm_config @@ -26,48 +23,20 @@ def get_version() -> str: return "" -def patch(): - if not asm_config._iast_enabled: - return - - if not getattr(os, "_datadog_cmdi_patch", False): - # all os.spawn* variants eventually use this one: - try_wrap_function_wrapper("os", "_spawnvef", _iast_cmdi_osspawn) - - if not getattr(subprocess, "_datadog_cmdi_patch", False): - try_wrap_function_wrapper("subprocess", "Popen.__init__", _iast_cmdi_subprocess_init) +_IAST_CMDI = "iast_cmdi" - os._datadog_cmdi_patch = True - subprocess._datadog_cmdi_patch = True - _set_metric_iast_instrumented_sink(VULN_CMDI) +def patch(): + if asm_config._iast_enabled: + subprocess_patch.patch() + subprocess_patch.add_str_callback(_IAST_CMDI, _iast_report_cmdi) + subprocess_patch.add_lst_callback(_IAST_CMDI, _iast_report_cmdi) + _set_metric_iast_instrumented_sink(VULN_CMDI) def unpatch() -> None: - try_unwrap("os", "system") - try_unwrap("os", "_spawnvef") - try_unwrap("subprocess", "Popen.__init__") - - os._datadog_cmdi_patch = False # type: ignore[attr-defined] - subprocess._datadog_cmdi_patch = False # type: ignore[attr-defined] - - -def _iast_cmdi_osspawn(wrapped, instance, args, kwargs): - mode, file, func_args, _, _ = args - _iast_report_cmdi(func_args) - - if hasattr(wrapped, "__func__"): - return wrapped.__func__(instance, *args, **kwargs) - return wrapped(*args, **kwargs) - - -def _iast_cmdi_subprocess_init(wrapped, instance, args, kwargs): - cmd_args = args[0] if len(args) else kwargs["args"] - _iast_report_cmdi(cmd_args) - - if hasattr(wrapped, "__func__"): - return wrapped.__func__(instance, *args, **kwargs) - return wrapped(*args, **kwargs) + subprocess_patch.del_str_callback(_IAST_CMDI) + subprocess_patch.del_lst_callback(_IAST_CMDI) @oce.register diff --git a/ddtrace/appsec/_metrics.py b/ddtrace/appsec/_metrics.py index f8713dc5ea7..3d5c7e3e59f 100644 --- a/ddtrace/appsec/_metrics.py +++ b/ddtrace/appsec/_metrics.py @@ -1,10 +1,11 @@ from ddtrace.appsec import _asm_request_context +from ddtrace.appsec import _constants from ddtrace.appsec._ddwaf import version as _version from ddtrace.appsec._deduplications import deduplication from ddtrace.internal import telemetry from ddtrace.internal.logger import get_logger from ddtrace.internal.telemetry.constants import TELEMETRY_LOG_LEVEL -from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE_TAG_APPSEC +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE log = get_logger(__name__) @@ -35,7 +36,7 @@ def _set_waf_updates_metric(info): tags = (("waf_version", DDWAF_VERSION),) telemetry.telemetry_writer.add_count_metric( - TELEMETRY_NAMESPACE_TAG_APPSEC, + TELEMETRY_NAMESPACE.APPSEC, "waf.updates", 1.0, tags=tags, @@ -55,7 +56,7 @@ def _set_waf_init_metric(info): tags = (("waf_version", DDWAF_VERSION),) telemetry.telemetry_writer.add_count_metric( - TELEMETRY_NAMESPACE_TAG_APPSEC, + TELEMETRY_NAMESPACE.APPSEC, "waf.init", 1.0, tags=tags, @@ -64,6 +65,15 @@ def _set_waf_init_metric(info): log.warning("Error reporting ASM WAF init metrics", exc_info=True) +_TYPES_AND_TAGS = { + _constants.EXPLOIT_PREVENTION.TYPE.CMDI: (("rule_type", "command_injection"), ("rule_variant", "exec")), + _constants.EXPLOIT_PREVENTION.TYPE.SHI: (("rule_type", "command_injection"), ("rule_variant", "shell")), + _constants.EXPLOIT_PREVENTION.TYPE.LFI: (("rule_type", "lfi"),), + _constants.EXPLOIT_PREVENTION.TYPE.SSRF: (("rule_type", "ssrf"),), + _constants.EXPLOIT_PREVENTION.TYPE.SQLI: (("rule_type", "sql_injection"),), +} + + def _set_waf_request_metrics(*args): try: result = _asm_request_context.get_waf_telemetry_results() @@ -80,7 +90,7 @@ def _set_waf_request_metrics(*args): ) telemetry.telemetry_writer.add_count_metric( - TELEMETRY_NAMESPACE_TAG_APPSEC, + TELEMETRY_NAMESPACE.APPSEC, "waf.requests", 1.0, tags=tags_request, @@ -91,13 +101,10 @@ def _set_waf_request_metrics(*args): for rule_type, value in rasp[t].items(): if value: telemetry.telemetry_writer.add_count_metric( - TELEMETRY_NAMESPACE_TAG_APPSEC, + TELEMETRY_NAMESPACE.APPSEC, n, float(value), - tags=( - ("rule_type", rule_type), - ("waf_version", DDWAF_VERSION), - ), + tags=_TYPES_AND_TAGS.get(rule_type, ()) + (("waf_version", DDWAF_VERSION),), ) except Exception: diff --git a/ddtrace/appsec/_processor.py b/ddtrace/appsec/_processor.py index 06328d1201a..030399f8a50 100644 --- a/ddtrace/appsec/_processor.py +++ b/ddtrace/appsec/_processor.py @@ -202,6 +202,10 @@ def _update_rules(self, new_rules: Dict[str, Any]) -> bool: def rasp_lfi_enabled(self) -> bool: return WAF_DATA_NAMES.LFI_ADDRESS in self._addresses_to_keep + @property + def rasp_shi_enabled(self) -> bool: + return WAF_DATA_NAMES.SHI_ADDRESS in self._addresses_to_keep + @property def rasp_cmdi_enabled(self) -> bool: return WAF_DATA_NAMES.CMDI_ADDRESS in self._addresses_to_keep @@ -258,6 +262,7 @@ def _waf_action( custom_data: Optional[Dict[str, Any]] = None, crop_trace: Optional[str] = None, rule_type: Optional[str] = None, + force_sent: bool = False, ) -> Optional[DDWaf_result]: """ Call the `WAF` with the given parameters. If `custom_data_names` is specified as @@ -289,7 +294,7 @@ def _waf_action( force_keys = custom_data.get("PROCESSOR_SETTINGS", {}).get("extract-schema", False) if custom_data else False for key, waf_name in iter_data: # type: ignore[attr-defined] - if key in data_already_sent: + if key in data_already_sent and not force_sent: continue # ensure ephemeral addresses are sent, event when value is None if waf_name not in WAF_DATA_NAMES.PERSISTENT_ADDRESSES and custom_data: diff --git a/ddtrace/appsec/_python_info/stdlib/__init__.py b/ddtrace/appsec/_python_info/stdlib/__init__.py index a040e57f859..e745c392f55 100644 --- a/ddtrace/appsec/_python_info/stdlib/__init__.py +++ b/ddtrace/appsec/_python_info/stdlib/__init__.py @@ -19,5 +19,5 @@ from .module_names_py312 import STDLIB_MODULE_NAMES -def _stdlib_for_python_version(): # type: () -> set +def _stdlib_for_python_version(): # type: () -> set[str] return STDLIB_MODULE_NAMES diff --git a/ddtrace/appsec/_trace_utils.py b/ddtrace/appsec/_trace_utils.py index 9f7404d0973..77cb1aaca3a 100644 --- a/ddtrace/appsec/_trace_utils.py +++ b/ddtrace/appsec/_trace_utils.py @@ -9,11 +9,13 @@ from ddtrace.appsec._asm_request_context import in_asm_context from ddtrace.appsec._constants import APPSEC from ddtrace.appsec._constants import LOGIN_EVENTS_MODE +from ddtrace.appsec._constants import WAF_ACTIONS from ddtrace.appsec._utils import _hash_user_id from ddtrace.contrib.trace_utils import set_user from ddtrace.ext import SpanTypes from ddtrace.ext import user from ddtrace.internal import core +from ddtrace.internal._exceptions import BlockingException from ddtrace.internal.logger import get_logger from ddtrace.settings.asm import config as asm_config @@ -71,6 +73,9 @@ def _track_user_login_common( span.set_tag_str("%s.%s" % (tag_metadata_prefix, k), str(v)) if login: + span.set_tag_str(f"{APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC}.{success_str}.usr.login", login) + if login_events_mode != LOGIN_EVENTS_MODE.SDK: + span.set_tag_str(APPSEC.USER_LOGIN_USERNAME, login) span.set_tag_str("%s.login" % tag_prefix, login) if email: @@ -118,19 +123,31 @@ def track_user_login_success_event( real_mode = login_events_mode if login_events_mode != LOGIN_EVENTS_MODE.AUTO else asm_config._user_event_mode if real_mode == LOGIN_EVENTS_MODE.DISABLED: return + initial_login = login + initial_user_id = user_id if real_mode == LOGIN_EVENTS_MODE.ANON: - login = name = email = None + name = email = None + login = None if login is None else _hash_user_id(str(login)) span = _track_user_login_common(tracer, True, metadata, login_events_mode, login, name, email, span) if not span: return - if real_mode == LOGIN_EVENTS_MODE.ANON and isinstance(user_id, str): user_id = _hash_user_id(user_id) - if in_asm_context(): - call_waf_callback(custom_data={"REQUEST_USER_ID": str(user_id), "LOGIN_SUCCESS": real_mode}) - + if login_events_mode != LOGIN_EVENTS_MODE.SDK: + span.set_tag_str(APPSEC.USER_LOGIN_USERID, str(user_id)) set_user(tracer, user_id, name, email, scope, role, session_id, propagate, span) + if in_asm_context(): + res = call_waf_callback( + custom_data={ + "REQUEST_USER_ID": str(initial_user_id) if initial_user_id else None, + "REQUEST_USERNAME": initial_login, + "LOGIN_SUCCESS": real_mode, + }, + force_sent=True, + ) + if res and any(action in [WAF_ACTIONS.BLOCK_ACTION, WAF_ACTIONS.REDIRECT_ACTION] for action in res.actions): + raise BlockingException(get_blocked()) def track_user_login_failure_event( @@ -154,7 +171,9 @@ def track_user_login_failure_event( real_mode = login_events_mode if login_events_mode != LOGIN_EVENTS_MODE.AUTO else asm_config._user_event_mode if real_mode == LOGIN_EVENTS_MODE.DISABLED: return - span = _track_user_login_common(tracer, False, metadata, login_events_mode) + if real_mode == LOGIN_EVENTS_MODE.ANON and isinstance(login, str): + login = _hash_user_id(login) + span = _track_user_login_common(tracer, False, metadata, login_events_mode, login) if not span: return if exists is not None: @@ -163,6 +182,8 @@ def track_user_login_failure_event( if user_id: if real_mode == LOGIN_EVENTS_MODE.ANON and isinstance(user_id, str): user_id = _hash_user_id(user_id) + if login_events_mode != LOGIN_EVENTS_MODE.SDK: + span.set_tag_str(APPSEC.USER_LOGIN_USERID, str(user_id)) span.set_tag_str("%s.failure.%s" % (APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC, user.ID), str(user_id)) # if called from the SDK, set the login, email and name if login_events_mode in (LOGIN_EVENTS_MODE.SDK, LOGIN_EVENTS_MODE.AUTO): @@ -183,7 +204,7 @@ def track_user_signup_event( if span: success_str = "true" if success else "false" span.set_tag_str(APPSEC.USER_SIGNUP_EVENT, success_str) - span.set_tag_str(user.ID, user_id) + span.set_tag_str(user.ID, str(user_id)) _asm_manual_keep(span) # This is used to mark if the call was done from the SDK of the automatic login events @@ -258,7 +279,7 @@ def should_block_user(tracer: Tracer, userid: str) -> bool: if get_blocked(): return True - _asm_request_context.call_waf_callback(custom_data={"REQUEST_USER_ID": str(userid)}) + _asm_request_context.call_waf_callback(custom_data={"REQUEST_USER_ID": str(userid)}, force_sent=True) return bool(get_blocked()) @@ -295,23 +316,16 @@ def block_request_if_user_blocked(tracer: Tracer, userid: str) -> None: _asm_request_context.block_request() -def _on_django_login( - pin, - request, - user, - mode, - info_retriever, - django_config, -): +def _on_django_login(pin, request, user, mode, info_retriever, django_config): if user: - from ddtrace.contrib.django.compat import user_is_authenticated + from ddtrace.contrib.internal.django.compat import user_is_authenticated + user_id, user_extra = info_retriever.get_user_info( + login=django_config.include_user_login, + email=django_config.include_user_email, + name=django_config.include_user_realname, + ) if user_is_authenticated(user): - user_id, user_extra = info_retriever.get_user_info( - login=django_config.include_user_login, - email=django_config.include_user_email, - name=django_config.include_user_realname, - ) with pin.tracer.trace("django.contrib.auth.login", span_type=SpanTypes.AUTH): session_key = getattr(request, "session_key", None) track_user_login_success_event( @@ -324,8 +338,10 @@ def _on_django_login( ) else: # Login failed and the user is unknown (may exist or not) - user_id = info_retriever.get_userid() - track_user_login_failure_event(pin.tracer, user_id=user_id, login_events_mode=mode) + # DEV: DEAD CODE? + track_user_login_failure_event( + pin.tracer, user_id=user_id, login_events_mode=mode, login=user_extra.get("login", None) + ) def _on_django_auth(result_user, mode, kwargs, pin, info_retriever, django_config): @@ -344,17 +360,18 @@ def _on_django_auth(result_user, mode, kwargs, pin, info_retriever, django_confi if not result_user: with pin.tracer.trace("django.contrib.auth.login", span_type=SpanTypes.AUTH): exists = info_retriever.user_exists() - if exists: - user_id, user_extra = info_retriever.get_user_info( - login=django_config.include_user_login, - email=django_config.include_user_email, - name=django_config.include_user_realname, - ) - track_user_login_failure_event( - pin.tracer, user_id=user_id, login_events_mode=mode, exists=True, **user_extra - ) - else: - track_user_login_failure_event(pin.tracer, user_id=user_id, login_events_mode=mode, exists=False) + user_id_found, user_extra = info_retriever.get_user_info( + login=django_config.include_user_login, + email=django_config.include_user_email, + name=django_config.include_user_realname, + ) + if user_extra.get("login") is None: + user_extra["login"] = user_id + user_id = user_id_found or user_id + + track_user_login_failure_event( + pin.tracer, user_id=user_id, login_events_mode=mode, exists=exists, **user_extra + ) return False, None diff --git a/ddtrace/contrib/aiomysql/__init__.py b/ddtrace/contrib/aiomysql/__init__.py index 98f78d3f3ab..06cd9987d81 100644 --- a/ddtrace/contrib/aiomysql/__init__.py +++ b/ddtrace/contrib/aiomysql/__init__.py @@ -19,7 +19,7 @@ To configure the integration on an per-connection basis use the ``Pin`` API:: - from ddtrace import Pin + from ddtrace.trace import Pin import asyncio import aiomysql diff --git a/ddtrace/contrib/aiopg/__init__.py b/ddtrace/contrib/aiopg/__init__.py index 11a572d12ed..c4cd51fdaa2 100644 --- a/ddtrace/contrib/aiopg/__init__.py +++ b/ddtrace/contrib/aiopg/__init__.py @@ -1,7 +1,8 @@ """ Instrument aiopg to report a span for each executed Postgres queries:: - from ddtrace import Pin, patch + from ddtrace import patch + from ddtrace.trace import Pin import aiopg # If not patched yet, you can patch aiopg specifically diff --git a/ddtrace/contrib/aioredis/__init__.py b/ddtrace/contrib/aioredis/__init__.py index 2bc3669a340..b390185d48d 100644 --- a/ddtrace/contrib/aioredis/__init__.py +++ b/ddtrace/contrib/aioredis/__init__.py @@ -64,7 +64,7 @@ ``Pin`` API:: import aioredis - from ddtrace import Pin + from ddtrace.trace import Pin myaioredis = aioredis.Aioredis() Pin.override(myaioredis, service="myaioredis") diff --git a/ddtrace/contrib/anthropic/__init__.py b/ddtrace/contrib/anthropic/__init__.py index c43ee4bb43c..f2d8ea8f353 100644 --- a/ddtrace/contrib/anthropic/__init__.py +++ b/ddtrace/contrib/anthropic/__init__.py @@ -76,7 +76,8 @@ ``Pin`` API:: import anthropic - from ddtrace import Pin, config + from ddtrace import config + from ddtrace.trace import Pin Pin.override(anthropic, service="my-anthropic-service") """ # noqa: E501 diff --git a/ddtrace/contrib/aredis/__init__.py b/ddtrace/contrib/aredis/__init__.py index 1af4c5db664..1d651b9c616 100644 --- a/ddtrace/contrib/aredis/__init__.py +++ b/ddtrace/contrib/aredis/__init__.py @@ -50,10 +50,10 @@ Instance Configuration ~~~~~~~~~~~~~~~~~~~~~~ -To configure particular aredis instances use the :class:`Pin ` API:: +To configure particular aredis instances use the :class:`Pin ` API:: import aredis - from ddtrace import Pin + from ddtrace.trace import Pin client = aredis.StrictRedis(host="localhost", port=6379) diff --git a/ddtrace/contrib/asyncpg/__init__.py b/ddtrace/contrib/asyncpg/__init__.py index c8e56511469..029cfd97790 100644 --- a/ddtrace/contrib/asyncpg/__init__.py +++ b/ddtrace/contrib/asyncpg/__init__.py @@ -38,7 +38,7 @@ basis use the ``Pin`` API:: import asyncpg - from ddtrace import Pin + from ddtrace.trace import Pin conn = asyncpg.connect("postgres://localhost:5432") Pin.override(conn, service="custom-service") diff --git a/ddtrace/contrib/cassandra/__init__.py b/ddtrace/contrib/cassandra/__init__.py index 1d0b6ad0afd..bcce866ad27 100644 --- a/ddtrace/contrib/cassandra/__init__.py +++ b/ddtrace/contrib/cassandra/__init__.py @@ -3,7 +3,8 @@ ``import ddtrace.auto`` will automatically patch your Cluster instance to make it work. :: - from ddtrace import Pin, patch + from ddtrace import patch + from ddtrace.trace import Pin from cassandra.cluster import Cluster # If not patched yet, you can patch cassandra specifically diff --git a/ddtrace/contrib/consul/__init__.py b/ddtrace/contrib/consul/__init__.py index a6317d0bce0..433c70c0e80 100644 --- a/ddtrace/contrib/consul/__init__.py +++ b/ddtrace/contrib/consul/__init__.py @@ -5,7 +5,8 @@ ``import ddtrace.auto`` will automatically patch your Consul client to make it work. :: - from ddtrace import Pin, patch + from ddtrace import patch + from ddtrace.trace import Pin import consul # If not patched yet, you can patch consul specifically diff --git a/ddtrace/contrib/dbapi/__init__.py b/ddtrace/contrib/dbapi/__init__.py index 358b928eadd..0b772ac04ec 100644 --- a/ddtrace/contrib/dbapi/__init__.py +++ b/ddtrace/contrib/dbapi/__init__.py @@ -20,7 +20,7 @@ from ...ext import SpanTypes from ...ext import db from ...ext import sql -from ...pin import Pin +from ...trace import Pin from ..trace_utils import ext_service from ..trace_utils import iswrapped diff --git a/ddtrace/contrib/dbapi_async/__init__.py b/ddtrace/contrib/dbapi_async/__init__.py index 6528d2b348a..d0c43fc1c2b 100644 --- a/ddtrace/contrib/dbapi_async/__init__.py +++ b/ddtrace/contrib/dbapi_async/__init__.py @@ -13,7 +13,7 @@ from ...constants import SPAN_MEASURED_KEY from ...ext import SpanKind from ...ext import SpanTypes -from ...pin import Pin +from ...trace import Pin from ..dbapi import TracedConnection from ..dbapi import TracedCursor from ..trace_utils import ext_service diff --git a/ddtrace/contrib/google_generativeai/__init__.py b/ddtrace/contrib/google_generativeai/__init__.py index d63a1134ab2..5066fc4f9a2 100644 --- a/ddtrace/contrib/google_generativeai/__init__.py +++ b/ddtrace/contrib/google_generativeai/__init__.py @@ -73,7 +73,8 @@ ``Pin`` API:: import google.generativeai as genai - from ddtrace import Pin, config + from ddtrace import config + from ddtrace.trace import Pin Pin.override(genai, service="my-gemini-service") """ # noqa: E501 diff --git a/ddtrace/contrib/graphql/__init__.py b/ddtrace/contrib/graphql/__init__.py index 5394f243533..e7ad66745d4 100644 --- a/ddtrace/contrib/graphql/__init__.py +++ b/ddtrace/contrib/graphql/__init__.py @@ -39,7 +39,7 @@ To configure the graphql integration using the ``Pin`` API:: - from ddtrace import Pin + from ddtrace.trace import Pin import graphql Pin.override(graphql, service="mygraphql") diff --git a/ddtrace/contrib/grpc/__init__.py b/ddtrace/contrib/grpc/__init__.py index ff5adb86aea..8ad2a705233 100644 --- a/ddtrace/contrib/grpc/__init__.py +++ b/ddtrace/contrib/grpc/__init__.py @@ -45,13 +45,14 @@ ``Pin`` API:: import grpc - from ddtrace import Pin, patch, Tracer + from ddtrace import patch + from ddtrace.trace import Pin + patch(grpc=True) - custom_tracer = Tracer() # override the pin on the client - Pin.override(grpc.Channel, service='mygrpc', tracer=custom_tracer) + Pin.override(grpc.Channel, service='mygrpc') with grpc.insecure_channel('localhost:50051') as channel: # create stubs and send requests pass @@ -61,13 +62,13 @@ import grpc from grpc.framework.foundation import logging_pool - from ddtrace import Pin, patch, Tracer + from ddtrace import patch + from ddtrace.trace import Pin patch(grpc=True) - custom_tracer = Tracer() # override the pin on the server - Pin.override(grpc.Server, service='mygrpc', tracer=custom_tracer) + Pin.override(grpc.Server, service='mygrpc') server = grpc.server(logging_pool.pool(2)) server.add_insecure_port('localhost:50051') add_MyServicer_to_server(MyServicer(), server) diff --git a/ddtrace/contrib/httpx/__init__.py b/ddtrace/contrib/httpx/__init__.py index 118d0a738b5..95762604687 100644 --- a/ddtrace/contrib/httpx/__init__.py +++ b/ddtrace/contrib/httpx/__init__.py @@ -57,10 +57,10 @@ Instance Configuration ~~~~~~~~~~~~~~~~~~~~~~ -To configure particular ``httpx`` client instances use the :class:`Pin ` API:: +To configure particular ``httpx`` client instances use the :class:`Pin ` API:: import httpx - from ddtrace import Pin + from ddtrace.trace import Pin client = httpx.Client() # Override service name for this instance diff --git a/ddtrace/contrib/internal/aiobotocore/patch.py b/ddtrace/contrib/internal/aiobotocore/patch.py index c5dafcaaa41..7431bd5c592 100644 --- a/ddtrace/contrib/internal/aiobotocore/patch.py +++ b/ddtrace/contrib/internal/aiobotocore/patch.py @@ -21,7 +21,7 @@ from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.formats import deep_getattr from ddtrace.internal.utils.version import parse_version -from ddtrace.pin import Pin +from ddtrace.trace import Pin aiobotocore_version_str = getattr(aiobotocore, "__version__", "") diff --git a/ddtrace/contrib/internal/aiohttp/patch.py b/ddtrace/contrib/internal/aiohttp/patch.py index 13b55ecb4fb..e0f0bc869e9 100644 --- a/ddtrace/contrib/internal/aiohttp/patch.py +++ b/ddtrace/contrib/internal/aiohttp/patch.py @@ -21,8 +21,8 @@ from ddtrace.internal.schema.span_attribute_schema import SpanDirection from ddtrace.internal.utils import get_argument_value from ddtrace.internal.utils.formats import asbool -from ddtrace.pin import Pin from ddtrace.propagation.http import HTTPPropagator +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/contrib/internal/aiohttp_jinja2/patch.py b/ddtrace/contrib/internal/aiohttp_jinja2/patch.py index 284352b54f0..84553899c39 100644 --- a/ddtrace/contrib/internal/aiohttp_jinja2/patch.py +++ b/ddtrace/contrib/internal/aiohttp_jinja2/patch.py @@ -1,6 +1,5 @@ import aiohttp_jinja2 -from ddtrace import Pin from ddtrace import config from ddtrace.contrib.trace_utils import unwrap from ddtrace.contrib.trace_utils import with_traced_module @@ -8,6 +7,7 @@ from ddtrace.ext import SpanTypes from ddtrace.internal.constants import COMPONENT from ddtrace.internal.utils import get_argument_value +from ddtrace.trace import Pin config._add( diff --git a/ddtrace/contrib/internal/aiomysql/patch.py b/ddtrace/contrib/internal/aiomysql/patch.py index 7f090b4c71d..0053e4f8a5b 100644 --- a/ddtrace/contrib/internal/aiomysql/patch.py +++ b/ddtrace/contrib/internal/aiomysql/patch.py @@ -1,7 +1,6 @@ import aiomysql import wrapt -from ddtrace import Pin from ddtrace import config from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY from ddtrace.constants import SPAN_KIND @@ -18,6 +17,7 @@ from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils.wrappers import unwrap from ddtrace.propagation._database_monitoring import _DBM_Propagator +from ddtrace.trace import Pin config._add( diff --git a/ddtrace/contrib/internal/aiopg/connection.py b/ddtrace/contrib/internal/aiopg/connection.py index b2522ae3888..1daf84b2987 100644 --- a/ddtrace/contrib/internal/aiopg/connection.py +++ b/ddtrace/contrib/internal/aiopg/connection.py @@ -15,7 +15,7 @@ from ddtrace.internal.schema import schematize_database_operation from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils.version import parse_version -from ddtrace.pin import Pin +from ddtrace.trace import Pin AIOPG_VERSION = parse_version(__version__) diff --git a/ddtrace/contrib/internal/aioredis/patch.py b/ddtrace/contrib/internal/aioredis/patch.py index 7915f652641..dc6004b9caa 100644 --- a/ddtrace/contrib/internal/aioredis/patch.py +++ b/ddtrace/contrib/internal/aioredis/patch.py @@ -27,7 +27,7 @@ from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.formats import stringify_cache_args from ddtrace.internal.utils.wrappers import unwrap as _u -from ddtrace.pin import Pin +from ddtrace.trace import Pin from ddtrace.vendor.packaging.version import parse as parse_version diff --git a/ddtrace/contrib/internal/algoliasearch/patch.py b/ddtrace/contrib/internal/algoliasearch/patch.py index 5217861409e..e3074225570 100644 --- a/ddtrace/contrib/internal/algoliasearch/patch.py +++ b/ddtrace/contrib/internal/algoliasearch/patch.py @@ -10,7 +10,7 @@ from ddtrace.internal.schema import schematize_cloud_api_operation from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils.wrappers import unwrap as _u -from ddtrace.pin import Pin +from ddtrace.trace import Pin from ddtrace.vendor.packaging.version import parse as parse_version diff --git a/ddtrace/contrib/internal/anthropic/patch.py b/ddtrace/contrib/internal/anthropic/patch.py index e82c4421e78..24f72f2b511 100644 --- a/ddtrace/contrib/internal/anthropic/patch.py +++ b/ddtrace/contrib/internal/anthropic/patch.py @@ -18,7 +18,7 @@ from ddtrace.internal.utils import get_argument_value from ddtrace.llmobs._integrations import AnthropicIntegration from ddtrace.llmobs._utils import _get_attr -from ddtrace.pin import Pin +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/contrib/internal/aredis/patch.py b/ddtrace/contrib/internal/aredis/patch.py index c9ba000ea36..bd8c5b4c750 100644 --- a/ddtrace/contrib/internal/aredis/patch.py +++ b/ddtrace/contrib/internal/aredis/patch.py @@ -12,7 +12,7 @@ from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.formats import stringify_cache_args from ddtrace.internal.utils.wrappers import unwrap -from ddtrace.pin import Pin +from ddtrace.trace import Pin config._add( diff --git a/ddtrace/contrib/internal/asgi/middleware.py b/ddtrace/contrib/internal/asgi/middleware.py index 98d352cf75f..2b3e23eb78b 100644 --- a/ddtrace/contrib/internal/asgi/middleware.py +++ b/ddtrace/contrib/internal/asgi/middleware.py @@ -150,7 +150,13 @@ async def __call__(self, scope, receive, send): if scope["type"] == "http": operation_name = schematize_url_operation(operation_name, direction=SpanDirection.INBOUND, protocol="http") - pin = ddtrace.pin.Pin(service="asgi", tracer=self.tracer) + # Calling ddtrace.trace.Pin(...) with the `tracer` argument is deprecated + # Remove this if statement when the `tracer` argument is removed + if self.tracer is ddtrace.tracer: + pin = ddtrace.trace.Pin(service="asgi") + else: + pin = ddtrace.trace.Pin(service="asgi", tracer=self.tracer) + with core.context_with_data( "asgi.__call__", remote_addr=scope.get("REMOTE_ADDR"), diff --git a/ddtrace/contrib/internal/asyncio/patch.py b/ddtrace/contrib/internal/asyncio/patch.py index 83f1918e9eb..ed64ca1bf5d 100644 --- a/ddtrace/contrib/internal/asyncio/patch.py +++ b/ddtrace/contrib/internal/asyncio/patch.py @@ -1,10 +1,10 @@ import asyncio -from ddtrace import Pin from ddtrace.internal.utils import get_argument_value from ddtrace.internal.utils import set_argument_value from ddtrace.internal.wrapping import unwrap from ddtrace.internal.wrapping import wrap +from ddtrace.trace import Pin def get_version(): diff --git a/ddtrace/contrib/internal/asyncpg/patch.py b/ddtrace/contrib/internal/asyncpg/patch.py index 7b2b269d5f2..ac1347a7de6 100644 --- a/ddtrace/contrib/internal/asyncpg/patch.py +++ b/ddtrace/contrib/internal/asyncpg/patch.py @@ -2,7 +2,7 @@ from types import ModuleType import asyncpg -from ddtrace import Pin +from ddtrace.trace import Pin from ddtrace import config from ddtrace.internal import core from ddtrace.internal.constants import COMPONENT diff --git a/ddtrace/contrib/internal/avro/patch.py b/ddtrace/contrib/internal/avro/patch.py index 6e66fbe20b0..3ef2adbcb0c 100644 --- a/ddtrace/contrib/internal/avro/patch.py +++ b/ddtrace/contrib/internal/avro/patch.py @@ -3,7 +3,7 @@ from ddtrace import config from ddtrace.internal.utils.wrappers import unwrap -from ddtrace.pin import Pin +from ddtrace.trace import Pin from .schema_iterator import SchemaExtractor diff --git a/ddtrace/contrib/internal/azure_functions/patch.py b/ddtrace/contrib/internal/azure_functions/patch.py index 15089a2e733..1c0c658a9eb 100644 --- a/ddtrace/contrib/internal/azure_functions/patch.py +++ b/ddtrace/contrib/internal/azure_functions/patch.py @@ -8,7 +8,7 @@ from ddtrace.internal import core from ddtrace.internal.schema import schematize_cloud_faas_operation from ddtrace.internal.schema import schematize_service_name -from ddtrace.pin import Pin +from ddtrace.trace import Pin config._add( diff --git a/ddtrace/contrib/internal/boto/patch.py b/ddtrace/contrib/internal/boto/patch.py index 8551056dfb3..e7418aba878 100644 --- a/ddtrace/contrib/internal/boto/patch.py +++ b/ddtrace/contrib/internal/boto/patch.py @@ -19,7 +19,7 @@ from ddtrace.internal.utils import get_argument_value from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.wrappers import unwrap -from ddtrace.pin import Pin +from ddtrace.trace import Pin # Original boto client class diff --git a/ddtrace/contrib/internal/botocore/patch.py b/ddtrace/contrib/internal/botocore/patch.py index febad29f982..07c0bd403e4 100644 --- a/ddtrace/contrib/internal/botocore/patch.py +++ b/ddtrace/contrib/internal/botocore/patch.py @@ -33,8 +33,8 @@ from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.formats import deep_getattr from ddtrace.llmobs._integrations import BedrockIntegration -from ddtrace.pin import Pin from ddtrace.settings.config import Config +from ddtrace.trace import Pin from .services.bedrock import patched_bedrock_api_call from .services.kinesis import patched_kinesis_api_call diff --git a/ddtrace/contrib/internal/botocore/services/bedrock.py b/ddtrace/contrib/internal/botocore/services/bedrock.py index 7c5f26b07a5..00e9aa5756f 100644 --- a/ddtrace/contrib/internal/botocore/services/bedrock.py +++ b/ddtrace/contrib/internal/botocore/services/bedrock.py @@ -24,6 +24,17 @@ _META = "meta" _STABILITY = "stability" +_MODEL_TYPE_IDENTIFIERS = ( + "foundation-model/", + "custom-model/", + "provisioned-model/", + "imported-model/", + "prompt/", + "endpoint/", + "inference-profile/", + "default-prompt-router/", +) + class TracedBotocoreStreamingBody(wrapt.ObjectProxy): """ @@ -320,14 +331,45 @@ def handle_bedrock_response( return result +def _parse_model_id(model_id: str): + """Best effort to extract the model provider and model name from the bedrock model ID. + model_id can be a 1/2 period-separated string or a full AWS ARN, based on the following formats: + 1. Base model: "{model_provider}.{model_name}" + 2. Cross-region model: "{region}.{model_provider}.{model_name}" + 3. Other: Prefixed by AWS ARN "arn:aws{+region?}:bedrock:{region}:{account-id}:" + a. Foundation model: ARN prefix + "foundation-model/{region?}.{model_provider}.{model_name}" + b. Custom model: ARN prefix + "custom-model/{model_provider}.{model_name}" + c. Provisioned model: ARN prefix + "provisioned-model/{model-id}" + d. Imported model: ARN prefix + "imported-module/{model-id}" + e. Prompt management: ARN prefix + "prompt/{prompt-id}" + f. Sagemaker: ARN prefix + "endpoint/{model-id}" + g. Inference profile: ARN prefix + "{application-?}inference-profile/{model-id}" + h. Default prompt router: ARN prefix + "default-prompt-router/{prompt-id}" + If model provider cannot be inferred from the model_id formatting, then default to "custom" + """ + if not model_id.startswith("arn:aws"): + model_meta = model_id.split(".") + if len(model_meta) < 2: + return "custom", model_meta[0] + return model_meta[-2], model_meta[-1] + for identifier in _MODEL_TYPE_IDENTIFIERS: + if identifier not in model_id: + continue + model_id = model_id.rsplit(identifier, 1)[-1] + if identifier in ("foundation-model/", "custom-model/"): + model_meta = model_id.split(".") + if len(model_meta) < 2: + return "custom", model_id + return model_meta[-2], model_meta[-1] + return "custom", model_id + return "custom", "custom" + + def patched_bedrock_api_call(original_func, instance, args, kwargs, function_vars): params = function_vars.get("params") pin = function_vars.get("pin") - model_meta = params.get("modelId").split(".") - if len(model_meta) == 2: - model_provider, model_name = model_meta - else: - _, model_provider, model_name = model_meta # cross-region inference + model_id = params.get("modelId") + model_provider, model_name = _parse_model_id(model_id) integration = function_vars.get("integration") submit_to_llmobs = integration.llmobs_enabled and "embed" not in model_name with core.context_with_data( diff --git a/ddtrace/contrib/internal/cassandra/session.py b/ddtrace/contrib/internal/cassandra/session.py index 33d307c13c7..7f02d8c0af6 100644 --- a/ddtrace/contrib/internal/cassandra/session.py +++ b/ddtrace/contrib/internal/cassandra/session.py @@ -39,7 +39,7 @@ from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils import get_argument_value from ddtrace.internal.utils.formats import deep_getattr -from ddtrace.pin import Pin +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/contrib/internal/celery/app.py b/ddtrace/contrib/internal/celery/app.py index 42eed2cb468..54ad5834769 100644 --- a/ddtrace/contrib/internal/celery/app.py +++ b/ddtrace/contrib/internal/celery/app.py @@ -3,8 +3,8 @@ import celery from celery import signals -from ddtrace import Pin from ddtrace import config +from ddtrace._trace.pin import _DD_PIN_NAME from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY from ddtrace.constants import SPAN_KIND from ddtrace.constants import SPAN_MEASURED_KEY @@ -19,7 +19,7 @@ from ddtrace.ext import SpanTypes from ddtrace.internal import core from ddtrace.internal.logger import get_logger -from ddtrace.pin import _DD_PIN_NAME +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/contrib/internal/celery/signals.py b/ddtrace/contrib/internal/celery/signals.py index 8f27fcc53b0..ea9d8c15863 100644 --- a/ddtrace/contrib/internal/celery/signals.py +++ b/ddtrace/contrib/internal/celery/signals.py @@ -3,7 +3,6 @@ from celery import current_app from celery import registry -from ddtrace import Pin from ddtrace import config from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY from ddtrace.constants import SPAN_KIND @@ -24,6 +23,7 @@ from ddtrace.internal.constants import COMPONENT from ddtrace.internal.logger import get_logger from ddtrace.propagation.http import HTTPPropagator +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/contrib/internal/consul/patch.py b/ddtrace/contrib/internal/consul/patch.py index b4725e807ba..b24b138b632 100644 --- a/ddtrace/contrib/internal/consul/patch.py +++ b/ddtrace/contrib/internal/consul/patch.py @@ -15,7 +15,7 @@ from ddtrace.internal.schema.span_attribute_schema import SpanDirection from ddtrace.internal.utils import get_argument_value from ddtrace.internal.utils.wrappers import unwrap as _u -from ddtrace.pin import Pin +from ddtrace.trace import Pin _KV_FUNCS = ["put", "get", "delete"] diff --git a/ddtrace/contrib/internal/django/patch.py b/ddtrace/contrib/internal/django/patch.py index d4b14487e39..8bc523dd1c1 100644 --- a/ddtrace/contrib/internal/django/patch.py +++ b/ddtrace/contrib/internal/django/patch.py @@ -17,7 +17,7 @@ import wrapt from wrapt.importer import when_imported -from ddtrace import Pin +import ddtrace from ddtrace import config from ddtrace.appsec._utils import _UserInfoRetriever from ddtrace.constants import SPAN_KIND @@ -49,6 +49,7 @@ from ddtrace.propagation._database_monitoring import _DBM_Propagator from ddtrace.settings.asm import config as asm_config from ddtrace.settings.integration import IntegrationConfig +from ddtrace.trace import Pin from ddtrace.vendor.packaging.version import parse as parse_version @@ -147,7 +148,12 @@ def cursor(django, pin, func, instance, args, kwargs): tags = {"django.db.vendor": vendor, "django.db.alias": alias} tags.update(getattr(conn, "_datadog_tags", {})) - pin = Pin(service, tags=tags, tracer=pin.tracer) + # Calling ddtrace.pin.Pin(...) with the `tracer` argument generates a deprecation warning. + # Remove this if statement when the `tracer` argument is removed + if pin.tracer is ddtrace.tracer: + pin = Pin(service, tags=tags) + else: + pin = Pin(service, tags=tags, tracer=pin.tracer) cursor = func(*args, **kwargs) diff --git a/ddtrace/contrib/internal/dogpile_cache/lock.py b/ddtrace/contrib/internal/dogpile_cache/lock.py index c592562f94f..76cdc2eb839 100644 --- a/ddtrace/contrib/internal/dogpile_cache/lock.py +++ b/ddtrace/contrib/internal/dogpile_cache/lock.py @@ -1,7 +1,7 @@ import dogpile from ddtrace.internal.utils.formats import asbool -from ddtrace.pin import Pin +from ddtrace.trace import Pin def _wrap_lock_ctor(func, instance, args, kwargs): diff --git a/ddtrace/contrib/internal/dogpile_cache/patch.py b/ddtrace/contrib/internal/dogpile_cache/patch.py index f4f41284a29..f78ea5cb23f 100644 --- a/ddtrace/contrib/internal/dogpile_cache/patch.py +++ b/ddtrace/contrib/internal/dogpile_cache/patch.py @@ -7,10 +7,10 @@ from wrapt import wrap_function_wrapper as _w +from ddtrace._trace.pin import _DD_PIN_NAME +from ddtrace._trace.pin import _DD_PIN_PROXY_NAME from ddtrace.internal.schema import schematize_service_name -from ddtrace.pin import _DD_PIN_NAME -from ddtrace.pin import _DD_PIN_PROXY_NAME -from ddtrace.pin import Pin +from ddtrace.trace import Pin from .lock import _wrap_lock_ctor from .region import _wrap_get_create diff --git a/ddtrace/contrib/internal/dogpile_cache/region.py b/ddtrace/contrib/internal/dogpile_cache/region.py index 04b70402e3d..0c89d2d84d9 100644 --- a/ddtrace/contrib/internal/dogpile_cache/region.py +++ b/ddtrace/contrib/internal/dogpile_cache/region.py @@ -7,7 +7,7 @@ from ddtrace.internal.schema import schematize_cache_operation from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils import get_argument_value -from ddtrace.pin import Pin +from ddtrace.trace import Pin def _wrap_get_create(func, instance, args, kwargs): diff --git a/ddtrace/contrib/internal/elasticsearch/patch.py b/ddtrace/contrib/internal/elasticsearch/patch.py index 455d0678d02..7c408db55a5 100644 --- a/ddtrace/contrib/internal/elasticsearch/patch.py +++ b/ddtrace/contrib/internal/elasticsearch/patch.py @@ -21,7 +21,7 @@ from ddtrace.internal.logger import get_logger from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils.wrappers import unwrap as _u -from ddtrace.pin import Pin +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/contrib/internal/fastapi/patch.py b/ddtrace/contrib/internal/fastapi/patch.py index b431f3c83f8..485c0424a5f 100644 --- a/ddtrace/contrib/internal/fastapi/patch.py +++ b/ddtrace/contrib/internal/fastapi/patch.py @@ -5,7 +5,6 @@ from wrapt import ObjectProxy from wrapt import wrap_function_wrapper as _w -from ddtrace import Pin from ddtrace import config from ddtrace.appsec._iast._utils import _is_iast_enabled from ddtrace.contrib.internal.asgi.middleware import TraceMiddleware @@ -15,6 +14,7 @@ from ddtrace.internal.logger import get_logger from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils.wrappers import unwrap as _u +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/contrib/internal/flask/patch.py b/ddtrace/contrib/internal/flask/patch.py index 429a9d05667..010df5218c5 100644 --- a/ddtrace/contrib/internal/flask/patch.py +++ b/ddtrace/contrib/internal/flask/patch.py @@ -29,7 +29,6 @@ from wrapt import wrap_function_wrapper as _w -from ddtrace import Pin from ddtrace import config from ddtrace.contrib.internal.wsgi.wsgi import _DDWSGIMiddlewareBase from ddtrace.contrib.trace_utils import unwrap as _u @@ -37,6 +36,7 @@ from ddtrace.internal.utils import get_argument_value from ddtrace.internal.utils.importlib import func_name from ddtrace.internal.utils.version import parse_version +from ddtrace.trace import Pin from .wrappers import _wrap_call_with_pin_check from .wrappers import get_current_app diff --git a/ddtrace/contrib/internal/flask/wrappers.py b/ddtrace/contrib/internal/flask/wrappers.py index 3aca2a1466a..d65697224ba 100644 --- a/ddtrace/contrib/internal/flask/wrappers.py +++ b/ddtrace/contrib/internal/flask/wrappers.py @@ -7,7 +7,7 @@ from ddtrace.internal.constants import COMPONENT from ddtrace.internal.logger import get_logger from ddtrace.internal.utils.importlib import func_name -from ddtrace.pin import Pin +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/contrib/internal/google_generativeai/patch.py b/ddtrace/contrib/internal/google_generativeai/patch.py index 29bc18dc756..3564f9ec1ec 100644 --- a/ddtrace/contrib/internal/google_generativeai/patch.py +++ b/ddtrace/contrib/internal/google_generativeai/patch.py @@ -14,7 +14,7 @@ from ddtrace.contrib.trace_utils import wrap from ddtrace.llmobs._integrations import GeminiIntegration from ddtrace.llmobs._integrations.utils import extract_model_name_google -from ddtrace.pin import Pin +from ddtrace.trace import Pin config._add( diff --git a/ddtrace/contrib/internal/graphql/patch.py b/ddtrace/contrib/internal/graphql/patch.py index b54df97e520..18916f4222a 100644 --- a/ddtrace/contrib/internal/graphql/patch.py +++ b/ddtrace/contrib/internal/graphql/patch.py @@ -39,7 +39,7 @@ from ddtrace.internal.utils.version import parse_version from ddtrace.internal.wrapping import unwrap from ddtrace.internal.wrapping import wrap -from ddtrace.pin import Pin +from ddtrace.trace import Pin _graphql_version_str = graphql.__version__ diff --git a/ddtrace/contrib/internal/grpc/aio_client_interceptor.py b/ddtrace/contrib/internal/grpc/aio_client_interceptor.py index bf6f156de7e..5c03d1b8527 100644 --- a/ddtrace/contrib/internal/grpc/aio_client_interceptor.py +++ b/ddtrace/contrib/internal/grpc/aio_client_interceptor.py @@ -11,7 +11,6 @@ from grpc.aio._typing import ResponseIterableType from grpc.aio._typing import ResponseType -from ddtrace import Pin from ddtrace import Span from ddtrace import config from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY @@ -30,6 +29,7 @@ from ddtrace.internal.schema import schematize_url_operation from ddtrace.internal.schema.span_attribute_schema import SpanDirection from ddtrace.propagation.http import HTTPPropagator +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/contrib/internal/grpc/aio_server_interceptor.py b/ddtrace/contrib/internal/grpc/aio_server_interceptor.py index 2361e3c3be9..d5ec9ed32ab 100644 --- a/ddtrace/contrib/internal/grpc/aio_server_interceptor.py +++ b/ddtrace/contrib/internal/grpc/aio_server_interceptor.py @@ -13,7 +13,6 @@ from grpc.aio._typing import ResponseType import wrapt -from ddtrace import Pin # noqa:F401 from ddtrace import Span # noqa:F401 from ddtrace import config from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY @@ -30,6 +29,7 @@ from ddtrace.internal.constants import COMPONENT from ddtrace.internal.schema import schematize_url_operation from ddtrace.internal.schema.span_attribute_schema import SpanDirection +from ddtrace.trace import Pin # noqa:F401 Continuation = Callable[[grpc.HandlerCallDetails], Awaitable[grpc.RpcMethodHandler]] diff --git a/ddtrace/contrib/internal/grpc/patch.py b/ddtrace/contrib/internal/grpc/patch.py index 9c41c5cc342..122893b030f 100644 --- a/ddtrace/contrib/internal/grpc/patch.py +++ b/ddtrace/contrib/internal/grpc/patch.py @@ -1,7 +1,6 @@ import grpc from wrapt import wrap_function_wrapper as _w -from ddtrace import Pin from ddtrace import config from ddtrace.contrib.internal.grpc import constants from ddtrace.contrib.internal.grpc import utils @@ -13,6 +12,7 @@ from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils import get_argument_value from ddtrace.internal.utils import set_argument_value +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/contrib/internal/httplib/patch.py b/ddtrace/contrib/internal/httplib/patch.py index e42241a2ca2..3e354aeedea 100644 --- a/ddtrace/contrib/internal/httplib/patch.py +++ b/ddtrace/contrib/internal/httplib/patch.py @@ -20,9 +20,9 @@ from ddtrace.internal.schema import schematize_url_operation from ddtrace.internal.schema.span_attribute_schema import SpanDirection from ddtrace.internal.utils.formats import asbool -from ddtrace.pin import Pin from ddtrace.propagation.http import HTTPPropagator from ddtrace.settings.asm import config as asm_config +from ddtrace.trace import Pin span_name = "http.client.request" diff --git a/ddtrace/contrib/internal/httpx/patch.py b/ddtrace/contrib/internal/httpx/patch.py index e6d1893880f..8a9e4eebc3a 100644 --- a/ddtrace/contrib/internal/httpx/patch.py +++ b/ddtrace/contrib/internal/httpx/patch.py @@ -22,8 +22,8 @@ from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.version import parse_version from ddtrace.internal.utils.wrappers import unwrap as _u -from ddtrace.pin import Pin from ddtrace.propagation.http import HTTPPropagator +from ddtrace.trace import Pin HTTPX_VERSION = parse_version(httpx.__version__) diff --git a/ddtrace/contrib/internal/jinja2/patch.py b/ddtrace/contrib/internal/jinja2/patch.py index 83aad083747..cdf1254527d 100644 --- a/ddtrace/contrib/internal/jinja2/patch.py +++ b/ddtrace/contrib/internal/jinja2/patch.py @@ -10,7 +10,7 @@ from ddtrace.internal.constants import COMPONENT from ddtrace.internal.utils import ArgumentError from ddtrace.internal.utils import get_argument_value -from ddtrace.pin import Pin +from ddtrace.trace import Pin from .constants import DEFAULT_TEMPLATE_NAME diff --git a/ddtrace/contrib/internal/kafka/patch.py b/ddtrace/contrib/internal/kafka/patch.py index 339e2469914..6f69cda3239 100644 --- a/ddtrace/contrib/internal/kafka/patch.py +++ b/ddtrace/contrib/internal/kafka/patch.py @@ -24,8 +24,8 @@ from ddtrace.internal.utils import set_argument_value from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.version import parse_version -from ddtrace.pin import Pin from ddtrace.propagation.http import HTTPPropagator as Propagator +from ddtrace.trace import Pin _Producer = confluent_kafka.Producer diff --git a/ddtrace/contrib/internal/kombu/patch.py b/ddtrace/contrib/internal/kombu/patch.py index fd571fd445c..fa63e5c4f86 100644 --- a/ddtrace/contrib/internal/kombu/patch.py +++ b/ddtrace/contrib/internal/kombu/patch.py @@ -22,8 +22,8 @@ from ddtrace.internal.utils import get_argument_value from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.wrappers import unwrap -from ddtrace.pin import Pin from ddtrace.propagation.http import HTTPPropagator +from ddtrace.trace import Pin from .constants import DEFAULT_SERVICE from .utils import HEADER_POS diff --git a/ddtrace/contrib/internal/langchain/patch.py b/ddtrace/contrib/internal/langchain/patch.py index b7513539da7..f9c58249cb2 100644 --- a/ddtrace/contrib/internal/langchain/patch.py +++ b/ddtrace/contrib/internal/langchain/patch.py @@ -62,7 +62,7 @@ from ddtrace.internal.utils.version import parse_version from ddtrace.llmobs._integrations import LangChainIntegration from ddtrace.llmobs._utils import safe_json -from ddtrace.pin import Pin +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/contrib/internal/mako/patch.py b/ddtrace/contrib/internal/mako/patch.py index 7db3b2e47df..d39a51238a2 100644 --- a/ddtrace/contrib/internal/mako/patch.py +++ b/ddtrace/contrib/internal/mako/patch.py @@ -11,7 +11,7 @@ from ddtrace.internal.constants import COMPONENT from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils.importlib import func_name -from ddtrace.pin import Pin +from ddtrace.trace import Pin from .constants import DEFAULT_TEMPLATE_NAME diff --git a/ddtrace/contrib/internal/mariadb/patch.py b/ddtrace/contrib/internal/mariadb/patch.py index b4ab267c5e3..1307403f6d4 100644 --- a/ddtrace/contrib/internal/mariadb/patch.py +++ b/ddtrace/contrib/internal/mariadb/patch.py @@ -3,7 +3,6 @@ import mariadb import wrapt -from ddtrace import Pin from ddtrace import config from ddtrace.contrib.dbapi import TracedConnection from ddtrace.ext import db @@ -11,6 +10,7 @@ from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.wrappers import unwrap +from ddtrace.trace import Pin config._add( diff --git a/ddtrace/contrib/internal/molten/patch.py b/ddtrace/contrib/internal/molten/patch.py index fd6fa53b195..7c60d37d0d6 100644 --- a/ddtrace/contrib/internal/molten/patch.py +++ b/ddtrace/contrib/internal/molten/patch.py @@ -4,7 +4,6 @@ import wrapt from wrapt import wrap_function_wrapper as _w -from ddtrace import Pin from ddtrace import config from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY from ddtrace.constants import SPAN_KIND @@ -21,6 +20,7 @@ from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.importlib import func_name from ddtrace.internal.utils.version import parse_version +from ddtrace.trace import Pin from .wrappers import MOLTEN_ROUTE from .wrappers import WrapperComponent diff --git a/ddtrace/contrib/internal/molten/wrappers.py b/ddtrace/contrib/internal/molten/wrappers.py index 7446224fe45..0a3e325ca0b 100644 --- a/ddtrace/contrib/internal/molten/wrappers.py +++ b/ddtrace/contrib/internal/molten/wrappers.py @@ -1,7 +1,6 @@ import molten import wrapt -from ddtrace import Pin from ddtrace import config from ddtrace.constants import SPAN_KIND from ddtrace.contrib import trace_utils @@ -9,6 +8,7 @@ from ddtrace.ext import http from ddtrace.internal.constants import COMPONENT from ddtrace.internal.utils.importlib import func_name +from ddtrace.trace import Pin MOLTEN_ROUTE = "molten.route" diff --git a/ddtrace/contrib/internal/mongoengine/trace.py b/ddtrace/contrib/internal/mongoengine/trace.py index c5f3e834aed..93868e096ce 100644 --- a/ddtrace/contrib/internal/mongoengine/trace.py +++ b/ddtrace/contrib/internal/mongoengine/trace.py @@ -23,12 +23,17 @@ class WrappedConnect(wrapt.ObjectProxy): def __init__(self, connect): super(WrappedConnect, self).__init__(connect) - ddtrace.Pin(_SERVICE, tracer=ddtrace.tracer).onto(self) + ddtrace.trace.Pin(_SERVICE).onto(self) def __call__(self, *args, **kwargs): client = self.__wrapped__(*args, **kwargs) - pin = ddtrace.Pin.get_from(self) + pin = ddtrace.trace.Pin.get_from(self) if pin: - ddtrace.Pin(service=pin.service, tracer=pin.tracer).onto(client) + # Calling ddtrace.pin.Pin(...) with the `tracer` argument generates a deprecation warning. + # Remove this if statement when the `tracer` argument is removed + if pin.tracer is ddtrace.tracer: + ddtrace.trace.Pin(service=pin.service).onto(client) + else: + ddtrace.trace.Pin(service=pin.service, tracer=pin.tracer).onto(client) return client diff --git a/ddtrace/contrib/internal/mysql/patch.py b/ddtrace/contrib/internal/mysql/patch.py index 2d5a8500cb3..d18d357d107 100644 --- a/ddtrace/contrib/internal/mysql/patch.py +++ b/ddtrace/contrib/internal/mysql/patch.py @@ -3,7 +3,6 @@ import mysql.connector import wrapt -from ddtrace import Pin from ddtrace import config from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION @@ -16,6 +15,7 @@ from ddtrace.internal.utils.formats import asbool from ddtrace.propagation._database_monitoring import _DBM_Propagator from ddtrace.settings.asm import config as asm_config +from ddtrace.trace import Pin config._add( diff --git a/ddtrace/contrib/internal/mysqldb/patch.py b/ddtrace/contrib/internal/mysqldb/patch.py index 291d6cb865e..8b6aa7bb7f2 100644 --- a/ddtrace/contrib/internal/mysqldb/patch.py +++ b/ddtrace/contrib/internal/mysqldb/patch.py @@ -3,7 +3,6 @@ import MySQLdb from wrapt import wrap_function_wrapper as _w -from ddtrace import Pin from ddtrace import config from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_sink from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION @@ -23,6 +22,7 @@ from ddtrace.internal.utils.wrappers import unwrap as _u from ddtrace.propagation._database_monitoring import _DBM_Propagator from ddtrace.settings.asm import config as asm_config +from ddtrace.trace import Pin config._add( diff --git a/ddtrace/contrib/internal/openai/_endpoint_hooks.py b/ddtrace/contrib/internal/openai/_endpoint_hooks.py index 73a2b2511c9..979e1774a8a 100644 --- a/ddtrace/contrib/internal/openai/_endpoint_hooks.py +++ b/ddtrace/contrib/internal/openai/_endpoint_hooks.py @@ -255,6 +255,14 @@ def _record_request(self, pin, integration, span, args, kwargs): span.set_tag_str("openai.request.messages.%d.content" % idx, integration.trunc(str(content))) span.set_tag_str("openai.request.messages.%d.role" % idx, str(role)) span.set_tag_str("openai.request.messages.%d.name" % idx, str(name)) + if parse_version(OPENAI_VERSION) >= (1, 26) and kwargs.get("stream"): + if kwargs.get("stream_options", {}).get("include_usage", None) is not None: + # Only perform token chunk auto-extraction if this option is not explicitly set + return + span._set_ctx_item("_dd.auto_extract_token_chunk", True) + stream_options = kwargs.get("stream_options", {}) + stream_options["include_usage"] = True + kwargs["stream_options"] = stream_options def _record_response(self, pin, integration, span, args, kwargs, resp, error): resp = super()._record_response(pin, integration, span, args, kwargs, resp, error) diff --git a/ddtrace/contrib/internal/openai/patch.py b/ddtrace/contrib/internal/openai/patch.py index 4ad76a17084..d87b06b3aba 100644 --- a/ddtrace/contrib/internal/openai/patch.py +++ b/ddtrace/contrib/internal/openai/patch.py @@ -13,7 +13,7 @@ from ddtrace.internal.utils.version import parse_version from ddtrace.internal.wrapping import wrap from ddtrace.llmobs._integrations import OpenAIIntegration -from ddtrace.pin import Pin +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/contrib/internal/openai/utils.py b/ddtrace/contrib/internal/openai/utils.py index d967383e366..f5dfc10efef 100644 --- a/ddtrace/contrib/internal/openai/utils.py +++ b/ddtrace/contrib/internal/openai/utils.py @@ -48,11 +48,28 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.__wrapped__.__exit__(exc_type, exc_val, exc_tb) def __iter__(self): - return self + exception_raised = False + try: + for chunk in self.__wrapped__: + self._extract_token_chunk(chunk) + yield chunk + _loop_handler(self._dd_span, chunk, self._streamed_chunks) + except Exception: + self._dd_span.set_exc_info(*sys.exc_info()) + exception_raised = True + raise + finally: + if not exception_raised: + _process_finished_stream( + self._dd_integration, self._dd_span, self._kwargs, self._streamed_chunks, self._is_completion + ) + self._dd_span.finish() + self._dd_integration.metric(self._dd_span, "dist", "request.duration", self._dd_span.duration_ns) def __next__(self): try: chunk = self.__wrapped__.__next__() + self._extract_token_chunk(chunk) _loop_handler(self._dd_span, chunk, self._streamed_chunks) return chunk except StopIteration: @@ -68,6 +85,22 @@ def __next__(self): self._dd_integration.metric(self._dd_span, "dist", "request.duration", self._dd_span.duration_ns) raise + def _extract_token_chunk(self, chunk): + """Attempt to extract the token chunk (last chunk in the stream) from the streamed response.""" + if not self._dd_span._get_ctx_item("_dd.auto_extract_token_chunk"): + return + choice = getattr(chunk, "choices", [None])[0] + if not getattr(choice, "finish_reason", None): + # Only the second-last chunk in the stream with token usage enabled will have finish_reason set + return + try: + # User isn't expecting last token chunk to be present since it's not part of the default streamed response, + # so we consume it and extract the token usage metadata before it reaches the user. + usage_chunk = self.__wrapped__.__next__() + self._streamed_chunks[0].insert(0, usage_chunk) + except (StopIteration, GeneratorExit): + return + class TracedOpenAIAsyncStream(BaseTracedOpenAIStream): async def __aenter__(self): @@ -77,12 +110,29 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc_val, exc_tb): await self.__wrapped__.__aexit__(exc_type, exc_val, exc_tb) - def __aiter__(self): - return self + async def __aiter__(self): + exception_raised = False + try: + async for chunk in self.__wrapped__: + await self._extract_token_chunk(chunk) + yield chunk + _loop_handler(self._dd_span, chunk, self._streamed_chunks) + except Exception: + self._dd_span.set_exc_info(*sys.exc_info()) + exception_raised = True + raise + finally: + if not exception_raised: + _process_finished_stream( + self._dd_integration, self._dd_span, self._kwargs, self._streamed_chunks, self._is_completion + ) + self._dd_span.finish() + self._dd_integration.metric(self._dd_span, "dist", "request.duration", self._dd_span.duration_ns) async def __anext__(self): try: chunk = await self.__wrapped__.__anext__() + await self._extract_token_chunk(chunk) _loop_handler(self._dd_span, chunk, self._streamed_chunks) return chunk except StopAsyncIteration: @@ -98,6 +148,19 @@ async def __anext__(self): self._dd_integration.metric(self._dd_span, "dist", "request.duration", self._dd_span.duration_ns) raise + async def _extract_token_chunk(self, chunk): + """Attempt to extract the token chunk (last chunk in the stream) from the streamed response.""" + if not self._dd_span._get_ctx_item("_dd.auto_extract_token_chunk"): + return + choice = getattr(chunk, "choices", [None])[0] + if not getattr(choice, "finish_reason", None): + return + try: + usage_chunk = await self.__wrapped__.__anext__() + self._streamed_chunks[0].insert(0, usage_chunk) + except (StopAsyncIteration, GeneratorExit): + return + def _compute_token_count(content, model): # type: (Union[str, List[int]], Optional[str]) -> Tuple[bool, int] diff --git a/ddtrace/contrib/internal/protobuf/patch.py b/ddtrace/contrib/internal/protobuf/patch.py index 607c29eb1c0..8ecdd7aefa5 100644 --- a/ddtrace/contrib/internal/protobuf/patch.py +++ b/ddtrace/contrib/internal/protobuf/patch.py @@ -4,7 +4,7 @@ from ddtrace import config from ddtrace.internal.utils.wrappers import unwrap -from ddtrace.pin import Pin +from ddtrace.trace import Pin from .schema_iterator import SchemaExtractor diff --git a/ddtrace/contrib/internal/psycopg/async_connection.py b/ddtrace/contrib/internal/psycopg/async_connection.py index 14ec854ffd1..72c8d70e7ec 100644 --- a/ddtrace/contrib/internal/psycopg/async_connection.py +++ b/ddtrace/contrib/internal/psycopg/async_connection.py @@ -1,4 +1,3 @@ -from ddtrace import Pin from ddtrace import config from ddtrace.constants import SPAN_KIND from ddtrace.constants import SPAN_MEASURED_KEY @@ -11,6 +10,7 @@ from ddtrace.ext import SpanTypes from ddtrace.ext import db from ddtrace.internal.constants import COMPONENT +from ddtrace.trace import Pin class Psycopg3TracedAsyncConnection(dbapi_async.TracedAsyncConnection): diff --git a/ddtrace/contrib/internal/psycopg/connection.py b/ddtrace/contrib/internal/psycopg/connection.py index c823e17dc61..a5e5353ad13 100644 --- a/ddtrace/contrib/internal/psycopg/connection.py +++ b/ddtrace/contrib/internal/psycopg/connection.py @@ -1,4 +1,3 @@ -from ddtrace import Pin from ddtrace import config from ddtrace.constants import SPAN_KIND from ddtrace.constants import SPAN_MEASURED_KEY @@ -15,6 +14,7 @@ from ddtrace.ext import net from ddtrace.ext import sql from ddtrace.internal.constants import COMPONENT +from ddtrace.trace import Pin class Psycopg3TracedConnection(dbapi.TracedConnection): diff --git a/ddtrace/contrib/internal/psycopg/patch.py b/ddtrace/contrib/internal/psycopg/patch.py index 7da5c1c73c7..9e24cee6696 100644 --- a/ddtrace/contrib/internal/psycopg/patch.py +++ b/ddtrace/contrib/internal/psycopg/patch.py @@ -3,9 +3,9 @@ import os from typing import List # noqa:F401 -from ddtrace import Pin from ddtrace import config from ddtrace.contrib import dbapi +from ddtrace.trace import Pin try: diff --git a/ddtrace/contrib/internal/pylibmc/client.py b/ddtrace/contrib/internal/pylibmc/client.py index 917a42b293e..3ea6f09c62c 100644 --- a/ddtrace/contrib/internal/pylibmc/client.py +++ b/ddtrace/contrib/internal/pylibmc/client.py @@ -51,7 +51,12 @@ def __init__(self, client=None, service=memcached.SERVICE, tracer=None, *args, * super(TracedClient, self).__init__(client) schematized_service = schematize_service_name(service) - pin = ddtrace.Pin(service=schematized_service, tracer=tracer) + # Calling ddtrace.pin.Pin(...) with the `tracer` argument generates a deprecation warning. + # Remove this if statement when the `tracer` argument is removed + if tracer is ddtrace.tracer: + pin = ddtrace.trace.Pin(service=schematized_service) + else: + pin = ddtrace.trace.Pin(service=schematized_service, tracer=tracer) pin.onto(self) # attempt to collect the pool of urls this client talks to @@ -64,7 +69,7 @@ def clone(self, *args, **kwargs): # rewrap new connections. cloned = self.__wrapped__.clone(*args, **kwargs) traced_client = TracedClient(cloned) - pin = ddtrace.Pin.get_from(self) + pin = ddtrace.trace.Pin.get_from(self) if pin: pin.clone().onto(traced_client) return traced_client @@ -155,7 +160,7 @@ def _no_span(self): def _span(self, cmd_name): """Return a span timing the given command.""" - pin = ddtrace.Pin.get_from(self) + pin = ddtrace.trace.Pin.get_from(self) if not pin or not pin.enabled(): return self._no_span() diff --git a/ddtrace/contrib/internal/pymemcache/client.py b/ddtrace/contrib/internal/pymemcache/client.py index 18a46a41053..37e14842a94 100644 --- a/ddtrace/contrib/internal/pymemcache/client.py +++ b/ddtrace/contrib/internal/pymemcache/client.py @@ -29,7 +29,7 @@ from ddtrace.internal.logger import get_logger from ddtrace.internal.schema import schematize_cache_operation from ddtrace.internal.utils.formats import asbool -from ddtrace.pin import Pin +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/contrib/internal/pymemcache/patch.py b/ddtrace/contrib/internal/pymemcache/patch.py index 07402680e9e..dd3687cd248 100644 --- a/ddtrace/contrib/internal/pymemcache/patch.py +++ b/ddtrace/contrib/internal/pymemcache/patch.py @@ -1,11 +1,11 @@ import pymemcache import pymemcache.client.hash +from ddtrace._trace.pin import _DD_PIN_NAME +from ddtrace._trace.pin import _DD_PIN_PROXY_NAME +from ddtrace._trace.pin import Pin from ddtrace.ext import memcached as memcachedx from ddtrace.internal.schema import schematize_service_name -from ddtrace.pin import _DD_PIN_NAME -from ddtrace.pin import _DD_PIN_PROXY_NAME -from ddtrace.pin import Pin from .client import WrappedClient from .client import WrappedHashClient diff --git a/ddtrace/contrib/internal/pymongo/client.py b/ddtrace/contrib/internal/pymongo/client.py index d5b2530d1f7..2cdf2185586 100644 --- a/ddtrace/contrib/internal/pymongo/client.py +++ b/ddtrace/contrib/internal/pymongo/client.py @@ -10,7 +10,6 @@ # project import ddtrace -from ddtrace import Pin from ddtrace import config from ddtrace.constants import _ANALYTICS_SAMPLE_RATE_KEY from ddtrace.constants import SPAN_KIND @@ -26,6 +25,7 @@ from ddtrace.internal.schema import schematize_database_operation from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils import get_argument_value +from ddtrace.trace import Pin from .parse import parse_msg from .parse import parse_query @@ -61,7 +61,7 @@ def __setddpin__(client, pin): pin.onto(client._topology) def __getddpin__(client): - return ddtrace.Pin.get_from(client._topology) + return ddtrace.trace.Pin.get_from(client._topology) # Set a pin on the mongoclient pin on the topology object # This allows us to pass the same pin to the server objects @@ -103,7 +103,7 @@ def _trace_topology_select_server(func, args, kwargs): # Ensure the pin used on the traced mongo client is passed down to the topology instance # This allows us to pass the same pin in traced server objects. topology_instance = get_argument_value(args, kwargs, 0, "self") - pin = ddtrace.Pin.get_from(topology_instance) + pin = ddtrace.trace.Pin.get_from(topology_instance) if pin is not None: pin.onto(server) @@ -125,7 +125,7 @@ def _datadog_trace_operation(operation, wrapped): log.exception("error parsing query") # Gets the pin from the mogno client (through the topology object) - pin = ddtrace.Pin.get_from(wrapped) + pin = ddtrace.trace.Pin.get_from(wrapped) # if we couldn't parse or shouldn't trace the message, just go. if not cmd or not pin or not pin.enabled(): return None @@ -220,7 +220,7 @@ def _trace_socket_command(func, args, kwargs): except Exception: log.exception("error parsing spec. skipping trace") - pin = ddtrace.Pin.get_from(socket_instance) + pin = ddtrace.trace.Pin.get_from(socket_instance) # skip tracing if we don't have a piece of data we need if not dbname or not cmd or not pin or not pin.enabled(): return func(*args, **kwargs) @@ -239,7 +239,7 @@ def _trace_socket_write_command(func, args, kwargs): except Exception: log.exception("error parsing msg") - pin = ddtrace.Pin.get_from(socket_instance) + pin = ddtrace.trace.Pin.get_from(socket_instance) # if we couldn't parse it, don't try to trace it. if not cmd or not pin or not pin.enabled(): return func(*args, **kwargs) @@ -252,7 +252,7 @@ def _trace_socket_write_command(func, args, kwargs): def _trace_cmd(cmd, socket_instance, address): - pin = ddtrace.Pin.get_from(socket_instance) + pin = ddtrace.trace.Pin.get_from(socket_instance) s = pin.tracer.trace( schematize_database_operation("pymongo.cmd", database_provider="mongodb"), span_type=SpanTypes.MONGODB, diff --git a/ddtrace/contrib/internal/pymongo/patch.py b/ddtrace/contrib/internal/pymongo/patch.py index 0c0927ffea1..200a4a902b8 100644 --- a/ddtrace/contrib/internal/pymongo/patch.py +++ b/ddtrace/contrib/internal/pymongo/patch.py @@ -2,7 +2,6 @@ import pymongo -from ddtrace import Pin from ddtrace import config from ddtrace.constants import SPAN_KIND from ddtrace.constants import SPAN_MEASURED_KEY @@ -15,6 +14,7 @@ from ddtrace.internal.utils import get_argument_value from ddtrace.internal.wrapping import unwrap as _u from ddtrace.internal.wrapping import wrap as _w +from ddtrace.trace import Pin from ....internal.schema import schematize_service_name diff --git a/ddtrace/contrib/internal/pymysql/patch.py b/ddtrace/contrib/internal/pymysql/patch.py index 00fee4f5ad7..a9a16d50608 100644 --- a/ddtrace/contrib/internal/pymysql/patch.py +++ b/ddtrace/contrib/internal/pymysql/patch.py @@ -3,7 +3,6 @@ import pymysql import wrapt -from ddtrace import Pin from ddtrace import config from ddtrace.contrib.dbapi import TracedConnection from ddtrace.contrib.trace_utils import _convert_to_string @@ -13,6 +12,7 @@ from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils.formats import asbool from ddtrace.propagation._database_monitoring import _DBM_Propagator +from ddtrace.trace import Pin config._add( diff --git a/ddtrace/contrib/internal/pynamodb/patch.py b/ddtrace/contrib/internal/pynamodb/patch.py index 15e1874ee77..be4ba00c893 100644 --- a/ddtrace/contrib/internal/pynamodb/patch.py +++ b/ddtrace/contrib/internal/pynamodb/patch.py @@ -20,7 +20,7 @@ from ddtrace.internal.utils import ArgumentError from ddtrace.internal.utils import get_argument_value from ddtrace.internal.utils.formats import deep_getattr -from ddtrace.pin import Pin +from ddtrace.trace import Pin # Pynamodb connection class diff --git a/ddtrace/contrib/internal/pyodbc/patch.py b/ddtrace/contrib/internal/pyodbc/patch.py index 40b561d2f53..180895a202e 100644 --- a/ddtrace/contrib/internal/pyodbc/patch.py +++ b/ddtrace/contrib/internal/pyodbc/patch.py @@ -2,7 +2,6 @@ import pyodbc -from ddtrace import Pin from ddtrace import config from ddtrace.contrib.dbapi import TracedConnection from ddtrace.contrib.dbapi import TracedCursor @@ -11,6 +10,7 @@ from ddtrace.ext import db from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils.formats import asbool +from ddtrace.trace import Pin config._add( diff --git a/ddtrace/contrib/internal/pytest/__init__.py b/ddtrace/contrib/internal/pytest/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ddtrace/contrib/pytest/_atr_utils.py b/ddtrace/contrib/internal/pytest/_atr_utils.py similarity index 94% rename from ddtrace/contrib/pytest/_atr_utils.py rename to ddtrace/contrib/internal/pytest/_atr_utils.py index 0d684486602..82d65c284b0 100644 --- a/ddtrace/contrib/pytest/_atr_utils.py +++ b/ddtrace/contrib/internal/pytest/_atr_utils.py @@ -3,15 +3,15 @@ import _pytest import pytest -from ddtrace.contrib.pytest._retry_utils import RetryOutcomes -from ddtrace.contrib.pytest._retry_utils import _get_outcome_from_retry -from ddtrace.contrib.pytest._retry_utils import _get_retry_attempt_string -from ddtrace.contrib.pytest._retry_utils import set_retry_num -from ddtrace.contrib.pytest._types import _pytest_report_teststatus_return_type -from ddtrace.contrib.pytest._types import pytest_TestReport -from ddtrace.contrib.pytest._utils import PYTEST_STATUS -from ddtrace.contrib.pytest._utils import _get_test_id_from_item -from ddtrace.contrib.pytest._utils import _TestOutcome +from ddtrace.contrib.internal.pytest._retry_utils import RetryOutcomes +from ddtrace.contrib.internal.pytest._retry_utils import _get_outcome_from_retry +from ddtrace.contrib.internal.pytest._retry_utils import _get_retry_attempt_string +from ddtrace.contrib.internal.pytest._retry_utils import set_retry_num +from ddtrace.contrib.internal.pytest._types import _pytest_report_teststatus_return_type +from ddtrace.contrib.internal.pytest._types import pytest_TestReport +from ddtrace.contrib.internal.pytest._utils import PYTEST_STATUS +from ddtrace.contrib.internal.pytest._utils import _get_test_id_from_item +from ddtrace.contrib.internal.pytest._utils import _TestOutcome from ddtrace.ext.test_visibility.api import TestStatus from ddtrace.internal.logger import get_logger from ddtrace.internal.test_visibility._internal_item_ids import InternalTestId diff --git a/ddtrace/contrib/pytest/_benchmark_utils.py b/ddtrace/contrib/internal/pytest/_benchmark_utils.py similarity index 87% rename from ddtrace/contrib/pytest/_benchmark_utils.py rename to ddtrace/contrib/internal/pytest/_benchmark_utils.py index 77dd6061b13..70a79c60700 100644 --- a/ddtrace/contrib/pytest/_benchmark_utils.py +++ b/ddtrace/contrib/internal/pytest/_benchmark_utils.py @@ -1,7 +1,7 @@ import pytest -from ddtrace.contrib.pytest._utils import _get_test_id_from_item -from ddtrace.contrib.pytest_benchmark.constants import PLUGIN_METRICS_V2 +from ddtrace.contrib.internal.pytest._utils import _get_test_id_from_item +from ddtrace.contrib.internal.pytest_benchmark.constants import PLUGIN_METRICS_V2 from ddtrace.internal.logger import get_logger from ddtrace.internal.test_visibility._benchmark_mixin import BenchmarkDurationData from ddtrace.internal.test_visibility.api import InternalTest diff --git a/ddtrace/contrib/pytest/_efd_utils.py b/ddtrace/contrib/internal/pytest/_efd_utils.py similarity index 94% rename from ddtrace/contrib/pytest/_efd_utils.py rename to ddtrace/contrib/internal/pytest/_efd_utils.py index 1e16934bb11..a64148cd574 100644 --- a/ddtrace/contrib/pytest/_efd_utils.py +++ b/ddtrace/contrib/internal/pytest/_efd_utils.py @@ -3,15 +3,15 @@ import _pytest import pytest -from ddtrace.contrib.pytest._retry_utils import RetryOutcomes -from ddtrace.contrib.pytest._retry_utils import _get_outcome_from_retry -from ddtrace.contrib.pytest._retry_utils import _get_retry_attempt_string -from ddtrace.contrib.pytest._retry_utils import set_retry_num -from ddtrace.contrib.pytest._types import _pytest_report_teststatus_return_type -from ddtrace.contrib.pytest._types import pytest_TestReport -from ddtrace.contrib.pytest._utils import PYTEST_STATUS -from ddtrace.contrib.pytest._utils import _get_test_id_from_item -from ddtrace.contrib.pytest._utils import _TestOutcome +from ddtrace.contrib.internal.pytest._retry_utils import RetryOutcomes +from ddtrace.contrib.internal.pytest._retry_utils import _get_outcome_from_retry +from ddtrace.contrib.internal.pytest._retry_utils import _get_retry_attempt_string +from ddtrace.contrib.internal.pytest._retry_utils import set_retry_num +from ddtrace.contrib.internal.pytest._types import _pytest_report_teststatus_return_type +from ddtrace.contrib.internal.pytest._types import pytest_TestReport +from ddtrace.contrib.internal.pytest._utils import PYTEST_STATUS +from ddtrace.contrib.internal.pytest._utils import _get_test_id_from_item +from ddtrace.contrib.internal.pytest._utils import _TestOutcome from ddtrace.ext.test_visibility.api import TestStatus from ddtrace.internal.logger import get_logger from ddtrace.internal.test_visibility._efd_mixins import EFDTestStatus diff --git a/ddtrace/contrib/pytest/_plugin_v1.py b/ddtrace/contrib/internal/pytest/_plugin_v1.py similarity index 98% rename from ddtrace/contrib/pytest/_plugin_v1.py rename to ddtrace/contrib/internal/pytest/_plugin_v1.py index e7b7b2caac9..fc4982bdc67 100644 --- a/ddtrace/contrib/pytest/_plugin_v1.py +++ b/ddtrace/contrib/internal/pytest/_plugin_v1.py @@ -30,15 +30,15 @@ from ddtrace.contrib.internal.coverage.patch import unpatch as unpatch_coverage from ddtrace.contrib.internal.coverage.utils import _is_coverage_invoked_by_coverage_run from ddtrace.contrib.internal.coverage.utils import _is_coverage_patched -from ddtrace.contrib.pytest._utils import _extract_span -from ddtrace.contrib.pytest._utils import _is_enabled_early -from ddtrace.contrib.pytest._utils import _is_pytest_8_or_later -from ddtrace.contrib.pytest._utils import _is_test_unskippable -from ddtrace.contrib.pytest.constants import FRAMEWORK -from ddtrace.contrib.pytest.constants import KIND -from ddtrace.contrib.pytest.constants import XFAIL_REASON -from ddtrace.contrib.pytest.plugin import is_enabled -from ddtrace.contrib.unittest import unpatch as unpatch_unittest +from ddtrace.contrib.internal.pytest._utils import _extract_span +from ddtrace.contrib.internal.pytest._utils import _is_enabled_early +from ddtrace.contrib.internal.pytest._utils import _is_pytest_8_or_later +from ddtrace.contrib.internal.pytest._utils import _is_test_unskippable +from ddtrace.contrib.internal.pytest.constants import FRAMEWORK +from ddtrace.contrib.internal.pytest.constants import KIND +from ddtrace.contrib.internal.pytest.constants import XFAIL_REASON +from ddtrace.contrib.internal.pytest.plugin import is_enabled +from ddtrace.contrib.internal.unittest.patch import unpatch as unpatch_unittest from ddtrace.ext import SpanTypes from ddtrace.ext import test from ddtrace.internal.ci_visibility import CIVisibility as _CIVisibility diff --git a/ddtrace/contrib/pytest/_plugin_v2.py b/ddtrace/contrib/internal/pytest/_plugin_v2.py similarity index 90% rename from ddtrace/contrib/pytest/_plugin_v2.py rename to ddtrace/contrib/internal/pytest/_plugin_v2.py index f15373a776a..ece2098e05e 100644 --- a/ddtrace/contrib/pytest/_plugin_v2.py +++ b/ddtrace/contrib/internal/pytest/_plugin_v2.py @@ -6,38 +6,38 @@ from ddtrace import DDTraceDeprecationWarning from ddtrace import config as dd_config -from ddtrace import patch +from ddtrace._monkey import patch from ddtrace.contrib.coverage import patch as patch_coverage from ddtrace.contrib.internal.coverage.constants import PCT_COVERED_KEY from ddtrace.contrib.internal.coverage.data import _coverage_data from ddtrace.contrib.internal.coverage.patch import run_coverage_report from ddtrace.contrib.internal.coverage.utils import _is_coverage_invoked_by_coverage_run from ddtrace.contrib.internal.coverage.utils import _is_coverage_patched -from ddtrace.contrib.pytest._benchmark_utils import _set_benchmark_data_from_item -from ddtrace.contrib.pytest._plugin_v1 import _extract_reason -from ddtrace.contrib.pytest._plugin_v1 import _is_pytest_cov_enabled -from ddtrace.contrib.pytest._types import _pytest_report_teststatus_return_type -from ddtrace.contrib.pytest._types import pytest_CallInfo -from ddtrace.contrib.pytest._types import pytest_Config -from ddtrace.contrib.pytest._types import pytest_TestReport -from ddtrace.contrib.pytest._utils import PYTEST_STATUS -from ddtrace.contrib.pytest._utils import _get_module_path_from_item -from ddtrace.contrib.pytest._utils import _get_names_from_item -from ddtrace.contrib.pytest._utils import _get_session_command -from ddtrace.contrib.pytest._utils import _get_source_file_info -from ddtrace.contrib.pytest._utils import _get_test_id_from_item -from ddtrace.contrib.pytest._utils import _get_test_parameters_json -from ddtrace.contrib.pytest._utils import _is_enabled_early -from ddtrace.contrib.pytest._utils import _is_test_unskippable -from ddtrace.contrib.pytest._utils import _pytest_marked_to_skip -from ddtrace.contrib.pytest._utils import _pytest_version_supports_atr -from ddtrace.contrib.pytest._utils import _pytest_version_supports_efd -from ddtrace.contrib.pytest._utils import _pytest_version_supports_retries -from ddtrace.contrib.pytest._utils import _TestOutcome -from ddtrace.contrib.pytest.constants import FRAMEWORK -from ddtrace.contrib.pytest.constants import XFAIL_REASON -from ddtrace.contrib.pytest.plugin import is_enabled -from ddtrace.contrib.unittest import unpatch as unpatch_unittest +from ddtrace.contrib.internal.pytest._benchmark_utils import _set_benchmark_data_from_item +from ddtrace.contrib.internal.pytest._plugin_v1 import _extract_reason +from ddtrace.contrib.internal.pytest._plugin_v1 import _is_pytest_cov_enabled +from ddtrace.contrib.internal.pytest._types import _pytest_report_teststatus_return_type +from ddtrace.contrib.internal.pytest._types import pytest_CallInfo +from ddtrace.contrib.internal.pytest._types import pytest_Config +from ddtrace.contrib.internal.pytest._types import pytest_TestReport +from ddtrace.contrib.internal.pytest._utils import PYTEST_STATUS +from ddtrace.contrib.internal.pytest._utils import _get_module_path_from_item +from ddtrace.contrib.internal.pytest._utils import _get_names_from_item +from ddtrace.contrib.internal.pytest._utils import _get_session_command +from ddtrace.contrib.internal.pytest._utils import _get_source_file_info +from ddtrace.contrib.internal.pytest._utils import _get_test_id_from_item +from ddtrace.contrib.internal.pytest._utils import _get_test_parameters_json +from ddtrace.contrib.internal.pytest._utils import _is_enabled_early +from ddtrace.contrib.internal.pytest._utils import _is_test_unskippable +from ddtrace.contrib.internal.pytest._utils import _pytest_marked_to_skip +from ddtrace.contrib.internal.pytest._utils import _pytest_version_supports_atr +from ddtrace.contrib.internal.pytest._utils import _pytest_version_supports_efd +from ddtrace.contrib.internal.pytest._utils import _pytest_version_supports_retries +from ddtrace.contrib.internal.pytest._utils import _TestOutcome +from ddtrace.contrib.internal.pytest.constants import FRAMEWORK +from ddtrace.contrib.internal.pytest.constants import XFAIL_REASON +from ddtrace.contrib.internal.pytest.plugin import is_enabled +from ddtrace.contrib.internal.unittest.patch import unpatch as unpatch_unittest from ddtrace.ext import test from ddtrace.ext.test_visibility import ITR_SKIPPING_LEVEL from ddtrace.ext.test_visibility.api import TestExcInfo @@ -63,21 +63,21 @@ if _pytest_version_supports_retries(): - from ddtrace.contrib.pytest._retry_utils import get_retry_num + from ddtrace.contrib.internal.pytest._retry_utils import get_retry_num if _pytest_version_supports_efd(): - from ddtrace.contrib.pytest._efd_utils import efd_get_failed_reports - from ddtrace.contrib.pytest._efd_utils import efd_get_teststatus - from ddtrace.contrib.pytest._efd_utils import efd_handle_retries - from ddtrace.contrib.pytest._efd_utils import efd_pytest_terminal_summary_post_yield + from ddtrace.contrib.internal.pytest._efd_utils import efd_get_failed_reports + from ddtrace.contrib.internal.pytest._efd_utils import efd_get_teststatus + from ddtrace.contrib.internal.pytest._efd_utils import efd_handle_retries + from ddtrace.contrib.internal.pytest._efd_utils import efd_pytest_terminal_summary_post_yield if _pytest_version_supports_atr(): - from ddtrace.contrib.pytest._atr_utils import atr_get_failed_reports - from ddtrace.contrib.pytest._atr_utils import atr_get_teststatus - from ddtrace.contrib.pytest._atr_utils import atr_handle_retries - from ddtrace.contrib.pytest._atr_utils import atr_pytest_terminal_summary_post_yield - from ddtrace.contrib.pytest._atr_utils import quarantine_atr_get_teststatus - from ddtrace.contrib.pytest._atr_utils import quarantine_pytest_terminal_summary_post_yield + from ddtrace.contrib.internal.pytest._atr_utils import atr_get_failed_reports + from ddtrace.contrib.internal.pytest._atr_utils import atr_get_teststatus + from ddtrace.contrib.internal.pytest._atr_utils import atr_handle_retries + from ddtrace.contrib.internal.pytest._atr_utils import atr_pytest_terminal_summary_post_yield + from ddtrace.contrib.internal.pytest._atr_utils import quarantine_atr_get_teststatus + from ddtrace.contrib.internal.pytest._atr_utils import quarantine_pytest_terminal_summary_post_yield log = get_logger(__name__) @@ -217,7 +217,7 @@ def pytest_configure(config: pytest_Config) -> None: # pytest-bdd plugin support if config.pluginmanager.hasplugin("pytest-bdd"): - from ddtrace.contrib.pytest._pytest_bdd_subplugin import _PytestBddSubPlugin + from ddtrace.contrib.internal.pytest._pytest_bdd_subplugin import _PytestBddSubPlugin config.pluginmanager.register(_PytestBddSubPlugin(), "_datadog-pytest-bdd") else: diff --git a/ddtrace/contrib/pytest/_pytest_bdd_subplugin.py b/ddtrace/contrib/internal/pytest/_pytest_bdd_subplugin.py similarity index 88% rename from ddtrace/contrib/pytest/_pytest_bdd_subplugin.py rename to ddtrace/contrib/internal/pytest/_pytest_bdd_subplugin.py index 7c964af3d59..4349f10654e 100644 --- a/ddtrace/contrib/pytest/_pytest_bdd_subplugin.py +++ b/ddtrace/contrib/internal/pytest/_pytest_bdd_subplugin.py @@ -13,13 +13,13 @@ import pytest -from ddtrace.contrib.pytest._utils import _get_test_id_from_item -from ddtrace.contrib.pytest_bdd import get_version -from ddtrace.contrib.pytest_bdd._plugin import _extract_span -from ddtrace.contrib.pytest_bdd._plugin import _get_step_func_args_json -from ddtrace.contrib.pytest_bdd._plugin import _store_span -from ddtrace.contrib.pytest_bdd.constants import FRAMEWORK -from ddtrace.contrib.pytest_bdd.constants import STEP_KIND +from ddtrace.contrib.internal.pytest._utils import _get_test_id_from_item +from ddtrace.contrib.internal.pytest_bdd._plugin import _extract_span +from ddtrace.contrib.internal.pytest_bdd._plugin import _get_step_func_args_json +from ddtrace.contrib.internal.pytest_bdd._plugin import _store_span +from ddtrace.contrib.internal.pytest_bdd.constants import FRAMEWORK +from ddtrace.contrib.internal.pytest_bdd.constants import STEP_KIND +from ddtrace.contrib.internal.pytest_bdd.patch import get_version from ddtrace.ext import test from ddtrace.internal.logger import get_logger from ddtrace.internal.test_visibility.api import InternalTest diff --git a/ddtrace/contrib/pytest/_retry_utils.py b/ddtrace/contrib/internal/pytest/_retry_utils.py similarity index 97% rename from ddtrace/contrib/pytest/_retry_utils.py rename to ddtrace/contrib/internal/pytest/_retry_utils.py index 6e38a2974c8..eab45f049be 100644 --- a/ddtrace/contrib/pytest/_retry_utils.py +++ b/ddtrace/contrib/internal/pytest/_retry_utils.py @@ -8,8 +8,8 @@ from _pytest.runner import CallInfo import pytest -from ddtrace.contrib.pytest._types import tmppath_result_key -from ddtrace.contrib.pytest._utils import _TestOutcome +from ddtrace.contrib.internal.pytest._types import tmppath_result_key +from ddtrace.contrib.internal.pytest._utils import _TestOutcome from ddtrace.ext.test_visibility.api import TestExcInfo from ddtrace.ext.test_visibility.api import TestStatus from ddtrace.internal import core diff --git a/ddtrace/contrib/pytest/_types.py b/ddtrace/contrib/internal/pytest/_types.py similarity index 90% rename from ddtrace/contrib/pytest/_types.py rename to ddtrace/contrib/internal/pytest/_types.py index ff1d07feb4d..8222bc7bc54 100644 --- a/ddtrace/contrib/pytest/_types.py +++ b/ddtrace/contrib/internal/pytest/_types.py @@ -1,6 +1,6 @@ import typing as t -from ddtrace.contrib.pytest._utils import _get_pytest_version_tuple +from ddtrace.contrib.internal.pytest._utils import _get_pytest_version_tuple if _get_pytest_version_tuple() >= (7, 0, 0): diff --git a/ddtrace/contrib/pytest/_utils.py b/ddtrace/contrib/internal/pytest/_utils.py similarity index 95% rename from ddtrace/contrib/pytest/_utils.py rename to ddtrace/contrib/internal/pytest/_utils.py index 8dc53ab0228..7e8b2bc2714 100644 --- a/ddtrace/contrib/pytest/_utils.py +++ b/ddtrace/contrib/internal/pytest/_utils.py @@ -7,10 +7,10 @@ import pytest -from ddtrace.contrib.pytest.constants import ATR_MIN_SUPPORTED_VERSION -from ddtrace.contrib.pytest.constants import EFD_MIN_SUPPORTED_VERSION -from ddtrace.contrib.pytest.constants import ITR_MIN_SUPPORTED_VERSION -from ddtrace.contrib.pytest.constants import RETRIES_MIN_SUPPORTED_VERSION +from ddtrace.contrib.internal.pytest.constants import ATR_MIN_SUPPORTED_VERSION +from ddtrace.contrib.internal.pytest.constants import EFD_MIN_SUPPORTED_VERSION +from ddtrace.contrib.internal.pytest.constants import ITR_MIN_SUPPORTED_VERSION +from ddtrace.contrib.internal.pytest.constants import RETRIES_MIN_SUPPORTED_VERSION from ddtrace.ext.test_visibility.api import TestExcInfo from ddtrace.ext.test_visibility.api import TestModuleId from ddtrace.ext.test_visibility.api import TestSourceFileInfo diff --git a/ddtrace/contrib/internal/pytest/constants.py b/ddtrace/contrib/internal/pytest/constants.py new file mode 100644 index 00000000000..cc5d768fc38 --- /dev/null +++ b/ddtrace/contrib/internal/pytest/constants.py @@ -0,0 +1,11 @@ +FRAMEWORK = "pytest" +KIND = "test" + + +# XFail Reason +XFAIL_REASON = "pytest.xfail.reason" + +ITR_MIN_SUPPORTED_VERSION = (7, 2, 0) +RETRIES_MIN_SUPPORTED_VERSION = (7, 0, 0) +EFD_MIN_SUPPORTED_VERSION = RETRIES_MIN_SUPPORTED_VERSION +ATR_MIN_SUPPORTED_VERSION = RETRIES_MIN_SUPPORTED_VERSION diff --git a/ddtrace/contrib/internal/pytest/newhooks.py b/ddtrace/contrib/internal/pytest/newhooks.py new file mode 100644 index 00000000000..c44fd0a1535 --- /dev/null +++ b/ddtrace/contrib/internal/pytest/newhooks.py @@ -0,0 +1,26 @@ +"""pytest-ddtrace hooks. + +These hooks are used to provide extra data used by the Datadog CI Visibility plugin. + +For example: module, suite, and test names for a given item. + +Note that these names will affect th display and reporting of tests in the Datadog UI, as well as information stored +the Intelligent Test Runner. Differing hook implementations may impact the behavior of Datadog CI Visibility products. +""" + +import pytest + + +@pytest.hookspec(firstresult=True) +def pytest_ddtrace_get_item_module_name(item: pytest.Item) -> str: + """Returns the module name to use when reporting CI Visibility results, should be unique""" + + +@pytest.hookspec(firstresult=True) +def pytest_ddtrace_get_item_suite_name(item: pytest.Item) -> str: + """Returns the suite name to use when reporting CI Visibility result, should be unique""" + + +@pytest.hookspec(firstresult=True) +def pytest_ddtrace_get_item_test_name(item: pytest.Item) -> str: + """Returns the test name to use when reporting CI Visibility result, should be unique""" diff --git a/ddtrace/contrib/internal/pytest/patch.py b/ddtrace/contrib/internal/pytest/patch.py new file mode 100644 index 00000000000..0299b665268 --- /dev/null +++ b/ddtrace/contrib/internal/pytest/patch.py @@ -0,0 +1,6 @@ +# Get version is imported from patch.py in _monkey.py +def get_version(): + # type: () -> str + import pytest + + return pytest.__version__ diff --git a/ddtrace/contrib/internal/pytest/plugin.py b/ddtrace/contrib/internal/pytest/plugin.py new file mode 100644 index 00000000000..52cf54a6f9c --- /dev/null +++ b/ddtrace/contrib/internal/pytest/plugin.py @@ -0,0 +1,178 @@ +""" +This custom pytest plugin implements tracing for pytest by using pytest hooks. The plugin registers tracing code +to be run at specific points during pytest execution. The most important hooks used are: + + * pytest_sessionstart: during pytest session startup, a custom trace filter is configured to the global tracer to + only send test spans, which are generated by the plugin. + * pytest_runtest_protocol: this wraps around the execution of a pytest test function, which we trace. Most span + tags are generated and added in this function. We also store the span on the underlying pytest test item to + retrieve later when we need to report test status/result. + * pytest_runtest_makereport: this hook is used to set the test status/result tag, including skipped tests and + expected failures. + +""" +import os +from typing import Dict # noqa:F401 + +import pytest + +from ddtrace import config +from ddtrace.appsec._iast._pytest_plugin import ddtrace_iast # noqa:F401 +from ddtrace.appsec._iast._utils import _is_iast_enabled +from ddtrace.contrib.internal.pytest._utils import _USE_PLUGIN_V2 +from ddtrace.contrib.internal.pytest._utils import _extract_span +from ddtrace.contrib.internal.pytest._utils import _pytest_version_supports_itr + + +# pytest default settings +config._add( + "pytest", + dict( + _default_service="pytest", + operation_name=os.getenv("DD_PYTEST_OPERATION_NAME", default="pytest.test"), + ), +) + + +DDTRACE_HELP_MSG = "Enable tracing of pytest functions." +NO_DDTRACE_HELP_MSG = "Disable tracing of pytest functions." +DDTRACE_INCLUDE_CLASS_HELP_MSG = "Prepend 'ClassName.' to names of class-based tests." +PATCH_ALL_HELP_MSG = "Call ddtrace.patch_all before running tests." + + +def is_enabled(config): + """Check if the ddtrace plugin is enabled.""" + return (config.getoption("ddtrace") or config.getini("ddtrace")) and not config.getoption("no-ddtrace") + + +def pytest_addoption(parser): + """Add ddtrace options.""" + group = parser.getgroup("ddtrace") + + group._addoption( + "--ddtrace", + action="store_true", + dest="ddtrace", + default=False, + help=DDTRACE_HELP_MSG, + ) + + group._addoption( + "--no-ddtrace", + action="store_true", + dest="no-ddtrace", + default=False, + help=NO_DDTRACE_HELP_MSG, + ) + + group._addoption( + "--ddtrace-patch-all", + action="store_true", + dest="ddtrace-patch-all", + default=False, + help=PATCH_ALL_HELP_MSG, + ) + + group._addoption( + "--ddtrace-include-class-name", + action="store_true", + dest="ddtrace-include-class-name", + default=False, + help=DDTRACE_INCLUDE_CLASS_HELP_MSG, + ) + + group._addoption( + "--ddtrace-iast-fail-tests", + action="store_true", + dest="ddtrace-iast-fail-tests", + default=False, + help=DDTRACE_INCLUDE_CLASS_HELP_MSG, + ) + + parser.addini("ddtrace", DDTRACE_HELP_MSG, type="bool") + parser.addini("no-ddtrace", DDTRACE_HELP_MSG, type="bool") + parser.addini("ddtrace-patch-all", PATCH_ALL_HELP_MSG, type="bool") + parser.addini("ddtrace-include-class-name", DDTRACE_INCLUDE_CLASS_HELP_MSG, type="bool") + if _is_iast_enabled(): + from ddtrace.appsec._iast import _iast_pytest_activation + + _iast_pytest_activation() + + +# Version-specific pytest hooks +if _USE_PLUGIN_V2: + from ddtrace.contrib.internal.pytest._plugin_v2 import pytest_collection_finish # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v2 import pytest_configure as _versioned_pytest_configure + from ddtrace.contrib.internal.pytest._plugin_v2 import pytest_ddtrace_get_item_module_name # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v2 import pytest_ddtrace_get_item_suite_name # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v2 import pytest_ddtrace_get_item_test_name # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v2 import pytest_load_initial_conftests # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v2 import pytest_report_teststatus # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v2 import pytest_runtest_makereport # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v2 import pytest_runtest_protocol # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v2 import pytest_sessionfinish # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v2 import pytest_sessionstart # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v2 import pytest_terminal_summary # noqa: F401 +else: + from ddtrace.contrib.internal.pytest._plugin_v1 import pytest_collection_modifyitems # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v1 import pytest_configure as _versioned_pytest_configure + from ddtrace.contrib.internal.pytest._plugin_v1 import pytest_ddtrace_get_item_module_name # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v1 import pytest_ddtrace_get_item_suite_name # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v1 import pytest_ddtrace_get_item_test_name # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v1 import pytest_load_initial_conftests # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v1 import pytest_runtest_makereport # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v1 import pytest_runtest_protocol # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v1 import pytest_sessionfinish # noqa: F401 + from ddtrace.contrib.internal.pytest._plugin_v1 import pytest_sessionstart # noqa: F401 + + # Internal coverage is only used for ITR at the moment, so the hook is only added if the pytest version supports it + if _pytest_version_supports_itr(): + from ddtrace.contrib.internal.pytest._plugin_v1 import pytest_terminal_summary # noqa: F401 + + +def pytest_configure(config): + config.addinivalue_line("markers", "dd_tags(**kwargs): add tags to current span") + if is_enabled(config): + _versioned_pytest_configure(config) + + +@pytest.hookimpl +def pytest_addhooks(pluginmanager): + from ddtrace.contrib.internal.pytest import newhooks + + pluginmanager.add_hookspecs(newhooks) + + +@pytest.fixture(scope="function") +def ddspan(request): + """Return the :class:`ddtrace._trace.span.Span` instance associated with the + current test when Datadog CI Visibility is enabled. + """ + from ddtrace.internal.ci_visibility import CIVisibility as _CIVisibility + + if _CIVisibility.enabled: + return _extract_span(request.node) + + +@pytest.fixture(scope="session") +def ddtracer(): + """Return the :class:`ddtrace.tracer.Tracer` instance for Datadog CI + visibility if it is enabled, otherwise return the default Datadog tracer. + """ + import ddtrace + from ddtrace.internal.ci_visibility import CIVisibility as _CIVisibility + + if _CIVisibility.enabled: + return _CIVisibility._instance.tracer + return ddtrace.tracer + + +@pytest.fixture(scope="session", autouse=True) +def patch_all(request): + """Patch all available modules for Datadog tracing when ddtrace-patch-all + is specified in command or .ini. + """ + import ddtrace + + if request.config.getoption("ddtrace-patch-all") or request.config.getini("ddtrace-patch-all"): + ddtrace.patch_all() diff --git a/ddtrace/contrib/internal/pytest_bdd/__init__.py b/ddtrace/contrib/internal/pytest_bdd/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ddtrace/contrib/pytest_bdd/_plugin.py b/ddtrace/contrib/internal/pytest_bdd/_plugin.py similarity index 94% rename from ddtrace/contrib/pytest_bdd/_plugin.py rename to ddtrace/contrib/internal/pytest_bdd/_plugin.py index 1ed8b8099e5..eb7bcc1028a 100644 --- a/ddtrace/contrib/pytest_bdd/_plugin.py +++ b/ddtrace/contrib/internal/pytest_bdd/_plugin.py @@ -4,10 +4,10 @@ import pytest -from ddtrace.contrib.pytest._utils import _extract_span as _extract_feature_span -from ddtrace.contrib.pytest_bdd import get_version -from ddtrace.contrib.pytest_bdd.constants import FRAMEWORK -from ddtrace.contrib.pytest_bdd.constants import STEP_KIND +from ddtrace.contrib.internal.pytest._utils import _extract_span as _extract_feature_span +from ddtrace.contrib.internal.pytest_bdd.constants import FRAMEWORK +from ddtrace.contrib.internal.pytest_bdd.constants import STEP_KIND +from ddtrace.contrib.internal.pytest_bdd.patch import get_version from ddtrace.ext import test from ddtrace.internal.ci_visibility import CIVisibility as _CIVisibility from ddtrace.internal.logger import get_logger diff --git a/ddtrace/contrib/internal/pytest_bdd/constants.py b/ddtrace/contrib/internal/pytest_bdd/constants.py new file mode 100644 index 00000000000..2dd377f7619 --- /dev/null +++ b/ddtrace/contrib/internal/pytest_bdd/constants.py @@ -0,0 +1,2 @@ +FRAMEWORK = "pytest_bdd" +STEP_KIND = "pytest_bdd.step" diff --git a/ddtrace/contrib/internal/pytest_bdd/patch.py b/ddtrace/contrib/internal/pytest_bdd/patch.py new file mode 100644 index 00000000000..efab83aee4b --- /dev/null +++ b/ddtrace/contrib/internal/pytest_bdd/patch.py @@ -0,0 +1,9 @@ +# ddtrace/_monkey.py expects all integrations to define get_version in /patch.py file +def get_version(): + # type: () -> str + try: + import importlib.metadata as importlib_metadata + except ImportError: + import importlib_metadata # type: ignore[no-redef] + + return str(importlib_metadata.version("pytest-bdd")) diff --git a/ddtrace/contrib/internal/pytest_bdd/plugin.py b/ddtrace/contrib/internal/pytest_bdd/plugin.py new file mode 100644 index 00000000000..22856056162 --- /dev/null +++ b/ddtrace/contrib/internal/pytest_bdd/plugin.py @@ -0,0 +1,30 @@ +from ddtrace import DDTraceDeprecationWarning +from ddtrace import config +from ddtrace.contrib.internal.pytest._utils import _USE_PLUGIN_V2 +from ddtrace.contrib.internal.pytest.plugin import is_enabled as is_ddtrace_enabled +from ddtrace.vendor.debtcollector import deprecate + + +# pytest-bdd default settings +config._add( + "pytest_bdd", + dict( + _default_service="pytest_bdd", + ), +) + + +def pytest_configure(config): + if config.pluginmanager.hasplugin("pytest-bdd") and config.pluginmanager.hasplugin("ddtrace"): + if not _USE_PLUGIN_V2: + if is_ddtrace_enabled(config): + from ._plugin import _PytestBddPlugin + + deprecate( + "the ddtrace.pytest_bdd plugin is deprecated", + message="it will be integrated with the main pytest ddtrace plugin", + removal_version="3.0.0", + category=DDTraceDeprecationWarning, + ) + + config.pluginmanager.register(_PytestBddPlugin(), "_datadog-pytest-bdd") diff --git a/ddtrace/contrib/internal/pytest_benchmark/__init__.py b/ddtrace/contrib/internal/pytest_benchmark/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ddtrace/contrib/pytest_benchmark/_plugin.py b/ddtrace/contrib/internal/pytest_benchmark/_plugin.py similarity index 73% rename from ddtrace/contrib/pytest_benchmark/_plugin.py rename to ddtrace/contrib/internal/pytest_benchmark/_plugin.py index ac6afa350d6..ee54660afc8 100644 --- a/ddtrace/contrib/pytest_benchmark/_plugin.py +++ b/ddtrace/contrib/internal/pytest_benchmark/_plugin.py @@ -1,9 +1,9 @@ import pytest -from ddtrace.contrib.pytest._utils import _extract_span -from ddtrace.contrib.pytest_benchmark.constants import BENCHMARK_INFO -from ddtrace.contrib.pytest_benchmark.constants import PLUGIN_METRICS -from ddtrace.contrib.pytest_benchmark.constants import PLUGIN_OUTLIERS +from ddtrace.contrib.internal.pytest._utils import _extract_span +from ddtrace.contrib.internal.pytest_benchmark.constants import BENCHMARK_INFO +from ddtrace.contrib.internal.pytest_benchmark.constants import PLUGIN_METRICS +from ddtrace.contrib.internal.pytest_benchmark.constants import PLUGIN_OUTLIERS from ddtrace.ext.test import TEST_TYPE diff --git a/ddtrace/contrib/internal/pytest_benchmark/constants.py b/ddtrace/contrib/internal/pytest_benchmark/constants.py new file mode 100644 index 00000000000..b4c4f7f5b27 --- /dev/null +++ b/ddtrace/contrib/internal/pytest_benchmark/constants.py @@ -0,0 +1,79 @@ +BENCHMARK_INFO = "benchmark.duration.info" +BENCHMARK_MEAN = "benchmark.duration.mean" +BENCHMARK_RUN = "benchmark.duration.runs" + +STATISTICS_HD15IQR = "benchmark.duration.statistics.hd15iqr" +STATISTICS_IQR = "benchmark.duration.statistics.iqr" +STATISTICS_IQR_OUTLIERS = "benchmark.duration.statistics.iqr_outliers" +STATISTICS_LD15IQR = "benchmark.duration.statistics.ld15iqr" +STATISTICS_MAX = "benchmark.duration.statistics.max" +STATISTICS_MEAN = "benchmark.duration.statistics.mean" +STATISTICS_MEDIAN = "benchmark.duration.statistics.median" +STATISTICS_MIN = "benchmark.duration.statistics.min" +STATISTICS_N = "benchmark.duration.statistics.n" +STATISTICS_OPS = "benchmark.duration.statistics.ops" +STATISTICS_OUTLIERS = "benchmark.duration.statistics.outliers" +STATISTICS_Q1 = "benchmark.duration.statistics.q1" +STATISTICS_Q3 = "benchmark.duration.statistics.q3" +STATISTICS_STDDEV = "benchmark.duration.statistics.std_dev" +STATISTICS_STDDEV_OUTLIERS = "benchmark.duration.statistics.std_dev_outliers" +STATISTICS_TOTAL = "benchmark.duration.statistics.total" + +PLUGIN_HD15IQR = "hd15iqr" +PLUGIN_IQR = "iqr" +PLUGIN_IQR_OUTLIERS = "iqr_outliers" +PLUGIN_LD15IQR = "ld15iqr" +PLUGIN_MAX = "max" +PLUGIN_MEAN = "mean" +PLUGIN_MEDIAN = "median" +PLUGIN_MIN = "min" +PLUGIN_OPS = "ops" +PLUGIN_OUTLIERS = "outliers" +PLUGIN_Q1 = "q1" +PLUGIN_Q3 = "q3" +PLUGIN_ROUNDS = "rounds" +PLUGIN_STDDEV = "stddev" +PLUGIN_STDDEV_OUTLIERS = "stddev_outliers" +PLUGIN_TOTAL = "total" + +PLUGIN_METRICS = { + BENCHMARK_MEAN: PLUGIN_MEAN, + BENCHMARK_RUN: PLUGIN_ROUNDS, + STATISTICS_HD15IQR: PLUGIN_HD15IQR, + STATISTICS_IQR: PLUGIN_IQR, + STATISTICS_IQR_OUTLIERS: PLUGIN_IQR_OUTLIERS, + STATISTICS_LD15IQR: PLUGIN_LD15IQR, + STATISTICS_MAX: PLUGIN_MAX, + STATISTICS_MEAN: PLUGIN_MEAN, + STATISTICS_MEDIAN: PLUGIN_MEDIAN, + STATISTICS_MIN: PLUGIN_MIN, + STATISTICS_OPS: PLUGIN_OPS, + STATISTICS_OUTLIERS: PLUGIN_OUTLIERS, + STATISTICS_Q1: PLUGIN_Q1, + STATISTICS_Q3: PLUGIN_Q3, + STATISTICS_N: PLUGIN_ROUNDS, + STATISTICS_STDDEV: PLUGIN_STDDEV, + STATISTICS_STDDEV_OUTLIERS: PLUGIN_STDDEV_OUTLIERS, + STATISTICS_TOTAL: PLUGIN_TOTAL, +} + +PLUGIN_METRICS_V2 = { + "duration_mean": PLUGIN_MEAN, + "duration_runs": PLUGIN_ROUNDS, + "statistics_hd15iqr": PLUGIN_HD15IQR, + "statistics_iqr": PLUGIN_IQR, + "statistics_iqr_outliers": PLUGIN_IQR_OUTLIERS, + "statistics_ld15iqr": PLUGIN_LD15IQR, + "statistics_max": PLUGIN_MAX, + "statistics_mean": PLUGIN_MEAN, + "statistics_median": PLUGIN_MEDIAN, + "statistics_min": PLUGIN_MIN, + "statistics_n": PLUGIN_ROUNDS, + "statistics_ops": PLUGIN_OPS, + "statistics_outliers": PLUGIN_OUTLIERS, + "statistics_q1": PLUGIN_Q1, + "statistics_q3": PLUGIN_Q3, + "statistics_std_dev": PLUGIN_STDDEV, + "statistics_std_dev_outliers": PLUGIN_STDDEV_OUTLIERS, + "statistics_total": PLUGIN_TOTAL, +} diff --git a/ddtrace/contrib/internal/pytest_benchmark/plugin.py b/ddtrace/contrib/internal/pytest_benchmark/plugin.py new file mode 100644 index 00000000000..04728f764a3 --- /dev/null +++ b/ddtrace/contrib/internal/pytest_benchmark/plugin.py @@ -0,0 +1,19 @@ +from ddtrace import DDTraceDeprecationWarning +from ddtrace.contrib.internal.pytest._utils import _USE_PLUGIN_V2 +from ddtrace.contrib.internal.pytest.plugin import is_enabled as is_ddtrace_enabled +from ddtrace.vendor.debtcollector import deprecate + + +def pytest_configure(config): + if config.pluginmanager.hasplugin("benchmark") and config.pluginmanager.hasplugin("ddtrace"): + if is_ddtrace_enabled(config): + deprecate( + "this version of the ddtrace.pytest_benchmark plugin is deprecated", + message="it will be integrated with the main pytest ddtrace plugin", + removal_version="3.0.0", + category=DDTraceDeprecationWarning, + ) + if not _USE_PLUGIN_V2: + from ._plugin import _PytestBenchmarkPlugin + + config.pluginmanager.register(_PytestBenchmarkPlugin(), "_datadog-pytest-benchmark") diff --git a/ddtrace/contrib/internal/redis/asyncio_patch.py b/ddtrace/contrib/internal/redis/asyncio_patch.py index 0115096ba0f..7c5bad354ab 100644 --- a/ddtrace/contrib/internal/redis/asyncio_patch.py +++ b/ddtrace/contrib/internal/redis/asyncio_patch.py @@ -4,7 +4,7 @@ from ddtrace._trace.utils_redis import _instrument_redis_execute_pipeline from ddtrace.contrib.redis_utils import _run_redis_command_async from ddtrace.internal.utils.formats import stringify_cache_args -from ddtrace.pin import Pin +from ddtrace.trace import Pin async def instrumented_async_execute_command(func, instance, args, kwargs): diff --git a/ddtrace/contrib/internal/redis/patch.py b/ddtrace/contrib/internal/redis/patch.py index 18b23fd68fa..33520e5894d 100644 --- a/ddtrace/contrib/internal/redis/patch.py +++ b/ddtrace/contrib/internal/redis/patch.py @@ -14,7 +14,7 @@ from ddtrace.internal.utils.formats import CMD_MAX_LEN from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.formats import stringify_cache_args -from ddtrace.pin import Pin +from ddtrace.trace import Pin config._add( diff --git a/ddtrace/contrib/internal/rediscluster/patch.py b/ddtrace/contrib/internal/rediscluster/patch.py index a415096ef10..c550df7e9ea 100644 --- a/ddtrace/contrib/internal/rediscluster/patch.py +++ b/ddtrace/contrib/internal/rediscluster/patch.py @@ -23,7 +23,7 @@ from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.formats import stringify_cache_args from ddtrace.internal.utils.wrappers import unwrap -from ddtrace.pin import Pin +from ddtrace.trace import Pin # DEV: In `2.0.0` `__version__` is a string and `VERSION` is a tuple, diff --git a/ddtrace/contrib/internal/requests/patch.py b/ddtrace/contrib/internal/requests/patch.py index a5867662d78..d4ec1f5182d 100644 --- a/ddtrace/contrib/internal/requests/patch.py +++ b/ddtrace/contrib/internal/requests/patch.py @@ -10,8 +10,8 @@ from ddtrace.contrib.trace_utils import unwrap as _u from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils.formats import asbool -from ddtrace.pin import Pin from ddtrace.settings.asm import config as asm_config +from ddtrace.trace import Pin from .connection import _wrap_send diff --git a/ddtrace/contrib/internal/requests/session.py b/ddtrace/contrib/internal/requests/session.py index 9551c70226c..783dda4ff7a 100644 --- a/ddtrace/contrib/internal/requests/session.py +++ b/ddtrace/contrib/internal/requests/session.py @@ -1,8 +1,8 @@ import requests from wrapt import wrap_function_wrapper as _w -from ddtrace import Pin from ddtrace import config +from ddtrace.trace import Pin from .connection import _wrap_send diff --git a/ddtrace/contrib/internal/rq/patch.py b/ddtrace/contrib/internal/rq/patch.py new file mode 100644 index 00000000000..c1f39431f57 --- /dev/null +++ b/ddtrace/contrib/internal/rq/patch.py @@ -0,0 +1,212 @@ +import os + +from ddtrace import config +from ddtrace.constants import SPAN_KIND +from ddtrace.internal import core +from ddtrace.internal.constants import COMPONENT +from ddtrace.internal.schema import schematize_messaging_operation +from ddtrace.internal.schema import schematize_service_name +from ddtrace.internal.schema.span_attribute_schema import SpanDirection +from ddtrace.internal.utils import get_argument_value +from ddtrace.internal.utils.formats import asbool +from ddtrace.trace import Pin + +from ....ext import SpanKind +from ....ext import SpanTypes +from ... import trace_utils + + +config._add( + "rq", + dict( + distributed_tracing_enabled=asbool(os.environ.get("DD_RQ_DISTRIBUTED_TRACING_ENABLED", True)), + _default_service=schematize_service_name("rq"), + ), +) + +config._add( + "rq_worker", + dict( + distributed_tracing_enabled=asbool(os.environ.get("DD_RQ_DISTRIBUTED_TRACING_ENABLED", True)), + _default_service=schematize_service_name("rq-worker"), + ), +) + + +JOB_ID = "job.id" +QUEUE_NAME = "queue.name" +JOB_FUNC_NAME = "job.func_name" + + +def get_version(): + # type: () -> str + import rq + + return str(getattr(rq, "__version__", "")) + + +@trace_utils.with_traced_module +def traced_queue_enqueue_job(rq, pin, func, instance, args, kwargs): + job = get_argument_value(args, kwargs, 0, "f") + + func_name = job.func_name + job_inst = job.instance + job_inst_str = "%s.%s" % (job_inst.__module__, job_inst.__class__.__name__) if job_inst else "" + + if job_inst_str: + resource = "%s.%s" % (job_inst_str, func_name) + else: + resource = func_name + + with core.context_with_data( + "rq.queue.enqueue_job", + span_name=schematize_messaging_operation( + "rq.queue.enqueue_job", provider="rq", direction=SpanDirection.OUTBOUND + ), + pin=pin, + service=trace_utils.int_service(pin, config.rq), + resource=resource, + span_type=SpanTypes.WORKER, + integration_config=config.rq_worker, + tags={ + COMPONENT: config.rq.integration_name, + SPAN_KIND: SpanKind.PRODUCER, + QUEUE_NAME: instance.name, + JOB_ID: job.get_id(), + JOB_FUNC_NAME: job.func_name, + }, + ) as ctx, ctx.span: + # If the queue is_async then add distributed tracing headers to the job + if instance.is_async: + core.dispatch("rq.queue.enqueue_job", [ctx, job.meta]) + return func(*args, **kwargs) + + +@trace_utils.with_traced_module +def traced_queue_fetch_job(rq, pin, func, instance, args, kwargs): + job_id = get_argument_value(args, kwargs, 0, "job_id") + with core.context_with_data( + "rq.traced_queue_fetch_job", + span_name=schematize_messaging_operation( + "rq.queue.fetch_job", provider="rq", direction=SpanDirection.PROCESSING + ), + pin=pin, + service=trace_utils.int_service(pin, config.rq), + tags={COMPONENT: config.rq.integration_name, JOB_ID: job_id}, + ) as ctx, ctx.span: + return func(*args, **kwargs) + + +@trace_utils.with_traced_module +def traced_perform_job(rq, pin, func, instance, args, kwargs): + """Trace rq.Worker.perform_job""" + # `perform_job` is executed in a freshly forked, short-lived instance + job = get_argument_value(args, kwargs, 0, "job") + + try: + with core.context_with_data( + "rq.worker.perform_job", + span_name="rq.worker.perform_job", + service=trace_utils.int_service(pin, config.rq_worker), + pin=pin, + span_type=SpanTypes.WORKER, + resource=job.func_name, + distributed_headers_config=config.rq_worker, + distributed_headers=job.meta, + tags={COMPONENT: config.rq.integration_name, SPAN_KIND: SpanKind.CONSUMER, JOB_ID: job.get_id()}, + ) as ctx, ctx.span: + try: + return func(*args, **kwargs) + finally: + # call _after_perform_job handler for job status and origin + span_tags = {"job.status": job.get_status() or "None", "job.origin": job.origin} + job_failed = job.is_failed + core.dispatch("rq.worker.perform_job", [ctx, job_failed, span_tags]) + + finally: + # Force flush to agent since the process `os.exit()`s + # immediately after this method returns + core.dispatch("rq.worker.after.perform.job", [ctx]) + + +@trace_utils.with_traced_module +def traced_job_perform(rq, pin, func, instance, args, kwargs): + """Trace rq.Job.perform(...)""" + job = instance + + # Inherit the service name from whatever parent exists. + # eg. in a worker, a perform_job parent span will exist with the worker + # service. + with core.context_with_data( + "rq.job.perform", + span_name="rq.job.perform", + resource=job.func_name, + pin=pin, + tags={COMPONENT: config.rq.integration_name, JOB_ID: job.get_id()}, + ) as ctx, ctx.span: + return func(*args, **kwargs) + + +@trace_utils.with_traced_module +def traced_job_fetch_many(rq, pin, func, instance, args, kwargs): + """Trace rq.Job.fetch_many(...)""" + job_ids = get_argument_value(args, kwargs, 0, "job_ids") + with core.context_with_data( + "rq.job.fetch_many", + span_name=schematize_messaging_operation( + "rq.job.fetch_many", provider="rq", direction=SpanDirection.PROCESSING + ), + service=trace_utils.ext_service(pin, config.rq_worker), + pin=pin, + tags={COMPONENT: config.rq.integration_name, JOB_ID: job_ids}, + ) as ctx, ctx.span: + return func(*args, **kwargs) + + +def patch(): + # Avoid importing rq at the module level, eventually will be an import hook + import rq + + if getattr(rq, "_datadog_patch", False): + return + + Pin().onto(rq) + + # Patch rq.job.Job + Pin().onto(rq.job.Job) + trace_utils.wrap(rq.job, "Job.perform", traced_job_perform(rq.job.Job)) + + # Patch rq.queue.Queue + Pin().onto(rq.queue.Queue) + trace_utils.wrap("rq.queue", "Queue.enqueue_job", traced_queue_enqueue_job(rq)) + trace_utils.wrap("rq.queue", "Queue.fetch_job", traced_queue_fetch_job(rq)) + + # Patch rq.worker.Worker + Pin().onto(rq.worker.Worker) + trace_utils.wrap(rq.worker, "Worker.perform_job", traced_perform_job(rq)) + + rq._datadog_patch = True + + +def unpatch(): + import rq + + if not getattr(rq, "_datadog_patch", False): + return + + Pin().remove_from(rq) + + # Unpatch rq.job.Job + Pin().remove_from(rq.job.Job) + trace_utils.unwrap(rq.job.Job, "perform") + + # Unpatch rq.queue.Queue + Pin().remove_from(rq.queue.Queue) + trace_utils.unwrap(rq.queue.Queue, "enqueue_job") + trace_utils.unwrap(rq.queue.Queue, "fetch_job") + + # Unpatch rq.worker.Worker + Pin().remove_from(rq.worker.Worker) + trace_utils.unwrap(rq.worker.Worker, "perform_job") + + rq._datadog_patch = False diff --git a/ddtrace/contrib/internal/sanic/patch.py b/ddtrace/contrib/internal/sanic/patch.py index 826267cd341..5d105cf2f32 100644 --- a/ddtrace/contrib/internal/sanic/patch.py +++ b/ddtrace/contrib/internal/sanic/patch.py @@ -17,7 +17,7 @@ from ddtrace.internal.schema import schematize_url_operation from ddtrace.internal.schema.span_attribute_schema import SpanDirection from ddtrace.internal.utils.wrappers import unwrap as _u -from ddtrace.pin import Pin +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/contrib/internal/snowflake/patch.py b/ddtrace/contrib/internal/snowflake/patch.py index 87896aab109..d28844ea992 100644 --- a/ddtrace/contrib/internal/snowflake/patch.py +++ b/ddtrace/contrib/internal/snowflake/patch.py @@ -2,7 +2,6 @@ import wrapt -from ddtrace import Pin from ddtrace import config from ddtrace.contrib.dbapi import TracedConnection from ddtrace.contrib.dbapi import TracedCursor @@ -11,6 +10,7 @@ from ddtrace.ext import net from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils.formats import asbool +from ddtrace.trace import Pin config._add( diff --git a/ddtrace/contrib/internal/sqlalchemy/engine.py b/ddtrace/contrib/internal/sqlalchemy/engine.py index 05bdb7ca0f1..3b5f96be9e7 100644 --- a/ddtrace/contrib/internal/sqlalchemy/engine.py +++ b/ddtrace/contrib/internal/sqlalchemy/engine.py @@ -29,7 +29,7 @@ from ddtrace.internal.constants import COMPONENT from ddtrace.internal.schema import schematize_database_operation from ddtrace.internal.schema import schematize_service_name -from ddtrace.pin import Pin +from ddtrace.trace import Pin def trace_engine(engine, tracer=None, service=None): @@ -67,7 +67,12 @@ def __init__(self, tracer, service, engine): self.name = schematize_database_operation("%s.query" % self.vendor, database_provider=self.vendor) # attach the PIN - Pin(tracer=tracer, service=self.service).onto(engine) + # Calling ddtrace.pin.Pin(...) with the `tracer` argument generates a deprecation warning. + # Remove this if statement when the `tracer` argument is removed + if self.tracer is ddtrace.tracer: + Pin(service=self.service).onto(engine) + else: + Pin(tracer=tracer, service=self.service).onto(engine) listen(engine, "before_cursor_execute", self._before_cur_exec) listen(engine, "after_cursor_execute", self._after_cur_exec) diff --git a/ddtrace/contrib/internal/sqlite3/patch.py b/ddtrace/contrib/internal/sqlite3/patch.py index dedf92c6297..f47906146bc 100644 --- a/ddtrace/contrib/internal/sqlite3/patch.py +++ b/ddtrace/contrib/internal/sqlite3/patch.py @@ -15,8 +15,8 @@ from ddtrace.internal.schema import schematize_database_operation from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils.formats import asbool -from ddtrace.pin import Pin from ddtrace.settings.asm import config as asm_config +from ddtrace.trace import Pin # Original connect method diff --git a/ddtrace/contrib/internal/starlette/patch.py b/ddtrace/contrib/internal/starlette/patch.py index b872a77ecd7..064722b67f1 100644 --- a/ddtrace/contrib/internal/starlette/patch.py +++ b/ddtrace/contrib/internal/starlette/patch.py @@ -12,7 +12,6 @@ from wrapt import ObjectProxy from wrapt import wrap_function_wrapper as _w -from ddtrace import Pin from ddtrace import config from ddtrace._trace.span import Span # noqa:F401 from ddtrace.appsec._iast import _is_iast_enabled @@ -28,6 +27,7 @@ from ddtrace.internal.utils import get_blocked from ddtrace.internal.utils import set_argument_value from ddtrace.internal.utils.wrappers import unwrap as _u +from ddtrace.trace import Pin from ddtrace.vendor.packaging.version import parse as parse_version diff --git a/ddtrace/contrib/internal/subprocess/patch.py b/ddtrace/contrib/internal/subprocess/patch.py index 7380e72fdaf..80d05b107bb 100644 --- a/ddtrace/contrib/internal/subprocess/patch.py +++ b/ddtrace/contrib/internal/subprocess/patch.py @@ -4,8 +4,8 @@ import os import re import shlex -import subprocess # nosec from threading import RLock +from typing import Callable # noqa:F401 from typing import Deque # noqa:F401 from typing import Dict # noqa:F401 from typing import List # noqa:F401 @@ -14,7 +14,6 @@ from typing import Union # noqa:F401 from typing import cast # noqa:F401 -from ddtrace import Pin from ddtrace import config from ddtrace.contrib import trace_utils from ddtrace.contrib.internal.subprocess.constants import COMMANDS @@ -23,6 +22,7 @@ from ddtrace.internal.compat import shjoin from ddtrace.internal.logger import get_logger from ddtrace.settings.asm import config as asm_config +from ddtrace.trace import Pin log = get_logger(__name__) @@ -33,45 +33,71 @@ ) -def get_version(): - # type: () -> str +def get_version() -> str: return "" -def patch(): - # type: () -> List[str] - patched = [] # type: List[str] - if not asm_config._asm_enabled: - return patched +_STR_CALLBACKS: Dict[str, Callable[[str], None]] = {} +_LST_CALLBACKS: Dict[str, Callable[[Union[List[str], str]], None]] = {} - import os - if not getattr(os, "_datadog_patch", False): - Pin().onto(os) - trace_utils.wrap(os, "system", _traced_ossystem(os)) - trace_utils.wrap(os, "fork", _traced_fork(os)) +def add_str_callback(name: str, callback: Callable[[str], None]): + _STR_CALLBACKS[name] = callback + + +def del_str_callback(name: str): + _STR_CALLBACKS.pop(name, None) + + +def add_lst_callback(name: str, callback: Callable[[Union[List[str], str]], None]): + _LST_CALLBACKS[name] = callback + + +def del_lst_callback(name: str): + _LST_CALLBACKS.pop(name, None) + - # all os.spawn* variants eventually use this one: - trace_utils.wrap(os, "_spawnvef", _traced_osspawn(os)) +def patch() -> List[str]: + if not (asm_config._asm_enabled or asm_config._iast_enabled): + return [] + patched: List[str] = [] + import os # nosec + import subprocess # nosec + + should_patch_system = not trace_utils.iswrapped(os.system) + should_patch_fork = not trace_utils.iswrapped(os.fork) + spawnvef = getattr(os, "_spawnvef", None) + should_patch_spawnvef = spawnvef is not None and not trace_utils.iswrapped(spawnvef) + + if should_patch_system or should_patch_fork or should_patch_spawnvef: + Pin().onto(os) + if should_patch_system: + trace_utils.wrap(os, "system", _traced_ossystem(os)) + if should_patch_fork: + trace_utils.wrap(os, "fork", _traced_fork(os)) + if should_patch_spawnvef: + # all os.spawn* variants eventually use this one: + trace_utils.wrap(os, "_spawnvef", _traced_osspawn(os)) patched.append("os") - if not getattr(subprocess, "_datadog_patch", False): + should_patch_Popen_init = not trace_utils.iswrapped(subprocess.Popen.__init__) + should_patch_Popen_wait = not trace_utils.iswrapped(subprocess.Popen.wait) + if should_patch_Popen_init or should_patch_Popen_wait: Pin().onto(subprocess) # We store the parameters on __init__ in the context and set the tags on wait # (where all the Popen objects eventually arrive, unless killed before it) - trace_utils.wrap(subprocess, "Popen.__init__", _traced_subprocess_init(subprocess)) - trace_utils.wrap(subprocess, "Popen.wait", _traced_subprocess_wait(subprocess)) - - os._datadog_patch = True - subprocess._datadog_patch = True + if should_patch_Popen_init: + trace_utils.wrap(subprocess, "Popen.__init__", _traced_subprocess_init(subprocess)) + if should_patch_Popen_wait: + trace_utils.wrap(subprocess, "Popen.wait", _traced_subprocess_wait(subprocess)) patched.append("subprocess") return patched @dataclass(eq=False) -class SubprocessCmdLineCacheEntry(object): +class SubprocessCmdLineCacheEntry: binary: Optional[str] = None arguments: Optional[List] = None truncated: bool = False @@ -80,10 +106,10 @@ class SubprocessCmdLineCacheEntry(object): as_string: Optional[str] = None -class SubprocessCmdLine(object): +class SubprocessCmdLine: # This catches the computed values into a SubprocessCmdLineCacheEntry object - _CACHE = {} # type: Dict[str, SubprocessCmdLineCacheEntry] - _CACHE_DEQUE = collections.deque() # type: Deque[str] + _CACHE: Dict[str, SubprocessCmdLineCacheEntry] = {} + _CACHE_DEQUE: Deque[str] = collections.deque() _CACHE_MAXSIZE = 32 _CACHE_LOCK = RLock() @@ -138,8 +164,7 @@ def _clear_cache(cls): ] _COMPILED_ENV_VAR_REGEXP = re.compile(r"\b[A-Z_]+=\w+") - def __init__(self, shell_args, shell=False): - # type: (Union[str, List[str]], bool) -> None + def __init__(self, shell_args: Union[str, List[str]], shell: bool = False) -> None: cache_key = str(shell_args) + str(shell) self._cache_entry = SubprocessCmdLine._CACHE.get(cache_key) if self._cache_entry: @@ -250,8 +275,7 @@ def scrub_arguments(self): self.arguments = new_args - def truncate_string(self, str_): - # type: (str) -> str + def truncate_string(self, str_: str) -> str: oversize = len(str_) - self.TRUNCATE_LIMIT if oversize <= 0: @@ -263,9 +287,7 @@ def truncate_string(self, str_): msg = ' "4kB argument truncated by %d characters"' % oversize return str_[0 : -(oversize + len(msg))] + msg - def _as_list_and_string(self): - # type: () -> Tuple[list[str], str] - + def _as_list_and_string(self) -> Tuple[List[str], str]: total_list = self.env_vars + [self.binary] + self.arguments truncated_str = self.truncate_string(shjoin(total_list)) truncated_list = shlex.split(truncated_str) @@ -290,8 +312,10 @@ def as_string(self): return str_res -def unpatch(): - # type: () -> None +def unpatch() -> None: + import os # nosec + import subprocess # nosec + trace_utils.unwrap(os, "system") trace_utils.unwrap(os, "_spawnvef") trace_utils.unwrap(subprocess.Popen, "__init__") @@ -299,13 +323,13 @@ def unpatch(): SubprocessCmdLine._clear_cache() - os._datadog_patch = False - subprocess._datadog_patch = False - @trace_utils.with_traced_module def _traced_ossystem(module, pin, wrapped, instance, args, kwargs): try: + if isinstance(args[0], str): + for callback in _STR_CALLBACKS.values(): + callback(args[0]) shellcmd = SubprocessCmdLine(args[0], shell=True) # nosec with pin.tracer.trace(COMMANDS.SPAN_NAME, resource=shellcmd.binary, span_type=SpanTypes.SYSTEM) as span: @@ -342,6 +366,10 @@ def _traced_fork(module, pin, wrapped, instance, args, kwargs): def _traced_osspawn(module, pin, wrapped, instance, args, kwargs): try: mode, file, func_args, _, _ = args + if isinstance(func_args, (list, tuple, str)): + commands = [file] + list(func_args) + for callback in _LST_CALLBACKS.values(): + callback(commands) shellcmd = SubprocessCmdLine(func_args, shell=False) with pin.tracer.trace(COMMANDS.SPAN_NAME, resource=shellcmd.binary, span_type=SpanTypes.SYSTEM) as span: @@ -366,6 +394,13 @@ def _traced_osspawn(module, pin, wrapped, instance, args, kwargs): def _traced_subprocess_init(module, pin, wrapped, instance, args, kwargs): try: cmd_args = args[0] if len(args) else kwargs["args"] + if isinstance(cmd_args, (list, tuple, str)): + if kwargs.get("shell", False): + for callback in _STR_CALLBACKS.values(): + callback(cmd_args) + else: + for callback in _LST_CALLBACKS.values(): + callback(cmd_args) cmd_args_list = shlex.split(cmd_args) if isinstance(cmd_args, str) else cmd_args is_shell = kwargs.get("shell", False) shellcmd = SubprocessCmdLine(cmd_args_list, shell=is_shell) # nosec diff --git a/ddtrace/contrib/internal/tornado/application.py b/ddtrace/contrib/internal/tornado/application.py index 3a7dc832b5e..86794689835 100644 --- a/ddtrace/contrib/internal/tornado/application.py +++ b/ddtrace/contrib/internal/tornado/application.py @@ -55,4 +55,9 @@ def tracer_config(__init__, app, args, kwargs): tracer.set_tags(tags) # configure the PIN object for template rendering - ddtrace.Pin(service=service, tracer=tracer).onto(template) + # Required for backwards compatibility. Remove the else clause when + # the `ddtrace.Pin` object no longer accepts the Pin argument. + if tracer is ddtrace.tracer: + ddtrace.trace.Pin(service=service).onto(template) + else: + ddtrace.trace.Pin(service=service, tracer=tracer).onto(template) diff --git a/ddtrace/contrib/internal/tornado/template.py b/ddtrace/contrib/internal/tornado/template.py index 5d94d2a358c..a47ee53b9e4 100644 --- a/ddtrace/contrib/internal/tornado/template.py +++ b/ddtrace/contrib/internal/tornado/template.py @@ -1,9 +1,9 @@ from tornado import template -from ddtrace import Pin from ddtrace import config from ddtrace.ext import SpanTypes from ddtrace.internal.constants import COMPONENT +from ddtrace.trace import Pin def generate(func, renderer, args, kwargs): diff --git a/ddtrace/contrib/internal/unittest/__init__.py b/ddtrace/contrib/internal/unittest/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ddtrace/contrib/internal/unittest/constants.py b/ddtrace/contrib/internal/unittest/constants.py new file mode 100644 index 00000000000..dc58863a2a5 --- /dev/null +++ b/ddtrace/contrib/internal/unittest/constants.py @@ -0,0 +1,8 @@ +COMPONENT_VALUE = "unittest" +FRAMEWORK = "unittest" +KIND = "test" + +TEST_OPERATION_NAME = "unittest.test" +SUITE_OPERATION_NAME = "unittest.test_suite" +SESSION_OPERATION_NAME = "unittest.test_session" +MODULE_OPERATION_NAME = "unittest.test_module" diff --git a/ddtrace/contrib/internal/unittest/patch.py b/ddtrace/contrib/internal/unittest/patch.py new file mode 100644 index 00000000000..74ce8d1d6a0 --- /dev/null +++ b/ddtrace/contrib/internal/unittest/patch.py @@ -0,0 +1,868 @@ +import inspect +import os +from typing import Union +import unittest + +import wrapt + +import ddtrace +from ddtrace import config +from ddtrace.constants import SPAN_KIND +from ddtrace.contrib.internal.coverage.data import _coverage_data +from ddtrace.contrib.internal.coverage.patch import patch as patch_coverage +from ddtrace.contrib.internal.coverage.patch import run_coverage_report +from ddtrace.contrib.internal.coverage.patch import unpatch as unpatch_coverage +from ddtrace.contrib.internal.coverage.utils import _is_coverage_invoked_by_coverage_run +from ddtrace.contrib.internal.coverage.utils import _is_coverage_patched +from ddtrace.contrib.internal.unittest.constants import COMPONENT_VALUE +from ddtrace.contrib.internal.unittest.constants import FRAMEWORK +from ddtrace.contrib.internal.unittest.constants import KIND +from ddtrace.contrib.internal.unittest.constants import MODULE_OPERATION_NAME +from ddtrace.contrib.internal.unittest.constants import SESSION_OPERATION_NAME +from ddtrace.contrib.internal.unittest.constants import SUITE_OPERATION_NAME +from ddtrace.ext import SpanTypes +from ddtrace.ext import test +from ddtrace.ext.ci import RUNTIME_VERSION +from ddtrace.ext.ci import _get_runtime_and_os_metadata +from ddtrace.internal.ci_visibility import CIVisibility as _CIVisibility +from ddtrace.internal.ci_visibility.constants import EVENT_TYPE as _EVENT_TYPE +from ddtrace.internal.ci_visibility.constants import ITR_CORRELATION_ID_TAG_NAME +from ddtrace.internal.ci_visibility.constants import ITR_UNSKIPPABLE_REASON +from ddtrace.internal.ci_visibility.constants import MODULE_ID as _MODULE_ID +from ddtrace.internal.ci_visibility.constants import MODULE_TYPE as _MODULE_TYPE +from ddtrace.internal.ci_visibility.constants import SESSION_ID as _SESSION_ID +from ddtrace.internal.ci_visibility.constants import SESSION_TYPE as _SESSION_TYPE +from ddtrace.internal.ci_visibility.constants import SKIPPED_BY_ITR_REASON +from ddtrace.internal.ci_visibility.constants import SUITE_ID as _SUITE_ID +from ddtrace.internal.ci_visibility.constants import SUITE_TYPE as _SUITE_TYPE +from ddtrace.internal.ci_visibility.constants import TEST +from ddtrace.internal.ci_visibility.coverage import _module_has_dd_coverage_enabled +from ddtrace.internal.ci_visibility.coverage import _report_coverage_to_span +from ddtrace.internal.ci_visibility.coverage import _start_coverage +from ddtrace.internal.ci_visibility.coverage import _stop_coverage +from ddtrace.internal.ci_visibility.coverage import _switch_coverage_context +from ddtrace.internal.ci_visibility.utils import _add_pct_covered_to_span +from ddtrace.internal.ci_visibility.utils import _add_start_end_source_file_path_data_to_span +from ddtrace.internal.ci_visibility.utils import _generate_fully_qualified_test_name +from ddtrace.internal.ci_visibility.utils import get_relative_or_absolute_path_for_path +from ddtrace.internal.constants import COMPONENT +from ddtrace.internal.logger import get_logger +from ddtrace.internal.utils.formats import asbool +from ddtrace.internal.utils.wrappers import unwrap as _u + + +log = get_logger(__name__) +_global_skipped_elements = 0 + +# unittest default settings +config._add( + "unittest", + dict( + _default_service="unittest", + operation_name=os.getenv("DD_UNITTEST_OPERATION_NAME", default="unittest.test"), + strict_naming=asbool(os.getenv("DD_CIVISIBILITY_UNITTEST_STRICT_NAMING", default=True)), + ), +) + + +def get_version(): + # type: () -> str + return "" + + +def _enable_unittest_if_not_started(): + _initialize_unittest_data() + if _CIVisibility.enabled: + return + _CIVisibility.enable(config=ddtrace.config.unittest) + + +def _initialize_unittest_data(): + if not hasattr(_CIVisibility, "_unittest_data"): + _CIVisibility._unittest_data = {} + if "suites" not in _CIVisibility._unittest_data: + _CIVisibility._unittest_data["suites"] = {} + if "modules" not in _CIVisibility._unittest_data: + _CIVisibility._unittest_data["modules"] = {} + if "unskippable_tests" not in _CIVisibility._unittest_data: + _CIVisibility._unittest_data["unskippable_tests"] = set() + + +def _set_tracer(tracer: ddtrace.tracer): + """Manually sets the tracer instance to `unittest.`""" + unittest._datadog_tracer = tracer + + +def _is_test_coverage_enabled(test_object) -> bool: + return _CIVisibility._instance._collect_coverage_enabled and not _is_skipped_test(test_object) + + +def _is_skipped_test(test_object) -> bool: + testMethod = getattr(test_object, test_object._testMethodName, "") + return ( + (hasattr(test_object.__class__, "__unittest_skip__") and test_object.__class__.__unittest_skip__) + or (hasattr(testMethod, "__unittest_skip__") and testMethod.__unittest_skip__) + or _is_skipped_by_itr(test_object) + ) + + +def _is_skipped_by_itr(test_object) -> bool: + return hasattr(test_object, "_dd_itr_skip") and test_object._dd_itr_skip + + +def _should_be_skipped_by_itr(args: tuple, test_module_suite_path: str, test_name: str, test_object) -> bool: + return ( + len(args) + and _CIVisibility._instance._should_skip_path(test_module_suite_path, test_name) + and not _is_skipped_test(test_object) + ) + + +def _is_marked_as_unskippable(test_object) -> bool: + test_suite_name = _extract_suite_name_from_test_method(test_object) + test_name = _extract_test_method_name(test_object) + test_module_path = _extract_module_file_path(test_object) + test_module_suite_name = _generate_fully_qualified_test_name(test_module_path, test_suite_name, test_name) + return ( + hasattr(_CIVisibility, "_unittest_data") + and test_module_suite_name in _CIVisibility._unittest_data["unskippable_tests"] + ) + + +def _update_skipped_elements_and_set_tags(test_module_span: ddtrace.Span, test_session_span: ddtrace.Span): + global _global_skipped_elements + _global_skipped_elements += 1 + + test_module_span._metrics[test.ITR_TEST_SKIPPING_COUNT] += 1 + test_module_span.set_tag_str(test.ITR_TEST_SKIPPING_TESTS_SKIPPED, "true") + test_module_span.set_tag_str(test.ITR_DD_CI_ITR_TESTS_SKIPPED, "true") + + test_session_span.set_tag_str(test.ITR_TEST_SKIPPING_TESTS_SKIPPED, "true") + test_session_span.set_tag_str(test.ITR_DD_CI_ITR_TESTS_SKIPPED, "true") + + +def _store_test_span(item, span: ddtrace.Span): + """Store datadog span at `unittest` test instance.""" + item._datadog_span = span + + +def _store_module_identifier(test_object: unittest.TextTestRunner): + """Store module identifier at `unittest` module instance, this is useful to classify event types.""" + if hasattr(test_object, "test") and hasattr(test_object.test, "_tests"): + for module in test_object.test._tests: + if len(module._tests) and _extract_module_name_from_module(module): + _set_identifier(module, "module") + + +def _store_suite_identifier(module): + """Store suite identifier at `unittest` suite instance, this is useful to classify event types.""" + if hasattr(module, "_tests"): + for suite in module._tests: + if len(suite._tests) and _extract_module_name_from_module(suite): + _set_identifier(suite, "suite") + + +def _is_test(item) -> bool: + if ( + type(item) == unittest.TestSuite + or not hasattr(item, "_testMethodName") + or (ddtrace.config.unittest.strict_naming and not item._testMethodName.startswith("test")) + ): + return False + return True + + +def _extract_span(item) -> Union[ddtrace.Span, None]: + return getattr(item, "_datadog_span", None) + + +def _extract_command_name_from_session(session: unittest.TextTestRunner) -> str: + if not hasattr(session, "progName"): + return "python -m unittest" + return getattr(session, "progName", "") + + +def _extract_test_method_name(test_object) -> str: + """Extract test method name from `unittest` instance.""" + return getattr(test_object, "_testMethodName", "") + + +def _extract_session_span() -> Union[ddtrace.Span, None]: + return getattr(_CIVisibility, "_datadog_session_span", None) + + +def _extract_module_span(module_identifier: str) -> Union[ddtrace.Span, None]: + if hasattr(_CIVisibility, "_unittest_data") and module_identifier in _CIVisibility._unittest_data["modules"]: + return _CIVisibility._unittest_data["modules"][module_identifier].get("module_span") + return None + + +def _extract_suite_span(suite_identifier: str) -> Union[ddtrace.Span, None]: + if hasattr(_CIVisibility, "_unittest_data") and suite_identifier in _CIVisibility._unittest_data["suites"]: + return _CIVisibility._unittest_data["suites"][suite_identifier].get("suite_span") + return None + + +def _update_status_item(item: ddtrace.Span, status: str): + """ + Sets the status for each Span implementing the test FAIL logic override. + """ + existing_status = item.get_tag(test.STATUS) + if existing_status and (status == test.Status.SKIP.value or existing_status == test.Status.FAIL.value): + return None + item.set_tag_str(test.STATUS, status) + return None + + +def _extract_suite_name_from_test_method(item) -> str: + item_type = type(item) + return getattr(item_type, "__name__", "") + + +def _extract_module_name_from_module(item) -> str: + if _is_test(item): + return type(item).__module__ + return "" + + +def _extract_test_reason(item: tuple) -> str: + """ + Given a tuple of type [test_class, str], it returns the test failure/skip reason + """ + return item[1] + + +def _extract_test_file_name(item) -> str: + return os.path.basename(inspect.getfile(item.__class__)) + + +def _extract_module_file_path(item) -> str: + if _is_test(item): + try: + test_module_object = inspect.getfile(item.__class__) + except TypeError: + log.debug( + "Tried to collect module file path but it is a built-in Python function", + ) + return "" + return get_relative_or_absolute_path_for_path(test_module_object, os.getcwd()) + + return "" + + +def _generate_test_resource(suite_name: str, test_name: str) -> str: + return "{}.{}".format(suite_name, test_name) + + +def _generate_suite_resource(test_suite: str) -> str: + return "{}".format(test_suite) + + +def _generate_module_resource(test_module: str) -> str: + return "{}".format(test_module) + + +def _generate_session_resource(test_command: str) -> str: + return "{}".format(test_command) + + +def _set_test_skipping_tags_to_span(span: ddtrace.Span): + span.set_tag_str(test.ITR_TEST_SKIPPING_ENABLED, "true") + span.set_tag_str(test.ITR_TEST_SKIPPING_TYPE, TEST) + span.set_tag_str(test.ITR_TEST_SKIPPING_TESTS_SKIPPED, "false") + span.set_tag_str(test.ITR_DD_CI_ITR_TESTS_SKIPPED, "false") + span.set_tag_str(test.ITR_FORCED_RUN, "false") + span.set_tag_str(test.ITR_UNSKIPPABLE, "false") + + +def _set_identifier(item, name: str): + """ + Adds an event type classification to a `unittest` test. + """ + item._datadog_object = name + + +def _is_valid_result(instance: unittest.TextTestRunner, args: tuple) -> bool: + return instance and isinstance(instance, unittest.runner.TextTestResult) and args + + +def _is_valid_test_call(kwargs: dict) -> bool: + """ + Validates that kwargs is empty to ensure that `unittest` is running a test + """ + return not len(kwargs) + + +def _is_valid_module_suite_call(func) -> bool: + """ + Validates that the mocked function is an actual function from `unittest` + """ + return type(func).__name__ == "method" or type(func).__name__ == "instancemethod" + + +def _is_invoked_by_cli(instance: unittest.TextTestRunner) -> bool: + return ( + hasattr(instance, "progName") + or hasattr(_CIVisibility, "_datadog_entry") + and _CIVisibility._datadog_entry == "cli" + ) + + +def _extract_test_method_object(test_object): + if hasattr(test_object, "_testMethodName"): + return getattr(test_object, test_object._testMethodName, None) + return None + + +def _is_invoked_by_text_test_runner() -> bool: + return hasattr(_CIVisibility, "_datadog_entry") and _CIVisibility._datadog_entry == "TextTestRunner" + + +def _generate_module_suite_path(test_module_path: str, test_suite_name: str) -> str: + return "{}.{}".format(test_module_path, test_suite_name) + + +def _populate_suites_and_modules(test_objects: list, seen_suites: dict, seen_modules: dict): + """ + Discovers suites and modules and initializes the seen_suites and seen_modules dictionaries. + """ + if not hasattr(test_objects, "__iter__"): + return + for test_object in test_objects: + if not _is_test(test_object): + _populate_suites_and_modules(test_object, seen_suites, seen_modules) + continue + test_module_path = _extract_module_file_path(test_object) + test_suite_name = _extract_suite_name_from_test_method(test_object) + test_module_suite_path = _generate_module_suite_path(test_module_path, test_suite_name) + if test_module_path not in seen_modules: + seen_modules[test_module_path] = { + "module_span": None, + "remaining_suites": 0, + } + if test_module_suite_path not in seen_suites: + seen_suites[test_module_suite_path] = { + "suite_span": None, + "remaining_tests": 0, + } + + seen_modules[test_module_path]["remaining_suites"] += 1 + + seen_suites[test_module_suite_path]["remaining_tests"] += 1 + + +def _finish_remaining_suites_and_modules(seen_suites: dict, seen_modules: dict): + """ + Forces all suite and module spans to finish and updates their statuses. + """ + for suite in seen_suites.values(): + test_suite_span = suite["suite_span"] + if test_suite_span and not test_suite_span.finished: + _finish_span(test_suite_span) + + for module in seen_modules.values(): + test_module_span = module["module_span"] + if test_module_span and not test_module_span.finished: + _finish_span(test_module_span) + del _CIVisibility._unittest_data + + +def _update_remaining_suites_and_modules( + test_module_suite_path: str, test_module_path: str, test_module_span: ddtrace.Span, test_suite_span: ddtrace.Span +): + """ + Updates the remaining test suite and test counter and finishes spans when these have finished their execution. + """ + suite_dict = _CIVisibility._unittest_data["suites"][test_module_suite_path] + modules_dict = _CIVisibility._unittest_data["modules"][test_module_path] + + suite_dict["remaining_tests"] -= 1 + if suite_dict["remaining_tests"] == 0: + modules_dict["remaining_suites"] -= 1 + _finish_span(test_suite_span) + if modules_dict["remaining_suites"] == 0: + _finish_span(test_module_span) + + +def _update_test_skipping_count_span(span: ddtrace.Span): + if _CIVisibility.test_skipping_enabled(): + span.set_metric(test.ITR_TEST_SKIPPING_COUNT, _global_skipped_elements) + + +def _extract_skip_if_reason(args, kwargs): + if len(args) >= 2: + return _extract_test_reason(args) + elif kwargs and "reason" in kwargs: + return kwargs["reason"] + return "" + + +def patch(): + """ + Patch the instrumented methods from unittest + """ + if getattr(unittest, "_datadog_patch", False) or _CIVisibility.enabled: + return + _initialize_unittest_data() + + unittest._datadog_patch = True + + _w = wrapt.wrap_function_wrapper + + _w(unittest, "TextTestResult.addSuccess", add_success_test_wrapper) + _w(unittest, "TextTestResult.addFailure", add_failure_test_wrapper) + _w(unittest, "TextTestResult.addError", add_failure_test_wrapper) + _w(unittest, "TextTestResult.addSkip", add_skip_test_wrapper) + _w(unittest, "TextTestResult.addExpectedFailure", add_xfail_test_wrapper) + _w(unittest, "TextTestResult.addUnexpectedSuccess", add_xpass_test_wrapper) + _w(unittest, "skipIf", skip_if_decorator) + _w(unittest, "TestCase.run", handle_test_wrapper) + _w(unittest, "TestSuite.run", collect_text_test_runner_session) + _w(unittest, "TextTestRunner.run", handle_text_test_runner_wrapper) + _w(unittest, "TestProgram.runTests", handle_cli_run) + + +def unpatch(): + """ + Undo patched instrumented methods from unittest + """ + if not getattr(unittest, "_datadog_patch", False): + return + + _u(unittest.TextTestResult, "addSuccess") + _u(unittest.TextTestResult, "addFailure") + _u(unittest.TextTestResult, "addError") + _u(unittest.TextTestResult, "addSkip") + _u(unittest.TextTestResult, "addExpectedFailure") + _u(unittest.TextTestResult, "addUnexpectedSuccess") + _u(unittest, "skipIf") + _u(unittest.TestSuite, "run") + _u(unittest.TestCase, "run") + _u(unittest.TextTestRunner, "run") + _u(unittest.TestProgram, "runTests") + + unittest._datadog_patch = False + _CIVisibility.disable() + + +def _set_test_span_status(test_item, status: str, exc_info: str = None, skip_reason: str = None): + span = _extract_span(test_item) + if not span: + log.debug("Tried setting test result for test but could not find span for %s", test_item) + return None + span.set_tag_str(test.STATUS, status) + if exc_info: + span.set_exc_info(exc_info[0], exc_info[1], exc_info[2]) + if status == test.Status.SKIP.value: + span.set_tag_str(test.SKIP_REASON, skip_reason) + + +def _set_test_xpass_xfail_result(test_item, result: str): + """ + Sets `test.result` and `test.status` to a XFAIL or XPASS test. + """ + span = _extract_span(test_item) + if not span: + log.debug("Tried setting test result for an xpass or xfail test but could not find span for %s", test_item) + return None + span.set_tag_str(test.RESULT, result) + status = span.get_tag(test.STATUS) + if result == test.Status.XFAIL.value: + if status == test.Status.PASS.value: + span.set_tag_str(test.STATUS, test.Status.FAIL.value) + elif status == test.Status.FAIL.value: + span.set_tag_str(test.STATUS, test.Status.PASS.value) + + +def add_success_test_wrapper(func, instance: unittest.TextTestRunner, args: tuple, kwargs: dict): + if _is_valid_result(instance, args): + _set_test_span_status(test_item=args[0], status=test.Status.PASS.value) + + return func(*args, **kwargs) + + +def add_failure_test_wrapper(func, instance: unittest.TextTestRunner, args: tuple, kwargs: dict): + if _is_valid_result(instance, args): + _set_test_span_status(test_item=args[0], exc_info=_extract_test_reason(args), status=test.Status.FAIL.value) + + return func(*args, **kwargs) + + +def add_xfail_test_wrapper(func, instance: unittest.TextTestRunner, args: tuple, kwargs: dict): + if _is_valid_result(instance, args): + _set_test_xpass_xfail_result(test_item=args[0], result=test.Status.XFAIL.value) + + return func(*args, **kwargs) + + +def add_skip_test_wrapper(func, instance: unittest.TextTestRunner, args: tuple, kwargs: dict): + if _is_valid_result(instance, args): + _set_test_span_status(test_item=args[0], skip_reason=_extract_test_reason(args), status=test.Status.SKIP.value) + + return func(*args, **kwargs) + + +def add_xpass_test_wrapper(func, instance, args: tuple, kwargs: dict): + if _is_valid_result(instance, args): + _set_test_xpass_xfail_result(test_item=args[0], result=test.Status.XPASS.value) + + return func(*args, **kwargs) + + +def _mark_test_as_unskippable(obj): + test_name = obj.__name__ + test_suite_name = str(obj).split(".")[0].split()[1] + test_module_path = get_relative_or_absolute_path_for_path(obj.__code__.co_filename, os.getcwd()) + test_module_suite_name = _generate_fully_qualified_test_name(test_module_path, test_suite_name, test_name) + _CIVisibility._unittest_data["unskippable_tests"].add(test_module_suite_name) + return obj + + +def _using_unskippable_decorator(args, kwargs): + return args[0] is False and _extract_skip_if_reason(args, kwargs) == ITR_UNSKIPPABLE_REASON + + +def skip_if_decorator(func, instance, args: tuple, kwargs: dict): + if _using_unskippable_decorator(args, kwargs): + return _mark_test_as_unskippable + return func(*args, **kwargs) + + +def handle_test_wrapper(func, instance, args: tuple, kwargs: dict): + """ + Creates module and suite spans for `unittest` test executions. + """ + if _is_valid_test_call(kwargs) and _is_test(instance) and hasattr(_CIVisibility, "_unittest_data"): + test_name = _extract_test_method_name(instance) + test_suite_name = _extract_suite_name_from_test_method(instance) + test_module_path = _extract_module_file_path(instance) + test_module_suite_path = _generate_module_suite_path(test_module_path, test_suite_name) + test_suite_span = _extract_suite_span(test_module_suite_path) + test_module_span = _extract_module_span(test_module_path) + if test_module_span is None and test_module_path in _CIVisibility._unittest_data["modules"]: + test_module_span = _start_test_module_span(instance) + _CIVisibility._unittest_data["modules"][test_module_path]["module_span"] = test_module_span + if test_suite_span is None and test_module_suite_path in _CIVisibility._unittest_data["suites"]: + test_suite_span = _start_test_suite_span(instance) + suite_dict = _CIVisibility._unittest_data["suites"][test_module_suite_path] + suite_dict["suite_span"] = test_suite_span + if not test_module_span or not test_suite_span: + log.debug("Suite and/or module span not found for test: %s", test_name) + return func(*args, **kwargs) + with _start_test_span(instance, test_suite_span) as span: + test_session_span = _CIVisibility._datadog_session_span + root_directory = os.getcwd() + fqn_test = _generate_fully_qualified_test_name(test_module_path, test_suite_name, test_name) + + if _CIVisibility.test_skipping_enabled(): + if ITR_CORRELATION_ID_TAG_NAME in _CIVisibility._instance._itr_meta: + span.set_tag_str( + ITR_CORRELATION_ID_TAG_NAME, _CIVisibility._instance._itr_meta[ITR_CORRELATION_ID_TAG_NAME] + ) + + if _is_marked_as_unskippable(instance): + span.set_tag_str(test.ITR_UNSKIPPABLE, "true") + test_module_span.set_tag_str(test.ITR_UNSKIPPABLE, "true") + test_session_span.set_tag_str(test.ITR_UNSKIPPABLE, "true") + test_module_suite_path_without_extension = "{}/{}".format( + os.path.splitext(test_module_path)[0], test_suite_name + ) + if _should_be_skipped_by_itr(args, test_module_suite_path_without_extension, test_name, instance): + if _is_marked_as_unskippable(instance): + span.set_tag_str(test.ITR_FORCED_RUN, "true") + test_module_span.set_tag_str(test.ITR_FORCED_RUN, "true") + test_session_span.set_tag_str(test.ITR_FORCED_RUN, "true") + else: + _update_skipped_elements_and_set_tags(test_module_span, test_session_span) + instance._dd_itr_skip = True + span.set_tag_str(test.ITR_SKIPPED, "true") + span.set_tag_str(test.SKIP_REASON, SKIPPED_BY_ITR_REASON) + + if _is_skipped_by_itr(instance): + result = args[0] + result.startTest(test=instance) + result.addSkip(test=instance, reason=SKIPPED_BY_ITR_REASON) + _set_test_span_status( + test_item=instance, skip_reason=SKIPPED_BY_ITR_REASON, status=test.Status.SKIP.value + ) + result.stopTest(test=instance) + else: + if _is_test_coverage_enabled(instance): + if not _module_has_dd_coverage_enabled(unittest, silent_mode=True): + unittest._dd_coverage = _start_coverage(root_directory) + _switch_coverage_context(unittest._dd_coverage, fqn_test) + result = func(*args, **kwargs) + _update_status_item(test_suite_span, span.get_tag(test.STATUS)) + if _is_test_coverage_enabled(instance): + _report_coverage_to_span(unittest._dd_coverage, span, root_directory) + + _update_remaining_suites_and_modules( + test_module_suite_path, test_module_path, test_module_span, test_suite_span + ) + return result + return func(*args, **kwargs) + + +def collect_text_test_runner_session(func, instance: unittest.TestSuite, args: tuple, kwargs: dict): + """ + Discovers test suites and tests for the current `unittest` `TextTestRunner` execution + """ + if not _is_valid_module_suite_call(func): + return func(*args, **kwargs) + _initialize_unittest_data() + if _is_invoked_by_text_test_runner(): + seen_suites = _CIVisibility._unittest_data["suites"] + seen_modules = _CIVisibility._unittest_data["modules"] + _populate_suites_and_modules(instance._tests, seen_suites, seen_modules) + + result = func(*args, **kwargs) + + return result + result = func(*args, **kwargs) + return result + + +def _start_test_session_span(instance) -> ddtrace.Span: + """ + Starts a test session span and sets the required tags for a `unittest` session instance. + """ + tracer = getattr(unittest, "_datadog_tracer", _CIVisibility._instance.tracer) + test_command = _extract_command_name_from_session(instance) + resource_name = _generate_session_resource(test_command) + test_session_span = tracer.trace( + SESSION_OPERATION_NAME, + service=_CIVisibility._instance._service, + span_type=SpanTypes.TEST, + resource=resource_name, + ) + test_session_span.set_tag_str(_EVENT_TYPE, _SESSION_TYPE) + test_session_span.set_tag_str(_SESSION_ID, str(test_session_span.span_id)) + + test_session_span.set_tag_str(COMPONENT, COMPONENT_VALUE) + test_session_span.set_tag_str(SPAN_KIND, KIND) + + test_session_span.set_tag_str(test.COMMAND, test_command) + test_session_span.set_tag_str(test.FRAMEWORK, FRAMEWORK) + test_session_span.set_tag_str(test.FRAMEWORK_VERSION, _get_runtime_and_os_metadata()[RUNTIME_VERSION]) + + test_session_span.set_tag_str(test.TEST_TYPE, SpanTypes.TEST) + test_session_span.set_tag_str( + test.ITR_TEST_CODE_COVERAGE_ENABLED, + "true" if _CIVisibility._instance._collect_coverage_enabled else "false", + ) + + _CIVisibility.set_test_session_name(test_command=test_command) + + if _CIVisibility.test_skipping_enabled(): + _set_test_skipping_tags_to_span(test_session_span) + else: + test_session_span.set_tag_str(test.ITR_TEST_SKIPPING_ENABLED, "false") + _store_module_identifier(instance) + if _is_coverage_invoked_by_coverage_run(): + patch_coverage() + return test_session_span + + +def _start_test_module_span(instance) -> ddtrace.Span: + """ + Starts a test module span and sets the required tags for a `unittest` module instance. + """ + tracer = getattr(unittest, "_datadog_tracer", _CIVisibility._instance.tracer) + test_session_span = _extract_session_span() + test_module_name = _extract_module_name_from_module(instance) + resource_name = _generate_module_resource(test_module_name) + test_module_span = tracer._start_span( + MODULE_OPERATION_NAME, + service=_CIVisibility._instance._service, + span_type=SpanTypes.TEST, + activate=True, + child_of=test_session_span, + resource=resource_name, + ) + test_module_span.set_tag_str(_EVENT_TYPE, _MODULE_TYPE) + test_module_span.set_tag_str(_SESSION_ID, str(test_session_span.span_id)) + test_module_span.set_tag_str(_MODULE_ID, str(test_module_span.span_id)) + + test_module_span.set_tag_str(COMPONENT, COMPONENT_VALUE) + test_module_span.set_tag_str(SPAN_KIND, KIND) + + test_module_span.set_tag_str(test.COMMAND, test_session_span.get_tag(test.COMMAND)) + test_module_span.set_tag_str(test.FRAMEWORK, FRAMEWORK) + test_module_span.set_tag_str(test.FRAMEWORK_VERSION, _get_runtime_and_os_metadata()[RUNTIME_VERSION]) + + test_module_span.set_tag_str(test.TEST_TYPE, SpanTypes.TEST) + test_module_span.set_tag_str(test.MODULE, test_module_name) + test_module_span.set_tag_str(test.MODULE_PATH, _extract_module_file_path(instance)) + test_module_span.set_tag_str( + test.ITR_TEST_CODE_COVERAGE_ENABLED, + "true" if _CIVisibility._instance._collect_coverage_enabled else "false", + ) + if _CIVisibility.test_skipping_enabled(): + _set_test_skipping_tags_to_span(test_module_span) + test_module_span.set_metric(test.ITR_TEST_SKIPPING_COUNT, 0) + else: + test_module_span.set_tag_str(test.ITR_TEST_SKIPPING_ENABLED, "false") + _store_suite_identifier(instance) + return test_module_span + + +def _start_test_suite_span(instance) -> ddtrace.Span: + """ + Starts a test suite span and sets the required tags for a `unittest` suite instance. + """ + tracer = getattr(unittest, "_datadog_tracer", _CIVisibility._instance.tracer) + test_module_path = _extract_module_file_path(instance) + test_module_span = _extract_module_span(test_module_path) + test_suite_name = _extract_suite_name_from_test_method(instance) + resource_name = _generate_suite_resource(test_suite_name) + test_suite_span = tracer._start_span( + SUITE_OPERATION_NAME, + service=_CIVisibility._instance._service, + span_type=SpanTypes.TEST, + child_of=test_module_span, + activate=True, + resource=resource_name, + ) + test_suite_span.set_tag_str(_EVENT_TYPE, _SUITE_TYPE) + test_suite_span.set_tag_str(_SESSION_ID, test_module_span.get_tag(_SESSION_ID)) + test_suite_span.set_tag_str(_SUITE_ID, str(test_suite_span.span_id)) + test_suite_span.set_tag_str(_MODULE_ID, str(test_module_span.span_id)) + + test_suite_span.set_tag_str(COMPONENT, COMPONENT_VALUE) + test_suite_span.set_tag_str(SPAN_KIND, KIND) + + test_suite_span.set_tag_str(test.COMMAND, test_module_span.get_tag(test.COMMAND)) + test_suite_span.set_tag_str(test.FRAMEWORK, FRAMEWORK) + test_suite_span.set_tag_str(test.FRAMEWORK_VERSION, _get_runtime_and_os_metadata()[RUNTIME_VERSION]) + + test_suite_span.set_tag_str(test.TEST_TYPE, SpanTypes.TEST) + test_suite_span.set_tag_str(test.SUITE, test_suite_name) + test_suite_span.set_tag_str(test.MODULE, test_module_span.get_tag(test.MODULE)) + test_suite_span.set_tag_str(test.MODULE_PATH, test_module_path) + return test_suite_span + + +def _start_test_span(instance, test_suite_span: ddtrace.Span) -> ddtrace.Span: + """ + Starts a test span and sets the required tags for a `unittest` test instance. + """ + tracer = getattr(unittest, "_datadog_tracer", _CIVisibility._instance.tracer) + test_name = _extract_test_method_name(instance) + test_method_object = _extract_test_method_object(instance) + test_suite_name = _extract_suite_name_from_test_method(instance) + resource_name = _generate_test_resource(test_suite_name, test_name) + span = tracer._start_span( + ddtrace.config.unittest.operation_name, + service=_CIVisibility._instance._service, + resource=resource_name, + span_type=SpanTypes.TEST, + child_of=test_suite_span, + activate=True, + ) + span.set_tag_str(_EVENT_TYPE, SpanTypes.TEST) + span.set_tag_str(_SESSION_ID, test_suite_span.get_tag(_SESSION_ID)) + span.set_tag_str(_MODULE_ID, test_suite_span.get_tag(_MODULE_ID)) + span.set_tag_str(_SUITE_ID, test_suite_span.get_tag(_SUITE_ID)) + + span.set_tag_str(COMPONENT, COMPONENT_VALUE) + span.set_tag_str(SPAN_KIND, KIND) + + span.set_tag_str(test.COMMAND, test_suite_span.get_tag(test.COMMAND)) + span.set_tag_str(test.FRAMEWORK, FRAMEWORK) + span.set_tag_str(test.FRAMEWORK_VERSION, _get_runtime_and_os_metadata()[RUNTIME_VERSION]) + + span.set_tag_str(test.TYPE, SpanTypes.TEST) + span.set_tag_str(test.NAME, test_name) + span.set_tag_str(test.SUITE, test_suite_name) + span.set_tag_str(test.MODULE, test_suite_span.get_tag(test.MODULE)) + span.set_tag_str(test.MODULE_PATH, test_suite_span.get_tag(test.MODULE_PATH)) + span.set_tag_str(test.STATUS, test.Status.FAIL.value) + span.set_tag_str(test.CLASS_HIERARCHY, test_suite_name) + + _CIVisibility.set_codeowners_of(_extract_test_file_name(instance), span=span) + + _add_start_end_source_file_path_data_to_span(span, test_method_object, test_name, os.getcwd()) + + _store_test_span(instance, span) + return span + + +def _finish_span(current_span: ddtrace.Span): + """ + Finishes active span and populates span status upwards + """ + current_status = current_span.get_tag(test.STATUS) + parent_span = current_span._parent + if current_status and parent_span: + _update_status_item(parent_span, current_status) + elif not current_status: + current_span.set_tag_str(test.SUITE, test.Status.FAIL.value) + current_span.finish() + + +def _finish_test_session_span(): + _finish_remaining_suites_and_modules( + _CIVisibility._unittest_data["suites"], _CIVisibility._unittest_data["modules"] + ) + _update_test_skipping_count_span(_CIVisibility._datadog_session_span) + if _CIVisibility._instance._collect_coverage_enabled and _module_has_dd_coverage_enabled(unittest): + _stop_coverage(unittest) + if _is_coverage_patched() and _is_coverage_invoked_by_coverage_run(): + run_coverage_report() + _add_pct_covered_to_span(_coverage_data, _CIVisibility._datadog_session_span) + unpatch_coverage() + _finish_span(_CIVisibility._datadog_session_span) + + +def handle_cli_run(func, instance: unittest.TestProgram, args: tuple, kwargs: dict): + """ + Creates session span and discovers test suites and tests for the current `unittest` CLI execution + """ + if _is_invoked_by_cli(instance): + _enable_unittest_if_not_started() + for parent_module in instance.test._tests: + for module in parent_module._tests: + _populate_suites_and_modules( + module, _CIVisibility._unittest_data["suites"], _CIVisibility._unittest_data["modules"] + ) + + test_session_span = _start_test_session_span(instance) + _CIVisibility._datadog_entry = "cli" + _CIVisibility._datadog_session_span = test_session_span + + try: + result = func(*args, **kwargs) + except SystemExit as e: + if _CIVisibility.enabled and _CIVisibility._datadog_session_span and hasattr(_CIVisibility, "_unittest_data"): + _finish_test_session_span() + + raise e + return result + + +def handle_text_test_runner_wrapper(func, instance: unittest.TextTestRunner, args: tuple, kwargs: dict): + """ + Creates session span if unittest is called through the `TextTestRunner` method + """ + if _is_invoked_by_cli(instance): + return func(*args, **kwargs) + _enable_unittest_if_not_started() + _CIVisibility._datadog_entry = "TextTestRunner" + if not hasattr(_CIVisibility, "_datadog_session_span"): + _CIVisibility._datadog_session_span = _start_test_session_span(instance) + _CIVisibility._datadog_expected_sessions = 0 + _CIVisibility._datadog_finished_sessions = 0 + _CIVisibility._datadog_expected_sessions += 1 + try: + result = func(*args, **kwargs) + except SystemExit as e: + _CIVisibility._datadog_finished_sessions += 1 + if _CIVisibility._datadog_finished_sessions == _CIVisibility._datadog_expected_sessions: + _finish_test_session_span() + del _CIVisibility._datadog_session_span + raise e + _CIVisibility._datadog_finished_sessions += 1 + if _CIVisibility._datadog_finished_sessions == _CIVisibility._datadog_expected_sessions: + _finish_test_session_span() + del _CIVisibility._datadog_session_span + return result diff --git a/ddtrace/contrib/internal/urllib3/patch.py b/ddtrace/contrib/internal/urllib3/patch.py index 624dd9efbc6..6c10526c125 100644 --- a/ddtrace/contrib/internal/urllib3/patch.py +++ b/ddtrace/contrib/internal/urllib3/patch.py @@ -22,9 +22,9 @@ from ddtrace.internal.utils import get_argument_value from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.wrappers import unwrap as _u -from ddtrace.pin import Pin from ddtrace.propagation.http import HTTPPropagator from ddtrace.settings.asm import config as asm_config +from ddtrace.trace import Pin # Ports which, if set, will not be used in hostnames/service names diff --git a/ddtrace/contrib/internal/vertexai/patch.py b/ddtrace/contrib/internal/vertexai/patch.py index 2dbce060234..bc6e46903c3 100644 --- a/ddtrace/contrib/internal/vertexai/patch.py +++ b/ddtrace/contrib/internal/vertexai/patch.py @@ -13,7 +13,7 @@ from ddtrace.contrib.trace_utils import wrap from ddtrace.llmobs._integrations import VertexAIIntegration from ddtrace.llmobs._integrations.utils import extract_model_name_google -from ddtrace.pin import Pin +from ddtrace.trace import Pin config._add( diff --git a/ddtrace/contrib/internal/vertica/patch.py b/ddtrace/contrib/internal/vertica/patch.py index 8e820248f14..b365ade8c05 100644 --- a/ddtrace/contrib/internal/vertica/patch.py +++ b/ddtrace/contrib/internal/vertica/patch.py @@ -18,7 +18,7 @@ from ddtrace.internal.schema import schematize_service_name from ddtrace.internal.utils import get_argument_value from ddtrace.internal.utils.wrappers import unwrap -from ddtrace.pin import Pin +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/contrib/internal/wsgi/wsgi.py b/ddtrace/contrib/internal/wsgi/wsgi.py index da86aa8f21e..44e1646f5f9 100644 --- a/ddtrace/contrib/internal/wsgi/wsgi.py +++ b/ddtrace/contrib/internal/wsgi/wsgi.py @@ -11,10 +11,10 @@ from typing import Mapping # noqa:F401 from typing import Optional # noqa:F401 - from ddtrace import Pin # noqa:F401 from ddtrace import Span # noqa:F401 from ddtrace import Tracer # noqa:F401 from ddtrace.settings import Config # noqa:F401 + from ddtrace.trace import Pin # noqa:F401 from urllib.parse import quote diff --git a/ddtrace/contrib/internal/yaaredis/patch.py b/ddtrace/contrib/internal/yaaredis/patch.py index eeba29994f6..58c5a47bda4 100644 --- a/ddtrace/contrib/internal/yaaredis/patch.py +++ b/ddtrace/contrib/internal/yaaredis/patch.py @@ -13,7 +13,7 @@ from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.formats import stringify_cache_args from ddtrace.internal.utils.wrappers import unwrap -from ddtrace.pin import Pin +from ddtrace.trace import Pin from ddtrace.vendor.debtcollector import deprecate diff --git a/ddtrace/contrib/kafka/__init__.py b/ddtrace/contrib/kafka/__init__.py index f3cf66f6f23..355d3a99f48 100644 --- a/ddtrace/contrib/kafka/__init__.py +++ b/ddtrace/contrib/kafka/__init__.py @@ -31,7 +31,7 @@ To configure the kafka integration using the ``Pin`` API:: - from ddtrace import Pin + from ddtrace.trace import Pin from ddtrace import patch # Make sure to patch before importing confluent_kafka diff --git a/ddtrace/contrib/kombu/__init__.py b/ddtrace/contrib/kombu/__init__.py index 49f3485f2ee..1a010892a7b 100644 --- a/ddtrace/contrib/kombu/__init__.py +++ b/ddtrace/contrib/kombu/__init__.py @@ -11,7 +11,8 @@ without the whole trace being dropped. :: - from ddtrace import Pin, patch + from ddtrace import patch + from ddtrace.trace import Pin import kombu # If not patched yet, you can patch kombu specifically diff --git a/ddtrace/contrib/mariadb/__init__.py b/ddtrace/contrib/mariadb/__init__.py index e5c7139ee74..ea245ab37e7 100644 --- a/ddtrace/contrib/mariadb/__init__.py +++ b/ddtrace/contrib/mariadb/__init__.py @@ -34,7 +34,7 @@ To configure the mariadb integration on an per-connection basis use the ``Pin`` API:: - from ddtrace import Pin + from ddtrace.trace import Pin from ddtrace import patch # Make sure to patch before importing mariadb diff --git a/ddtrace/contrib/mongoengine/__init__.py b/ddtrace/contrib/mongoengine/__init__.py index 1522ac1438b..eed76b32f4c 100644 --- a/ddtrace/contrib/mongoengine/__init__.py +++ b/ddtrace/contrib/mongoengine/__init__.py @@ -3,7 +3,8 @@ ``import ddtrace.auto`` will automatically patch your mongoengine connect method to make it work. :: - from ddtrace import Pin, patch + from ddtrace import patch + from ddtrace.trace import Pin import mongoengine # If not patched yet, you can patch mongoengine specifically diff --git a/ddtrace/contrib/mysql/__init__.py b/ddtrace/contrib/mysql/__init__.py index 1c3f6064e55..5fa835be709 100644 --- a/ddtrace/contrib/mysql/__init__.py +++ b/ddtrace/contrib/mysql/__init__.py @@ -41,7 +41,7 @@ To configure the mysql integration on an per-connection basis use the ``Pin`` API:: - from ddtrace import Pin + from ddtrace.trace import Pin # Make sure to import mysql.connector and not the 'connect' function, # otherwise you won't have access to the patched version import mysql.connector diff --git a/ddtrace/contrib/mysqldb/__init__.py b/ddtrace/contrib/mysqldb/__init__.py index 17bb76b08a8..81dd4b62c37 100644 --- a/ddtrace/contrib/mysqldb/__init__.py +++ b/ddtrace/contrib/mysqldb/__init__.py @@ -55,7 +55,7 @@ # Make sure to import MySQLdb and not the 'connect' function, # otherwise you won't have access to the patched version - from ddtrace import Pin + from ddtrace.trace import Pin import MySQLdb # This will report a span with the default settings diff --git a/ddtrace/contrib/openai/__init__.py b/ddtrace/contrib/openai/__init__.py index 79a5b488834..bf482c93913 100644 --- a/ddtrace/contrib/openai/__init__.py +++ b/ddtrace/contrib/openai/__init__.py @@ -242,7 +242,8 @@ ``Pin`` API:: import openai - from ddtrace import Pin, config + from ddtrace import config + from ddtrace.trace import Pin Pin.override(openai, service="my-openai-service") """ # noqa: E501 diff --git a/ddtrace/contrib/psycopg/__init__.py b/ddtrace/contrib/psycopg/__init__.py index 48869af6cd7..a747be2310a 100644 --- a/ddtrace/contrib/psycopg/__init__.py +++ b/ddtrace/contrib/psycopg/__init__.py @@ -50,7 +50,7 @@ To configure the psycopg integration on an per-connection basis use the ``Pin`` API:: - from ddtrace import Pin + from ddtrace.trace import Pin import psycopg db = psycopg.connect(connection_factory=factory) diff --git a/ddtrace/contrib/pylibmc/__init__.py b/ddtrace/contrib/pylibmc/__init__.py index 5689fcd9070..a4ca6aa7692 100644 --- a/ddtrace/contrib/pylibmc/__init__.py +++ b/ddtrace/contrib/pylibmc/__init__.py @@ -5,7 +5,8 @@ # Be sure to import pylibmc and not pylibmc.Client directly, # otherwise you won't have access to the patched version - from ddtrace import Pin, patch + from ddtrace import patch + from ddtrace.trace import Pin import pylibmc # If not patched yet, you can patch pylibmc specifically diff --git a/ddtrace/contrib/pymemcache/__init__.py b/ddtrace/contrib/pymemcache/__init__.py index 871d8ee0f6c..894359fb007 100644 --- a/ddtrace/contrib/pymemcache/__init__.py +++ b/ddtrace/contrib/pymemcache/__init__.py @@ -2,7 +2,8 @@ ``import ddtrace.auto`` will automatically patch the pymemcache ``Client``:: - from ddtrace import Pin, patch + from ddtrace import patch + from ddtrace.trace import Pin # If not patched yet, patch pymemcache specifically patch(pymemcache=True) diff --git a/ddtrace/contrib/pymongo/__init__.py b/ddtrace/contrib/pymongo/__init__.py index 60394b6c2f3..a9363e65a04 100644 --- a/ddtrace/contrib/pymongo/__init__.py +++ b/ddtrace/contrib/pymongo/__init__.py @@ -8,7 +8,8 @@ # Be sure to import pymongo and not pymongo.MongoClient directly, # otherwise you won't have access to the patched version - from ddtrace import Pin, patch + from ddtrace import patch + from ddtrace.trace import Pin import pymongo # If not patched yet, you can patch pymongo specifically diff --git a/ddtrace/contrib/pymysql/__init__.py b/ddtrace/contrib/pymysql/__init__.py index d4b24e2cd5f..bd0e36c6be8 100644 --- a/ddtrace/contrib/pymysql/__init__.py +++ b/ddtrace/contrib/pymysql/__init__.py @@ -41,7 +41,7 @@ To configure the integration on an per-connection basis use the ``Pin`` API:: - from ddtrace import Pin + from ddtrace.trace import Pin from pymysql import connect # This will report a span with the default settings diff --git a/ddtrace/contrib/pyodbc/__init__.py b/ddtrace/contrib/pyodbc/__init__.py index 44605b7cdc9..bc7d2b3e9b3 100644 --- a/ddtrace/contrib/pyodbc/__init__.py +++ b/ddtrace/contrib/pyodbc/__init__.py @@ -41,7 +41,7 @@ To configure the integration on an per-connection basis use the ``Pin`` API:: - from ddtrace import Pin + from ddtrace.trace import Pin import pyodbc # This will report a span with the default settings diff --git a/ddtrace/contrib/pytest/__init__.py b/ddtrace/contrib/pytest/__init__.py index 30be6789602..0037949af50 100644 --- a/ddtrace/contrib/pytest/__init__.py +++ b/ddtrace/contrib/pytest/__init__.py @@ -60,27 +60,15 @@ Default: ``"pytest.test"`` """ +from ddtrace.contrib.internal.pytest.patch import get_version # noqa: F401 +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate -import os -from ddtrace import config - - -# pytest default settings -config._add( - "pytest", - dict( - _default_service="pytest", - operation_name=os.getenv("DD_PYTEST_OPERATION_NAME", default="pytest.test"), - ), +deprecate( + ("%s is deprecated" % (__name__)), + message="Avoid using this package directly. " + "Use ``ddtrace.auto`` or the ``ddtrace-run`` command to enable and configure this integration.", + category=DDTraceDeprecationWarning, + removal_version="3.0.0", ) - - -def get_version(): - # type: () -> str - import pytest - - return pytest.__version__ - - -__all__ = ["get_version"] diff --git a/ddtrace/contrib/pytest/constants.py b/ddtrace/contrib/pytest/constants.py index cc5d768fc38..695c48e5b95 100644 --- a/ddtrace/contrib/pytest/constants.py +++ b/ddtrace/contrib/pytest/constants.py @@ -1,11 +1,14 @@ -FRAMEWORK = "pytest" -KIND = "test" +from ddtrace.contrib.internal.pytest.constants import * # noqa: F403 +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate -# XFail Reason -XFAIL_REASON = "pytest.xfail.reason" +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) -ITR_MIN_SUPPORTED_VERSION = (7, 2, 0) -RETRIES_MIN_SUPPORTED_VERSION = (7, 0, 0) -EFD_MIN_SUPPORTED_VERSION = RETRIES_MIN_SUPPORTED_VERSION -ATR_MIN_SUPPORTED_VERSION = RETRIES_MIN_SUPPORTED_VERSION + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/pytest/newhooks.py b/ddtrace/contrib/pytest/newhooks.py index c44fd0a1535..b54e146fde9 100644 --- a/ddtrace/contrib/pytest/newhooks.py +++ b/ddtrace/contrib/pytest/newhooks.py @@ -1,26 +1,14 @@ -"""pytest-ddtrace hooks. +from ddtrace.contrib.internal.pytest.newhooks import * # noqa: F403 +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate -These hooks are used to provide extra data used by the Datadog CI Visibility plugin. -For example: module, suite, and test names for a given item. +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) -Note that these names will affect th display and reporting of tests in the Datadog UI, as well as information stored -the Intelligent Test Runner. Differing hook implementations may impact the behavior of Datadog CI Visibility products. -""" - -import pytest - - -@pytest.hookspec(firstresult=True) -def pytest_ddtrace_get_item_module_name(item: pytest.Item) -> str: - """Returns the module name to use when reporting CI Visibility results, should be unique""" - - -@pytest.hookspec(firstresult=True) -def pytest_ddtrace_get_item_suite_name(item: pytest.Item) -> str: - """Returns the suite name to use when reporting CI Visibility result, should be unique""" - - -@pytest.hookspec(firstresult=True) -def pytest_ddtrace_get_item_test_name(item: pytest.Item) -> str: - """Returns the test name to use when reporting CI Visibility result, should be unique""" + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/pytest/plugin.py b/ddtrace/contrib/pytest/plugin.py index a09a81be49a..05002fc74d4 100644 --- a/ddtrace/contrib/pytest/plugin.py +++ b/ddtrace/contrib/pytest/plugin.py @@ -1,166 +1,14 @@ -""" -This custom pytest plugin implements tracing for pytest by using pytest hooks. The plugin registers tracing code -to be run at specific points during pytest execution. The most important hooks used are: +from ddtrace.contrib.internal.pytest.plugin import * # noqa: F403 +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate - * pytest_sessionstart: during pytest session startup, a custom trace filter is configured to the global tracer to - only send test spans, which are generated by the plugin. - * pytest_runtest_protocol: this wraps around the execution of a pytest test function, which we trace. Most span - tags are generated and added in this function. We also store the span on the underlying pytest test item to - retrieve later when we need to report test status/result. - * pytest_runtest_makereport: this hook is used to set the test status/result tag, including skipped tests and - expected failures. -""" -from typing import Dict # noqa:F401 - -import pytest - -from ddtrace.appsec._iast._pytest_plugin import ddtrace_iast # noqa:F401 -from ddtrace.appsec._iast._utils import _is_iast_enabled -from ddtrace.contrib.pytest._utils import _USE_PLUGIN_V2 -from ddtrace.contrib.pytest._utils import _extract_span -from ddtrace.contrib.pytest._utils import _pytest_version_supports_itr - - -DDTRACE_HELP_MSG = "Enable tracing of pytest functions." -NO_DDTRACE_HELP_MSG = "Disable tracing of pytest functions." -DDTRACE_INCLUDE_CLASS_HELP_MSG = "Prepend 'ClassName.' to names of class-based tests." -PATCH_ALL_HELP_MSG = "Call ddtrace.patch_all before running tests." - - -def is_enabled(config): - """Check if the ddtrace plugin is enabled.""" - return (config.getoption("ddtrace") or config.getini("ddtrace")) and not config.getoption("no-ddtrace") - - -def pytest_addoption(parser): - """Add ddtrace options.""" - group = parser.getgroup("ddtrace") - - group._addoption( - "--ddtrace", - action="store_true", - dest="ddtrace", - default=False, - help=DDTRACE_HELP_MSG, +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, ) - group._addoption( - "--no-ddtrace", - action="store_true", - dest="no-ddtrace", - default=False, - help=NO_DDTRACE_HELP_MSG, - ) - - group._addoption( - "--ddtrace-patch-all", - action="store_true", - dest="ddtrace-patch-all", - default=False, - help=PATCH_ALL_HELP_MSG, - ) - - group._addoption( - "--ddtrace-include-class-name", - action="store_true", - dest="ddtrace-include-class-name", - default=False, - help=DDTRACE_INCLUDE_CLASS_HELP_MSG, - ) - - group._addoption( - "--ddtrace-iast-fail-tests", - action="store_true", - dest="ddtrace-iast-fail-tests", - default=False, - help=DDTRACE_INCLUDE_CLASS_HELP_MSG, - ) - - parser.addini("ddtrace", DDTRACE_HELP_MSG, type="bool") - parser.addini("no-ddtrace", DDTRACE_HELP_MSG, type="bool") - parser.addini("ddtrace-patch-all", PATCH_ALL_HELP_MSG, type="bool") - parser.addini("ddtrace-include-class-name", DDTRACE_INCLUDE_CLASS_HELP_MSG, type="bool") - if _is_iast_enabled(): - from ddtrace.appsec._iast import _iast_pytest_activation - - _iast_pytest_activation() - - -# Version-specific pytest hooks -if _USE_PLUGIN_V2: - from ddtrace.contrib.pytest._plugin_v2 import pytest_collection_finish # noqa: F401 - from ddtrace.contrib.pytest._plugin_v2 import pytest_configure as _versioned_pytest_configure - from ddtrace.contrib.pytest._plugin_v2 import pytest_ddtrace_get_item_module_name # noqa: F401 - from ddtrace.contrib.pytest._plugin_v2 import pytest_ddtrace_get_item_suite_name # noqa: F401 - from ddtrace.contrib.pytest._plugin_v2 import pytest_ddtrace_get_item_test_name # noqa: F401 - from ddtrace.contrib.pytest._plugin_v2 import pytest_load_initial_conftests # noqa: F401 - from ddtrace.contrib.pytest._plugin_v2 import pytest_report_teststatus # noqa: F401 - from ddtrace.contrib.pytest._plugin_v2 import pytest_runtest_makereport # noqa: F401 - from ddtrace.contrib.pytest._plugin_v2 import pytest_runtest_protocol # noqa: F401 - from ddtrace.contrib.pytest._plugin_v2 import pytest_sessionfinish # noqa: F401 - from ddtrace.contrib.pytest._plugin_v2 import pytest_sessionstart # noqa: F401 - from ddtrace.contrib.pytest._plugin_v2 import pytest_terminal_summary # noqa: F401 -else: - from ddtrace.contrib.pytest._plugin_v1 import pytest_collection_modifyitems # noqa: F401 - from ddtrace.contrib.pytest._plugin_v1 import pytest_configure as _versioned_pytest_configure - from ddtrace.contrib.pytest._plugin_v1 import pytest_ddtrace_get_item_module_name # noqa: F401 - from ddtrace.contrib.pytest._plugin_v1 import pytest_ddtrace_get_item_suite_name # noqa: F401 - from ddtrace.contrib.pytest._plugin_v1 import pytest_ddtrace_get_item_test_name # noqa: F401 - from ddtrace.contrib.pytest._plugin_v1 import pytest_load_initial_conftests # noqa: F401 - from ddtrace.contrib.pytest._plugin_v1 import pytest_runtest_makereport # noqa: F401 - from ddtrace.contrib.pytest._plugin_v1 import pytest_runtest_protocol # noqa: F401 - from ddtrace.contrib.pytest._plugin_v1 import pytest_sessionfinish # noqa: F401 - from ddtrace.contrib.pytest._plugin_v1 import pytest_sessionstart # noqa: F401 - - # Internal coverage is only used for ITR at the moment, so the hook is only added if the pytest version supports it - if _pytest_version_supports_itr(): - from ddtrace.contrib.pytest._plugin_v1 import pytest_terminal_summary # noqa: F401 - - -def pytest_configure(config): - config.addinivalue_line("markers", "dd_tags(**kwargs): add tags to current span") - if is_enabled(config): - _versioned_pytest_configure(config) - - -@pytest.hookimpl -def pytest_addhooks(pluginmanager): - from ddtrace.contrib.pytest import newhooks - - pluginmanager.add_hookspecs(newhooks) - - -@pytest.fixture(scope="function") -def ddspan(request): - """Return the :class:`ddtrace._trace.span.Span` instance associated with the - current test when Datadog CI Visibility is enabled. - """ - from ddtrace.internal.ci_visibility import CIVisibility as _CIVisibility - - if _CIVisibility.enabled: - return _extract_span(request.node) - - -@pytest.fixture(scope="session") -def ddtracer(): - """Return the :class:`ddtrace.tracer.Tracer` instance for Datadog CI - visibility if it is enabled, otherwise return the default Datadog tracer. - """ - import ddtrace - from ddtrace.internal.ci_visibility import CIVisibility as _CIVisibility - - if _CIVisibility.enabled: - return _CIVisibility._instance.tracer - return ddtrace.tracer - - -@pytest.fixture(scope="session", autouse=True) -def patch_all(request): - """Patch all available modules for Datadog tracing when ddtrace-patch-all - is specified in command or .ini. - """ - import ddtrace - - if request.config.getoption("ddtrace-patch-all") or request.config.getini("ddtrace-patch-all"): - ddtrace.patch_all() + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/pytest_bdd/__init__.py b/ddtrace/contrib/pytest_bdd/__init__.py index b1cc6701fda..2e91392914d 100644 --- a/ddtrace/contrib/pytest_bdd/__init__.py +++ b/ddtrace/contrib/pytest_bdd/__init__.py @@ -21,27 +21,18 @@ for more details. """ +from ddtrace.contrib.internal.pytest_bdd.patch import get_version +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate -from ddtrace import config - -# pytest-bdd default settings -config._add( - "pytest_bdd", - dict( - _default_service="pytest_bdd", - ), +deprecate( + ("%s is deprecated" % (__name__)), + message="Avoid using this package directly. " + "Use ``ddtrace.auto`` or the ``ddtrace-run`` command to enable and configure this integration.", + category=DDTraceDeprecationWarning, + removal_version="3.0.0", ) -def get_version(): - # type: () -> str - try: - import importlib.metadata as importlib_metadata - except ImportError: - import importlib_metadata # type: ignore[no-redef] - - return str(importlib_metadata.version("pytest-bdd")) - - __all__ = ["get_version"] diff --git a/ddtrace/contrib/pytest_bdd/constants.py b/ddtrace/contrib/pytest_bdd/constants.py index 2dd377f7619..9c2e907debd 100644 --- a/ddtrace/contrib/pytest_bdd/constants.py +++ b/ddtrace/contrib/pytest_bdd/constants.py @@ -1,2 +1,14 @@ -FRAMEWORK = "pytest_bdd" -STEP_KIND = "pytest_bdd.step" +from ddtrace.contrib.internal.pytest_bdd.constants import * # noqa: F403 +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + + +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/pytest_bdd/plugin.py b/ddtrace/contrib/pytest_bdd/plugin.py index 1dc714c89c5..88645368d38 100644 --- a/ddtrace/contrib/pytest_bdd/plugin.py +++ b/ddtrace/contrib/pytest_bdd/plugin.py @@ -1,20 +1,14 @@ -from ddtrace import DDTraceDeprecationWarning -from ddtrace.contrib.pytest._utils import _USE_PLUGIN_V2 -from ddtrace.contrib.pytest.plugin import is_enabled as is_ddtrace_enabled +from ddtrace.contrib.internal.pytest_bdd.plugin import * # noqa: F403 +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning from ddtrace.vendor.debtcollector import deprecate -def pytest_configure(config): - if config.pluginmanager.hasplugin("pytest-bdd") and config.pluginmanager.hasplugin("ddtrace"): - if not _USE_PLUGIN_V2: - if is_ddtrace_enabled(config): - from ._plugin import _PytestBddPlugin +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) - deprecate( - "the ddtrace.pytest_bdd plugin is deprecated", - message="it will be integrated with the main pytest ddtrace plugin", - removal_version="3.0.0", - category=DDTraceDeprecationWarning, - ) - - config.pluginmanager.register(_PytestBddPlugin(), "_datadog-pytest-bdd") + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/pytest_benchmark/__init__.py b/ddtrace/contrib/pytest_benchmark/__init__.py index e69de29bb2d..3829deeb38a 100644 --- a/ddtrace/contrib/pytest_benchmark/__init__.py +++ b/ddtrace/contrib/pytest_benchmark/__init__.py @@ -0,0 +1,11 @@ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + + +deprecate( + ("%s is deprecated" % (__name__)), + message="Avoid using this package directly. " + "Use ``ddtrace.auto`` or the ``ddtrace-run`` command to enable and configure this integration.", + category=DDTraceDeprecationWarning, + removal_version="3.0.0", +) diff --git a/ddtrace/contrib/pytest_benchmark/constants.py b/ddtrace/contrib/pytest_benchmark/constants.py index b4c4f7f5b27..522f664d4b8 100644 --- a/ddtrace/contrib/pytest_benchmark/constants.py +++ b/ddtrace/contrib/pytest_benchmark/constants.py @@ -1,79 +1,14 @@ -BENCHMARK_INFO = "benchmark.duration.info" -BENCHMARK_MEAN = "benchmark.duration.mean" -BENCHMARK_RUN = "benchmark.duration.runs" +from ddtrace.contrib.internal.pytest_benchmark.constants import * # noqa: F403 +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate -STATISTICS_HD15IQR = "benchmark.duration.statistics.hd15iqr" -STATISTICS_IQR = "benchmark.duration.statistics.iqr" -STATISTICS_IQR_OUTLIERS = "benchmark.duration.statistics.iqr_outliers" -STATISTICS_LD15IQR = "benchmark.duration.statistics.ld15iqr" -STATISTICS_MAX = "benchmark.duration.statistics.max" -STATISTICS_MEAN = "benchmark.duration.statistics.mean" -STATISTICS_MEDIAN = "benchmark.duration.statistics.median" -STATISTICS_MIN = "benchmark.duration.statistics.min" -STATISTICS_N = "benchmark.duration.statistics.n" -STATISTICS_OPS = "benchmark.duration.statistics.ops" -STATISTICS_OUTLIERS = "benchmark.duration.statistics.outliers" -STATISTICS_Q1 = "benchmark.duration.statistics.q1" -STATISTICS_Q3 = "benchmark.duration.statistics.q3" -STATISTICS_STDDEV = "benchmark.duration.statistics.std_dev" -STATISTICS_STDDEV_OUTLIERS = "benchmark.duration.statistics.std_dev_outliers" -STATISTICS_TOTAL = "benchmark.duration.statistics.total" -PLUGIN_HD15IQR = "hd15iqr" -PLUGIN_IQR = "iqr" -PLUGIN_IQR_OUTLIERS = "iqr_outliers" -PLUGIN_LD15IQR = "ld15iqr" -PLUGIN_MAX = "max" -PLUGIN_MEAN = "mean" -PLUGIN_MEDIAN = "median" -PLUGIN_MIN = "min" -PLUGIN_OPS = "ops" -PLUGIN_OUTLIERS = "outliers" -PLUGIN_Q1 = "q1" -PLUGIN_Q3 = "q3" -PLUGIN_ROUNDS = "rounds" -PLUGIN_STDDEV = "stddev" -PLUGIN_STDDEV_OUTLIERS = "stddev_outliers" -PLUGIN_TOTAL = "total" +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) -PLUGIN_METRICS = { - BENCHMARK_MEAN: PLUGIN_MEAN, - BENCHMARK_RUN: PLUGIN_ROUNDS, - STATISTICS_HD15IQR: PLUGIN_HD15IQR, - STATISTICS_IQR: PLUGIN_IQR, - STATISTICS_IQR_OUTLIERS: PLUGIN_IQR_OUTLIERS, - STATISTICS_LD15IQR: PLUGIN_LD15IQR, - STATISTICS_MAX: PLUGIN_MAX, - STATISTICS_MEAN: PLUGIN_MEAN, - STATISTICS_MEDIAN: PLUGIN_MEDIAN, - STATISTICS_MIN: PLUGIN_MIN, - STATISTICS_OPS: PLUGIN_OPS, - STATISTICS_OUTLIERS: PLUGIN_OUTLIERS, - STATISTICS_Q1: PLUGIN_Q1, - STATISTICS_Q3: PLUGIN_Q3, - STATISTICS_N: PLUGIN_ROUNDS, - STATISTICS_STDDEV: PLUGIN_STDDEV, - STATISTICS_STDDEV_OUTLIERS: PLUGIN_STDDEV_OUTLIERS, - STATISTICS_TOTAL: PLUGIN_TOTAL, -} - -PLUGIN_METRICS_V2 = { - "duration_mean": PLUGIN_MEAN, - "duration_runs": PLUGIN_ROUNDS, - "statistics_hd15iqr": PLUGIN_HD15IQR, - "statistics_iqr": PLUGIN_IQR, - "statistics_iqr_outliers": PLUGIN_IQR_OUTLIERS, - "statistics_ld15iqr": PLUGIN_LD15IQR, - "statistics_max": PLUGIN_MAX, - "statistics_mean": PLUGIN_MEAN, - "statistics_median": PLUGIN_MEDIAN, - "statistics_min": PLUGIN_MIN, - "statistics_n": PLUGIN_ROUNDS, - "statistics_ops": PLUGIN_OPS, - "statistics_outliers": PLUGIN_OUTLIERS, - "statistics_q1": PLUGIN_Q1, - "statistics_q3": PLUGIN_Q3, - "statistics_std_dev": PLUGIN_STDDEV, - "statistics_std_dev_outliers": PLUGIN_STDDEV_OUTLIERS, - "statistics_total": PLUGIN_TOTAL, -} + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/pytest_benchmark/plugin.py b/ddtrace/contrib/pytest_benchmark/plugin.py index 4cb76148dbc..7a33bbf838d 100644 --- a/ddtrace/contrib/pytest_benchmark/plugin.py +++ b/ddtrace/contrib/pytest_benchmark/plugin.py @@ -1,19 +1,14 @@ -from ddtrace import DDTraceDeprecationWarning -from ddtrace.contrib.pytest._utils import _USE_PLUGIN_V2 -from ddtrace.contrib.pytest.plugin import is_enabled as is_ddtrace_enabled +from ddtrace.contrib.internal.pytest_benchmark.plugin import * # noqa: F403 +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning from ddtrace.vendor.debtcollector import deprecate -def pytest_configure(config): - if config.pluginmanager.hasplugin("benchmark") and config.pluginmanager.hasplugin("ddtrace"): - if is_ddtrace_enabled(config): - deprecate( - "this version of the ddtrace.pytest_benchmark plugin is deprecated", - message="it will be integrated with the main pytest ddtrace plugin", - removal_version="3.0.0", - category=DDTraceDeprecationWarning, - ) - if not _USE_PLUGIN_V2: - from ._plugin import _PytestBenchmarkPlugin +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) - config.pluginmanager.register(_PytestBenchmarkPlugin(), "_datadog-pytest-benchmark") + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/redis/__init__.py b/ddtrace/contrib/redis/__init__.py index 4fddef1c742..9b498614e4b 100644 --- a/ddtrace/contrib/redis/__init__.py +++ b/ddtrace/contrib/redis/__init__.py @@ -52,10 +52,10 @@ Instance Configuration ~~~~~~~~~~~~~~~~~~~~~~ -To configure particular redis instances use the :class:`Pin ` API:: +To configure particular redis instances use the :class:`Pin ` API:: import redis - from ddtrace import Pin + from ddtrace.trace import Pin client = redis.StrictRedis(host="localhost", port=6379) diff --git a/ddtrace/contrib/rediscluster/__init__.py b/ddtrace/contrib/rediscluster/__init__.py index cb14eb9aa30..65209053b97 100644 --- a/ddtrace/contrib/rediscluster/__init__.py +++ b/ddtrace/contrib/rediscluster/__init__.py @@ -3,7 +3,8 @@ ``import ddtrace.auto`` will automatically patch your Redis Cluster client to make it work. :: - from ddtrace import Pin, patch + from ddtrace import patch + from ddtrace.trace import Pin import rediscluster # If not patched yet, you can patch redis specifically diff --git a/ddtrace/contrib/rq/__init__.py b/ddtrace/contrib/rq/__init__.py index 56ff26eb587..0ce23d17984 100644 --- a/ddtrace/contrib/rq/__init__.py +++ b/ddtrace/contrib/rq/__init__.py @@ -28,7 +28,7 @@ To override the service name for a queue:: - from ddtrace import Pin + from ddtrace.trace import Pin connection = redis.Redis() queue = rq.Queue(connection=connection) @@ -76,219 +76,15 @@ .. __: https://python-rq.org/ """ +from ddtrace.contrib.internal.rq.patch import * # noqa: F403 +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate -import os -from ddtrace import Pin -from ddtrace import config -from ddtrace.constants import SPAN_KIND -from ddtrace.internal import core -from ddtrace.internal.constants import COMPONENT -from ddtrace.internal.schema import schematize_messaging_operation -from ddtrace.internal.schema import schematize_service_name -from ddtrace.internal.schema.span_attribute_schema import SpanDirection -from ddtrace.internal.utils import get_argument_value -from ddtrace.internal.utils.formats import asbool - -from ...ext import SpanKind -from ...ext import SpanTypes -from .. import trace_utils - - -__all__ = ["patch", "unpatch", "get_version"] - - -config._add( - "rq", - dict( - distributed_tracing_enabled=asbool(os.environ.get("DD_RQ_DISTRIBUTED_TRACING_ENABLED", True)), - _default_service=schematize_service_name("rq"), - ), +deprecate( + ("%s is deprecated" % (__name__)), + message="Avoid using this package directly. " + "Use ``ddtrace.auto`` or the ``ddtrace-run`` command to enable and configure this integration.", + category=DDTraceDeprecationWarning, + removal_version="3.0.0", ) - -config._add( - "rq_worker", - dict( - distributed_tracing_enabled=asbool(os.environ.get("DD_RQ_DISTRIBUTED_TRACING_ENABLED", True)), - _default_service=schematize_service_name("rq-worker"), - ), -) - - -JOB_ID = "job.id" -QUEUE_NAME = "queue.name" -JOB_FUNC_NAME = "job.func_name" - - -def get_version(): - # type: () -> str - import rq - - return str(getattr(rq, "__version__", "")) - - -@trace_utils.with_traced_module -def traced_queue_enqueue_job(rq, pin, func, instance, args, kwargs): - job = get_argument_value(args, kwargs, 0, "f") - - func_name = job.func_name - job_inst = job.instance - job_inst_str = "%s.%s" % (job_inst.__module__, job_inst.__class__.__name__) if job_inst else "" - - if job_inst_str: - resource = "%s.%s" % (job_inst_str, func_name) - else: - resource = func_name - - with core.context_with_data( - "rq.queue.enqueue_job", - span_name=schematize_messaging_operation( - "rq.queue.enqueue_job", provider="rq", direction=SpanDirection.OUTBOUND - ), - pin=pin, - service=trace_utils.int_service(pin, config.rq), - resource=resource, - span_type=SpanTypes.WORKER, - integration_config=config.rq_worker, - tags={ - COMPONENT: config.rq.integration_name, - SPAN_KIND: SpanKind.PRODUCER, - QUEUE_NAME: instance.name, - JOB_ID: job.get_id(), - JOB_FUNC_NAME: job.func_name, - }, - ) as ctx, ctx.span: - # If the queue is_async then add distributed tracing headers to the job - if instance.is_async: - core.dispatch("rq.queue.enqueue_job", [ctx, job.meta]) - return func(*args, **kwargs) - - -@trace_utils.with_traced_module -def traced_queue_fetch_job(rq, pin, func, instance, args, kwargs): - job_id = get_argument_value(args, kwargs, 0, "job_id") - with core.context_with_data( - "rq.traced_queue_fetch_job", - span_name=schematize_messaging_operation( - "rq.queue.fetch_job", provider="rq", direction=SpanDirection.PROCESSING - ), - pin=pin, - service=trace_utils.int_service(pin, config.rq), - tags={COMPONENT: config.rq.integration_name, JOB_ID: job_id}, - ) as ctx, ctx.span: - return func(*args, **kwargs) - - -@trace_utils.with_traced_module -def traced_perform_job(rq, pin, func, instance, args, kwargs): - """Trace rq.Worker.perform_job""" - # `perform_job` is executed in a freshly forked, short-lived instance - job = get_argument_value(args, kwargs, 0, "job") - - try: - with core.context_with_data( - "rq.worker.perform_job", - span_name="rq.worker.perform_job", - service=trace_utils.int_service(pin, config.rq_worker), - pin=pin, - span_type=SpanTypes.WORKER, - resource=job.func_name, - distributed_headers_config=config.rq_worker, - distributed_headers=job.meta, - tags={COMPONENT: config.rq.integration_name, SPAN_KIND: SpanKind.CONSUMER, JOB_ID: job.get_id()}, - ) as ctx, ctx.span: - try: - return func(*args, **kwargs) - finally: - # call _after_perform_job handler for job status and origin - span_tags = {"job.status": job.get_status() or "None", "job.origin": job.origin} - job_failed = job.is_failed - core.dispatch("rq.worker.perform_job", [ctx, job_failed, span_tags]) - - finally: - # Force flush to agent since the process `os.exit()`s - # immediately after this method returns - core.dispatch("rq.worker.after.perform.job", [ctx]) - - -@trace_utils.with_traced_module -def traced_job_perform(rq, pin, func, instance, args, kwargs): - """Trace rq.Job.perform(...)""" - job = instance - - # Inherit the service name from whatever parent exists. - # eg. in a worker, a perform_job parent span will exist with the worker - # service. - with core.context_with_data( - "rq.job.perform", - span_name="rq.job.perform", - resource=job.func_name, - pin=pin, - tags={COMPONENT: config.rq.integration_name, JOB_ID: job.get_id()}, - ) as ctx, ctx.span: - return func(*args, **kwargs) - - -@trace_utils.with_traced_module -def traced_job_fetch_many(rq, pin, func, instance, args, kwargs): - """Trace rq.Job.fetch_many(...)""" - job_ids = get_argument_value(args, kwargs, 0, "job_ids") - with core.context_with_data( - "rq.job.fetch_many", - span_name=schematize_messaging_operation( - "rq.job.fetch_many", provider="rq", direction=SpanDirection.PROCESSING - ), - service=trace_utils.ext_service(pin, config.rq_worker), - pin=pin, - tags={COMPONENT: config.rq.integration_name, JOB_ID: job_ids}, - ) as ctx, ctx.span: - return func(*args, **kwargs) - - -def patch(): - # Avoid importing rq at the module level, eventually will be an import hook - import rq - - if getattr(rq, "_datadog_patch", False): - return - - Pin().onto(rq) - - # Patch rq.job.Job - Pin().onto(rq.job.Job) - trace_utils.wrap(rq.job, "Job.perform", traced_job_perform(rq.job.Job)) - - # Patch rq.queue.Queue - Pin().onto(rq.queue.Queue) - trace_utils.wrap("rq.queue", "Queue.enqueue_job", traced_queue_enqueue_job(rq)) - trace_utils.wrap("rq.queue", "Queue.fetch_job", traced_queue_fetch_job(rq)) - - # Patch rq.worker.Worker - Pin().onto(rq.worker.Worker) - trace_utils.wrap(rq.worker, "Worker.perform_job", traced_perform_job(rq)) - - rq._datadog_patch = True - - -def unpatch(): - import rq - - if not getattr(rq, "_datadog_patch", False): - return - - Pin().remove_from(rq) - - # Unpatch rq.job.Job - Pin().remove_from(rq.job.Job) - trace_utils.unwrap(rq.job.Job, "perform") - - # Unpatch rq.queue.Queue - Pin().remove_from(rq.queue.Queue) - trace_utils.unwrap(rq.queue.Queue, "enqueue_job") - trace_utils.unwrap(rq.queue.Queue, "fetch_job") - - # Unpatch rq.worker.Worker - Pin().remove_from(rq.worker.Worker) - trace_utils.unwrap(rq.worker.Worker, "perform_job") - - rq._datadog_patch = False diff --git a/ddtrace/contrib/snowflake/__init__.py b/ddtrace/contrib/snowflake/__init__.py index e675ff7a067..db137130214 100644 --- a/ddtrace/contrib/snowflake/__init__.py +++ b/ddtrace/contrib/snowflake/__init__.py @@ -9,13 +9,19 @@ The integration is not enabled automatically when using :ref:`ddtrace-run` or :ref:`import ddtrace.auto`. -Use :func:`patch()` to manually enable the integration:: +Use ``DD_TRACE_SNOWFLAKE_ENABLED=true`` to enable it with ``ddtrace-run`` - from ddtrace import patch, patch_all +or :func:`patch()` to manually enable the integration:: + + from ddtrace import patch patch(snowflake=True) + +or use :func:`patch_all()` to manually enable the integration:: + + from ddtrace import patch_all patch_all(snowflake=True) -or the ``DD_TRACE_SNOWFLAKE_ENABLED=true`` to enable it with ``ddtrace-run``. + Global Configuration @@ -45,7 +51,7 @@ To configure the integration on an per-connection basis use the ``Pin`` API:: - from ddtrace import Pin + from ddtrace.trace import Pin from snowflake.connector import connect # This will report a span with the default settings diff --git a/ddtrace/contrib/sqlalchemy/__init__.py b/ddtrace/contrib/sqlalchemy/__init__.py index c294b8c976c..cc4d1775a44 100644 --- a/ddtrace/contrib/sqlalchemy/__init__.py +++ b/ddtrace/contrib/sqlalchemy/__init__.py @@ -7,7 +7,8 @@ using the patch method that **must be called before** importing sqlalchemy:: # patch before importing `create_engine` - from ddtrace import Pin, patch + from ddtrace import patch + from ddtrace.trace import Pin patch(sqlalchemy=True) # use SQLAlchemy as usual diff --git a/ddtrace/contrib/sqlite3/__init__.py b/ddtrace/contrib/sqlite3/__init__.py index 42499cf0447..adf3ea061d6 100644 --- a/ddtrace/contrib/sqlite3/__init__.py +++ b/ddtrace/contrib/sqlite3/__init__.py @@ -41,7 +41,7 @@ To configure the integration on an per-connection basis use the ``Pin`` API:: - from ddtrace import Pin + from ddtrace.trace import Pin import sqlite3 # This will report a span with the default settings diff --git a/ddtrace/contrib/tornado/__init__.py b/ddtrace/contrib/tornado/__init__.py index ad0adef2dd5..10390e77e6e 100644 --- a/ddtrace/contrib/tornado/__init__.py +++ b/ddtrace/contrib/tornado/__init__.py @@ -76,11 +76,6 @@ def log_exception(self, typ, value, tb): 'default_service': 'my-tornado-app', 'tags': {'env': 'production'}, 'distributed_tracing': False, - 'settings': { - 'FILTERS': [ - FilterRequestsOnUrl(r'http://test\\.example\\.com'), - ], - }, }, } diff --git a/ddtrace/contrib/trace_utils.py b/ddtrace/contrib/trace_utils.py index 974ee0ff525..7d2ea0c9986 100644 --- a/ddtrace/contrib/trace_utils.py +++ b/ddtrace/contrib/trace_utils.py @@ -20,7 +20,6 @@ import wrapt -from ddtrace import Pin from ddtrace import config from ddtrace.ext import http from ddtrace.ext import net @@ -37,6 +36,7 @@ import ddtrace.internal.utils.wrappers from ddtrace.propagation.http import HTTPPropagator from ddtrace.settings.asm import config as asm_config +from ddtrace.trace import Pin if TYPE_CHECKING: # pragma: no cover @@ -577,9 +577,9 @@ def activate_distributed_headers(tracer, int_config=None, request_headers=None, context = HTTPPropagator.extract(request_headers) # Only need to activate the new context if something was propagated - if not context.trace_id: + # The new context must have one of these values in order for it to be activated + if not context.trace_id and not context._baggage and not context._span_links: return None - # Do not reactivate a context with the same trace id # DEV: An example could be nested web frameworks, when one layer already # parsed request headers and activated them. @@ -589,7 +589,14 @@ def activate_distributed_headers(tracer, int_config=None, request_headers=None, # app = Flask(__name__) # Traced via Flask instrumentation # app = DDWSGIMiddleware(app) # Extra layer on top for WSGI current_context = tracer.current_trace_context() - if current_context and current_context.trace_id == context.trace_id: + + # We accept incoming contexts with only baggage or only span_links, however if we + # already have a current_context then an incoming context not + # containing a trace_id or containing the same trace_id + # should not be activated. + if current_context and ( + not context.trace_id or (context.trace_id and context.trace_id == current_context.trace_id) + ): log.debug( "will not activate extracted Context(trace_id=%r, span_id=%r), a context with that trace id is already active", # noqa: E501 context.trace_id, diff --git a/ddtrace/contrib/trace_utils_async.py b/ddtrace/contrib/trace_utils_async.py index 63a3325db50..f58cc4e34bb 100644 --- a/ddtrace/contrib/trace_utils_async.py +++ b/ddtrace/contrib/trace_utils_async.py @@ -3,8 +3,8 @@ Note that this module should only be imported in Python 3.5+. """ -from ddtrace import Pin from ddtrace.internal.logger import get_logger +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/contrib/unittest/__init__.py b/ddtrace/contrib/unittest/__init__.py index 5180b59c959..43a1e8a740c 100644 --- a/ddtrace/contrib/unittest/__init__.py +++ b/ddtrace/contrib/unittest/__init__.py @@ -34,11 +34,18 @@ Default: ``True`` """ +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate +from ..internal.unittest.patch import get_version # noqa: F401 +from ..internal.unittest.patch import patch # noqa: F401 +from ..internal.unittest.patch import unpatch # noqa: F401 -from .patch import get_version -from .patch import patch -from .patch import unpatch - -__all__ = ["patch", "unpatch", "get_version"] +deprecate( + ("%s is deprecated" % (__name__)), + message="Avoid using this package directly. " + "Use ``ddtrace.auto`` or the ``ddtrace-run`` command to enable and configure this integration.", + category=DDTraceDeprecationWarning, + removal_version="3.0.0", +) diff --git a/ddtrace/contrib/unittest/constants.py b/ddtrace/contrib/unittest/constants.py index dc58863a2a5..fc8643d5e06 100644 --- a/ddtrace/contrib/unittest/constants.py +++ b/ddtrace/contrib/unittest/constants.py @@ -1,8 +1,14 @@ -COMPONENT_VALUE = "unittest" -FRAMEWORK = "unittest" -KIND = "test" +from ddtrace.contrib.internal.unittest.constants import * # noqa: F403 +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate -TEST_OPERATION_NAME = "unittest.test" -SUITE_OPERATION_NAME = "unittest.test_suite" -SESSION_OPERATION_NAME = "unittest.test_session" -MODULE_OPERATION_NAME = "unittest.test_module" + +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/unittest/patch.py b/ddtrace/contrib/unittest/patch.py index 2c8bdd299a6..277b3b421c6 100644 --- a/ddtrace/contrib/unittest/patch.py +++ b/ddtrace/contrib/unittest/patch.py @@ -1,868 +1,14 @@ -import inspect -import os -from typing import Union -import unittest +from ddtrace.contrib.internal.unittest.patch import * # noqa: F403 +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate -import wrapt -import ddtrace -from ddtrace import config -from ddtrace.constants import SPAN_KIND -from ddtrace.contrib.internal.coverage.data import _coverage_data -from ddtrace.contrib.internal.coverage.patch import patch as patch_coverage -from ddtrace.contrib.internal.coverage.patch import run_coverage_report -from ddtrace.contrib.internal.coverage.patch import unpatch as unpatch_coverage -from ddtrace.contrib.internal.coverage.utils import _is_coverage_invoked_by_coverage_run -from ddtrace.contrib.internal.coverage.utils import _is_coverage_patched -from ddtrace.contrib.unittest.constants import COMPONENT_VALUE -from ddtrace.contrib.unittest.constants import FRAMEWORK -from ddtrace.contrib.unittest.constants import KIND -from ddtrace.contrib.unittest.constants import MODULE_OPERATION_NAME -from ddtrace.contrib.unittest.constants import SESSION_OPERATION_NAME -from ddtrace.contrib.unittest.constants import SUITE_OPERATION_NAME -from ddtrace.ext import SpanTypes -from ddtrace.ext import test -from ddtrace.ext.ci import RUNTIME_VERSION -from ddtrace.ext.ci import _get_runtime_and_os_metadata -from ddtrace.internal.ci_visibility import CIVisibility as _CIVisibility -from ddtrace.internal.ci_visibility.constants import EVENT_TYPE as _EVENT_TYPE -from ddtrace.internal.ci_visibility.constants import ITR_CORRELATION_ID_TAG_NAME -from ddtrace.internal.ci_visibility.constants import ITR_UNSKIPPABLE_REASON -from ddtrace.internal.ci_visibility.constants import MODULE_ID as _MODULE_ID -from ddtrace.internal.ci_visibility.constants import MODULE_TYPE as _MODULE_TYPE -from ddtrace.internal.ci_visibility.constants import SESSION_ID as _SESSION_ID -from ddtrace.internal.ci_visibility.constants import SESSION_TYPE as _SESSION_TYPE -from ddtrace.internal.ci_visibility.constants import SKIPPED_BY_ITR_REASON -from ddtrace.internal.ci_visibility.constants import SUITE_ID as _SUITE_ID -from ddtrace.internal.ci_visibility.constants import SUITE_TYPE as _SUITE_TYPE -from ddtrace.internal.ci_visibility.constants import TEST -from ddtrace.internal.ci_visibility.coverage import _module_has_dd_coverage_enabled -from ddtrace.internal.ci_visibility.coverage import _report_coverage_to_span -from ddtrace.internal.ci_visibility.coverage import _start_coverage -from ddtrace.internal.ci_visibility.coverage import _stop_coverage -from ddtrace.internal.ci_visibility.coverage import _switch_coverage_context -from ddtrace.internal.ci_visibility.utils import _add_pct_covered_to_span -from ddtrace.internal.ci_visibility.utils import _add_start_end_source_file_path_data_to_span -from ddtrace.internal.ci_visibility.utils import _generate_fully_qualified_test_name -from ddtrace.internal.ci_visibility.utils import get_relative_or_absolute_path_for_path -from ddtrace.internal.constants import COMPONENT -from ddtrace.internal.logger import get_logger -from ddtrace.internal.utils.formats import asbool -from ddtrace.internal.utils.wrappers import unwrap as _u - - -log = get_logger(__name__) -_global_skipped_elements = 0 - -# unittest default settings -config._add( - "unittest", - dict( - _default_service="unittest", - operation_name=os.getenv("DD_UNITTEST_OPERATION_NAME", default="unittest.test"), - strict_naming=asbool(os.getenv("DD_CIVISIBILITY_UNITTEST_STRICT_NAMING", default=True)), - ), -) - - -def get_version(): - # type: () -> str - return "" - - -def _enable_unittest_if_not_started(): - _initialize_unittest_data() - if _CIVisibility.enabled: - return - _CIVisibility.enable(config=ddtrace.config.unittest) - - -def _initialize_unittest_data(): - if not hasattr(_CIVisibility, "_unittest_data"): - _CIVisibility._unittest_data = {} - if "suites" not in _CIVisibility._unittest_data: - _CIVisibility._unittest_data["suites"] = {} - if "modules" not in _CIVisibility._unittest_data: - _CIVisibility._unittest_data["modules"] = {} - if "unskippable_tests" not in _CIVisibility._unittest_data: - _CIVisibility._unittest_data["unskippable_tests"] = set() - - -def _set_tracer(tracer: ddtrace.tracer): - """Manually sets the tracer instance to `unittest.`""" - unittest._datadog_tracer = tracer - - -def _is_test_coverage_enabled(test_object) -> bool: - return _CIVisibility._instance._collect_coverage_enabled and not _is_skipped_test(test_object) - - -def _is_skipped_test(test_object) -> bool: - testMethod = getattr(test_object, test_object._testMethodName, "") - return ( - (hasattr(test_object.__class__, "__unittest_skip__") and test_object.__class__.__unittest_skip__) - or (hasattr(testMethod, "__unittest_skip__") and testMethod.__unittest_skip__) - or _is_skipped_by_itr(test_object) - ) - - -def _is_skipped_by_itr(test_object) -> bool: - return hasattr(test_object, "_dd_itr_skip") and test_object._dd_itr_skip - - -def _should_be_skipped_by_itr(args: tuple, test_module_suite_path: str, test_name: str, test_object) -> bool: - return ( - len(args) - and _CIVisibility._instance._should_skip_path(test_module_suite_path, test_name) - and not _is_skipped_test(test_object) - ) - - -def _is_marked_as_unskippable(test_object) -> bool: - test_suite_name = _extract_suite_name_from_test_method(test_object) - test_name = _extract_test_method_name(test_object) - test_module_path = _extract_module_file_path(test_object) - test_module_suite_name = _generate_fully_qualified_test_name(test_module_path, test_suite_name, test_name) - return ( - hasattr(_CIVisibility, "_unittest_data") - and test_module_suite_name in _CIVisibility._unittest_data["unskippable_tests"] - ) - - -def _update_skipped_elements_and_set_tags(test_module_span: ddtrace.Span, test_session_span: ddtrace.Span): - global _global_skipped_elements - _global_skipped_elements += 1 - - test_module_span._metrics[test.ITR_TEST_SKIPPING_COUNT] += 1 - test_module_span.set_tag_str(test.ITR_TEST_SKIPPING_TESTS_SKIPPED, "true") - test_module_span.set_tag_str(test.ITR_DD_CI_ITR_TESTS_SKIPPED, "true") - - test_session_span.set_tag_str(test.ITR_TEST_SKIPPING_TESTS_SKIPPED, "true") - test_session_span.set_tag_str(test.ITR_DD_CI_ITR_TESTS_SKIPPED, "true") - - -def _store_test_span(item, span: ddtrace.Span): - """Store datadog span at `unittest` test instance.""" - item._datadog_span = span - - -def _store_module_identifier(test_object: unittest.TextTestRunner): - """Store module identifier at `unittest` module instance, this is useful to classify event types.""" - if hasattr(test_object, "test") and hasattr(test_object.test, "_tests"): - for module in test_object.test._tests: - if len(module._tests) and _extract_module_name_from_module(module): - _set_identifier(module, "module") - - -def _store_suite_identifier(module): - """Store suite identifier at `unittest` suite instance, this is useful to classify event types.""" - if hasattr(module, "_tests"): - for suite in module._tests: - if len(suite._tests) and _extract_module_name_from_module(suite): - _set_identifier(suite, "suite") - - -def _is_test(item) -> bool: - if ( - type(item) == unittest.TestSuite - or not hasattr(item, "_testMethodName") - or (ddtrace.config.unittest.strict_naming and not item._testMethodName.startswith("test")) - ): - return False - return True - - -def _extract_span(item) -> Union[ddtrace.Span, None]: - return getattr(item, "_datadog_span", None) - - -def _extract_command_name_from_session(session: unittest.TextTestRunner) -> str: - if not hasattr(session, "progName"): - return "python -m unittest" - return getattr(session, "progName", "") - - -def _extract_test_method_name(test_object) -> str: - """Extract test method name from `unittest` instance.""" - return getattr(test_object, "_testMethodName", "") - - -def _extract_session_span() -> Union[ddtrace.Span, None]: - return getattr(_CIVisibility, "_datadog_session_span", None) - - -def _extract_module_span(module_identifier: str) -> Union[ddtrace.Span, None]: - if hasattr(_CIVisibility, "_unittest_data") and module_identifier in _CIVisibility._unittest_data["modules"]: - return _CIVisibility._unittest_data["modules"][module_identifier].get("module_span") - return None - - -def _extract_suite_span(suite_identifier: str) -> Union[ddtrace.Span, None]: - if hasattr(_CIVisibility, "_unittest_data") and suite_identifier in _CIVisibility._unittest_data["suites"]: - return _CIVisibility._unittest_data["suites"][suite_identifier].get("suite_span") - return None - - -def _update_status_item(item: ddtrace.Span, status: str): - """ - Sets the status for each Span implementing the test FAIL logic override. - """ - existing_status = item.get_tag(test.STATUS) - if existing_status and (status == test.Status.SKIP.value or existing_status == test.Status.FAIL.value): - return None - item.set_tag_str(test.STATUS, status) - return None - - -def _extract_suite_name_from_test_method(item) -> str: - item_type = type(item) - return getattr(item_type, "__name__", "") - - -def _extract_module_name_from_module(item) -> str: - if _is_test(item): - return type(item).__module__ - return "" - - -def _extract_test_reason(item: tuple) -> str: - """ - Given a tuple of type [test_class, str], it returns the test failure/skip reason - """ - return item[1] - - -def _extract_test_file_name(item) -> str: - return os.path.basename(inspect.getfile(item.__class__)) - - -def _extract_module_file_path(item) -> str: - if _is_test(item): - try: - test_module_object = inspect.getfile(item.__class__) - except TypeError: - log.debug( - "Tried to collect module file path but it is a built-in Python function", - ) - return "" - return get_relative_or_absolute_path_for_path(test_module_object, os.getcwd()) - - return "" - - -def _generate_test_resource(suite_name: str, test_name: str) -> str: - return "{}.{}".format(suite_name, test_name) - - -def _generate_suite_resource(test_suite: str) -> str: - return "{}".format(test_suite) - - -def _generate_module_resource(test_module: str) -> str: - return "{}".format(test_module) - - -def _generate_session_resource(test_command: str) -> str: - return "{}".format(test_command) - - -def _set_test_skipping_tags_to_span(span: ddtrace.Span): - span.set_tag_str(test.ITR_TEST_SKIPPING_ENABLED, "true") - span.set_tag_str(test.ITR_TEST_SKIPPING_TYPE, TEST) - span.set_tag_str(test.ITR_TEST_SKIPPING_TESTS_SKIPPED, "false") - span.set_tag_str(test.ITR_DD_CI_ITR_TESTS_SKIPPED, "false") - span.set_tag_str(test.ITR_FORCED_RUN, "false") - span.set_tag_str(test.ITR_UNSKIPPABLE, "false") - - -def _set_identifier(item, name: str): - """ - Adds an event type classification to a `unittest` test. - """ - item._datadog_object = name - - -def _is_valid_result(instance: unittest.TextTestRunner, args: tuple) -> bool: - return instance and isinstance(instance, unittest.runner.TextTestResult) and args - - -def _is_valid_test_call(kwargs: dict) -> bool: - """ - Validates that kwargs is empty to ensure that `unittest` is running a test - """ - return not len(kwargs) - - -def _is_valid_module_suite_call(func) -> bool: - """ - Validates that the mocked function is an actual function from `unittest` - """ - return type(func).__name__ == "method" or type(func).__name__ == "instancemethod" - - -def _is_invoked_by_cli(instance: unittest.TextTestRunner) -> bool: - return ( - hasattr(instance, "progName") - or hasattr(_CIVisibility, "_datadog_entry") - and _CIVisibility._datadog_entry == "cli" +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, ) - -def _extract_test_method_object(test_object): - if hasattr(test_object, "_testMethodName"): - return getattr(test_object, test_object._testMethodName, None) - return None - - -def _is_invoked_by_text_test_runner() -> bool: - return hasattr(_CIVisibility, "_datadog_entry") and _CIVisibility._datadog_entry == "TextTestRunner" - - -def _generate_module_suite_path(test_module_path: str, test_suite_name: str) -> str: - return "{}.{}".format(test_module_path, test_suite_name) - - -def _populate_suites_and_modules(test_objects: list, seen_suites: dict, seen_modules: dict): - """ - Discovers suites and modules and initializes the seen_suites and seen_modules dictionaries. - """ - if not hasattr(test_objects, "__iter__"): - return - for test_object in test_objects: - if not _is_test(test_object): - _populate_suites_and_modules(test_object, seen_suites, seen_modules) - continue - test_module_path = _extract_module_file_path(test_object) - test_suite_name = _extract_suite_name_from_test_method(test_object) - test_module_suite_path = _generate_module_suite_path(test_module_path, test_suite_name) - if test_module_path not in seen_modules: - seen_modules[test_module_path] = { - "module_span": None, - "remaining_suites": 0, - } - if test_module_suite_path not in seen_suites: - seen_suites[test_module_suite_path] = { - "suite_span": None, - "remaining_tests": 0, - } - - seen_modules[test_module_path]["remaining_suites"] += 1 - - seen_suites[test_module_suite_path]["remaining_tests"] += 1 - - -def _finish_remaining_suites_and_modules(seen_suites: dict, seen_modules: dict): - """ - Forces all suite and module spans to finish and updates their statuses. - """ - for suite in seen_suites.values(): - test_suite_span = suite["suite_span"] - if test_suite_span and not test_suite_span.finished: - _finish_span(test_suite_span) - - for module in seen_modules.values(): - test_module_span = module["module_span"] - if test_module_span and not test_module_span.finished: - _finish_span(test_module_span) - del _CIVisibility._unittest_data - - -def _update_remaining_suites_and_modules( - test_module_suite_path: str, test_module_path: str, test_module_span: ddtrace.Span, test_suite_span: ddtrace.Span -): - """ - Updates the remaining test suite and test counter and finishes spans when these have finished their execution. - """ - suite_dict = _CIVisibility._unittest_data["suites"][test_module_suite_path] - modules_dict = _CIVisibility._unittest_data["modules"][test_module_path] - - suite_dict["remaining_tests"] -= 1 - if suite_dict["remaining_tests"] == 0: - modules_dict["remaining_suites"] -= 1 - _finish_span(test_suite_span) - if modules_dict["remaining_suites"] == 0: - _finish_span(test_module_span) - - -def _update_test_skipping_count_span(span: ddtrace.Span): - if _CIVisibility.test_skipping_enabled(): - span.set_metric(test.ITR_TEST_SKIPPING_COUNT, _global_skipped_elements) - - -def _extract_skip_if_reason(args, kwargs): - if len(args) >= 2: - return _extract_test_reason(args) - elif kwargs and "reason" in kwargs: - return kwargs["reason"] - return "" - - -def patch(): - """ - Patch the instrumented methods from unittest - """ - if getattr(unittest, "_datadog_patch", False) or _CIVisibility.enabled: - return - _initialize_unittest_data() - - unittest._datadog_patch = True - - _w = wrapt.wrap_function_wrapper - - _w(unittest, "TextTestResult.addSuccess", add_success_test_wrapper) - _w(unittest, "TextTestResult.addFailure", add_failure_test_wrapper) - _w(unittest, "TextTestResult.addError", add_failure_test_wrapper) - _w(unittest, "TextTestResult.addSkip", add_skip_test_wrapper) - _w(unittest, "TextTestResult.addExpectedFailure", add_xfail_test_wrapper) - _w(unittest, "TextTestResult.addUnexpectedSuccess", add_xpass_test_wrapper) - _w(unittest, "skipIf", skip_if_decorator) - _w(unittest, "TestCase.run", handle_test_wrapper) - _w(unittest, "TestSuite.run", collect_text_test_runner_session) - _w(unittest, "TextTestRunner.run", handle_text_test_runner_wrapper) - _w(unittest, "TestProgram.runTests", handle_cli_run) - - -def unpatch(): - """ - Undo patched instrumented methods from unittest - """ - if not getattr(unittest, "_datadog_patch", False): - return - - _u(unittest.TextTestResult, "addSuccess") - _u(unittest.TextTestResult, "addFailure") - _u(unittest.TextTestResult, "addError") - _u(unittest.TextTestResult, "addSkip") - _u(unittest.TextTestResult, "addExpectedFailure") - _u(unittest.TextTestResult, "addUnexpectedSuccess") - _u(unittest, "skipIf") - _u(unittest.TestSuite, "run") - _u(unittest.TestCase, "run") - _u(unittest.TextTestRunner, "run") - _u(unittest.TestProgram, "runTests") - - unittest._datadog_patch = False - _CIVisibility.disable() - - -def _set_test_span_status(test_item, status: str, exc_info: str = None, skip_reason: str = None): - span = _extract_span(test_item) - if not span: - log.debug("Tried setting test result for test but could not find span for %s", test_item) - return None - span.set_tag_str(test.STATUS, status) - if exc_info: - span.set_exc_info(exc_info[0], exc_info[1], exc_info[2]) - if status == test.Status.SKIP.value: - span.set_tag_str(test.SKIP_REASON, skip_reason) - - -def _set_test_xpass_xfail_result(test_item, result: str): - """ - Sets `test.result` and `test.status` to a XFAIL or XPASS test. - """ - span = _extract_span(test_item) - if not span: - log.debug("Tried setting test result for an xpass or xfail test but could not find span for %s", test_item) - return None - span.set_tag_str(test.RESULT, result) - status = span.get_tag(test.STATUS) - if result == test.Status.XFAIL.value: - if status == test.Status.PASS.value: - span.set_tag_str(test.STATUS, test.Status.FAIL.value) - elif status == test.Status.FAIL.value: - span.set_tag_str(test.STATUS, test.Status.PASS.value) - - -def add_success_test_wrapper(func, instance: unittest.TextTestRunner, args: tuple, kwargs: dict): - if _is_valid_result(instance, args): - _set_test_span_status(test_item=args[0], status=test.Status.PASS.value) - - return func(*args, **kwargs) - - -def add_failure_test_wrapper(func, instance: unittest.TextTestRunner, args: tuple, kwargs: dict): - if _is_valid_result(instance, args): - _set_test_span_status(test_item=args[0], exc_info=_extract_test_reason(args), status=test.Status.FAIL.value) - - return func(*args, **kwargs) - - -def add_xfail_test_wrapper(func, instance: unittest.TextTestRunner, args: tuple, kwargs: dict): - if _is_valid_result(instance, args): - _set_test_xpass_xfail_result(test_item=args[0], result=test.Status.XFAIL.value) - - return func(*args, **kwargs) - - -def add_skip_test_wrapper(func, instance: unittest.TextTestRunner, args: tuple, kwargs: dict): - if _is_valid_result(instance, args): - _set_test_span_status(test_item=args[0], skip_reason=_extract_test_reason(args), status=test.Status.SKIP.value) - - return func(*args, **kwargs) - - -def add_xpass_test_wrapper(func, instance, args: tuple, kwargs: dict): - if _is_valid_result(instance, args): - _set_test_xpass_xfail_result(test_item=args[0], result=test.Status.XPASS.value) - - return func(*args, **kwargs) - - -def _mark_test_as_unskippable(obj): - test_name = obj.__name__ - test_suite_name = str(obj).split(".")[0].split()[1] - test_module_path = get_relative_or_absolute_path_for_path(obj.__code__.co_filename, os.getcwd()) - test_module_suite_name = _generate_fully_qualified_test_name(test_module_path, test_suite_name, test_name) - _CIVisibility._unittest_data["unskippable_tests"].add(test_module_suite_name) - return obj - - -def _using_unskippable_decorator(args, kwargs): - return args[0] is False and _extract_skip_if_reason(args, kwargs) == ITR_UNSKIPPABLE_REASON - - -def skip_if_decorator(func, instance, args: tuple, kwargs: dict): - if _using_unskippable_decorator(args, kwargs): - return _mark_test_as_unskippable - return func(*args, **kwargs) - - -def handle_test_wrapper(func, instance, args: tuple, kwargs: dict): - """ - Creates module and suite spans for `unittest` test executions. - """ - if _is_valid_test_call(kwargs) and _is_test(instance) and hasattr(_CIVisibility, "_unittest_data"): - test_name = _extract_test_method_name(instance) - test_suite_name = _extract_suite_name_from_test_method(instance) - test_module_path = _extract_module_file_path(instance) - test_module_suite_path = _generate_module_suite_path(test_module_path, test_suite_name) - test_suite_span = _extract_suite_span(test_module_suite_path) - test_module_span = _extract_module_span(test_module_path) - if test_module_span is None and test_module_path in _CIVisibility._unittest_data["modules"]: - test_module_span = _start_test_module_span(instance) - _CIVisibility._unittest_data["modules"][test_module_path]["module_span"] = test_module_span - if test_suite_span is None and test_module_suite_path in _CIVisibility._unittest_data["suites"]: - test_suite_span = _start_test_suite_span(instance) - suite_dict = _CIVisibility._unittest_data["suites"][test_module_suite_path] - suite_dict["suite_span"] = test_suite_span - if not test_module_span or not test_suite_span: - log.debug("Suite and/or module span not found for test: %s", test_name) - return func(*args, **kwargs) - with _start_test_span(instance, test_suite_span) as span: - test_session_span = _CIVisibility._datadog_session_span - root_directory = os.getcwd() - fqn_test = _generate_fully_qualified_test_name(test_module_path, test_suite_name, test_name) - - if _CIVisibility.test_skipping_enabled(): - if ITR_CORRELATION_ID_TAG_NAME in _CIVisibility._instance._itr_meta: - span.set_tag_str( - ITR_CORRELATION_ID_TAG_NAME, _CIVisibility._instance._itr_meta[ITR_CORRELATION_ID_TAG_NAME] - ) - - if _is_marked_as_unskippable(instance): - span.set_tag_str(test.ITR_UNSKIPPABLE, "true") - test_module_span.set_tag_str(test.ITR_UNSKIPPABLE, "true") - test_session_span.set_tag_str(test.ITR_UNSKIPPABLE, "true") - test_module_suite_path_without_extension = "{}/{}".format( - os.path.splitext(test_module_path)[0], test_suite_name - ) - if _should_be_skipped_by_itr(args, test_module_suite_path_without_extension, test_name, instance): - if _is_marked_as_unskippable(instance): - span.set_tag_str(test.ITR_FORCED_RUN, "true") - test_module_span.set_tag_str(test.ITR_FORCED_RUN, "true") - test_session_span.set_tag_str(test.ITR_FORCED_RUN, "true") - else: - _update_skipped_elements_and_set_tags(test_module_span, test_session_span) - instance._dd_itr_skip = True - span.set_tag_str(test.ITR_SKIPPED, "true") - span.set_tag_str(test.SKIP_REASON, SKIPPED_BY_ITR_REASON) - - if _is_skipped_by_itr(instance): - result = args[0] - result.startTest(test=instance) - result.addSkip(test=instance, reason=SKIPPED_BY_ITR_REASON) - _set_test_span_status( - test_item=instance, skip_reason=SKIPPED_BY_ITR_REASON, status=test.Status.SKIP.value - ) - result.stopTest(test=instance) - else: - if _is_test_coverage_enabled(instance): - if not _module_has_dd_coverage_enabled(unittest, silent_mode=True): - unittest._dd_coverage = _start_coverage(root_directory) - _switch_coverage_context(unittest._dd_coverage, fqn_test) - result = func(*args, **kwargs) - _update_status_item(test_suite_span, span.get_tag(test.STATUS)) - if _is_test_coverage_enabled(instance): - _report_coverage_to_span(unittest._dd_coverage, span, root_directory) - - _update_remaining_suites_and_modules( - test_module_suite_path, test_module_path, test_module_span, test_suite_span - ) - return result - return func(*args, **kwargs) - - -def collect_text_test_runner_session(func, instance: unittest.TestSuite, args: tuple, kwargs: dict): - """ - Discovers test suites and tests for the current `unittest` `TextTestRunner` execution - """ - if not _is_valid_module_suite_call(func): - return func(*args, **kwargs) - _initialize_unittest_data() - if _is_invoked_by_text_test_runner(): - seen_suites = _CIVisibility._unittest_data["suites"] - seen_modules = _CIVisibility._unittest_data["modules"] - _populate_suites_and_modules(instance._tests, seen_suites, seen_modules) - - result = func(*args, **kwargs) - - return result - result = func(*args, **kwargs) - return result - - -def _start_test_session_span(instance) -> ddtrace.Span: - """ - Starts a test session span and sets the required tags for a `unittest` session instance. - """ - tracer = getattr(unittest, "_datadog_tracer", _CIVisibility._instance.tracer) - test_command = _extract_command_name_from_session(instance) - resource_name = _generate_session_resource(test_command) - test_session_span = tracer.trace( - SESSION_OPERATION_NAME, - service=_CIVisibility._instance._service, - span_type=SpanTypes.TEST, - resource=resource_name, - ) - test_session_span.set_tag_str(_EVENT_TYPE, _SESSION_TYPE) - test_session_span.set_tag_str(_SESSION_ID, str(test_session_span.span_id)) - - test_session_span.set_tag_str(COMPONENT, COMPONENT_VALUE) - test_session_span.set_tag_str(SPAN_KIND, KIND) - - test_session_span.set_tag_str(test.COMMAND, test_command) - test_session_span.set_tag_str(test.FRAMEWORK, FRAMEWORK) - test_session_span.set_tag_str(test.FRAMEWORK_VERSION, _get_runtime_and_os_metadata()[RUNTIME_VERSION]) - - test_session_span.set_tag_str(test.TEST_TYPE, SpanTypes.TEST) - test_session_span.set_tag_str( - test.ITR_TEST_CODE_COVERAGE_ENABLED, - "true" if _CIVisibility._instance._collect_coverage_enabled else "false", - ) - - _CIVisibility.set_test_session_name(test_command=test_command) - - if _CIVisibility.test_skipping_enabled(): - _set_test_skipping_tags_to_span(test_session_span) - else: - test_session_span.set_tag_str(test.ITR_TEST_SKIPPING_ENABLED, "false") - _store_module_identifier(instance) - if _is_coverage_invoked_by_coverage_run(): - patch_coverage() - return test_session_span - - -def _start_test_module_span(instance) -> ddtrace.Span: - """ - Starts a test module span and sets the required tags for a `unittest` module instance. - """ - tracer = getattr(unittest, "_datadog_tracer", _CIVisibility._instance.tracer) - test_session_span = _extract_session_span() - test_module_name = _extract_module_name_from_module(instance) - resource_name = _generate_module_resource(test_module_name) - test_module_span = tracer._start_span( - MODULE_OPERATION_NAME, - service=_CIVisibility._instance._service, - span_type=SpanTypes.TEST, - activate=True, - child_of=test_session_span, - resource=resource_name, - ) - test_module_span.set_tag_str(_EVENT_TYPE, _MODULE_TYPE) - test_module_span.set_tag_str(_SESSION_ID, str(test_session_span.span_id)) - test_module_span.set_tag_str(_MODULE_ID, str(test_module_span.span_id)) - - test_module_span.set_tag_str(COMPONENT, COMPONENT_VALUE) - test_module_span.set_tag_str(SPAN_KIND, KIND) - - test_module_span.set_tag_str(test.COMMAND, test_session_span.get_tag(test.COMMAND)) - test_module_span.set_tag_str(test.FRAMEWORK, FRAMEWORK) - test_module_span.set_tag_str(test.FRAMEWORK_VERSION, _get_runtime_and_os_metadata()[RUNTIME_VERSION]) - - test_module_span.set_tag_str(test.TEST_TYPE, SpanTypes.TEST) - test_module_span.set_tag_str(test.MODULE, test_module_name) - test_module_span.set_tag_str(test.MODULE_PATH, _extract_module_file_path(instance)) - test_module_span.set_tag_str( - test.ITR_TEST_CODE_COVERAGE_ENABLED, - "true" if _CIVisibility._instance._collect_coverage_enabled else "false", - ) - if _CIVisibility.test_skipping_enabled(): - _set_test_skipping_tags_to_span(test_module_span) - test_module_span.set_metric(test.ITR_TEST_SKIPPING_COUNT, 0) - else: - test_module_span.set_tag_str(test.ITR_TEST_SKIPPING_ENABLED, "false") - _store_suite_identifier(instance) - return test_module_span - - -def _start_test_suite_span(instance) -> ddtrace.Span: - """ - Starts a test suite span and sets the required tags for a `unittest` suite instance. - """ - tracer = getattr(unittest, "_datadog_tracer", _CIVisibility._instance.tracer) - test_module_path = _extract_module_file_path(instance) - test_module_span = _extract_module_span(test_module_path) - test_suite_name = _extract_suite_name_from_test_method(instance) - resource_name = _generate_suite_resource(test_suite_name) - test_suite_span = tracer._start_span( - SUITE_OPERATION_NAME, - service=_CIVisibility._instance._service, - span_type=SpanTypes.TEST, - child_of=test_module_span, - activate=True, - resource=resource_name, - ) - test_suite_span.set_tag_str(_EVENT_TYPE, _SUITE_TYPE) - test_suite_span.set_tag_str(_SESSION_ID, test_module_span.get_tag(_SESSION_ID)) - test_suite_span.set_tag_str(_SUITE_ID, str(test_suite_span.span_id)) - test_suite_span.set_tag_str(_MODULE_ID, str(test_module_span.span_id)) - - test_suite_span.set_tag_str(COMPONENT, COMPONENT_VALUE) - test_suite_span.set_tag_str(SPAN_KIND, KIND) - - test_suite_span.set_tag_str(test.COMMAND, test_module_span.get_tag(test.COMMAND)) - test_suite_span.set_tag_str(test.FRAMEWORK, FRAMEWORK) - test_suite_span.set_tag_str(test.FRAMEWORK_VERSION, _get_runtime_and_os_metadata()[RUNTIME_VERSION]) - - test_suite_span.set_tag_str(test.TEST_TYPE, SpanTypes.TEST) - test_suite_span.set_tag_str(test.SUITE, test_suite_name) - test_suite_span.set_tag_str(test.MODULE, test_module_span.get_tag(test.MODULE)) - test_suite_span.set_tag_str(test.MODULE_PATH, test_module_path) - return test_suite_span - - -def _start_test_span(instance, test_suite_span: ddtrace.Span) -> ddtrace.Span: - """ - Starts a test span and sets the required tags for a `unittest` test instance. - """ - tracer = getattr(unittest, "_datadog_tracer", _CIVisibility._instance.tracer) - test_name = _extract_test_method_name(instance) - test_method_object = _extract_test_method_object(instance) - test_suite_name = _extract_suite_name_from_test_method(instance) - resource_name = _generate_test_resource(test_suite_name, test_name) - span = tracer._start_span( - ddtrace.config.unittest.operation_name, - service=_CIVisibility._instance._service, - resource=resource_name, - span_type=SpanTypes.TEST, - child_of=test_suite_span, - activate=True, - ) - span.set_tag_str(_EVENT_TYPE, SpanTypes.TEST) - span.set_tag_str(_SESSION_ID, test_suite_span.get_tag(_SESSION_ID)) - span.set_tag_str(_MODULE_ID, test_suite_span.get_tag(_MODULE_ID)) - span.set_tag_str(_SUITE_ID, test_suite_span.get_tag(_SUITE_ID)) - - span.set_tag_str(COMPONENT, COMPONENT_VALUE) - span.set_tag_str(SPAN_KIND, KIND) - - span.set_tag_str(test.COMMAND, test_suite_span.get_tag(test.COMMAND)) - span.set_tag_str(test.FRAMEWORK, FRAMEWORK) - span.set_tag_str(test.FRAMEWORK_VERSION, _get_runtime_and_os_metadata()[RUNTIME_VERSION]) - - span.set_tag_str(test.TYPE, SpanTypes.TEST) - span.set_tag_str(test.NAME, test_name) - span.set_tag_str(test.SUITE, test_suite_name) - span.set_tag_str(test.MODULE, test_suite_span.get_tag(test.MODULE)) - span.set_tag_str(test.MODULE_PATH, test_suite_span.get_tag(test.MODULE_PATH)) - span.set_tag_str(test.STATUS, test.Status.FAIL.value) - span.set_tag_str(test.CLASS_HIERARCHY, test_suite_name) - - _CIVisibility.set_codeowners_of(_extract_test_file_name(instance), span=span) - - _add_start_end_source_file_path_data_to_span(span, test_method_object, test_name, os.getcwd()) - - _store_test_span(instance, span) - return span - - -def _finish_span(current_span: ddtrace.Span): - """ - Finishes active span and populates span status upwards - """ - current_status = current_span.get_tag(test.STATUS) - parent_span = current_span._parent - if current_status and parent_span: - _update_status_item(parent_span, current_status) - elif not current_status: - current_span.set_tag_str(test.SUITE, test.Status.FAIL.value) - current_span.finish() - - -def _finish_test_session_span(): - _finish_remaining_suites_and_modules( - _CIVisibility._unittest_data["suites"], _CIVisibility._unittest_data["modules"] - ) - _update_test_skipping_count_span(_CIVisibility._datadog_session_span) - if _CIVisibility._instance._collect_coverage_enabled and _module_has_dd_coverage_enabled(unittest): - _stop_coverage(unittest) - if _is_coverage_patched() and _is_coverage_invoked_by_coverage_run(): - run_coverage_report() - _add_pct_covered_to_span(_coverage_data, _CIVisibility._datadog_session_span) - unpatch_coverage() - _finish_span(_CIVisibility._datadog_session_span) - - -def handle_cli_run(func, instance: unittest.TestProgram, args: tuple, kwargs: dict): - """ - Creates session span and discovers test suites and tests for the current `unittest` CLI execution - """ - if _is_invoked_by_cli(instance): - _enable_unittest_if_not_started() - for parent_module in instance.test._tests: - for module in parent_module._tests: - _populate_suites_and_modules( - module, _CIVisibility._unittest_data["suites"], _CIVisibility._unittest_data["modules"] - ) - - test_session_span = _start_test_session_span(instance) - _CIVisibility._datadog_entry = "cli" - _CIVisibility._datadog_session_span = test_session_span - - try: - result = func(*args, **kwargs) - except SystemExit as e: - if _CIVisibility.enabled and _CIVisibility._datadog_session_span and hasattr(_CIVisibility, "_unittest_data"): - _finish_test_session_span() - - raise e - return result - - -def handle_text_test_runner_wrapper(func, instance: unittest.TextTestRunner, args: tuple, kwargs: dict): - """ - Creates session span if unittest is called through the `TextTestRunner` method - """ - if _is_invoked_by_cli(instance): - return func(*args, **kwargs) - _enable_unittest_if_not_started() - _CIVisibility._datadog_entry = "TextTestRunner" - if not hasattr(_CIVisibility, "_datadog_session_span"): - _CIVisibility._datadog_session_span = _start_test_session_span(instance) - _CIVisibility._datadog_expected_sessions = 0 - _CIVisibility._datadog_finished_sessions = 0 - _CIVisibility._datadog_expected_sessions += 1 - try: - result = func(*args, **kwargs) - except SystemExit as e: - _CIVisibility._datadog_finished_sessions += 1 - if _CIVisibility._datadog_finished_sessions == _CIVisibility._datadog_expected_sessions: - _finish_test_session_span() - del _CIVisibility._datadog_session_span - raise e - _CIVisibility._datadog_finished_sessions += 1 - if _CIVisibility._datadog_finished_sessions == _CIVisibility._datadog_expected_sessions: - _finish_test_session_span() - del _CIVisibility._datadog_session_span - return result + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/vertexai/__init__.py b/ddtrace/contrib/vertexai/__init__.py index f472d28790d..80597e0c8ff 100644 --- a/ddtrace/contrib/vertexai/__init__.py +++ b/ddtrace/contrib/vertexai/__init__.py @@ -77,7 +77,8 @@ ``Pin`` API:: import vertexai - from ddtrace import Pin, config + from ddtrace import config + from ddtrace.trace import Pin Pin.override(vertexai, service="my-vertexai-service") """ # noqa: E501 diff --git a/ddtrace/contrib/vertica/__init__.py b/ddtrace/contrib/vertica/__init__.py index 3ec424fbb53..7271c1c92ad 100644 --- a/ddtrace/contrib/vertica/__init__.py +++ b/ddtrace/contrib/vertica/__init__.py @@ -27,16 +27,16 @@ To configure the Vertica integration on an instance-per-instance basis use the ``Pin`` API:: - from ddtrace import Pin, patch, Tracer + from ddtrace import patch + from ddtrace.trace import Pin patch(vertica=True) import vertica_python - custom_tracer = Tracer() conn = vertica_python.connect(**YOUR_VERTICA_CONFIG) - # override the service and tracer to be used - Pin.override(conn, service='myverticaservice', tracer=custom_tracer) + # override the service + Pin.override(conn, service='myverticaservice') """ diff --git a/ddtrace/contrib/yaaredis/__init__.py b/ddtrace/contrib/yaaredis/__init__.py index 03d76db11c0..7c0c9bd1b21 100644 --- a/ddtrace/contrib/yaaredis/__init__.py +++ b/ddtrace/contrib/yaaredis/__init__.py @@ -50,10 +50,10 @@ Instance Configuration ~~~~~~~~~~~~~~~~~~~~~~ -To configure particular yaaredis instances use the :class:`Pin ` API:: +To configure particular yaaredis instances use the :class:`Pin ` API:: import yaaredis - from ddtrace import Pin + from ddtrace.trace import Pin client = yaaredis.StrictRedis(host="localhost", port=6379) diff --git a/ddtrace/debugging/_safety.py b/ddtrace/debugging/_safety.py index 118deddef40..92b38ff6bdc 100644 --- a/ddtrace/debugging/_safety.py +++ b/ddtrace/debugging/_safety.py @@ -1,5 +1,6 @@ from inspect import CO_VARARGS from inspect import CO_VARKEYWORDS +from itertools import chain from types import FrameType from typing import Any from typing import Dict @@ -23,11 +24,11 @@ def get_args(frame: FrameType) -> Iterator[Tuple[str, Any]]: def get_locals(frame: FrameType) -> Iterator[Tuple[str, Any]]: code = frame.f_code + _locals = frame.f_locals nargs = code.co_argcount + bool(code.co_flags & CO_VARARGS) + bool(code.co_flags & CO_VARKEYWORDS) - names = code.co_varnames[nargs:] - values = (frame.f_locals.get(name) for name in names) - - return zip(names, values) + return ( + (name, _locals.get(name)) for name in chain(code.co_varnames[nargs:], code.co_freevars, code.co_cellvars) + ) # include freevars and cellvars def get_globals(frame: FrameType) -> Iterator[Tuple[str, Any]]: diff --git a/ddtrace/filters.py b/ddtrace/filters.py index a2e6884f05c..bd6367d5635 100644 --- a/ddtrace/filters.py +++ b/ddtrace/filters.py @@ -1,72 +1,10 @@ -import abc -import re -from typing import TYPE_CHECKING # noqa:F401 -from typing import List # noqa:F401 -from typing import Optional # noqa:F401 -from typing import Union # noqa:F401 +from ddtrace._trace.filters import * # noqa: F403 +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate -from ddtrace._trace.processor import TraceProcessor -from ddtrace.ext import http - -if TYPE_CHECKING: # pragma: no cover - from ddtrace._trace.span import Span # noqa:F401 - - -class TraceFilter(TraceProcessor): - @abc.abstractmethod - def process_trace(self, trace): - # type: (List[Span]) -> Optional[List[Span]] - """Processes a trace. - - None can be returned to prevent the trace from being exported. - """ - pass - - -class FilterRequestsOnUrl(TraceFilter): - r"""Filter out traces from incoming http requests based on the request's url. - - This class takes as argument a list of regular expression patterns - representing the urls to be excluded from tracing. A trace will be excluded - if its root span contains a ``http.url`` tag and if this tag matches any of - the provided regular expression using the standard python regexp match - semantic (https://docs.python.org/3/library/re.html#re.match). - - :param list regexps: a list of regular expressions (or a single string) defining - the urls that should be filtered out. - - Examples: - To filter out http calls to domain api.example.com:: - - FilterRequestsOnUrl(r'http://api\\.example\\.com') - - To filter out http calls to all first level subdomains from example.com:: - - FilterRequestOnUrl(r'http://.*+\\.example\\.com') - - To filter out calls to both http://test.example.com and http://example.com/healthcheck:: - - FilterRequestOnUrl([r'http://test\\.example\\.com', r'http://example\\.com/healthcheck']) - """ - - def __init__(self, regexps: Union[str, List[str]]): - if isinstance(regexps, str): - regexps = [regexps] - self._regexps = [re.compile(regexp) for regexp in regexps] - - def process_trace(self, trace): - # type: (List[Span]) -> Optional[List[Span]] - """ - When the filter is registered in the tracer, process_trace is called by - on each trace before it is sent to the agent, the returned value will - be fed to the next filter in the list. If process_trace returns None, - the whole trace is discarded. - """ - for span in trace: - url = span.get_tag(http.URL) - if span.parent_id is None and url is not None: - for regexp in self._regexps: - if regexp.match(url): - return None - return trace +deprecate( + "The ddtrace.filters module and the ``FilterRequestsOnUrl`` class is deprecated and will be removed.", + message="Import ``TraceFilter`` from the ddtrace.trace package.", + category=DDTraceDeprecationWarning, +) diff --git a/ddtrace/internal/ci_visibility/api/_test.py b/ddtrace/internal/ci_visibility/api/_test.py index 0f8a2efd41d..a7ab21dd459 100644 --- a/ddtrace/internal/ci_visibility/api/_test.py +++ b/ddtrace/internal/ci_visibility/api/_test.py @@ -5,7 +5,7 @@ from typing import Optional from typing import Union -from ddtrace.contrib.pytest_benchmark.constants import BENCHMARK_INFO +from ddtrace.contrib.internal.pytest_benchmark.constants import BENCHMARK_INFO from ddtrace.ext import SpanTypes from ddtrace.ext import test from ddtrace.ext.test_visibility import ITR_SKIPPING_LEVEL diff --git a/ddtrace/internal/ci_visibility/filters.py b/ddtrace/internal/ci_visibility/filters.py index c90e7324533..f1b22d97e13 100644 --- a/ddtrace/internal/ci_visibility/filters.py +++ b/ddtrace/internal/ci_visibility/filters.py @@ -8,7 +8,7 @@ from ddtrace.constants import AUTO_KEEP from ddtrace.ext import SpanTypes from ddtrace.ext import ci -from ddtrace.filters import TraceFilter +from ddtrace.trace import TraceFilter if TYPE_CHECKING: diff --git a/ddtrace/internal/ci_visibility/recorder.py b/ddtrace/internal/ci_visibility/recorder.py index 609475506d3..a69d1d72c6b 100644 --- a/ddtrace/internal/ci_visibility/recorder.py +++ b/ddtrace/internal/ci_visibility/recorder.py @@ -145,6 +145,12 @@ def _do_request(method, url, payload, headers, timeout=DEFAULT_TIMEOUT): return result +class CIVisibilityTracer(Tracer): + def __init__(self, *args, **kwargs): + # Allows for multiple instances of the civis tracer to be created without logging a warning + super(CIVisibilityTracer, self).__init__(*args, **kwargs) + + class CIVisibility(Service): _instance = None # type: Optional[CIVisibility] enabled = False @@ -166,7 +172,7 @@ def __init__(self, tracer=None, config=None, service=None): log.debug("Using _CI_DD_AGENT_URL for CI Visibility tracer: %s", env_agent_url) url = env_agent_url - self.tracer = Tracer(context_provider=CIContextProvider(), url=url) + self.tracer = CIVisibilityTracer(context_provider=CIContextProvider(), url=url) else: self.tracer = ddtrace.tracer diff --git a/ddtrace/internal/ci_visibility/telemetry/api_request.py b/ddtrace/internal/ci_visibility/telemetry/api_request.py index 076cc0cca77..77f3ea5f626 100644 --- a/ddtrace/internal/ci_visibility/telemetry/api_request.py +++ b/ddtrace/internal/ci_visibility/telemetry/api_request.py @@ -1,10 +1,10 @@ import dataclasses from typing import Optional -from ddtrace.internal.ci_visibility.telemetry.constants import CIVISIBILITY_TELEMETRY_NAMESPACE as _NAMESPACE from ddtrace.internal.ci_visibility.telemetry.constants import ERROR_TYPES from ddtrace.internal.logger import get_logger from ddtrace.internal.telemetry import telemetry_writer +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE log = get_logger(__name__) @@ -32,13 +32,15 @@ def record_api_request( error, ) - telemetry_writer.add_count_metric(_NAMESPACE, f"{metric_names.count}", 1) - telemetry_writer.add_distribution_metric(_NAMESPACE, f"{metric_names.duration}", duration) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, f"{metric_names.count}", 1) + telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, f"{metric_names.duration}", duration) if response_bytes is not None: if metric_names.response_bytes is not None: # We don't always want to record response bytes (for settings requests), so assume that no metric name # means we don't want to record it. - telemetry_writer.add_distribution_metric(_NAMESPACE, f"{metric_names.response_bytes}", response_bytes) + telemetry_writer.add_distribution_metric( + TELEMETRY_NAMESPACE.CIVISIBILITY, f"{metric_names.response_bytes}", response_bytes + ) if error is not None: record_api_request_error(metric_names.error, error) @@ -46,4 +48,4 @@ def record_api_request( def record_api_request_error(error_metric_name: str, error: ERROR_TYPES): log.debug("Recording early flake detection request error telemetry: %s", error) - telemetry_writer.add_count_metric(_NAMESPACE, error_metric_name, 1, (("error_type", error),)) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, error_metric_name, 1, (("error_type", error),)) diff --git a/ddtrace/internal/ci_visibility/telemetry/constants.py b/ddtrace/internal/ci_visibility/telemetry/constants.py index dad54511c04..191338e86e9 100644 --- a/ddtrace/internal/ci_visibility/telemetry/constants.py +++ b/ddtrace/internal/ci_visibility/telemetry/constants.py @@ -1,9 +1,6 @@ from enum import Enum -CIVISIBILITY_TELEMETRY_NAMESPACE = "civisibility" - - class ERROR_TYPES(str, Enum): TIMEOUT = "timeout" NETWORK = "network" diff --git a/ddtrace/internal/ci_visibility/telemetry/coverage.py b/ddtrace/internal/ci_visibility/telemetry/coverage.py index e3370fbee6e..392196f7236 100644 --- a/ddtrace/internal/ci_visibility/telemetry/coverage.py +++ b/ddtrace/internal/ci_visibility/telemetry/coverage.py @@ -3,10 +3,10 @@ from typing import Optional from typing import Tuple -from ddtrace.internal.ci_visibility.telemetry.constants import CIVISIBILITY_TELEMETRY_NAMESPACE as _NAMESPACE from ddtrace.internal.ci_visibility.telemetry.constants import TEST_FRAMEWORKS from ddtrace.internal.logger import get_logger from ddtrace.internal.telemetry import telemetry_writer +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE log = get_logger(__name__) @@ -30,7 +30,7 @@ def record_code_coverage_started(coverage_library: COVERAGE_LIBRARY, test_framew _tags: List[Tuple[str, str]] = [("library", coverage_library)] if test_framework is not None: _tags.append(("test_framework", test_framework)) - telemetry_writer.add_count_metric(_NAMESPACE, COVERAGE_TELEMETRY.STARTED, 1, tuple(_tags)) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, COVERAGE_TELEMETRY.STARTED, 1, tuple(_tags)) def record_code_coverage_finished(coverage_library: COVERAGE_LIBRARY, test_framework: Optional[TEST_FRAMEWORKS] = None): @@ -38,19 +38,19 @@ def record_code_coverage_finished(coverage_library: COVERAGE_LIBRARY, test_frame _tags: List[Tuple[str, str]] = [("library", coverage_library)] if test_framework is not None: _tags.append(("test_framework", test_framework)) - telemetry_writer.add_count_metric(_NAMESPACE, COVERAGE_TELEMETRY.FINISHED, 1, tuple(_tags)) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, COVERAGE_TELEMETRY.FINISHED, 1, tuple(_tags)) def record_code_coverage_empty(): log.debug("Recording code coverage empty telemetry") - telemetry_writer.add_count_metric(_NAMESPACE, COVERAGE_TELEMETRY.IS_EMPTY, 1) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, COVERAGE_TELEMETRY.IS_EMPTY, 1) def record_code_coverage_files(count_files: int): log.debug("Recording code coverage files telemetry: %s", count_files) - telemetry_writer.add_distribution_metric(_NAMESPACE, COVERAGE_TELEMETRY.FILES, count_files) + telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, COVERAGE_TELEMETRY.FILES, count_files) def record_code_coverage_error(): log.debug("Recording code coverage error telemetry") - telemetry_writer.add_count_metric(_NAMESPACE, COVERAGE_TELEMETRY.ERRORS, 1) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, COVERAGE_TELEMETRY.ERRORS, 1) diff --git a/ddtrace/internal/ci_visibility/telemetry/early_flake_detection.py b/ddtrace/internal/ci_visibility/telemetry/early_flake_detection.py index f8a512e7048..b9e9e48d021 100644 --- a/ddtrace/internal/ci_visibility/telemetry/early_flake_detection.py +++ b/ddtrace/internal/ci_visibility/telemetry/early_flake_detection.py @@ -1,8 +1,8 @@ from enum import Enum -from ddtrace.internal.ci_visibility.telemetry.constants import CIVISIBILITY_TELEMETRY_NAMESPACE as _NAMESPACE from ddtrace.internal.logger import get_logger from ddtrace.internal.telemetry import telemetry_writer +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE log = get_logger(__name__) @@ -19,5 +19,7 @@ class EARLY_FLAKE_DETECTION_TELEMETRY(str, Enum): def record_early_flake_detection_tests_count(early_flake_detection_count: int): log.debug("Recording early flake detection tests count telemetry: %s", early_flake_detection_count) telemetry_writer.add_distribution_metric( - _NAMESPACE, EARLY_FLAKE_DETECTION_TELEMETRY.RESPONSE_TESTS.value, early_flake_detection_count + TELEMETRY_NAMESPACE.CIVISIBILITY, + EARLY_FLAKE_DETECTION_TELEMETRY.RESPONSE_TESTS.value, + early_flake_detection_count, ) diff --git a/ddtrace/internal/ci_visibility/telemetry/events.py b/ddtrace/internal/ci_visibility/telemetry/events.py index 34c603c3b03..b630ee96413 100644 --- a/ddtrace/internal/ci_visibility/telemetry/events.py +++ b/ddtrace/internal/ci_visibility/telemetry/events.py @@ -3,11 +3,11 @@ from typing import Optional from typing import Tuple -from ddtrace.internal.ci_visibility.telemetry.constants import CIVISIBILITY_TELEMETRY_NAMESPACE as _NAMESPACE from ddtrace.internal.ci_visibility.telemetry.constants import EVENT_TYPES from ddtrace.internal.ci_visibility.telemetry.constants import TEST_FRAMEWORKS from ddtrace.internal.logger import get_logger from ddtrace.internal.telemetry import telemetry_writer +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE log = get_logger(__name__) @@ -67,7 +67,7 @@ def _record_event( if early_flake_detection_abort_reason and event == EVENTS_TELEMETRY.FINISHED and event_type == EVENT_TYPES.SESSION: _tags.append(("early_flake_detection_abort_reason", early_flake_detection_abort_reason)) - telemetry_writer.add_count_metric(_NAMESPACE, event.value, 1, tuple(_tags)) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, event.value, 1, tuple(_tags)) def record_event_created( @@ -117,11 +117,19 @@ def record_event_finished( def record_manual_api_event_created(event_type: EVENT_TYPES): # Note: _created suffix is added in cases we were to change the metric name in the future. # The current metric applies to event creation even though it does not specify it - telemetry_writer.add_count_metric(_NAMESPACE, EVENTS_TELEMETRY.MANUAL_API_EVENT, 1, (("event_type", event_type),)) + telemetry_writer.add_count_metric( + TELEMETRY_NAMESPACE.CIVISIBILITY, + EVENTS_TELEMETRY.MANUAL_API_EVENT, + 1, + (("event_type", event_type),) + ) def record_events_enqueued_for_serialization(events_count: int): - telemetry_writer.add_count_metric(_NAMESPACE, EVENTS_TELEMETRY.ENQUEUED_FOR_SERIALIZATION, events_count) + telemetry_writer.add_count_metric( + TELEMETRY_NAMESPACE.CIVISIBILITY, + EVENTS_TELEMETRY.ENQUEUED_FOR_SERIALIZATION, + events_count) def record_event_created_test( @@ -139,7 +147,7 @@ def record_event_created_test( if is_benchmark: tags.append(("is_benchmark", "true")) - telemetry_writer.add_count_metric(_NAMESPACE, EVENTS_TELEMETRY.FINISHED, 1, tuple(tags)) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, EVENTS_TELEMETRY.FINISHED, 1, tuple(tags)) def record_event_finished_test( @@ -190,4 +198,4 @@ def record_event_finished_test( if is_quarantined: tags.append(("is_quarantined", "true")) - telemetry_writer.add_count_metric(_NAMESPACE, EVENTS_TELEMETRY.FINISHED, 1, tuple(tags)) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, EVENTS_TELEMETRY.FINISHED, 1, tuple(tags)) diff --git a/ddtrace/internal/ci_visibility/telemetry/git.py b/ddtrace/internal/ci_visibility/telemetry/git.py index faf01621cde..41bca64a8fd 100644 --- a/ddtrace/internal/ci_visibility/telemetry/git.py +++ b/ddtrace/internal/ci_visibility/telemetry/git.py @@ -1,11 +1,11 @@ from typing import Optional -from ddtrace.internal.ci_visibility.telemetry.constants import CIVISIBILITY_TELEMETRY_NAMESPACE as _NAMESPACE from ddtrace.internal.ci_visibility.telemetry.constants import ERROR_TYPES from ddtrace.internal.ci_visibility.telemetry.constants import GIT_TELEMETRY from ddtrace.internal.ci_visibility.telemetry.constants import GIT_TELEMETRY_COMMANDS from ddtrace.internal.logger import get_logger from ddtrace.internal.telemetry import telemetry_writer +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE log = get_logger(__name__) @@ -14,35 +14,45 @@ def record_git_command(command: GIT_TELEMETRY_COMMANDS, duration: float, exit_code: Optional[int]) -> None: log.debug("Recording git command telemetry: %s, %s, %s", command, duration, exit_code) tags = (("command", command),) - telemetry_writer.add_count_metric(_NAMESPACE, GIT_TELEMETRY.COMMAND_COUNT, 1, tags) - telemetry_writer.add_distribution_metric(_NAMESPACE, GIT_TELEMETRY.COMMAND_MS, duration, tags) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, GIT_TELEMETRY.COMMAND_COUNT, 1, tags) + telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, GIT_TELEMETRY.COMMAND_MS, duration, tags) if exit_code is not None and exit_code != 0: error_tags = (("command", command), ("exit_code", str(exit_code))) - telemetry_writer.add_count_metric(_NAMESPACE, GIT_TELEMETRY.COMMAND_ERRORS, 1, error_tags) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, GIT_TELEMETRY.COMMAND_ERRORS, 1, error_tags) def record_search_commits(duration: float, error: Optional[ERROR_TYPES] = None) -> None: log.debug("Recording search commits telemetry: %s, %s", duration, error) - telemetry_writer.add_count_metric(_NAMESPACE, GIT_TELEMETRY.SEARCH_COMMITS_COUNT, 1) - telemetry_writer.add_distribution_metric(_NAMESPACE, GIT_TELEMETRY.SEARCH_COMMITS_MS, duration) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, GIT_TELEMETRY.SEARCH_COMMITS_COUNT, 1) + telemetry_writer.add_distribution_metric( + TELEMETRY_NAMESPACE.CIVISIBILITY, GIT_TELEMETRY.SEARCH_COMMITS_MS, duration + ) if error is not None: error_tags = (("error_type", str(error)),) - telemetry_writer.add_count_metric(_NAMESPACE, GIT_TELEMETRY.SEARCH_COMMITS_ERRORS, 1, error_tags) + telemetry_writer.add_count_metric( + TELEMETRY_NAMESPACE.CIVISIBILITY, GIT_TELEMETRY.SEARCH_COMMITS_ERRORS, 1, error_tags + ) def record_objects_pack_request(duration: float, error: Optional[ERROR_TYPES] = None) -> None: log.debug("Recording objects pack request telmetry: %s, %s", duration, error) - telemetry_writer.add_count_metric(_NAMESPACE, GIT_TELEMETRY.OBJECTS_PACK_COUNT, 1) - telemetry_writer.add_distribution_metric(_NAMESPACE, GIT_TELEMETRY.OBJECTS_PACK_MS, duration) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, GIT_TELEMETRY.OBJECTS_PACK_COUNT, 1) + telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, GIT_TELEMETRY.OBJECTS_PACK_MS, duration) if error is not None: error_tags = (("error", error),) - telemetry_writer.add_count_metric(_NAMESPACE, GIT_TELEMETRY.OBJECTS_PACK_ERRORS, 1, error_tags) + telemetry_writer.add_count_metric( + TELEMETRY_NAMESPACE.CIVISIBILITY, GIT_TELEMETRY.OBJECTS_PACK_ERRORS, 1, error_tags + ) def record_objects_pack_data(num_files: int, num_bytes: int) -> None: log.debug("Recording objects pack data telemetry: %s, %s", num_files, num_bytes) - telemetry_writer.add_distribution_metric(_NAMESPACE, GIT_TELEMETRY.OBJECTS_PACK_BYTES, num_bytes) - telemetry_writer.add_distribution_metric(_NAMESPACE, GIT_TELEMETRY.OBJECTS_PACK_FILES, num_files) + telemetry_writer.add_distribution_metric( + TELEMETRY_NAMESPACE.CIVISIBILITY, GIT_TELEMETRY.OBJECTS_PACK_BYTES, num_bytes + ) + telemetry_writer.add_distribution_metric( + TELEMETRY_NAMESPACE.CIVISIBILITY, GIT_TELEMETRY.OBJECTS_PACK_FILES, num_files + ) def record_settings_response( @@ -87,4 +97,6 @@ def record_settings_response( response_tags.append(("quarantine_enabled", "true")) if response_tags: - telemetry_writer.add_count_metric(_NAMESPACE, GIT_TELEMETRY.SETTINGS_RESPONSE, 1, tuple(response_tags)) + telemetry_writer.add_count_metric( + TELEMETRY_NAMESPACE.CIVISIBILITY, GIT_TELEMETRY.SETTINGS_RESPONSE, 1, tuple(response_tags) + ) diff --git a/ddtrace/internal/ci_visibility/telemetry/itr.py b/ddtrace/internal/ci_visibility/telemetry/itr.py index 210a4103734..b8bf6889471 100644 --- a/ddtrace/internal/ci_visibility/telemetry/itr.py +++ b/ddtrace/internal/ci_visibility/telemetry/itr.py @@ -2,10 +2,10 @@ import functools from ddtrace.internal.ci_visibility.constants import SUITE -from ddtrace.internal.ci_visibility.telemetry.constants import CIVISIBILITY_TELEMETRY_NAMESPACE as _NAMESPACE from ddtrace.internal.ci_visibility.telemetry.constants import EVENT_TYPES from ddtrace.internal.logger import get_logger from ddtrace.internal.telemetry import telemetry_writer +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE log = get_logger(__name__) @@ -40,18 +40,24 @@ def wrapper(event_type: str): @_enforce_event_is_test_or_suite def record_itr_skipped(event_type: EVENT_TYPES): log.debug("Recording itr skipped telemetry for %s", event_type) - telemetry_writer.add_count_metric(_NAMESPACE, ITR_TELEMETRY.SKIPPED, 1, (("event_type", event_type.value),)) + telemetry_writer.add_count_metric( + TELEMETRY_NAMESPACE.CIVISIBILITY, ITR_TELEMETRY.SKIPPED, 1, (("event_type", event_type.value),) + ) @_enforce_event_is_test_or_suite def record_itr_unskippable(event_type: EVENT_TYPES): log.debug("Recording itr unskippable telemetry for %s", event_type) - telemetry_writer.add_count_metric(_NAMESPACE, ITR_TELEMETRY.UNSKIPPABLE, 1, (("event_type", event_type.value),)) + telemetry_writer.add_count_metric( + TELEMETRY_NAMESPACE.CIVISIBILITY, ITR_TELEMETRY.UNSKIPPABLE, 1, (("event_type", event_type.value),) + ) def record_itr_forced_run(event_type: EVENT_TYPES): log.debug("Recording itr forced run telemetry for %s", event_type) - telemetry_writer.add_count_metric(_NAMESPACE, ITR_TELEMETRY.FORCED_RUN, 1, (("event_type", event_type.value),)) + telemetry_writer.add_count_metric( + TELEMETRY_NAMESPACE.CIVISIBILITY, ITR_TELEMETRY.FORCED_RUN, 1, (("event_type", event_type.value),) + ) def record_skippable_count(skippable_count: int, skipping_level: str): @@ -60,4 +66,4 @@ def record_skippable_count(skippable_count: int, skipping_level: str): if skipping_level == SUITE else SKIPPABLE_TESTS_TELEMETRY.RESPONSE_TESTS ) - telemetry_writer.add_count_metric(_NAMESPACE, skippable_count_metric, skippable_count) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.CIVISIBILITY, skippable_count_metric, skippable_count) diff --git a/ddtrace/internal/ci_visibility/telemetry/payload.py b/ddtrace/internal/ci_visibility/telemetry/payload.py index 1cf41d306ff..f5dd7a9ca00 100644 --- a/ddtrace/internal/ci_visibility/telemetry/payload.py +++ b/ddtrace/internal/ci_visibility/telemetry/payload.py @@ -1,8 +1,8 @@ from enum import Enum -from ddtrace.internal.ci_visibility.telemetry.constants import CIVISIBILITY_TELEMETRY_NAMESPACE as _NAMESPACE from ddtrace.internal.logger import get_logger from ddtrace.internal.telemetry import telemetry_writer +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE log = get_logger(__name__) @@ -31,38 +31,46 @@ class REQUEST_ERROR_TYPE(str, Enum): def record_endpoint_payload_bytes(endpoint: ENDPOINT, nbytes: int) -> None: log.debug("Recording endpoint payload bytes: %s, %s", endpoint, nbytes) tags = (("endpoint", endpoint.value),) - telemetry_writer.add_distribution_metric(_NAMESPACE, ENDPOINT_PAYLOAD_TELEMETRY.BYTES.value, nbytes, tags) + telemetry_writer.add_distribution_metric( + TELEMETRY_NAMESPACE.CIVISIBILITY, ENDPOINT_PAYLOAD_TELEMETRY.BYTES.value, nbytes, tags + ) def record_endpoint_payload_request(endpoint: ENDPOINT) -> None: log.debug("Recording endpoint payload request: %s", endpoint) tags = (("endpoint", endpoint.value),) - telemetry_writer.add_count_metric(_NAMESPACE, ENDPOINT_PAYLOAD_TELEMETRY.REQUESTS_COUNT.value, 1, tags) + telemetry_writer.add_count_metric( + TELEMETRY_NAMESPACE.CIVISIBILITY, ENDPOINT_PAYLOAD_TELEMETRY.REQUESTS_COUNT.value, 1, tags + ) def record_endpoint_payload_request_time(endpoint: ENDPOINT, seconds: float) -> None: log.debug("Recording endpoint payload request time: %s, %s seconds", endpoint, seconds) tags = (("endpoint", endpoint.value),) telemetry_writer.add_distribution_metric( - _NAMESPACE, ENDPOINT_PAYLOAD_TELEMETRY.REQUESTS_MS.value, seconds * 1000, tags + TELEMETRY_NAMESPACE.CIVISIBILITY, ENDPOINT_PAYLOAD_TELEMETRY.REQUESTS_MS.value, seconds * 1000, tags ) def record_endpoint_payload_request_error(endpoint: ENDPOINT, error_type: REQUEST_ERROR_TYPE) -> None: log.debug("Recording endpoint payload request error: %s, %s", endpoint, error_type) tags = (("endpoint", endpoint.value), ("error_type", error_type)) - telemetry_writer.add_count_metric(_NAMESPACE, ENDPOINT_PAYLOAD_TELEMETRY.REQUESTS_ERRORS.value, 1, tags) + telemetry_writer.add_count_metric( + TELEMETRY_NAMESPACE.CIVISIBILITY, ENDPOINT_PAYLOAD_TELEMETRY.REQUESTS_ERRORS.value, 1, tags + ) def record_endpoint_payload_events_count(endpoint: ENDPOINT, count: int) -> None: log.debug("Recording endpoint payload events count: %s, %s", endpoint, count) tags = (("endpoint", endpoint.value),) - telemetry_writer.add_distribution_metric(_NAMESPACE, ENDPOINT_PAYLOAD_TELEMETRY.EVENTS_COUNT.value, count, tags) + telemetry_writer.add_distribution_metric( + TELEMETRY_NAMESPACE.CIVISIBILITY, ENDPOINT_PAYLOAD_TELEMETRY.EVENTS_COUNT.value, count, tags + ) def record_endpoint_payload_events_serialization_time(endpoint: ENDPOINT, seconds: float) -> None: log.debug("Recording endpoint payload serialization time: %s, %s seconds", endpoint, seconds) tags = (("endpoint", endpoint.value),) telemetry_writer.add_distribution_metric( - _NAMESPACE, ENDPOINT_PAYLOAD_TELEMETRY.EVENTS_SERIALIZATION_MS.value, seconds * 1000, tags + TELEMETRY_NAMESPACE.CIVISIBILITY, ENDPOINT_PAYLOAD_TELEMETRY.EVENTS_SERIALIZATION_MS.value, seconds * 1000, tags ) diff --git a/ddtrace/internal/compat.py b/ddtrace/internal/compat.py index 7f00043f049..bc61ebdf0a4 100644 --- a/ddtrace/internal/compat.py +++ b/ddtrace/internal/compat.py @@ -56,32 +56,16 @@ def ensure_text(s, encoding="utf-8", errors="ignore") -> str: if isinstance(s, str): return s - if isinstance(s, bytes): return s.decode(encoding, errors) - - # Skip the check for Mock objects as they are used in tests - from unittest.mock import Mock - - if isinstance(s, Mock): - return str(s) - raise TypeError("Expected str or bytes but received %r" % (s.__class__)) def ensure_binary(s, encoding="utf-8", errors="ignore") -> bytes: if isinstance(s, bytes): return s - - # Skip the check for Mock objects as they are used in tests - from unittest.mock import Mock - - if isinstance(s, Mock): - return bytes(s) - if not isinstance(s, str): raise TypeError("Expected str or bytes but received %r" % (s.__class__)) - return s.encode(encoding, errors) @@ -185,9 +169,6 @@ def get_connection_response( return conn.getresponse() -CONTEXTVARS_IS_AVAILABLE = True - - try: from collections.abc import Iterable # noqa:F401 except ImportError: diff --git a/ddtrace/internal/constants.py b/ddtrace/internal/constants.py index 4efdc754ef3..c4255035c41 100644 --- a/ddtrace/internal/constants.py +++ b/ddtrace/internal/constants.py @@ -19,6 +19,10 @@ _PROPAGATION_STYLE_NONE, _PROPAGATION_STYLE_BAGGAGE, ) +_PROPAGATION_BEHAVIOR_CONTINUE = "continue" +_PROPAGATION_BEHAVIOR_IGNORE = "ignore" +_PROPAGATION_BEHAVIOR_RESTART = "restart" +_PROPAGATION_BEHAVIOR_DEFAULT = _PROPAGATION_BEHAVIOR_CONTINUE W3C_TRACESTATE_KEY = "tracestate" W3C_TRACEPARENT_KEY = "traceparent" W3C_TRACESTATE_PARENT_ID_KEY = "p" diff --git a/ddtrace/internal/coverage/code.py b/ddtrace/internal/coverage/code.py index b6f5d379661..5227e08fa42 100644 --- a/ddtrace/internal/coverage/code.py +++ b/ddtrace/internal/coverage/code.py @@ -1,4 +1,5 @@ from collections import defaultdict +from contextvars import ContextVar from copy import deepcopy from inspect import getmodule import os @@ -19,14 +20,13 @@ from ddtrace.internal.packages import purelib_path from ddtrace.internal.packages import stdlib_path from ddtrace.internal.test_visibility.coverage_lines import CoverageLines -from ddtrace.vendor.contextvars import ContextVar log = get_logger(__name__) _original_exec = exec -ctx_covered = ContextVar("ctx_covered", default=None) +ctx_covered: ContextVar[t.List[t.DefaultDict[str, CoverageLines]]] = ContextVar("ctx_covered", default=[]) ctx_is_import_coverage = ContextVar("ctx_is_import_coverage", default=False) ctx_coverage_enabled = ContextVar("ctx_coverage_enabled", default=False) diff --git a/ddtrace/internal/datastreams/kombu.py b/ddtrace/internal/datastreams/kombu.py index 0d163021d52..fd04ebc163a 100644 --- a/ddtrace/internal/datastreams/kombu.py +++ b/ddtrace/internal/datastreams/kombu.py @@ -1,8 +1,8 @@ from ddtrace import config -from ddtrace.contrib.kombu.utils import HEADER_POS -from ddtrace.contrib.kombu.utils import PUBLISH_BODY_IDX -from ddtrace.contrib.kombu.utils import get_exchange_from_args -from ddtrace.contrib.kombu.utils import get_routing_key_from_args +from ddtrace.contrib.internal.kombu.utils import HEADER_POS +from ddtrace.contrib.internal.kombu.utils import PUBLISH_BODY_IDX +from ddtrace.contrib.internal.kombu.utils import get_exchange_from_args +from ddtrace.contrib.internal.kombu.utils import get_routing_key_from_args from ddtrace.internal import core from ddtrace.internal.datastreams.processor import DsmPathwayCodec from ddtrace.internal.datastreams.utils import _calculate_byte_size diff --git a/ddtrace/internal/debug.py b/ddtrace/internal/debug.py index 4d533b604b6..ec4048b59d0 100644 --- a/ddtrace/internal/debug.py +++ b/ddtrace/internal/debug.py @@ -10,12 +10,12 @@ from typing import Union # noqa:F401 import ddtrace +from ddtrace._trace.sampler import DatadogSampler from ddtrace.internal import agent from ddtrace.internal.packages import get_distributions from ddtrace.internal.utils.cache import callonce from ddtrace.internal.writer import AgentWriter from ddtrace.internal.writer import LogWriter -from ddtrace.sampler import DatadogSampler from ddtrace.settings.asm import config as asm_config from .logger import get_logger @@ -117,8 +117,8 @@ def collect(tracer): from ddtrace._trace.tracer import log return dict( - # Timestamp UTC ISO 8601 - date=datetime.datetime.utcnow().isoformat(), + # Timestamp UTC ISO 8601 with the trailing +00:00 removed + date=datetime.datetime.now(datetime.timezone.utc).isoformat()[0:-6], # eg. "Linux", "Darwin" os_name=platform.system(), # eg. 12.5.0 diff --git a/ddtrace/internal/sampling.py b/ddtrace/internal/sampling.py index d97833bde24..997d3af77fd 100644 --- a/ddtrace/internal/sampling.py +++ b/ddtrace/internal/sampling.py @@ -14,6 +14,7 @@ except ImportError: from typing_extensions import TypedDict +from ddtrace._trace.sampling_rule import SamplingRule # noqa:F401 from ddtrace.constants import _SINGLE_SPAN_SAMPLING_MAX_PER_SEC from ddtrace.constants import _SINGLE_SPAN_SAMPLING_MAX_PER_SEC_NO_LIMIT from ddtrace.constants import _SINGLE_SPAN_SAMPLING_MECHANISM @@ -26,7 +27,6 @@ from ddtrace.internal.constants import SAMPLING_DECISION_TRACE_TAG_KEY from ddtrace.internal.glob_matching import GlobMatcher from ddtrace.internal.logger import get_logger -from ddtrace.sampling_rule import SamplingRule # noqa:F401 from ddtrace.settings import _config as config from .rate_limiter import RateLimiter diff --git a/ddtrace/internal/telemetry/constants.py b/ddtrace/internal/telemetry/constants.py index 3298fdd7616..a809b5f2f4f 100644 --- a/ddtrace/internal/telemetry/constants.py +++ b/ddtrace/internal/telemetry/constants.py @@ -1,9 +1,13 @@ from enum import Enum -TELEMETRY_NAMESPACE_TAG_TRACER = "tracers" -TELEMETRY_NAMESPACE_TAG_APPSEC = "appsec" -TELEMETRY_NAMESPACE_TAG_IAST = "iast" +class TELEMETRY_NAMESPACE(Enum): + TRACERS = "tracers" + APPSEC = "appsec" + IAST = "iast" + CIVISIBILITY = "civisibility" + MLOBS = "mlobs" + TELEMETRY_TYPE_GENERATE_METRICS = "generate-metrics" TELEMETRY_TYPE_DISTRIBUTION = "distributions" diff --git a/ddtrace/internal/telemetry/metrics_namespaces.py b/ddtrace/internal/telemetry/metrics_namespaces.py index 927f6de775d..4b432ba330c 100644 --- a/ddtrace/internal/telemetry/metrics_namespaces.py +++ b/ddtrace/internal/telemetry/metrics_namespaces.py @@ -5,6 +5,7 @@ from typing import Type # noqa:F401 from ddtrace.internal import forksafe +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE from ddtrace.internal.telemetry.constants import TELEMETRY_TYPE_DISTRIBUTION from ddtrace.internal.telemetry.constants import TELEMETRY_TYPE_GENERATE_METRICS from ddtrace.internal.telemetry.metrics import DistributionMetric @@ -34,23 +35,31 @@ def flush(self): } return namespace_metrics - def add_metric(self, metric_class, namespace, name, value=1.0, tags=None, interval=None): - # type: (Type[Metric], str, str, float, MetricTagType, Optional[float]) -> None + def add_metric( + self, + metric_class: Type[Metric], + namespace: TELEMETRY_NAMESPACE, + name: str, + value: float = 1.0, + tags: MetricTagType = None, + interval: Optional[float] = None, + ) -> None: """ Telemetry Metrics are stored in DD dashboards, check the metrics in datadoghq.com/metric/explorer. The metric will store in dashboard as "dd.instrumentation_telemetry_data." + namespace + "." + name """ - metric_id = Metric.get_id(name, namespace, tags, metric_class.metric_type) + namespace_str = namespace.value + metric_id = Metric.get_id(name, namespace_str, tags, metric_class.metric_type) if metric_class is DistributionMetric: metrics_type_payload = TELEMETRY_TYPE_DISTRIBUTION else: metrics_type_payload = TELEMETRY_TYPE_GENERATE_METRICS with self._lock: - existing_metric = self._metrics_data[metrics_type_payload][namespace].get(metric_id) + existing_metric = self._metrics_data[metrics_type_payload][namespace_str].get(metric_id) if existing_metric: existing_metric.add_point(value) else: - new_metric = metric_class(namespace, name, tags=tags, common=True, interval=interval) + new_metric = metric_class(namespace_str, name, tags=tags, common=True, interval=interval) new_metric.add_point(value) - self._metrics_data[metrics_type_payload][namespace][metric_id] = new_metric + self._metrics_data[metrics_type_payload][namespace_str][metric_id] = new_metric diff --git a/ddtrace/internal/telemetry/writer.py b/ddtrace/internal/telemetry/writer.py index 71de6b03907..35a73d5e235 100644 --- a/ddtrace/internal/telemetry/writer.py +++ b/ddtrace/internal/telemetry/writer.py @@ -31,6 +31,7 @@ from . import modules from .constants import TELEMETRY_APM_PRODUCT from .constants import TELEMETRY_LOG_LEVEL # noqa:F401 +from .constants import TELEMETRY_NAMESPACE from .constants import TELEMETRY_TYPE_DISTRIBUTION from .constants import TELEMETRY_TYPE_GENERATE_METRICS from .constants import TELEMETRY_TYPE_LOGS @@ -118,11 +119,17 @@ def send_event(self, request: Dict) -> Optional[httplib.HTTPResponse]: conn.request("POST", self._endpoint, rb_json, headers) resp = get_connection_response(conn) if resp.status < 300: - log.debug("sent %d in %.5fs to %s. response: %s", len(rb_json), sw.elapsed(), self.url, resp.status) + log.debug( + "Instrumentation Telemetry sent %d in %.5fs to %s. response: %s", + len(rb_json), + sw.elapsed(), + self.url, + resp.status, + ) else: - log.debug("failed to send telemetry to %s. response: %s", self.url, resp.status) - except Exception: - log.debug("failed to send telemetry to %s.", self.url, exc_info=True) + log.debug("Failed to send Instrumentation Telemetry to %s. response: %s", self.url, resp.status) + except Exception as e: + log.debug("Failed to send Instrumentation Telemetry to %s. Error: %s", self.url, str(e)) finally: if conn is not None: conn.close() @@ -331,7 +338,7 @@ def _app_started(self, register_app_shutdown=True): } # SOABI should help us identify which wheels people are getting from PyPI - self.add_configurations(get_python_config_vars()) # type: ignore + self.add_configurations(get_python_config_vars()) payload = { "configuration": self._flush_configuration_queue(), @@ -468,7 +475,6 @@ def add_configuration(self, configuration_name, configuration_value, origin="unk } def add_configurations(self, configuration_list): - # type: (List[Tuple[str, Union[bool, float, str], str]]) -> None """Creates and queues a list of configurations""" with self._service_lock: for name, value, _origin in configuration_list: @@ -479,7 +485,6 @@ def add_configurations(self, configuration_list): } def add_log(self, level, message, stack_trace="", tags=None): - # type: (TELEMETRY_LOG_LEVEL, str, str, Optional[Dict]) -> None """ Queues log. This event is meant to send library logs to Datadog’s backend through the Telemetry intake. This will make support cycles easier and ensure we know about potentially silent issues in libraries. @@ -501,8 +506,7 @@ def add_log(self, level, message, stack_trace="", tags=None): data["stack_trace"] = stack_trace self._logs.add(data) - def add_gauge_metric(self, namespace, name, value, tags=None): - # type: (str,str, float, MetricTagType) -> None + def add_gauge_metric(self, namespace: TELEMETRY_NAMESPACE, name: str, value: float, tags: MetricTagType = None): """ Queues gauge metric """ @@ -516,8 +520,7 @@ def add_gauge_metric(self, namespace, name, value, tags=None): self.interval, ) - def add_rate_metric(self, namespace, name, value=1.0, tags=None): - # type: (str,str, float, MetricTagType) -> None + def add_rate_metric(self, namespace: TELEMETRY_NAMESPACE, name: str, value: float, tags: MetricTagType = None): """ Queues rate metric """ @@ -531,8 +534,7 @@ def add_rate_metric(self, namespace, name, value=1.0, tags=None): self.interval, ) - def add_count_metric(self, namespace, name, value=1.0, tags=None): - # type: (str,str, float, MetricTagType) -> None + def add_count_metric(self, namespace: TELEMETRY_NAMESPACE, name: str, value: int = 1, tags: MetricTagType = None): """ Queues count metric """ @@ -545,8 +547,7 @@ def add_count_metric(self, namespace, name, value=1.0, tags=None): tags, ) - def add_distribution_metric(self, namespace, name, value=1.0, tags=None): - # type: (str,str, float, MetricTagType) -> None + def add_distribution_metric(self, namespace: TELEMETRY_NAMESPACE, name: str, value, tags: MetricTagType = None): """ Queues distributions metric """ @@ -702,7 +703,7 @@ def _telemetry_excepthook(self, tp, value, root_traceback): internal_index = dir_parts.index("internal") integration_name = dir_parts[internal_index + 1] self.add_count_metric( - "tracers", + TELEMETRY_NAMESPACE.TRACERS, "integration_errors", 1, (("integration_name", integration_name), ("error_type", tp.__name__)), diff --git a/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py b/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py new file mode 100644 index 00000000000..9a640e08454 --- /dev/null +++ b/ddtrace/llmobs/_evaluators/ragas/answer_relevancy.py @@ -0,0 +1,146 @@ +import math +from typing import Optional +from typing import Tuple +from typing import Union + +from ddtrace.internal.logger import get_logger +from ddtrace.llmobs._constants import EVALUATION_SPAN_METADATA +from ddtrace.llmobs._evaluators.ragas.base import BaseRagasEvaluator +from ddtrace.llmobs._evaluators.ragas.base import _get_ml_app_for_ragas_trace + + +logger = get_logger(__name__) + + +class RagasAnswerRelevancyEvaluator(BaseRagasEvaluator): + """A class used by EvaluatorRunner to conduct ragas answer relevancy evaluations + on LLM Observability span events. The job of an Evaluator is to take a span and + submit evaluation metrics based on the span's attributes. + """ + + LABEL = "ragas_answer_relevancy" + METRIC_TYPE = "score" + + def __init__(self, llmobs_service): + """ + Initialize an evaluator that uses the ragas library to generate a context precision score on finished LLM spans. + + answer relevancy focuses on assessing how pertinent the generated answer is to a given question. + A lower score is assigned to answers that are incomplete or contain redundant information and higher scores + indicate better relevancy. This metric is computed using the question, contexts, and answer. + + For more information, see https://docs.ragas.io/en/latest/concepts/metrics/available_metrics/answer_relevance/ + + The `ragas.metrics.answer_relevancy` instance is used for answer relevancy scores. + If there is no llm attribute set on this instance, it will be set to the + default `llm_factory()` from ragas which uses openai. + If there is no embedding attribute set on this instance, it will be to to the + default `embedding_factory()` from ragas which uses openai + + :param llmobs_service: An instance of the LLM Observability service used for tracing the evaluation and + submitting evaluation metrics. + + Raises: NotImplementedError if the ragas library is not found or if ragas version is not supported. + """ + super().__init__(llmobs_service) + self.ragas_answer_relevancy_instance = self._get_answer_relevancy_instance() + self.answer_relevancy_output_parser = self.ragas_dependencies.RagasoutputParser( + pydantic_object=self.ragas_dependencies.AnswerRelevanceClassification + ) + + def _get_answer_relevancy_instance(self): + """ + This helper function ensures the answer relevancy instance used in + ragas evaluator is updated with the latest ragas answer relevancy instance + instance AND has an non-null llm + """ + if self.ragas_dependencies.answer_relevancy is None: + return None + ragas_answer_relevancy_instance = self.ragas_dependencies.answer_relevancy + if not ragas_answer_relevancy_instance.llm: + ragas_answer_relevancy_instance.llm = self.ragas_dependencies.llm_factory() + if not ragas_answer_relevancy_instance.embeddings: + ragas_answer_relevancy_instance.embeddings = self.ragas_dependencies.embedding_factory() + return ragas_answer_relevancy_instance + + def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]]: + """ + Performs a answer relevancy evaluation on an llm span event, returning either + - answer relevancy score (float) OR failure reason (str) + - evaluation metadata (dict) + If the ragas answer relevancy instance does not have `llm` set, we set `llm` using the `llm_factory()` + method from ragas which currently defaults to openai's gpt-4o-turbo. + """ + self.ragas_answer_relevancy_instance = self._get_answer_relevancy_instance() + if not self.ragas_answer_relevancy_instance: + return "fail_answer_relevancy_is_none", {} + + evaluation_metadata = {} # type: dict[str, Union[str, dict, list]] + trace_metadata = {} # type: dict[str, Union[str, dict, list]] + + # initialize data we annotate for tracing ragas + score, answer_classifications = (math.nan, None) + + with self.llmobs_service.workflow( + "dd-ragas.answer_relevancy", ml_app=_get_ml_app_for_ragas_trace(span_event) + ) as ragas_ar_workflow: + try: + evaluation_metadata[EVALUATION_SPAN_METADATA] = self.llmobs_service.export_span(span=ragas_ar_workflow) + + answer_relevancy_inputs = self._extract_evaluation_inputs_from_span(span_event) + if answer_relevancy_inputs is None: + logger.debug( + "Failed to extract question and contexts from " + "span sampled for `ragas_answer_relevancy` evaluation" + ) + return "fail_extract_answer_relevancy_inputs", evaluation_metadata + + prompt = self.ragas_answer_relevancy_instance.question_generation.format( + answer=answer_relevancy_inputs["answer"], + context="\n".join(answer_relevancy_inputs["contexts"]), + ) + + trace_metadata["strictness"] = self.ragas_answer_relevancy_instance.strictness + result = self.ragas_answer_relevancy_instance.llm.generate_text( + prompt, n=self.ragas_answer_relevancy_instance.strictness + ) + + try: + answers = [self.answer_relevancy_output_parser.parse(res.text) for res in result.generations[0]] + answers = [answer for answer in answers if answer is not None] + except Exception as e: + logger.debug("Failed to parse answer relevancy output: %s", e) + return "fail_parse_answer_relevancy_output", evaluation_metadata + + gen_questions = [answer.question for answer in answers] + answer_classifications = [ + {"question": answer.question, "noncommittal": answer.noncommittal} for answer in answers + ] + trace_metadata["answer_classifications"] = answer_classifications + if all(q == "" for q in gen_questions): + logger.warning("Invalid JSON response. Expected dictionary with key 'question'") + return "fail_parse_answer_relevancy_output", evaluation_metadata + + # calculate cosine similarity between the question and generated questions + with self.llmobs_service.workflow("dd-ragas.calculate_similarity") as ragas_cs_workflow: + cosine_sim = self.ragas_answer_relevancy_instance.calculate_similarity( + answer_relevancy_inputs["question"], gen_questions + ) + self.llmobs_service.annotate( + span=ragas_cs_workflow, + input_data={ + "question": answer_relevancy_inputs["question"], + "generated_questions": gen_questions, + }, + output_data=cosine_sim.mean(), + ) + + score = cosine_sim.mean() * int(not any(answer.noncommittal for answer in answers)) + return score, evaluation_metadata + finally: + self.llmobs_service.annotate( + span=ragas_ar_workflow, + input_data=span_event, + output_data=score, + metadata=trace_metadata, + ) diff --git a/ddtrace/llmobs/_evaluators/ragas/base.py b/ddtrace/llmobs/_evaluators/ragas/base.py new file mode 100644 index 00000000000..798c8e2fccc --- /dev/null +++ b/ddtrace/llmobs/_evaluators/ragas/base.py @@ -0,0 +1,231 @@ +import traceback +from typing import List +from typing import Optional +from typing import Tuple +from typing import Union + +from ddtrace.internal.logger import get_logger +from ddtrace.internal.telemetry import telemetry_writer +from ddtrace.internal.telemetry.constants import TELEMETRY_LOG_LEVEL +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE +from ddtrace.internal.utils.version import parse_version +from ddtrace.llmobs._constants import INTERNAL_CONTEXT_VARIABLE_KEYS +from ddtrace.llmobs._constants import INTERNAL_QUERY_VARIABLE_KEYS +from ddtrace.llmobs._constants import RAGAS_ML_APP_PREFIX + + +logger = get_logger(__name__) + + +class RagasDependencies: + """ + A helper class to store instances of ragas classes and functions + that may or may not exist in a user's environment. + """ + + def __init__(self): + import ragas + + self.ragas_version = ragas.__version__ # type: str + + parsed_version = parse_version(ragas.__version__) + if parsed_version >= (0, 2, 0) or parsed_version < (0, 1, 10): + raise NotImplementedError( + "Ragas version: {} is not supported".format(self.ragas_version), + ) + + from ragas.llms import llm_factory + + self.llm_factory = llm_factory + + from ragas.llms.output_parser import RagasoutputParser + + self.RagasoutputParser = RagasoutputParser + + from ragas.metrics import context_precision + + self.context_precision = context_precision + + from ragas.metrics.base import ensembler + + self.ensembler = ensembler + + from ragas.metrics import faithfulness + + self.faithfulness = faithfulness + + from ragas.metrics.base import get_segmenter + + self.get_segmenter = get_segmenter + + from ragas.metrics import answer_relevancy + + self.answer_relevancy = answer_relevancy + + from ragas.embeddings import embedding_factory + + self.embedding_factory = embedding_factory + + from ddtrace.llmobs._evaluators.ragas.models import ContextPrecisionVerification + + self.ContextPrecisionVerification = ContextPrecisionVerification + + from ddtrace.llmobs._evaluators.ragas.models import StatementFaithfulnessAnswers + + self.StatementFaithfulnessAnswers = StatementFaithfulnessAnswers + + from ddtrace.llmobs._evaluators.ragas.models import StatementsAnswers + + self.StatementsAnswers = StatementsAnswers + + from ddtrace.llmobs._evaluators.ragas.models import AnswerRelevanceClassification + + self.AnswerRelevanceClassification = AnswerRelevanceClassification + + +def _get_ml_app_for_ragas_trace(span_event: dict) -> str: + """ + The `ml_app` spans generated from traces of ragas will be named as `dd-ragas-` + or `dd-ragas` if `ml_app` is not present in the span event. + """ + tags: List[str] = span_event.get("tags", []) + ml_app = None + for tag in tags: + if isinstance(tag, str) and tag.startswith("ml_app:"): + ml_app = tag.split(":")[1] + break + if not ml_app: + return RAGAS_ML_APP_PREFIX + return "{}-{}".format(RAGAS_ML_APP_PREFIX, ml_app) + + +class BaseRagasEvaluator: + """A class used by EvaluatorRunner to conduct ragas evaluations + on LLM Observability span events. The job of an Evaluator is to take a span and + submit evaluation metrics based on the span's attributes. + + Extenders of this class should only need to implement the `evaluate` method. + """ + + LABEL = "ragas" + METRIC_TYPE = "score" + + def __init__(self, llmobs_service): + """ + Initialize an evaluator that uses the ragas library to generate a score on finished LLM spans. + + :param llmobs_service: An instance of the LLM Observability service used for tracing the evaluation and + submitting evaluation metrics. + + Raises: NotImplementedError if the ragas library is not found or if ragas version is not supported. + """ + self.llmobs_service = llmobs_service + self.ragas_version = "unknown" + telemetry_state = "ok" + try: + self.ragas_dependencies = RagasDependencies() + self.ragas_version = self.ragas_dependencies.ragas_version + except ImportError as e: + telemetry_state = "fail_import_error" + raise NotImplementedError("Failed to load dependencies for `{}` evaluator".format(self.LABEL)) from e + except AttributeError as e: + telemetry_state = "fail_attribute_error" + raise NotImplementedError("Failed to load dependencies for `{}` evaluator".format(self.LABEL)) from e + except NotImplementedError as e: + telemetry_state = "fail_not_supported" + raise NotImplementedError("Failed to load dependencies for `{}` evaluator".format(self.LABEL)) from e + except Exception as e: + telemetry_state = "fail_unknown" + raise NotImplementedError("Failed to load dependencies for `{}` evaluator".format(self.LABEL)) from e + finally: + telemetry_writer.add_count_metric( + namespace=TELEMETRY_NAMESPACE.MLOBS, + name="evaluators.init", + value=1, + tags=( + ("evaluator_label", self.LABEL), + ("state", telemetry_state), + ("evaluator_version", self.ragas_version), + ), + ) + if telemetry_state != "ok": + telemetry_writer.add_log( + level=TELEMETRY_LOG_LEVEL.ERROR, + message="Failed to import Ragas dependencies", + stack_trace=traceback.format_exc(), + tags={"evaluator_version": self.ragas_version}, + ) + + def run_and_submit_evaluation(self, span_event: dict): + if not span_event: + return + score_result_or_failure, metric_metadata = self.evaluate(span_event) + telemetry_writer.add_count_metric( + TELEMETRY_NAMESPACE.MLOBS, + "evaluators.run", + 1, + tags=( + ("evaluator_label", self.LABEL), + ("state", score_result_or_failure if isinstance(score_result_or_failure, str) else "success"), + ("evaluator_version", self.ragas_version), + ), + ) + if isinstance(score_result_or_failure, float): + self.llmobs_service.submit_evaluation( + span_context={"trace_id": span_event.get("trace_id"), "span_id": span_event.get("span_id")}, + label=self.LABEL, + metric_type=self.METRIC_TYPE, + value=score_result_or_failure, + metadata=metric_metadata, + ) + + def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]]: + raise NotImplementedError("evaluate method must be implemented by individual evaluators") + + def _extract_evaluation_inputs_from_span(self, span_event: dict) -> Optional[dict]: + """ + Extracts the question, answer, and context used as inputs for a ragas evaluation on a span event. + """ + with self.llmobs_service.workflow("dd-ragas.extract_evaluation_inputs_from_span") as extract_inputs_workflow: + self.llmobs_service.annotate(span=extract_inputs_workflow, input_data=span_event) + question, answer, contexts = None, None, None + + meta_io = span_event.get("meta") + if meta_io is None: + return None + + meta_input = meta_io.get("input") + meta_output = meta_io.get("output") + + if not (meta_input and meta_output): + return None + + prompt = meta_input.get("prompt") + if prompt is None: + logger.debug("Failed to extract `prompt` from span for ragas evaluation") + return None + prompt_variables = prompt.get("variables") + + input_messages = meta_input.get("messages") + + messages = meta_output.get("messages") + if messages is not None and len(messages) > 0: + answer = messages[-1].get("content") + + if prompt_variables: + context_keys = prompt.get(INTERNAL_CONTEXT_VARIABLE_KEYS, ["context"]) + question_keys = prompt.get(INTERNAL_QUERY_VARIABLE_KEYS, ["question"]) + contexts = [prompt_variables.get(key) for key in context_keys if prompt_variables.get(key)] + question = " ".join([prompt_variables.get(key) for key in question_keys if prompt_variables.get(key)]) + + if not question and input_messages is not None and len(input_messages) > 0: + question = input_messages[-1].get("content") + + self.llmobs_service.annotate( + span=extract_inputs_workflow, output_data={"question": question, "contexts": contexts, "answer": answer} + ) + if any(field is None for field in (question, contexts, answer)): + logger.debug("Failed to extract inputs required for ragas evaluation") + return None + + return {"question": question, "contexts": contexts, "answer": answer} diff --git a/ddtrace/llmobs/_evaluators/ragas/context_precision.py b/ddtrace/llmobs/_evaluators/ragas/context_precision.py new file mode 100644 index 00000000000..990302931c8 --- /dev/null +++ b/ddtrace/llmobs/_evaluators/ragas/context_precision.py @@ -0,0 +1,153 @@ +import math +from typing import Optional +from typing import Tuple +from typing import Union + +from ddtrace.internal.logger import get_logger +from ddtrace.llmobs._constants import EVALUATION_KIND_METADATA +from ddtrace.llmobs._constants import EVALUATION_SPAN_METADATA +from ddtrace.llmobs._evaluators.ragas.base import BaseRagasEvaluator +from ddtrace.llmobs._evaluators.ragas.base import _get_ml_app_for_ragas_trace + + +logger = get_logger(__name__) + + +class RagasContextPrecisionEvaluator(BaseRagasEvaluator): + """ + A class used by EvaluatorRunner to conduct ragas context precision evaluations + on LLM Observability span events. + """ + + LABEL = "ragas_context_precision" + METRIC_TYPE = "score" + + def __init__(self, llmobs_service): + """ + Initialize an evaluator that uses the ragas library to generate a context precision score on finished LLM spans. + + Context Precision is a metric that verifies if the context was useful in arriving at the given answer. + We compute this by dividing the number of relevant contexts by the total number of contexts. + Note that this is slightly modified from the original context precision metric in ragas, which computes + the mean of the precision @ rank k for each chunk in the context (where k is the number of + retrieved context chunks). + + For more information, see https://docs.ragas.io/en/latest/concepts/metrics/available_metrics/context_precision/ + + The `ragas.metrics.context_precision` instance is used for context precision scores. + If there is no llm attribute set on this instance, it will be set to the + default `llm_factory()` which uses openai. + + :param llmobs_service: An instance of the LLM Observability service used for tracing the evaluation and + submitting evaluation metrics. + + Raises: NotImplementedError if the ragas library is not found or if ragas version is not supported. + """ + super().__init__(llmobs_service) + self.ragas_context_precision_instance = self._get_context_precision_instance() + self.context_precision_output_parser = self.ragas_dependencies.RagasoutputParser( + pydantic_object=self.ragas_dependencies.ContextPrecisionVerification + ) + + def _get_context_precision_instance(self): + """ + This helper function ensures the context precision instance used in + ragas evaluator is updated with the latest ragas context precision instance + instance AND has an non-null llm + """ + if self.ragas_dependencies.context_precision is None: + return None + ragas_context_precision_instance = self.ragas_dependencies.context_precision + if not ragas_context_precision_instance.llm: + ragas_context_precision_instance.llm = self.ragas_dependencies.llm_factory() + return ragas_context_precision_instance + + def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]]: + """ + Performs a context precision evaluation on an llm span event, returning either + - context precision score (float) OR failure reason (str) + - evaluation metadata (dict) + If the ragas context precision instance does not have `llm` set, we set `llm` using the `llm_factory()` + method from ragas which currently defaults to openai's gpt-4o-turbo. + """ + self.ragas_context_precision_instance = self._get_context_precision_instance() + if not self.ragas_context_precision_instance: + return "fail_context_precision_is_none", {} + + evaluation_metadata = {EVALUATION_KIND_METADATA: "context_precision"} # type: dict[str, Union[str, dict, list]] + + # initialize data we annotate for tracing ragas + score = math.nan + + with self.llmobs_service.workflow( + "dd-ragas.context_precision", ml_app=_get_ml_app_for_ragas_trace(span_event) + ) as ragas_cp_workflow: + try: + evaluation_metadata[EVALUATION_SPAN_METADATA] = self.llmobs_service.export_span(span=ragas_cp_workflow) + + ctx_precision_inputs = self._extract_evaluation_inputs_from_span(span_event) + if ctx_precision_inputs is None: + logger.debug( + "Failed to extract evaluation inputs from " + "span sampled for `ragas_context_precision` evaluation" + ) + return "fail_extract_context_precision_inputs", evaluation_metadata + + # create a prompt to evaluate the relevancy of each context chunk + context_precision_prompts = [ + self.ragas_context_precision_instance.context_precision_prompt.format( + question=ctx_precision_inputs["question"], + context=c, + answer=ctx_precision_inputs["answer"], + ) + for c in ctx_precision_inputs["contexts"] + ] + + responses = [] + + for prompt in context_precision_prompts: + result = self.ragas_context_precision_instance.llm.generate_text(prompt) + reproducibility = getattr(self.ragas_context_precision_instance, "_reproducibility", 1) + + results = [result.generations[0][i].text for i in range(reproducibility)] + try: + responses.append( + [ + res.dict() + for res in [self.context_precision_output_parser.parse(text) for text in results] + if res is not None + ] + ) + except Exception as e: + logger.debug( + "Failed to parse context precision verification for `ragas_context_precision`", + exc_info=e, + ) + return "fail_context_precision_parsing", evaluation_metadata + + answers = [] + for response in responses: + agg_answer = self.ragas_dependencies.ensembler.from_discrete([response], "verdict") + if agg_answer: + try: + agg_answer = self.ragas_dependencies.ContextPrecisionVerification.parse_obj(agg_answer[0]) + except Exception as e: + logger.debug( + "Failed to parse context precision verification for `ragas_context_precision`", + exc_info=e, + ) + return "fail_context_precision_parsing", evaluation_metadata + answers.append(agg_answer) + + if len(answers) == 0: + return "fail_no_answers", evaluation_metadata + + verdict_list = [1 if ver.verdict else 0 for ver in answers] + score = sum(verdict_list) / len(verdict_list) + return score, evaluation_metadata + finally: + self.llmobs_service.annotate( + span=ragas_cp_workflow, + input_data=span_event, + output_data=score, + ) diff --git a/ddtrace/llmobs/_evaluators/ragas/faithfulness.py b/ddtrace/llmobs/_evaluators/ragas/faithfulness.py index d651c2443a4..98725b1f27e 100644 --- a/ddtrace/llmobs/_evaluators/ragas/faithfulness.py +++ b/ddtrace/llmobs/_evaluators/ragas/faithfulness.py @@ -1,73 +1,22 @@ import json import math -import traceback from typing import List from typing import Optional from typing import Tuple from typing import Union from ddtrace.internal.logger import get_logger -from ddtrace.internal.telemetry import telemetry_writer -from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT -from ddtrace.internal.telemetry.constants import TELEMETRY_LOG_LEVEL -from ddtrace.internal.utils.version import parse_version from ddtrace.llmobs._constants import EVALUATION_KIND_METADATA from ddtrace.llmobs._constants import EVALUATION_SPAN_METADATA from ddtrace.llmobs._constants import FAITHFULNESS_DISAGREEMENTS_METADATA -from ddtrace.llmobs._constants import INTERNAL_CONTEXT_VARIABLE_KEYS -from ddtrace.llmobs._constants import INTERNAL_QUERY_VARIABLE_KEYS -from ddtrace.llmobs._constants import RAGAS_ML_APP_PREFIX +from ddtrace.llmobs._evaluators.ragas.base import BaseRagasEvaluator +from ddtrace.llmobs._evaluators.ragas.base import _get_ml_app_for_ragas_trace logger = get_logger(__name__) -class MiniRagas: - """ - A helper class to store instances of ragas classes and functions - that may or may not exist in a user's environment. - """ - - llm_factory = None - RagasoutputParser = None - faithfulness = None - ensembler = None - get_segmenter = None - StatementFaithfulnessAnswers = None - StatementsAnswers = None - - -def _get_ml_app_for_ragas_trace(span_event: dict) -> str: - """ - The `ml_app` spans generated from traces of ragas will be named as `dd-ragas-` - or `dd-ragas` if `ml_app` is not present in the span event. - """ - tags = span_event.get("tags", []) # list[str] - ml_app = None - for tag in tags: - if isinstance(tag, str) and tag.startswith("ml_app:"): - ml_app = tag.split(":")[1] - break - if not ml_app: - return RAGAS_ML_APP_PREFIX - return "{}-{}".format(RAGAS_ML_APP_PREFIX, ml_app) - - -def _get_faithfulness_instance() -> Optional[object]: - """ - This helper function ensures the faithfulness instance used in - ragas evaluator is updated with the latest ragas faithfulness - instance AND has an non-null llm - """ - if MiniRagas.faithfulness is None: - return None - ragas_faithfulness_instance = MiniRagas.faithfulness - if not ragas_faithfulness_instance.llm: - ragas_faithfulness_instance.llm = MiniRagas.llm_factory() - return ragas_faithfulness_instance - - -class RagasFaithfulnessEvaluator: +class RagasFaithfulnessEvaluator(BaseRagasEvaluator): """A class used by EvaluatorRunner to conduct ragas faithfulness evaluations on LLM Observability span events. The job of an Evaluator is to take a span and submit evaluation metrics based on the span's attributes. @@ -95,98 +44,30 @@ def __init__(self, llmobs_service): Raises: NotImplementedError if the ragas library is not found or if ragas version is not supported. """ - self.llmobs_service = llmobs_service - self.ragas_version = "unknown" - telemetry_state = "ok" - try: - import ragas - - self.ragas_version = parse_version(ragas.__version__) - if self.ragas_version >= (0, 2, 0) or self.ragas_version < (0, 1, 10): - raise NotImplementedError( - "Ragas version: {} is not supported for `ragas_faithfulness` evaluator".format(self.ragas_version), - ) - - from ragas.llms import llm_factory - - MiniRagas.llm_factory = llm_factory - - from ragas.llms.output_parser import RagasoutputParser - - MiniRagas.RagasoutputParser = RagasoutputParser - - from ragas.metrics import faithfulness - - MiniRagas.faithfulness = faithfulness - - from ragas.metrics.base import ensembler - - MiniRagas.ensembler = ensembler - - from ragas.metrics.base import get_segmenter - - MiniRagas.get_segmenter = get_segmenter - - from ddtrace.llmobs._evaluators.ragas.models import StatementFaithfulnessAnswers - - MiniRagas.StatementFaithfulnessAnswers = StatementFaithfulnessAnswers - - from ddtrace.llmobs._evaluators.ragas.models import StatementsAnswers - - MiniRagas.StatementsAnswers = StatementsAnswers - except Exception as e: - telemetry_state = "fail" - telemetry_writer.add_log( - level=TELEMETRY_LOG_LEVEL.ERROR, - message="Failed to import Ragas dependencies", - stack_trace=traceback.format_exc(), - tags={"ragas_version": self.ragas_version}, - ) - raise NotImplementedError("Failed to load dependencies for `ragas_faithfulness` evaluator") from e - finally: - telemetry_writer.add_count_metric( - namespace=TELEMETRY_APM_PRODUCT.LLMOBS, - name="evaluators.init", - value=1, - tags=( - ("evaluator_label", self.LABEL), - ("state", telemetry_state), - ("ragas_version", self.ragas_version), - ), - ) - - self.ragas_faithfulness_instance = _get_faithfulness_instance() - self.llm_output_parser_for_generated_statements = MiniRagas.RagasoutputParser( - pydantic_object=MiniRagas.StatementsAnswers + super().__init__(llmobs_service) + self.ragas_faithfulness_instance = self._get_faithfulness_instance() + self.llm_output_parser_for_generated_statements = self.ragas_dependencies.RagasoutputParser( + pydantic_object=self.ragas_dependencies.StatementsAnswers ) - self.llm_output_parser_for_faithfulness_score = MiniRagas.RagasoutputParser( - pydantic_object=MiniRagas.StatementFaithfulnessAnswers + self.llm_output_parser_for_faithfulness_score = self.ragas_dependencies.RagasoutputParser( + pydantic_object=self.ragas_dependencies.StatementFaithfulnessAnswers ) - self.split_answer_into_sentences = MiniRagas.get_segmenter( + self.split_answer_into_sentences = self.ragas_dependencies.get_segmenter( language=self.ragas_faithfulness_instance.nli_statements_message.language, clean=False ) - def run_and_submit_evaluation(self, span_event: dict): - if not span_event: - return - score_result_or_failure, metric_metadata = self.evaluate(span_event) - telemetry_writer.add_count_metric( - TELEMETRY_APM_PRODUCT.LLMOBS, - "evaluators.run", - 1, - tags=( - ("evaluator_label", self.LABEL), - ("state", score_result_or_failure if isinstance(score_result_or_failure, str) else "success"), - ), - ) - if isinstance(score_result_or_failure, float): - self.llmobs_service.submit_evaluation( - span_context={"trace_id": span_event.get("trace_id"), "span_id": span_event.get("span_id")}, - label=RagasFaithfulnessEvaluator.LABEL, - metric_type=RagasFaithfulnessEvaluator.METRIC_TYPE, - value=score_result_or_failure, - metadata=metric_metadata, - ) + def _get_faithfulness_instance(self) -> Optional[object]: + """ + This helper function ensures the faithfulness instance used in + ragas evaluator is updated with the latest ragas faithfulness + instance AND has an non-null llm + """ + if self.ragas_dependencies.faithfulness is None: + return None + ragas_faithfulness_instance = self.ragas_dependencies.faithfulness + if not ragas_faithfulness_instance.llm: + ragas_faithfulness_instance.llm = self.ragas_dependencies.llm_factory() + return ragas_faithfulness_instance def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]]: """ @@ -196,7 +77,7 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] If the ragas faithfulness instance does not have `llm` set, we set `llm` using the `llm_factory()` method from ragas which defaults to openai's gpt-4o-turbo. """ - self.ragas_faithfulness_instance = _get_faithfulness_instance() + self.ragas_faithfulness_instance = self._get_faithfulness_instance() if not self.ragas_faithfulness_instance: return "fail_faithfulness_is_none", {} @@ -220,16 +101,16 @@ def evaluate(self, span_event: dict) -> Tuple[Union[float, str], Optional[dict]] span=ragas_faithfulness_workflow ) - faithfulness_inputs = self._extract_faithfulness_inputs(span_event) + faithfulness_inputs = self._extract_evaluation_inputs_from_span(span_event) if faithfulness_inputs is None: logger.debug( - "Failed to extract question and context from span sampled for ragas_faithfulness evaluation" + "Failed to extract evaluation inputs from span sampled for `ragas_faithfulness` evaluation" ) return "fail_extract_faithfulness_inputs", evaluation_metadata question = faithfulness_inputs["question"] answer = faithfulness_inputs["answer"] - context = faithfulness_inputs["context"] + context = " ".join(faithfulness_inputs["contexts"]) statements = self._create_statements(question, answer) if statements is None: @@ -318,9 +199,9 @@ def _create_verdicts(self, context: str, statements: List[str]): return None # collapse multiple generations into a single faithfulness list - faithfulness_list = MiniRagas.ensembler.from_discrete(raw_faithfulness_list, "verdict") # type: ignore + faithfulness_list = self.ragas_dependencies.ensembler.from_discrete(raw_faithfulness_list, "verdict") try: - return MiniRagas.StatementFaithfulnessAnswers.parse_obj(faithfulness_list) # type: ignore + return self.ragas_dependencies.StatementFaithfulnessAnswers.parse_obj(faithfulness_list) except Exception as e: logger.debug("Failed to parse faithfulness_list", exc_info=e) return None @@ -330,59 +211,6 @@ def _create_verdicts(self, context: str, statements: List[str]): output_data=faithfulness_list, ) - def _extract_faithfulness_inputs(self, span_event: dict) -> Optional[dict]: - """ - Extracts the question, answer, and context used as inputs to faithfulness - evaluation from a span event. - - question - input.prompt.variables.question OR input.messages[-1].content - context - input.prompt.variables.context - answer - output.messages[-1].content - """ - with self.llmobs_service.workflow("dd-ragas.extract_faithfulness_inputs") as extract_inputs_workflow: - self.llmobs_service.annotate(span=extract_inputs_workflow, input_data=span_event) - question, answer, context = None, None, None - - meta_io = span_event.get("meta") - if meta_io is None: - return None - - meta_input = meta_io.get("input") - meta_output = meta_io.get("output") - - if not (meta_input and meta_output): - return None - - prompt = meta_input.get("prompt") - if prompt is None: - logger.debug("Failed to extract `prompt` from span for `ragas_faithfulness` evaluation") - return None - prompt_variables = prompt.get("variables") - - input_messages = meta_input.get("messages") - - messages = meta_output.get("messages") - if messages is not None and len(messages) > 0: - answer = messages[-1].get("content") - - if prompt_variables: - context_keys = prompt.get(INTERNAL_CONTEXT_VARIABLE_KEYS, ["context"]) - question_keys = prompt.get(INTERNAL_QUERY_VARIABLE_KEYS, ["question"]) - context = " ".join([prompt_variables.get(key) for key in context_keys if prompt_variables.get(key)]) - question = " ".join([prompt_variables.get(key) for key in question_keys if prompt_variables.get(key)]) - - if not question and input_messages is not None and len(input_messages) > 0: - question = input_messages[-1].get("content") - - self.llmobs_service.annotate( - span=extract_inputs_workflow, output_data={"question": question, "context": context, "answer": answer} - ) - if any(field is None for field in (question, context, answer)): - logger.debug("Failed to extract inputs required for faithfulness evaluation") - return None - - return {"question": question, "context": context, "answer": answer} - def _create_statements_prompt(self, answer, question): # Returns: `ragas.llms.PromptValue` object with self.llmobs_service.task("dd-ragas.create_statements_prompt"): diff --git a/ddtrace/llmobs/_evaluators/ragas/models.py b/ddtrace/llmobs/_evaluators/ragas/models.py index 5ee4d433c33..c5b37ee2b7f 100644 --- a/ddtrace/llmobs/_evaluators/ragas/models.py +++ b/ddtrace/llmobs/_evaluators/ragas/models.py @@ -11,6 +11,18 @@ """ +class AnswerRelevanceClassification(BaseModel): + question: str + noncommittal: int + + +class ContextPrecisionVerification(BaseModel): + """Answer for the verification task whether the context was useful.""" + + reason: str = Field(..., description="Reason for verification") + verdict: int = Field(..., description="Binary (0/1) verdict of verification") + + class StatementFaithfulnessAnswer(BaseModel): statement: str = Field(..., description="the original statement, word-by-word") reason: str = Field(..., description="the reason of the verdict") diff --git a/ddtrace/llmobs/_evaluators/runner.py b/ddtrace/llmobs/_evaluators/runner.py index bf45e618e01..056a80000e4 100644 --- a/ddtrace/llmobs/_evaluators/runner.py +++ b/ddtrace/llmobs/_evaluators/runner.py @@ -7,7 +7,9 @@ from ddtrace.internal.logger import get_logger from ddtrace.internal.periodic import PeriodicService from ddtrace.internal.telemetry import telemetry_writer -from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE +from ddtrace.llmobs._evaluators.ragas.answer_relevancy import RagasAnswerRelevancyEvaluator +from ddtrace.llmobs._evaluators.ragas.context_precision import RagasContextPrecisionEvaluator from ddtrace.llmobs._evaluators.ragas.faithfulness import RagasFaithfulnessEvaluator from ddtrace.llmobs._evaluators.sampler import EvaluatorRunnerSampler @@ -17,6 +19,8 @@ SUPPORTED_EVALUATORS = { RagasFaithfulnessEvaluator.LABEL: RagasFaithfulnessEvaluator, + RagasAnswerRelevancyEvaluator.LABEL: RagasAnswerRelevancyEvaluator, + RagasContextPrecisionEvaluator.LABEL: RagasContextPrecisionEvaluator, } @@ -56,7 +60,7 @@ def __init__(self, interval: float, llmobs_service=None, evaluators=None): raise e finally: telemetry_writer.add_count_metric( - namespace=TELEMETRY_APM_PRODUCT.LLMOBS, + namespace=TELEMETRY_NAMESPACE.MLOBS, name="evaluators.init", value=1, tags=( @@ -64,13 +68,15 @@ def __init__(self, interval: float, llmobs_service=None, evaluators=None): ("state", evaluator_init_state), ), ) + else: + raise ValueError("Parsed unsupported evaluator: {}".format(evaluator)) def start(self, *args, **kwargs): if not self.evaluators: logger.debug("no evaluators configured, not starting %r", self.__class__.__name__) return super(EvaluatorRunner, self).start() - logger.debug("started %r to %r", self.__class__.__name__) + logger.debug("started %r", self.__class__.__name__) def _stop_service(self) -> None: """ @@ -109,20 +115,12 @@ def periodic(self, _wait_sync=False) -> None: self._buffer = [] try: - if not _wait_sync: - for evaluator in self.evaluators: - self.executor.map( - lambda span_event: evaluator.run_and_submit_evaluation(span_event), - [ - span_event - for span_event, span in span_events_and_spans - if self.sampler.sample(evaluator.LABEL, span) - ], - ) - else: - for evaluator in self.evaluators: - for span_event, span in span_events_and_spans: - if self.sampler.sample(evaluator.LABEL, span): + for evaluator in self.evaluators: + for span_event, span in span_events_and_spans: + if self.sampler.sample(evaluator.LABEL, span): + if not _wait_sync: + self.executor.submit(evaluator.run_and_submit_evaluation, span_event) + else: evaluator.run_and_submit_evaluation(span_event) except RuntimeError as e: logger.debug("failed to run evaluation: %s", e) diff --git a/ddtrace/llmobs/_evaluators/sampler.py b/ddtrace/llmobs/_evaluators/sampler.py index a959e127606..3598e90f7f3 100644 --- a/ddtrace/llmobs/_evaluators/sampler.py +++ b/ddtrace/llmobs/_evaluators/sampler.py @@ -6,11 +6,11 @@ from typing import Union from ddtrace import config +from ddtrace._trace.sampling_rule import SamplingRule from ddtrace.internal.logger import get_logger from ddtrace.internal.telemetry import telemetry_writer -from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT from ddtrace.internal.telemetry.constants import TELEMETRY_LOG_LEVEL -from ddtrace.sampling_rule import SamplingRule +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE logger = get_logger(__name__) @@ -67,7 +67,7 @@ def parsing_failed_because(msg, maybe_throw_this): TELEMETRY_LOG_LEVEL.ERROR, message="Evaluator sampling parsing failure because: {}".format(msg) ) telemetry_writer.add_count_metric( - namespace=TELEMETRY_APM_PRODUCT.LLMOBS, + namespace=TELEMETRY_NAMESPACE.MLOBS, name="evaluators.error", value=1, tags=(("reason", "sampling_rule_parsing_failure"),), @@ -104,7 +104,7 @@ def parsing_failed_because(msg, maybe_throw_this): span_name = rule.get(EvaluatorRunnerSamplingRule.SPAN_NAME_KEY, SamplingRule.NO_RULE) evaluator_label = rule.get(EvaluatorRunnerSamplingRule.EVALUATOR_LABEL_KEY, SamplingRule.NO_RULE) telemetry_writer.add_distribution_metric( - TELEMETRY_APM_PRODUCT.LLMOBS, + TELEMETRY_NAMESPACE.MLOBS, "evaluators.rule_sample_rate", sample_rate, tags=(("evaluator_label", evaluator_label), ("span_name", span_name)), diff --git a/ddtrace/llmobs/_integrations/base.py b/ddtrace/llmobs/_integrations/base.py index a6968ce0d83..2e892904720 100644 --- a/ddtrace/llmobs/_integrations/base.py +++ b/ddtrace/llmobs/_integrations/base.py @@ -6,8 +6,8 @@ from typing import List # noqa:F401 from typing import Optional # noqa:F401 -from ddtrace import Pin from ddtrace import config +from ddtrace._trace.sampler import RateSampler from ddtrace._trace.span import Span from ddtrace.constants import SPAN_MEASURED_KEY from ddtrace.contrib.trace_utils import int_service @@ -22,8 +22,8 @@ from ddtrace.llmobs._llmobs import LLMObs from ddtrace.llmobs._log_writer import V2LogWriter from ddtrace.llmobs._utils import _get_llmobs_parent_id -from ddtrace.sampler import RateSampler from ddtrace.settings import IntegrationConfig +from ddtrace.trace import Pin log = get_logger(__name__) diff --git a/ddtrace/llmobs/_integrations/openai.py b/ddtrace/llmobs/_integrations/openai.py index bd727b1a5a2..ea660f53f68 100644 --- a/ddtrace/llmobs/_integrations/openai.py +++ b/ddtrace/llmobs/_integrations/openai.py @@ -24,7 +24,7 @@ from ddtrace.llmobs._integrations.base import BaseLLMIntegration from ddtrace.llmobs._utils import _get_attr from ddtrace.llmobs.utils import Document -from ddtrace.pin import Pin +from ddtrace.trace import Pin class OpenAIIntegration(BaseLLMIntegration): diff --git a/ddtrace/llmobs/_llmobs.py b/ddtrace/llmobs/_llmobs.py index 49815151118..b4f1dc1b2f6 100644 --- a/ddtrace/llmobs/_llmobs.py +++ b/ddtrace/llmobs/_llmobs.py @@ -3,7 +3,9 @@ import time from typing import Any from typing import Dict +from typing import List from typing import Optional +from typing import Tuple from typing import Union import ddtrace @@ -11,8 +13,12 @@ from ddtrace import config from ddtrace import patch from ddtrace._trace.context import Context +from ddtrace.constants import ERROR_MSG +from ddtrace.constants import ERROR_STACK +from ddtrace.constants import ERROR_TYPE from ddtrace.ext import SpanTypes from ddtrace.internal import atexit +from ddtrace.internal import core from ddtrace.internal import forksafe from ddtrace.internal._rand import rand64bits from ddtrace.internal.compat import ensure_text @@ -22,8 +28,10 @@ from ddtrace.internal.service import ServiceStatusError from ddtrace.internal.telemetry import telemetry_writer from ddtrace.internal.telemetry.constants import TELEMETRY_APM_PRODUCT +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.formats import parse_tags_str +from ddtrace.llmobs import _constants as constants from ddtrace.llmobs._constants import ANNOTATIONS_CONTEXT_ID from ddtrace.llmobs._constants import INPUT_DOCUMENTS from ddtrace.llmobs._constants import INPUT_MESSAGES @@ -45,11 +53,11 @@ from ddtrace.llmobs._constants import SPAN_START_WHILE_DISABLED_WARNING from ddtrace.llmobs._constants import TAGS from ddtrace.llmobs._evaluators.runner import EvaluatorRunner -from ddtrace.llmobs._trace_processor import LLMObsTraceProcessor from ddtrace.llmobs._utils import AnnotationContext from ddtrace.llmobs._utils import _get_llmobs_parent_id from ddtrace.llmobs._utils import _get_ml_app from ddtrace.llmobs._utils import _get_session_id +from ddtrace.llmobs._utils import _get_span_name from ddtrace.llmobs._utils import _inject_llmobs_parent_id from ddtrace.llmobs._utils import safe_json from ddtrace.llmobs._utils import validate_prompt @@ -59,6 +67,7 @@ from ddtrace.llmobs.utils import ExportedLLMObsSpan from ddtrace.llmobs.utils import Messages from ddtrace.propagation.http import HTTPPropagator +from ddtrace.vendor.debtcollector import deprecate log = get_logger(__name__) @@ -81,34 +90,157 @@ class LLMObs(Service): def __init__(self, tracer=None): super(LLMObs, self).__init__() self.tracer = tracer or ddtrace.tracer - self._llmobs_span_writer = None - self._llmobs_span_writer = LLMObsSpanWriter( is_agentless=config._llmobs_agentless_enabled, interval=float(os.getenv("_DD_LLMOBS_WRITER_INTERVAL", 1.0)), timeout=float(os.getenv("_DD_LLMOBS_WRITER_TIMEOUT", 5.0)), ) - self._llmobs_eval_metric_writer = LLMObsEvalMetricWriter( site=config._dd_site, api_key=config._dd_api_key, interval=float(os.getenv("_DD_LLMOBS_WRITER_INTERVAL", 1.0)), timeout=float(os.getenv("_DD_LLMOBS_WRITER_TIMEOUT", 5.0)), ) - self._evaluator_runner = EvaluatorRunner( interval=float(os.getenv("_DD_LLMOBS_EVALUATOR_INTERVAL", 1.0)), llmobs_service=self, ) - self._trace_processor = LLMObsTraceProcessor(self._llmobs_span_writer, self._evaluator_runner) forksafe.register(self._child_after_fork) self._annotations = [] self._annotation_context_lock = forksafe.RLock() - self.tracer.on_start_span(self._do_annotations) - def _do_annotations(self, span): + def _on_span_start(self, span): + if self.enabled and span.span_type == SpanTypes.LLM: + self._do_annotations(span) + + def _on_span_finish(self, span): + if self.enabled and span.span_type == SpanTypes.LLM: + self._submit_llmobs_span(span) + + def _submit_llmobs_span(self, span: Span) -> None: + """Generate and submit an LLMObs span event to be sent to LLMObs.""" + span_event = None + is_llm_span = span._get_ctx_item(SPAN_KIND) == "llm" + is_ragas_integration_span = False + try: + span_event, is_ragas_integration_span = self._llmobs_span_event(span) + self._llmobs_span_writer.enqueue(span_event) + except (KeyError, TypeError): + log.error( + "Error generating LLMObs span event for span %s, likely due to malformed span", span, exc_info=True + ) + finally: + if not span_event or not is_llm_span or is_ragas_integration_span: + return + if self._evaluator_runner: + self._evaluator_runner.enqueue(span_event, span) + + @classmethod + def _llmobs_span_event(cls, span: Span) -> Tuple[Dict[str, Any], bool]: + """Span event object structure.""" + span_kind = span._get_ctx_item(SPAN_KIND) + if not span_kind: + raise KeyError("Span kind not found in span context") + meta: Dict[str, Any] = {"span.kind": span_kind, "input": {}, "output": {}} + if span_kind in ("llm", "embedding") and span._get_ctx_item(MODEL_NAME) is not None: + meta["model_name"] = span._get_ctx_item(MODEL_NAME) + meta["model_provider"] = (span._get_ctx_item(MODEL_PROVIDER) or "custom").lower() + meta["metadata"] = span._get_ctx_item(METADATA) or {} + if span._get_ctx_item(INPUT_PARAMETERS): + meta["input"]["parameters"] = span._get_ctx_item(INPUT_PARAMETERS) + if span_kind == "llm" and span._get_ctx_item(INPUT_MESSAGES) is not None: + meta["input"]["messages"] = span._get_ctx_item(INPUT_MESSAGES) + if span._get_ctx_item(INPUT_VALUE) is not None: + meta["input"]["value"] = safe_json(span._get_ctx_item(INPUT_VALUE)) + if span_kind == "llm" and span._get_ctx_item(OUTPUT_MESSAGES) is not None: + meta["output"]["messages"] = span._get_ctx_item(OUTPUT_MESSAGES) + if span_kind == "embedding" and span._get_ctx_item(INPUT_DOCUMENTS) is not None: + meta["input"]["documents"] = span._get_ctx_item(INPUT_DOCUMENTS) + if span._get_ctx_item(OUTPUT_VALUE) is not None: + meta["output"]["value"] = safe_json(span._get_ctx_item(OUTPUT_VALUE)) + if span_kind == "retrieval" and span._get_ctx_item(OUTPUT_DOCUMENTS) is not None: + meta["output"]["documents"] = span._get_ctx_item(OUTPUT_DOCUMENTS) + if span._get_ctx_item(INPUT_PROMPT) is not None: + prompt_json_str = span._get_ctx_item(INPUT_PROMPT) + if span_kind != "llm": + log.warning( + "Dropping prompt on non-LLM span kind, annotating prompts is only supported for LLM span kinds." + ) + else: + meta["input"]["prompt"] = prompt_json_str + if span.error: + meta.update( + { + ERROR_MSG: span.get_tag(ERROR_MSG), + ERROR_STACK: span.get_tag(ERROR_STACK), + ERROR_TYPE: span.get_tag(ERROR_TYPE), + } + ) + if not meta["input"]: + meta.pop("input") + if not meta["output"]: + meta.pop("output") + metrics = span._get_ctx_item(METRICS) or {} + ml_app = _get_ml_app(span) + + is_ragas_integration_span = False + + if ml_app.startswith(constants.RAGAS_ML_APP_PREFIX): + is_ragas_integration_span = True + + span._set_ctx_item(ML_APP, ml_app) + parent_id = str(_get_llmobs_parent_id(span) or "undefined") + + llmobs_span_event = { + "trace_id": "{:x}".format(span.trace_id), + "span_id": str(span.span_id), + "parent_id": parent_id, + "name": _get_span_name(span), + "start_ns": span.start_ns, + "duration": span.duration_ns, + "status": "error" if span.error else "ok", + "meta": meta, + "metrics": metrics, + } + session_id = _get_session_id(span) + if session_id is not None: + span._set_ctx_item(SESSION_ID, session_id) + llmobs_span_event["session_id"] = session_id + + llmobs_span_event["tags"] = cls._llmobs_tags( + span, ml_app, session_id, is_ragas_integration_span=is_ragas_integration_span + ) + return llmobs_span_event, is_ragas_integration_span + + @staticmethod + def _llmobs_tags( + span: Span, ml_app: str, session_id: Optional[str] = None, is_ragas_integration_span: bool = False + ) -> List[str]: + tags = { + "version": config.version or "", + "env": config.env or "", + "service": span.service or "", + "source": "integration", + "ml_app": ml_app, + "ddtrace.version": ddtrace.__version__, + "language": "python", + "error": span.error, + } + err_type = span.get_tag(ERROR_TYPE) + if err_type: + tags["error_type"] = err_type + if session_id: + tags["session_id"] = session_id + if is_ragas_integration_span: + tags[constants.RUNNER_IS_INTEGRATION_SPAN_TAG] = "ragas" + existing_tags = span._get_ctx_item(TAGS) + if existing_tags is not None: + tags.update(existing_tags) + return ["{}:{}".format(k, v) for k, v in tags.items()] + + def _do_annotations(self, span: Span) -> None: # get the current span context # only do the annotations if it matches the context if span.span_type != SpanTypes.LLM: # do this check to avoid the warning log in `annotate` @@ -120,20 +252,14 @@ def _do_annotations(self, span): if current_context_id == context_id: self.annotate(span, **annotation_kwargs) - def _child_after_fork(self): + def _child_after_fork(self) -> None: self._llmobs_span_writer = self._llmobs_span_writer.recreate() self._llmobs_eval_metric_writer = self._llmobs_eval_metric_writer.recreate() self._evaluator_runner = self._evaluator_runner.recreate() - self._trace_processor._span_writer = self._llmobs_span_writer - self._trace_processor._evaluator_runner = self._evaluator_runner if self.enabled: self._start_service() def _start_service(self) -> None: - tracer_filters = self.tracer._filters - if not any(isinstance(tracer_filter, LLMObsTraceProcessor) for tracer_filter in tracer_filters): - tracer_filters += [self._trace_processor] - self.tracer.configure(settings={"FILTERS": tracer_filters}) try: self._llmobs_span_writer.start() self._llmobs_eval_metric_writer.start() @@ -146,6 +272,10 @@ def _start_service(self) -> None: log.debug("Error starting evaluator runner") def _stop_service(self) -> None: + # Remove listener hooks for span events + core.reset_listeners("trace.span_start", self._on_span_start) + core.reset_listeners("trace.span_finish", self._on_span_finish) + try: self._evaluator_runner.stop() # flush remaining evaluation spans & evaluations @@ -160,11 +290,7 @@ def _stop_service(self) -> None: except ServiceStatusError: log.debug("Error stopping LLMObs writers") - try: - forksafe.unregister(self._child_after_fork) - self.tracer.shutdown() - except Exception: - log.warning("Failed to shutdown tracer", exc_info=True) + forksafe.unregister(self._child_after_fork) @classmethod def enable( @@ -244,6 +370,10 @@ def enable( cls.enabled = True cls._instance.start() + # Register hooks for span events + core.on("trace.span_start", cls._instance._on_span_start) + core.on("trace.span_finish", cls._instance._on_span_finish) + atexit.register(cls.disable) telemetry_writer.product_activated(TELEMETRY_APM_PRODUCT.LLMOBS, True) @@ -265,7 +395,6 @@ def disable(cls) -> None: cls._instance.stop() cls.enabled = False - cls._instance.tracer.deregister_on_start_span(cls._instance._do_annotations) telemetry_writer.product_activated(TELEMETRY_APM_PRODUCT.LLMOBS, False) log.debug("%s disabled", cls.__name__) @@ -785,6 +914,127 @@ def _tag_metrics(span: Span, metrics: Dict[str, Any]) -> None: return span._set_ctx_item(METRICS, metrics) + @classmethod + def submit_evaluation_for( + cls, + label: str, + metric_type: str, + value: Union[str, int, float], + span: Optional[dict] = None, + span_with_tag_value: Optional[Dict[str, str]] = None, + tags: Optional[Dict[str, str]] = None, + ml_app: Optional[str] = None, + timestamp_ms: Optional[int] = None, + ) -> None: + """ + Submits a custom evaluation metric for a given span. + + :param str label: The name of the evaluation metric. + :param str metric_type: The type of the evaluation metric. One of "categorical", "score". + :param value: The value of the evaluation metric. + Must be a string (categorical), integer (score), or float (score). + :param dict span: A dictionary of shape {'span_id': str, 'trace_id': str} uniquely identifying + the span associated with this evaluation. + :param dict span_with_tag_value: A dictionary with the format {'tag_key': str, 'tag_value': str} + uniquely identifying the span associated with this evaluation. + :param tags: A dictionary of string key-value pairs to tag the evaluation metric with. + :param str ml_app: The name of the ML application + :param int timestamp_ms: The unix timestamp in milliseconds when the evaluation metric result was generated. + If not set, the current time will be used. + """ + if cls.enabled is False: + log.debug( + "LLMObs.submit_evaluation_for() called when LLMObs is not enabled. ", + "Evaluation metric data will not be sent.", + ) + return + + has_exactly_one_joining_key = (span is not None) ^ (span_with_tag_value is not None) + + if not has_exactly_one_joining_key: + raise ValueError( + "Exactly one of `span` or `span_with_tag_value` must be specified to submit an evaluation metric." + ) + + join_on = {} + if span is not None: + if ( + not isinstance(span, dict) + or not isinstance(span.get("span_id"), str) + or not isinstance(span.get("trace_id"), str) + ): + raise TypeError( + "`span` must be a dictionary containing both span_id and trace_id keys. " + "LLMObs.export_span() can be used to generate this dictionary from a given span." + ) + join_on["span"] = span + elif span_with_tag_value is not None: + if ( + not isinstance(span_with_tag_value, dict) + or not isinstance(span_with_tag_value.get("tag_key"), str) + or not isinstance(span_with_tag_value.get("tag_value"), str) + ): + raise TypeError( + "`span_with_tag_value` must be a dict with keys 'tag_key' and 'tag_value' containing string values" + ) + join_on["tag"] = { + "key": span_with_tag_value.get("tag_key"), + "value": span_with_tag_value.get("tag_value"), + } + + timestamp_ms = timestamp_ms if timestamp_ms else int(time.time() * 1000) + + if not isinstance(timestamp_ms, int) or timestamp_ms < 0: + raise ValueError("timestamp_ms must be a non-negative integer. Evaluation metric data will not be sent") + + if not label: + raise ValueError("label must be the specified name of the evaluation metric.") + + metric_type = metric_type.lower() + if metric_type not in ("categorical", "score"): + raise ValueError("metric_type must be one of 'categorical' or 'score'.") + + if metric_type == "categorical" and not isinstance(value, str): + raise TypeError("value must be a string for a categorical metric.") + if metric_type == "score" and not isinstance(value, (int, float)): + raise TypeError("value must be an integer or float for a score metric.") + + if tags is not None and not isinstance(tags, dict): + log.warning("tags must be a dictionary of string key-value pairs.") + tags = {} + + evaluation_tags = { + "ddtrace.version": ddtrace.__version__, + "ml_app": ml_app, + } + + if tags: + for k, v in tags.items(): + try: + evaluation_tags[ensure_text(k)] = ensure_text(v) + except TypeError: + log.warning("Failed to parse tags. Tags for evaluation metrics must be strings.") + + ml_app = ml_app if ml_app else config._llmobs_ml_app + if not ml_app: + log.warning( + "ML App name is required for sending evaluation metrics. Evaluation metric data will not be sent. " + "Ensure this configuration is set before running your application." + ) + return + + evaluation_metric = { + "join_on": join_on, + "label": str(label), + "metric_type": metric_type, + "timestamp_ms": timestamp_ms, + "{}_value".format(metric_type): value, + "ml_app": ml_app, + "tags": ["{}:{}".format(k, v) for k, v in evaluation_tags.items()], + } + + cls._instance._llmobs_eval_metric_writer.enqueue(evaluation_metric) + @classmethod def submit_evaluation( cls, @@ -797,6 +1047,13 @@ def submit_evaluation( timestamp_ms: Optional[int] = None, metadata: Optional[Dict[str, object]] = None, ) -> None: + deprecate( + "Using `LLMObs.submit_evaluation` is deprecated", + message="Please use `LLMObs.submit_evaluation_for` instead.", + removal_version="3.0.0", + category=DDTraceDeprecationWarning, + ) + """ Submits a custom evaluation metric for a given span ID and trace ID. @@ -812,7 +1069,7 @@ def submit_evaluation( evaluation metric. """ if cls.enabled is False: - log.warning( + log.debug( "LLMObs.submit_evaluation() called when LLMObs is not enabled. Evaluation metric data will not be sent." ) return @@ -888,8 +1145,7 @@ def submit_evaluation( log.warning("Failed to parse tags. Tags for evaluation metrics must be strings.") evaluation_metric = { - "span_id": span_id, - "trace_id": trace_id, + "join_on": {"span": {"span_id": span_id, "trace_id": trace_id}}, "label": str(label), "metric_type": metric_type.lower(), "timestamp_ms": timestamp_ms, diff --git a/ddtrace/llmobs/_trace_processor.py b/ddtrace/llmobs/_trace_processor.py deleted file mode 100644 index 231d53d7626..00000000000 --- a/ddtrace/llmobs/_trace_processor.py +++ /dev/null @@ -1,177 +0,0 @@ -from typing import Any -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple - -import ddtrace -from ddtrace import Span -from ddtrace import config -from ddtrace._trace.processor import TraceProcessor -from ddtrace.constants import ERROR_MSG -from ddtrace.constants import ERROR_STACK -from ddtrace.constants import ERROR_TYPE -from ddtrace.ext import SpanTypes -from ddtrace.internal.logger import get_logger -from ddtrace.llmobs._constants import INPUT_DOCUMENTS -from ddtrace.llmobs._constants import INPUT_MESSAGES -from ddtrace.llmobs._constants import INPUT_PARAMETERS -from ddtrace.llmobs._constants import INPUT_PROMPT -from ddtrace.llmobs._constants import INPUT_VALUE -from ddtrace.llmobs._constants import METADATA -from ddtrace.llmobs._constants import METRICS -from ddtrace.llmobs._constants import ML_APP -from ddtrace.llmobs._constants import MODEL_NAME -from ddtrace.llmobs._constants import MODEL_PROVIDER -from ddtrace.llmobs._constants import OUTPUT_DOCUMENTS -from ddtrace.llmobs._constants import OUTPUT_MESSAGES -from ddtrace.llmobs._constants import OUTPUT_VALUE -from ddtrace.llmobs._constants import RAGAS_ML_APP_PREFIX -from ddtrace.llmobs._constants import RUNNER_IS_INTEGRATION_SPAN_TAG -from ddtrace.llmobs._constants import SESSION_ID -from ddtrace.llmobs._constants import SPAN_KIND -from ddtrace.llmobs._constants import TAGS -from ddtrace.llmobs._utils import _get_llmobs_parent_id -from ddtrace.llmobs._utils import _get_ml_app -from ddtrace.llmobs._utils import _get_session_id -from ddtrace.llmobs._utils import _get_span_name -from ddtrace.llmobs._utils import safe_json - - -log = get_logger(__name__) - - -class LLMObsTraceProcessor(TraceProcessor): - """ - Processor that extracts LLM-type spans in a trace to submit as separate LLMObs span events to LLM Observability. - """ - - def __init__(self, llmobs_span_writer, evaluator_runner=None): - self._span_writer = llmobs_span_writer - self._evaluator_runner = evaluator_runner - - def process_trace(self, trace: List[Span]) -> Optional[List[Span]]: - if not trace: - return None - for span in trace: - if span.span_type == SpanTypes.LLM: - self.submit_llmobs_span(span) - return None if config._llmobs_agentless_enabled else trace - - def submit_llmobs_span(self, span: Span) -> None: - """Generate and submit an LLMObs span event to be sent to LLMObs.""" - span_event = None - is_llm_span = span._get_ctx_item(SPAN_KIND) == "llm" - is_ragas_integration_span = False - try: - span_event, is_ragas_integration_span = self._llmobs_span_event(span) - self._span_writer.enqueue(span_event) - except (KeyError, TypeError): - log.error("Error generating LLMObs span event for span %s, likely due to malformed span", span) - finally: - if not span_event or not is_llm_span or is_ragas_integration_span: - return - if self._evaluator_runner: - self._evaluator_runner.enqueue(span_event, span) - - def _llmobs_span_event(self, span: Span) -> Tuple[Dict[str, Any], bool]: - """Span event object structure.""" - span_kind = span._get_ctx_item(SPAN_KIND) - if not span_kind: - raise KeyError("Span kind not found in span context") - meta: Dict[str, Any] = {"span.kind": span_kind, "input": {}, "output": {}} - if span_kind in ("llm", "embedding") and span._get_ctx_item(MODEL_NAME) is not None: - meta["model_name"] = span._get_ctx_item(MODEL_NAME) - meta["model_provider"] = (span._get_ctx_item(MODEL_PROVIDER) or "custom").lower() - meta["metadata"] = span._get_ctx_item(METADATA) or {} - if span._get_ctx_item(INPUT_PARAMETERS): - meta["input"]["parameters"] = span._get_ctx_item(INPUT_PARAMETERS) - if span_kind == "llm" and span._get_ctx_item(INPUT_MESSAGES) is not None: - meta["input"]["messages"] = span._get_ctx_item(INPUT_MESSAGES) - if span._get_ctx_item(INPUT_VALUE) is not None: - meta["input"]["value"] = safe_json(span._get_ctx_item(INPUT_VALUE)) - if span_kind == "llm" and span._get_ctx_item(OUTPUT_MESSAGES) is not None: - meta["output"]["messages"] = span._get_ctx_item(OUTPUT_MESSAGES) - if span_kind == "embedding" and span._get_ctx_item(INPUT_DOCUMENTS) is not None: - meta["input"]["documents"] = span._get_ctx_item(INPUT_DOCUMENTS) - if span._get_ctx_item(OUTPUT_VALUE) is not None: - meta["output"]["value"] = safe_json(span._get_ctx_item(OUTPUT_VALUE)) - if span_kind == "retrieval" and span._get_ctx_item(OUTPUT_DOCUMENTS) is not None: - meta["output"]["documents"] = span._get_ctx_item(OUTPUT_DOCUMENTS) - if span._get_ctx_item(INPUT_PROMPT) is not None: - prompt_json_str = span._get_ctx_item(INPUT_PROMPT) - if span_kind != "llm": - log.warning( - "Dropping prompt on non-LLM span kind, annotating prompts is only supported for LLM span kinds." - ) - else: - meta["input"]["prompt"] = prompt_json_str - if span.error: - meta.update( - { - ERROR_MSG: span.get_tag(ERROR_MSG), - ERROR_STACK: span.get_tag(ERROR_STACK), - ERROR_TYPE: span.get_tag(ERROR_TYPE), - } - ) - if not meta["input"]: - meta.pop("input") - if not meta["output"]: - meta.pop("output") - metrics = span._get_ctx_item(METRICS) or {} - ml_app = _get_ml_app(span) - - is_ragas_integration_span = False - - if ml_app.startswith(RAGAS_ML_APP_PREFIX): - is_ragas_integration_span = True - - span._set_ctx_item(ML_APP, ml_app) - parent_id = str(_get_llmobs_parent_id(span) or "undefined") - - llmobs_span_event = { - "trace_id": "{:x}".format(span.trace_id), - "span_id": str(span.span_id), - "parent_id": parent_id, - "name": _get_span_name(span), - "start_ns": span.start_ns, - "duration": span.duration_ns, - "status": "error" if span.error else "ok", - "meta": meta, - "metrics": metrics, - } - session_id = _get_session_id(span) - if session_id is not None: - span._set_ctx_item(SESSION_ID, session_id) - llmobs_span_event["session_id"] = session_id - - llmobs_span_event["tags"] = self._llmobs_tags( - span, ml_app, session_id, is_ragas_integration_span=is_ragas_integration_span - ) - return llmobs_span_event, is_ragas_integration_span - - @staticmethod - def _llmobs_tags( - span: Span, ml_app: str, session_id: Optional[str] = None, is_ragas_integration_span: bool = False - ) -> List[str]: - tags = { - "version": config.version or "", - "env": config.env or "", - "service": span.service or "", - "source": "integration", - "ml_app": ml_app, - "ddtrace.version": ddtrace.__version__, - "language": "python", - "error": span.error, - } - err_type = span.get_tag(ERROR_TYPE) - if err_type: - tags["error_type"] = err_type - if session_id: - tags["session_id"] = session_id - if is_ragas_integration_span: - tags[RUNNER_IS_INTEGRATION_SPAN_TAG] = "ragas" - existing_tags = span._get_ctx_item(TAGS) - if existing_tags is not None: - tags.update(existing_tags) - return ["{}:{}".format(k, v) for k, v in tags.items()] diff --git a/ddtrace/llmobs/_utils.py b/ddtrace/llmobs/_utils.py index c1b1c4a776c..dd616db8bef 100644 --- a/ddtrace/llmobs/_utils.py +++ b/ddtrace/llmobs/_utils.py @@ -135,9 +135,12 @@ def _get_ml_app(span: Span) -> str: ml_app = span._get_ctx_item(ML_APP) if ml_app: return ml_app - nearest_llmobs_ancestor = _get_nearest_llmobs_ancestor(span) - if nearest_llmobs_ancestor: - ml_app = nearest_llmobs_ancestor._get_ctx_item(ML_APP) + llmobs_parent = _get_nearest_llmobs_ancestor(span) + while llmobs_parent: + ml_app = llmobs_parent._get_ctx_item(ML_APP) + if ml_app is not None: + return ml_app + llmobs_parent = _get_nearest_llmobs_ancestor(llmobs_parent) return ml_app or config._llmobs_ml_app or "unknown-ml-app" @@ -149,9 +152,12 @@ def _get_session_id(span: Span) -> Optional[str]: session_id = span._get_ctx_item(SESSION_ID) if session_id: return session_id - nearest_llmobs_ancestor = _get_nearest_llmobs_ancestor(span) - if nearest_llmobs_ancestor: - session_id = nearest_llmobs_ancestor._get_ctx_item(SESSION_ID) + llmobs_parent = _get_nearest_llmobs_ancestor(span) + while llmobs_parent: + session_id = llmobs_parent._get_ctx_item(SESSION_ID) + if session_id is not None: + return session_id + llmobs_parent = _get_nearest_llmobs_ancestor(llmobs_parent) return session_id diff --git a/ddtrace/llmobs/_writer.py b/ddtrace/llmobs/_writer.py index 5a293f05c4e..5880019d67f 100644 --- a/ddtrace/llmobs/_writer.py +++ b/ddtrace/llmobs/_writer.py @@ -55,8 +55,7 @@ class LLMObsSpanEvent(TypedDict): class LLMObsEvaluationMetricEvent(TypedDict, total=False): - span_id: str - trace_id: str + join_on: Dict[str, Dict[str, str]] metric_type: str label: str categorical_value: str @@ -107,6 +106,13 @@ def periodic(self) -> None: events = self._buffer self._buffer = [] + if not self._headers.get("DD-API-KEY"): + logger.warning( + "DD_API_KEY is required for sending evaluation metrics. Evaluation metric data will not be sent. ", + "Ensure this configuration is set before running your application.", + ) + return + data = self._data(events) enc_llm_events = safe_json(data) conn = httplib.HTTPSConnection(self._intake, 443, timeout=self._timeout) @@ -154,7 +160,7 @@ def __init__(self, site: str, api_key: str, interval: float, timeout: float) -> super(LLMObsEvalMetricWriter, self).__init__(site, api_key, interval, timeout) self._event_type = "evaluation_metric" self._buffer = [] - self._endpoint = "/api/intake/llm-obs/v1/eval-metric" + self._endpoint = "/api/intake/llm-obs/v2/eval-metric" self._intake = "api.%s" % self._site # type: str def enqueue(self, event: LLMObsEvaluationMetricEvent) -> None: diff --git a/ddtrace/opentracer/tracer.py b/ddtrace/opentracer/tracer.py index 489c025037a..110fad2401c 100644 --- a/ddtrace/opentracer/tracer.py +++ b/ddtrace/opentracer/tracer.py @@ -18,6 +18,7 @@ from ddtrace.internal.constants import SPAN_API_OPENTRACING from ddtrace.internal.utils.config import get_application_name from ddtrace.settings import ConfigException +from ddtrace.vendor.debtcollector import deprecate from ..internal.logger import get_logger from .propagation import HTTPPropagator @@ -70,8 +71,8 @@ def __init__( If ``None`` is provided, defaults to :class:`opentracing.scope_managers.ThreadLocalScopeManager`. :param dd_tracer: (optional) the Datadog tracer for this tracer to use. This - should only be passed if a custom Datadog tracer is being used. Defaults - to the global ``ddtrace.tracer`` tracer. + parameter is deprecated and will be removed in v3.0.0. The + to the global tracer (``ddtrace.tracer``) should always be used. """ # Merge the given config with the default into a new dict self._config = DEFAULT_CONFIG.copy() @@ -99,7 +100,14 @@ def __init__( self._scope_manager = scope_manager or ThreadLocalScopeManager() dd_context_provider = get_context_provider_for_scope_manager(self._scope_manager) - self._dd_tracer = dd_tracer or ddtrace.tracer or DatadogTracer() + if dd_tracer is not None: + deprecate( + "The ``dd_tracer`` parameter is deprecated", + message="The global tracer (``ddtrace.tracer``) will be used instead.", + removal_version="3.0.0", + ) + + self._dd_tracer = dd_tracer or ddtrace.tracer self._dd_tracer.set_tags(self._config.get(keys.GLOBAL_TAGS)) # type: ignore[arg-type] self._dd_tracer.configure( enabled=self._config.get(keys.ENABLED), diff --git a/ddtrace/pin.py b/ddtrace/pin.py index 926918b6cea..0e683b3b22e 100644 --- a/ddtrace/pin.py +++ b/ddtrace/pin.py @@ -1,209 +1,10 @@ -from typing import TYPE_CHECKING # noqa:F401 -from typing import Any # noqa:F401 -from typing import Dict # noqa:F401 -from typing import Optional # noqa:F401 +from ddtrace._trace.pin import * # noqa: F403 +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate -import wrapt -import ddtrace - -from .internal.logger import get_logger - - -log = get_logger(__name__) - - -# To set attributes on wrapt proxy objects use this prefix: -# http://wrapt.readthedocs.io/en/latest/wrappers.html -_DD_PIN_NAME = "_datadog_pin" -_DD_PIN_PROXY_NAME = "_self_" + _DD_PIN_NAME - - -class Pin(object): - """Pin (a.k.a Patch INfo) is a small class which is used to - set tracing metadata on a particular traced connection. - This is useful if you wanted to, say, trace two different - database clusters. - - >>> conn = sqlite.connect('/tmp/user.db') - >>> # Override a pin for a specific connection - >>> pin = Pin.override(conn, service='user-db') - >>> conn = sqlite.connect('/tmp/image.db') - """ - - __slots__ = ["tags", "tracer", "_target", "_config", "_initialized"] - - def __init__( - self, - service=None, # type: Optional[str] - tags=None, # type: Optional[Dict[str, str]] - tracer=None, - _config=None, # type: Optional[Dict[str, Any]] - ): - # type: (...) -> None - tracer = tracer or ddtrace.tracer - self.tags = tags - self.tracer = tracer - self._target = None # type: Optional[int] - # keep the configuration attribute internal because the - # public API to access it is not the Pin class - self._config = _config or {} # type: Dict[str, Any] - # [Backward compatibility]: service argument updates the `Pin` config - self._config["service_name"] = service - self._initialized = True - - @property - def service(self): - # type: () -> str - """Backward compatibility: accessing to `pin.service` returns the underlying - configuration value. - """ - return self._config["service_name"] - - def __setattr__(self, name, value): - if getattr(self, "_initialized", False) and name != "_target": - raise AttributeError("can't mutate a pin, use override() or clone() instead") - super(Pin, self).__setattr__(name, value) - - def __repr__(self): - return "Pin(service=%s, tags=%s, tracer=%s)" % (self.service, self.tags, self.tracer) - - @staticmethod - def _find(*objs): - # type: (Any) -> Optional[Pin] - """ - Return the first :class:`ddtrace.pin.Pin` found on any of the provided objects or `None` if none were found - - - >>> pin = Pin._find(wrapper, instance, conn) - - :param objs: The objects to search for a :class:`ddtrace.pin.Pin` on - :type objs: List of objects - :rtype: :class:`ddtrace.pin.Pin`, None - :returns: The first found :class:`ddtrace.pin.Pin` or `None` is none was found - """ - for obj in objs: - pin = Pin.get_from(obj) - if pin: - return pin - return None - - @staticmethod - def get_from(obj): - # type: (Any) -> Optional[Pin] - """Return the pin associated with the given object. If a pin is attached to - `obj` but the instance is not the owner of the pin, a new pin is cloned and - attached. This ensures that a pin inherited from a class is a copy for the new - instance, avoiding that a specific instance overrides other pins values. - - >>> pin = Pin.get_from(conn) - - :param obj: The object to look for a :class:`ddtrace.pin.Pin` on - :type obj: object - :rtype: :class:`ddtrace.pin.Pin`, None - :returns: :class:`ddtrace.pin.Pin` associated with the object, or None if none was found - """ - if hasattr(obj, "__getddpin__"): - return obj.__getddpin__() - - pin_name = _DD_PIN_PROXY_NAME if isinstance(obj, wrapt.ObjectProxy) else _DD_PIN_NAME - pin = getattr(obj, pin_name, None) - # detect if the PIN has been inherited from a class - if pin is not None and pin._target != id(obj): - pin = pin.clone() - pin.onto(obj) - return pin - - @classmethod - def override( - cls, - obj, # type: Any - service=None, # type: Optional[str] - tags=None, # type: Optional[Dict[str, str]] - tracer=None, - ): - # type: (...) -> None - """Override an object with the given attributes. - - That's the recommended way to customize an already instrumented client, without - losing existing attributes. - - >>> conn = sqlite.connect('/tmp/user.db') - >>> # Override a pin for a specific connection - >>> Pin.override(conn, service='user-db') - """ - if not obj: - return - - pin = cls.get_from(obj) - if pin is None: - Pin(service=service, tags=tags, tracer=tracer).onto(obj) - else: - pin.clone(service=service, tags=tags, tracer=tracer).onto(obj) - - def enabled(self): - # type: () -> bool - """Return true if this pin's tracer is enabled.""" - # inline to avoid circular imports - from ddtrace.settings.asm import config as asm_config - - return bool(self.tracer) and (self.tracer.enabled or asm_config._apm_opt_out) - - def onto(self, obj, send=True): - # type: (Any, bool) -> None - """Patch this pin onto the given object. If send is true, it will also - queue the metadata to be sent to the server. - """ - # Actually patch it on the object. - try: - if hasattr(obj, "__setddpin__"): - return obj.__setddpin__(self) - - pin_name = _DD_PIN_PROXY_NAME if isinstance(obj, wrapt.ObjectProxy) else _DD_PIN_NAME - - # set the target reference; any get_from, clones and retarget the new PIN - self._target = id(obj) - if self.service: - ddtrace.config._add_extra_service(self.service) - return setattr(obj, pin_name, self) - except AttributeError: - log.debug("can't pin onto object. skipping", exc_info=True) - - def remove_from(self, obj): - # type: (Any) -> None - # Remove pin from the object. - try: - pin_name = _DD_PIN_PROXY_NAME if isinstance(obj, wrapt.ObjectProxy) else _DD_PIN_NAME - - pin = Pin.get_from(obj) - if pin is not None: - delattr(obj, pin_name) - except AttributeError: - log.debug("can't remove pin from object. skipping", exc_info=True) - - def clone( - self, - service=None, # type: Optional[str] - tags=None, # type: Optional[Dict[str, str]] - tracer=None, - ): - # type: (...) -> Pin - """Return a clone of the pin with the given attributes replaced.""" - # do a shallow copy of Pin dicts - if not tags and self.tags: - tags = self.tags.copy() - - # we use a copy instead of a deepcopy because we expect configurations - # to have only a root level dictionary without nested objects. Using - # deepcopy introduces a big overhead: - # - # copy: 0.00654911994934082 - # deepcopy: 0.2787208557128906 - config = self._config.copy() - - return Pin( - service=service or self.service, - tags=tags, - tracer=tracer or self.tracer, # do not clone the Tracer - _config=config, - ) +deprecate( + "The ddtrace.trace.Pin module is deprecated and will be removed.", + message="Import ``Pin`` from the ddtrace.trace package.", + category=DDTraceDeprecationWarning, +) diff --git a/ddtrace/profiling/collector/_lock.py b/ddtrace/profiling/collector/_lock.py index 6dedf3295f7..ec0c438b2b9 100644 --- a/ddtrace/profiling/collector/_lock.py +++ b/ddtrace/profiling/collector/_lock.py @@ -179,69 +179,64 @@ def acquire(self, *args, **kwargs): def _release(self, inner_func, *args, **kwargs): # type (typing.Any, typing.Any) -> None + + start = None + if hasattr(self, "_self_acquired_at"): + # _self_acquired_at is only set when the acquire was captured + # if it's not set, we're not capturing the release + start = self._self_acquired_at + del self._self_acquired_at + try: return inner_func(*args, **kwargs) finally: - try: - if hasattr(self, "_self_acquired_at"): - try: - end = time.monotonic_ns() - thread_id, thread_name = _current_thread() - task_id, task_name, task_frame = _task.get_task(thread_id) - lock_name = ( - "%s:%s" % (self._self_init_loc, self._self_name) if self._self_name else self._self_init_loc - ) - - if task_frame is None: - # See the comments in _acquire - frame = sys._getframe(2) - else: - frame = task_frame - - frames, nframes = _traceback.pyframe_to_frames(frame, self._self_max_nframes) - - if self._self_export_libdd_enabled: - thread_native_id = _threading.get_thread_native_id(thread_id) - - handle = ddup.SampleHandle() - handle.push_monotonic_ns(end) - handle.push_lock_name(lock_name) - handle.push_release( - end - self._self_acquired_at, 1 - ) # AFAICT, capture_pct does not adjust anything here - handle.push_threadinfo(thread_id, thread_native_id, thread_name) - handle.push_task_id(task_id) - handle.push_task_name(task_name) - - if self._self_tracer is not None: - handle.push_span(self._self_tracer.current_span()) - for frame in frames: - handle.push_frame(frame.function_name, frame.file_name, 0, frame.lineno) - handle.flush_sample() - else: - event = self.RELEASE_EVENT_CLASS( - lock_name=lock_name, - frames=frames, - nframes=nframes, - thread_id=thread_id, - thread_name=thread_name, - task_id=task_id, - task_name=task_name, - locked_for_ns=end - self._self_acquired_at, - sampling_pct=self._self_capture_sampler.capture_pct, - ) - - if self._self_tracer is not None: - event.set_trace_info( - self._self_tracer.current_span(), self._self_endpoint_collection_enabled - ) - - self._self_recorder.push_event(event) - finally: - del self._self_acquired_at - except Exception as e: - LOG.warning("Error recording lock release event: %s", e) - pass # nosec + if start is not None: + end = time.monotonic_ns() + thread_id, thread_name = _current_thread() + task_id, task_name, task_frame = _task.get_task(thread_id) + lock_name = "%s:%s" % (self._self_init_loc, self._self_name) if self._self_name else self._self_init_loc + + if task_frame is None: + # See the comments in _acquire + frame = sys._getframe(2) + else: + frame = task_frame + + frames, nframes = _traceback.pyframe_to_frames(frame, self._self_max_nframes) + + if self._self_export_libdd_enabled: + thread_native_id = _threading.get_thread_native_id(thread_id) + + handle = ddup.SampleHandle() + handle.push_monotonic_ns(end) + handle.push_lock_name(lock_name) + handle.push_release(end - start, 1) # AFAICT, capture_pct does not adjust anything here + handle.push_threadinfo(thread_id, thread_native_id, thread_name) + handle.push_task_id(task_id) + handle.push_task_name(task_name) + + if self._self_tracer is not None: + handle.push_span(self._self_tracer.current_span()) + for frame in frames: + handle.push_frame(frame.function_name, frame.file_name, 0, frame.lineno) + handle.flush_sample() + else: + event = self.RELEASE_EVENT_CLASS( + lock_name=lock_name, + frames=frames, + nframes=nframes, + thread_id=thread_id, + thread_name=thread_name, + task_id=task_id, + task_name=task_name, + locked_for_ns=end - start, + sampling_pct=self._self_capture_sampler.capture_pct, + ) + + if self._self_tracer is not None: + event.set_trace_info(self._self_tracer.current_span(), self._self_endpoint_collection_enabled) + + self._self_recorder.push_event(event) def release(self, *args, **kwargs): return self._release(self.__wrapped__.release, *args, **kwargs) diff --git a/ddtrace/profiling/collector/_memalloc.c b/ddtrace/profiling/collector/_memalloc.c index b55e9ebcfab..1f2b87e0433 100644 --- a/ddtrace/profiling/collector/_memalloc.c +++ b/ddtrace/profiling/collector/_memalloc.c @@ -109,13 +109,19 @@ memalloc_init() } static void -memalloc_add_event(memalloc_context_t* ctx, void* ptr, size_t size) +memalloc_assert_gil() { if (g_crash_on_no_gil && !PyGILState_Check()) { int* p = NULL; *p = 0; abort(); // should never reach here } +} + +static void +memalloc_add_event(memalloc_context_t* ctx, void* ptr, size_t size) +{ + memalloc_assert_gil(); uint64_t alloc_count = atomic_add_clamped(&global_alloc_tracker->alloc_count, 1, ALLOC_TRACKER_MAX_COUNT); @@ -332,6 +338,8 @@ memalloc_stop(PyObject* Py_UNUSED(module), PyObject* Py_UNUSED(args)) return NULL; } + memalloc_assert_gil(); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &global_memalloc_ctx.pymem_allocator_obj); memalloc_tb_deinit(); if (memlock_trylock(&g_memalloc_lock)) { @@ -389,6 +397,8 @@ iterevents_new(PyTypeObject* type, PyObject* Py_UNUSED(args), PyObject* Py_UNUSE if (!iestate) return NULL; + memalloc_assert_gil(); + /* reset the current traceback list */ if (memlock_trylock(&g_memalloc_lock)) { iestate->alloc_tracker = global_alloc_tracker; diff --git a/ddtrace/profiling/exporter/http.py b/ddtrace/profiling/exporter/http.py index 6700e584ade..b4ec6994d72 100644 --- a/ddtrace/profiling/exporter/http.py +++ b/ddtrace/profiling/exporter/http.py @@ -220,8 +220,18 @@ def export( "family": "python", "attachments": [item["filename"].decode("utf-8") for item in data], "tags_profiler": self._get_tags(service), - "start": (datetime.datetime.utcfromtimestamp(start_time_ns / 1e9).replace(microsecond=0).isoformat() + "Z"), - "end": (datetime.datetime.utcfromtimestamp(end_time_ns / 1e9).replace(microsecond=0).isoformat() + "Z"), + "start": ( + datetime.datetime.fromtimestamp(start_time_ns / 1e9, tz=datetime.timezone.utc) + .replace(microsecond=0) + .isoformat()[0:-6] # removes the trailing +00:00 portion of the time + + "Z" + ), + "end": ( + datetime.datetime.fromtimestamp(end_time_ns / 1e9, tz=datetime.timezone.utc) + .replace(microsecond=0) + .isoformat()[0:-6] # removes the trailing +00:00 portion of the time + + "Z" + ), } # type: Dict[str, Any] if self.endpoint_call_counter_span_processor is not None: diff --git a/ddtrace/propagation/http.py b/ddtrace/propagation/http.py index a1664664ace..fdaf97410ad 100644 --- a/ddtrace/propagation/http.py +++ b/ddtrace/propagation/http.py @@ -40,8 +40,8 @@ from ..internal._tagset import decode_tagset_string from ..internal._tagset import encode_tagset_values from ..internal.compat import ensure_text +from ..internal.constants import _PROPAGATION_BEHAVIOR_RESTART from ..internal.constants import _PROPAGATION_STYLE_BAGGAGE -from ..internal.constants import _PROPAGATION_STYLE_NONE from ..internal.constants import _PROPAGATION_STYLE_W3C_TRACECONTEXT from ..internal.constants import DD_TRACE_BAGGAGE_MAX_BYTES from ..internal.constants import DD_TRACE_BAGGAGE_MAX_ITEMS @@ -101,6 +101,7 @@ def _possible_header(header): _POSSIBLE_HTTP_HEADER_B3_FLAGS = _possible_header(_HTTP_HEADER_B3_FLAGS) _POSSIBLE_HTTP_HEADER_TRACEPARENT = _possible_header(_HTTP_HEADER_TRACEPARENT) _POSSIBLE_HTTP_HEADER_TRACESTATE = _possible_header(_HTTP_HEADER_TRACESTATE) +_POSSIBLE_HTTP_BAGGAGE_HEADER = _possible_header(_HTTP_HEADER_BAGGAGE) # https://www.w3.org/TR/trace-context/#traceparent-header-field-values @@ -877,20 +878,6 @@ def _inject(span_context, headers): headers[_HTTP_HEADER_TRACESTATE] = span_context._tracestate -class _NOP_Propagator: - @staticmethod - def _extract(headers): - # type: (Dict[str, str]) -> None - return None - - # this method technically isn't needed with the current way we have HTTPPropagator.inject setup - # but if it changes then we might want it - @staticmethod - def _inject(span_context, headers): - # type: (Context , Dict[str, str]) -> Dict[str, str] - return headers - - class _BaggageHeader: """Helper class to inject/extract Baggage Headers""" @@ -937,7 +924,7 @@ def _inject(span_context: Context, headers: Dict[str, str]) -> None: @staticmethod def _extract(headers: Dict[str, str]) -> Context: - header_value = headers.get(_HTTP_HEADER_BAGGAGE) + header_value = _extract_header_value(_POSSIBLE_HTTP_BAGGAGE_HEADER, headers) if not header_value: return Context(baggage={}) @@ -962,7 +949,6 @@ def _extract(headers: Dict[str, str]) -> Context: PROPAGATION_STYLE_B3_MULTI: _B3MultiHeader, PROPAGATION_STYLE_B3_SINGLE: _B3SingleHeader, _PROPAGATION_STYLE_W3C_TRACECONTEXT: _TraceContext, - _PROPAGATION_STYLE_NONE: _NOP_Propagator, _PROPAGATION_STYLE_BAGGAGE: _BaggageHeader, } @@ -973,12 +959,12 @@ class HTTPPropagator(object): """ @staticmethod - def _extract_configured_contexts_avail(normalized_headers): + def _extract_configured_contexts_avail(normalized_headers: Dict[str, str]) -> Tuple[List[Context], List[str]]: contexts = [] styles_w_ctx = [] for prop_style in config._propagation_style_extract: propagator = _PROP_STYLES[prop_style] - context = propagator._extract(normalized_headers) + context = propagator._extract(normalized_headers) # type: ignore # baggage is handled separately if prop_style == _PROPAGATION_STYLE_BAGGAGE: continue @@ -987,6 +973,24 @@ def _extract_configured_contexts_avail(normalized_headers): styles_w_ctx.append(prop_style) return contexts, styles_w_ctx + @staticmethod + def _context_to_span_link(context: Context, style: str, reason: str) -> Optional[SpanLink]: + # encoding expects at least trace_id and span_id + if context.span_id and context.trace_id: + return SpanLink( + context.trace_id, + context.span_id, + flags=1 if context.sampling_priority and context.sampling_priority > 0 else 0, + tracestate=( + context._meta.get(W3C_TRACESTATE_KEY, "") if style == _PROPAGATION_STYLE_W3C_TRACECONTEXT else None + ), + attributes={ + "reason": reason, + "context_headers": style, + }, + ) + return None + @staticmethod def _resolve_contexts(contexts, styles_w_ctx, normalized_headers): primary_context = contexts[0] @@ -995,23 +999,14 @@ def _resolve_contexts(contexts, styles_w_ctx, normalized_headers): for context in contexts[1:]: style_w_ctx = styles_w_ctx[contexts.index(context)] # encoding expects at least trace_id and span_id - if context.span_id and context.trace_id and context.trace_id != primary_context.trace_id: - links.append( - SpanLink( - context.trace_id, - context.span_id, - flags=1 if context.sampling_priority and context.sampling_priority > 0 else 0, - tracestate=( - context._meta.get(W3C_TRACESTATE_KEY, "") - if style_w_ctx == _PROPAGATION_STYLE_W3C_TRACECONTEXT - else None - ), - attributes={ - "reason": "terminated_context", - "context_headers": style_w_ctx, - }, - ) + if context.trace_id and context.trace_id != primary_context.trace_id: + link = HTTPPropagator._context_to_span_link( + context, + style_w_ctx, + "terminated_context", ) + if link: + links.append(link) # if trace_id matches and the propagation style is tracecontext # add the tracestate to the primary context elif style_w_ctx == _PROPAGATION_STYLE_W3C_TRACECONTEXT: @@ -1057,6 +1052,8 @@ def parent_call(): :param dict headers: HTTP headers to extend with tracing attributes. :param Span non_active_span: Only to be used if injecting a non-active span. """ + if not config._propagation_style_inject: + return if non_active_span is not None and non_active_span.context is not span_context: log.error( "span_context and non_active_span.context are not the same, but should be. non_active_span.context " @@ -1129,17 +1126,19 @@ def my_controller(url, headers): :param dict headers: HTTP headers to extract tracing attributes. :return: New `Context` with propagated attributes. """ - if not headers: - return Context() + context = Context() + if not headers or not config._propagation_style_extract: + return context try: + style = "" normalized_headers = {name.lower(): v for name, v in headers.items()} - context = Context() # tracer configured to extract first only if config._propagation_extract_first: # loop through the extract propagation styles specified in order, return whatever context we get first for prop_style in config._propagation_style_extract: propagator = _PROP_STYLES[prop_style] context = propagator._extract(normalized_headers) + style = prop_style if config.propagation_http_baggage_enabled is True: _attach_baggage_to_context(normalized_headers, context) break @@ -1147,6 +1146,9 @@ def my_controller(url, headers): # loop through all extract propagation styles else: contexts, styles_w_ctx = HTTPPropagator._extract_configured_contexts_avail(normalized_headers) + # check that styles_w_ctx is not empty + if styles_w_ctx: + style = styles_w_ctx[0] if contexts: context = HTTPPropagator._resolve_contexts(contexts, styles_w_ctx, normalized_headers) @@ -1158,9 +1160,12 @@ def my_controller(url, headers): baggage_context = _BaggageHeader._extract(normalized_headers) if baggage_context._baggage != {}: if context: - context._baggage = baggage_context._baggage + context._baggage = baggage_context.get_all_baggage_items() else: context = baggage_context + if config._propagation_behavior_extract == _PROPAGATION_BEHAVIOR_RESTART: + link = HTTPPropagator._context_to_span_link(context, style, "propagation_behavior_extract") + context = Context(baggage=context.get_all_baggage_items(), span_links=[link] if link else []) return context diff --git a/ddtrace/sampler.py b/ddtrace/sampler.py index be6f8898ef0..c7f4b9d499a 100644 --- a/ddtrace/sampler.py +++ b/ddtrace/sampler.py @@ -1,368 +1,10 @@ -"""Samplers manage the client-side trace sampling +from ddtrace._trace.sampler import * # noqa: F403 +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate -Any `sampled = False` trace won't be written, and can be ignored by the instrumentation. -""" -import abc -import json -from typing import TYPE_CHECKING # noqa:F401 -from typing import Dict # noqa:F401 -from typing import List # noqa:F401 -from typing import Optional # noqa:F401 -from typing import Tuple # noqa:F401 - -from ddtrace import config -from ddtrace.constants import SAMPLING_LIMIT_DECISION - -from .constants import ENV_KEY -from .internal.constants import _PRIORITY_CATEGORY -from .internal.constants import DEFAULT_SAMPLING_RATE_LIMIT -from .internal.constants import MAX_UINT_64BITS as _MAX_UINT_64BITS -from .internal.logger import get_logger -from .internal.rate_limiter import RateLimiter -from .internal.sampling import _get_highest_precedence_rule_matching -from .internal.sampling import _set_sampling_tags -from .sampling_rule import SamplingRule -from .settings import _config as ddconfig - - -PROVENANCE_ORDER = ["customer", "dynamic", "default"] - -try: - from json.decoder import JSONDecodeError -except ImportError: - # handling python 2.X import error - JSONDecodeError = ValueError # type: ignore - -if TYPE_CHECKING: # pragma: no cover - from ddtrace._trace.span import Span # noqa:F401 - - -log = get_logger(__name__) - -# All references to MAX_TRACE_ID were replaced with _MAX_UINT_64BITS. -# Now that ddtrace supports generating 128bit trace_ids, -# the max trace id should be 2**128 - 1 (not 2**64 -1) -# MAX_TRACE_ID is no longer used and should be removed. -MAX_TRACE_ID = _MAX_UINT_64BITS -# Has to be the same factor and key as the Agent to allow chained sampling -KNUTH_FACTOR = 1111111111111111111 - - -class SamplingError(Exception): - pass - - -class BaseSampler(metaclass=abc.ABCMeta): - __slots__ = () - - @abc.abstractmethod - def sample(self, span): - # type: (Span) -> bool - pass - - -class BasePrioritySampler(BaseSampler): - __slots__ = () - - @abc.abstractmethod - def update_rate_by_service_sample_rates(self, sample_rates): - pass - - -class AllSampler(BaseSampler): - """Sampler sampling all the traces""" - - def sample(self, span): - return True - - -class RateSampler(BaseSampler): - """Sampler based on a rate - - Keep (100 * `sample_rate`)% of the traces. - It samples randomly, its main purpose is to reduce the instrumentation footprint. - """ - - def __init__(self, sample_rate: float = 1.0) -> None: - """sample_rate is clamped between 0 and 1 inclusive""" - sample_rate = min(1, max(0, sample_rate)) - self.set_sample_rate(sample_rate) - log.debug("initialized RateSampler, sample %s%% of traces", 100 * sample_rate) - - def set_sample_rate(self, sample_rate: float) -> None: - self.sample_rate = float(sample_rate) - self.sampling_id_threshold = self.sample_rate * _MAX_UINT_64BITS - - def sample(self, span): - sampled = ((span._trace_id_64bits * KNUTH_FACTOR) % _MAX_UINT_64BITS) <= self.sampling_id_threshold - return sampled - - -class _AgentRateSampler(RateSampler): - pass - - -class RateByServiceSampler(BasePrioritySampler): - """Sampler based on a rate, by service - - Keep (100 * `sample_rate`)% of the traces. - The sample rate is kept independently for each service/env tuple. - """ - - __slots__ = ("sample_rate", "_by_service_samplers", "_default_sampler") - - _default_key = "service:,env:" - - @staticmethod - def _key( - service=None, # type: Optional[str] - env=None, # type: Optional[str] - ): - # type: (...) -> str - """Compute a key with the same format used by the Datadog agent API.""" - service = service or "" - env = env or "" - return "service:" + service + ",env:" + env - - def __init__(self, sample_rate=1.0): - # type: (float) -> None - self.sample_rate = sample_rate - self._default_sampler = RateSampler(self.sample_rate) - self._by_service_samplers = {} # type: Dict[str, RateSampler] - - def set_sample_rate( - self, - sample_rate, # type: float - service=None, # type: Optional[str] - env=None, # type: Optional[str] - ): - # type: (...) -> None - - # if we have a blank service, we need to match it to the config.service - if service is None: - service = config.service - if env is None: - env = config.env - - self._by_service_samplers[self._key(service, env)] = _AgentRateSampler(sample_rate) - - def sample(self, span): - sampled, sampler = self._make_sampling_decision(span) - _set_sampling_tags( - span, - sampled, - sampler.sample_rate, - self._choose_priority_category(sampler), - ) - return sampled - - def _choose_priority_category(self, sampler): - # type: (BaseSampler) -> str - if sampler is self._default_sampler: - return _PRIORITY_CATEGORY.DEFAULT - elif isinstance(sampler, _AgentRateSampler): - return _PRIORITY_CATEGORY.AUTO - else: - return _PRIORITY_CATEGORY.RULE_DEF - - def _make_sampling_decision(self, span): - # type: (Span) -> Tuple[bool, BaseSampler] - env = span.get_tag(ENV_KEY) - key = self._key(span.service, env) - sampler = self._by_service_samplers.get(key) or self._default_sampler - sampled = sampler.sample(span) - return sampled, sampler - - def update_rate_by_service_sample_rates(self, rate_by_service): - # type: (Dict[str, float]) -> None - samplers = {} # type: Dict[str, RateSampler] - for key, sample_rate in rate_by_service.items(): - samplers[key] = _AgentRateSampler(sample_rate) - - self._by_service_samplers = samplers - - -class DatadogSampler(RateByServiceSampler): - """ - By default, this sampler relies on dynamic sample rates provided by the trace agent - to determine which traces are kept or dropped. - - You can also configure a static sample rate via ``default_sample_rate`` to use for sampling. - When a ``default_sample_rate`` is configured, that is the only sample rate used, and the agent - provided rates are ignored. - - You may also supply a list of ``SamplingRule`` instances to set sample rates for specific - services. - - Example rules:: - - DatadogSampler(rules=[ - SamplingRule(sample_rate=1.0, service="my-svc"), - SamplingRule(sample_rate=0.0, service="less-important"), - ]) - - Rules are evaluated in the order they are provided, and the first rule that matches is used. - If no rule matches, then the agent sample rates are used. - - This sampler can be configured with a rate limit. This will ensure the max number of - sampled traces per second does not exceed the supplied limit. The default is 100 traces kept - per second. - """ - - __slots__ = ("limiter", "rules", "default_sample_rate", "_rate_limit_always_on") - - NO_RATE_LIMIT = -1 - # deprecate and remove the DEFAULT_RATE_LIMIT field from DatadogSampler - DEFAULT_RATE_LIMIT = DEFAULT_SAMPLING_RATE_LIMIT - - def __init__( - self, - rules=None, # type: Optional[List[SamplingRule]] - default_sample_rate=None, # type: Optional[float] - rate_limit=None, # type: Optional[int] - rate_limit_window=1e9, # type: float - rate_limit_always_on=False, # type: bool - ): - # type: (...) -> None - """ - Constructor for DatadogSampler sampler - - :param rules: List of :class:`SamplingRule` rules to apply to the root span of every trace, default no rules - :param default_sample_rate: The default sample rate to apply if no rules matched (default: ``None`` / - Use :class:`RateByServiceSampler` only) - :param rate_limit: Global rate limit (traces per second) to apply to all traces regardless of the rules - applied to them, (default: ``100``) - """ - # Use default sample rate of 1.0 - super(DatadogSampler, self).__init__() - self.default_sample_rate = default_sample_rate - effective_sample_rate = default_sample_rate - if default_sample_rate is None: - if ddconfig._get_source("_trace_sample_rate") != "default": - effective_sample_rate = float(ddconfig._trace_sample_rate) - - if rate_limit is None: - rate_limit = int(ddconfig._trace_rate_limit) - - self._rate_limit_always_on = rate_limit_always_on - - if rules is None: - env_sampling_rules = ddconfig._trace_sampling_rules - if env_sampling_rules: - rules = self._parse_rules_from_str(env_sampling_rules) - else: - rules = [] - self.rules = rules - else: - self.rules = [] - # Validate that rules is a list of SampleRules - for rule in rules: - if isinstance(rule, SamplingRule): - self.rules.append(rule) - elif config._raise: - raise TypeError("Rule {!r} must be a sub-class of type ddtrace.sampler.SamplingRules".format(rule)) - - # DEV: sampling rule must come last - if effective_sample_rate is not None: - self.rules.append(SamplingRule(sample_rate=effective_sample_rate)) - - # Configure rate limiter - self.limiter = RateLimiter(rate_limit, rate_limit_window) - - log.debug("initialized %r", self) - - def __str__(self): - rates = {key: sampler.sample_rate for key, sampler in self._by_service_samplers.items()} - return "{}(agent_rates={!r}, limiter={!r}, rules={!r})".format( - self.__class__.__name__, rates, self.limiter, self.rules - ) - - __repr__ = __str__ - - @staticmethod - def _parse_rules_from_str(rules): - # type: (str) -> List[SamplingRule] - sampling_rules = [] - try: - json_rules = json.loads(rules) - except JSONDecodeError: - if config._raise: - raise ValueError("Unable to parse DD_TRACE_SAMPLING_RULES={}".format(rules)) - for rule in json_rules: - if "sample_rate" not in rule: - if config._raise: - raise KeyError("No sample_rate provided for sampling rule: {}".format(json.dumps(rule))) - continue - sample_rate = float(rule["sample_rate"]) - service = rule.get("service", SamplingRule.NO_RULE) - name = rule.get("name", SamplingRule.NO_RULE) - resource = rule.get("resource", SamplingRule.NO_RULE) - tags = rule.get("tags", SamplingRule.NO_RULE) - provenance = rule.get("provenance", "default") - try: - sampling_rule = SamplingRule( - sample_rate=sample_rate, - service=service, - name=name, - resource=resource, - tags=tags, - provenance=provenance, - ) - except ValueError as e: - if config._raise: - raise ValueError("Error creating sampling rule {}: {}".format(json.dumps(rule), e)) - continue - sampling_rules.append(sampling_rule) - - # Sort the sampling_rules list using a lambda function as the key - sampling_rules = sorted(sampling_rules, key=lambda rule: PROVENANCE_ORDER.index(rule.provenance)) - return sampling_rules - - def sample(self, span): - span.context._update_tags(span) - - matched_rule = _get_highest_precedence_rule_matching(span, self.rules) - - sampler = self._default_sampler # type: BaseSampler - sample_rate = self.sample_rate - if matched_rule: - # Client based sampling - sampled = matched_rule.sample(span) - sample_rate = matched_rule.sample_rate - else: - # Agent based sampling - sampled, sampler = super(DatadogSampler, self)._make_sampling_decision(span) - if isinstance(sampler, RateSampler): - sample_rate = sampler.sample_rate - - if matched_rule or self._rate_limit_always_on: - # Avoid rate limiting when trace sample rules and/or sample rates are NOT provided - # by users. In this scenario tracing should default to agent based sampling. ASM - # uses DatadogSampler._rate_limit_always_on to override this functionality. - if sampled: - sampled = self.limiter.is_allowed() - span.set_metric(SAMPLING_LIMIT_DECISION, self.limiter.effective_rate) - _set_sampling_tags( - span, - sampled, - sample_rate, - self._choose_priority_category_with_rule(matched_rule, sampler), - ) - - return sampled - - def _choose_priority_category_with_rule(self, rule, sampler): - # type: (Optional[SamplingRule], BaseSampler) -> str - if rule: - provenance = rule.provenance - if provenance == "customer": - return _PRIORITY_CATEGORY.RULE_CUSTOMER - if provenance == "dynamic": - return _PRIORITY_CATEGORY.RULE_DYNAMIC - return _PRIORITY_CATEGORY.RULE_DEF - elif self._rate_limit_always_on: - # backwards compaitbiility for ASM, when the rate limit is always on (ASM standalone mode) - # we want spans to be set to a MANUAL priority to avoid agent based sampling - return _PRIORITY_CATEGORY.USER - return super(DatadogSampler, self)._choose_priority_category(sampler) +deprecate( + "The ddtrace.sampler module is deprecated and will be removed.", + message="Use DD_TRACE_SAMPLING_RULES to configure sampling rates.", + category=DDTraceDeprecationWarning, +) diff --git a/ddtrace/sampling_rule.py b/ddtrace/sampling_rule.py index 532a0b71f51..244cebddd31 100644 --- a/ddtrace/sampling_rule.py +++ b/ddtrace/sampling_rule.py @@ -1,244 +1,10 @@ -from typing import TYPE_CHECKING # noqa:F401 -from typing import Any -from typing import Optional -from typing import Tuple - -from ddtrace.internal.compat import pattern_type -from ddtrace.internal.constants import MAX_UINT_64BITS as _MAX_UINT_64BITS -from ddtrace.internal.glob_matching import GlobMatcher -from ddtrace.internal.logger import get_logger -from ddtrace.internal.utils.cache import cachedmethod +from ddtrace._trace.sampling_rule import * # noqa: F403 from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning from ddtrace.vendor.debtcollector import deprecate -if TYPE_CHECKING: # pragma: no cover - from ddtrace._trace.span import Span # noqa:F401 - -log = get_logger(__name__) -KNUTH_FACTOR = 1111111111111111111 - - -class SamplingRule(object): - """ - Definition of a sampling rule used by :class:`DatadogSampler` for applying a sample rate on a span - """ - - NO_RULE = object() - - def __init__( - self, - sample_rate: float, - service: Any = NO_RULE, - name: Any = NO_RULE, - resource: Any = NO_RULE, - tags: Any = NO_RULE, - provenance: str = "default", - ) -> None: - """ - Configure a new :class:`SamplingRule` - - .. code:: python - - DatadogSampler([ - # Sample 100% of any trace - SamplingRule(sample_rate=1.0), - - # Sample no healthcheck traces - SamplingRule(sample_rate=0, name='flask.request'), - - # Sample all services ending in `-db` based on a regular expression - SamplingRule(sample_rate=0.5, service=re.compile('-db$')), - - # Sample based on service name using custom function - SamplingRule(sample_rate=0.75, service=lambda service: 'my-app' in service), - ]) - - :param sample_rate: The sample rate to apply to any matching spans - :type sample_rate: :obj:`float` clamped between 0.0 and 1.0 inclusive - :param service: Rule to match the `span.service` on, default no rule defined - :type service: :obj:`object` to directly compare, :obj:`function` to evaluate, or :class:`re.Pattern` to match - :param name: Rule to match the `span.name` on, default no rule defined - :type name: :obj:`object` to directly compare, :obj:`function` to evaluate, or :class:`re.Pattern` to match - :param tags: A dictionary whose keys exactly match the names of tags expected to appear on spans, and whose - values are glob-matches with the expected span tag values. Glob matching supports "*" meaning any - number of characters, and "?" meaning any one character. If all tags specified in a SamplingRule are - matches with a given span, that span is considered to have matching tags with the rule. - """ - self.sample_rate = float(min(1, max(0, sample_rate))) - # since span.py converts None to 'None' for tags, and does not accept 'None' for metrics - # we can just create a GlobMatcher for 'None' and it will match properly - self._tag_value_matchers = ( - {k: GlobMatcher(str(v)) for k, v in tags.items()} if tags != SamplingRule.NO_RULE else {} - ) - self.tags = tags - self.service = self.choose_matcher(service) - self.name = self.choose_matcher(name) - self.resource = self.choose_matcher(resource) - self.provenance = provenance - - @property - def sample_rate(self) -> float: - return self._sample_rate - - @sample_rate.setter - def sample_rate(self, sample_rate: float) -> None: - self._sample_rate = sample_rate - self._sampling_id_threshold = sample_rate * _MAX_UINT_64BITS - - def _pattern_matches(self, prop, pattern): - # If the rule is not set, then assume it matches - # DEV: Having no rule and being `None` are different things - # e.g. ignoring `span.service` vs `span.service == None` - if pattern is self.NO_RULE: - return True - if isinstance(pattern, GlobMatcher): - return pattern.match(str(prop)) - - # If the pattern is callable (e.g. a function) then call it passing the prop - # The expected return value is a boolean so cast the response in case it isn't - if callable(pattern): - try: - return bool(pattern(prop)) - except Exception: - log.warning("%r pattern %r failed with %r", self, pattern, prop, exc_info=True) - # Their function failed to validate, assume it is a False - return False - - # The pattern is a regular expression and the prop is a string - if isinstance(pattern, pattern_type): - try: - return bool(pattern.match(str(prop))) - except (ValueError, TypeError): - # This is to guard us against the casting to a string (shouldn't happen, but still) - log.warning("%r pattern %r failed with %r", self, pattern, prop, exc_info=True) - return False - - # Exact match on the values - return prop == pattern - - @cachedmethod() - def _matches(self, key: Tuple[Optional[str], str, Optional[str]]) -> bool: - # self._matches exists to maintain legacy pattern values such as regex and functions - service, name, resource = key - for prop, pattern in [(service, self.service), (name, self.name), (resource, self.resource)]: - if not self._pattern_matches(prop, pattern): - return False - else: - return True - - def matches(self, span): - # type: (Span) -> bool - """ - Return if this span matches this rule - - :param span: The span to match against - :type span: :class:`ddtrace._trace.span.Span` - :returns: Whether this span matches or not - :rtype: :obj:`bool` - """ - tags_match = self.tags_match(span) - return tags_match and self._matches((span.service, span.name, span.resource)) - - def tags_match(self, span): - # type: (Span) -> bool - tag_match = True - if self._tag_value_matchers: - tag_match = self.check_tags(span.get_tags(), span.get_metrics()) - return tag_match - - def check_tags(self, meta, metrics): - if meta is None and metrics is None: - return False - - tag_match = False - for tag_key in self._tag_value_matchers.keys(): - value = meta.get(tag_key) - tag_match = self._tag_value_matchers[tag_key].match(str(value)) - # If the value doesn't match in meta, check the metrics - if tag_match is False: - value = metrics.get(tag_key) - # Floats: Matching floating point values with a non-zero decimal part is not supported. - # For floating point values with a non-zero decimal part, any all * pattern always returns true. - # Other patterns always return false. - if isinstance(value, float): - if not value.is_integer(): - if self._tag_value_matchers[tag_key].pattern == "*": - tag_match = True - else: - return False - continue - else: - value = int(value) - - tag_match = self._tag_value_matchers[tag_key].match(str(value)) - else: - continue - # if we don't match with all specified tags for a rule, it's not a match - if tag_match is False: - return False - - return tag_match - - def sample(self, span): - """ - Return if this rule chooses to sample the span - - :param span: The span to sample against - :type span: :class:`ddtrace._trace.span.Span` - :returns: Whether this span was sampled - :rtype: :obj:`bool` - """ - if self.sample_rate == 1: - return True - elif self.sample_rate == 0: - return False - - return ((span._trace_id_64bits * KNUTH_FACTOR) % _MAX_UINT_64BITS) <= self._sampling_id_threshold - - def _no_rule_or_self(self, val): - if val is self.NO_RULE: - return "NO_RULE" - elif val is None: - return "None" - elif type(val) == GlobMatcher: - return val.pattern - else: - return val - - def choose_matcher(self, prop): - # We currently support the ability to pass in a function, a regular expression, or a string - # If a string is passed in we create a GlobMatcher to handle the matching - if callable(prop) or isinstance(prop, pattern_type): - # deprecated: passing a function or a regular expression' - deprecate( - "Using methods or regular expressions for SamplingRule matching is deprecated. ", - message="Please move to passing in a string for Glob matching.", - removal_version="3.0.0", - category=DDTraceDeprecationWarning, - ) - return prop - # Name and Resource will never be None, but service can be, since we str() - # whatever we pass into the GlobMatcher, we can just use its matching - elif prop is None: - prop = "None" - else: - return GlobMatcher(prop) if prop != SamplingRule.NO_RULE else SamplingRule.NO_RULE - - def __repr__(self): - return "{}(sample_rate={!r}, service={!r}, name={!r}, resource={!r}, tags={!r}, provenance={!r})".format( - self.__class__.__name__, - self.sample_rate, - self._no_rule_or_self(self.service), - self._no_rule_or_self(self.name), - self._no_rule_or_self(self.resource), - self._no_rule_or_self(self.tags), - self.provenance, - ) - - __str__ = __repr__ - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, SamplingRule): - return False - return str(self) == str(other) +deprecate( + "The ddtrace.sample_rule module is deprecated and will be removed.", + message="Use DD_TRACE_SAMPLING_RULES to set sampling rules.", + category=DDTraceDeprecationWarning, +) diff --git a/ddtrace/settings/_otel_remapper.py b/ddtrace/settings/_otel_remapper.py index 8bdb313fdef..d3501c2e3fa 100644 --- a/ddtrace/settings/_otel_remapper.py +++ b/ddtrace/settings/_otel_remapper.py @@ -28,7 +28,7 @@ def __class_getitem__(self, item): from ..constants import VERSION_KEY from ..internal.logger import get_logger from ..internal.telemetry import telemetry_writer -from ..internal.telemetry.constants import TELEMETRY_NAMESPACE_TAG_TRACER +from ..internal.telemetry.constants import TELEMETRY_NAMESPACE log = get_logger(__name__) @@ -169,7 +169,7 @@ def otel_remapping(): if otel_env.startswith("OTEL_") and otel_env != "OTEL_PYTHON_CONTEXT": log.warning("OpenTelemetry configuration %s is not supported by Datadog.", otel_env) telemetry_writer.add_count_metric( - TELEMETRY_NAMESPACE_TAG_TRACER, + TELEMETRY_NAMESPACE.TRACERS, "otel.env.unsupported", 1, (("config_opentelemetry", otel_env.lower()),), @@ -185,7 +185,7 @@ def otel_remapping(): otel_value, ) telemetry_writer.add_count_metric( - TELEMETRY_NAMESPACE_TAG_TRACER, + TELEMETRY_NAMESPACE.TRACERS, "otel.env.hiding", 1, (("config_opentelemetry", otel_env.lower()), ("config_datadog", dd_env.lower())), @@ -205,7 +205,7 @@ def otel_remapping(): otel_value, ) telemetry_writer.add_count_metric( - TELEMETRY_NAMESPACE_TAG_TRACER, + TELEMETRY_NAMESPACE.TRACERS, "otel.env.invalid", 1, (("config_opentelemetry", otel_env.lower()), ("config_datadog", dd_env.lower())), diff --git a/ddtrace/settings/config.py b/ddtrace/settings/config.py index 0de3cdfc706..b35975a2e5d 100644 --- a/ddtrace/settings/config.py +++ b/ddtrace/settings/config.py @@ -19,8 +19,12 @@ from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning from ddtrace.vendor.debtcollector import deprecate +from .._trace.pin import Pin from ..internal import gitmetadata +from ..internal.constants import _PROPAGATION_BEHAVIOR_DEFAULT +from ..internal.constants import _PROPAGATION_BEHAVIOR_IGNORE from ..internal.constants import _PROPAGATION_STYLE_DEFAULT +from ..internal.constants import _PROPAGATION_STYLE_NONE from ..internal.constants import DEFAULT_BUFFER_SIZE from ..internal.constants import DEFAULT_MAX_PAYLOAD_SIZE from ..internal.constants import DEFAULT_PROCESSING_INTERVAL @@ -34,7 +38,6 @@ from ..internal.serverless import in_aws_lambda from ..internal.utils.formats import asbool from ..internal.utils.formats import parse_tags_str -from ..pin import Pin from ._core import get_config as _get_config from ._inferred_base_service import detect_service from ._otel_remapper import otel_remapping as _otel_remapping @@ -147,12 +150,15 @@ def _parse_propagation_styles(styles_str): category=DDTraceDeprecationWarning, ) style = PROPAGATION_STYLE_B3_SINGLE - if not style: + if not style or style == _PROPAGATION_STYLE_NONE: continue if style not in PROPAGATION_STYLE_ALL: log.warning("Unknown DD_TRACE_PROPAGATION_STYLE: {!r}, allowed values are %r", style, PROPAGATION_STYLE_ALL) continue styles.append(style) + # Remove "none" if it's present since it lacks a propagator + if _PROPAGATION_STYLE_NONE in styles: + styles.remove(_PROPAGATION_STYLE_NONE) return styles @@ -529,11 +535,23 @@ def __init__(self): # Propagation styles # DD_TRACE_PROPAGATION_STYLE_EXTRACT and DD_TRACE_PROPAGATION_STYLE_INJECT # take precedence over DD_TRACE_PROPAGATION_STYLE - self._propagation_style_extract = _parse_propagation_styles( - _get_config( - ["DD_TRACE_PROPAGATION_STYLE_EXTRACT", "DD_TRACE_PROPAGATION_STYLE"], _PROPAGATION_STYLE_DEFAULT - ) + # if DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT is set to ignore + # we set DD_TRACE_PROPAGATION_STYLE_EXTRACT to [_PROPAGATION_STYLE_NONE] since no extraction will heeded + self._propagation_behavior_extract = _get_config( + ["DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT"], _PROPAGATION_BEHAVIOR_DEFAULT, self._lower ) + if self._propagation_behavior_extract != _PROPAGATION_BEHAVIOR_IGNORE: + self._propagation_style_extract = _parse_propagation_styles( + _get_config( + ["DD_TRACE_PROPAGATION_STYLE_EXTRACT", "DD_TRACE_PROPAGATION_STYLE"], _PROPAGATION_STYLE_DEFAULT + ) + ) + else: + log.debug( + """DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT is set to ignore, + setting DD_TRACE_PROPAGATION_STYLE_EXTRACT to empty list""" + ) + self._propagation_style_extract = [_PROPAGATION_STYLE_NONE] self._propagation_style_inject = _parse_propagation_styles( _get_config(["DD_TRACE_PROPAGATION_STYLE_INJECT", "DD_TRACE_PROPAGATION_STYLE"], _PROPAGATION_STYLE_DEFAULT) ) @@ -981,3 +999,6 @@ def convert_rc_trace_sampling_rules(self, rc_rules: List[Dict[str, Any]]) -> Opt return json.dumps(rc_rules) else: return None + + def _lower(self, value): + return value.lower() diff --git a/ddtrace/settings/profiling.py b/ddtrace/settings/profiling.py index 94d71f1778b..38a4b49d6d6 100644 --- a/ddtrace/settings/profiling.py +++ b/ddtrace/settings/profiling.py @@ -334,7 +334,7 @@ class ProfilingConfigStack(En): _v2_enabled = En.v( bool, "v2_enabled", - default=False, + default=True, help_type="Boolean", help="Whether to enable the v2 stack profiler. Also enables the libdatadog collector.", ) diff --git a/ddtrace/trace/__init__.py b/ddtrace/trace/__init__.py index 90c662cebc7..f709310d589 100644 --- a/ddtrace/trace/__init__.py +++ b/ddtrace/trace/__init__.py @@ -1,7 +1,18 @@ from ddtrace._trace.context import Context +from ddtrace._trace.filters import TraceFilter +from ddtrace._trace.pin import Pin +from ddtrace._trace.span import Span +from ddtrace._trace.tracer import Tracer -# TODO: Move `ddtrace.Pin`, `ddtrace.Tracer`, `ddtrace.Span`, and `ddtrace.tracer` to this module +# a global tracer instance with integration settings +tracer = Tracer() + __all__ = [ "Context", + "Pin", + "TraceFilter", + "Tracer", + "Span", + "tracer", ] diff --git a/ddtrace/vendor/__init__.py b/ddtrace/vendor/__init__.py index c74ff435562..c8487e48a8c 100644 --- a/ddtrace/vendor/__init__.py +++ b/ddtrace/vendor/__init__.py @@ -47,20 +47,6 @@ License: BSD 3 -contextvars -------------- - -Source: https://github.com/MagicStack/contextvars -Version: 2.4 -License: Apache License 2.0 - -Notes: - - removal of metaclass usage - - formatting - - use a plain old dict instead of immutables.Map - - removal of `*` syntax - - sqlcommenter ------------ diff --git a/ddtrace/vendor/contextvars/__init__.py b/ddtrace/vendor/contextvars/__init__.py deleted file mode 100644 index f4c5c62dcc6..00000000000 --- a/ddtrace/vendor/contextvars/__init__.py +++ /dev/null @@ -1,160 +0,0 @@ -import threading - -try: - from collections.abc import Mapping -except ImportError: - from collections import Mapping - - -__all__ = ("ContextVar", "Context", "Token", "copy_context") - - -_NO_DEFAULT = object() - - -class Context(Mapping): - def __init__(self): - self._data = {} - self._prev_context = None - - def run(self, callable, *args, **kwargs): - if self._prev_context is not None: - raise RuntimeError("cannot enter context: {} is already entered".format(self)) - - self._prev_context = _get_context() - try: - _set_context(self) - return callable(*args, **kwargs) - finally: - _set_context(self._prev_context) - self._prev_context = None - - def copy(self): - new = Context() - new._data = self._data.copy() - return new - - def __getitem__(self, var): - if not isinstance(var, ContextVar): - raise TypeError("a ContextVar key was expected, got {!r}".format(var)) - return self._data[var] - - def __contains__(self, var): - if not isinstance(var, ContextVar): - raise TypeError("a ContextVar key was expected, got {!r}".format(var)) - return var in self._data - - def __len__(self): - return len(self._data) - - def __iter__(self): - return iter(self._data) - - -class ContextVar(object): - def __init__(self, name, default=_NO_DEFAULT): - if not isinstance(name, str): - raise TypeError("context variable name must be a str") - self._name = name - self._default = default - - @property - def name(self): - return self._name - - def get(self, default=_NO_DEFAULT): - ctx = _get_context() - try: - return ctx[self] - except KeyError: - pass - - if default is not _NO_DEFAULT: - return default - - if self._default is not _NO_DEFAULT: - return self._default - - raise LookupError - - def set(self, value): - ctx = _get_context() - data = ctx._data - try: - old_value = data[self] - except KeyError: - old_value = Token.MISSING - - updated_data = data.copy() - updated_data[self] = value - ctx._data = updated_data - return Token(ctx, self, old_value) - - def reset(self, token): - if token._used: - raise RuntimeError("Token has already been used once") - - if token._var is not self: - raise ValueError("Token was created by a different ContextVar") - - if token._context is not _get_context(): - raise ValueError("Token was created in a different Context") - - ctx = token._context - if token._old_value is Token.MISSING: - del ctx._data[token._var] - else: - ctx._data[token._var] = token._old_value - - token._used = True - - def __repr__(self): - r = "".format(id(self)) - - -class Token(object): - - MISSING = object() - - def __init__(self, context, var, old_value): - self._context = context - self._var = var - self._old_value = old_value - self._used = False - - @property - def var(self): - return self._var - - @property - def old_value(self): - return self._old_value - - def __repr__(self): - r = "` as early as possible +To enable full ddtrace support (library instrumentation, profiling, application security monitoring, dynamic instrumentation, etc.) call :ref:`import ddtrace.auto` as the very first thing in your application. This will only work if your application is not running under ``ddtrace-run``. -``patch_all`` -------------- - -For fine-grained control over instrumentation setup, use ``patch_all`` as early as possible -in the application:: - - from ddtrace import patch_all - patch_all() - -To toggle instrumentation for a particular module:: - - from ddtrace import patch_all - patch_all(redis=False, cassandra=False) - -By default all supported libraries will be instrumented when ``patch_all`` is -used. - -**Note:** To ensure that the supported libraries are instrumented properly in -the application, they must be patched *prior* to being imported. So make sure -to call ``patch_all`` *before* importing libraries that are to be instrumented. - -More information about ``patch_all`` is available in the :py:func:`patch_all` API -documentation. +Note: Some Datadog products and instrumentation are disabled by default. Products and instrumentation can be enabled/disable via environment variables, see :ref:`configurations ` page for more details. +Tracing +~~~~~~~ Manual Instrumentation ---------------------- diff --git a/docs/configuration.rst b/docs/configuration.rst index 35bb63fac20..455272f318d 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -368,6 +368,26 @@ Trace Context propagation version_added: v1.7.0: The ``b3multi`` propagation style was added and ``b3`` was deprecated in favor it. + DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT: + default: | + ``continue`` + + description: | + String for how to handle incoming request headers that are extracted for propagation of trace info. + + The supported values are ``continue``, ``restart``, and ``ignore``. + + After extracting the headers for propagation, this configuration determines what is done with them. + + The default value is ``continue`` which always propagates valid headers. + ``ignore`` ignores all incoming headers and ``restart`` turns the first extracted valid propagation header + into a span link and propagates baggage if present. + + Example: ``DD_TRACE_PROPAGATION_STYLE_EXTRACT="ignore"`` to ignore all incoming headers and to start a root span without a parent. + + version_added: + v2.20.0: + DD_TRACE_PROPAGATION_STYLE_INJECT: default: | ``tracecontext,datadog`` diff --git a/docs/contributing-integrations.rst b/docs/contributing-integrations.rst index 9d7d2d202ee..0dab68b5053 100644 --- a/docs/contributing-integrations.rst +++ b/docs/contributing-integrations.rst @@ -30,7 +30,7 @@ into the runtime execution of third-party libraries. The essential task of writi the functions in the third-party library that would serve as useful entrypoints and wrapping them with ``wrap_function_wrapper``. There are exceptions, but this is generally a useful starting point. -The Pin API in ``ddtrace.pin`` is used to configure the instrumentation at runtime. It provides a ``Pin`` class +The Pin API in ``ddtrace.trace.Pin`` is used to configure the instrumentation at runtime. It provides a ``Pin`` class that can store configuration data in memory in a manner that is accessible from within functions wrapped by Wrapt. ``Pin`` objects are most often used for storing configuration data scoped to a given integration, such as enable/disable flags and service name overrides. diff --git a/docs/index.rst b/docs/index.rst index 3d19a4fbf14..23c0cd48a06 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -186,7 +186,7 @@ contacting support. .. [1] Libraries that are automatically instrumented when the :ref:`ddtrace-run` command is used or the ``import ddtrace.auto`` import - is used. Always use ``patch()``, ``patch_all()``, and ``import ddtrace.auto`` as soon + is used. Always use ``import ddtrace.auto`` as soon as possible in your Python entrypoint. .. [2] only the synchronous client diff --git a/docs/installation_quickstart.rst b/docs/installation_quickstart.rst index 4cfc4d670d1..2d879a072b3 100644 --- a/docs/installation_quickstart.rst +++ b/docs/installation_quickstart.rst @@ -59,24 +59,6 @@ When ``ddtrace-run`` cannot be used, a similar start-up behavior can be achieved with the import of ``ddtrace.auto``. This should normally be imported as the first thing during the application start-up. -If neither ``ddtrace-run`` nor ``import ddtrace.auto`` are suitable for your application, then -:py:func:`ddtrace.patch_all` can be used to configure the tracer:: - - from ddtrace import config, patch_all - - config.env = "dev" # the environment the application is in - config.service = "app" # name of your application - config.version = "0.1" # version of your application - patch_all() - - -.. note:: - We recommend the use of ``ddtrace-run`` when possible. If you are importing - ``ddtrace.auto`` as a programmatic replacement for ``ddtrace``, then note - that integrations will take their configuration from the environment - variables. A call to :py:func:`ddtrace.patch_all` cannot be used to disable - an integration at this point. - Service names also need to be configured for libraries that query other services (``requests``, ``grpc``, database libraries, etc). Check out the diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index c9cc13a5a9e..ff2cfc09c6d 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -31,6 +31,8 @@ autopatching autoreload autoreloading aws +AWS +ARN backend backends backport diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst index 73dde525de9..04bd56d0ba9 100644 --- a/docs/troubleshooting.rst +++ b/docs/troubleshooting.rst @@ -67,7 +67,7 @@ This can be a problem for users who want to see error details from a child span While this is default behavior for integrations, users can add a trace filter to propagate the error details up to the root span:: from ddtrace import Span, tracer - from ddtrace.filters import TraceFilter + from ddtrace.trace import TraceFilter class ErrorFilter(TraceFilter): diff --git a/hatch.toml b/hatch.toml index 0baca1fd235..7269db2d497 100644 --- a/hatch.toml +++ b/hatch.toml @@ -214,11 +214,11 @@ test = [ # if you add or remove a version here, please also update the parallelism parameter # in .circleci/config.templ.yml [[envs.appsec_threats_django.matrix]] -python = ["3.7", "3.9"] +python = ["3.8", "3.9"] django = ["~=2.2"] [[envs.appsec_threats_django.matrix]] -python = ["3.7", "3.9", "3.10"] +python = ["3.8", "3.9", "3.10"] django = ["~=3.2"] [[envs.appsec_threats_django.matrix]] @@ -226,11 +226,11 @@ python = ["3.8", "3.10"] django = ["==4.0.10"] [[envs.appsec_threats_django.matrix]] -python = ["3.8", "3.10", "3.12"] +python = ["3.8", "3.11", "3.13"] django = ["~=4.2"] [[envs.appsec_threats_django.matrix]] -python = ["3.10", "3.12"] +python = ["3.10", "3.13"] django = ["~=5.1"] @@ -262,21 +262,21 @@ test = [ # if you add or remove a version here, please also update the parallelism parameter # in .circleci/config.templ.yml [[envs.appsec_threats_flask.matrix]] -python = ["3.7", "3.9"] +python = ["3.8", "3.9"] flask = ["~=1.1"] markupsafe = ["~=1.1"] [[envs.appsec_threats_flask.matrix]] -python = ["3.7", "3.9"] +python = ["3.8", "3.9"] flask = ["==2.1.3"] werkzeug = ["<3.0"] [[envs.appsec_threats_flask.matrix]] -python = ["3.8", "3.9", "3.12"] +python = ["3.8", "3.10", "3.13"] flask = ["~=2.3"] [[envs.appsec_threats_flask.matrix]] -python = ["3.8", "3.10", "3.12"] +python = ["3.8", "3.11", "3.13"] flask = ["~=3.0"] ## ASM Native IAST module @@ -299,6 +299,108 @@ test = [ [[envs.appsec_iast_native.matrix]] python = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] +## ASM appsec_iast_packages + +[envs.appsec_iast_packages] +template = "appsec_iast_packages" +dependencies = [ + "pytest", + "pytest-cov", + "requests", + "hypothesis", + "astunparse", + "flask", + "virtualenv-clone" +] + +[envs.appsec_iast_packages.scripts] +test = [ + "uname -a", + "pip freeze", + "DD_CIVISIBILITY_ITR_ENABLED=0 DD_IAST_REQUEST_SAMPLING=100 _DD_APPSEC_DEDUPLICATION_ENABLED=false python -m pytest tests/appsec/iast_packages", +] + +[[envs.appsec_iast_packages.matrix]] +python = ["3.9", "3.10", "3.11", "3.12"] + +## ASM appsec_integrations_django + +[envs.appsec_integrations_django] +template = "appsec_integrations_django" +dependencies = [ + "pytest", + "pytest-cov", + "requests", + "hypothesis", + "pylibmc", + "bcrypt==4.2.1", + "pytest-django[testing]==3.10.0", + "Django{matrix:django}", +] + +[envs.appsec_integrations_django.scripts] +test = [ + "uname -a", + "pip freeze", + "DD_CIVISIBILITY_ITR_ENABLED=0 DD_IAST_REQUEST_SAMPLING=100 _DD_APPSEC_DEDUPLICATION_ENABLED=false python -m pytest -vvv {args:tests/appsec/integrations/django_tests/}", +] + +[[envs.appsec_integrations_django.matrix]] +python = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] +django = ["~=4.0"] + +[[envs.appsec_integrations_django.matrix]] +python = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] +django = ["~=3.2"] + +## ASM appsec_integrations_flask + +[envs.appsec_integrations_flask] +template = "appsec_integrations_flask" +dependencies = [ + "pytest", + "pytest-cov", + "requests", + "hypothesis", + "gunicorn", + "psycopg2-binary~=2.9.9", + "MarkupSafe{matrix:markupsafe:}", + "itsdangerous{matrix:itsdangerous:}", + "Werkzeug{matrix:werkzeug:}", + "flask{matrix:flask}", +] + +[envs.appsec_integrations_flask.scripts] +test = [ + "uname -a", + "pip freeze", + "DD_TRACE_AGENT_URL=http://localhost:9126 DD_CIVISIBILITY_ITR_ENABLED=0 DD_IAST_REQUEST_SAMPLING=100 _DD_APPSEC_DEDUPLICATION_ENABLED=false python -m pytest -vvv {args:tests/appsec/integrations/flask_tests/}", +] + +[[envs.appsec_integrations_flask.matrix]] +python = ["3.8", "3.9"] +flask = ["~=1.1"] +# https://github.com/pallets/markupsafe/issues/282 +# DEV: Breaking change made in 2.1.0 release +markupsafe = ["~=1.1"] +itsdangerous = ["==2.0.1"] +# DEV: Flask 1.0.x is missing a maximum version for werkzeug dependency +werkzeug = ["==2.0.3"] + +[[envs.appsec_integrations_flask.matrix]] +python = ["3.8", "3.9", "3.10", "3.11"] +flask = ["~=2.2"] + +[[envs.appsec_integrations_flask.matrix]] +python = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] +flask = ["~=2.2"] + +[[envs.appsec_integrations_flask.matrix]] +python = ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] +flask = ["~=3.0"] + + + ## ASM FastAPI [envs.appsec_threats_fastapi] @@ -327,16 +429,16 @@ test = [ # if you add or remove a version here, please also update the parallelism parameter # in .circleci/config.templ.yml [[envs.appsec_threats_fastapi.matrix]] -python = ["3.7", "3.9", "3.11"] +python = ["3.8", "3.10", "3.13"] fastapi = ["==0.86.0"] anyio = ["==3.7.1"] [[envs.appsec_threats_fastapi.matrix]] -python = ["3.7", "3.9", "3.12"] +python = ["3.8", "3.10", "3.13"] fastapi = ["==0.94.1"] [[envs.appsec_threats_fastapi.matrix]] -python = ["3.8", "3.10", "3.12"] +python = ["3.8", "3.10", "3.13"] fastapi = ["~=0.114.2"] diff --git a/lib-injection/sources/sitecustomize.py b/lib-injection/sources/sitecustomize.py index 0f87b770edd..32ab1c31ff3 100644 --- a/lib-injection/sources/sitecustomize.py +++ b/lib-injection/sources/sitecustomize.py @@ -45,6 +45,7 @@ def parse_version(version): TELEMETRY_ENABLED = "DD_INJECTION_ENABLED" in os.environ DEBUG_MODE = os.environ.get("DD_TRACE_DEBUG", "").lower() in ("true", "1", "t") INSTALLED_PACKAGES = {} +DDTRACE_VERSION = "unknown" PYTHON_VERSION = "unknown" PYTHON_RUNTIME = "unknown" PKGS_ALLOW_LIST = {} @@ -133,7 +134,7 @@ def create_count_metric(metric, tags=None): } -def gen_telemetry_payload(telemetry_events, ddtrace_version="unknown"): +def gen_telemetry_payload(telemetry_events, ddtrace_version): return { "metadata": { "language_name": "python", @@ -233,6 +234,7 @@ def get_first_incompatible_sysarg(): def _inject(): + global DDTRACE_VERSION global INSTALLED_PACKAGES global PYTHON_VERSION global PYTHON_RUNTIME @@ -353,10 +355,7 @@ def _inject(): if not os.path.exists(site_pkgs_path): _log("ddtrace site-packages not found in %r, aborting" % site_pkgs_path, level="error") TELEMETRY_DATA.append( - gen_telemetry_payload( - [create_count_metric("library_entrypoint.abort", ["reason:missing_" + site_pkgs_path])], - DDTRACE_VERSION, - ) + create_count_metric("library_entrypoint.abort", ["reason:missing_" + site_pkgs_path]), ) return @@ -369,14 +368,9 @@ def _inject(): except BaseException as e: _log("failed to load ddtrace module: %s" % e, level="error") TELEMETRY_DATA.append( - gen_telemetry_payload( - [ - create_count_metric( - "library_entrypoint.error", ["error_type:import_ddtrace_" + type(e).__name__.lower()] - ) - ], - DDTRACE_VERSION, - ) + create_count_metric( + "library_entrypoint.error", ["error_type:import_ddtrace_" + type(e).__name__.lower()] + ), ) return @@ -408,28 +402,18 @@ def _inject(): _log("successfully configured ddtrace package, python path is %r" % os.environ["PYTHONPATH"]) TELEMETRY_DATA.append( - gen_telemetry_payload( + create_count_metric( + "library_entrypoint.complete", [ - create_count_metric( - "library_entrypoint.complete", - [ - "injection_forced:" + str(runtime_incomp or integration_incomp).lower(), - ], - ) + "injection_forced:" + str(runtime_incomp or integration_incomp).lower(), ], - DDTRACE_VERSION, - ) + ), ) except Exception as e: TELEMETRY_DATA.append( - gen_telemetry_payload( - [ - create_count_metric( - "library_entrypoint.error", ["error_type:init_ddtrace_" + type(e).__name__.lower()] - ) - ], - DDTRACE_VERSION, - ) + create_count_metric( + "library_entrypoint.error", ["error_type:init_ddtrace_" + type(e).__name__.lower()] + ), ) _log("failed to load ddtrace.bootstrap.sitecustomize: %s" % e, level="error") return @@ -451,12 +435,11 @@ def _inject(): _inject() except Exception as e: TELEMETRY_DATA.append( - gen_telemetry_payload( - [create_count_metric("library_entrypoint.error", ["error_type:main_" + type(e).__name__.lower()])] - ) + create_count_metric("library_entrypoint.error", ["error_type:main_" + type(e).__name__.lower()]) ) finally: if TELEMETRY_DATA: - send_telemetry(TELEMETRY_DATA) + payload = gen_telemetry_payload(TELEMETRY_DATA, DDTRACE_VERSION) + send_telemetry(payload) except Exception: pass # absolutely never allow exceptions to propagate to the app diff --git a/pyproject.toml b/pyproject.toml index df5fbdcdbb2..03560d1b171 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,9 +55,9 @@ ddtrace-run = "ddtrace.commands.ddtrace_run:main" ddcontextvars_context = "ddtrace.internal.opentelemetry.context:DDRuntimeContext" [project.entry-points.pytest11] -ddtrace = "ddtrace.contrib.pytest.plugin" -"ddtrace.pytest_bdd" = "ddtrace.contrib.pytest_bdd.plugin" -"ddtrace.pytest_benchmark" = "ddtrace.contrib.pytest_benchmark.plugin" +ddtrace = "ddtrace.contrib.internal.pytest.plugin" +"ddtrace.pytest_bdd" = "ddtrace.contrib.internal.pytest_bdd.plugin" +"ddtrace.pytest_benchmark" = "ddtrace.contrib.internal.pytest_benchmark.plugin" [project.entry-points.'ddtrace.products'] "code-origin-for-spans" = "ddtrace.debugging._products.code_origin.span" diff --git a/releasenotes/notes/ATO_V3-e7f73ecf00d1474b.yaml b/releasenotes/notes/ATO_V3-e7f73ecf00d1474b.yaml new file mode 100644 index 00000000000..0a2757dba6a --- /dev/null +++ b/releasenotes/notes/ATO_V3-e7f73ecf00d1474b.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + ASM: This introduces full support for Automated user lifecycle tracking for login events (success and failure) diff --git a/releasenotes/notes/case_insensitive_baggage_header_extraction-63167c492474da6f.yaml b/releasenotes/notes/case_insensitive_baggage_header_extraction-63167c492474da6f.yaml new file mode 100644 index 00000000000..ad0eacb28e8 --- /dev/null +++ b/releasenotes/notes/case_insensitive_baggage_header_extraction-63167c492474da6f.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + tracer: This fix resolves an issue where baggage header extraction was case sensitive and didn't accept the header prepended with HTTP. + Now the baggage header will be extracted regardless of casing and the HTTP format. + diff --git a/releasenotes/notes/ddtrace-resourcefilter-deprecated-52b1c92d388b0518.yaml b/releasenotes/notes/ddtrace-resourcefilter-deprecated-52b1c92d388b0518.yaml new file mode 100644 index 00000000000..183249aa688 --- /dev/null +++ b/releasenotes/notes/ddtrace-resourcefilter-deprecated-52b1c92d388b0518.yaml @@ -0,0 +1,4 @@ +--- +deprecations: + - | + tracing: Deprecates ``ddtrace.filters.FilterRequestsOnUrl``. Spans should be filtered/sampled using DD_TRACE_SAMPLING_RULES configuration. diff --git a/releasenotes/notes/denylist-extend-more-f0d96917c50d43cf.yaml b/releasenotes/notes/denylist-extend-more-f0d96917c50d43cf.yaml new file mode 100644 index 00000000000..b0c378dadaa --- /dev/null +++ b/releasenotes/notes/denylist-extend-more-f0d96917c50d43cf.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Add more modules to the IAST patching denylist to improve startup time diff --git a/releasenotes/notes/deprecate-multiple-tracer-instances-078b920081ba4a36.yaml b/releasenotes/notes/deprecate-multiple-tracer-instances-078b920081ba4a36.yaml new file mode 100644 index 00000000000..7b96d366269 --- /dev/null +++ b/releasenotes/notes/deprecate-multiple-tracer-instances-078b920081ba4a36.yaml @@ -0,0 +1,4 @@ +--- +deprecations: + - | + tracing: Deprecates the use of multiple tracer instances in the same process. The global tracer (``ddtrace.tracer``) `should be used instead. diff --git a/releasenotes/notes/feat-openai-streamed-chunk-auto-extract-4cbaea8870b1df13.yaml b/releasenotes/notes/feat-openai-streamed-chunk-auto-extract-4cbaea8870b1df13.yaml new file mode 100644 index 00000000000..afaf95876d5 --- /dev/null +++ b/releasenotes/notes/feat-openai-streamed-chunk-auto-extract-4cbaea8870b1df13.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + openai: Introduces automatic extraction of token usage from streamed chat completions. + Unless ``stream_options: {"include_usage": False}`` is explicitly set on your streamed chat completion request, + the OpenAI integration will add ``stream_options: {"include_usage": True}`` to your request and automatically extract the token usage chunk from the streamed response. diff --git a/releasenotes/notes/fix-bedrock-model-id-parsing-611aea2ca2e00656.yaml b/releasenotes/notes/fix-bedrock-model-id-parsing-611aea2ca2e00656.yaml new file mode 100644 index 00000000000..c3e13ea3d38 --- /dev/null +++ b/releasenotes/notes/fix-bedrock-model-id-parsing-611aea2ca2e00656.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + botocore: Resolves formatting errors in the bedrock integration when parsing request model IDs, which can now accept AWS ARNs. diff --git a/releasenotes/notes/fix-er-include-nonlocals-bbeecfbbbde35496.yaml b/releasenotes/notes/fix-er-include-nonlocals-bbeecfbbbde35496.yaml new file mode 100644 index 00000000000..4d77fddb710 --- /dev/null +++ b/releasenotes/notes/fix-er-include-nonlocals-bbeecfbbbde35496.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + exception replay: include missing nonlocal variables in snapshot log messages. diff --git a/releasenotes/notes/fix-llmobs-default-writer-hooks-5e456c2f7dfd4381.yaml b/releasenotes/notes/fix-llmobs-default-writer-hooks-5e456c2f7dfd4381.yaml new file mode 100644 index 00000000000..702e2538b99 --- /dev/null +++ b/releasenotes/notes/fix-llmobs-default-writer-hooks-5e456c2f7dfd4381.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + LLM Observability: Resolves an issue where enabling LLM Observability in agentless mode would result in traces also being sent to the agent proxy endpoint. \ No newline at end of file diff --git a/releasenotes/notes/fix-llmobs-processor-4afd715a84323d32.yaml b/releasenotes/notes/fix-llmobs-processor-4afd715a84323d32.yaml new file mode 100644 index 00000000000..5912a415022 --- /dev/null +++ b/releasenotes/notes/fix-llmobs-processor-4afd715a84323d32.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + LLM Observability: Resolves an issue where configuring custom trace filters/processors onto the tracer would disable LLM Observability. + Note that if LLM Observability is enabled in agentless mode, writing APM traces must be explicitly disabled by setting `DD_TRACE_ENABLED=0`. diff --git a/releasenotes/notes/fix-ssi-telemetry-events-a0a01ad0b6ef63b5.yaml b/releasenotes/notes/fix-ssi-telemetry-events-a0a01ad0b6ef63b5.yaml new file mode 100644 index 00000000000..a1eba938bb8 --- /dev/null +++ b/releasenotes/notes/fix-ssi-telemetry-events-a0a01ad0b6ef63b5.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + lib-injection: Fixes incorrect telemetry data payload format. diff --git a/releasenotes/notes/iast-feat-code-injection-0213a27bc3340505.yaml b/releasenotes/notes/iast-feat-code-injection-0213a27bc3340505.yaml new file mode 100644 index 00000000000..0ca2d6fc398 --- /dev/null +++ b/releasenotes/notes/iast-feat-code-injection-0213a27bc3340505.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + Code Security (IAST): Code Injection vulnerability detection, + which will be displayed on your DataDog Vulnerability Explorer dashboard. + See the `Application Vulnerability Management `_ documentation for more information about this feature. diff --git a/releasenotes/notes/internalize-samplers-b4f014f680f872f9.yaml b/releasenotes/notes/internalize-samplers-b4f014f680f872f9.yaml new file mode 100644 index 00000000000..28c8dec3619 --- /dev/null +++ b/releasenotes/notes/internalize-samplers-b4f014f680f872f9.yaml @@ -0,0 +1,4 @@ +--- +deprecations: + - | + tracer: Deprecates support for configuring samplers via a programmatic API. In v3.0.0 samplers will only be configurable via environment variables or remote configuration. \ No newline at end of file diff --git a/releasenotes/notes/make-ci-vis-ints-internal-532bc22d19bb62ab.yaml b/releasenotes/notes/make-ci-vis-ints-internal-532bc22d19bb62ab.yaml new file mode 100644 index 00000000000..334a65b1cfb --- /dev/null +++ b/releasenotes/notes/make-ci-vis-ints-internal-532bc22d19bb62ab.yaml @@ -0,0 +1,5 @@ +--- +deprecations: + - | + ci vis: Moves the implementational details of the pytest, pytest_benchmark, pytest_bdd, and unittest integrations + from ``ddtrace.contrib.`` to ``ddtrace.contrib.internal.``. diff --git a/releasenotes/notes/make-rq-imp-details-internal-c54afd12ba7b4bb7.yaml b/releasenotes/notes/make-rq-imp-details-internal-c54afd12ba7b4bb7.yaml new file mode 100644 index 00000000000..856c3367ee5 --- /dev/null +++ b/releasenotes/notes/make-rq-imp-details-internal-c54afd12ba7b4bb7.yaml @@ -0,0 +1,5 @@ +--- +deprecations: + - | + rq: Ensures the implementation details of the rq integration are internal to ddtrace library. + In ``ddtrace>=3.0.0`` this integration should only be enabled and configured via ``ddtrace.patch(..)``, ``import ddtrace.auto`` or the ``ddtrace-run`` command. diff --git a/releasenotes/notes/move-pin-and-filters-to-trace-package-2f47fa2d2592b413.yaml b/releasenotes/notes/move-pin-and-filters-to-trace-package-2f47fa2d2592b413.yaml new file mode 100644 index 00000000000..29b3ebe277d --- /dev/null +++ b/releasenotes/notes/move-pin-and-filters-to-trace-package-2f47fa2d2592b413.yaml @@ -0,0 +1,6 @@ +--- +deprecations: + - | + tracing: Deprecates ``ddtrace.pin`` module and moves the ``Pin`` class to ``ddtrace.trace`` package. In v3.0.0 the ``ddtrace/pin.py`` will be removed. + - | + tracing: Deprecates ``ddtrace.filters`` module and moves the ``TraceFilter`` and ``FilterRequestsOnUrl`` classes to ``ddtrace.trace`` package. In v3.0.0 the ``ddtrace/filters.py`` will be removed. \ No newline at end of file diff --git a/releasenotes/notes/profiling-lock-acquired-at-e308547cffdca9f7.yaml b/releasenotes/notes/profiling-lock-acquired-at-e308547cffdca9f7.yaml new file mode 100644 index 00000000000..de86c8227b6 --- /dev/null +++ b/releasenotes/notes/profiling-lock-acquired-at-e308547cffdca9f7.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + profiling: This fix resolves a data race issue accessing lock's acquired + time, leading to an ``AttributeError``: ``_Profiled_ThreadingLock`` object + has no attribute ``self_acquired_at`` diff --git a/releasenotes/notes/profiling-lock-acquired-at-remove-c8b5b96130a46ca8.yaml b/releasenotes/notes/profiling-lock-acquired-at-remove-c8b5b96130a46ca8.yaml new file mode 100644 index 00000000000..fdb268dcd76 --- /dev/null +++ b/releasenotes/notes/profiling-lock-acquired-at-remove-c8b5b96130a46ca8.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + profiling: resolves an issue where lock release would have been captured + with a wrong acquire timestamp diff --git a/releasenotes/notes/profiling-stack-v2-default-ecd535ccf0c73ce0.yaml b/releasenotes/notes/profiling-stack-v2-default-ecd535ccf0c73ce0.yaml new file mode 100644 index 00000000000..a998fe652fc --- /dev/null +++ b/releasenotes/notes/profiling-stack-v2-default-ecd535ccf0c73ce0.yaml @@ -0,0 +1,19 @@ +--- +features: + - | + profiling: Stack V2 is enabled by default. It is the new stack sampler + implementation for CPython 3.8+. It enhances the performance, accuracy, + and reliability of Python CPU profiling. This feature activates our new + stack sampling, collection and export system. + + The following are known issues and missing features from Stack V2 + + - Services using ``gunicorn`` with Stack V2 results in performance degradation + - Support for ``gevent`` is lacking + - Exception sampling is missing + + If you find these as a blocker for enabling Stack V2 for your services, you + can turn it off via setting ``DD_PROFILING_STACK_V2_ENABLED=0``. If you + find any other issue, then please proceed to escalate using appropriate + support channels or file an issue on the repository. + diff --git a/releasenotes/notes/propagation_behavior_extract-3d16765cfd07485b.yaml b/releasenotes/notes/propagation_behavior_extract-3d16765cfd07485b.yaml new file mode 100644 index 00000000000..6e1def89993 --- /dev/null +++ b/releasenotes/notes/propagation_behavior_extract-3d16765cfd07485b.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + propagation: Introduces the environment variable ``DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT`` + to control the behavior of the extraction of distributed tracing headers. The values, ``continue`` (default), + ``ignore``, and ``restart``, are supported. The default value is ``continue`` which has no change from the current behavior of always propagating valid headers. + ``ignore`` ignores all incoming headers, never propagating the incoming trace information + and ``restart`` turns the first extracted propagation style into a span link and propagates baggage if extracted. + +fixes: + - | + propagation: Fixes an issue where the baggage header was not being propagated when the baggage header was the only header extracted. + With this fix, the baggage header is now propagated when it is the only header extracted. diff --git a/releasenotes/notes/rasp_cmdi-3c7819ee9e33e447.yaml b/releasenotes/notes/rasp_cmdi-3c7819ee9e33e447.yaml new file mode 100644 index 00000000000..89744bf9be2 --- /dev/null +++ b/releasenotes/notes/rasp_cmdi-3c7819ee9e33e447.yaml @@ -0,0 +1,6 @@ +--- +features: + - | + ASM: This introduces the support for command injection for Exploit Prevention. With previous support of shell injection with os.system, + this provides automatic instrumentation for subprocess module functions and os.spawn* functions, + ensuring monitoring and blocking for Exploit Prevention on those endpoints. diff --git a/releasenotes/notes/remove-multi-tracer-support-from-pin-f2f20ca3fa731929.yaml b/releasenotes/notes/remove-multi-tracer-support-from-pin-f2f20ca3fa731929.yaml new file mode 100644 index 00000000000..18c70a15b04 --- /dev/null +++ b/releasenotes/notes/remove-multi-tracer-support-from-pin-f2f20ca3fa731929.yaml @@ -0,0 +1,4 @@ +--- +deprecations: + - | + tracer: Deprecates the ability to use multiple tracer instances with ddtrace.Pin. In v3.0.0 pin objects will only use the global tracer. diff --git a/releasenotes/notes/resolves-gevent-asyncio-incompatiblities-246028676b10bea9.yaml b/releasenotes/notes/resolves-gevent-asyncio-incompatiblities-246028676b10bea9.yaml new file mode 100644 index 00000000000..08a5448b3dc --- /dev/null +++ b/releasenotes/notes/resolves-gevent-asyncio-incompatiblities-246028676b10bea9.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + asyncio: Resolves an issue where asyncio event loops fail to register when ``ddtrace-run``/``import ddtrace.auto`` is used and gevent is installed. diff --git a/releasenotes/notes/submit-evaluation-for-01096d803d969e3e.yaml b/releasenotes/notes/submit-evaluation-for-01096d803d969e3e.yaml new file mode 100644 index 00000000000..c2e4b25f255 --- /dev/null +++ b/releasenotes/notes/submit-evaluation-for-01096d803d969e3e.yaml @@ -0,0 +1,17 @@ +--- +features: + - | + LLM Observability: This introduces the `LLMObs.submit_evaluation_for` method, which provides the ability to join a custom evaluation + to a span using a tag key-value pair on the span. The tag key-value pair is expected to uniquely identify a single span. + Tag-based joining is an alternative to the existing method of joining evaluations to spans using trace and span IDs. + Example usage: + - Evaluation joined by tag: `LLMObs.submit_evaluation_for(span_with_tag_value={"tag_key": "message_id", "tag_value": "dummy_message_id"}, label="rating", ...)`. + - Evaluation joined by trace/span ID: `LLMObs.submit_evaluation_for(span={"trace_id": "...", "span_id": "..."}, label="rating", ...)`. +deprecations: + - | + LLM Observability: `LLMObs.submit_evaluation` is deprecated and will be removed in ddtrace 3.0.0. + As an alternative to `LLMObs.submit_evaluation`, you can use `LLMObs.submit_evaluation_for` instead. + To migrate, replace `LLMObs.submit_evaluation(span_context={"span_id": ..., "trace_id": ...}, ...)` with: + `LLMObs.submit_evaluation_for(span={"span_id": ..., "trace_id": ...}, ...) + You may also join an evaluation to a span using a tag key-value pair like so: + `LLMObs.submit_evaluation_for(span_with_tag_value={"tag_key": ..., "tag_val": ...}, ...)`. diff --git a/releasenotes/notes/threethirteen-botocore-ee2431d065f99d7b.yaml b/releasenotes/notes/threethirteen-botocore-ee2431d065f99d7b.yaml new file mode 100644 index 00000000000..8bbeaa4d294 --- /dev/null +++ b/releasenotes/notes/threethirteen-botocore-ee2431d065f99d7b.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + botocore: Makes boto-related integrations compatible with Python 3.13 diff --git a/riotfile.py b/riotfile.py index 0d9f66ca925..b172e648e74 100644 --- a/riotfile.py +++ b/riotfile.py @@ -228,50 +228,16 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT }, ), Venv( - name="appsec_integrations", - command="pytest {cmdargs} tests/appsec/integrations/", + name="appsec_integrations_pygoat", + pys=select_pys(min_version="3.10"), + command="pytest {cmdargs} tests/appsec/integrations/pygoat_tests/", pkgs={ "requests": latest, - "gunicorn": latest, - "psycopg2-binary": "~=2.9.9", }, env={ "DD_CIVISIBILITY_ITR_ENABLED": "0", "DD_IAST_REQUEST_SAMPLING": "100", # Override default 30% to analyze all IAST requests }, - venvs=[ - # Flask 1.x.x - Venv( - pys=select_pys(min_version="3.7", max_version="3.9"), - pkgs={ - "flask": "~=1.0", - # https://github.com/pallets/itsdangerous/issues/290 - # DEV: Breaking change made in 2.1.0 release - "itsdangerous": "<2.1.0", - # https://github.com/pallets/markupsafe/issues/282 - # DEV: Breaking change made in 2.1.0 release - "markupsafe": "<2.0", - # DEV: Flask 1.0.x is missing a maximum version for werkzeug dependency - "werkzeug": "<2.0", - }, - ), - # Flask 2.x.x - Venv( - pys=select_pys(min_version="3.7", max_version="3.11"), - pkgs={ - "flask": "~=2.2", - }, - ), - # Flask 3.x.x - Venv( - pys=select_pys(min_version="3.8", max_version="3.12"), - pkgs={ - "flask": "~=3.0", - "langchain": "==0.0.354", - "langchain_experimental": "==0.0.47", - }, - ), - ], ), Venv( name="profile-diff", @@ -330,9 +296,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT Venv( name="telemetry", command="pytest {cmdargs} tests/telemetry/", - env={ - "DD_PROFILING__FORCE_LEGACY_EXPORTER": "1", - }, pys=select_pys(), pkgs={ "requests": latest, @@ -411,7 +374,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT name="internal", env={ "DD_TRACE_AGENT_URL": "http://ddagent:8126", - "DD_PROFILING__FORCE_LEGACY_EXPORTER": "1", "DD_INSTRUMENTATION_TELEMETRY_ENABLED": "0", }, command="pytest -v {cmdargs} tests/internal/", @@ -516,9 +478,6 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( name="ddtracerun", - env={ - "DD_PROFILING__FORCE_LEGACY_EXPORTER": "1", - }, command="pytest {cmdargs} --no-cov tests/commands/test_runner.py", venvs=[ Venv( @@ -1462,7 +1421,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT }, venvs=[ Venv( - pys=select_pys(min_version="3.8", max_version="3.12"), + pys=select_pys(min_version="3.8"), pkgs={"botocore": "==1.34.49", "boto3": "==1.34.49"}, ), ], @@ -1573,7 +1532,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( name="aiobotocore", - command="pytest {cmdargs} tests/contrib/aiobotocore", + command="pytest {cmdargs} --no-cov tests/contrib/aiobotocore", pkgs={ "pytest-asyncio": "==0.21.1", "async_generator": ["~=1.10"], @@ -1587,7 +1546,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT }, ), Venv( - pys=select_pys(min_version="3.12", max_version="3.12"), + pys=select_pys(min_version="3.12"), pkgs={"aiobotocore": latest}, ), ], @@ -2700,6 +2659,11 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT pys=select_pys(min_version="3.10", max_version="3.12"), pkgs={"tornado": ["==6.2", "==6.3.1"]}, ), + Venv( + # tornado fixed a bug affecting 3.13 in 6.4.1 + pys=select_pys(min_version="3.13"), + pkgs={"tornado": "==6.4.1"}, + ), ], ), Venv( @@ -2945,6 +2909,17 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT env={ "DD_AGENT_PORT": "9126", }, + venvs=[ + # Python 3.8 + Venv( + pys=["3.8"], + pkgs={"greenlet": "==3.1.0"}, + ), + # Python 3.9+ + Venv( + pys=select_pys(min_version="3.9"), + ), + ], ), Venv( name="subprocess", @@ -2958,8 +2933,8 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT name="llmobs", command="pytest {cmdargs} tests/llmobs", pkgs={"vcrpy": latest, "pytest-asyncio": "==0.21.1"}, - pys=select_pys(min_version="3.7"), venvs=[ + Venv(pys="3.7"), Venv(pys=select_pys(min_version="3.8"), pkgs={"ragas": "==0.1.21", "langchain": latest}), ], ), @@ -2969,6 +2944,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT command="python -m tests.profiling.run pytest -v --no-cov --capture=no --benchmark-disable {cmdargs} tests/profiling", # noqa: E501 env={ "DD_PROFILING_ENABLE_ASSERTS": "1", + "DD_PROFILING_STACK_V2_ENABLED": "0", "DD_PROFILING__FORCE_LEGACY_EXPORTER": "1", "CPUCOUNT": "12", }, diff --git a/scripts/gen_circleci_config.py b/scripts/gen_circleci_config.py index bc51f2c5519..3a68a1a7975 100644 --- a/scripts/gen_circleci_config.py +++ b/scripts/gen_circleci_config.py @@ -17,10 +17,13 @@ def gen_required_suites(template: dict) -> None: required_suites = template["requires_tests"]["requires"] = [] for_each_testrun_needed( suites=sorted( - set(n.rpartition("::")[-1] for n, s in get_suites().items() if not s.get("skip", False)) - & set(template["jobs"].keys()) + set( + n + for n, s in get_suites().items() + if not s.get("skip", False) and n.rpartition("::")[-1] in template["jobs"] + ) ), - action=lambda suite: required_suites.append(suite), + action=lambda suite: required_suites.append(suite.rpartition("::")[-1]), git_selections=extract_git_commit_selections(os.getenv("GIT_COMMIT_DESC", "")), ) diff --git a/scripts/gen_gitlab_config.py b/scripts/gen_gitlab_config.py index 8dc9e5b178f..96dfd5a4ff0 100644 --- a/scripts/gen_gitlab_config.py +++ b/scripts/gen_gitlab_config.py @@ -22,6 +22,7 @@ class JobSpec: timeout: t.Optional[int] = None skip: bool = False paths: t.Optional[t.Set[str]] = None # ignored + only: t.Optional[t.Set[str]] = None # ignored def __str__(self) -> str: lines = [] @@ -60,6 +61,11 @@ def __str__(self) -> str: for key, value in env.items(): lines.append(f" {key}: {value}") + if self.only: + lines.append(" only:") + for value in self.only: + lines.append(f" - {value}") + if self.parallelism is not None: lines.append(f" parallel: {self.parallelism}") diff --git a/tests/appsec/app.py b/tests/appsec/app.py index eb5beb666cf..e7e5dbaf231 100644 --- a/tests/appsec/app.py +++ b/tests/appsec/app.py @@ -87,7 +87,7 @@ from tests.appsec.iast_packages.packages.pkg_wrapt import pkg_wrapt from tests.appsec.iast_packages.packages.pkg_yarl import pkg_yarl from tests.appsec.iast_packages.packages.pkg_zipp import pkg_zipp -import tests.appsec.integrations.module_with_import_errors as module_with_import_errors +import tests.appsec.integrations.flask_tests.module_with_import_errors as module_with_import_errors app = Flask(__name__) diff --git a/tests/appsec/appsec/rules-rasp-blocking.json b/tests/appsec/appsec/rules-rasp-blocking.json index f2f8c4d7955..e5038e4a7c2 100644 --- a/tests/appsec/appsec/rules-rasp-blocking.json +++ b/tests/appsec/appsec/rules-rasp-blocking.json @@ -201,6 +201,55 @@ "stack_trace", "block" ] + }, + { + "id": "rasp-932-110", + "name": "OS command injection exploit", + "tags": { + "type": "command_injection", + "category": "vulnerability_trigger", + "cwe": "77", + "capec": "1000/152/248/88", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.sys.exec.cmd" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ] + }, + "operator": "cmdi_detector" + } + ], + "transformers": [], + "on_match": [ + "stack_trace", + "block" + ] } ] } \ No newline at end of file diff --git a/tests/appsec/appsec/rules-rasp-disabled.json b/tests/appsec/appsec/rules-rasp-disabled.json index 4a0943a34fb..ec67b186732 100644 --- a/tests/appsec/appsec/rules-rasp-disabled.json +++ b/tests/appsec/appsec/rules-rasp-disabled.json @@ -201,6 +201,55 @@ "on_match": [ "stack_trace" ] + }, + { + "id": "rasp-932-110", + "name": "OS command injection exploit", + "enabled": false, + "tags": { + "type": "command_injection", + "category": "vulnerability_trigger", + "cwe": "77", + "capec": "1000/152/248/88", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.sys.exec.cmd" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ] + }, + "operator": "cmdi_detector" + } + ], + "transformers": [], + "on_match": [ + "stack_trace" + ] } ] } \ No newline at end of file diff --git a/tests/appsec/appsec/rules-rasp-redirecting.json b/tests/appsec/appsec/rules-rasp-redirecting.json index a7a53db6e3b..6e2080b2dbf 100644 --- a/tests/appsec/appsec/rules-rasp-redirecting.json +++ b/tests/appsec/appsec/rules-rasp-redirecting.json @@ -211,6 +211,55 @@ "stack_trace", "block" ] + }, + { + "id": "rasp-932-110", + "name": "OS command injection exploit", + "tags": { + "type": "command_injection", + "category": "vulnerability_trigger", + "cwe": "77", + "capec": "1000/152/248/88", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.sys.exec.cmd" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ] + }, + "operator": "cmdi_detector" + } + ], + "transformers": [], + "on_match": [ + "stack_trace", + "block" + ] } ] } \ No newline at end of file diff --git a/tests/appsec/appsec/rules-rasp.json b/tests/appsec/appsec/rules-rasp.json index c1a6822d261..d73672392af 100644 --- a/tests/appsec/appsec/rules-rasp.json +++ b/tests/appsec/appsec/rules-rasp.json @@ -197,6 +197,54 @@ "on_match": [ "stack_trace" ] + }, + { + "id": "rasp-932-110", + "name": "OS command injection exploit", + "tags": { + "type": "command_injection", + "category": "vulnerability_trigger", + "cwe": "77", + "capec": "1000/152/248/88", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.sys.exec.cmd" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ] + }, + "operator": "cmdi_detector" + } + ], + "transformers": [], + "on_match": [ + "stack_trace" + ] } ] } \ No newline at end of file diff --git a/tests/appsec/appsec/test_processor.py b/tests/appsec/appsec/test_processor.py index 3fec599237b..ad3c9e6827a 100644 --- a/tests/appsec/appsec/test_processor.py +++ b/tests/appsec/appsec/test_processor.py @@ -638,7 +638,23 @@ def test_asm_context_registration(tracer): "name": "test required", "tags": {"category": "attack_attempt", "custom": "1", "type": "custom"}, "transformers": [], - } + }, + { + "conditions": [ + { + "operator": "match_regex", + "parameters": { + "inputs": [{"address": "usr.login"}], + "options": {"case_sensitive": False}, + "regex": "GET", + }, + } + ], + "id": "32b243c7-26eb-4046-bbbb-custom", + "name": "test required", + "tags": {"category": "attack_attempt", "custom": "1", "type": "custom"}, + "transformers": [], + }, ] } @@ -672,6 +688,7 @@ def test_required_addresses(): "server.request.query", "server.response.headers.no_cookies", "usr.id", + "usr.login", } diff --git a/tests/appsec/appsec/test_remoteconfiguration.py b/tests/appsec/appsec/test_remoteconfiguration.py index f00167706dc..1d2c47bc190 100644 --- a/tests/appsec/appsec/test_remoteconfiguration.py +++ b/tests/appsec/appsec/test_remoteconfiguration.py @@ -117,7 +117,7 @@ def test_rc_activation_states_off(tracer, appsec_enabled, rc_value, remote_confi @pytest.mark.parametrize( "rc_enabled, appsec_enabled, capability", [ - (True, "true", "D4HkA/w="), # All capabilities except ASM_ACTIVATION + (True, "true", "L4HkA/w="), # All capabilities except ASM_ACTIVATION (False, "true", ""), (True, "false", "gAAAAA=="), (False, "false", ""), @@ -142,7 +142,7 @@ def test_rc_capabilities(rc_enabled, appsec_enabled, capability, tracer): @pytest.mark.parametrize( "env_rules, expected", [ - ({}, "D4HkA/4="), # All capabilities + ({}, "L4HkA/4="), # All capabilities ({"_asm_static_rule_file": DEFAULT.RULES}, "gAAAAg=="), # Only ASM_FEATURES ], ) diff --git a/tests/appsec/appsec/test_telemetry.py b/tests/appsec/appsec/test_telemetry.py index 8678820e8d6..6932a7bdba0 100644 --- a/tests/appsec/appsec/test_telemetry.py +++ b/tests/appsec/appsec/test_telemetry.py @@ -12,7 +12,7 @@ from ddtrace.appsec._processor import AppSecSpanProcessor from ddtrace.contrib.trace_utils import set_http_meta from ddtrace.ext import SpanTypes -from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE_TAG_APPSEC +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE from ddtrace.internal.telemetry.constants import TELEMETRY_TYPE_DISTRIBUTION from ddtrace.internal.telemetry.constants import TELEMETRY_TYPE_GENERATE_METRICS import tests.appsec.rules as rules @@ -27,7 +27,7 @@ def _assert_generate_metrics(metrics_result, is_rule_triggered=False, is_blocked_request=False): - generate_metrics = metrics_result[TELEMETRY_TYPE_GENERATE_METRICS][TELEMETRY_NAMESPACE_TAG_APPSEC] + generate_metrics = metrics_result[TELEMETRY_TYPE_GENERATE_METRICS][TELEMETRY_NAMESPACE.APPSEC.value] assert len(generate_metrics) == 2, "Expected 2 generate_metrics" for _metric_id, metric in generate_metrics.items(): if metric.name == "waf.requests": @@ -44,7 +44,7 @@ def _assert_generate_metrics(metrics_result, is_rule_triggered=False, is_blocked def _assert_distributions_metrics(metrics_result, is_rule_triggered=False, is_blocked_request=False): - distributions_metrics = metrics_result[TELEMETRY_TYPE_DISTRIBUTION][TELEMETRY_NAMESPACE_TAG_APPSEC] + distributions_metrics = metrics_result[TELEMETRY_TYPE_DISTRIBUTION][TELEMETRY_NAMESPACE.APPSEC.value] assert len(distributions_metrics) == 2, "Expected 2 distributions_metrics" for _metric_id, metric in distributions_metrics.items(): @@ -69,8 +69,8 @@ def test_metrics_when_appsec_doesnt_runs(telemetry_writer, tracer): rules.Config(), ) metrics_data = telemetry_writer._namespace._metrics_data - assert len(metrics_data[TELEMETRY_TYPE_GENERATE_METRICS][TELEMETRY_NAMESPACE_TAG_APPSEC]) == 0 - assert len(metrics_data[TELEMETRY_TYPE_DISTRIBUTION][TELEMETRY_NAMESPACE_TAG_APPSEC]) == 0 + assert len(metrics_data[TELEMETRY_TYPE_GENERATE_METRICS][TELEMETRY_NAMESPACE.APPSEC.value]) == 0 + assert len(metrics_data[TELEMETRY_TYPE_DISTRIBUTION][TELEMETRY_NAMESPACE.APPSEC.value]) == 0 def test_metrics_when_appsec_runs(telemetry_writer, tracer): @@ -136,7 +136,7 @@ def test_log_metric_error_ddwaf_timeout(telemetry_writer, tracer): assert len(list_metrics_logs) == 0 generate_metrics = telemetry_writer._namespace._metrics_data[TELEMETRY_TYPE_GENERATE_METRICS][ - TELEMETRY_NAMESPACE_TAG_APPSEC + TELEMETRY_NAMESPACE.APPSEC.value ] timeout_found = False diff --git a/tests/appsec/contrib_appsec/django_app/urls.py b/tests/appsec/contrib_appsec/django_app/urls.py index 77ad7a7f0a6..1b691d43a53 100644 --- a/tests/appsec/contrib_appsec/django_app/urls.py +++ b/tests/appsec/contrib_appsec/django_app/urls.py @@ -1,5 +1,6 @@ import os import sqlite3 +import subprocess import tempfile import django @@ -129,13 +130,33 @@ def rasp(request, endpoint: str): res.append(f"Error: {e}") tracer.current_span()._local_root.set_tag("rasp.request.done", endpoint) return HttpResponse("<\\br>\n".join(res)) + elif endpoint == "shell_injection": + res = ["shell_injection endpoint"] + for param in query_params: + if param.startswith("cmd"): + cmd = query_params[param] + try: + if param.startswith("cmdsys"): + res.append(f'cmd stdout: {os.system(f"ls {cmd}")}') + else: + res.append(f'cmd stdout: {subprocess.run(f"ls {cmd}", shell=True)}') + except Exception as e: + res.append(f"Error: {e}") + tracer.current_span()._local_root.set_tag("rasp.request.done", endpoint) + return HttpResponse("<\\br>\n".join(res)) elif endpoint == "command_injection": res = ["command_injection endpoint"] for param in query_params: - if param.startswith("cmd"): + if param.startswith("cmda"): + cmd = query_params[param] + try: + res.append(f'cmd stdout: {subprocess.run([cmd, "-c", "3", "localhost"])}') + except Exception as e: + res.append(f"Error: {e}") + elif param.startswith("cmds"): cmd = query_params[param] try: - res.append(f'cmd stdout: {os.system(f"ls {cmd}")}') + res.append(f"cmd stdout: {subprocess.run(cmd)}") except Exception as e: res.append(f"Error: {e}") tracer.current_span()._local_root.set_tag("rasp.request.done", endpoint) @@ -175,7 +196,7 @@ def login_user(request): def new_service(request, service_name: str): import ddtrace - ddtrace.Pin.override(django, service=service_name, tracer=ddtrace.tracer) + ddtrace.trace.Pin.override(django, service=service_name, tracer=ddtrace.tracer) return HttpResponse(service_name, status=200) diff --git a/tests/appsec/contrib_appsec/fastapi_app/app.py b/tests/appsec/contrib_appsec/fastapi_app/app.py index 10b7b430543..3403df6f844 100644 --- a/tests/appsec/contrib_appsec/fastapi_app/app.py +++ b/tests/appsec/contrib_appsec/fastapi_app/app.py @@ -1,6 +1,7 @@ import asyncio import os import sqlite3 +import subprocess from typing import Optional from fastapi import FastAPI @@ -103,7 +104,7 @@ async def multi_view_no_param(request: Request): # noqa: B008 async def new_service(service_name: str, request: Request): # noqa: B008 import ddtrace - ddtrace.Pin.override(app, service=service_name, tracer=ddtrace.tracer) + ddtrace.trace.Pin.override(app, service=service_name, tracer=ddtrace.tracer) return HTMLResponse(service_name, 200) async def slow_numbers(minimum, maximum): @@ -178,13 +179,33 @@ async def rasp(endpoint: str, request: Request): res.append(f"Error: {e}") tracer.current_span()._local_root.set_tag("rasp.request.done", endpoint) return HTMLResponse("<\\br>\n".join(res)) + elif endpoint == "shell_injection": + res = ["shell_injection endpoint"] + for param in query_params: + if param.startswith("cmd"): + cmd = query_params[param] + try: + if param.startswith("cmdsys"): + res.append(f'cmd stdout: {os.system(f"ls {cmd}")}') + else: + res.append(f'cmd stdout: {subprocess.run(f"ls {cmd}", shell=True)}') + except Exception as e: + res.append(f"Error: {e}") + tracer.current_span()._local_root.set_tag("rasp.request.done", endpoint) + return HTMLResponse("<\\br>\n".join(res)) elif endpoint == "command_injection": res = ["command_injection endpoint"] for param in query_params: - if param.startswith("cmd"): + if param.startswith("cmda"): cmd = query_params[param] try: - res.append(f'cmd stdout: {os.system(f"ls {cmd}")}') + res.append(f'cmd stdout: {subprocess.run([cmd, "-c", "3", "localhost"])}') + except Exception as e: + res.append(f"Error: {e}") + elif param.startswith("cmds"): + cmd = query_params[param] + try: + res.append(f"cmd stdout: {subprocess.run(cmd)}") except Exception as e: res.append(f"Error: {e}") tracer.current_span()._local_root.set_tag("rasp.request.done", endpoint) @@ -214,23 +235,25 @@ def authenticate(username: str, password: str) -> Optional[str]: return USERS[username]["id"] else: appsec_trace_utils.track_user_login_failure_event( - tracer, user_id=USERS[username]["id"], exists=True, login_events_mode="auto" + tracer, user_id=USERS[username]["id"], exists=True, login_events_mode="auto", login=username ) return None appsec_trace_utils.track_user_login_failure_event( - tracer, user_id=username, exists=False, login_events_mode="auto" + tracer, user_id=username, exists=False, login_events_mode="auto", login=username ) return None - def login(user_id: str) -> None: + def login(user_id: str, username: str) -> None: """login user""" - appsec_trace_utils.track_user_login_success_event(tracer, user_id=user_id, login_events_mode="auto") + appsec_trace_utils.track_user_login_success_event( + tracer, user_id=user_id, login_events_mode="auto", login=username + ) username = request.query_params.get("username") password = request.query_params.get("password") user_id = authenticate(username=username, password=password) if user_id is not None: - login(user_id) + login(user_id, username) return HTMLResponse("OK") return HTMLResponse("login failure", status_code=401) diff --git a/tests/appsec/contrib_appsec/flask_app/app.py b/tests/appsec/contrib_appsec/flask_app/app.py index 5270229d3e9..aee42be7b54 100644 --- a/tests/appsec/contrib_appsec/flask_app/app.py +++ b/tests/appsec/contrib_appsec/flask_app/app.py @@ -1,5 +1,6 @@ import os import sqlite3 +import subprocess from typing import Optional from flask import Flask @@ -59,7 +60,7 @@ def multi_view(param_int=0, param_str=""): def new_service(service_name: str): import ddtrace - ddtrace.Pin.override(Flask, service=service_name, tracer=ddtrace.tracer) + ddtrace.trace.Pin.override(Flask, service=service_name, tracer=ddtrace.tracer) return service_name @@ -126,13 +127,33 @@ def rasp(endpoint: str): res.append(f"Error: {e}") tracer.current_span()._local_root.set_tag("rasp.request.done", endpoint) return "<\\br>\n".join(res) + elif endpoint == "shell_injection": + res = ["shell_injection endpoint"] + for param in query_params: + if param.startswith("cmd"): + cmd = query_params[param] + try: + if param.startswith("cmdsys"): + res.append(f'cmd stdout: {os.system(f"ls {cmd}")}') + else: + res.append(f'cmd stdout: {subprocess.run(f"ls {cmd}", shell=True)}') + except Exception as e: + res.append(f"Error: {e}") + tracer.current_span()._local_root.set_tag("rasp.request.done", endpoint) + return "<\\br>\n".join(res) elif endpoint == "command_injection": res = ["command_injection endpoint"] for param in query_params: - if param.startswith("cmd"): + if param.startswith("cmda"): cmd = query_params[param] try: - res.append(f'cmd stdout: {os.system(f"ls {cmd}")}') + res.append(f'cmd stdout: {subprocess.run([cmd, "-c", "3", "localhost"])}') + except Exception as e: + res.append(f"Error: {e}") + elif param.startswith("cmds"): + cmd = query_params[param] + try: + res.append(f"cmd stdout: {subprocess.run(cmd)}") except Exception as e: res.append(f"Error: {e}") tracer.current_span()._local_root.set_tag("rasp.request.done", endpoint) @@ -167,22 +188,24 @@ def authenticate(username: str, password: str) -> Optional[str]: return USERS[username]["id"] else: appsec_trace_utils.track_user_login_failure_event( - tracer, user_id=USERS[username]["id"], exists=True, login_events_mode="auto" + tracer, user_id=USERS[username]["id"], exists=True, login_events_mode="auto", login=username ) return None appsec_trace_utils.track_user_login_failure_event( - tracer, user_id=username, exists=False, login_events_mode="auto" + tracer, user_id=username, exists=False, login_events_mode="auto", login=username ) return None - def login(user_id: str) -> None: + def login(user_id: str, login: str) -> None: """login user""" - appsec_trace_utils.track_user_login_success_event(tracer, user_id=user_id, login_events_mode="auto") + appsec_trace_utils.track_user_login_success_event( + tracer, user_id=user_id, login_events_mode="auto", login=login + ) username = request.args.get("username") password = request.args.get("password") user_id = authenticate(username=username, password=password) if user_id is not None: - login(user_id) + login(user_id, username) return "OK" return "login failure", 401 diff --git a/tests/appsec/contrib_appsec/test_flask.py b/tests/appsec/contrib_appsec/test_flask.py index 90a35ac0c88..b497de98bf9 100644 --- a/tests/appsec/contrib_appsec/test_flask.py +++ b/tests/appsec/contrib_appsec/test_flask.py @@ -1,8 +1,8 @@ from flask.testing import FlaskClient import pytest -from ddtrace import Pin from ddtrace.internal.packages import get_version_for_package +from ddtrace.trace import Pin from tests.appsec.contrib_appsec import utils from tests.utils import TracerTestCase diff --git a/tests/appsec/contrib_appsec/utils.py b/tests/appsec/contrib_appsec/utils.py index 315caa49a5d..6400cacb625 100644 --- a/tests/appsec/contrib_appsec/utils.py +++ b/tests/appsec/contrib_appsec/utils.py @@ -1308,11 +1308,19 @@ def test_stream_response( + [("sql_injection", "user_id_1=1 OR 1=1&user_id_2=1 OR 1=1", "rasp-942-100", ("dispatch",))] + [ ( - "command_injection", - "cmd_1=$(cat /etc/passwd 1>%262 ; echo .)&cmd_2=$(uname -a 1>%262 ; echo .)", + "shell_injection", + "cmdsys_1=$(cat /etc/passwd 1>%262 ; echo .)&cmdrun_2=$(uname -a 1>%262 ; echo .)", "rasp-932-100", ("system", "rasp"), ) + ] + + [ + ( + "command_injection", + "cmda_1=/sbin/ping&cmds_2=/usr/bin/ls%20-la", + "rasp-932-110", + ("Popen", "rasp"), + ) ], ) @pytest.mark.parametrize( @@ -1371,7 +1379,7 @@ def validate_top_function(trace): assert get_tag(http.STATUS_CODE) == str(code), (get_tag(http.STATUS_CODE), code) if code == 200: assert self.body(response).startswith(f"{endpoint} endpoint") - telemetry_calls = {(c.__name__, f"{ns}.{nm}", t): v for (c, ns, nm, v, t), _ in mocked.call_args_list} + telemetry_calls = {(c.__name__, f"{ns.value}.{nm}", t): v for (c, ns, nm, v, t), _ in mocked.call_args_list} if asm_enabled and ep_enabled and action_level > 0: self.check_rules_triggered([rule] * (1 if action_level == 2 else 2), root_span) assert self.check_for_stack_trace(root_span) @@ -1381,11 +1389,23 @@ def validate_top_function(trace): trace ), f"unknown top function {trace['frames'][0]} {[t['function'] for t in trace['frames'][:4]]}" # assert mocked.call_args_list == [] + expected_rule_type = "command_injection" if endpoint == "shell_injection" else endpoint + expected_variant = ( + "exec" if endpoint == "command_injection" else "shell" if endpoint == "shell_injection" else None + ) matches = [t for c, n, t in telemetry_calls if c == "CountMetric" and n == "appsec.rasp.rule.match"] - assert matches == [(("rule_type", endpoint), ("waf_version", DDWAF_VERSION))], matches + if expected_variant: + expected_tags = ( + ("rule_type", expected_rule_type), + ("rule_variant", expected_variant), + ("waf_version", DDWAF_VERSION), + ) + else: + expected_tags = (("rule_type", expected_rule_type), ("waf_version", DDWAF_VERSION)) + assert matches == [expected_tags], matches evals = [t for c, n, t in telemetry_calls if c == "CountMetric" and n == "appsec.rasp.rule.eval"] # there may have been multiple evaluations of other rules too - assert (("rule_type", endpoint), ("waf_version", DDWAF_VERSION)) in evals + assert expected_tags in evals if action_level == 2: assert get_tag("rasp.request.done") is None, get_tag("rasp.request.done") else: @@ -1458,9 +1478,16 @@ def test_auto_user_events( assert get_tag("_dd.appsec.events.users.login.failure.sdk") == "true" else: assert get_tag("_dd.appsec.events.users.login.success.sdk") is None + if mode == "identification": + assert get_tag("_dd.appsec.usr.login") == user + elif mode == "anonymization": + assert get_tag("_dd.appsec.usr.login") == _hash_user_id(user) else: assert get_tag("appsec.events.users.login.success.track") == "true" assert get_tag("usr.id") == user_id_hash + assert get_tag("_dd.appsec.usr.id") == user_id_hash + if mode == "identification": + assert get_tag("_dd.appsec.usr.login") == user # check for manual instrumentation tag in manual instrumented frameworks if interface.name in ["flask", "fastapi"]: assert get_tag("_dd.appsec.events.users.login.success.sdk") == "true" @@ -1509,7 +1536,7 @@ def test_fingerprinting(self, interface, root_span, get_tag, asm_enabled, user_a def test_iast(self, interface, root_span, get_tag): from ddtrace.ext import http - url = "/rasp/command_injection/?cmd=." + url = "/rasp/command_injection/?cmds=." self.update_tracer(interface) response = interface.client.get(url) assert self.status(response) == 200 @@ -1540,8 +1567,8 @@ def test_tracer(): @contextmanager def post_tracer(interface): - original_tracer = getattr(ddtrace.Pin.get_from(interface.framework), "tracer", None) - ddtrace.Pin.override(interface.framework, tracer=interface.tracer) + original_tracer = getattr(ddtrace.trace.Pin.get_from(interface.framework), "tracer", None) + ddtrace.trace.Pin.override(interface.framework, tracer=interface.tracer) yield if original_tracer is not None: - ddtrace.Pin.override(interface.framework, tracer=original_tracer) + ddtrace.trace.Pin.override(interface.framework, tracer=original_tracer) diff --git a/tests/appsec/iast/_ast/test_ast_patching.py b/tests/appsec/iast/_ast/test_ast_patching.py index cf0fabd14e4..213737ecbce 100644 --- a/tests/appsec/iast/_ast/test_ast_patching.py +++ b/tests/appsec/iast/_ast/test_ast_patching.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import logging +import os import sys import astunparse @@ -9,7 +10,9 @@ from ddtrace.appsec._constants import IAST from ddtrace.appsec._iast._ast.ast_patching import _in_python_stdlib from ddtrace.appsec._iast._ast.ast_patching import _should_iast_patch +from ddtrace.appsec._iast._ast.ast_patching import _trie_has_prefix_for from ddtrace.appsec._iast._ast.ast_patching import astpatch_module +from ddtrace.appsec._iast._ast.ast_patching import build_trie from ddtrace.appsec._iast._ast.ast_patching import visit_ast from ddtrace.internal.utils.formats import asbool from tests.utils import override_env @@ -18,6 +21,15 @@ _PREFIX = IAST.PATCH_ADDED_SYMBOL_PREFIX +@pytest.fixture(autouse=True, scope="module") +def clear_iast_env_vars(): + if IAST.PATCH_MODULES in os.environ: + os.environ.pop("_DD_IAST_PATCH_MODULES") + if IAST.DENY_MODULES in os.environ: + os.environ.pop("_DD_IAST_DENY_MODULES") + yield + + @pytest.mark.parametrize( "source_text, module_path, module_name", [ @@ -146,15 +158,34 @@ def test_astpatch_source_unchanged(module_name): assert ("", None) == astpatch_module(__import__(module_name, fromlist=[None])) -def test_module_should_iast_patch(): +def test_should_iast_patch_allow_first_party(): + assert _should_iast_patch("tests.appsec.iast.integration.main") + assert _should_iast_patch("tests.appsec.iast.integration.print_str") + + +def test_should_not_iast_patch_if_vendored(): + assert not _should_iast_patch("foobar.vendor.requests") + assert not _should_iast_patch(("vendored.foobar.requests")) + + +def test_should_iast_patch_deny_by_default_if_third_party(): + # note that modules here must be in the ones returned by get_package_distributions() + # but not in ALLOWLIST or DENYLIST. So please don't put astunparse there :) + assert not _should_iast_patch("astunparse.foo.bar.not.in.deny.or.allow.list") + + +def test_should_not_iast_patch_if_in_denylist(): assert not _should_iast_patch("ddtrace.internal.module") assert not _should_iast_patch("ddtrace.appsec._iast") + assert not _should_iast_patch("pip.foo.bar") + + +def test_should_not_iast_patch_if_stdlib(): assert not _should_iast_patch("base64") - assert not _should_iast_patch("envier") assert not _should_iast_patch("itertools") assert not _should_iast_patch("http") - assert _should_iast_patch("tests.appsec.iast.integration.main") - assert _should_iast_patch("tests.appsec.iast.integration.print_str") + assert not _should_iast_patch("os.path") + assert not _should_iast_patch("sys.platform") @pytest.mark.parametrize( @@ -308,3 +339,87 @@ def test_astpatch_dir_patched_with_or_without_custom_dir(module_name, expected_n # Check that all the symbols in the expected set are in the patched dir() result for name in expected_names: assert name in patched_dir + + +def test_build_trie(): + from ddtrace.appsec._iast._ast.ast_patching import build_trie + + trie = build_trie(["abc", "def", "ghi", "jkl", "mno", "pqr", "stu", "vwx", "yz"]) + assert dict(trie) == { + "a": { + "b": { + "c": {"": None}, + }, + }, + "d": { + "e": { + "f": {"": None}, + }, + }, + "g": { + "h": { + "i": {"": None}, + }, + }, + "j": { + "k": { + "l": {"": None}, + }, + }, + "m": { + "n": { + "o": {"": None}, + }, + }, + "p": { + "q": { + "r": {"": None}, + }, + }, + "s": { + "t": { + "u": {"": None}, + }, + }, + "v": { + "w": { + "x": {"": None}, + }, + }, + "y": { + "z": {"": None}, + }, + } + + +def test_trie_has_string_match(): + trie = build_trie(["abc", "def", "ghi", "jkl", "mno", "pqr", "stu", "vwx", "yz"]) + assert _trie_has_prefix_for(trie, "abc") + assert not _trie_has_prefix_for(trie, "ab") + assert _trie_has_prefix_for(trie, "abcd") + assert _trie_has_prefix_for(trie, "def") + assert not _trie_has_prefix_for(trie, "de") + assert _trie_has_prefix_for(trie, "defg") + assert _trie_has_prefix_for(trie, "ghi") + assert not _trie_has_prefix_for(trie, "gh") + assert _trie_has_prefix_for(trie, "ghij") + assert _trie_has_prefix_for(trie, "jkl") + assert not _trie_has_prefix_for(trie, "jk") + assert _trie_has_prefix_for(trie, "jklm") + assert _trie_has_prefix_for(trie, "mno") + assert not _trie_has_prefix_for(trie, "mn") + assert _trie_has_prefix_for(trie, "mnop") + assert _trie_has_prefix_for(trie, "pqr") + assert not _trie_has_prefix_for(trie, "pq") + assert _trie_has_prefix_for(trie, "pqrs") + assert _trie_has_prefix_for(trie, "stu") + assert not _trie_has_prefix_for(trie, "st") + assert _trie_has_prefix_for(trie, "stuv") + assert _trie_has_prefix_for(trie, "vwx") + assert not _trie_has_prefix_for(trie, "vw") + assert _trie_has_prefix_for(trie, "vwxy") + assert _trie_has_prefix_for(trie, "yz") + assert not _trie_has_prefix_for(trie, "y") + assert _trie_has_prefix_for(trie, "yza") + assert not _trie_has_prefix_for(trie, "z") + assert not _trie_has_prefix_for(trie, "zzz") diff --git a/tests/appsec/iast/conftest.py b/tests/appsec/iast/conftest.py index 3daa3611f51..85d516dd154 100644 --- a/tests/appsec/iast/conftest.py +++ b/tests/appsec/iast/conftest.py @@ -2,6 +2,7 @@ import os import re import subprocess +import time import pytest @@ -14,7 +15,8 @@ from ddtrace.appsec._iast._iast_request_context import start_iast_context from ddtrace.appsec._iast._patches.json_tainting import patch as json_patch from ddtrace.appsec._iast._patches.json_tainting import unpatch_iast as json_unpatch -from ddtrace.appsec._iast.taint_sinks._base import VulnerabilityBase +from ddtrace.appsec._iast.taint_sinks.code_injection import patch as code_injection_patch +from ddtrace.appsec._iast.taint_sinks.code_injection import unpatch as code_injection_unpatch from ddtrace.appsec._iast.taint_sinks.command_injection import patch as cmdi_patch from ddtrace.appsec._iast.taint_sinks.command_injection import unpatch as cmdi_unpatch from ddtrace.appsec._iast.taint_sinks.header_injection import patch as header_injection_patch @@ -23,8 +25,10 @@ from ddtrace.appsec._iast.taint_sinks.weak_cipher import unpatch_iast as weak_cipher_unpatch from ddtrace.appsec._iast.taint_sinks.weak_hash import patch as weak_hash_patch from ddtrace.appsec._iast.taint_sinks.weak_hash import unpatch_iast as weak_hash_unpatch -from ddtrace.contrib.sqlite3.patch import patch as sqli_sqlite_patch -from ddtrace.contrib.sqlite3.patch import unpatch as sqli_sqlite_unpatch +from ddtrace.contrib.internal.sqlite3.patch import patch as sqli_sqlite_patch +from ddtrace.contrib.internal.sqlite3.patch import unpatch as sqli_sqlite_unpatch +from ddtrace.internal.utils.http import Response +from ddtrace.internal.utils.http import get_connection from tests.utils import override_env from tests.utils import override_global_config @@ -60,20 +64,20 @@ def _end_iast_context_and_oce(span=None): def iast_context(env, request_sampling=100.0, deduplication=False, asm_enabled=False): try: - from ddtrace.contrib.langchain.patch import patch as langchain_patch - from ddtrace.contrib.langchain.patch import unpatch as langchain_unpatch + from ddtrace.contrib.internal.langchain.patch import patch as langchain_patch + from ddtrace.contrib.internal.langchain.patch import unpatch as langchain_unpatch except Exception: langchain_patch = lambda: True # noqa: E731 langchain_unpatch = lambda: True # noqa: E731 try: - from ddtrace.contrib.sqlalchemy.patch import patch as sqlalchemy_patch - from ddtrace.contrib.sqlalchemy.patch import unpatch as sqlalchemy_unpatch + from ddtrace.contrib.internal.sqlalchemy.patch import patch as sqlalchemy_patch + from ddtrace.contrib.internal.sqlalchemy.patch import unpatch as sqlalchemy_unpatch except Exception: sqlalchemy_patch = lambda: True # noqa: E731 sqlalchemy_unpatch = lambda: True # noqa: E731 try: - from ddtrace.contrib.psycopg.patch import patch as psycopg_patch - from ddtrace.contrib.psycopg.patch import unpatch as psycopg_unpatch + from ddtrace.contrib.internal.psycopg.patch import patch as psycopg_patch + from ddtrace.contrib.internal.psycopg.patch import unpatch as psycopg_unpatch except Exception: psycopg_patch = lambda: True # noqa: E731 psycopg_unpatch = lambda: True # noqa: E731 @@ -90,7 +94,6 @@ class MockSpan: _iast_request_sampling=request_sampling, ) ), override_env(env): - VulnerabilityBase._reset_cache_for_testing() _start_iast_context_and_oce(MockSpan()) weak_hash_patch() weak_cipher_patch() @@ -100,6 +103,7 @@ class MockSpan: sqlalchemy_patch() cmdi_patch() header_injection_patch() + code_injection_patch() langchain_patch() patch_common_modules() yield @@ -112,6 +116,7 @@ class MockSpan: sqlalchemy_unpatch() cmdi_unpatch() header_injection_unpatch() + code_injection_unpatch() langchain_unpatch() _end_iast_context_and_oce() @@ -160,11 +165,27 @@ def check_native_code_exception_in_each_python_aspect_test(request, caplog): @pytest.fixture(scope="session") def configuration_endpoint(): current_dir = os.path.dirname(__file__) - cmd = [ - "python", - os.path.join(current_dir, "fixtures", "integration", "http_config_server.py"), - CONFIG_SERVER_PORT, - ] - process = subprocess.Popen(cmd, cwd=current_dir) + status = None + retries = 0 + while status != 200 and retries < 5: + cmd = [ + "python", + os.path.join(current_dir, "fixtures", "integration", "http_config_server.py"), + CONFIG_SERVER_PORT, + ] + process = subprocess.Popen(cmd, cwd=current_dir) + time.sleep(0.2) + + url = f"http://localhost:{CONFIG_SERVER_PORT}/" + conn = get_connection(url) + conn.request("GET", "/") + response = conn.getresponse() + result = Response.from_http_response(response) + status = result.status + retries += 1 + + if retries == 5: + pytest.skip("Failed to start the configuration server") + yield process.kill() diff --git a/tests/appsec/iast/fixtures/taint_sinks/code_injection.py b/tests/appsec/iast/fixtures/taint_sinks/code_injection.py new file mode 100644 index 00000000000..822ab547434 --- /dev/null +++ b/tests/appsec/iast/fixtures/taint_sinks/code_injection.py @@ -0,0 +1,55 @@ +""" +CAVEAT: the line number is important to some IAST tests, be careful to modify this file and update the tests if you +make some changes +""" +from ast import literal_eval + + +def pt_eval(origin_string): + r = eval(origin_string) + return r + + +def pt_eval_globals(origin_string): + context = {"x": 5, "y": 10} + r = eval(origin_string, context) + return r + + +def pt_eval_globals_locals(origin_string): + z = 15 # noqa: F841 + globals_dict = {"x": 10} + locals_dict = {"y": 20} + r = eval(origin_string, globals_dict, locals_dict) + return r + + +def pt_eval_lambda(fun): + return eval("lambda v,fun=fun:not fun(v)") + + +def is_true(value): + return value is True + + +def pt_eval_lambda_globals(origin_string): + globals_dict = {"square": lambda x: x * x} + r = eval(origin_string, globals=globals_dict) + return r + + +def pt_literal_eval(origin_string): + r = literal_eval(origin_string) + return r + + +def pt_exec(origin_string): + exec(origin_string) + return "OR: " + origin_string + + +def pt_exec_with_globals(origin_string): + my_var_in_pt_exec_with_globals = "abc" + exec(origin_string) + my_var_in_pt_exec_with_globals += "def" + return my_var_in_pt_exec_with_globals diff --git a/tests/appsec/iast/taint_sinks/test_code_injection.py b/tests/appsec/iast/taint_sinks/test_code_injection.py new file mode 100644 index 00000000000..a38600795ff --- /dev/null +++ b/tests/appsec/iast/taint_sinks/test_code_injection.py @@ -0,0 +1,167 @@ +import os + +import pytest + +from ddtrace.appsec._iast._taint_tracking import OriginType +from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject +from ddtrace.appsec._iast.constants import VULN_CODE_INJECTION +from tests.appsec.iast.aspects.conftest import _iast_patched_module +from tests.appsec.iast.taint_sinks.conftest import _get_iast_data +from tests.appsec.iast.taint_sinks.conftest import _get_span_report + + +ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) +mod = _iast_patched_module("tests.appsec.iast.fixtures.taint_sinks.code_injection") + + +def test_code_injection_eval(iast_context_defaults): + code_string = '"abc" + "def"' + + tainted_string = taint_pyobject( + code_string, source_name="path", source_value=code_string, source_origin=OriginType.PATH + ) + mod.pt_eval(tainted_string) + + data = _get_iast_data() + + assert len(data["vulnerabilities"]) == 1 + vulnerability = data["vulnerabilities"][0] + source = data["sources"][0] + assert vulnerability["type"] == VULN_CODE_INJECTION + assert source["name"] == "path" + assert source["origin"] == OriginType.PATH + assert source["value"] == '"abc" + "def"' + assert vulnerability["evidence"]["valueParts"] == [{"source": 0, "value": '"abc" + "def"'}] + assert "value" not in vulnerability["evidence"].keys() + assert vulnerability["evidence"].get("pattern") is None + assert vulnerability["evidence"].get("redacted") is None + + +def test_code_injection_eval_globals(iast_context_defaults): + """Validate globals and locals of the function""" + + code_string = "x + y" + + tainted_string = taint_pyobject( + code_string, source_name="path", source_value=code_string, source_origin=OriginType.PATH + ) + mod.pt_eval_globals(tainted_string) + + data = _get_iast_data() + + assert len(data["vulnerabilities"]) == 1 + vulnerability = data["vulnerabilities"][0] + source = data["sources"][0] + assert vulnerability["type"] == VULN_CODE_INJECTION + assert source["name"] == "path" + assert source["origin"] == OriginType.PATH + assert source["value"] == "x + y" + assert vulnerability["evidence"]["valueParts"] == [{"source": 0, "value": "x + y"}] + assert "value" not in vulnerability["evidence"].keys() + assert vulnerability["evidence"].get("pattern") is None + assert vulnerability["evidence"].get("redacted") is None + + +def test_code_injection_eval_globals_locals(iast_context_defaults): + """Validate globals and locals of the function""" + + code_string = "x + y" + + tainted_string = taint_pyobject( + code_string, source_name="path", source_value=code_string, source_origin=OriginType.PATH + ) + + mod.pt_eval_globals_locals(tainted_string) + + data = _get_iast_data() + + assert len(data["vulnerabilities"]) == 1 + vulnerability = data["vulnerabilities"][0] + source = data["sources"][0] + assert vulnerability["type"] == VULN_CODE_INJECTION + assert source["name"] == "path" + assert source["origin"] == OriginType.PATH + assert source["value"] == "x + y" + assert vulnerability["evidence"]["valueParts"] == [{"source": 0, "value": "x + y"}] + assert "value" not in vulnerability["evidence"].keys() + assert vulnerability["evidence"].get("pattern") is None + assert vulnerability["evidence"].get("redacted") is None + + +def test_code_injection_eval_globals_locals_override(iast_context_defaults): + """Validate globals and locals of the function""" + + code_string = "x + y + z" + + tainted_string = taint_pyobject( + code_string, source_name="path", source_value=code_string, source_origin=OriginType.PATH + ) + with pytest.raises(NameError): + mod.pt_eval_globals_locals(tainted_string) + + data = _get_iast_data() + + assert len(data["vulnerabilities"]) == 1 + vulnerability = data["vulnerabilities"][0] + source = data["sources"][0] + assert vulnerability["type"] == VULN_CODE_INJECTION + assert source["name"] == "path" + assert source["origin"] == OriginType.PATH + assert source["value"] == "x + y + z" + assert vulnerability["evidence"]["valueParts"] == [{"source": 0, "value": "x + y + z"}] + assert "value" not in vulnerability["evidence"].keys() + assert vulnerability["evidence"].get("pattern") is None + assert vulnerability["evidence"].get("redacted") is None + + +def test_code_injection_eval_lambda(iast_context_defaults): + """Validate globals and locals of the function""" + mod = _iast_patched_module("tests.appsec.iast.fixtures.taint_sinks.code_injection") + + def pt_eval_lambda_no_tainted(fun): + return eval("lambda v,fun=fun:not fun(v)") + + def is_true_no_tainted(value): + return value is True + + assert mod.pt_eval_lambda(mod.is_true)(True) is pt_eval_lambda_no_tainted(is_true_no_tainted)(True) + + +def test_code_injection_eval_globals_kwargs_lambda(iast_context_defaults): + """Validate globals and locals of the function""" + + code_string = "square(5)" + + tainted_string = taint_pyobject( + code_string, source_name="path", source_value=code_string, source_origin=OriginType.PATH + ) + + mod.pt_eval_lambda_globals(tainted_string) + + data = _get_iast_data() + + assert len(data["vulnerabilities"]) == 1 + vulnerability = data["vulnerabilities"][0] + source = data["sources"][0] + assert vulnerability["type"] == VULN_CODE_INJECTION + assert source["name"] == "path" + assert source["origin"] == OriginType.PATH + assert source["value"] == "square(5)" + assert vulnerability["evidence"]["valueParts"] == [{"source": 0, "value": "square(5)"}] + assert "value" not in vulnerability["evidence"].keys() + assert vulnerability["evidence"].get("pattern") is None + assert vulnerability["evidence"].get("redacted") is None + + +def test_code_injection_literal_eval(iast_context_defaults): + mod = _iast_patched_module("tests.appsec.iast.fixtures.taint_sinks.code_injection") + code_string = "[1, 2, 3]" + + tainted_string = taint_pyobject( + code_string, source_name="path", source_value=code_string, source_origin=OriginType.PATH + ) + mod.pt_literal_eval(tainted_string) + + data = _get_span_report() + + assert data is None diff --git a/tests/appsec/iast/taint_sinks/test_command_injection.py b/tests/appsec/iast/taint_sinks/test_command_injection.py index b716f594e85..ab611c1969b 100644 --- a/tests/appsec/iast/taint_sinks/test_command_injection.py +++ b/tests/appsec/iast/taint_sinks/test_command_injection.py @@ -123,7 +123,7 @@ def test_popen_wait_shell_true(iast_context_defaults): _assert_vulnerability("test_popen_wait_shell_true", source_name=source_name) -@pytest.mark.skipif(sys.platform != "linux", reason="Only for Linux") +@pytest.mark.skipif(sys.platform not in ["linux", "darwin"], reason="Only for Unix") @pytest.mark.parametrize( "function,mode,arguments,tag", [ @@ -156,11 +156,11 @@ def test_osspawn_variants(iast_context_defaults, function, mode, arguments, tag) if "spawnv" in cleaned_name: # label test_osspawn_variants2 - function(mode, copied_args[0], copied_args) + function(mode, copied_args[0], copied_args[1:]) label = "test_osspawn_variants2" else: # label test_osspawn_variants1 - function(mode, copied_args[0], *copied_args) + function(mode, copied_args[0], *copied_args[1:]) label = "test_osspawn_variants1" _assert_vulnerability( @@ -171,7 +171,7 @@ def test_osspawn_variants(iast_context_defaults, function, mode, arguments, tag) ) -@pytest.mark.skipif(sys.platform != "linux", reason="Only for Linux") +@pytest.mark.skipif(sys.platform not in ["linux", "darwin"], reason="Only for Unix") def test_multiple_cmdi(iast_context_defaults): _BAD_DIR = taint_pyobject( pyobject=_BAD_DIR_DEFAULT, @@ -193,7 +193,7 @@ def test_multiple_cmdi(iast_context_defaults): assert len(list(data["vulnerabilities"])) == 2 -@pytest.mark.skipif(sys.platform != "linux", reason="Only for Linux") +@pytest.mark.skipif(sys.platform not in ["linux", "darwin"], reason="Only for Unix") def test_string_cmdi(iast_context_defaults): cmd = taint_pyobject( pyobject="dir -l .", diff --git a/tests/appsec/iast/taint_sinks/test_ssrf.py b/tests/appsec/iast/taint_sinks/test_ssrf.py index f6f3ea0fb58..13d3443d930 100644 --- a/tests/appsec/iast/taint_sinks/test_ssrf.py +++ b/tests/appsec/iast/taint_sinks/test_ssrf.py @@ -2,16 +2,16 @@ from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect from ddtrace.appsec._iast.constants import VULN_SSRF -from ddtrace.contrib.httplib.patch import patch as httplib_patch -from ddtrace.contrib.httplib.patch import unpatch as httplib_unpatch -from ddtrace.contrib.requests.patch import patch as requests_patch -from ddtrace.contrib.requests.patch import unpatch as requests_unpatch -from ddtrace.contrib.urllib.patch import patch as urllib_patch -from ddtrace.contrib.urllib.patch import unpatch as urllib_unpatch -from ddtrace.contrib.urllib3.patch import patch as urllib3_patch -from ddtrace.contrib.urllib3.patch import unpatch as urllib3_unpatch -from ddtrace.contrib.webbrowser.patch import patch as webbrowser_patch -from ddtrace.contrib.webbrowser.patch import unpatch as webbrowser_unpatch +from ddtrace.contrib.internal.httplib.patch import patch as httplib_patch +from ddtrace.contrib.internal.httplib.patch import unpatch as httplib_unpatch +from ddtrace.contrib.internal.requests.patch import patch as requests_patch +from ddtrace.contrib.internal.requests.patch import unpatch as requests_unpatch +from ddtrace.contrib.internal.urllib.patch import patch as urllib_patch +from ddtrace.contrib.internal.urllib.patch import unpatch as urllib_unpatch +from ddtrace.contrib.internal.urllib3.patch import patch as urllib3_patch +from ddtrace.contrib.internal.urllib3.patch import unpatch as urllib3_unpatch +from ddtrace.contrib.internal.webbrowser.patch import patch as webbrowser_patch +from ddtrace.contrib.internal.webbrowser.patch import unpatch as webbrowser_unpatch from tests.appsec.iast.conftest import _end_iast_context_and_oce from tests.appsec.iast.conftest import _start_iast_context_and_oce from tests.appsec.iast.iast_utils import get_line_and_hash diff --git a/tests/appsec/iast/test_telemetry.py b/tests/appsec/iast/test_telemetry.py index dc07754bdc5..139dab79918 100644 --- a/tests/appsec/iast/test_telemetry.py +++ b/tests/appsec/iast/test_telemetry.py @@ -16,16 +16,19 @@ from ddtrace.appsec._iast._taint_tracking import origin_to_str from ddtrace.appsec._iast._taint_tracking._taint_objects import taint_pyobject from ddtrace.appsec._iast.constants import VULN_CMDI +from ddtrace.appsec._iast.constants import VULN_CODE_INJECTION from ddtrace.appsec._iast.constants import VULN_HEADER_INJECTION from ddtrace.appsec._iast.constants import VULN_PATH_TRAVERSAL from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION +from ddtrace.appsec._iast.taint_sinks.code_injection import patch as code_injection_patch +from ddtrace.appsec._iast.taint_sinks.code_injection import unpatch as code_injection_unpatch from ddtrace.appsec._iast.taint_sinks.command_injection import patch as cmdi_patch from ddtrace.appsec._iast.taint_sinks.header_injection import patch as header_injection_patch from ddtrace.appsec._iast.taint_sinks.header_injection import unpatch as header_injection_unpatch -from ddtrace.contrib.sqlalchemy import patch as sqli_sqlalchemy_patch -from ddtrace.contrib.sqlite3 import patch as sqli_sqlite3_patch +from ddtrace.contrib.internal.sqlalchemy.patch import patch as sqli_sqlalchemy_patch +from ddtrace.contrib.internal.sqlite3.patch import patch as sqli_sqlite3_patch from ddtrace.ext import SpanTypes -from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE_TAG_IAST +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE from ddtrace.internal.telemetry.constants import TELEMETRY_TYPE_GENERATE_METRICS from tests.appsec.iast.aspects.conftest import _iast_patched_module from tests.appsec.utils import asm_context @@ -35,7 +38,7 @@ def _assert_instrumented_sink(telemetry_writer, vuln_type): metrics_result = telemetry_writer._namespace._metrics_data - generate_metrics = metrics_result[TELEMETRY_TYPE_GENERATE_METRICS][TELEMETRY_NAMESPACE_TAG_IAST] + generate_metrics = metrics_result[TELEMETRY_TYPE_GENERATE_METRICS][TELEMETRY_NAMESPACE.IAST.value] assert len(generate_metrics) == 1, "Expected 1 generate_metrics" assert [metric.name for metric in generate_metrics.values()] == ["instrumented.sink"] assert [metric._tags for metric in generate_metrics.values()] == [(("vulnerability_type", vuln_type),)] @@ -84,7 +87,7 @@ def test_metric_executed_sink(no_request_sampling, telemetry_writer, caplog): metrics_result = telemetry_writer._namespace._metrics_data - generate_metrics = metrics_result[TELEMETRY_TYPE_GENERATE_METRICS][TELEMETRY_NAMESPACE_TAG_IAST].values() + generate_metrics = metrics_result[TELEMETRY_TYPE_GENERATE_METRICS][TELEMETRY_NAMESPACE.IAST.value].values() assert len(generate_metrics) == 1 # Remove potential sinks from internal usage of the lib (like http.client, used to communicate with # the agent) @@ -120,6 +123,15 @@ def test_metric_instrumented_header_injection(no_request_sampling, telemetry_wri _assert_instrumented_sink(telemetry_writer, VULN_HEADER_INJECTION) +def test_metric_instrumented_code_injection(no_request_sampling, telemetry_writer): + # We need to unpatch first because ddtrace.appsec._iast._patch_modules loads at runtime this patch function + code_injection_unpatch() + with override_global_config(dict(_iast_enabled=True, _iast_telemetry_report_lvl=TELEMETRY_INFORMATION_NAME)): + code_injection_patch() + + _assert_instrumented_sink(telemetry_writer, VULN_CODE_INJECTION) + + def test_metric_instrumented_sqli_sqlite3(no_request_sampling, telemetry_writer): with override_global_config(dict(_iast_enabled=True, _iast_telemetry_report_lvl=TELEMETRY_INFORMATION_NAME)): sqli_sqlite3_patch() @@ -139,7 +151,7 @@ def test_metric_instrumented_propagation(no_request_sampling, telemetry_writer): _iast_patched_module("benchmarks.bm.iast_fixtures.str_methods") metrics_result = telemetry_writer._namespace._metrics_data - generate_metrics = metrics_result[TELEMETRY_TYPE_GENERATE_METRICS][TELEMETRY_NAMESPACE_TAG_IAST] + generate_metrics = metrics_result[TELEMETRY_TYPE_GENERATE_METRICS][TELEMETRY_NAMESPACE.IAST.value] # Remove potential sinks from internal usage of the lib (like http.client, used to communicate with # the agent) filtered_metrics = [metric.name for metric in generate_metrics.values() if metric.name != "executed.sink"] @@ -163,7 +175,7 @@ def test_metric_request_tainted(no_request_sampling, telemetry_writer): metrics_result = telemetry_writer._namespace._metrics_data - generate_metrics = metrics_result[TELEMETRY_TYPE_GENERATE_METRICS][TELEMETRY_NAMESPACE_TAG_IAST] + generate_metrics = metrics_result[TELEMETRY_TYPE_GENERATE_METRICS][TELEMETRY_NAMESPACE.IAST.value] # Remove potential sinks from internal usage of the lib (like http.client, used to communicate with # the agent) filtered_metrics = [metric.name for metric in generate_metrics.values() if metric.name != "executed.sink"] diff --git a/tests/appsec/iast_packages/packages/pkg_pyjwt.py b/tests/appsec/iast_packages/packages/pkg_pyjwt.py index 4712f6cee0f..ec43d8a17d2 100644 --- a/tests/appsec/iast_packages/packages/pkg_pyjwt.py +++ b/tests/appsec/iast_packages/packages/pkg_pyjwt.py @@ -3,6 +3,7 @@ https://pypi.org/project/PyJWT/ """ + import datetime from flask import Blueprint @@ -25,7 +26,10 @@ def pkg_pyjwt_view(): secret_key = "your-256-bit-secret" user_payload = request.args.get("package_param", "default-user") - payload = {"user": user_payload, "exp": datetime.datetime.utcnow() + datetime.timedelta(seconds=30)} + payload = { + "user": user_payload, + "exp": datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(seconds=30), + } try: # Encode the payload to create a JWT diff --git a/tests/appsec/iast_packages/test_packages.py b/tests/appsec/iast_packages/test_packages.py index 86aad989007..83e53ae92c9 100644 --- a/tests/appsec/iast_packages/test_packages.py +++ b/tests/appsec/iast_packages/test_packages.py @@ -216,24 +216,29 @@ def uninstall(self, python_cmd): import_module_to_validate="boto3.session", ), PackageForTesting("botocore", "1.34.110", "", "", "", test_e2e=False), - PackageForTesting( - "cffi", "1.16.0", "", 30, "", import_module_to_validate="cffi.model", extras=[("setuptools", "72.1.0")] - ), - PackageForTesting( - "certifi", "2024.2.2", "", "The path to the CA bundle is", "", import_module_to_validate="certifi.core" - ), - PackageForTesting( - "charset-normalizer", - "3.3.2", - "my-bytes-string", - "my-bytes-string", - "", - import_name="charset_normalizer", - import_module_to_validate="charset_normalizer.api", - test_propagation=True, - fixme_propagation_fails=True, - ), - PackageForTesting("click", "8.1.7", "", "Hello World!\nHello World!\n", "", import_module_to_validate="click.core"), + ## Skip due to cffi added to the denylist + # PackageForTesting( + # "cffi", "1.16.0", "", 30, "", import_module_to_validate="cffi.model", extras=[("setuptools", "72.1.0")] + # ), + ## Skip due to certifi added to the denylist + # PackageForTesting( + # "certifi", "2024.2.2", "", "The path to the CA bundle is", "", import_module_to_validate="certifi.core" + # ), + ## Skip due to charset-normalizer added to the denylist + # PackageForTesting( + # "charset-normalizer", + # "3.3.2", + # "my-bytes-string", + # "my-bytes-string", + # "", + # import_name="charset_normalizer", + # import_module_to_validate="charset_normalizer.api", + # test_propagation=True, + # fixme_propagation_fails=True, + # ), + ## Skip due to click added to the denylist + # PackageForTesting("click", "8.1.7", "", "Hello World!\nHello World!\n", "", + # import_module_to_validate="click.core"), PackageForTesting( "cryptography", "42.0.7", @@ -247,15 +252,16 @@ def uninstall(self, python_cmd): PackageForTesting( "distlib", "0.3.8", "", "Name: example-package\nVersion: 0.1", "", import_module_to_validate="distlib.util" ), - PackageForTesting( - "exceptiongroup", - "1.2.1", - "foobar", - "ValueError: First error with foobar\nTypeError: Second error with foobar", - "", - import_module_to_validate="exceptiongroup._formatting", - test_propagation=True, - ), + ## Skip due to docopt added to the denylist + # PackageForTesting( + # "exceptiongroup", + # "1.2.1", + # "foobar", + # "ValueError: First error with foobar\nTypeError: Second error with foobar", + # "", + # import_module_to_validate="exceptiongroup._formatting", + # test_propagation=True, + # ), PackageForTesting( "filelock", "3.14.0", @@ -327,14 +333,15 @@ def uninstall(self, python_cmd): "", import_module_to_validate="isodate.duration", ), - PackageForTesting( - "itsdangerous", - "2.2.0", - "foobar", - "Signed value: foobar.generated_signature\nUnsigned value: foobar", - "", - import_module_to_validate="itsdangerous.serializer", - ), + ## Skip due to itsdangerous added to the denylist + # PackageForTesting( + # "itsdangerous", + # "2.2.0", + # "foobar", + # "Signed value: foobar.generated_signature\nUnsigned value: foobar", + # "", + # import_module_to_validate="itsdangerous.serializer", + # ), PackageForTesting( "jinja2", "3.1.4", @@ -402,16 +409,17 @@ def uninstall(self, python_cmd): import_module_to_validate="multidict._multidict_py", test_propagation=True, ), + ## Skip due to numpy added to the denylist # Python 3.12 fails in all steps with "import error" when import numpy - PackageForTesting( - "numpy", - "1.24.4", - "9 8 7 6 5 4 3", - [3, 4, 5, 6, 7, 8, 9], - 5, - skip_python_version=[(3, 12)], - import_module_to_validate="numpy.core._internal", - ), + # PackageForTesting( + # "numpy", + # "1.24.4", + # "9 8 7 6 5 4 3", + # [3, 4, 5, 6, 7, 8, 9], + # 5, + # skip_python_version=[(3, 12)], + # import_module_to_validate="numpy.core._internal", + # ), PackageForTesting( "oauthlib", "3.2.2", @@ -423,15 +431,18 @@ def uninstall(self, python_cmd): PackageForTesting( "openpyxl", "3.1.2", "foobar", "Written value: foobar", "", import_module_to_validate="openpyxl.chart.axis" ), - PackageForTesting( - "packaging", - "24.0", - "", - {"is_version_valid": True, "requirement": "example-package>=1.0.0", "specifier": ">=1.0.0", "version": "1.2.3"}, - "", - ), + ## Skip due to packaging added to the denylist + # PackageForTesting( + # "packaging", + # "24.0", + # "", + # {"is_version_valid": True, "requirement": "example-package>=1.0.0", + # "specifier": ">=1.0.0", "version": "1.2.3"}, + # "", + # ), + ## Skip due to pandas added to the denylist # Pandas dropped Python 3.8 support in pandas>2.0.3 - PackageForTesting("pandas", "2.2.2", "foobar", "Written value: foobar", "", skip_python_version=[(3, 8)]), + # PackageForTesting("pandas", "2.2.2", "foobar", "Written value: foobar", "", skip_python_version=[(3, 8)]), PackageForTesting( "platformdirs", "4.2.2", @@ -441,14 +452,15 @@ def uninstall(self, python_cmd): import_module_to_validate="platformdirs.unix", test_propagation=True, ), - PackageForTesting( - "pluggy", - "1.5.0", - "foobar", - "Hook result: Plugin received: foobar", - "", - import_module_to_validate="pluggy._hooks", - ), + ## Skip due to pluggy added to the denylist + # PackageForTesting( + # "pluggy", + # "1.5.0", + # "foobar", + # "Hook result: Plugin received: foobar", + # "", + # import_module_to_validate="pluggy._hooks", + # ), PackageForTesting( "pyasn1", "0.6.0", @@ -459,7 +471,8 @@ def uninstall(self, python_cmd): test_propagation=True, fixme_propagation_fails=True, ), - PackageForTesting("pycparser", "2.22", "", "", ""), + ## Skip due to pygments added to the denylist + # PackageForTesting("pycparser", "2.22", "", "", ""), PackageForTesting( "pydantic", "2.7.1", @@ -477,17 +490,16 @@ def uninstall(self, python_cmd): import_name="pydantic_core", import_module_to_validate="pydantic_core.core_schema", ), - # # TODO: patching Pytest fails: ImportError: cannot import name 'Dir' from '_pytest.main' - # PackageForTesting("pytest", "8.2.1", "", "", "", test_e2e=False), - PackageForTesting( - "python-dateutil", - "2.8.2", - "Sat Oct 11 17:13:46 UTC 2003", - "Sat, 11 Oct 2003 17:13:46 GMT", - "And the Easter of that year is: 2004-04-11", - import_name="dateutil", - import_module_to_validate="dateutil.relativedelta", - ), + ## Skip due to python-dateutil added to the denylist + # PackageForTesting( + # "python-dateutil", + # "2.8.2", + # "Sat Oct 11 17:13:46 UTC 2003", + # "Sat, 11 Oct 2003 17:13:46 GMT", + # "And the Easter of that year is: 2004-04-11", + # import_name="dateutil", + # import_module_to_validate="dateutil.relativedelta", + # ), PackageForTesting( "python-multipart", "0.0.5", # this version validates APPSEC-55240 issue, don't upgrade it @@ -499,13 +511,14 @@ def uninstall(self, python_cmd): test_import=False, test_propagation=True, ), - PackageForTesting( - "pytz", - "2024.1", - "America/New_York", - "Current time in America/New_York: replaced_time", - "", - ), + ## Skip due to pytz added to the denylist + # PackageForTesting( + # "pytz", + # "2024.1", + # "America/New_York", + # "Current time in America/New_York: replaced_time", + # "", + # ), PackageForTesting( "PyYAML", "6.0.1", @@ -584,7 +597,8 @@ def uninstall(self, python_cmd): "", import_module_to_validate="tomlkit.items", ), - PackageForTesting("tqdm", "4.66.4", "", "", "", test_e2e=False, import_module_to_validate="tqdm.std"), + ## Skip due to tqdm added to the denylist + # PackageForTesting("tqdm", "4.66.4", "", "", "", test_e2e=False, import_module_to_validate="tqdm.std"), # Python 3.8 and 3.9 fail with ImportError: cannot import name 'get_host' from 'urllib3.util.url' PackageForTesting( "urllib3", @@ -616,15 +630,16 @@ def uninstall(self, python_cmd): test_propagation=True, fixme_propagation_fails=True, ), - PackageForTesting( - "werkzeug", - "3.0.3", - "your-password", - "Original password: your-password\nHashed password: replaced_hashed\nPassword match: True", - "", - import_module_to_validate="werkzeug.http", - skip_python_version=[(3, 6), (3, 7), (3, 8)], - ), + ## Skip due to werkzeug added to the denylist + # PackageForTesting( + # "werkzeug", + # "3.0.3", + # "your-password", + # "Original password: your-password\nHashed password: replaced_hashed\nPassword match: True", + # "", + # import_module_to_validate="werkzeug.http", + # skip_python_version=[(3, 6), (3, 7), (3, 8)], + # ), PackageForTesting( "yarl", "1.9.4", @@ -637,24 +652,26 @@ def uninstall(self, python_cmd): test_propagation=True, fixme_propagation_fails=True, ), - PackageForTesting( - "zipp", - "3.18.2", - "example.zip", - "Contents of example.zip: ['example.zip/example.txt']", - "", - skip_python_version=[(3, 6), (3, 7), (3, 8)], - ), - PackageForTesting( - "typing-extensions", - "4.11.0", - "", - "", - "", - import_name="typing_extensions", - test_e2e=False, - skip_python_version=[(3, 6), (3, 7), (3, 8)], - ), + ## Skip due to zipp added to the denylist + # PackageForTesting( + # "zipp", + # "3.18.2", + # "example.zip", + # "Contents of example.zip: ['example.zip/example.txt']", + # "", + # skip_python_version=[(3, 6), (3, 7), (3, 8)], + # ), + ## Skip due to typing-extensions added to the denylist + # PackageForTesting( + # "typing-extensions", + # "4.11.0", + # "", + # "", + # "", + # import_name="typing_extensions", + # test_e2e=False, + # skip_python_version=[(3, 6), (3, 7), (3, 8)], + # ), PackageForTesting( "six", "1.16.0", @@ -663,15 +680,16 @@ def uninstall(self, python_cmd): "", skip_python_version=[(3, 6), (3, 7), (3, 8)], ), - PackageForTesting( - "pillow", - "10.3.0", - "Hello, Pillow!", - "Image correctly generated", - "", - import_name="PIL.Image", - skip_python_version=[(3, 6), (3, 7), (3, 8)], - ), + ## Skip due to pillow added to the denylist + # PackageForTesting( + # "pillow", + # "10.3.0", + # "Hello, Pillow!", + # "Image correctly generated", + # "", + # import_name="PIL.Image", + # skip_python_version=[(3, 6), (3, 7), (3, 8)], + # ), PackageForTesting( "aiobotocore", "2.13.0", "", "", "", test_e2e=False, test_import=False, import_name="aiobotocore.session" ), @@ -683,14 +701,15 @@ def uninstall(self, python_cmd): "", import_name="jwt", ), - PackageForTesting( - "wrapt", - "1.16.0", - "some-value", - "Function executed with param: some-value", - "", - test_propagation=True, - ), + ## Skip due to pyarrow added to the denylist + # PackageForTesting( + # "wrapt", + # "1.16.0", + # "some-value", + # "Function executed with param: some-value", + # "", + # test_propagation=True, + # ), PackageForTesting( "cachetools", "5.3.3", @@ -745,16 +764,17 @@ def uninstall(self, python_cmd): "", test_e2e=False, ), - # scipy dropped Python 3.8 support in scipy > 1.10.1 - PackageForTesting( - "scipy", - "1.13.0", - "1,2,3,4,5", - "Mean: 3.0, Standard Deviation: 1.581", - "", - import_name="scipy.special", - skip_python_version=[(3, 8)], - ), + ## Skip due to scipy added to the denylist + # # scipy dropped Python 3.8 support in scipy > 1.10.1 + # PackageForTesting( + # "scipy", + # "1.13.0", + # "1,2,3,4,5", + # "Mean: 3.0, Standard Deviation: 1.581", + # "", + # import_name="scipy.special", + # skip_python_version=[(3, 8)], + # ), PackageForTesting( "iniconfig", "2.0.0", @@ -799,16 +819,17 @@ def uninstall(self, python_cmd): "", import_name="OpenSSL.SSL", ), - PackageForTesting( - "moto[s3]", - "5.0.11", - "some_bucket", - "right_result", - "", - import_name="moto.s3.models", - test_e2e=True, - extras=[("boto3", "1.34.143")], - ), + ## Skip due to pyarrow added to the denylist + # PackageForTesting( + # "moto[s3]", + # "5.0.11", + # "some_bucket", + # "right_result", + # "", + # import_name="moto.s3.models", + # test_e2e=True, + # extras=[("boto3", "1.34.143")], + # ), PackageForTesting("decorator", "5.1.1", "World", "Decorated result: Hello, World!", ""), # TODO: e2e implemented but fails unpatched: "RateLimiter object has no attribute _is_allowed" PackageForTesting( diff --git a/tests/appsec/integrations/django_tests/__init__.py b/tests/appsec/integrations/django_tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/appsec/integrations/django_tests/conftest.py b/tests/appsec/integrations/django_tests/conftest.py new file mode 100644 index 00000000000..76ffa4a3763 --- /dev/null +++ b/tests/appsec/integrations/django_tests/conftest.py @@ -0,0 +1,66 @@ +import os + +import django +from django.conf import settings +import pytest + +from ddtrace import Pin +from ddtrace.appsec._iast import enable_iast_propagation +from ddtrace.contrib.internal.django.patch import patch +from tests.appsec.iast.conftest import _end_iast_context_and_oce +from tests.appsec.iast.conftest import _start_iast_context_and_oce +from tests.utils import DummyTracer +from tests.utils import TracerSpanContainer +from tests.utils import override_global_config + + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tests.appsec.integrations.django_tests.django_app.settings") + + +# `pytest` automatically calls this function once when tests are run. +def pytest_configure(): + with override_global_config( + dict( + _iast_enabled=True, + _deduplication_enabled=False, + _iast_request_sampling=100.0, + ) + ): + settings.DEBUG = False + enable_iast_propagation() + patch() + django.setup() + + +@pytest.fixture +def tracer(): + tracer = DummyTracer() + # Patch Django and override tracer to be our test tracer + pin = Pin.get_from(django) + original_tracer = pin.tracer + Pin.override(django, tracer=tracer) + + # Yield to our test + yield tracer + tracer.pop() + + # Reset the tracer pinned to Django and unpatch + # DEV: unable to properly unpatch and reload django app with each test + # unpatch() + Pin.override(django, tracer=original_tracer) + + +@pytest.fixture +def test_spans(tracer): + with override_global_config( + dict( + _iast_enabled=True, + _deduplication_enabled=False, + _iast_request_sampling=100.0, + ) + ): + container = TracerSpanContainer(tracer) + _start_iast_context_and_oce() + yield container + _end_iast_context_and_oce() + container.reset() diff --git a/tests/appsec/integrations/django_tests/django_app/__init__.py b/tests/appsec/integrations/django_tests/django_app/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/appsec/integrations/django_tests/django_app/settings.py b/tests/appsec/integrations/django_tests/django_app/settings.py new file mode 100644 index 00000000000..9883d69f5ba --- /dev/null +++ b/tests/appsec/integrations/django_tests/django_app/settings.py @@ -0,0 +1,77 @@ +import os + +from ddtrace import tracer +from tests.webclient import PingFilter + + +tracer.configure( + settings={ + "FILTERS": [PingFilter()], + } +) + + +ALLOWED_HOSTS = [ + "testserver", + "localhost", +] + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +DATABASES = { + "default": {"ENGINE": "django.db.backends.sqlite3", "NAME": ":memory:"}, +} + + +CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + "LOCATION": "unique-snowflake", + }, + "pylibmc": { + "BACKEND": "django.core.cache.backends.memcached.PyLibMCCache", + "LOCATION": "127.0.0.1:11211", + }, +} + +SITE_ID = 1 +SECRET_KEY = "not_very_secret_in_tests" +USE_I18N = True +USE_L10N = True +STATIC_URL = "/static/" +ROOT_URLCONF = "tests.appsec.integrations.django_tests.django_app.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [ + os.path.join(BASE_DIR, "templates"), + ], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +MIDDLEWARE = [ + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "django.middleware.security.SecurityMiddleware", +] + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", +] diff --git a/tests/appsec/integrations/django_tests/django_app/urls.py b/tests/appsec/integrations/django_tests/django_app/urls.py new file mode 100644 index 00000000000..be2d142baa2 --- /dev/null +++ b/tests/appsec/integrations/django_tests/django_app/urls.py @@ -0,0 +1,84 @@ +import django +from django.http import HttpResponse +from django.urls import path + +from ddtrace import tracer +from tests.appsec.integrations.django_tests.django_app import views + + +# django.conf.urls.url was deprecated in django 3 and removed in django 4 +if django.VERSION < (4, 0, 0): + from django.conf.urls import url as handler +else: + from django.urls import re_path as handler + + +def shutdown(request): + # Endpoint used to flush traces to the agent when doing snapshots. + tracer.shutdown() + return HttpResponse(status=200) + + +urlpatterns = [ + handler(r"^$", views.index), + # This must precede composed-view. + handler("appsec/response-header/$", views.magic_header_key, name="response-header"), + handler("appsec/body/$", views.body_view, name="body_view"), + handler("appsec/view_with_exception/$", views.view_with_exception, name="view_with_exception"), + handler("appsec/weak-hash/$", views.weak_hash_view, name="weak_hash"), + handler("appsec/block/$", views.block_callable_view, name="block"), + handler("appsec/command-injection/$", views.command_injection, name="command_injection"), + handler("appsec/header-injection/$", views.header_injection, name="header_injection"), + handler("appsec/taint-checking-enabled/$", views.taint_checking_enabled_view, name="taint_checking_enabled_view"), + handler( + "appsec/taint-checking-disabled/$", views.taint_checking_disabled_view, name="taint_checking_disabled_view" + ), + handler( + "appsec/sqli_http_request_parameter/$", views.sqli_http_request_parameter, name="sqli_http_request_parameter" + ), + handler( + "appsec/sqli_http_request_parameter_name_get/$", + views.sqli_http_request_parameter_name_get, + name="sqli_http_request_parameter_name_get", + ), + handler( + "appsec/sqli_http_request_parameter_name_post/$", + views.sqli_http_request_parameter_name_post, + name="sqli_http_request_parameter_name_post", + ), + handler( + "appsec/sqli_http_request_header_name/$", + views.sqli_http_request_header_name, + name="sqli_http_request_header_name", + ), + handler( + "appsec/sqli_http_request_header_value/$", + views.sqli_http_request_header_value, + name="sqli_http_request_header_value", + ), + handler( + "appsec/sqli_http_request_cookie_name/$", + views.sqli_http_request_cookie_name, + name="sqli_http_request_cookie_name", + ), + handler( + "appsec/sqli_http_request_cookie_value/$", + views.sqli_http_request_cookie_value, + name="sqli_http_request_cookie_value", + ), + handler("appsec/sqli_http_request_body/$", views.sqli_http_request_body, name="sqli_http_request_body"), + handler("appsec/source/body/$", views.source_body_view, name="source_body"), + handler("appsec/insecure-cookie/test_insecure_2_1/$", views.view_insecure_cookies_two_insecure_one_secure), + handler("appsec/insecure-cookie/test_insecure_special/$", views.view_insecure_cookies_insecure_special_chars), + handler("appsec/insecure-cookie/test_insecure/$", views.view_insecure_cookies_insecure), + handler("appsec/insecure-cookie/test_secure/$", views.view_insecure_cookies_secure), + handler("appsec/insecure-cookie/test_empty_cookie/$", views.view_insecure_cookies_empty), + path( + "appsec/sqli_http_path_parameter//", + views.sqli_http_path_parameter, + name="sqli_http_path_parameter", + ), + handler("appsec/validate_querydict/$", views.validate_querydict, name="validate_querydict"), + path("appsec/path-params///", views.path_params_view, name="path-params-view"), + path("appsec/checkuser//", views.checkuser_view, name="checkuser"), +] diff --git a/tests/contrib/django/django_app/appsec_urls.py b/tests/appsec/integrations/django_tests/django_app/views.py similarity index 54% rename from tests/contrib/django/django_app/appsec_urls.py rename to tests/appsec/integrations/django_tests/django_app/views.py index f5b3f359445..ef4fd78b138 100644 --- a/tests/contrib/django/django_app/appsec_urls.py +++ b/tests/appsec/integrations/django_tests/django_app/views.py @@ -1,39 +1,32 @@ +""" +Class based views used for Django tests. +""" import hashlib import os -from typing import TYPE_CHECKING # noqa:F401 +from typing import Any -import django from django.db import connection from django.http import HttpResponse from django.http import JsonResponse from ddtrace import tracer from ddtrace.appsec import _asm_request_context -from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect -from ddtrace.appsec._iast._taint_tracking.aspects import decode_aspect -from ddtrace.appsec._iast._utils import _is_python_version_supported as python_supported_by_iast +from ddtrace.appsec._iast._taint_tracking import OriginType +from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted +from ddtrace.appsec._iast.reporter import IastSpanReporter from ddtrace.appsec._trace_utils import block_request_if_user_blocked -from tests.utils import override_env -# django.conf.urls.url was deprecated in django 3 and removed in django 4 -if django.VERSION < (4, 0, 0): - from django.conf.urls import url as handler -else: - from django.urls import re_path as handler +def assert_origin(parameter: Any, origin_type: Any) -> None: + assert is_pyobject_tainted(parameter) + sources, _ = IastSpanReporter.taint_ranges_as_evidence_info(parameter) + assert sources[0].origin == origin_type -if django.VERSION >= (2, 0, 0): - from django.urls import path -else: - from django.conf.urls import url as path - -if TYPE_CHECKING: - from typing import Any # noqa:F401 - - -def include_view(request): - return HttpResponse(status=200) +def index(request): + response = HttpResponse("Hello, test app.") + response["my-response-header"] = "my_response_value" + return response def path_params_view(request, year, month): @@ -52,6 +45,8 @@ def body_view(request): return HttpResponse(data, status=200) else: data = request.POST + first_post_key = list(request.POST.keys())[0] + assert_origin(first_post_key, OriginType.PARAMETER_NAME) return HttpResponse(str(dict(data)), status=200) @@ -81,7 +76,25 @@ def sqli_http_request_parameter(request): obj = password_django.encode("i'm a password", bcrypt.gensalt()) with connection.cursor() as cursor: # label iast_enabled_sqli_http_request_parameter - cursor.execute(add_aspect(add_aspect(request.GET["q"], obj), "'")) + cursor.execute(request.GET["q"] + obj + "'") + + return HttpResponse(request.META["HTTP_USER_AGENT"], status=200) + + +def sqli_http_request_parameter_name_get(request): + obj = " 1" + with connection.cursor() as cursor: + # label iast_enabled_sqli_http_request_parameter_name_get + cursor.execute(list(request.GET.keys())[0] + obj) + + return HttpResponse(request.META["HTTP_USER_AGENT"], status=200) + + +def sqli_http_request_parameter_name_post(request): + obj = " 1" + with connection.cursor() as cursor: + # label iast_enabled_sqli_http_request_parameter_name_post + cursor.execute(list(request.POST.keys())[0] + obj) return HttpResponse(request.META["HTTP_USER_AGENT"], status=200) @@ -91,7 +104,7 @@ def sqli_http_request_header_name(request): with connection.cursor() as cursor: # label iast_enabled_sqli_http_request_header_name - cursor.execute(add_aspect("SELECT 1 FROM sqlite_", key)) + cursor.execute("SELECT 1 FROM sqlite_" + key) return HttpResponse(request.META["master"], status=200) @@ -99,7 +112,7 @@ def sqli_http_request_header_name(request): def sqli_http_request_header_value(request): value = [x for x in request.META.values() if x == "master"][0] with connection.cursor() as cursor: - query = add_aspect("SELECT 1 FROM sqlite_", value) + query = "SELECT 1 FROM sqlite_" + value # label iast_enabled_sqli_http_request_header_value cursor.execute(query) @@ -107,11 +120,8 @@ def sqli_http_request_header_value(request): def sqli_http_path_parameter(request, q_http_path_parameter): - with override_env({"DD_IAST_ENABLED": "True"}): - from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect - with connection.cursor() as cursor: - query = add_aspect("SELECT 1 from ", q_http_path_parameter) + query = "SELECT 1 from " + q_http_path_parameter # label iast_enabled_full_sqli_http_path_parameter cursor.execute(query) @@ -119,49 +129,28 @@ def sqli_http_path_parameter(request, q_http_path_parameter): def taint_checking_enabled_view(request): - if python_supported_by_iast(): - with override_env({"DD_IAST_ENABLED": "True"}): - from ddtrace.appsec._iast._taint_tracking import OriginType - from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted - from ddtrace.appsec._iast.reporter import IastSpanReporter - - def assert_origin_path(path): # type: (Any) -> None - assert is_pyobject_tainted(path) - sources, tainted_ranges_to_dict = IastSpanReporter.taint_ranges_as_evidence_info(path) - assert sources[0].origin == OriginType.PATH - - else: - - def assert_origin_path(pyobject): # type: (Any) -> bool - return True - - def is_pyobject_tainted(pyobject): # type: (Any) -> bool - return True - # TODO: Taint request body # assert is_pyobject_tainted(request.body) + first_get_key = list(request.GET.keys())[0] assert is_pyobject_tainted(request.GET["q"]) + assert is_pyobject_tainted(first_get_key) assert is_pyobject_tainted(request.META["QUERY_STRING"]) assert is_pyobject_tainted(request.META["HTTP_USER_AGENT"]) # TODO: Taint request headers # assert is_pyobject_tainted(request.headers["User-Agent"]) - assert_origin_path(request.path_info) - assert_origin_path(request.path) - assert_origin_path(request.META["PATH_INFO"]) + assert_origin(request.path_info, OriginType.PATH) + assert_origin(request.path, OriginType.PATH) + assert_origin(request.META["PATH_INFO"], OriginType.PATH) + assert_origin(request.GET["q"], OriginType.PARAMETER) + assert_origin(first_get_key, OriginType.PARAMETER_NAME) + return HttpResponse(request.META["HTTP_USER_AGENT"], status=200) def taint_checking_disabled_view(request): - if python_supported_by_iast(): - with override_env({"DD_IAST_ENABLED": "True"}): - from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted - else: - - def is_pyobject_tainted(pyobject): # type: (Any) -> bool - return False - assert not is_pyobject_tainted(request.body) assert not is_pyobject_tainted(request.GET["q"]) + assert not is_pyobject_tainted(list(request.GET.keys())[0]) assert not is_pyobject_tainted(request.META["QUERY_STRING"]) assert not is_pyobject_tainted(request.META["HTTP_USER_AGENT"]) assert not is_pyobject_tainted(request.headers["User-Agent"]) @@ -181,7 +170,7 @@ def sqli_http_request_cookie_name(request): with connection.cursor() as cursor: # label iast_enabled_sqli_http_cookies_name - cursor.execute(add_aspect("SELECT 1 FROM sqlite_", key)) + cursor.execute("SELECT 1 FROM sqlite_" + key) return HttpResponse(request.COOKIES["master"], status=200) @@ -191,7 +180,7 @@ def sqli_http_request_cookie_value(request): with connection.cursor() as cursor: # label iast_enabled_sqli_http_cookies_value - cursor.execute(add_aspect("SELECT 1 FROM sqlite_", value)) + cursor.execute("SELECT 1 FROM sqlite_" + value) return HttpResponse(request.COOKIES["master"], status=200) @@ -201,19 +190,19 @@ def sqli_http_request_body(request): if key in request.POST: value = request.POST[key] else: - value = decode_aspect(bytes.decode, 1, request.body) + value = request.body.decode() with connection.cursor() as cursor: # label iast_enabled_sqli_http_body - cursor.execute(add_aspect("SELECT 1 FROM sqlite_", value)) + cursor.execute("SELECT 1 FROM sqlite_" + value) return HttpResponse(value, status=200) def source_body_view(request): - value = decode_aspect(bytes.decode, 1, request.body) + value = request.body.decode() with connection.cursor() as cursor: # label source_body_view - cursor.execute(add_aspect("SELECT 1 FROM sqlite_master WHERE type='1'", value)) + cursor.execute("SELECT 1 FROM sqlite_master WHERE type='1'" + value) return HttpResponse(value, status=200) @@ -260,15 +249,15 @@ def view_insecure_cookies_insecure_special_chars(request): def command_injection(request): - value = decode_aspect(bytes.decode, 1, request.body) + value = request.body.decode() # label iast_command_injection - os.system(add_aspect("dir -l ", value)) + os.system("dir -l " + value) return HttpResponse("OK", status=200) def header_injection(request): - value = decode_aspect(bytes.decode, 1, request.body) + value = request.body.decode() response = HttpResponse("OK", status=200) # label iast_header_injection @@ -284,45 +273,3 @@ def validate_querydict(request): return HttpResponse( "x=%s, all=%s, keys=%s, urlencode=%s" % (str(res), str(lres), str(keys), qd.urlencode()), status=200 ) - - -urlpatterns = [ - handler("response-header/$", magic_header_key, name="response-header"), - handler("body/$", body_view, name="body_view"), - handler("view_with_exception/$", view_with_exception, name="view_with_exception"), - handler("weak-hash/$", weak_hash_view, name="weak_hash"), - handler("block/$", block_callable_view, name="block"), - handler("command-injection/$", command_injection, name="command_injection"), - handler("header-injection/$", header_injection, name="header_injection"), - handler("taint-checking-enabled/$", taint_checking_enabled_view, name="taint_checking_enabled_view"), - handler("taint-checking-disabled/$", taint_checking_disabled_view, name="taint_checking_disabled_view"), - handler("sqli_http_request_parameter/$", sqli_http_request_parameter, name="sqli_http_request_parameter"), - handler("sqli_http_request_header_name/$", sqli_http_request_header_name, name="sqli_http_request_header_name"), - handler("sqli_http_request_header_value/$", sqli_http_request_header_value, name="sqli_http_request_header_value"), - handler("sqli_http_request_cookie_name/$", sqli_http_request_cookie_name, name="sqli_http_request_cookie_name"), - handler("sqli_http_request_cookie_value/$", sqli_http_request_cookie_value, name="sqli_http_request_cookie_value"), - handler("sqli_http_request_body/$", sqli_http_request_body, name="sqli_http_request_body"), - handler("source/body/$", source_body_view, name="source_body"), - handler("insecure-cookie/test_insecure_2_1/$", view_insecure_cookies_two_insecure_one_secure), - handler("insecure-cookie/test_insecure_special/$", view_insecure_cookies_insecure_special_chars), - handler("insecure-cookie/test_insecure/$", view_insecure_cookies_insecure), - handler("insecure-cookie/test_secure/$", view_insecure_cookies_secure), - handler("insecure-cookie/test_empty_cookie/$", view_insecure_cookies_empty), - path( - "sqli_http_path_parameter//", - sqli_http_path_parameter, - name="sqli_http_path_parameter", - ), - handler("validate_querydict/$", validate_querydict, name="validate_querydict"), -] - -if django.VERSION >= (2, 0, 0): - urlpatterns += [ - path("path-params///", path_params_view, name="path-params-view"), - path("checkuser//", checkuser_view, name="checkuser"), - ] -else: - urlpatterns += [ - path(r"path-params/(?P[0-9]{4})/(?P\w+)/$", path_params_view, name="path-params-view"), - path(r"checkuser/(?P\w+)/$", checkuser_view, name="checkuser"), - ] diff --git a/tests/contrib/django/test_django_appsec.py b/tests/appsec/integrations/django_tests/test_django_appsec.py similarity index 98% rename from tests/contrib/django/test_django_appsec.py rename to tests/appsec/integrations/django_tests/test_django_appsec.py index 3c5cb399739..2a00657e14a 100644 --- a/tests/contrib/django/test_django_appsec.py +++ b/tests/appsec/integrations/django_tests/test_django_appsec.py @@ -235,7 +235,9 @@ def test_django_login_sucess_anonymization(client, test_spans, tracer, use_login assert login_span.get_tag(user.ID) == "1" assert login_span.get_tag("appsec.events.users.login.success.track") == "true" assert login_span.get_tag(APPSEC.AUTO_LOGIN_EVENTS_SUCCESS_MODE) == LOGIN_EVENTS_MODE.ANON - assert login_span.get_tag(APPSEC.USER_LOGIN_EVENT_PREFIX + ".success.login") is None + assert login_span.get_tag(APPSEC.USER_LOGIN_EVENT_PREFIX + ".success.login") == ( + "anon_d1ad1f735a4381c2e8dbed0222db1136" if use_login else None + ) assert login_span.get_tag(APPSEC.USER_LOGIN_EVENT_PREFIX + ".success.email") is None assert login_span.get_tag(APPSEC.USER_LOGIN_EVENT_PREFIX + ".success.username") is None @@ -368,7 +370,10 @@ def test_django_login_sucess_anonymization_but_user_set_login(client, test_spans assert login_span.get_tag(user.ID) == "anon_d1ad1f735a4381c2e8dbed0222db1136" assert login_span.get_tag("appsec.events.users.login.success.track") == "true" assert login_span.get_tag(APPSEC.AUTO_LOGIN_EVENTS_SUCCESS_MODE) == LOGIN_EVENTS_MODE.ANON - assert not login_span.get_tag(APPSEC.USER_LOGIN_EVENT_PREFIX + ".success.login") + assert ( + login_span.get_tag(APPSEC.USER_LOGIN_EVENT_PREFIX + ".success.login") + == "anon_d1ad1f735a4381c2e8dbed0222db1136" + ) assert not login_span.get_tag(APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC + ".success.email") assert not login_span.get_tag(APPSEC.USER_LOGIN_EVENT_PREFIX_PUBLIC + ".success.username") diff --git a/tests/contrib/django/test_django_appsec_iast.py b/tests/appsec/integrations/django_tests/test_django_appsec_iast.py similarity index 87% rename from tests/contrib/django/test_django_appsec_iast.py rename to tests/appsec/integrations/django_tests/test_django_appsec_iast.py index ee5cb069331..2b18ae0f7d6 100644 --- a/tests/contrib/django/test_django_appsec_iast.py +++ b/tests/appsec/integrations/django_tests/test_django_appsec_iast.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- import json -import logging -import re import pytest @@ -19,7 +17,7 @@ from tests.utils import override_global_config -TEST_FILE = "tests/contrib/django/django_app/appsec_urls.py" +TEST_FILE = "tests/appsec/integrations/django_tests/django_app/views.py" @pytest.fixture(autouse=True) @@ -30,29 +28,6 @@ def iast_context(): yield -# The log contains "[IAST]" but "[IAST] create_context" or "[IAST] reset_context" are valid -IAST_VALID_LOG = re.compile(r"(?=.*\[IAST\] )(?!.*\[IAST\] (create_context|reset_context))") - - -@pytest.fixture(autouse=True) -def check_native_code_exception_in_each_django_test(request, caplog, telemetry_writer): - if "skip_iast_check_logs" in request.keywords: - yield - else: - caplog.set_level(logging.DEBUG) - with override_env({"_DD_IAST_USE_ROOT_SPAN": "false"}), override_global_config( - dict(_iast_debug=True) - ), caplog.at_level(logging.DEBUG): - yield - - log_messages = [record.message for record in caplog.get_records("call")] - for message in log_messages: - if IAST_VALID_LOG.search(message): - pytest.fail(message) - list_metrics_logs = list(telemetry_writer._logs) - assert len(list_metrics_logs) == 0 - - def _aux_appsec_get_root_span( client, test_spans, @@ -245,6 +220,105 @@ def test_django_tainted_user_agent_iast_enabled_sqli_http_request_parameter(clie assert loaded["vulnerabilities"][0]["hash"] == hash_value +@pytest.mark.django_db() +@pytest.mark.skipif(not python_supported_by_iast(), reason="Python version not supported by IAST") +def test_django_tainted_user_agent_iast_enabled_sqli_http_request_parameter_name_get(client, test_spans, tracer): + with override_global_config(dict(_iast_enabled=True, _deduplication_enabled=False, _iast_request_sampling=100.0)): + root_span, response = _aux_appsec_get_root_span( + client, + test_spans, + tracer, + content_type="application/x-www-form-urlencoded", + url="/appsec/sqli_http_request_parameter_name_get/?SELECT=unused", + headers={"HTTP_USER_AGENT": "test/1.2.3"}, + ) + + vuln_type = "SQL_INJECTION" + + assert response.status_code == 200 + assert response.content == b"test/1.2.3" + + loaded = json.loads(root_span.get_tag(IAST.JSON)) + + line, hash_value = get_line_and_hash( + "iast_enabled_sqli_http_request_parameter_name_get", vuln_type, filename=TEST_FILE + ) + + assert loaded["sources"] == [ + { + "name": "SELECT", + "origin": "http.request.parameter.name", + "value": "SELECT", + } + ] + + assert loaded["vulnerabilities"][0]["type"] == vuln_type + assert loaded["vulnerabilities"][0]["evidence"] == { + "valueParts": [ + {"source": 0, "value": "SELECT"}, + { + "value": " ", + }, + { + "redacted": True, + }, + ] + } + assert loaded["vulnerabilities"][0]["location"]["path"] == TEST_FILE + assert loaded["vulnerabilities"][0]["location"]["line"] == line + assert loaded["vulnerabilities"][0]["hash"] == hash_value + + +@pytest.mark.django_db() +@pytest.mark.skipif(not python_supported_by_iast(), reason="Python version not supported by IAST") +def test_django_tainted_user_agent_iast_enabled_sqli_http_request_parameter_name_post(client, test_spans, tracer): + with override_global_config(dict(_iast_enabled=True, _deduplication_enabled=False, _iast_request_sampling=100.0)): + root_span, response = _aux_appsec_get_root_span( + client, + test_spans, + tracer, + payload=urlencode({"SELECT": "unused"}), + content_type="application/x-www-form-urlencoded", + url="/appsec/sqli_http_request_parameter_name_post/", + headers={"HTTP_USER_AGENT": "test/1.2.3"}, + ) + + vuln_type = "SQL_INJECTION" + + assert response.status_code == 200 + assert response.content == b"test/1.2.3" + + loaded = json.loads(root_span.get_tag(IAST.JSON)) + + line, hash_value = get_line_and_hash( + "iast_enabled_sqli_http_request_parameter_name_post", vuln_type, filename=TEST_FILE + ) + + assert loaded["sources"] == [ + { + "name": "SELECT", + "origin": "http.request.parameter.name", + "value": "SELECT", + } + ] + + assert loaded["vulnerabilities"][0]["type"] == vuln_type + assert loaded["vulnerabilities"][0]["evidence"] == { + "valueParts": [ + {"source": 0, "value": "SELECT"}, + { + "value": " ", + }, + { + "redacted": True, + }, + ] + } + assert loaded["vulnerabilities"][0]["location"]["path"] == TEST_FILE + assert loaded["vulnerabilities"][0]["location"]["line"] == line + assert loaded["vulnerabilities"][0]["hash"] == hash_value + + @pytest.mark.django_db() @pytest.mark.skipif(not python_supported_by_iast(), reason="Python version not supported by IAST") def test_django_tainted_user_agent_iast_enabled_sqli_http_request_header_value(client, test_spans, tracer): @@ -364,37 +438,36 @@ def test_django_tainted_user_agent_iast_disabled_sqli_http_request_header_name(c @pytest.mark.django_db() @pytest.mark.skipif(not python_supported_by_iast(), reason="Python version not supported by IAST") def test_django_iast_enabled_full_sqli_http_path_parameter(client, test_spans, tracer): - with override_global_config(dict(_iast_enabled=True)): - root_span, response = _aux_appsec_get_root_span( - client, - test_spans, - tracer, - url="/appsec/sqli_http_path_parameter/sqlite_master/", - headers={"HTTP_USER_AGENT": "test/1.2.3"}, - ) - assert response.status_code == 200 - assert response.content == b"test/1.2.3" - - loaded = json.loads(root_span.get_tag(IAST.JSON)) - - assert loaded["sources"] == [ - {"origin": "http.request.path.parameter", "name": "q_http_path_parameter", "value": "sqlite_master"} + root_span, response = _aux_appsec_get_root_span( + client, + test_spans, + tracer, + url="/appsec/sqli_http_path_parameter/sqlite_master/", + headers={"HTTP_USER_AGENT": "test/1.2.3"}, + ) + assert response.status_code == 200 + assert response.content == b"test/1.2.3" + + loaded = json.loads(root_span.get_tag(IAST.JSON)) + + assert loaded["sources"] == [ + {"origin": "http.request.path.parameter", "name": "q_http_path_parameter", "value": "sqlite_master"} + ] + assert loaded["vulnerabilities"][0]["type"] == VULN_SQL_INJECTION + assert loaded["vulnerabilities"][0]["evidence"] == { + "valueParts": [ + {"value": "SELECT "}, + {"redacted": True}, + {"value": " from "}, + {"value": "sqlite_master", "source": 0}, ] - assert loaded["vulnerabilities"][0]["type"] == VULN_SQL_INJECTION - assert loaded["vulnerabilities"][0]["evidence"] == { - "valueParts": [ - {"value": "SELECT "}, - {"redacted": True}, - {"value": " from "}, - {"value": "sqlite_master", "source": 0}, - ] - } - line, hash_value = get_line_and_hash( - "iast_enabled_full_sqli_http_path_parameter", VULN_SQL_INJECTION, filename=TEST_FILE - ) - assert loaded["vulnerabilities"][0]["location"]["path"] == TEST_FILE - assert loaded["vulnerabilities"][0]["location"]["line"] == line - assert loaded["vulnerabilities"][0]["hash"] == hash_value + } + line, hash_value = get_line_and_hash( + "iast_enabled_full_sqli_http_path_parameter", VULN_SQL_INJECTION, filename=TEST_FILE + ) + assert loaded["vulnerabilities"][0]["location"]["path"] == TEST_FILE + assert loaded["vulnerabilities"][0]["location"]["line"] == line + assert loaded["vulnerabilities"][0]["hash"] == hash_value @pytest.mark.django_db() diff --git a/tests/appsec/integrations/flask_tests/__init__.py b/tests/appsec/integrations/flask_tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/appsec/integrations/module_with_import_errors.py b/tests/appsec/integrations/flask_tests/module_with_import_errors.py similarity index 100% rename from tests/appsec/integrations/module_with_import_errors.py rename to tests/appsec/integrations/flask_tests/module_with_import_errors.py diff --git a/tests/appsec/integrations/test_flask_remoteconfig.py b/tests/appsec/integrations/flask_tests/test_flask_remoteconfig.py similarity index 94% rename from tests/appsec/integrations/test_flask_remoteconfig.py rename to tests/appsec/integrations/flask_tests/test_flask_remoteconfig.py index eaa69e04182..4ddf1ee8f64 100644 --- a/tests/appsec/integrations/test_flask_remoteconfig.py +++ b/tests/appsec/integrations/flask_tests/test_flask_remoteconfig.py @@ -14,9 +14,9 @@ from ddtrace.internal.compat import httplib from ddtrace.internal.compat import parse from tests.appsec.appsec_utils import gunicorn_server -from tests.appsec.integrations.utils import _PORT -from tests.appsec.integrations.utils import _multi_requests -from tests.appsec.integrations.utils import _request_200 +from tests.appsec.integrations.flask_tests.utils import _PORT +from tests.appsec.integrations.flask_tests.utils import _multi_requests +from tests.appsec.integrations.flask_tests.utils import _request_200 from tests.utils import flaky @@ -187,6 +187,7 @@ def _request_403(client, debug_mode=False, max_retries=40, sleep_time=1): raise AssertionError("request_403 failed, max_retries=%d, sleep_time=%f" % (max_retries, sleep_time)) +@flaky(until=1706677200, reason="TODO(avara1986): We need to migrate testagent to gitlab") @pytest.mark.skipif(sys.version_info >= (3, 11), reason="Gunicorn is only supported up to 3.10") def test_load_testing_appsec_ip_blocking_gunicorn_rc_disabled(): token = "test_load_testing_appsec_ip_blocking_gunicorn_rc_disabled_{}".format(str(uuid.uuid4())) @@ -202,6 +203,7 @@ def test_load_testing_appsec_ip_blocking_gunicorn_rc_disabled(): _unblock_ip(token) +@flaky(until=1706677200, reason="TODO(avara1986): We need to migrate testagent to gitlab") @pytest.mark.skipif(sys.version_info >= (3, 11), reason="Gunicorn is only supported up to 3.10") def test_load_testing_appsec_ip_blocking_gunicorn_block(): token = "test_load_testing_appsec_ip_blocking_gunicorn_block_{}".format(str(uuid.uuid4())) @@ -219,6 +221,7 @@ def test_load_testing_appsec_ip_blocking_gunicorn_block(): _request_200(gunicorn_client) +@flaky(until=1706677200, reason="TODO(avara1986): We need to migrate testagent to gitlab") @pytest.mark.skipif(list(sys.version_info[:2]) != [3, 10], reason="Run this tests in python 3.10") def test_load_testing_appsec_ip_blocking_gunicorn_block_and_kill_child_worker(): token = "test_load_testing_appsec_ip_blocking_gunicorn_block_and_kill_child_worker_{}".format(str(uuid.uuid4())) @@ -267,7 +270,7 @@ def test_load_testing_appsec_1click_and_ip_blocking_gunicorn_block_and_kill_chil _request_200(gunicorn_client, debug_mode=False) -@pytest.mark.subprocess(ddtrace_run=True, out=b"success") +@pytest.mark.subprocess(ddtrace_run=True, err=None, out=b"success") def test_compatiblity_with_multiprocessing(): import multiprocessing from multiprocessing import Array diff --git a/tests/appsec/integrations/test_gunicorn_handlers.py b/tests/appsec/integrations/flask_tests/test_gunicorn_handlers.py similarity index 100% rename from tests/appsec/integrations/test_gunicorn_handlers.py rename to tests/appsec/integrations/flask_tests/test_gunicorn_handlers.py diff --git a/tests/appsec/integrations/test_flask_entrypoint_iast_patches.py b/tests/appsec/integrations/flask_tests/test_iast_flask_entrypoint_iast_patches.py similarity index 85% rename from tests/appsec/integrations/test_flask_entrypoint_iast_patches.py rename to tests/appsec/integrations/flask_tests/test_iast_flask_entrypoint_iast_patches.py index 4f54bc675c3..7b15868c270 100644 --- a/tests/appsec/integrations/test_flask_entrypoint_iast_patches.py +++ b/tests/appsec/integrations/flask_tests/test_iast_flask_entrypoint_iast_patches.py @@ -1,9 +1,10 @@ -import pytest +import sys -from tests.utils import flaky +import pytest -@pytest.mark.subprocess() +@pytest.mark.skipif(sys.version_info >= (3, 13, 0), reason="Test not compatible with Python 3.13") +@pytest.mark.subprocess(err=None) def test_ddtrace_iast_flask_patch(): import dis import io @@ -35,7 +36,35 @@ def test_ddtrace_iast_flask_patch(): del sys.modules["tests.appsec.iast.fixtures.entrypoint.app_main_patched"] -@pytest.mark.subprocess() +@pytest.mark.skipif(sys.version_info < (3, 13, 0), reason="Test compatible with Python 3.13") +@pytest.mark.subprocess(err=None) +def test_ddtrace_iast_flask_patch_py313(): + import dis + import io + import re + import sys + + from tests.utils import override_env + from tests.utils import override_global_config + + PATTERN = r"""LOAD_GLOBAL 0 \(_ddtrace_aspects\)""" + + with override_global_config(dict(_iast_enabled=True)), override_env( + dict(DD_IAST_ENABLED="true", DD_IAST_REQUEST_SAMPLING="100") + ): + import tests.appsec.iast.fixtures.entrypoint.app_main_patched as flask_entrypoint + + dis_output = io.StringIO() + dis.dis(flask_entrypoint, file=dis_output) + str_output = dis_output.getvalue() + # Should have replaced the binary op with the aspect in add_test: + assert re.search(PATTERN, str_output), str_output + # Should have replaced the app.run() with a pass: + # assert "Disassembly of run" not in str_output, str_output + del sys.modules["tests.appsec.iast.fixtures.entrypoint.app_main_patched"] + + +@pytest.mark.subprocess(err=None) def test_ddtrace_iast_flask_patch_iast_disabled(): import dis import io @@ -62,7 +91,7 @@ def _uninstall_watchdog_and_reload(): del sys.modules["tests.appsec.iast.fixtures.entrypoint.app_main_patched"] -@pytest.mark.subprocess() +@pytest.mark.subprocess(err=None) def test_ddtrace_iast_flask_no_patch(): import dis import io @@ -93,7 +122,7 @@ def _uninstall_watchdog_and_reload(): del sys.modules["tests.appsec.iast.fixtures.entrypoint.app"] -@pytest.mark.subprocess() +@pytest.mark.subprocess(err=None) def test_ddtrace_iast_flask_app_create_app_enable_iast_propagation(): import dis import io @@ -125,7 +154,7 @@ def _uninstall_watchdog_and_reload(): del sys.modules["tests.appsec.iast.fixtures.entrypoint.views"] -@pytest.mark.subprocess() +@pytest.mark.subprocess(err=None) def test_ddtrace_iast_flask_app_create_app_patch_all(): import dis import io @@ -155,8 +184,7 @@ def _uninstall_watchdog_and_reload(): del sys.modules["tests.appsec.iast.fixtures.entrypoint.views"] -@flaky(1736035200) -@pytest.mark.subprocess(check_logs=False) +@pytest.mark.subprocess(err=None) def test_ddtrace_iast_flask_app_create_app_patch_all_enable_iast_propagation(): import dis import io @@ -187,7 +215,7 @@ def _uninstall_watchdog_and_reload(): del sys.modules["tests.appsec.iast.fixtures.entrypoint.views"] -@pytest.mark.subprocess() +@pytest.mark.subprocess(err=None) def test_ddtrace_iast_flask_app_create_app_patch_all_enable_iast_propagation_disabled(): import dis import io diff --git a/tests/appsec/integrations/test_flask_iast_patching.py b/tests/appsec/integrations/flask_tests/test_iast_flask_patching.py similarity index 93% rename from tests/appsec/integrations/test_flask_iast_patching.py rename to tests/appsec/integrations/flask_tests/test_iast_flask_patching.py index 3291297ea92..8cb3b2e7730 100644 --- a/tests/appsec/integrations/test_flask_iast_patching.py +++ b/tests/appsec/integrations/flask_tests/test_iast_flask_patching.py @@ -2,9 +2,8 @@ from tests.appsec.appsec_utils import flask_server from tests.appsec.appsec_utils import gunicorn_server -from tests.appsec.integrations.utils import _PORT -from tests.appsec.integrations.utils import _request_200 -from tests.utils import flaky +from tests.appsec.integrations.flask_tests.utils import _PORT +from tests.appsec.integrations.flask_tests.utils import _request_200 def test_flask_iast_ast_patching_import_error(): @@ -28,7 +27,6 @@ def test_flask_iast_ast_patching_import_error(): assert response.content == b"False" -@flaky(until=1706677200, reason="TODO(avara1986): Re.Match contains errors. APPSEC-55239") @pytest.mark.parametrize("style", ["re_module", "re_object"]) @pytest.mark.parametrize("endpoint", ["re", "non-re"]) @pytest.mark.parametrize( diff --git a/tests/appsec/integrations/test_flask_telemetry.py b/tests/appsec/integrations/flask_tests/test_iast_flask_telemetry.py similarity index 100% rename from tests/appsec/integrations/test_flask_telemetry.py rename to tests/appsec/integrations/flask_tests/test_iast_flask_telemetry.py diff --git a/tests/appsec/integrations/test_langchain.py b/tests/appsec/integrations/flask_tests/test_iast_langchain.py similarity index 100% rename from tests/appsec/integrations/test_langchain.py rename to tests/appsec/integrations/flask_tests/test_iast_langchain.py diff --git a/tests/appsec/integrations/test_psycopg2.py b/tests/appsec/integrations/flask_tests/test_iast_psycopg2.py similarity index 100% rename from tests/appsec/integrations/test_psycopg2.py rename to tests/appsec/integrations/flask_tests/test_iast_psycopg2.py diff --git a/tests/appsec/integrations/utils.py b/tests/appsec/integrations/flask_tests/utils.py similarity index 100% rename from tests/appsec/integrations/utils.py rename to tests/appsec/integrations/flask_tests/utils.py index 18bf20a1608..3935a9447b4 100644 --- a/tests/appsec/integrations/utils.py +++ b/tests/appsec/integrations/flask_tests/utils.py @@ -2,6 +2,9 @@ import time +_PORT = 8040 + + def _multi_requests(client, url="/", debug_mode=False): if debug_mode: results = [ @@ -47,9 +50,6 @@ def _request_200( raise AssertionError("request_200 failed, max_retries=%d, sleep_time=%f" % (max_retries, sleep_time)) -_PORT = 8040 - - def _request(client, url="/"): response = client.get(url, headers={"X-Forwarded-For": "123.45.67.88"}) return response diff --git a/tests/appsec/suitespec.yml b/tests/appsec/suitespec.yml index e2b15e2336c..f075ba2da4a 100644 --- a/tests/appsec/suitespec.yml +++ b/tests/appsec/suitespec.yml @@ -74,7 +74,15 @@ suites: retry: 2 runner: hatch appsec_iast_packages: - parallelism: 5 + parallelism: 4 + paths: + - '@appsec_iast' + - tests/appsec/iast_packages/* + retry: 2 + runner: hatch + timeout: 50m + appsec_integrations_pygoat: + parallelism: 7 paths: - '@bootstrap' - '@core' @@ -82,12 +90,13 @@ suites: - '@appsec' - '@appsec_iast' - '@remoteconfig' - - tests/appsec/iast/* - - tests/appsec/iast_packages/* + - tests/appsec/integrations/pygoat_tests/* + - tests/snapshots/tests.appsec.* + retry: 2 runner: riot snapshot: true - appsec_integrations: - parallelism: 7 + appsec_integrations_flask: + parallelism: 6 paths: - '@bootstrap' - '@core' @@ -95,11 +104,23 @@ suites: - '@appsec' - '@appsec_iast' - '@remoteconfig' - - tests/appsec/* - - tests/snapshots/tests.appsec.* + - tests/appsec/integrations/flask_tests/* retry: 2 - runner: riot - snapshot: true + runner: hatch + timeout: 30m + appsec_integrations_django: + parallelism: 6 + paths: + - '@bootstrap' + - '@core' + - '@tracing' + - '@appsec' + - '@appsec_iast' + - '@remoteconfig' + - tests/appsec/integrations/django_tests/* + retry: 2 + runner: hatch + timeout: 30m appsec_threats_django: parallelism: 12 paths: diff --git a/tests/commands/ddtrace_run_integration.py b/tests/commands/ddtrace_run_integration.py index e52a0c9b8b0..12f8ab857e2 100644 --- a/tests/commands/ddtrace_run_integration.py +++ b/tests/commands/ddtrace_run_integration.py @@ -5,7 +5,7 @@ import redis -from ddtrace import Pin +from ddtrace.trace import Pin from tests.contrib.config import REDIS_CONFIG from tests.utils import DummyWriter diff --git a/tests/contrib/aiobotocore/test.py b/tests/contrib/aiobotocore/test.py index 9b31b61abb4..f535dfb76d3 100644 --- a/tests/contrib/aiobotocore/test.py +++ b/tests/contrib/aiobotocore/test.py @@ -5,8 +5,8 @@ import pytest from ddtrace.constants import ERROR_MSG -from ddtrace.contrib.aiobotocore.patch import patch -from ddtrace.contrib.aiobotocore.patch import unpatch +from ddtrace.contrib.internal.aiobotocore.patch import patch +from ddtrace.contrib.internal.aiobotocore.patch import unpatch from tests.conftest import DEFAULT_DDTRACE_SUBPROCESS_TEST_SERVICE_NAME from tests.utils import assert_is_measured from tests.utils import assert_span_http_status_code @@ -419,8 +419,8 @@ def test_schematized_env_specified_service(ddtrace_run_python_code_in_subprocess service_name, schema_version, expected_service_name, expected_operation_name = schema_params code = """ import asyncio -from ddtrace.contrib.aiobotocore.patch import patch -from ddtrace.contrib.aiobotocore.patch import unpatch +from ddtrace.contrib.internal.aiobotocore.patch import patch +from ddtrace.contrib.internal.aiobotocore.patch import unpatch from tests.contrib.aiobotocore.utils import * from tests.conftest import * diff --git a/tests/contrib/aiobotocore/test_aiobotocore_patch.py b/tests/contrib/aiobotocore/test_aiobotocore_patch.py index 7f56b2c90f4..4a452336388 100644 --- a/tests/contrib/aiobotocore/test_aiobotocore_patch.py +++ b/tests/contrib/aiobotocore/test_aiobotocore_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.aiobotocore import get_version -from ddtrace.contrib.aiobotocore.patch import patch +from ddtrace.contrib.internal.aiobotocore.patch import get_version +from ddtrace.contrib.internal.aiobotocore.patch import patch try: - from ddtrace.contrib.aiobotocore.patch import unpatch + from ddtrace.contrib.internal.aiobotocore.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/aiobotocore/utils.py b/tests/contrib/aiobotocore/utils.py index 061f8d37847..b51b6550327 100644 --- a/tests/contrib/aiobotocore/utils.py +++ b/tests/contrib/aiobotocore/utils.py @@ -3,7 +3,7 @@ from async_generator import asynccontextmanager from async_generator import yield_ -from ddtrace import Pin +from ddtrace.trace import Pin LOCALSTACK_ENDPOINT_URL = { diff --git a/tests/contrib/aiohttp/app/web.py b/tests/contrib/aiohttp/app/web.py index e97100e4144..d4eeb9b1646 100644 --- a/tests/contrib/aiohttp/app/web.py +++ b/tests/contrib/aiohttp/app/web.py @@ -11,7 +11,7 @@ else: import jinja2 -from ddtrace.contrib.aiohttp.middlewares import CONFIG_KEY +from ddtrace.contrib.internal.aiohttp.middlewares import CONFIG_KEY BASE_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/contrib/aiohttp/conftest.py b/tests/contrib/aiohttp/conftest.py index cb687b20280..4166049bd23 100644 --- a/tests/contrib/aiohttp/conftest.py +++ b/tests/contrib/aiohttp/conftest.py @@ -1,8 +1,8 @@ import aiohttp # noqa:F401 import pytest -from ddtrace.contrib.aiohttp.middlewares import trace_app -from ddtrace.contrib.aiohttp.patch import unpatch +from ddtrace.contrib.internal.aiohttp.middlewares import trace_app +from ddtrace.contrib.internal.aiohttp.patch import unpatch from ddtrace.internal.utils import version # noqa:F401 from .app.web import setup_app diff --git a/tests/contrib/aiohttp/test_aiohttp_client.py b/tests/contrib/aiohttp/test_aiohttp_client.py index dd2f51bbaa9..2b2b51c2650 100644 --- a/tests/contrib/aiohttp/test_aiohttp_client.py +++ b/tests/contrib/aiohttp/test_aiohttp_client.py @@ -3,10 +3,10 @@ import aiohttp import pytest -from ddtrace import Pin -from ddtrace.contrib.aiohttp import patch -from ddtrace.contrib.aiohttp import unpatch -from ddtrace.contrib.aiohttp.patch import extract_netloc_and_query_info_from_url +from ddtrace.contrib.internal.aiohttp.patch import extract_netloc_and_query_info_from_url +from ddtrace.contrib.internal.aiohttp.patch import patch +from ddtrace.contrib.internal.aiohttp.patch import unpatch +from ddtrace.trace import Pin from tests.utils import override_config from tests.utils import override_http_config @@ -101,7 +101,7 @@ async def test_distributed_tracing_disabled(ddtrace_run_python_code_in_subproces import asyncio import sys import aiohttp -from ddtrace import Pin +from ddtrace.trace import Pin from tests.contrib.aiohttp.test_aiohttp_client import URL async def test(): @@ -184,7 +184,7 @@ def test_configure_service_name_pin(ddtrace_run_python_code_in_subprocess): import asyncio import sys import aiohttp -from ddtrace import Pin +from ddtrace.trace import Pin from tests.contrib.aiohttp.test_aiohttp_client import URL_200 async def test(): diff --git a/tests/contrib/aiohttp/test_aiohttp_patch.py b/tests/contrib/aiohttp/test_aiohttp_patch.py index e721bfc666c..e00297e8bf4 100644 --- a/tests/contrib/aiohttp/test_aiohttp_patch.py +++ b/tests/contrib/aiohttp/test_aiohttp_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.aiohttp import get_version -from ddtrace.contrib.aiohttp.patch import patch +from ddtrace.contrib.internal.aiohttp.patch import get_version +from ddtrace.contrib.internal.aiohttp.patch import patch try: - from ddtrace.contrib.aiohttp.patch import unpatch + from ddtrace.contrib.internal.aiohttp.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/aiohttp/test_middleware.py b/tests/contrib/aiohttp/test_middleware.py index 58a6023ac98..30d40654314 100644 --- a/tests/contrib/aiohttp/test_middleware.py +++ b/tests/contrib/aiohttp/test_middleware.py @@ -3,14 +3,14 @@ from opentracing.scope_managers.asyncio import AsyncioScopeManager import pytest +from ddtrace._trace.sampler import RateSampler from ddtrace.constants import ERROR_MSG from ddtrace.constants import SAMPLING_PRIORITY_KEY from ddtrace.constants import USER_KEEP -from ddtrace.contrib.aiohttp.middlewares import CONFIG_KEY -from ddtrace.contrib.aiohttp.middlewares import trace_app -from ddtrace.contrib.aiohttp.middlewares import trace_middleware +from ddtrace.contrib.internal.aiohttp.middlewares import CONFIG_KEY +from ddtrace.contrib.internal.aiohttp.middlewares import trace_app +from ddtrace.contrib.internal.aiohttp.middlewares import trace_middleware from ddtrace.ext import http -from ddtrace.sampler import RateSampler from tests.opentracer.utils import init_tracer from tests.utils import assert_span_http_status_code from tests.utils import flaky diff --git a/tests/contrib/aiohttp/test_request.py b/tests/contrib/aiohttp/test_request.py index cde0f311521..36e9d8a399a 100644 --- a/tests/contrib/aiohttp/test_request.py +++ b/tests/contrib/aiohttp/test_request.py @@ -3,7 +3,7 @@ from urllib import request from ddtrace import config -from ddtrace.contrib.aiohttp.middlewares import trace_app +from ddtrace.contrib.internal.aiohttp.middlewares import trace_app from tests.utils import assert_is_measured from tests.utils import override_global_config diff --git a/tests/contrib/aiohttp_jinja2/conftest.py b/tests/contrib/aiohttp_jinja2/conftest.py index aca4965c318..a58b72f7f49 100644 --- a/tests/contrib/aiohttp_jinja2/conftest.py +++ b/tests/contrib/aiohttp_jinja2/conftest.py @@ -1,9 +1,9 @@ import aiohttp_jinja2 import pytest -from ddtrace.contrib.aiohttp_jinja2 import patch -from ddtrace.contrib.aiohttp_jinja2 import unpatch -from ddtrace.pin import Pin +from ddtrace.contrib.internal.aiohttp_jinja2.patch import patch +from ddtrace.contrib.internal.aiohttp_jinja2.patch import unpatch +from ddtrace.trace import Pin from tests.contrib.aiohttp.conftest import app_tracer # noqa:F401 from tests.contrib.aiohttp.conftest import patched_app_tracer # noqa:F401 from tests.contrib.aiohttp.conftest import untraced_app_tracer # noqa:F401 diff --git a/tests/contrib/aiohttp_jinja2/test_aiohttp_jinja2.py b/tests/contrib/aiohttp_jinja2/test_aiohttp_jinja2.py index 222522d66f6..8889d828752 100644 --- a/tests/contrib/aiohttp_jinja2/test_aiohttp_jinja2.py +++ b/tests/contrib/aiohttp_jinja2/test_aiohttp_jinja2.py @@ -1,9 +1,9 @@ import aiohttp_jinja2 import pytest -from ddtrace import Pin from ddtrace import tracer from ddtrace.constants import ERROR_MSG +from ddtrace.trace import Pin from tests.contrib.aiohttp.app.web import set_filesystem_loader from tests.contrib.aiohttp.app.web import set_package_loader import tests.contrib.aiohttp.conftest # noqa:F401 diff --git a/tests/contrib/aiohttp_jinja2/test_aiohttp_jinja2_patch.py b/tests/contrib/aiohttp_jinja2/test_aiohttp_jinja2_patch.py index e9dfc0d5e1b..c2b684418d5 100644 --- a/tests/contrib/aiohttp_jinja2/test_aiohttp_jinja2_patch.py +++ b/tests/contrib/aiohttp_jinja2/test_aiohttp_jinja2_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.aiohttp_jinja2 import get_version -from ddtrace.contrib.aiohttp_jinja2.patch import patch +from ddtrace.contrib.internal.aiohttp_jinja2.patch import get_version +from ddtrace.contrib.internal.aiohttp_jinja2.patch import patch try: - from ddtrace.contrib.aiohttp_jinja2.patch import unpatch + from ddtrace.contrib.internal.aiohttp_jinja2.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/aiomysql/test_aiomysql.py b/tests/contrib/aiomysql/test_aiomysql.py index ce6cfd7ce3c..0bf8839dc96 100644 --- a/tests/contrib/aiomysql/test_aiomysql.py +++ b/tests/contrib/aiomysql/test_aiomysql.py @@ -5,11 +5,11 @@ import pymysql import pytest -from ddtrace import Pin from ddtrace import Tracer -from ddtrace.contrib.aiomysql import patch -from ddtrace.contrib.aiomysql import unpatch +from ddtrace.contrib.internal.aiomysql.patch import patch +from ddtrace.contrib.internal.aiomysql.patch import unpatch from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME +from ddtrace.trace import Pin from tests.contrib import shared_tests_async as shared_tests from tests.contrib.asyncio.utils import AsyncioTestCase from tests.contrib.asyncio.utils import mark_asyncio diff --git a/tests/contrib/aiomysql/test_aiomysql_patch.py b/tests/contrib/aiomysql/test_aiomysql_patch.py index cbdb900d48f..29ea79357a2 100644 --- a/tests/contrib/aiomysql/test_aiomysql_patch.py +++ b/tests/contrib/aiomysql/test_aiomysql_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.aiomysql import get_version -from ddtrace.contrib.aiomysql.patch import patch +from ddtrace.contrib.internal.aiomysql.patch import get_version +from ddtrace.contrib.internal.aiomysql.patch import patch try: - from ddtrace.contrib.aiomysql.patch import unpatch + from ddtrace.contrib.internal.aiomysql.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/aiopg/test.py b/tests/contrib/aiopg/test.py index c2b5fc49804..eb738e009d8 100644 --- a/tests/contrib/aiopg/test.py +++ b/tests/contrib/aiopg/test.py @@ -4,11 +4,12 @@ from psycopg2 import extras import pytest -# project -from ddtrace import Pin -from ddtrace.contrib.aiopg.patch import patch -from ddtrace.contrib.aiopg.patch import unpatch +from ddtrace.contrib.internal.aiopg.patch import patch +from ddtrace.contrib.internal.aiopg.patch import unpatch from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME + +# project +from ddtrace.trace import Pin from tests.contrib.asyncio.utils import AsyncioTestCase from tests.contrib.config import POSTGRES_CONFIG from tests.opentracer.utils import init_tracer diff --git a/tests/contrib/aiopg/test_aiopg_patch.py b/tests/contrib/aiopg/test_aiopg_patch.py index edbb57005cb..459f64cb69b 100644 --- a/tests/contrib/aiopg/test_aiopg_patch.py +++ b/tests/contrib/aiopg/test_aiopg_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.aiopg import get_version -from ddtrace.contrib.aiopg.patch import patch +from ddtrace.contrib.internal.aiopg.patch import get_version +from ddtrace.contrib.internal.aiopg.patch import patch try: - from ddtrace.contrib.aiopg.patch import unpatch + from ddtrace.contrib.internal.aiopg.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/algoliasearch/test.py b/tests/contrib/algoliasearch/test.py index 3b81e195899..87f5f7b6910 100644 --- a/tests/contrib/algoliasearch/test.py +++ b/tests/contrib/algoliasearch/test.py @@ -1,9 +1,9 @@ from ddtrace import config from ddtrace import patch_all -from ddtrace.contrib.algoliasearch.patch import algoliasearch_version -from ddtrace.contrib.algoliasearch.patch import patch -from ddtrace.contrib.algoliasearch.patch import unpatch -from ddtrace.pin import Pin +from ddtrace.contrib.internal.algoliasearch.patch import algoliasearch_version +from ddtrace.contrib.internal.algoliasearch.patch import patch +from ddtrace.contrib.internal.algoliasearch.patch import unpatch +from ddtrace.trace import Pin from ddtrace.vendor.packaging.version import parse as parse_version from tests.utils import TracerTestCase from tests.utils import assert_is_measured diff --git a/tests/contrib/algoliasearch/test_algoliasearch_patch.py b/tests/contrib/algoliasearch/test_algoliasearch_patch.py index 324e1d37f78..c8381976166 100644 --- a/tests/contrib/algoliasearch/test_algoliasearch_patch.py +++ b/tests/contrib/algoliasearch/test_algoliasearch_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.algoliasearch import get_version -from ddtrace.contrib.algoliasearch.patch import patch +from ddtrace.contrib.internal.algoliasearch.patch import get_version +from ddtrace.contrib.internal.algoliasearch.patch import patch try: - from ddtrace.contrib.algoliasearch.patch import unpatch + from ddtrace.contrib.internal.algoliasearch.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/anthropic/conftest.py b/tests/contrib/anthropic/conftest.py index 8ae466dd0bc..a6a4b53cd4c 100644 --- a/tests/contrib/anthropic/conftest.py +++ b/tests/contrib/anthropic/conftest.py @@ -3,10 +3,10 @@ import mock import pytest -from ddtrace import Pin -from ddtrace.contrib.anthropic.patch import patch -from ddtrace.contrib.anthropic.patch import unpatch +from ddtrace.contrib.internal.anthropic.patch import patch +from ddtrace.contrib.internal.anthropic.patch import unpatch from ddtrace.llmobs import LLMObs +from ddtrace.trace import Pin from tests.contrib.anthropic.utils import get_request_vcr from tests.utils import DummyTracer from tests.utils import DummyWriter diff --git a/tests/contrib/anthropic/test_anthropic_patch.py b/tests/contrib/anthropic/test_anthropic_patch.py index 52675cc1341..3e6234393e4 100644 --- a/tests/contrib/anthropic/test_anthropic_patch.py +++ b/tests/contrib/anthropic/test_anthropic_patch.py @@ -1,6 +1,6 @@ -from ddtrace.contrib.anthropic import get_version -from ddtrace.contrib.anthropic import patch -from ddtrace.contrib.anthropic import unpatch +from ddtrace.contrib.internal.anthropic.patch import get_version +from ddtrace.contrib.internal.anthropic.patch import patch +from ddtrace.contrib.internal.anthropic.patch import unpatch from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/aredis/test_aredis.py b/tests/contrib/aredis/test_aredis.py index 4949a3b19d0..e62cfa974be 100644 --- a/tests/contrib/aredis/test_aredis.py +++ b/tests/contrib/aredis/test_aredis.py @@ -5,9 +5,9 @@ import pytest from wrapt import ObjectProxy -from ddtrace import Pin -from ddtrace.contrib.aredis.patch import patch -from ddtrace.contrib.aredis.patch import unpatch +from ddtrace.contrib.internal.aredis.patch import patch +from ddtrace.contrib.internal.aredis.patch import unpatch +from ddtrace.trace import Pin from tests.conftest import DEFAULT_DDTRACE_SUBPROCESS_TEST_SERVICE_NAME from tests.opentracer.utils import init_tracer from tests.utils import override_config @@ -152,7 +152,7 @@ def test_schematization_of_service_and_operation(ddtrace_run_python_code_in_subp import pytest import sys from tests.conftest import * -from ddtrace.pin import Pin +from ddtrace.trace import Pin import aredis from tests.contrib.config import REDIS_CONFIG from tests.contrib.aredis.test_aredis import traced_aredis diff --git a/tests/contrib/aredis/test_aredis_patch.py b/tests/contrib/aredis/test_aredis_patch.py index f6d8c550883..7faf3a84e0f 100644 --- a/tests/contrib/aredis/test_aredis_patch.py +++ b/tests/contrib/aredis/test_aredis_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.aredis import get_version -from ddtrace.contrib.aredis.patch import patch +from ddtrace.contrib.internal.aredis.patch import get_version +from ddtrace.contrib.internal.aredis.patch import patch try: - from ddtrace.contrib.aredis.patch import unpatch + from ddtrace.contrib.internal.aredis.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/asgi/test_asgi.py b/tests/contrib/asgi/test_asgi.py index c1e1f1c7328..f2c461e56cd 100644 --- a/tests/contrib/asgi/test_asgi.py +++ b/tests/contrib/asgi/test_asgi.py @@ -9,9 +9,9 @@ import pytest from ddtrace.constants import ERROR_MSG -from ddtrace.contrib.asgi import TraceMiddleware -from ddtrace.contrib.asgi import span_from_scope +from ddtrace.contrib.internal.asgi.middleware import TraceMiddleware from ddtrace.contrib.internal.asgi.middleware import _parse_response_cookies +from ddtrace.contrib.internal.asgi.middleware import span_from_scope from ddtrace.propagation import http as http_propagation from tests.conftest import DEFAULT_DDTRACE_SUBPROCESS_TEST_SERVICE_NAME from tests.utils import DummyTracer @@ -195,8 +195,8 @@ def test_span_attribute_schema_operation_name(ddtrace_run_python_code_in_subproc from tests.contrib.asgi.test_asgi import scope from tests.contrib.asgi.test_asgi import tracer from asgiref.testing import ApplicationCommunicator -from ddtrace.contrib.asgi import TraceMiddleware -from ddtrace.contrib.asgi import span_from_scope +from ddtrace.contrib.internal.asgi.middleware import TraceMiddleware +from ddtrace.contrib.internal.asgi.middleware import span_from_scope @pytest.mark.asyncio @@ -254,8 +254,8 @@ def test_span_attribute_schema_service_name(ddtrace_run_python_code_in_subproces from tests.contrib.asgi.test_asgi import scope from tests.contrib.asgi.test_asgi import tracer from asgiref.testing import ApplicationCommunicator -from ddtrace.contrib.asgi import TraceMiddleware -from ddtrace.contrib.asgi import span_from_scope +from ddtrace.contrib.internal.asgi.middleware import TraceMiddleware +from ddtrace.contrib.internal.asgi.middleware import span_from_scope @pytest.mark.asyncio async def test(scope, tracer, test_spans): diff --git a/tests/contrib/asyncio/test_lazyimport.py b/tests/contrib/asyncio/test_lazyimport.py index adca84973db..07c96bc799f 100644 --- a/tests/contrib/asyncio/test_lazyimport.py +++ b/tests/contrib/asyncio/test_lazyimport.py @@ -16,3 +16,16 @@ def test_lazy_import(): assert tracer.context_provider.active() is span span.finish() assert tracer.context_provider.active() is None + + +@pytest.mark.subprocess() +def test_asyncio_not_imported_by_auto_instrumentation(): + # Module unloading is not supported for asyncio, a simple workaround + # is to ensure asyncio is not imported by ddtrace.auto or ddtrace-run. + # If asyncio is imported by ddtrace.auto the asyncio event loop with fail + # to register new loops in some platforms (e.g. Ubuntuu). + import sys + + import ddtrace.auto # noqa: F401 + + assert "asyncio" not in sys.modules diff --git a/tests/contrib/asyncio/test_propagation.py b/tests/contrib/asyncio/test_propagation.py index b62f004f281..fa78e975c9f 100644 --- a/tests/contrib/asyncio/test_propagation.py +++ b/tests/contrib/asyncio/test_propagation.py @@ -5,8 +5,8 @@ from ddtrace._trace.context import Context from ddtrace._trace.provider import DefaultContextProvider -from ddtrace.contrib.asyncio.patch import patch -from ddtrace.contrib.asyncio.patch import unpatch +from ddtrace.contrib.internal.asyncio.patch import patch +from ddtrace.contrib.internal.asyncio.patch import unpatch from tests.opentracer.utils import init_tracer diff --git a/tests/contrib/asyncio/test_tracer.py b/tests/contrib/asyncio/test_tracer.py index 16d16215b5e..e7aaa94d903 100644 --- a/tests/contrib/asyncio/test_tracer.py +++ b/tests/contrib/asyncio/test_tracer.py @@ -4,9 +4,9 @@ import pytest from ddtrace.constants import ERROR_MSG -from ddtrace.contrib.asyncio import patch -from ddtrace.contrib.asyncio import unpatch from ddtrace.contrib.asyncio.compat import asyncio_current_task +from ddtrace.contrib.internal.asyncio.patch import patch +from ddtrace.contrib.internal.asyncio.patch import unpatch @pytest.fixture(autouse=True) diff --git a/tests/contrib/asyncpg/test_asyncpg.py b/tests/contrib/asyncpg/test_asyncpg.py index 8a43e3f68ab..032a0f91731 100644 --- a/tests/contrib/asyncpg/test_asyncpg.py +++ b/tests/contrib/asyncpg/test_asyncpg.py @@ -5,11 +5,11 @@ import mock import pytest -from ddtrace import Pin from ddtrace import tracer -from ddtrace.contrib.asyncpg import patch -from ddtrace.contrib.asyncpg import unpatch +from ddtrace.contrib.internal.asyncpg.patch import patch +from ddtrace.contrib.internal.asyncpg.patch import unpatch from ddtrace.contrib.trace_utils import iswrapped +from ddtrace.trace import Pin from tests.contrib.asyncio.utils import AsyncioTestCase from tests.contrib.asyncio.utils import mark_asyncio from tests.contrib.config import POSTGRES_CONFIG diff --git a/tests/contrib/asyncpg/test_asyncpg_patch.py b/tests/contrib/asyncpg/test_asyncpg_patch.py index 3e9a1dcac71..23a7e69abc0 100644 --- a/tests/contrib/asyncpg/test_asyncpg_patch.py +++ b/tests/contrib/asyncpg/test_asyncpg_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.asyncpg import get_version -from ddtrace.contrib.asyncpg.patch import patch +from ddtrace.contrib.internal.asyncpg.patch import get_version +from ddtrace.contrib.internal.asyncpg.patch import patch try: - from ddtrace.contrib.asyncpg.patch import unpatch + from ddtrace.contrib.internal.asyncpg.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/asynctest/test_asynctest.py b/tests/contrib/asynctest/test_asynctest.py index 44e0c0c2387..cd325e0ec8b 100644 --- a/tests/contrib/asynctest/test_asynctest.py +++ b/tests/contrib/asynctest/test_asynctest.py @@ -5,7 +5,7 @@ import pytest import ddtrace -from ddtrace.contrib.pytest.plugin import is_enabled +from ddtrace.contrib.internal.pytest.plugin import is_enabled from ddtrace.ext import test from ddtrace.internal.ci_visibility import CIVisibility from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings diff --git a/tests/contrib/avro/test_avro.py b/tests/contrib/avro/test_avro.py index 7be5e2f6351..3db10460a23 100644 --- a/tests/contrib/avro/test_avro.py +++ b/tests/contrib/avro/test_avro.py @@ -4,11 +4,11 @@ from avro.io import DatumWriter from wrapt import ObjectProxy -from ddtrace import Pin from ddtrace.constants import AUTO_KEEP from ddtrace.contrib.internal.avro.patch import patch from ddtrace.contrib.internal.avro.patch import unpatch from ddtrace.ext import schema as SCHEMA_TAGS +from ddtrace.trace import Pin OPENAPI_USER_SCHEMA_DEF = ( diff --git a/tests/contrib/aws_lambda/test_aws_lambda.py b/tests/contrib/aws_lambda/test_aws_lambda.py index 2de2b286e03..dd99607000c 100644 --- a/tests/contrib/aws_lambda/test_aws_lambda.py +++ b/tests/contrib/aws_lambda/test_aws_lambda.py @@ -1,7 +1,7 @@ import pytest -from ddtrace.contrib.aws_lambda import patch -from ddtrace.contrib.aws_lambda import unpatch +from ddtrace.contrib.internal.aws_lambda.patch import patch +from ddtrace.contrib.internal.aws_lambda.patch import unpatch from tests.contrib.aws_lambda.handlers import class_handler from tests.contrib.aws_lambda.handlers import datadog from tests.contrib.aws_lambda.handlers import finishing_spans_early_handler diff --git a/tests/contrib/azure_functions/test_azure_functions_patch.py b/tests/contrib/azure_functions/test_azure_functions_patch.py index acc58df654a..2dfa71ade15 100644 --- a/tests/contrib/azure_functions/test_azure_functions_patch.py +++ b/tests/contrib/azure_functions/test_azure_functions_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.azure_functions import get_version -from ddtrace.contrib.azure_functions.patch import patch +from ddtrace.contrib.internal.azure_functions.patch import get_version +from ddtrace.contrib.internal.azure_functions.patch import patch try: - from ddtrace.contrib.azure_functions.patch import unpatch + from ddtrace.contrib.internal.azure_functions.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/boto/test.py b/tests/contrib/boto/test.py index f0b9cb57fd1..2570ca9c65c 100644 --- a/tests/contrib/boto/test.py +++ b/tests/contrib/boto/test.py @@ -14,12 +14,13 @@ from moto import mock_s3 from moto import mock_sts -# project -from ddtrace import Pin -from ddtrace.contrib.boto.patch import patch -from ddtrace.contrib.boto.patch import unpatch +from ddtrace.contrib.internal.boto.patch import patch +from ddtrace.contrib.internal.boto.patch import unpatch from ddtrace.ext import http from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME + +# project +from ddtrace.trace import Pin from tests.opentracer.utils import init_tracer from tests.utils import TracerTestCase from tests.utils import assert_is_measured diff --git a/tests/contrib/boto/test_boto_patch.py b/tests/contrib/boto/test_boto_patch.py index 4dcc92ee184..831d8eaa957 100644 --- a/tests/contrib/boto/test_boto_patch.py +++ b/tests/contrib/boto/test_boto_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.boto import get_version -from ddtrace.contrib.boto.patch import patch +from ddtrace.contrib.internal.boto.patch import get_version +from ddtrace.contrib.internal.boto.patch import patch try: - from ddtrace.contrib.boto.patch import unpatch + from ddtrace.contrib.internal.boto.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/botocore/bedrock_cassettes/amazon_invoke_model_arn.yaml b/tests/contrib/botocore/bedrock_cassettes/amazon_invoke_model_arn.yaml new file mode 100644 index 00000000000..cd2283c0ce7 --- /dev/null +++ b/tests/contrib/botocore/bedrock_cassettes/amazon_invoke_model_arn.yaml @@ -0,0 +1,52 @@ +interactions: +- request: + body: '{"inputText": "Command: can you explain what Datadog is to someone not + in the tech industry?", "textGenerationConfig": {"maxTokenCount": 50, "stopSequences": + [], "temperature": 0, "topP": 0.9}}' + headers: + Content-Length: + - '193' + User-Agent: + - !!binary | + Qm90bzMvMS4zNC40OSBtZC9Cb3RvY29yZSMxLjM0LjQ5IHVhLzIuMCBvcy9tYWNvcyMyNC4yLjAg + bWQvYXJjaCNhcm02NCBsYW5nL3B5dGhvbiMzLjEwLjUgbWQvcHlpbXBsI0NQeXRob24gY2ZnL3Jl + dHJ5LW1vZGUjbGVnYWN5IEJvdG9jb3JlLzEuMzQuNDk= + X-Amz-Date: + - !!binary | + MjAyNTAxMTRUMjIwNDAyWg== + amz-sdk-invocation-id: + - !!binary | + YjY5NGZlNDgtNDBmNy00YTJlLWI1YTgtYjRiZGVhZTU5MjQ0 + amz-sdk-request: + - !!binary | + YXR0ZW1wdD0x + method: POST + uri: https://bedrock-runtime.us-east-1.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-east-1%3A%3Afoundation-model%2Famazon.titan-tg1-large/invoke + response: + body: + string: '{"inputTextTokenCount":18,"results":[{"tokenCount":50,"outputText":"\n\nDatadog + is a monitoring and analytics platform for IT operations, DevOps, and software + development teams. It provides real-time monitoring of infrastructure, applications, + and services, allowing users to identify and resolve issues quickly. Datadog + collects","completionReason":"LENGTH"}]}' + headers: + Connection: + - keep-alive + Content-Length: + - '361' + Content-Type: + - application/json + Date: + - Tue, 14 Jan 2025 22:04:05 GMT + X-Amzn-Bedrock-Input-Token-Count: + - '18' + X-Amzn-Bedrock-Invocation-Latency: + - '2646' + X-Amzn-Bedrock-Output-Token-Count: + - '50' + x-amzn-RequestId: + - b2d0fd44-c29a-4cd4-a97a-6901a48f6264 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/contrib/botocore/test.py b/tests/contrib/botocore/test.py index 085dcbbc1e9..9e4f91cb61f 100644 --- a/tests/contrib/botocore/test.py +++ b/tests/contrib/botocore/test.py @@ -23,6 +23,7 @@ from ddtrace._trace._span_pointer import _SpanPointer from ddtrace._trace._span_pointer import _SpanPointerDirection +from ddtrace._trace.utils_botocore import span_tags from tests.utils import get_128_bit_trace_id_from_headers @@ -32,20 +33,20 @@ except ImportError: from moto import mock_kinesis as mock_firehose -from ddtrace import Pin from ddtrace import config from ddtrace.constants import ERROR_MSG from ddtrace.constants import ERROR_STACK from ddtrace.constants import ERROR_TYPE -from ddtrace.contrib.botocore.patch import patch -from ddtrace.contrib.botocore.patch import patch_submodules -from ddtrace.contrib.botocore.patch import unpatch +from ddtrace.contrib.internal.botocore.patch import patch +from ddtrace.contrib.internal.botocore.patch import patch_submodules +from ddtrace.contrib.internal.botocore.patch import unpatch from ddtrace.internal.compat import PYTHON_VERSION_INFO from ddtrace.internal.datastreams.processor import PROPAGATION_KEY_BASE_64 from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME from ddtrace.internal.utils.version import parse_version from ddtrace.propagation.http import HTTP_HEADER_PARENT_ID from ddtrace.propagation.http import HTTP_HEADER_TRACE_ID +from ddtrace.trace import Pin from tests.opentracer.utils import init_tracer from tests.utils import TracerTestCase from tests.utils import assert_is_measured @@ -104,6 +105,12 @@ def setUp(self): super(BotocoreTest, self).setUp() Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(botocore.parsers.ResponseParser) + # Setting the validated flag to False ensures the redaction paths configurations are re-validated + # FIXME: Ensure AWSPayloadTagging._REQUEST_REDACTION_PATHS_DEFAULTS is always in sync with + # config.botocore.payload_tagging_request + # FIXME: Ensure AWSPayloadTagging._RESPONSE_REDACTION_PATHS_DEFAULTS is always in sync with + # config.botocore.payload_tagging_response + span_tags._PAYLOAD_TAGGER.validated = False def tearDown(self): super(BotocoreTest, self).tearDown() @@ -3968,6 +3975,7 @@ def test_aws_payload_tagging_s3_invalid_config(self): with pytest.raises(Exception): s3.list_objects(bucket="mybucket") + @pytest.mark.skip(reason="broken during period of skipping on main branch") @pytest.mark.snapshot(ignores=snapshot_ignores) @mock_s3 def test_aws_payload_tagging_s3_valid_config(self): diff --git a/tests/contrib/botocore/test_bedrock.py b/tests/contrib/botocore/test_bedrock.py index 1001aff0dac..1cf5618bd0e 100644 --- a/tests/contrib/botocore/test_bedrock.py +++ b/tests/contrib/botocore/test_bedrock.py @@ -4,9 +4,9 @@ import mock import pytest -from ddtrace import Pin from ddtrace.contrib.internal.botocore.patch import patch from ddtrace.contrib.internal.botocore.patch import unpatch +from ddtrace.trace import Pin from tests.contrib.botocore.bedrock_utils import _MODELS from tests.contrib.botocore.bedrock_utils import _REQUEST_BODIES from tests.contrib.botocore.bedrock_utils import get_request_vcr @@ -222,6 +222,15 @@ def test_meta_invoke(bedrock_client, request_vcr): json.loads(response.get("body").read()) +@pytest.mark.snapshot +def test_invoke_model_using_aws_arn_model_id(bedrock_client, request_vcr): + body = json.dumps(_REQUEST_BODIES["amazon"]) + model = "arn:aws:bedrock:us-east-1::foundation-model/amazon.titan-tg1-large" + with request_vcr.use_cassette("amazon_invoke_model_arn.yaml"): + response = bedrock_client.invoke_model(body=body, modelId=model) + json.loads(response.get("body").read()) + + @pytest.mark.snapshot def test_amazon_invoke_stream(bedrock_client, request_vcr): body, model = json.dumps(_REQUEST_BODIES["amazon"]), _MODELS["amazon"] diff --git a/tests/contrib/botocore/test_bedrock_llmobs.py b/tests/contrib/botocore/test_bedrock_llmobs.py index ced2dc37265..790b86f0704 100644 --- a/tests/contrib/botocore/test_bedrock_llmobs.py +++ b/tests/contrib/botocore/test_bedrock_llmobs.py @@ -4,10 +4,10 @@ import mock import pytest -from ddtrace import Pin -from ddtrace.contrib.botocore.patch import patch -from ddtrace.contrib.botocore.patch import unpatch +from ddtrace.contrib.internal.botocore.patch import patch +from ddtrace.contrib.internal.botocore.patch import unpatch from ddtrace.llmobs import LLMObs +from ddtrace.trace import Pin from tests.contrib.botocore.bedrock_utils import _MODELS from tests.contrib.botocore.bedrock_utils import _REQUEST_BODIES from tests.contrib.botocore.bedrock_utils import get_request_vcr diff --git a/tests/contrib/botocore/test_botocore_patch.py b/tests/contrib/botocore/test_botocore_patch.py index abb24d96dc5..7c4e710b6c3 100644 --- a/tests/contrib/botocore/test_botocore_patch.py +++ b/tests/contrib/botocore/test_botocore_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.botocore import get_version -from ddtrace.contrib.botocore.patch import patch +from ddtrace.contrib.internal.botocore.patch import get_version +from ddtrace.contrib.internal.botocore.patch import patch try: - from ddtrace.contrib.botocore.patch import unpatch + from ddtrace.contrib.internal.botocore.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/botocore/test_stepfunctions.py b/tests/contrib/botocore/test_stepfunctions.py index aaf17eb6051..f350e967d4c 100644 --- a/tests/contrib/botocore/test_stepfunctions.py +++ b/tests/contrib/botocore/test_stepfunctions.py @@ -1,9 +1,9 @@ import json -from ddtrace import Pin from ddtrace.contrib.internal.botocore.services.stepfunctions import update_stepfunction_input from ddtrace.ext import SpanTypes from ddtrace.internal import core +from ddtrace.trace import Pin def test_update_stepfunction_input(): diff --git a/tests/contrib/bottle/test.py b/tests/contrib/bottle/test.py index 9f0323ed9a5..ab0f6db9c8d 100644 --- a/tests/contrib/bottle/test.py +++ b/tests/contrib/bottle/test.py @@ -3,7 +3,7 @@ import ddtrace from ddtrace import config -from ddtrace.contrib.bottle import TracePlugin +from ddtrace.contrib.internal.bottle.patch import TracePlugin from ddtrace.ext import http from ddtrace.internal import compat from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME diff --git a/tests/contrib/bottle/test_bottle_patch.py b/tests/contrib/bottle/test_bottle_patch.py index 1b255e1da2b..c7e1c0f2ee2 100644 --- a/tests/contrib/bottle/test_bottle_patch.py +++ b/tests/contrib/bottle/test_bottle_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.bottle import get_version -from ddtrace.contrib.bottle.patch import patch +from ddtrace.contrib.internal.bottle.patch import get_version +from ddtrace.contrib.internal.bottle.patch import patch try: - from ddtrace.contrib.bottle.patch import unpatch + from ddtrace.contrib.internal.bottle.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/cassandra/test.py b/tests/contrib/cassandra/test.py index b3395a61e6e..21b98d6396f 100644 --- a/tests/contrib/cassandra/test.py +++ b/tests/contrib/cassandra/test.py @@ -9,16 +9,16 @@ from cassandra.query import SimpleStatement import mock -from ddtrace import Pin from ddtrace import config from ddtrace.constants import ERROR_MSG from ddtrace.constants import ERROR_TYPE -from ddtrace.contrib.cassandra.patch import patch -from ddtrace.contrib.cassandra.patch import unpatch -from ddtrace.contrib.cassandra.session import SERVICE +from ddtrace.contrib.internal.cassandra.patch import patch +from ddtrace.contrib.internal.cassandra.patch import unpatch +from ddtrace.contrib.internal.cassandra.session import SERVICE from ddtrace.ext import cassandra as cassx from ddtrace.ext import net from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME +from ddtrace.trace import Pin from tests.contrib.config import CASSANDRA_CONFIG from tests.opentracer.utils import init_tracer from tests.utils import DummyTracer diff --git a/tests/contrib/cassandra/test_cassandra_patch.py b/tests/contrib/cassandra/test_cassandra_patch.py index b1da5f99262..19a09daccf4 100644 --- a/tests/contrib/cassandra/test_cassandra_patch.py +++ b/tests/contrib/cassandra/test_cassandra_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.cassandra import get_version -from ddtrace.contrib.cassandra.patch import patch +from ddtrace.contrib.internal.cassandra.patch import patch +from ddtrace.contrib.internal.cassandra.session import get_version try: - from ddtrace.contrib.cassandra.patch import unpatch + from ddtrace.contrib.internal.cassandra.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/celery/autopatch.py b/tests/contrib/celery/autopatch.py index 128ad9f8415..08eeed5acad 100644 --- a/tests/contrib/celery/autopatch.py +++ b/tests/contrib/celery/autopatch.py @@ -1,4 +1,4 @@ -from ddtrace import Pin +from ddtrace.trace import Pin if __name__ == "__main__": diff --git a/tests/contrib/celery/base.py b/tests/contrib/celery/base.py index 3a14f92c33f..c2b7de22a54 100644 --- a/tests/contrib/celery/base.py +++ b/tests/contrib/celery/base.py @@ -3,9 +3,9 @@ import celery import pytest -from ddtrace import Pin -from ddtrace.contrib.celery import patch -from ddtrace.contrib.celery import unpatch +from ddtrace.contrib.internal.celery.patch import patch +from ddtrace.contrib.internal.celery.patch import unpatch +from ddtrace.trace import Pin from tests.utils import TracerTestCase from ..config import RABBITMQ_CONFIG diff --git a/tests/contrib/celery/test_app.py b/tests/contrib/celery/test_app.py index ed26f00878d..6218d77f061 100644 --- a/tests/contrib/celery/test_app.py +++ b/tests/contrib/celery/test_app.py @@ -1,7 +1,7 @@ import celery -from ddtrace import Pin -from ddtrace.contrib.celery import unpatch_app +from ddtrace.contrib.internal.celery.patch import unpatch_app +from ddtrace.trace import Pin from .base import CeleryBaseTestCase diff --git a/tests/contrib/celery/test_integration.py b/tests/contrib/celery/test_integration.py index c3696f27307..717ed1de359 100644 --- a/tests/contrib/celery/test_integration.py +++ b/tests/contrib/celery/test_integration.py @@ -8,13 +8,13 @@ import mock import pytest -from ddtrace import Pin from ddtrace._trace.context import Context from ddtrace.constants import ERROR_MSG -from ddtrace.contrib.celery import patch -from ddtrace.contrib.celery import unpatch +from ddtrace.contrib.internal.celery.patch import patch +from ddtrace.contrib.internal.celery.patch import unpatch import ddtrace.internal.forksafe as forksafe from ddtrace.propagation.http import HTTPPropagator +from ddtrace.trace import Pin from tests.opentracer.utils import init_tracer from tests.utils import flaky diff --git a/tests/contrib/celery/test_patch.py b/tests/contrib/celery/test_patch.py index 28d34552551..3892fc79fb1 100644 --- a/tests/contrib/celery/test_patch.py +++ b/tests/contrib/celery/test_patch.py @@ -1,6 +1,6 @@ import unittest -from ddtrace import Pin +from ddtrace.trace import Pin from tests.contrib.patch import emit_integration_and_version_to_test_agent @@ -9,7 +9,7 @@ def test_patch_after_import(self): import celery from ddtrace import patch - from ddtrace.contrib.celery import unpatch + from ddtrace.contrib.internal.celery.patch import unpatch patch(celery=True) @@ -19,7 +19,7 @@ def test_patch_after_import(self): def test_patch_before_import(self): from ddtrace import patch - from ddtrace.contrib.celery import unpatch + from ddtrace.contrib.internal.celery.patch import unpatch patch(celery=True) import celery @@ -29,7 +29,7 @@ def test_patch_before_import(self): unpatch() def test_and_emit_get_version(self): - from ddtrace.contrib.celery import get_version + from ddtrace.contrib.internal.celery.patch import get_version version = get_version() assert type(version) == str diff --git a/tests/contrib/celery/test_tagging.py b/tests/contrib/celery/test_tagging.py index 2809364ba13..6b88acf9434 100644 --- a/tests/contrib/celery/test_tagging.py +++ b/tests/contrib/celery/test_tagging.py @@ -5,9 +5,9 @@ from celery.contrib.testing.worker import start_worker import pytest -from ddtrace import Pin -from ddtrace.contrib.celery import patch -from ddtrace.contrib.celery import unpatch +from ddtrace.contrib.internal.celery.patch import patch +from ddtrace.contrib.internal.celery.patch import unpatch +from ddtrace.trace import Pin from tests.utils import DummyTracer from .base import AMQP_BROKER_URL diff --git a/tests/contrib/celery/test_utils.py b/tests/contrib/celery/test_utils.py index 3bbeb959ce1..3c9e0f09d16 100644 --- a/tests/contrib/celery/test_utils.py +++ b/tests/contrib/celery/test_utils.py @@ -4,11 +4,11 @@ import pytest from ddtrace._trace.span import Span -from ddtrace.contrib.celery.utils import attach_span -from ddtrace.contrib.celery.utils import detach_span -from ddtrace.contrib.celery.utils import retrieve_span -from ddtrace.contrib.celery.utils import retrieve_task_id -from ddtrace.contrib.celery.utils import set_tags_from_context +from ddtrace.contrib.internal.celery.utils import attach_span +from ddtrace.contrib.internal.celery.utils import detach_span +from ddtrace.contrib.internal.celery.utils import retrieve_span +from ddtrace.contrib.internal.celery.utils import retrieve_task_id +from ddtrace.contrib.internal.celery.utils import set_tags_from_context @pytest.fixture diff --git a/tests/contrib/cherrypy/test_middleware.py b/tests/contrib/cherrypy/test_middleware.py index 0a524120de9..000f15610a0 100644 --- a/tests/contrib/cherrypy/test_middleware.py +++ b/tests/contrib/cherrypy/test_middleware.py @@ -15,7 +15,7 @@ from ddtrace.constants import ERROR_STACK from ddtrace.constants import ERROR_TYPE from ddtrace.constants import SAMPLING_PRIORITY_KEY -from ddtrace.contrib.cherrypy import TraceMiddleware +from ddtrace.contrib.internal.cherrypy.middleware import TraceMiddleware from ddtrace.ext import http from tests.contrib.patch import emit_integration_and_version_to_test_agent from tests.utils import TracerTestCase @@ -55,7 +55,7 @@ def setUp(self): ) def test_and_emit_get_version(self): - from ddtrace.contrib.cherrypy import get_version + from ddtrace.contrib.internal.cherrypy.middleware import get_version version = get_version() assert type(version) == str @@ -543,7 +543,7 @@ def test_service_name_schema(ddtrace_run_python_code_in_subprocess, schema_versi from cherrypy.test import helper from tests.utils import TracerTestCase from tests.contrib.cherrypy.web import StubApp -from ddtrace.contrib.cherrypy import TraceMiddleware +from ddtrace.contrib.internal.cherrypy.middleware import TraceMiddleware class TestCherrypy(TracerTestCase, helper.CPWebCase): @staticmethod def setup_server(): @@ -602,7 +602,7 @@ def test_operation_name_schema(ddtrace_run_python_code_in_subprocess, schema_ver from cherrypy.test import helper from tests.utils import TracerTestCase from tests.contrib.cherrypy.web import StubApp -from ddtrace.contrib.cherrypy import TraceMiddleware +from ddtrace.contrib.internal.cherrypy.middleware import TraceMiddleware class TestCherrypy(TracerTestCase, helper.CPWebCase): @staticmethod def setup_server(): diff --git a/tests/contrib/consul/test.py b/tests/contrib/consul/test.py index 3abcbeb9957..285287f9e95 100644 --- a/tests/contrib/consul/test.py +++ b/tests/contrib/consul/test.py @@ -1,11 +1,11 @@ import consul from wrapt import BoundFunctionWrapper -from ddtrace import Pin -from ddtrace.contrib.consul.patch import patch -from ddtrace.contrib.consul.patch import unpatch +from ddtrace.contrib.internal.consul.patch import patch +from ddtrace.contrib.internal.consul.patch import unpatch from ddtrace.ext import consul as consulx from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME +from ddtrace.trace import Pin from tests.utils import TracerTestCase from tests.utils import assert_is_measured diff --git a/tests/contrib/consul/test_consul_patch.py b/tests/contrib/consul/test_consul_patch.py index 3ba449356e8..6435e95ed4c 100644 --- a/tests/contrib/consul/test_consul_patch.py +++ b/tests/contrib/consul/test_consul_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.consul import get_version -from ddtrace.contrib.consul.patch import patch +from ddtrace.contrib.internal.consul.patch import get_version +from ddtrace.contrib.internal.consul.patch import patch try: - from ddtrace.contrib.consul.patch import unpatch + from ddtrace.contrib.internal.consul.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/dbapi/test_dbapi.py b/tests/contrib/dbapi/test_dbapi.py index 80135734a6a..1f6be1d66f5 100644 --- a/tests/contrib/dbapi/test_dbapi.py +++ b/tests/contrib/dbapi/test_dbapi.py @@ -1,7 +1,6 @@ import mock import pytest -from ddtrace import Pin from ddtrace._trace.span import Span # noqa:F401 from ddtrace.contrib.dbapi import FetchTracedCursor from ddtrace.contrib.dbapi import TracedConnection @@ -9,6 +8,7 @@ from ddtrace.propagation._database_monitoring import _DBM_Propagator from ddtrace.settings import Config from ddtrace.settings.integration import IntegrationConfig +from ddtrace.trace import Pin from tests.utils import TracerTestCase from tests.utils import assert_is_measured from tests.utils import assert_is_not_measured diff --git a/tests/contrib/dbapi/test_dbapi_appsec.py b/tests/contrib/dbapi/test_dbapi_appsec.py index b60b3ac05c0..d43d9c37e3c 100644 --- a/tests/contrib/dbapi/test_dbapi_appsec.py +++ b/tests/contrib/dbapi/test_dbapi_appsec.py @@ -1,12 +1,12 @@ import mock import pytest -from ddtrace import Pin from ddtrace.appsec._iast import oce from ddtrace.appsec._iast._utils import _is_python_version_supported from ddtrace.contrib.dbapi import TracedCursor from ddtrace.settings import Config from ddtrace.settings.integration import IntegrationConfig +from ddtrace.trace import Pin from tests.appsec.iast.conftest import _end_iast_context_and_oce from tests.appsec.iast.conftest import _start_iast_context_and_oce from tests.utils import TracerTestCase diff --git a/tests/contrib/dbapi_async/test_dbapi_async.py b/tests/contrib/dbapi_async/test_dbapi_async.py index f7151fe1390..7343e875829 100644 --- a/tests/contrib/dbapi_async/test_dbapi_async.py +++ b/tests/contrib/dbapi_async/test_dbapi_async.py @@ -1,7 +1,6 @@ import mock import pytest -from ddtrace import Pin from ddtrace._trace.span import Span # noqa:F401 from ddtrace.contrib.dbapi_async import FetchTracedAsyncCursor from ddtrace.contrib.dbapi_async import TracedAsyncConnection @@ -9,6 +8,7 @@ from ddtrace.propagation._database_monitoring import _DBM_Propagator from ddtrace.settings import Config from ddtrace.settings.integration import IntegrationConfig +from ddtrace.trace import Pin from tests.contrib.asyncio.utils import AsyncioTestCase from tests.contrib.asyncio.utils import mark_asyncio from tests.utils import assert_is_measured diff --git a/tests/contrib/django/asgi.py b/tests/contrib/django/asgi.py index 96627e7e34c..602246bacab 100644 --- a/tests/contrib/django/asgi.py +++ b/tests/contrib/django/asgi.py @@ -4,7 +4,7 @@ from django.core.asgi import get_asgi_application from django.urls import re_path -from ddtrace.contrib.asgi import TraceMiddleware +from ddtrace.contrib.internal.asgi.middleware import TraceMiddleware application = get_asgi_application() diff --git a/tests/contrib/django/conftest.py b/tests/contrib/django/conftest.py index 63190d7d566..3dd992681b4 100644 --- a/tests/contrib/django/conftest.py +++ b/tests/contrib/django/conftest.py @@ -4,8 +4,8 @@ from django.conf import settings import pytest -from ddtrace import Pin -from ddtrace.contrib.django import patch +from ddtrace.contrib.internal.django.patch import patch +from ddtrace.trace import Pin from tests.utils import DummyTracer from tests.utils import TracerSpanContainer diff --git a/tests/contrib/django/django_app/urls.py b/tests/contrib/django/django_app/urls.py index caa7b33653f..523a250d0c8 100644 --- a/tests/contrib/django/django_app/urls.py +++ b/tests/contrib/django/django_app/urls.py @@ -78,7 +78,6 @@ def shutdown(request): re_path(r"re-path.*/", repath_view), path("path/", path_view), path("include/", include("tests.contrib.django.django_app.extra_urls")), - path("appsec/", include("tests.contrib.django.django_app.appsec_urls")), # This must precede composed-view. handler(r"^some-static-view/$", TemplateView.as_view(template_name="my-template.html")), handler(r"^composed-template-view/$", views.ComposedTemplateView.as_view(), name="composed-template-view"), diff --git a/tests/contrib/django/test_django.py b/tests/contrib/django/test_django.py index 1bd223539c9..7b8a0e18ef7 100644 --- a/tests/contrib/django/test_django.py +++ b/tests/contrib/django/test_django.py @@ -24,9 +24,9 @@ from ddtrace.constants import SAMPLING_PRIORITY_KEY from ddtrace.constants import USER_KEEP from ddtrace.contrib import trace_utils -from ddtrace.contrib.django.patch import instrument_view -from ddtrace.contrib.django.patch import traced_get_response -from ddtrace.contrib.django.utils import get_request_uri +from ddtrace.contrib.internal.django.patch import instrument_view +from ddtrace.contrib.internal.django.patch import traced_get_response +from ddtrace.contrib.internal.django.utils import get_request_uri from ddtrace.ext import http from ddtrace.ext import user from ddtrace.internal.compat import ensure_text @@ -1888,7 +1888,8 @@ def test_collecting_requests_handles_improperly_configured_error(client, test_sp """ # patch django._patch - django.__init__.py imports patch.py module as _patch with mock.patch( - "ddtrace.contrib.django.utils.user_is_authenticated", side_effect=django.core.exceptions.ImproperlyConfigured + "ddtrace.contrib.internal.django.utils.user_is_authenticated", + side_effect=django.core.exceptions.ImproperlyConfigured, ): # If ImproperlyConfigured error bubbles up, should automatically fail the test. resp = client.get("/") diff --git a/tests/contrib/django/test_django_dbm.py b/tests/contrib/django/test_django_dbm.py index 00edf1c0815..d44f90f3208 100644 --- a/tests/contrib/django/test_django_dbm.py +++ b/tests/contrib/django/test_django_dbm.py @@ -1,7 +1,7 @@ from django.db import connections import mock -from ddtrace import Pin +from ddtrace.trace import Pin from tests.contrib import shared_tests from tests.utils import DummyTracer from tests.utils import override_config diff --git a/tests/contrib/django/test_django_patch.py b/tests/contrib/django/test_django_patch.py index cec30ac004f..41e1de6b3e7 100644 --- a/tests/contrib/django/test_django_patch.py +++ b/tests/contrib/django/test_django_patch.py @@ -1,5 +1,5 @@ -from ddtrace.contrib.django import get_version -from ddtrace.contrib.django import patch +from ddtrace.contrib.internal.django.patch import get_version +from ddtrace.contrib.internal.django.patch import patch from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/django/test_django_snapshots.py b/tests/contrib/django/test_django_snapshots.py index d7402e37083..547095c01f9 100644 --- a/tests/contrib/django/test_django_snapshots.py +++ b/tests/contrib/django/test_django_snapshots.py @@ -156,8 +156,8 @@ def test_404_exceptions(client): def psycopg2_patched(transactional_db): from django.db import connections - from ddtrace.contrib.psycopg.patch import patch - from ddtrace.contrib.psycopg.patch import unpatch + from ddtrace.contrib.internal.psycopg.patch import patch + from ddtrace.contrib.internal.psycopg.patch import unpatch patch() @@ -204,8 +204,8 @@ def psycopg3_patched(transactional_db): else: from django.db import connections - from ddtrace.contrib.psycopg.patch import patch - from ddtrace.contrib.psycopg.patch import unpatch + from ddtrace.contrib.internal.psycopg.patch import patch + from ddtrace.contrib.internal.psycopg.patch import unpatch patch() diff --git a/tests/contrib/django/test_django_utils.py b/tests/contrib/django/test_django_utils.py index b6e4b582fb2..6d1f2b9711e 100644 --- a/tests/contrib/django/test_django_utils.py +++ b/tests/contrib/django/test_django_utils.py @@ -1,7 +1,7 @@ from django.test.client import RequestFactory import pytest -from ddtrace.contrib.django.utils import DJANGO22 +from ddtrace.contrib.internal.django.utils import DJANGO22 from ddtrace.contrib.internal.django.utils import _get_request_headers diff --git a/tests/contrib/django/test_django_wsgi.py b/tests/contrib/django/test_django_wsgi.py index edc065377d8..1891e5cea0e 100644 --- a/tests/contrib/django/test_django_wsgi.py +++ b/tests/contrib/django/test_django_wsgi.py @@ -10,7 +10,7 @@ from django.urls import path import pytest -from ddtrace.contrib.wsgi import DDWSGIMiddleware +from ddtrace.contrib.internal.wsgi.wsgi import DDWSGIMiddleware from ddtrace.internal.compat import PYTHON_VERSION_INFO from tests.contrib.django.utils import make_soap_request from tests.webclient import Client diff --git a/tests/contrib/django_hosts/conftest.py b/tests/contrib/django_hosts/conftest.py index 9406a29a66e..a4b18a51b67 100644 --- a/tests/contrib/django_hosts/conftest.py +++ b/tests/contrib/django_hosts/conftest.py @@ -3,7 +3,7 @@ import django from django.conf import settings -from ddtrace.contrib.django import patch +from ddtrace.contrib.internal.django.patch import patch # We manually designate which settings we will be using in an environment variable diff --git a/tests/contrib/djangorestframework/conftest.py b/tests/contrib/djangorestframework/conftest.py index efffd1b090a..1b2cc1ab4ad 100644 --- a/tests/contrib/djangorestframework/conftest.py +++ b/tests/contrib/djangorestframework/conftest.py @@ -5,7 +5,7 @@ import django from django.conf import settings -from ddtrace.contrib.django import patch +from ddtrace.contrib.internal.django.patch import patch from ..django.conftest import test_spans from ..django.conftest import tracer diff --git a/tests/contrib/dogpile_cache/test_dogpile_cache_patch.py b/tests/contrib/dogpile_cache/test_dogpile_cache_patch.py index 9c4de2a1985..0cb74993254 100644 --- a/tests/contrib/dogpile_cache/test_dogpile_cache_patch.py +++ b/tests/contrib/dogpile_cache/test_dogpile_cache_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.dogpile_cache import get_version -from ddtrace.contrib.dogpile_cache.patch import patch +from ddtrace.contrib.internal.dogpile_cache.patch import get_version +from ddtrace.contrib.internal.dogpile_cache.patch import patch try: - from ddtrace.contrib.dogpile_cache.patch import unpatch + from ddtrace.contrib.internal.dogpile_cache.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/dogpile_cache/test_tracing.py b/tests/contrib/dogpile_cache/test_tracing.py index fbf10ab4a83..fec78818eda 100644 --- a/tests/contrib/dogpile_cache/test_tracing.py +++ b/tests/contrib/dogpile_cache/test_tracing.py @@ -3,9 +3,9 @@ import dogpile import pytest -from ddtrace import Pin -from ddtrace.contrib.dogpile_cache.patch import patch -from ddtrace.contrib.dogpile_cache.patch import unpatch +from ddtrace.contrib.internal.dogpile_cache.patch import patch +from ddtrace.contrib.internal.dogpile_cache.patch import unpatch +from ddtrace.trace import Pin from tests.conftest import DEFAULT_DDTRACE_SUBPROCESS_TEST_SERVICE_NAME from tests.utils import DummyTracer from tests.utils import TracerSpanContainer diff --git a/tests/contrib/dramatiq/test_integration.py b/tests/contrib/dramatiq/test_integration.py index 2897cc7edcf..ba8d836181c 100644 --- a/tests/contrib/dramatiq/test_integration.py +++ b/tests/contrib/dramatiq/test_integration.py @@ -3,9 +3,9 @@ import dramatiq import pytest -from ddtrace.contrib.dramatiq import patch -from ddtrace.contrib.dramatiq import unpatch -from ddtrace.pin import Pin +from ddtrace.contrib.internal.dramatiq.patch import patch +from ddtrace.contrib.internal.dramatiq.patch import unpatch +from ddtrace.trace import Pin from tests.utils import DummyTracer from tests.utils import snapshot diff --git a/tests/contrib/dramatiq/test_patch_manual.py b/tests/contrib/dramatiq/test_patch_manual.py index d7e8619c390..67bbbe9e7ee 100644 --- a/tests/contrib/dramatiq/test_patch_manual.py +++ b/tests/contrib/dramatiq/test_patch_manual.py @@ -6,7 +6,7 @@ class DramatiqPatchTest(unittest.TestCase): def test_patch_before_import(self): from ddtrace import patch - from ddtrace.contrib.dramatiq import unpatch + from ddtrace.contrib.internal.dramatiq.patch import unpatch # Patch dramatiq before dramatiq imports patch(dramatiq=True) @@ -36,7 +36,7 @@ def test_patch_after_import(self): from dramatiq.brokers.stub import StubBroker from ddtrace import patch - from ddtrace.contrib.dramatiq import unpatch + from ddtrace.contrib.internal.dramatiq.patch import unpatch # Patch after all dramatiq imports patch(dramatiq=True) diff --git a/tests/contrib/elasticsearch/test_elasticsearch.py b/tests/contrib/elasticsearch/test_elasticsearch.py index b80b4486e71..6e381bc1e31 100644 --- a/tests/contrib/elasticsearch/test_elasticsearch.py +++ b/tests/contrib/elasticsearch/test_elasticsearch.py @@ -1,18 +1,19 @@ import datetime from http.client import HTTPConnection from importlib import import_module +import json import time import pytest -from ddtrace import Pin from ddtrace import config -from ddtrace.contrib.elasticsearch.patch import get_version -from ddtrace.contrib.elasticsearch.patch import get_versions -from ddtrace.contrib.elasticsearch.patch import patch -from ddtrace.contrib.elasticsearch.patch import unpatch +from ddtrace.contrib.internal.elasticsearch.patch import get_version +from ddtrace.contrib.internal.elasticsearch.patch import get_versions +from ddtrace.contrib.internal.elasticsearch.patch import patch +from ddtrace.contrib.internal.elasticsearch.patch import unpatch from ddtrace.ext import http from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME +from ddtrace.trace import Pin from tests.contrib.patch import emit_integration_and_version_to_test_agent from tests.utils import TracerTestCase @@ -167,7 +168,12 @@ def test_elasticsearch(self): es.index(id=10, body={"name": "ten", "created": datetime.date(2016, 1, 1)}, **args) es.index(id=11, body={"name": "eleven", "created": datetime.date(2016, 2, 1)}, **args) es.index(id=12, body={"name": "twelve", "created": datetime.date(2016, 3, 1)}, **args) - result = es.search(sort=["name:desc"], size=100, body={"query": {"match_all": {}}}, **args) + result = es.search( + sort={"name": {"order": "desc", "unmapped_type": "keyword"}}, + size=100, + body={"query": {"match_all": {}}}, + **args, + ) assert len(result["hits"]["hits"]) == 3, result spans = self.get_spans() @@ -183,13 +189,25 @@ def test_elasticsearch(self): assert url.endswith("/_search") assert url == span.get_tag("elasticsearch.url") if elasticsearch.__version__ >= (8, 0, 0): - assert span.get_tag("elasticsearch.body").replace(" ", "") == '{"query":{"match_all":{}},"size":100}' - assert set(span.get_tag("elasticsearch.params").split("&")) == {"sort=name%3Adesc"} - assert set(span.get_tag(http.QUERY_STRING).split("&")) == {"sort=name%3Adesc"} + # Key order is not consistent, parse into dict to compare + body = json.loads(span.get_tag("elasticsearch.body")) + assert body == { + "query": {"match_all": {}}, + "sort": {"name": {"order": "desc", "unmapped_type": "keyword"}}, + "size": 100, + } + assert not span.get_tag("elasticsearch.params") + assert not span.get_tag(http.QUERY_STRING) else: assert span.get_tag("elasticsearch.body").replace(" ", "") == '{"query":{"match_all":{}}}' - assert set(span.get_tag("elasticsearch.params").split("&")) == {"sort=name%3Adesc", "size=100"} - assert set(span.get_tag(http.QUERY_STRING).split("&")) == {"sort=name%3Adesc", "size=100"} + assert set(span.get_tag("elasticsearch.params").split("&")) == { + "sort=%7B%27name%27%3A+%7B%27order%27%3A+%27desc%27%2C+%27unmapped_type%27%3A+%27keyword%27%7D%7D", + "size=100", + } + assert set(span.get_tag(http.QUERY_STRING).split("&")) == { + "sort=%7B%27name%27%3A+%7B%27order%27%3A+%27desc%27%2C+%27unmapped_type%27%3A+%27keyword%27%7D%7D", + "size=100", + } assert span.get_tag("component") == "elasticsearch" assert span.get_tag("span.kind") == "client" diff --git a/tests/contrib/falcon/app/app.py b/tests/contrib/falcon/app/app.py index a89de18fbfd..d3f17900485 100644 --- a/tests/contrib/falcon/app/app.py +++ b/tests/contrib/falcon/app/app.py @@ -1,7 +1,7 @@ import falcon -from ddtrace.contrib.falcon import TraceMiddleware -from ddtrace.contrib.falcon.patch import FALCON_VERSION +from ddtrace.contrib.internal.falcon.middleware import TraceMiddleware +from ddtrace.contrib.internal.falcon.patch import FALCON_VERSION from . import resources diff --git a/tests/contrib/falcon/test_distributed_tracing.py b/tests/contrib/falcon/test_distributed_tracing.py index c789118ccce..c9c545430aa 100644 --- a/tests/contrib/falcon/test_distributed_tracing.py +++ b/tests/contrib/falcon/test_distributed_tracing.py @@ -1,7 +1,7 @@ from falcon import testing from ddtrace import config -from ddtrace.contrib.falcon.patch import FALCON_VERSION +from ddtrace.contrib.internal.falcon.patch import FALCON_VERSION from tests.utils import DummyTracer from tests.utils import TracerTestCase diff --git a/tests/contrib/falcon/test_falcon_patch.py b/tests/contrib/falcon/test_falcon_patch.py index 013c5abc333..b6ddfa80a4d 100644 --- a/tests/contrib/falcon/test_falcon_patch.py +++ b/tests/contrib/falcon/test_falcon_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.falcon import get_version -from ddtrace.contrib.falcon.patch import patch +from ddtrace.contrib.internal.falcon.patch import get_version +from ddtrace.contrib.internal.falcon.patch import patch try: - from ddtrace.contrib.falcon.patch import unpatch + from ddtrace.contrib.internal.falcon.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/falcon/test_middleware.py b/tests/contrib/falcon/test_middleware.py index b4153994924..466e3cdd2db 100644 --- a/tests/contrib/falcon/test_middleware.py +++ b/tests/contrib/falcon/test_middleware.py @@ -1,6 +1,6 @@ from falcon import testing -from ddtrace.contrib.falcon.patch import FALCON_VERSION +from ddtrace.contrib.internal.falcon.patch import FALCON_VERSION from tests.utils import TracerTestCase from .app import get_app diff --git a/tests/contrib/falcon/test_schematization.py b/tests/contrib/falcon/test_schematization.py index 5cfd4e71d2e..438433b5dac 100644 --- a/tests/contrib/falcon/test_schematization.py +++ b/tests/contrib/falcon/test_schematization.py @@ -16,7 +16,7 @@ def test_schematized_service_name(ddtrace_run_python_code_in_subprocess, schema_ import sys from falcon import testing -from ddtrace.contrib.falcon.patch import FALCON_VERSION +from ddtrace.contrib.internal.falcon.patch import FALCON_VERSION from tests.contrib.falcon.test_suite import FalconTestMixin from tests.utils import TracerTestCase @@ -64,7 +64,7 @@ def test_schematized_operation_name(ddtrace_run_python_code_in_subprocess, schem import sys from falcon import testing -from ddtrace.contrib.falcon.patch import FALCON_VERSION +from ddtrace.contrib.internal.falcon.patch import FALCON_VERSION from tests.contrib.falcon.test_suite import FalconTestMixin from tests.utils import TracerTestCase diff --git a/tests/contrib/falcon/test_suite.py b/tests/contrib/falcon/test_suite.py index ecec0b09a1e..cdf26d401ac 100644 --- a/tests/contrib/falcon/test_suite.py +++ b/tests/contrib/falcon/test_suite.py @@ -1,6 +1,6 @@ from ddtrace import config from ddtrace.constants import ERROR_TYPE -from ddtrace.contrib.falcon.patch import FALCON_VERSION +from ddtrace.contrib.internal.falcon.patch import FALCON_VERSION from ddtrace.ext import http as httpx from tests.opentracer.utils import init_tracer from tests.utils import assert_is_measured diff --git a/tests/contrib/fastapi/conftest.py b/tests/contrib/fastapi/conftest.py index 821ef269414..e47a6a3208c 100644 --- a/tests/contrib/fastapi/conftest.py +++ b/tests/contrib/fastapi/conftest.py @@ -2,8 +2,8 @@ import pytest import ddtrace -from ddtrace.contrib.fastapi import patch as fastapi_patch -from ddtrace.contrib.fastapi import unpatch as fastapi_unpatch +from ddtrace.contrib.internal.fastapi.patch import patch as fastapi_patch +from ddtrace.contrib.internal.fastapi.patch import unpatch as fastapi_unpatch from tests.utils import DummyTracer from tests.utils import TracerSpanContainer diff --git a/tests/contrib/fastapi/test_fastapi.py b/tests/contrib/fastapi/test_fastapi.py index f4adc720f8b..7619a35f043 100644 --- a/tests/contrib/fastapi/test_fastapi.py +++ b/tests/contrib/fastapi/test_fastapi.py @@ -6,8 +6,8 @@ import httpx import pytest -from ddtrace.contrib.starlette.patch import patch as patch_starlette -from ddtrace.contrib.starlette.patch import unpatch as unpatch_starlette +from ddtrace.contrib.internal.starlette.patch import patch as patch_starlette +from ddtrace.contrib.internal.starlette.patch import unpatch as unpatch_starlette from ddtrace.internal.utils.version import parse_version from ddtrace.propagation import http as http_propagation from tests.conftest import DEFAULT_DDTRACE_SUBPROCESS_TEST_SERVICE_NAME diff --git a/tests/contrib/fastapi/test_fastapi_appsec_iast.py b/tests/contrib/fastapi/test_fastapi_appsec_iast.py index dd3050771c6..19502912b8d 100644 --- a/tests/contrib/fastapi/test_fastapi_appsec_iast.py +++ b/tests/contrib/fastapi/test_fastapi_appsec_iast.py @@ -25,7 +25,7 @@ from ddtrace.appsec._iast.constants import VULN_NO_SAMESITE_COOKIE from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION from ddtrace.contrib.internal.fastapi.patch import patch as patch_fastapi -from ddtrace.contrib.sqlite3.patch import patch as patch_sqlite_sqli +from ddtrace.contrib.internal.sqlite3.patch import patch as patch_sqlite_sqli from tests.appsec.iast.iast_utils import get_line_and_hash from tests.utils import override_env from tests.utils import override_global_config @@ -34,6 +34,8 @@ TEST_FILE_PATH = "tests/contrib/fastapi/test_fastapi_appsec_iast.py" fastapi_version = tuple([int(v) for v in _fastapi_version.split(".")]) +if sys.version_info > (3, 12): + pytest.skip(reason="IAST only supports Py3.12 and older", allow_module_level=True) def _aux_appsec_prepare_tracer(tracer): diff --git a/tests/contrib/fastapi/test_fastapi_patch.py b/tests/contrib/fastapi/test_fastapi_patch.py index 62cf0ab589e..77ba91a755f 100644 --- a/tests/contrib/fastapi/test_fastapi_patch.py +++ b/tests/contrib/fastapi/test_fastapi_patch.py @@ -1,6 +1,6 @@ -from ddtrace.contrib.fastapi import get_version -from ddtrace.contrib.fastapi import patch -from ddtrace.contrib.fastapi import unpatch +from ddtrace.contrib.internal.fastapi.patch import get_version +from ddtrace.contrib.internal.fastapi.patch import patch +from ddtrace.contrib.internal.fastapi.patch import unpatch from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/flask/__init__.py b/tests/contrib/flask/__init__.py index bbec059ec0f..a512a79f196 100644 --- a/tests/contrib/flask/__init__.py +++ b/tests/contrib/flask/__init__.py @@ -2,9 +2,9 @@ from flask.testing import FlaskClient import wrapt -from ddtrace import Pin -from ddtrace.contrib.flask import patch -from ddtrace.contrib.flask import unpatch +from ddtrace.contrib.internal.flask.patch import patch +from ddtrace.contrib.internal.flask.patch import unpatch +from ddtrace.trace import Pin from tests.utils import TracerTestCase diff --git a/tests/contrib/flask/test_appsec_flask_snapshot.py b/tests/contrib/flask/test_appsec_flask_snapshot.py index 29629c69834..47465883395 100644 --- a/tests/contrib/flask/test_appsec_flask_snapshot.py +++ b/tests/contrib/flask/test_appsec_flask_snapshot.py @@ -11,7 +11,7 @@ import pytest from ddtrace.appsec._constants import APPSEC -from ddtrace.contrib.flask.patch import flask_version +from ddtrace.contrib.internal.flask.patch import flask_version from ddtrace.internal.constants import BLOCKED_RESPONSE_HTML from ddtrace.internal.constants import BLOCKED_RESPONSE_JSON from ddtrace.internal.utils.retry import RetryError diff --git a/tests/contrib/flask/test_blueprint.py b/tests/contrib/flask/test_blueprint.py index 93b6c2d7ffc..96401dfa1a9 100644 --- a/tests/contrib/flask/test_blueprint.py +++ b/tests/contrib/flask/test_blueprint.py @@ -1,7 +1,7 @@ import flask -from ddtrace import Pin -from ddtrace.contrib.flask import unpatch +from ddtrace.contrib.internal.flask.patch import unpatch +from ddtrace.trace import Pin from . import BaseFlaskTestCase diff --git a/tests/contrib/flask/test_flask_appsec.py b/tests/contrib/flask/test_flask_appsec.py index 7fd045c61f2..642585fc035 100644 --- a/tests/contrib/flask/test_flask_appsec.py +++ b/tests/contrib/flask/test_flask_appsec.py @@ -2,7 +2,7 @@ from ddtrace.appsec._constants import SPAN_DATA_NAMES from ddtrace.appsec._trace_utils import block_request_if_user_blocked -from ddtrace.contrib.sqlite3.patch import patch +from ddtrace.contrib.internal.sqlite3.patch import patch from ddtrace.ext import http from ddtrace.internal import constants import tests.appsec.rules as rules diff --git a/tests/contrib/flask/test_flask_appsec_iast.py b/tests/contrib/flask/test_flask_appsec_iast.py index 94948bde0b2..c49dca7c29f 100644 --- a/tests/contrib/flask/test_flask_appsec_iast.py +++ b/tests/contrib/flask/test_flask_appsec_iast.py @@ -16,7 +16,7 @@ from ddtrace.appsec._iast.constants import VULN_NO_SAMESITE_COOKIE from ddtrace.appsec._iast.constants import VULN_SQL_INJECTION from ddtrace.appsec._iast.taint_sinks.header_injection import patch as patch_header_injection -from ddtrace.contrib.sqlite3.patch import patch as patch_sqlite_sqli +from ddtrace.contrib.internal.sqlite3.patch import patch as patch_sqlite_sqli from tests.appsec.iast.iast_utils import get_line_and_hash from tests.contrib.flask import BaseFlaskTestCase from tests.utils import override_env @@ -72,6 +72,7 @@ def sqli_1(param_str): dict( _iast_enabled=True, _deduplication_enabled=False, + _iast_request_sampling=100.0, ) ): resp = self.client.post("/sqli/sqlite_master/", data={"name": "test"}) @@ -525,6 +526,122 @@ def sqli_9(): assert vulnerability["location"]["path"] == TEST_FILE_PATH assert vulnerability["hash"] == hash_value + @pytest.mark.skipif(not python_supported_by_iast(), reason="Python version not supported by IAST") + def test_flask_full_sqli_iast_http_request_parameter_name_post(self): + @self.app.route("/sqli/", methods=["POST"]) + def sqli_13(): + import sqlite3 + + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect + + for i in request.form.keys(): + assert is_pyobject_tainted(i) + + first_param = list(request.form.keys())[0] + + con = sqlite3.connect(":memory:") + cur = con.cursor() + # label test_flask_full_sqli_iast_http_request_parameter_name_post + cur.execute(add_aspect("SELECT 1 FROM ", first_param)) + + return "OK", 200 + + with override_global_config( + dict( + _iast_enabled=True, + _deduplication_enabled=False, + _iast_request_sampling=100.0, + ) + ): + resp = self.client.post("/sqli/", data={"sqlite_master": "unused"}) + assert resp.status_code == 200 + + root_span = self.pop_spans()[0] + assert root_span.get_metric(IAST.ENABLED) == 1.0 + + loaded = json.loads(root_span.get_tag(IAST.JSON)) + assert loaded["sources"] == [ + {"origin": "http.request.parameter.name", "name": "sqlite_master", "value": "sqlite_master"} + ] + + line, hash_value = get_line_and_hash( + "test_flask_full_sqli_iast_http_request_parameter_name_post", + VULN_SQL_INJECTION, + filename=TEST_FILE_PATH, + ) + vulnerability = loaded["vulnerabilities"][0] + assert vulnerability["type"] == VULN_SQL_INJECTION + assert vulnerability["evidence"] == { + "valueParts": [ + {"value": "SELECT "}, + {"redacted": True}, + {"value": " FROM "}, + {"value": "sqlite_master", "source": 0}, + ] + } + assert vulnerability["location"]["line"] == line + assert vulnerability["location"]["path"] == TEST_FILE_PATH + assert vulnerability["hash"] == hash_value + + @pytest.mark.skipif(not python_supported_by_iast(), reason="Python version not supported by IAST") + def test_flask_full_sqli_iast_http_request_parameter_name_get(self): + @self.app.route("/sqli/", methods=["GET"]) + def sqli_14(): + import sqlite3 + + from ddtrace.appsec._iast._taint_tracking._taint_objects import is_pyobject_tainted + from ddtrace.appsec._iast._taint_tracking.aspects import add_aspect + + for i in request.args.keys(): + assert is_pyobject_tainted(i) + + first_param = list(request.args.keys())[0] + + con = sqlite3.connect(":memory:") + cur = con.cursor() + # label test_flask_full_sqli_iast_http_request_parameter_name_get + cur.execute(add_aspect("SELECT 1 FROM ", first_param)) + + return "OK", 200 + + with override_global_config( + dict( + _iast_enabled=True, + _deduplication_enabled=False, + _iast_request_sampling=100.0, + ) + ): + resp = self.client.get("/sqli/", query_string={"sqlite_master": "unused"}) + assert resp.status_code == 200 + + root_span = self.pop_spans()[0] + assert root_span.get_metric(IAST.ENABLED) == 1.0 + + loaded = json.loads(root_span.get_tag(IAST.JSON)) + assert loaded["sources"] == [ + {"origin": "http.request.parameter.name", "name": "sqlite_master", "value": "sqlite_master"} + ] + + line, hash_value = get_line_and_hash( + "test_flask_full_sqli_iast_http_request_parameter_name_get", + VULN_SQL_INJECTION, + filename=TEST_FILE_PATH, + ) + vulnerability = loaded["vulnerabilities"][0] + assert vulnerability["type"] == VULN_SQL_INJECTION + assert vulnerability["evidence"] == { + "valueParts": [ + {"value": "SELECT "}, + {"redacted": True}, + {"value": " FROM "}, + {"value": "sqlite_master", "source": 0}, + ] + } + assert vulnerability["location"]["line"] == line + assert vulnerability["location"]["path"] == TEST_FILE_PATH + assert vulnerability["hash"] == hash_value + @pytest.mark.skipif(not python_supported_by_iast(), reason="Python version not supported by IAST") def test_flask_request_body(self): @self.app.route("/sqli/body/", methods=("POST",)) diff --git a/tests/contrib/flask/test_flask_helpers.py b/tests/contrib/flask/test_flask_helpers.py index 73f6fb1bf98..d3672213a0a 100644 --- a/tests/contrib/flask/test_flask_helpers.py +++ b/tests/contrib/flask/test_flask_helpers.py @@ -2,10 +2,10 @@ import flask -from ddtrace import Pin -from ddtrace.contrib.flask import unpatch -from ddtrace.contrib.flask.patch import flask_version +from ddtrace.contrib.internal.flask.patch import flask_version +from ddtrace.contrib.internal.flask.patch import unpatch from ddtrace.internal.compat import StringIO +from ddtrace.trace import Pin from . import BaseFlaskTestCase diff --git a/tests/contrib/flask/test_flask_patch.py b/tests/contrib/flask/test_flask_patch.py index af738988991..7495caed35f 100644 --- a/tests/contrib/flask/test_flask_patch.py +++ b/tests/contrib/flask/test_flask_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.flask import get_version -from ddtrace.contrib.flask.patch import patch +from ddtrace.contrib.internal.flask.patch import get_version +from ddtrace.contrib.internal.flask.patch import patch try: - from ddtrace.contrib.flask.patch import unpatch + from ddtrace.contrib.internal.flask.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/flask/test_flask_snapshot.py b/tests/contrib/flask/test_flask_snapshot.py index e75e2e81bdb..d51014b9538 100644 --- a/tests/contrib/flask/test_flask_snapshot.py +++ b/tests/contrib/flask/test_flask_snapshot.py @@ -9,7 +9,7 @@ import pytest -from ddtrace.contrib.flask.patch import flask_version +from ddtrace.contrib.internal.flask.patch import flask_version from ddtrace.internal.utils.retry import RetryError from tests.utils import flaky from tests.webclient import Client diff --git a/tests/contrib/flask/test_hooks.py b/tests/contrib/flask/test_hooks.py index cdf237f0922..9a3af9cf17c 100644 --- a/tests/contrib/flask/test_hooks.py +++ b/tests/contrib/flask/test_hooks.py @@ -1,7 +1,7 @@ from flask import Blueprint import pytest -from ddtrace.contrib.flask.patch import flask_version +from ddtrace.contrib.internal.flask.patch import flask_version from ddtrace.ext import http from tests.contrib.flask.test_errorhandler import EXPECTED_METADATA from tests.utils import assert_span_http_status_code diff --git a/tests/contrib/flask/test_idempotency.py b/tests/contrib/flask/test_idempotency.py index 53625076d76..0b7758d24bc 100644 --- a/tests/contrib/flask/test_idempotency.py +++ b/tests/contrib/flask/test_idempotency.py @@ -4,10 +4,10 @@ import mock import wrapt -from ddtrace.contrib.flask import patch -from ddtrace.contrib.flask import unpatch from ddtrace.contrib.internal.flask.patch import _u from ddtrace.contrib.internal.flask.patch import _w +from ddtrace.contrib.internal.flask.patch import patch +from ddtrace.contrib.internal.flask.patch import unpatch class FlaskIdempotencyTestCase(unittest.TestCase): diff --git a/tests/contrib/flask/test_request.py b/tests/contrib/flask/test_request.py index 62df061a198..eec1a4c4a1c 100644 --- a/tests/contrib/flask/test_request.py +++ b/tests/contrib/flask/test_request.py @@ -12,7 +12,7 @@ import pytest from ddtrace.constants import ERROR_MSG -from ddtrace.contrib.flask.patch import flask_version +from ddtrace.contrib.internal.flask.patch import flask_version from ddtrace.ext import http from ddtrace.propagation.http import HTTP_HEADER_PARENT_ID from ddtrace.propagation.http import HTTP_HEADER_TRACE_ID diff --git a/tests/contrib/flask/test_signals.py b/tests/contrib/flask/test_signals.py index 1e8ac54e5b0..b86e8989047 100644 --- a/tests/contrib/flask/test_signals.py +++ b/tests/contrib/flask/test_signals.py @@ -1,9 +1,9 @@ import flask import mock -from ddtrace import Pin -from ddtrace.contrib.flask import unpatch -from ddtrace.contrib.flask.patch import flask_version +from ddtrace.contrib.internal.flask.patch import flask_version +from ddtrace.contrib.internal.flask.patch import unpatch +from ddtrace.trace import Pin from . import BaseFlaskTestCase diff --git a/tests/contrib/flask/test_static.py b/tests/contrib/flask/test_static.py index 14baf6ce0d6..3aa70426ab8 100644 --- a/tests/contrib/flask/test_static.py +++ b/tests/contrib/flask/test_static.py @@ -1,5 +1,5 @@ from ddtrace.constants import ERROR_MSG -from ddtrace.contrib.flask.patch import flask_version +from ddtrace.contrib.internal.flask.patch import flask_version from ddtrace.ext import http from tests.utils import assert_span_http_status_code diff --git a/tests/contrib/flask/test_template.py b/tests/contrib/flask/test_template.py index f15c67aa207..a38311d3b86 100644 --- a/tests/contrib/flask/test_template.py +++ b/tests/contrib/flask/test_template.py @@ -1,8 +1,8 @@ import flask -from ddtrace import Pin -from ddtrace.contrib.flask import unpatch -from ddtrace.contrib.flask.patch import flask_version +from ddtrace.contrib.internal.flask.patch import flask_version +from ddtrace.contrib.internal.flask.patch import unpatch +from ddtrace.trace import Pin from . import BaseFlaskTestCase diff --git a/tests/contrib/flask_autopatch/test_flask_autopatch.py b/tests/contrib/flask_autopatch/test_flask_autopatch.py index 3da871f27f4..27c4b47e2d0 100644 --- a/tests/contrib/flask_autopatch/test_flask_autopatch.py +++ b/tests/contrib/flask_autopatch/test_flask_autopatch.py @@ -2,9 +2,9 @@ import flask import wrapt -from ddtrace import Pin -from ddtrace.contrib.flask.patch import flask_version +from ddtrace.contrib.internal.flask.patch import flask_version from ddtrace.ext import http +from ddtrace.trace import Pin from tests.utils import TracerTestCase from tests.utils import assert_is_measured from tests.utils import assert_span_http_status_code diff --git a/tests/contrib/flask_cache/test.py b/tests/contrib/flask_cache/test.py index 62d3bb11d9c..b7be36a10eb 100644 --- a/tests/contrib/flask_cache/test.py +++ b/tests/contrib/flask_cache/test.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- from flask import Flask -from ddtrace.contrib.flask_cache import get_traced_cache -from ddtrace.contrib.flask_cache.tracers import CACHE_BACKEND +from ddtrace.contrib.internal.flask_cache.tracers import CACHE_BACKEND +from ddtrace.contrib.internal.flask_cache.tracers import get_traced_cache from ddtrace.ext import net from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME from tests.opentracer.utils import init_tracer diff --git a/tests/contrib/flask_cache/test_utils.py b/tests/contrib/flask_cache/test_utils.py index a527f443033..7d7e200cd9f 100644 --- a/tests/contrib/flask_cache/test_utils.py +++ b/tests/contrib/flask_cache/test_utils.py @@ -3,7 +3,7 @@ from flask import Flask from ddtrace._trace.tracer import Tracer -from ddtrace.contrib.flask_cache import get_traced_cache +from ddtrace.contrib.internal.flask_cache.tracers import get_traced_cache from ddtrace.contrib.internal.flask_cache.utils import _extract_client from ddtrace.contrib.internal.flask_cache.utils import _extract_conn_tags from ddtrace.contrib.internal.flask_cache.utils import _resource_from_cache_prefix diff --git a/tests/contrib/flask_cache/test_wrapper_safety.py b/tests/contrib/flask_cache/test_wrapper_safety.py index d389138e8b1..0b98e1e2557 100644 --- a/tests/contrib/flask_cache/test_wrapper_safety.py +++ b/tests/contrib/flask_cache/test_wrapper_safety.py @@ -3,8 +3,8 @@ import pytest from redis.exceptions import ConnectionError -from ddtrace.contrib.flask_cache import get_traced_cache -from ddtrace.contrib.flask_cache.tracers import CACHE_BACKEND +from ddtrace.contrib.internal.flask_cache.tracers import CACHE_BACKEND +from ddtrace.contrib.internal.flask_cache.tracers import get_traced_cache from ddtrace.ext import net from tests.utils import TracerTestCase diff --git a/tests/contrib/freezegun/test_freezegun.py b/tests/contrib/freezegun/test_freezegun.py index a1e624ed0eb..95198486598 100644 --- a/tests/contrib/freezegun/test_freezegun.py +++ b/tests/contrib/freezegun/test_freezegun.py @@ -12,8 +12,8 @@ class TestFreezegunTestCase: @pytest.fixture(autouse=True) def _patch_freezegun(self): - from ddtrace.contrib.freezegun import patch - from ddtrace.contrib.freezegun import unpatch + from ddtrace.contrib.internal.freezegun.patch import patch + from ddtrace.contrib.internal.freezegun.patch import unpatch patch() yield @@ -22,7 +22,7 @@ def _patch_freezegun(self): def test_freezegun_unpatch(self): import freezegun - from ddtrace.contrib.freezegun import unpatch + from ddtrace.contrib.internal.freezegun.patch import unpatch unpatch() @@ -75,7 +75,7 @@ def test_freezegun_pytest_plugin(self): """Tests that pytest's patching of freezegun in the v1 plugin version works""" import sys - from ddtrace.contrib.freezegun import unpatch + from ddtrace.contrib.internal.freezegun.patch import unpatch unpatch() if "freezegun" in sys.modules: diff --git a/tests/contrib/futures/test_futures_patch.py b/tests/contrib/futures/test_futures_patch.py index fca6a316c00..b805dea2e9a 100644 --- a/tests/contrib/futures/test_futures_patch.py +++ b/tests/contrib/futures/test_futures_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.futures import get_version -from ddtrace.contrib.futures.patch import patch +from ddtrace.contrib.internal.futures.patch import get_version +from ddtrace.contrib.internal.futures.patch import patch try: - from ddtrace.contrib.futures.patch import unpatch + from ddtrace.contrib.internal.futures.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/futures/test_propagation.py b/tests/contrib/futures/test_propagation.py index 037aebd9ce5..7f177c566ca 100644 --- a/tests/contrib/futures/test_propagation.py +++ b/tests/contrib/futures/test_propagation.py @@ -4,8 +4,8 @@ import pytest -from ddtrace.contrib.futures import patch -from ddtrace.contrib.futures import unpatch +from ddtrace.contrib.internal.futures.patch import patch +from ddtrace.contrib.internal.futures.patch import unpatch from tests.opentracer.utils import init_tracer from tests.utils import DummyTracer from tests.utils import TracerTestCase diff --git a/tests/contrib/gevent/test_tracer.py b/tests/contrib/gevent/test_tracer.py index a40a0f3cdaf..c34f06d41d6 100644 --- a/tests/contrib/gevent/test_tracer.py +++ b/tests/contrib/gevent/test_tracer.py @@ -9,8 +9,8 @@ from ddtrace.constants import SAMPLING_PRIORITY_KEY from ddtrace.constants import USER_KEEP from ddtrace._trace.context import Context -from ddtrace.contrib.gevent import patch -from ddtrace.contrib.gevent import unpatch +from ddtrace.contrib.internal.gevent.patch import patch +from ddtrace.contrib.internal.gevent.patch import unpatch from tests.opentracer.utils import init_tracer from tests.utils import TracerTestCase diff --git a/tests/contrib/google_generativeai/conftest.py b/tests/contrib/google_generativeai/conftest.py index 7da872255c3..0cadd515f84 100644 --- a/tests/contrib/google_generativeai/conftest.py +++ b/tests/contrib/google_generativeai/conftest.py @@ -3,10 +3,10 @@ import mock import pytest -from ddtrace.contrib.google_generativeai import patch -from ddtrace.contrib.google_generativeai import unpatch +from ddtrace.contrib.internal.google_generativeai.patch import patch +from ddtrace.contrib.internal.google_generativeai.patch import unpatch from ddtrace.llmobs import LLMObs -from ddtrace.pin import Pin +from ddtrace.trace import Pin from tests.contrib.google_generativeai.utils import MockGenerativeModelAsyncClient from tests.contrib.google_generativeai.utils import MockGenerativeModelClient from tests.utils import DummyTracer diff --git a/tests/contrib/google_generativeai/test_google_generativeai_patch.py b/tests/contrib/google_generativeai/test_google_generativeai_patch.py index 470e292a303..a98ad7e2d6a 100644 --- a/tests/contrib/google_generativeai/test_google_generativeai_patch.py +++ b/tests/contrib/google_generativeai/test_google_generativeai_patch.py @@ -1,6 +1,6 @@ -from ddtrace.contrib.google_generativeai import get_version -from ddtrace.contrib.google_generativeai import patch -from ddtrace.contrib.google_generativeai import unpatch +from ddtrace.contrib.internal.google_generativeai.patch import get_version +from ddtrace.contrib.internal.google_generativeai.patch import patch +from ddtrace.contrib.internal.google_generativeai.patch import unpatch from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/graphene/test_graphene.py b/tests/contrib/graphene/test_graphene.py index 47b842f2d75..5dca40212ee 100644 --- a/tests/contrib/graphene/test_graphene.py +++ b/tests/contrib/graphene/test_graphene.py @@ -1,8 +1,8 @@ import graphene import pytest -from ddtrace.contrib.graphql import patch -from ddtrace.contrib.graphql import unpatch +from ddtrace.contrib.internal.graphql.patch import patch +from ddtrace.contrib.internal.graphql.patch import unpatch from tests.utils import override_config diff --git a/tests/contrib/graphql/test_graphql.py b/tests/contrib/graphql/test_graphql.py index 67c5befd7fe..f6eca690b36 100644 --- a/tests/contrib/graphql/test_graphql.py +++ b/tests/contrib/graphql/test_graphql.py @@ -4,9 +4,9 @@ import pytest from ddtrace import tracer -from ddtrace.contrib.graphql import patch -from ddtrace.contrib.graphql import unpatch from ddtrace.contrib.internal.graphql.patch import _graphql_version as graphql_version +from ddtrace.contrib.internal.graphql.patch import patch +from ddtrace.contrib.internal.graphql.patch import unpatch from tests.utils import override_config from tests.utils import snapshot diff --git a/tests/contrib/graphql/test_graphql_patch.py b/tests/contrib/graphql/test_graphql_patch.py index 951e69998f3..5504a29ef06 100644 --- a/tests/contrib/graphql/test_graphql_patch.py +++ b/tests/contrib/graphql/test_graphql_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.graphql import get_version -from ddtrace.contrib.graphql.patch import patch +from ddtrace.contrib.internal.graphql.patch import get_version +from ddtrace.contrib.internal.graphql.patch import patch try: - from ddtrace.contrib.graphql.patch import unpatch + from ddtrace.contrib.internal.graphql.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/grpc/common.py b/tests/contrib/grpc/common.py index e45db8e6473..e67e4f32a92 100644 --- a/tests/contrib/grpc/common.py +++ b/tests/contrib/grpc/common.py @@ -2,10 +2,10 @@ from grpc._grpcio_metadata import __version__ as _GRPC_VERSION from grpc.framework.foundation import logging_pool -from ddtrace import Pin -from ddtrace.contrib.grpc import constants -from ddtrace.contrib.grpc import patch -from ddtrace.contrib.grpc import unpatch +from ddtrace.contrib.internal.grpc import constants +from ddtrace.contrib.internal.grpc.patch import patch +from ddtrace.contrib.internal.grpc.patch import unpatch +from ddtrace.trace import Pin from tests.utils import TracerTestCase from .hello_pb2_grpc import add_HelloServicer_to_server diff --git a/tests/contrib/grpc/test_constants.py b/tests/contrib/grpc/test_constants.py index 5ff9e4b6aa2..8da5b97172f 100644 --- a/tests/contrib/grpc/test_constants.py +++ b/tests/contrib/grpc/test_constants.py @@ -4,7 +4,7 @@ def test_not_deprecated(): - from ddtrace.contrib.grpc import constants as grpc_constants + from ddtrace.contrib.internal.grpc import constants as grpc_constants with warnings.catch_warnings(record=True) as warns: warnings.simplefilter("always") @@ -15,4 +15,4 @@ def test_not_deprecated(): def test_invalid(): with pytest.raises(ImportError): - from ddtrace.contrib.grpc.constants import INVALID_CONSTANT # noqa:F401 + from ddtrace.contrib.internal.grpc.constants import INVALID_CONSTANT # noqa:F401 diff --git a/tests/contrib/grpc/test_grpc.py b/tests/contrib/grpc/test_grpc.py index fdde555c83d..93dacb4cb45 100644 --- a/tests/contrib/grpc/test_grpc.py +++ b/tests/contrib/grpc/test_grpc.py @@ -5,16 +5,16 @@ from grpc.framework.foundation import logging_pool import pytest -from ddtrace import Pin from ddtrace._trace.span import _get_64_highest_order_bits_as_hex from ddtrace.constants import ERROR_MSG from ddtrace.constants import ERROR_STACK from ddtrace.constants import ERROR_TYPE -from ddtrace.contrib.grpc import constants -from ddtrace.contrib.grpc import patch -from ddtrace.contrib.grpc import unpatch +from ddtrace.contrib.internal.grpc import constants from ddtrace.contrib.internal.grpc.patch import _unpatch_server +from ddtrace.contrib.internal.grpc.patch import patch +from ddtrace.contrib.internal.grpc.patch import unpatch from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME +from ddtrace.trace import Pin from tests.utils import TracerTestCase from tests.utils import flaky from tests.utils import snapshot diff --git a/tests/contrib/grpc/test_grpc_patch.py b/tests/contrib/grpc/test_grpc_patch.py index 805b6996a3c..6a50342367f 100644 --- a/tests/contrib/grpc/test_grpc_patch.py +++ b/tests/contrib/grpc/test_grpc_patch.py @@ -1,5 +1,5 @@ -from ddtrace.contrib.grpc import get_version -from ddtrace.contrib.grpc import patch +from ddtrace.contrib.internal.grpc.patch import get_version +from ddtrace.contrib.internal.grpc.patch import patch from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/grpc_aio/test_grpc_aio.py b/tests/contrib/grpc_aio/test_grpc_aio.py index ff09cf198ce..885765e91ca 100644 --- a/tests/contrib/grpc_aio/test_grpc_aio.py +++ b/tests/contrib/grpc_aio/test_grpc_aio.py @@ -7,15 +7,15 @@ from grpc import aio import pytest -from ddtrace import Pin from ddtrace.constants import ERROR_MSG from ddtrace.constants import ERROR_STACK from ddtrace.constants import ERROR_TYPE -from ddtrace.contrib.grpc import patch -from ddtrace.contrib.grpc import unpatch -from ddtrace.contrib.grpc.patch import GRPC_AIO_PIN_MODULE_CLIENT -from ddtrace.contrib.grpc.patch import GRPC_AIO_PIN_MODULE_SERVER +from ddtrace.contrib.internal.grpc.patch import GRPC_AIO_PIN_MODULE_CLIENT +from ddtrace.contrib.internal.grpc.patch import GRPC_AIO_PIN_MODULE_SERVER +from ddtrace.contrib.internal.grpc.patch import patch +from ddtrace.contrib.internal.grpc.patch import unpatch from ddtrace.contrib.internal.grpc.utils import _parse_rpc_repr_string +from ddtrace.trace import Pin import ddtrace.vendor.packaging.version as packaging_version from tests.contrib.grpc.hello_pb2 import HelloReply from tests.contrib.grpc.hello_pb2 import HelloRequest diff --git a/tests/contrib/gunicorn/wsgi_mw_app.py b/tests/contrib/gunicorn/wsgi_mw_app.py index 26f46033e65..ac3931923c7 100644 --- a/tests/contrib/gunicorn/wsgi_mw_app.py +++ b/tests/contrib/gunicorn/wsgi_mw_app.py @@ -11,7 +11,7 @@ import json from ddtrace import tracer -from ddtrace.contrib.wsgi import DDWSGIMiddleware +from ddtrace.contrib.internal.wsgi.wsgi import DDWSGIMiddleware from ddtrace.profiling import bootstrap import ddtrace.profiling.auto # noqa:F401 from tests.webclient import PingFilter diff --git a/tests/contrib/httplib/test_httplib.py b/tests/contrib/httplib/test_httplib.py index edc888ad3eb..24a5fe3f051 100644 --- a/tests/contrib/httplib/test_httplib.py +++ b/tests/contrib/httplib/test_httplib.py @@ -9,15 +9,15 @@ import wrapt from ddtrace import config -from ddtrace.contrib.httplib import patch -from ddtrace.contrib.httplib import unpatch -from ddtrace.contrib.httplib.patch import should_skip_request +from ddtrace.contrib.internal.httplib.patch import patch +from ddtrace.contrib.internal.httplib.patch import should_skip_request +from ddtrace.contrib.internal.httplib.patch import unpatch from ddtrace.ext import http from ddtrace.internal.compat import httplib from ddtrace.internal.compat import parse from ddtrace.internal.constants import _HTTPLIB_NO_TRACE_REQUEST from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME -from ddtrace.pin import Pin +from ddtrace.trace import Pin from tests.opentracer.utils import init_tracer from tests.utils import TracerTestCase from tests.utils import assert_span_http_status_code diff --git a/tests/contrib/httplib/test_httplib_distributed.py b/tests/contrib/httplib/test_httplib_distributed.py index 3552f65a51a..40e5e891662 100644 --- a/tests/contrib/httplib/test_httplib_distributed.py +++ b/tests/contrib/httplib/test_httplib_distributed.py @@ -7,7 +7,7 @@ from ddtrace import config from ddtrace._trace.span import _get_64_highest_order_bits_as_hex from ddtrace.internal.compat import httplib -from ddtrace.pin import Pin +from ddtrace.trace import Pin from tests.utils import TracerTestCase from .test_httplib import SOCKET diff --git a/tests/contrib/httplib/test_httplib_patch.py b/tests/contrib/httplib/test_httplib_patch.py index 420f76bc302..dd8018f27ed 100644 --- a/tests/contrib/httplib/test_httplib_patch.py +++ b/tests/contrib/httplib/test_httplib_patch.py @@ -1,9 +1,9 @@ -from ddtrace.contrib.httplib.patch import get_version -from ddtrace.contrib.httplib.patch import patch +from ddtrace.contrib.internal.httplib.patch import get_version +from ddtrace.contrib.internal.httplib.patch import patch try: - from ddtrace.contrib.httplib.patch import unpatch + from ddtrace.contrib.internal.httplib.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/httpx/test_httpx.py b/tests/contrib/httpx/test_httpx.py index 43841d73f23..33ecadb825f 100644 --- a/tests/contrib/httpx/test_httpx.py +++ b/tests/contrib/httpx/test_httpx.py @@ -3,11 +3,11 @@ from wrapt import ObjectProxy from ddtrace import config -from ddtrace.contrib.httpx.patch import HTTPX_VERSION -from ddtrace.contrib.httpx.patch import patch -from ddtrace.contrib.httpx.patch import unpatch -from ddtrace.pin import Pin +from ddtrace.contrib.internal.httpx.patch import HTTPX_VERSION +from ddtrace.contrib.internal.httpx.patch import patch +from ddtrace.contrib.internal.httpx.patch import unpatch from ddtrace.settings.http import HttpConfig +from ddtrace.trace import Pin from tests.utils import flaky from tests.utils import override_config from tests.utils import override_http_config @@ -165,7 +165,7 @@ def test_configure_service_name_env(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -202,7 +202,7 @@ def test_schematized_configure_global_service_name_env_default(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -236,7 +236,7 @@ def test_schematized_configure_global_service_name_env_v0(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -270,7 +270,7 @@ def test_schematized_configure_global_service_name_env_v1(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -303,7 +303,7 @@ def test_schematized_unspecified_service_name_env_default(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -336,7 +336,7 @@ def test_schematized_unspecified_service_name_env_v0(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -368,7 +368,7 @@ def test_schematized_unspecified_service_name_env_v1(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -401,7 +401,7 @@ def test_schematized_operation_name_env_v0(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -433,7 +433,7 @@ def test_schematized_operation_name_env_v1(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -598,7 +598,7 @@ def test_distributed_tracing_disabled_env(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url patch() diff --git a/tests/contrib/httpx/test_httpx_patch.py b/tests/contrib/httpx/test_httpx_patch.py index 5062fb2c575..e3a62fc99ef 100644 --- a/tests/contrib/httpx/test_httpx_patch.py +++ b/tests/contrib/httpx/test_httpx_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.httpx import get_version -from ddtrace.contrib.httpx.patch import patch +from ddtrace.contrib.internal.httpx.patch import get_version +from ddtrace.contrib.internal.httpx.patch import patch try: - from ddtrace.contrib.httpx.patch import unpatch + from ddtrace.contrib.internal.httpx.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/httpx/test_httpx_pre_0_11.py b/tests/contrib/httpx/test_httpx_pre_0_11.py index ed4a70e047d..315c53cb29c 100644 --- a/tests/contrib/httpx/test_httpx_pre_0_11.py +++ b/tests/contrib/httpx/test_httpx_pre_0_11.py @@ -3,11 +3,11 @@ from wrapt import ObjectProxy from ddtrace import config -from ddtrace.contrib.httpx.patch import HTTPX_VERSION -from ddtrace.contrib.httpx.patch import patch -from ddtrace.contrib.httpx.patch import unpatch -from ddtrace.pin import Pin +from ddtrace.contrib.internal.httpx.patch import HTTPX_VERSION +from ddtrace.contrib.internal.httpx.patch import patch +from ddtrace.contrib.internal.httpx.patch import unpatch from ddtrace.settings.http import HttpConfig +from ddtrace.trace import Pin from tests.utils import override_config from tests.utils import override_http_config @@ -143,7 +143,7 @@ def test_configure_service_name_env(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -172,7 +172,7 @@ def test_configure_global_service_name_env(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -201,7 +201,7 @@ def test_schematized_configure_global_service_name_env_default(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -230,7 +230,7 @@ def test_schematized_configure_global_service_name_env_v0(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -259,7 +259,7 @@ def test_schematized_configure_global_service_name_env_v1(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -287,7 +287,7 @@ def test_schematized_unspecified_service_name_env_default(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -315,7 +315,7 @@ def test_schematized_unspecified_service_name_env_v0(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -343,7 +343,7 @@ def test_schematized_unspecified_service_name_env_v1(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -371,7 +371,7 @@ def test_schematized_operation_name_env_v0(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -399,7 +399,7 @@ def test_schematized_operation_name_env_v1(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url from tests.utils import snapshot_context @@ -536,7 +536,7 @@ def test_distributed_tracing_disabled_env(): import httpx - from ddtrace.contrib.httpx import patch + from ddtrace.contrib.internal.httpx.patch import patch from tests.contrib.httpx.test_httpx import get_url patch() diff --git a/tests/contrib/jinja2/test_jinja2.py b/tests/contrib/jinja2/test_jinja2.py index fe675ffd9b3..64002fd6555 100644 --- a/tests/contrib/jinja2/test_jinja2.py +++ b/tests/contrib/jinja2/test_jinja2.py @@ -4,10 +4,10 @@ # 3rd party import jinja2 -from ddtrace import Pin from ddtrace import config -from ddtrace.contrib.jinja2 import patch -from ddtrace.contrib.jinja2 import unpatch +from ddtrace.contrib.internal.jinja2.patch import patch +from ddtrace.contrib.internal.jinja2.patch import unpatch +from ddtrace.trace import Pin from tests.utils import TracerTestCase from tests.utils import assert_is_measured from tests.utils import assert_is_not_measured diff --git a/tests/contrib/jinja2/test_jinja2_patch.py b/tests/contrib/jinja2/test_jinja2_patch.py index ec65e5276e3..4b4256e672f 100644 --- a/tests/contrib/jinja2/test_jinja2_patch.py +++ b/tests/contrib/jinja2/test_jinja2_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.jinja2 import get_version -from ddtrace.contrib.jinja2.patch import patch +from ddtrace.contrib.internal.jinja2.patch import get_version +from ddtrace.contrib.internal.jinja2.patch import patch try: - from ddtrace.contrib.jinja2.patch import unpatch + from ddtrace.contrib.internal.jinja2.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/kafka/test_kafka.py b/tests/contrib/kafka/test_kafka.py index 9d2786151da..38f6c783fc1 100644 --- a/tests/contrib/kafka/test_kafka.py +++ b/tests/contrib/kafka/test_kafka.py @@ -11,18 +11,18 @@ import mock import pytest -from ddtrace import Pin from ddtrace import Tracer -from ddtrace.contrib.kafka.patch import TracedConsumer -from ddtrace.contrib.kafka.patch import patch -from ddtrace.contrib.kafka.patch import unpatch -from ddtrace.filters import TraceFilter +from ddtrace.contrib.internal.kafka.patch import TracedConsumer +from ddtrace.contrib.internal.kafka.patch import patch +from ddtrace.contrib.internal.kafka.patch import unpatch import ddtrace.internal.datastreams # noqa: F401 - used as part of mock patching from ddtrace.internal.datastreams.processor import PROPAGATION_KEY_BASE_64 from ddtrace.internal.datastreams.processor import ConsumerPartitionKey from ddtrace.internal.datastreams.processor import DataStreamsCtx from ddtrace.internal.datastreams.processor import PartitionKey from ddtrace.internal.utils.retry import fibonacci_backoff_with_jitter +from ddtrace.trace import Pin +from ddtrace.trace import TraceFilter from tests.contrib.config import KAFKA_CONFIG from tests.datastreams.test_public_api import MockedTracer from tests.utils import DummyTracer @@ -499,8 +499,8 @@ def test_data_streams_kafka(dsm_processor, consumer, producer, kafka_topic): def _generate_in_subprocess(random_topic): import ddtrace - from ddtrace.contrib.kafka.patch import patch - from ddtrace.contrib.kafka.patch import unpatch + from ddtrace.contrib.internal.kafka.patch import patch + from ddtrace.contrib.internal.kafka.patch import unpatch from tests.contrib.kafka.test_kafka import KafkaConsumerPollFilter PAYLOAD = bytes("hueh hueh hueh", encoding="utf-8") @@ -518,8 +518,8 @@ def _generate_in_subprocess(random_topic): "auto.offset.reset": "earliest", } ) - ddtrace.Pin.override(producer, tracer=ddtrace.tracer) - ddtrace.Pin.override(consumer, tracer=ddtrace.tracer) + ddtrace.trace.Pin.override(producer, tracer=ddtrace.tracer) + ddtrace.trace.Pin.override(consumer, tracer=ddtrace.tracer) # We run all of these commands with retry attempts because the kafka-confluent API # sys.exits on connection failures, which causes the test to fail. We want to retry @@ -799,8 +799,8 @@ def test_tracing_context_is_propagated_when_enabled(ddtrace_run_python_code_in_s import random import sys -from ddtrace import Pin -from ddtrace.contrib.kafka.patch import patch +from ddtrace.trace import Pin +from ddtrace.contrib.internal.kafka.patch import patch from tests.contrib.kafka.test_kafka import consumer from tests.contrib.kafka.test_kafka import kafka_topic @@ -1039,8 +1039,8 @@ def test_does_not_trace_empty_poll_when_disabled(ddtrace_run_python_code_in_subp import random import sys -from ddtrace import Pin -from ddtrace.contrib.kafka.patch import patch +from ddtrace.trace import Pin +from ddtrace.contrib.internal.kafka.patch import patch from ddtrace import config from tests.contrib.kafka.test_kafka import consumer diff --git a/tests/contrib/kafka/test_kafka_patch.py b/tests/contrib/kafka/test_kafka_patch.py index 23adc8503af..53e82ca2ee5 100644 --- a/tests/contrib/kafka/test_kafka_patch.py +++ b/tests/contrib/kafka/test_kafka_patch.py @@ -1,6 +1,6 @@ -from ddtrace.contrib.kafka.patch import get_version -from ddtrace.contrib.kafka.patch import patch -from ddtrace.contrib.kafka.patch import unpatch +from ddtrace.contrib.internal.kafka.patch import get_version +from ddtrace.contrib.internal.kafka.patch import patch +from ddtrace.contrib.internal.kafka.patch import unpatch from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/kombu/test.py b/tests/contrib/kombu/test.py index bb666d7a52e..b56ecdf0d0f 100644 --- a/tests/contrib/kombu/test.py +++ b/tests/contrib/kombu/test.py @@ -2,13 +2,13 @@ import kombu import mock -from ddtrace import Pin -from ddtrace.contrib.kombu import utils -from ddtrace.contrib.kombu.patch import patch -from ddtrace.contrib.kombu.patch import unpatch +from ddtrace.contrib.internal.kombu import utils +from ddtrace.contrib.internal.kombu.patch import patch +from ddtrace.contrib.internal.kombu.patch import unpatch from ddtrace.ext import kombu as kombux from ddtrace.internal.datastreams.processor import PROPAGATION_KEY_BASE_64 from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME +from ddtrace.trace import Pin from tests.utils import TracerTestCase from tests.utils import assert_is_measured diff --git a/tests/contrib/kombu/test_kombu_patch.py b/tests/contrib/kombu/test_kombu_patch.py index 908cd920489..dde0dfbb1c1 100644 --- a/tests/contrib/kombu/test_kombu_patch.py +++ b/tests/contrib/kombu/test_kombu_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.kombu import get_version -from ddtrace.contrib.kombu.patch import patch +from ddtrace.contrib.internal.kombu.patch import get_version +from ddtrace.contrib.internal.kombu.patch import patch try: - from ddtrace.contrib.kombu.patch import unpatch + from ddtrace.contrib.internal.kombu.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/langchain/conftest.py b/tests/contrib/langchain/conftest.py index 5403417f42e..5184f1093b7 100644 --- a/tests/contrib/langchain/conftest.py +++ b/tests/contrib/langchain/conftest.py @@ -3,9 +3,9 @@ import mock import pytest -from ddtrace import Pin -from ddtrace import patch -from ddtrace.contrib.langchain.patch import unpatch +from ddtrace.contrib.internal.langchain.patch import patch +from ddtrace.contrib.internal.langchain.patch import unpatch +from ddtrace.trace import Pin from tests.utils import DummyTracer from tests.utils import DummyWriter from tests.utils import override_config @@ -91,7 +91,7 @@ def langchain(ddtrace_config_langchain, mock_logs, mock_metrics): AI21_API_KEY=os.getenv("AI21_API_KEY", ""), ) ): - patch(langchain=True) + patch() import langchain mock_logs.reset_mock() diff --git a/tests/contrib/langchain/test_langchain_patch.py b/tests/contrib/langchain/test_langchain_patch.py index 16bd03b1327..10df4ab60e9 100644 --- a/tests/contrib/langchain/test_langchain_patch.py +++ b/tests/contrib/langchain/test_langchain_patch.py @@ -1,9 +1,9 @@ -from ddtrace.contrib.langchain import get_version -from ddtrace.contrib.langchain import patch -from ddtrace.contrib.langchain import unpatch -from ddtrace.contrib.langchain.constants import text_embedding_models -from ddtrace.contrib.langchain.constants import vectorstore_classes -from ddtrace.contrib.langchain.patch import PATCH_LANGCHAIN_V0 +from ddtrace.contrib.internal.langchain.constants import text_embedding_models +from ddtrace.contrib.internal.langchain.constants import vectorstore_classes +from ddtrace.contrib.internal.langchain.patch import PATCH_LANGCHAIN_V0 +from ddtrace.contrib.internal.langchain.patch import get_version +from ddtrace.contrib.internal.langchain.patch import patch +from ddtrace.contrib.internal.langchain.patch import unpatch from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/logbook/test_logbook_logging.py b/tests/contrib/logbook/test_logbook_logging.py index f8b858449b4..d9d67da82f4 100644 --- a/tests/contrib/logbook/test_logbook_logging.py +++ b/tests/contrib/logbook/test_logbook_logging.py @@ -7,8 +7,8 @@ from ddtrace.constants import ENV_KEY from ddtrace.constants import SERVICE_KEY from ddtrace.constants import VERSION_KEY -from ddtrace.contrib.logbook import patch -from ddtrace.contrib.logbook import unpatch +from ddtrace.contrib.internal.logbook.patch import patch +from ddtrace.contrib.internal.logbook.patch import unpatch from ddtrace.internal.constants import MAX_UINT_64BITS from tests.utils import override_global_config @@ -78,8 +78,8 @@ def test_log_trace(): from ddtrace import config from ddtrace import tracer - from ddtrace.contrib.logbook import patch - from ddtrace.contrib.logbook import unpatch + from ddtrace.contrib.internal.logbook.patch import patch + from ddtrace.contrib.internal.logbook.patch import unpatch config.service = "logging" config.env = "global.env" @@ -116,8 +116,8 @@ def test_log_trace_128bit_trace_ids(): from ddtrace import config from ddtrace import tracer - from ddtrace.contrib.logbook import patch - from ddtrace.contrib.logbook import unpatch + from ddtrace.contrib.internal.logbook.patch import patch + from ddtrace.contrib.internal.logbook.patch import unpatch from ddtrace.internal.constants import MAX_UINT_64BITS config.service = "logging" @@ -151,8 +151,8 @@ def test_log_DD_TAGS(): from logbook import TestHandler from ddtrace import tracer - from ddtrace.contrib.logbook import patch - from ddtrace.contrib.logbook import unpatch + from ddtrace.contrib.internal.logbook.patch import patch + from ddtrace.contrib.internal.logbook.patch import unpatch from ddtrace.internal.constants import MAX_UINT_64BITS handler = TestHandler() diff --git a/tests/contrib/logbook/test_logbook_patch.py b/tests/contrib/logbook/test_logbook_patch.py index 0be19c170db..ad434c8fba6 100644 --- a/tests/contrib/logbook/test_logbook_patch.py +++ b/tests/contrib/logbook/test_logbook_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.logbook import get_version -from ddtrace.contrib.logbook.patch import patch +from ddtrace.contrib.internal.logbook.patch import get_version +from ddtrace.contrib.internal.logbook.patch import patch try: - from ddtrace.contrib.logbook.patch import unpatch + from ddtrace.contrib.internal.logbook.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/logging/test_logging.py b/tests/contrib/logging/test_logging.py index 2e85d73e386..4da09fa45dc 100644 --- a/tests/contrib/logging/test_logging.py +++ b/tests/contrib/logging/test_logging.py @@ -5,10 +5,10 @@ import ddtrace from ddtrace.constants import ENV_KEY from ddtrace.constants import VERSION_KEY -from ddtrace.contrib.logging import patch -from ddtrace.contrib.logging import unpatch -from ddtrace.contrib.logging.constants import RECORD_ATTR_SPAN_ID -from ddtrace.contrib.logging.constants import RECORD_ATTR_TRACE_ID +from ddtrace.contrib.internal.logging.constants import RECORD_ATTR_SPAN_ID +from ddtrace.contrib.internal.logging.constants import RECORD_ATTR_TRACE_ID +from ddtrace.contrib.internal.logging.patch import patch +from ddtrace.contrib.internal.logging.patch import unpatch from ddtrace.internal.compat import StringIO from ddtrace.internal.constants import MAX_UINT_64BITS from tests.utils import TracerTestCase diff --git a/tests/contrib/logging/test_logging_patch.py b/tests/contrib/logging/test_logging_patch.py index 94e4c975a81..e4f92436eda 100644 --- a/tests/contrib/logging/test_logging_patch.py +++ b/tests/contrib/logging/test_logging_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.logging import get_version -from ddtrace.contrib.logging.patch import patch +from ddtrace.contrib.internal.logging.patch import get_version +from ddtrace.contrib.internal.logging.patch import patch try: - from ddtrace.contrib.logging.patch import unpatch + from ddtrace.contrib.internal.logging.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/loguru/test_loguru_logging.py b/tests/contrib/loguru/test_loguru_logging.py index 0c2361aff41..7ea8baddddb 100644 --- a/tests/contrib/loguru/test_loguru_logging.py +++ b/tests/contrib/loguru/test_loguru_logging.py @@ -9,8 +9,8 @@ from ddtrace.constants import ENV_KEY from ddtrace.constants import SERVICE_KEY from ddtrace.constants import VERSION_KEY -from ddtrace.contrib.loguru import patch -from ddtrace.contrib.loguru import unpatch +from ddtrace.contrib.internal.loguru.patch import patch +from ddtrace.contrib.internal.loguru.patch import unpatch from ddtrace.internal.constants import MAX_UINT_64BITS from tests.utils import override_global_config diff --git a/tests/contrib/loguru/test_loguru_patch.py b/tests/contrib/loguru/test_loguru_patch.py index fa9166fa251..4e919050ad0 100644 --- a/tests/contrib/loguru/test_loguru_patch.py +++ b/tests/contrib/loguru/test_loguru_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.loguru import get_version -from ddtrace.contrib.loguru.patch import patch +from ddtrace.contrib.internal.loguru.patch import get_version +from ddtrace.contrib.internal.loguru.patch import patch try: - from ddtrace.contrib.loguru.patch import unpatch + from ddtrace.contrib.internal.loguru.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/mako/test_mako.py b/tests/contrib/mako/test_mako.py index dca66e7a38b..7e690b04a43 100644 --- a/tests/contrib/mako/test_mako.py +++ b/tests/contrib/mako/test_mako.py @@ -4,13 +4,13 @@ from mako.runtime import Context from mako.template import Template -from ddtrace import Pin -from ddtrace.contrib.mako import patch -from ddtrace.contrib.mako import unpatch -from ddtrace.contrib.mako.constants import DEFAULT_TEMPLATE_NAME +from ddtrace.contrib.internal.mako.constants import DEFAULT_TEMPLATE_NAME +from ddtrace.contrib.internal.mako.patch import patch +from ddtrace.contrib.internal.mako.patch import unpatch from ddtrace.internal.compat import StringIO from ddtrace.internal.compat import to_unicode from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME +from ddtrace.trace import Pin from tests.utils import TracerTestCase from tests.utils import assert_is_measured diff --git a/tests/contrib/mako/test_mako_patch.py b/tests/contrib/mako/test_mako_patch.py index 17e23677b0d..e83f0a2f9fb 100644 --- a/tests/contrib/mako/test_mako_patch.py +++ b/tests/contrib/mako/test_mako_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.mako import get_version -from ddtrace.contrib.mako.patch import patch +from ddtrace.contrib.internal.mako.patch import get_version +from ddtrace.contrib.internal.mako.patch import patch try: - from ddtrace.contrib.mako.patch import unpatch + from ddtrace.contrib.internal.mako.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/mariadb/test_mariadb.py b/tests/contrib/mariadb/test_mariadb.py index 3c5768389bc..7ea8cd27feb 100644 --- a/tests/contrib/mariadb/test_mariadb.py +++ b/tests/contrib/mariadb/test_mariadb.py @@ -4,9 +4,9 @@ import mariadb import pytest -from ddtrace import Pin -from ddtrace.contrib.mariadb import patch -from ddtrace.contrib.mariadb import unpatch +from ddtrace.contrib.internal.mariadb.patch import patch +from ddtrace.contrib.internal.mariadb.patch import unpatch +from ddtrace.trace import Pin from tests.contrib.config import MARIADB_CONFIG from tests.utils import DummyTracer from tests.utils import assert_dict_issuperset diff --git a/tests/contrib/mariadb/test_mariadb_patch.py b/tests/contrib/mariadb/test_mariadb_patch.py index 8dd47b216b3..8ad25102bfb 100644 --- a/tests/contrib/mariadb/test_mariadb_patch.py +++ b/tests/contrib/mariadb/test_mariadb_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.mariadb import get_version -from ddtrace.contrib.mariadb.patch import patch +from ddtrace.contrib.internal.mariadb.patch import get_version +from ddtrace.contrib.internal.mariadb.patch import patch try: - from ddtrace.contrib.mariadb.patch import unpatch + from ddtrace.contrib.internal.mariadb.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/molten/test_molten.py b/tests/contrib/molten/test_molten.py index 08a15551ae7..cc73ceef861 100644 --- a/tests/contrib/molten/test_molten.py +++ b/tests/contrib/molten/test_molten.py @@ -2,16 +2,16 @@ from molten.testing import TestClient import pytest -from ddtrace import Pin from ddtrace import config from ddtrace.constants import ERROR_MSG -from ddtrace.contrib.molten import patch -from ddtrace.contrib.molten import unpatch -from ddtrace.contrib.molten.patch import MOLTEN_VERSION +from ddtrace.contrib.internal.molten.patch import MOLTEN_VERSION +from ddtrace.contrib.internal.molten.patch import patch +from ddtrace.contrib.internal.molten.patch import unpatch from ddtrace.ext import http from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME from ddtrace.propagation.http import HTTP_HEADER_PARENT_ID from ddtrace.propagation.http import HTTP_HEADER_TRACE_ID +from ddtrace.trace import Pin from tests.utils import TracerTestCase from tests.utils import assert_is_measured from tests.utils import assert_span_http_status_code diff --git a/tests/contrib/molten/test_molten_di.py b/tests/contrib/molten/test_molten_di.py index 8c6f03a4786..d360698f4cb 100644 --- a/tests/contrib/molten/test_molten_di.py +++ b/tests/contrib/molten/test_molten_di.py @@ -3,9 +3,9 @@ import molten from molten import DependencyInjector -from ddtrace import Pin -from ddtrace.contrib.molten import patch -from ddtrace.contrib.molten import unpatch +from ddtrace.contrib.internal.molten.patch import patch +from ddtrace.contrib.internal.molten.patch import unpatch +from ddtrace.trace import Pin from tests.utils import TracerTestCase diff --git a/tests/contrib/molten/test_molten_patch.py b/tests/contrib/molten/test_molten_patch.py index 73a13a3e463..ac0b7c962a6 100644 --- a/tests/contrib/molten/test_molten_patch.py +++ b/tests/contrib/molten/test_molten_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.molten import get_version -from ddtrace.contrib.molten.patch import patch +from ddtrace.contrib.internal.molten.patch import get_version +from ddtrace.contrib.internal.molten.patch import patch try: - from ddtrace.contrib.molten.patch import unpatch + from ddtrace.contrib.internal.molten.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/mongoengine/test.py b/tests/contrib/mongoengine/test.py index b34d500291c..b3961e3808c 100644 --- a/tests/contrib/mongoengine/test.py +++ b/tests/contrib/mongoengine/test.py @@ -3,11 +3,11 @@ import mongoengine import pymongo -from ddtrace import Pin -from ddtrace.contrib.mongoengine.patch import patch -from ddtrace.contrib.mongoengine.patch import unpatch +from ddtrace.contrib.internal.mongoengine.patch import patch +from ddtrace.contrib.internal.mongoengine.patch import unpatch from ddtrace.ext import mongo as mongox from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME +from ddtrace.trace import Pin from tests.opentracer.utils import init_tracer from tests.utils import DummyTracer from tests.utils import TracerTestCase diff --git a/tests/contrib/mongoengine/test_mongoengine_patch.py b/tests/contrib/mongoengine/test_mongoengine_patch.py index f6d994cec0f..6f219d1566e 100644 --- a/tests/contrib/mongoengine/test_mongoengine_patch.py +++ b/tests/contrib/mongoengine/test_mongoengine_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.mongoengine import get_version -from ddtrace.contrib.mongoengine.patch import patch +from ddtrace.contrib.internal.mongoengine.patch import get_version +from ddtrace.contrib.internal.mongoengine.patch import patch try: - from ddtrace.contrib.mongoengine.patch import unpatch + from ddtrace.contrib.internal.mongoengine.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/mysql/test_mysql.py b/tests/contrib/mysql/test_mysql.py index 777b5d341fa..08626890fac 100644 --- a/tests/contrib/mysql/test_mysql.py +++ b/tests/contrib/mysql/test_mysql.py @@ -1,9 +1,9 @@ import mock import mysql -from ddtrace import Pin -from ddtrace.contrib.mysql.patch import patch -from ddtrace.contrib.mysql.patch import unpatch +from ddtrace.contrib.internal.mysql.patch import patch +from ddtrace.contrib.internal.mysql.patch import unpatch +from ddtrace.trace import Pin from tests.contrib import shared_tests from tests.contrib.config import MYSQL_CONFIG from tests.opentracer.utils import init_tracer diff --git a/tests/contrib/mysql/test_mysql_patch.py b/tests/contrib/mysql/test_mysql_patch.py index e69f9781cc4..6e10950dac9 100644 --- a/tests/contrib/mysql/test_mysql_patch.py +++ b/tests/contrib/mysql/test_mysql_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.mysql import get_version -from ddtrace.contrib.mysql.patch import patch +from ddtrace.contrib.internal.mysql.patch import get_version +from ddtrace.contrib.internal.mysql.patch import patch try: - from ddtrace.contrib.mysql.patch import unpatch + from ddtrace.contrib.internal.mysql.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/mysqldb/test_mysqldb.py b/tests/contrib/mysqldb/test_mysqldb.py index 9d0d4e29a5e..5d2c98a752c 100644 --- a/tests/contrib/mysqldb/test_mysqldb.py +++ b/tests/contrib/mysqldb/test_mysqldb.py @@ -2,10 +2,10 @@ import MySQLdb import pytest -from ddtrace import Pin -from ddtrace.contrib.mysqldb.patch import patch -from ddtrace.contrib.mysqldb.patch import unpatch +from ddtrace.contrib.internal.mysqldb.patch import patch +from ddtrace.contrib.internal.mysqldb.patch import unpatch from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME +from ddtrace.trace import Pin from tests.contrib import shared_tests from tests.opentracer.utils import init_tracer from tests.utils import TracerTestCase diff --git a/tests/contrib/mysqldb/test_mysqldb_patch.py b/tests/contrib/mysqldb/test_mysqldb_patch.py index e17ce852f56..1822231851f 100644 --- a/tests/contrib/mysqldb/test_mysqldb_patch.py +++ b/tests/contrib/mysqldb/test_mysqldb_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.mysqldb import get_version -from ddtrace.contrib.mysqldb.patch import patch +from ddtrace.contrib.internal.mysqldb.patch import get_version +from ddtrace.contrib.internal.mysqldb.patch import patch try: - from ddtrace.contrib.mysqldb.patch import unpatch + from ddtrace.contrib.internal.mysqldb.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/openai/conftest.py b/tests/contrib/openai/conftest.py index d7cb896b704..02311649bde 100644 --- a/tests/contrib/openai/conftest.py +++ b/tests/contrib/openai/conftest.py @@ -7,11 +7,11 @@ import mock import pytest -from ddtrace import Pin -from ddtrace import patch -from ddtrace.contrib.openai.patch import unpatch -from ddtrace.filters import TraceFilter +from ddtrace.contrib.internal.openai.patch import patch +from ddtrace.contrib.internal.openai.patch import unpatch from ddtrace.llmobs import LLMObs +from ddtrace.trace import Pin +from ddtrace.trace import TraceFilter from tests.utils import DummyTracer from tests.utils import DummyWriter from tests.utils import override_config @@ -165,7 +165,7 @@ def patch_openai(ddtrace_global_config, ddtrace_config_openai, openai_api_key, o if api_key_in_env: openai.api_key = openai_api_key openai.organization = openai_organization - patch(openai=True) + patch() yield unpatch() diff --git a/tests/contrib/openai/test_openai_llmobs.py b/tests/contrib/openai/test_openai_llmobs.py index a1a2b93a5ca..a145877c8c8 100644 --- a/tests/contrib/openai/test_openai_llmobs.py +++ b/tests/contrib/openai/test_openai_llmobs.py @@ -518,11 +518,17 @@ async def test_chat_completion_azure_async( ) ) - def test_chat_completion_stream(self, openai, ddtrace_global_config, mock_llmobs_writer, mock_tracer): + @pytest.mark.skipif( + parse_version(openai_module.version.VERSION) < (1, 26), reason="Stream options only available openai >= 1.26" + ) + def test_chat_completion_stream_explicit_no_tokens( + self, openai, ddtrace_global_config, mock_llmobs_writer, mock_tracer + ): """Ensure llmobs records are emitted for chat completion endpoints when configured. Also ensure the llmobs records have the correct tagging including trace/span ID for trace correlation. """ + with get_openai_vcr(subdirectory_name="v1").use_cassette("chat_completion_streamed.yaml"): with mock.patch("ddtrace.contrib.internal.openai.utils.encoding_for_model", create=True) as mock_encoding: with mock.patch("ddtrace.contrib.internal.openai.utils._est_tokens") as mock_est: @@ -534,7 +540,11 @@ def test_chat_completion_stream(self, openai, ddtrace_global_config, mock_llmobs expected_completion = "The Los Angeles Dodgers won the World Series in 2020." client = openai.OpenAI() resp = client.chat.completions.create( - model=model, messages=input_messages, stream=True, user="ddtrace-test" + model=model, + messages=input_messages, + stream=True, + user="ddtrace-test", + stream_options={"include_usage": False}, ) for chunk in resp: resp_model = chunk.model @@ -547,7 +557,7 @@ def test_chat_completion_stream(self, openai, ddtrace_global_config, mock_llmobs model_provider="openai", input_messages=input_messages, output_messages=[{"content": expected_completion, "role": "assistant"}], - metadata={"stream": True, "user": "ddtrace-test"}, + metadata={"stream": True, "stream_options": {"include_usage": False}, "user": "ddtrace-test"}, token_metrics={"input_tokens": 8, "output_tokens": 8, "total_tokens": 16}, tags={"ml_app": "", "service": "tests.contrib.openai"}, ) @@ -557,20 +567,14 @@ def test_chat_completion_stream(self, openai, ddtrace_global_config, mock_llmobs parse_version(openai_module.version.VERSION) < (1, 26, 0), reason="Streamed tokens available in 1.26.0+" ) def test_chat_completion_stream_tokens(self, openai, ddtrace_global_config, mock_llmobs_writer, mock_tracer): - """ - Ensure llmobs records are emitted for chat completion endpoints when configured - with the `stream_options={"include_usage": True}`. - Also ensure the llmobs records have the correct tagging including trace/span ID for trace correlation. - """ + """Assert that streamed token chunk extraction logic works when options are not explicitly passed from user.""" with get_openai_vcr(subdirectory_name="v1").use_cassette("chat_completion_streamed_tokens.yaml"): model = "gpt-3.5-turbo" resp_model = model input_messages = [{"role": "user", "content": "Who won the world series in 2020?"}] expected_completion = "The Los Angeles Dodgers won the World Series in 2020." client = openai.OpenAI() - resp = client.chat.completions.create( - model=model, messages=input_messages, stream=True, stream_options={"include_usage": True} - ) + resp = client.chat.completions.create(model=model, messages=input_messages, stream=True) for chunk in resp: resp_model = chunk.model span = mock_tracer.pop_traces()[0][0] @@ -671,7 +675,6 @@ def test_chat_completion_tool_call_stream(self, openai, ddtrace_global_config, m messages=[{"role": "user", "content": chat_completion_input_description}], user="ddtrace-test", stream=True, - stream_options={"include_usage": True}, ) for chunk in resp: resp_model = chunk.model diff --git a/tests/contrib/openai/test_openai_patch.py b/tests/contrib/openai/test_openai_patch.py index e2fa8cb88c3..caab79117cf 100644 --- a/tests/contrib/openai/test_openai_patch.py +++ b/tests/contrib/openai/test_openai_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.openai import get_version -from ddtrace.contrib.openai.patch import patch +from ddtrace.contrib.internal.openai.patch import get_version +from ddtrace.contrib.internal.openai.patch import patch try: - from ddtrace.contrib.openai.patch import unpatch + from ddtrace.contrib.internal.openai.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/openai/test_openai_v0.py b/tests/contrib/openai/test_openai_v0.py index 04654f4a4cf..c9ab8aac716 100644 --- a/tests/contrib/openai/test_openai_v0.py +++ b/tests/contrib/openai/test_openai_v0.py @@ -1476,7 +1476,7 @@ def test_integration_sync(openai_api_key, ddtrace_run_python_code_in_subprocess) import ddtrace from tests.contrib.openai.conftest import FilterOrg from tests.contrib.openai.test_openai_v0 import get_openai_vcr -pin = ddtrace.Pin.get_from(openai) +pin = ddtrace.trace.Pin.get_from(openai) pin.tracer.configure(settings={"FILTERS": [FilterOrg()]}) with get_openai_vcr(subdirectory_name="v0").use_cassette("completion.yaml"): resp = openai.Completion.create(model="ada", prompt="Hello world", temperature=0.8, n=2, stop=".", max_tokens=10) @@ -1527,7 +1527,7 @@ def test_integration_async(openai_api_key, ddtrace_run_python_code_in_subprocess import ddtrace from tests.contrib.openai.conftest import FilterOrg from tests.contrib.openai.test_openai_v0 import get_openai_vcr -pin = ddtrace.Pin.get_from(openai) +pin = ddtrace.trace.Pin.get_from(openai) pin.tracer.configure(settings={"FILTERS": [FilterOrg()]}) async def task(): with get_openai_vcr(subdirectory_name="v0").use_cassette("completion_async.yaml"): @@ -1900,7 +1900,7 @@ def test_integration_service_name(openai_api_key, ddtrace_run_python_code_in_sub import ddtrace from tests.contrib.openai.conftest import FilterOrg from tests.contrib.openai.test_openai_v0 import get_openai_vcr -pin = ddtrace.Pin.get_from(openai) +pin = ddtrace.trace.Pin.get_from(openai) pin.tracer.configure(settings={"FILTERS": [FilterOrg()]}) with get_openai_vcr(subdirectory_name="v0").use_cassette("completion.yaml"): resp = openai.Completion.create(model="ada", prompt="Hello world", temperature=0.8, n=2, stop=".", max_tokens=10) diff --git a/tests/contrib/openai/test_openai_v1.py b/tests/contrib/openai/test_openai_v1.py index f13de144fc5..9c9738cbb69 100644 --- a/tests/contrib/openai/test_openai_v1.py +++ b/tests/contrib/openai/test_openai_v1.py @@ -921,128 +921,78 @@ def test_span_finish_on_stream_error(openai, openai_vcr, snapshot_tracer): ) -def test_completion_stream(openai, openai_vcr, mock_metrics, mock_tracer): +@pytest.mark.snapshot +@pytest.mark.skipif(TIKTOKEN_AVAILABLE, reason="This test estimates token counts") +def test_completion_stream_est_tokens(openai, openai_vcr, mock_metrics, snapshot_tracer): with openai_vcr.use_cassette("completion_streamed.yaml"): with mock.patch("ddtrace.contrib.internal.openai.utils.encoding_for_model", create=True) as mock_encoding: mock_encoding.return_value.encode.side_effect = lambda x: [1, 2] - expected_completion = '! ... A page layouts page drawer? ... Interesting. The "Tools" is' client = openai.OpenAI() resp = client.completions.create(model="ada", prompt="Hello world", stream=True, n=None) - chunks = [c for c in resp] - - completion = "".join([c.choices[0].text for c in chunks]) - assert completion == expected_completion + _ = [c for c in resp] - traces = mock_tracer.pop_traces() - assert len(traces) == 1 - assert len(traces[0]) == 1 - assert traces[0][0].get_tag("openai.response.choices.0.text") == expected_completion - assert traces[0][0].get_tag("openai.response.choices.0.finish_reason") == "length" - expected_tags = [ - "version:", - "env:", - "service:tests.contrib.openai", - "openai.request.model:ada", - "model:ada", - "openai.request.endpoint:/v1/completions", - "openai.request.method:POST", - "openai.organization.id:", - "openai.organization.name:datadog-4", - "openai.user.api_key:sk-...key>", - "error:0", - "openai.estimated:true", - ] - if TIKTOKEN_AVAILABLE: - expected_tags = expected_tags[:-1] - assert mock.call.distribution("tokens.prompt", 2, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.distribution("tokens.completion", mock.ANY, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.distribution("tokens.total", mock.ANY, tags=expected_tags) in mock_metrics.mock_calls +@pytest.mark.skipif(not TIKTOKEN_AVAILABLE, reason="This test computes token counts using tiktoken") +@pytest.mark.snapshot(token="tests.contrib.openai.test_openai.test_completion_stream") +def test_completion_stream(openai, openai_vcr, mock_metrics, snapshot_tracer): + with openai_vcr.use_cassette("completion_streamed.yaml"): + with mock.patch("ddtrace.contrib.internal.openai.utils.encoding_for_model", create=True) as mock_encoding: + mock_encoding.return_value.encode.side_effect = lambda x: [1, 2] + client = openai.OpenAI() + resp = client.completions.create(model="ada", prompt="Hello world", stream=True, n=None) + _ = [c for c in resp] -async def test_completion_async_stream(openai, openai_vcr, mock_metrics, mock_tracer): +@pytest.mark.skipif(not TIKTOKEN_AVAILABLE, reason="This test computes token counts using tiktoken") +@pytest.mark.snapshot(token="tests.contrib.openai.test_openai.test_completion_stream") +async def test_completion_async_stream(openai, openai_vcr, mock_metrics, snapshot_tracer): with openai_vcr.use_cassette("completion_streamed.yaml"): with mock.patch("ddtrace.contrib.internal.openai.utils.encoding_for_model", create=True) as mock_encoding: mock_encoding.return_value.encode.side_effect = lambda x: [1, 2] - expected_completion = '! ... A page layouts page drawer? ... Interesting. The "Tools" is' client = openai.AsyncOpenAI() - resp = await client.completions.create(model="ada", prompt="Hello world", stream=True) - chunks = [c async for c in resp] - - completion = "".join([c.choices[0].text for c in chunks]) - assert completion == expected_completion - - traces = mock_tracer.pop_traces() - assert len(traces) == 1 - assert len(traces[0]) == 1 - assert traces[0][0].get_tag("openai.response.choices.0.text") == expected_completion - assert traces[0][0].get_tag("openai.response.choices.0.finish_reason") == "length" - - expected_tags = [ - "version:", - "env:", - "service:tests.contrib.openai", - "openai.request.model:ada", - "model:ada", - "openai.request.endpoint:/v1/completions", - "openai.request.method:POST", - "openai.organization.id:", - "openai.organization.name:datadog-4", - "openai.user.api_key:sk-...key>", - "error:0", - "openai.estimated:true", - ] - if TIKTOKEN_AVAILABLE: - expected_tags = expected_tags[:-1] - assert mock.call.distribution("tokens.prompt", 2, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.distribution("tokens.completion", mock.ANY, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.distribution("tokens.total", mock.ANY, tags=expected_tags) in mock_metrics.mock_calls + resp = await client.completions.create(model="ada", prompt="Hello world", stream=True, n=None) + _ = [c async for c in resp] @pytest.mark.skipif( - parse_version(openai_module.version.VERSION) < (1, 6, 0), + parse_version(openai_module.version.VERSION) < (1, 6, 0) or not TIKTOKEN_AVAILABLE, reason="Streamed response context managers are only available v1.6.0+", ) -def test_completion_stream_context_manager(openai, openai_vcr, mock_metrics, mock_tracer): +@pytest.mark.snapshot(token="tests.contrib.openai.test_openai.test_completion_stream") +def test_completion_stream_context_manager(openai, openai_vcr, mock_metrics, snapshot_tracer): with openai_vcr.use_cassette("completion_streamed.yaml"): with mock.patch("ddtrace.contrib.internal.openai.utils.encoding_for_model", create=True) as mock_encoding: mock_encoding.return_value.encode.side_effect = lambda x: [1, 2] - expected_completion = '! ... A page layouts page drawer? ... Interesting. The "Tools" is' client = openai.OpenAI() with client.completions.create(model="ada", prompt="Hello world", stream=True, n=None) as resp: - chunks = [c for c in resp] + _ = [c for c in resp] - completion = "".join([c.choices[0].text for c in chunks]) - assert completion == expected_completion - - traces = mock_tracer.pop_traces() - assert len(traces) == 1 - assert len(traces[0]) == 1 - assert traces[0][0].get_tag("openai.response.choices.0.text") == expected_completion - assert traces[0][0].get_tag("openai.response.choices.0.finish_reason") == "length" - expected_tags = [ - "version:", - "env:", - "service:tests.contrib.openai", - "openai.request.model:ada", - "model:ada", - "openai.request.endpoint:/v1/completions", - "openai.request.method:POST", - "openai.organization.id:", - "openai.organization.name:datadog-4", - "openai.user.api_key:sk-...key>", - "error:0", - "openai.estimated:true", - ] - if TIKTOKEN_AVAILABLE: - expected_tags = expected_tags[:-1] - assert mock.call.distribution("tokens.prompt", 2, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.distribution("tokens.completion", mock.ANY, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.distribution("tokens.total", mock.ANY, tags=expected_tags) in mock_metrics.mock_calls +@pytest.mark.skipif( + parse_version(openai_module.version.VERSION) < (1, 26), reason="Stream options only available openai >= 1.26" +) +@pytest.mark.snapshot(token="tests.contrib.openai.test_openai.test_chat_completion_stream") +def test_chat_completion_stream(openai, openai_vcr, mock_metrics, snapshot_tracer): + """Assert that streamed token chunk extraction logic works automatically.""" + with openai_vcr.use_cassette("chat_completion_streamed_tokens.yaml"): + with mock.patch("ddtrace.contrib.internal.openai.utils.encoding_for_model", create=True) as mock_encoding: + mock_encoding.return_value.encode.side_effect = lambda x: [1, 2, 3, 4, 5, 6, 7, 8] + client = openai.OpenAI() + resp = client.chat.completions.create( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Who won the world series in 2020?"}], + stream=True, + user="ddtrace-test", + n=None, + ) + _ = [c for c in resp] -def test_chat_completion_stream(openai, openai_vcr, mock_metrics, snapshot_tracer): +@pytest.mark.skipif( + parse_version(openai_module.version.VERSION) < (1, 26), reason="Stream options only available openai >= 1.26" +) +def test_chat_completion_stream_explicit_no_tokens(openai, openai_vcr, mock_metrics, snapshot_tracer): + """Assert that streamed token chunk extraction logic is avoided if explicitly set to False by the user.""" with openai_vcr.use_cassette("chat_completion_streamed.yaml"): with mock.patch("ddtrace.contrib.internal.openai.utils.encoding_for_model", create=True) as mock_encoding: mock_encoding.return_value.encode.side_effect = lambda x: [1, 2, 3, 4, 5, 6, 7, 8] @@ -1054,20 +1004,16 @@ def test_chat_completion_stream(openai, openai_vcr, mock_metrics, snapshot_trace {"role": "user", "content": "Who won the world series in 2020?"}, ], stream=True, + stream_options={"include_usage": False}, user="ddtrace-test", n=None, ) - prompt_tokens = 8 span = snapshot_tracer.current_span() chunks = [c for c in resp] assert len(chunks) == 15 completion = "".join([c.choices[0].delta.content for c in chunks if c.choices[0].delta.content is not None]) assert completion == expected_completion - assert span.get_tag("openai.response.choices.0.message.content") == expected_completion - assert span.get_tag("openai.response.choices.0.message.role") == "assistant" - assert span.get_tag("openai.response.choices.0.finish_reason") == "stop" - expected_tags = [ "version:", "env:", @@ -1087,16 +1033,19 @@ def test_chat_completion_stream(openai, openai_vcr, mock_metrics, snapshot_trace expected_tags += ["openai.estimated:true"] if TIKTOKEN_AVAILABLE: expected_tags = expected_tags[:-1] - assert mock.call.distribution("tokens.prompt", prompt_tokens, tags=expected_tags) in mock_metrics.mock_calls + assert mock.call.distribution("tokens.prompt", 8, tags=expected_tags) in mock_metrics.mock_calls assert mock.call.distribution("tokens.completion", mock.ANY, tags=expected_tags) in mock_metrics.mock_calls assert mock.call.distribution("tokens.total", mock.ANY, tags=expected_tags) in mock_metrics.mock_calls +@pytest.mark.skipif( + parse_version(openai_module.version.VERSION) < (1, 26, 0), reason="Streamed tokens available in 1.26.0+" +) +@pytest.mark.snapshot(token="tests.contrib.openai.test_openai.test_chat_completion_stream") async def test_chat_completion_async_stream(openai, openai_vcr, mock_metrics, snapshot_tracer): - with openai_vcr.use_cassette("chat_completion_streamed.yaml"): + with openai_vcr.use_cassette("chat_completion_streamed_tokens.yaml"): with mock.patch("ddtrace.contrib.internal.openai.utils.encoding_for_model", create=True) as mock_encoding: mock_encoding.return_value.encode.side_effect = lambda x: [1, 2, 3, 4, 5, 6, 7, 8] - expected_completion = "The Los Angeles Dodgers won the World Series in 2020." client = openai.AsyncOpenAI() resp = await client.chat.completions.create( model="gpt-3.5-turbo", @@ -1104,99 +1053,21 @@ async def test_chat_completion_async_stream(openai, openai_vcr, mock_metrics, sn {"role": "user", "content": "Who won the world series in 2020?"}, ], stream=True, + n=None, user="ddtrace-test", ) - prompt_tokens = 8 - span = snapshot_tracer.current_span() - chunks = [c async for c in resp] - assert len(chunks) == 15 - completion = "".join([c.choices[0].delta.content for c in chunks if c.choices[0].delta.content is not None]) - assert completion == expected_completion - - assert span.get_tag("openai.response.choices.0.message.content") == expected_completion - assert span.get_tag("openai.response.choices.0.message.role") == "assistant" - assert span.get_tag("openai.response.choices.0.finish_reason") == "stop" - - expected_tags = [ - "version:", - "env:", - "service:tests.contrib.openai", - "openai.request.model:gpt-3.5-turbo", - "model:gpt-3.5-turbo", - "openai.request.endpoint:/v1/chat/completions", - "openai.request.method:POST", - "openai.organization.id:", - "openai.organization.name:datadog-4", - "openai.user.api_key:sk-...key>", - "error:0", - ] - assert mock.call.distribution("request.duration", span.duration_ns, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.gauge("ratelimit.requests", 3000, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.gauge("ratelimit.remaining.requests", 2999, tags=expected_tags) in mock_metrics.mock_calls - expected_tags += ["openai.estimated:true"] - if TIKTOKEN_AVAILABLE: - expected_tags = expected_tags[:-1] - assert mock.call.distribution("tokens.prompt", prompt_tokens, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.distribution("tokens.completion", mock.ANY, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.distribution("tokens.total", mock.ANY, tags=expected_tags) in mock_metrics.mock_calls - - -@pytest.mark.skipif( - parse_version(openai_module.version.VERSION) < (1, 26, 0), reason="Streamed tokens available in 1.26.0+" -) -def test_chat_completion_stream_tokens(openai, openai_vcr, mock_metrics, snapshot_tracer): - with openai_vcr.use_cassette("chat_completion_streamed_tokens.yaml"): - expected_completion = "The Los Angeles Dodgers won the World Series in 2020." - client = openai.OpenAI() - resp = client.chat.completions.create( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Who won the world series in 2020?"}], - stream=True, - user="ddtrace-test", - n=None, - stream_options={"include_usage": True}, - ) - span = snapshot_tracer.current_span() - chunks = [c for c in resp] - completion = "".join( - [c.choices[0].delta.content for c in chunks if c.choices and c.choices[0].delta.content is not None] - ) - assert completion == expected_completion - - assert span.get_tag("openai.response.choices.0.message.content") == expected_completion - assert span.get_tag("openai.response.choices.0.message.role") == "assistant" - assert span.get_tag("openai.response.choices.0.finish_reason") == "stop" - - expected_tags = [ - "version:", - "env:", - "service:tests.contrib.openai", - "openai.request.model:gpt-3.5-turbo", - "model:gpt-3.5-turbo", - "openai.request.endpoint:/v1/chat/completions", - "openai.request.method:POST", - "openai.organization.id:", - "openai.organization.name:datadog-4", - "openai.user.api_key:sk-...key>", - "error:0", - ] - assert mock.call.distribution("request.duration", span.duration_ns, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.gauge("ratelimit.requests", 3000, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.gauge("ratelimit.remaining.requests", 2999, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.distribution("tokens.prompt", 17, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.distribution("tokens.completion", 19, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.distribution("tokens.total", 36, tags=expected_tags) in mock_metrics.mock_calls + _ = [c async for c in resp] @pytest.mark.skipif( - parse_version(openai_module.version.VERSION) < (1, 6, 0), - reason="Streamed response context managers are only available v1.6.0+", + parse_version(openai_module.version.VERSION) < (1, 26, 0), + reason="Streamed response context managers are only available v1.6.0+, tokens available 1.26.0+", ) +@pytest.mark.snapshot(token="tests.contrib.openai.test_openai.test_chat_completion_stream") async def test_chat_completion_async_stream_context_manager(openai, openai_vcr, mock_metrics, snapshot_tracer): - with openai_vcr.use_cassette("chat_completion_streamed.yaml"): + with openai_vcr.use_cassette("chat_completion_streamed_tokens.yaml"): with mock.patch("ddtrace.contrib.internal.openai.utils.encoding_for_model", create=True) as mock_encoding: mock_encoding.return_value.encode.side_effect = lambda x: [1, 2, 3, 4, 5, 6, 7, 8] - expected_completion = "The Los Angeles Dodgers won the World Series in 2020." client = openai.AsyncOpenAI() async with await client.chat.completions.create( model="gpt-3.5-turbo", @@ -1207,41 +1078,7 @@ async def test_chat_completion_async_stream_context_manager(openai, openai_vcr, user="ddtrace-test", n=None, ) as resp: - prompt_tokens = 8 - span = snapshot_tracer.current_span() - chunks = [c async for c in resp] - assert len(chunks) == 15 - completion = "".join( - [c.choices[0].delta.content for c in chunks if c.choices[0].delta.content is not None] - ) - assert completion == expected_completion - - assert span.get_tag("openai.response.choices.0.message.content") == expected_completion - assert span.get_tag("openai.response.choices.0.message.role") == "assistant" - assert span.get_tag("openai.response.choices.0.finish_reason") == "stop" - - expected_tags = [ - "version:", - "env:", - "service:tests.contrib.openai", - "openai.request.model:gpt-3.5-turbo", - "model:gpt-3.5-turbo", - "openai.request.endpoint:/v1/chat/completions", - "openai.request.method:POST", - "openai.organization.id:", - "openai.organization.name:datadog-4", - "openai.user.api_key:sk-...key>", - "error:0", - ] - assert mock.call.distribution("request.duration", span.duration_ns, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.gauge("ratelimit.requests", 3000, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.gauge("ratelimit.remaining.requests", 2999, tags=expected_tags) in mock_metrics.mock_calls - expected_tags += ["openai.estimated:true"] - if TIKTOKEN_AVAILABLE: - expected_tags = expected_tags[:-1] - assert mock.call.distribution("tokens.prompt", prompt_tokens, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.distribution("tokens.completion", mock.ANY, tags=expected_tags) in mock_metrics.mock_calls - assert mock.call.distribution("tokens.total", mock.ANY, tags=expected_tags) in mock_metrics.mock_calls + _ = [c async for c in resp] @pytest.mark.snapshot( @@ -1274,7 +1111,7 @@ def test_integration_sync(openai_api_key, ddtrace_run_python_code_in_subprocess) import ddtrace from tests.contrib.openai.conftest import FilterOrg from tests.contrib.openai.test_openai_v1 import get_openai_vcr -pin = ddtrace.Pin.get_from(openai) +pin = ddtrace.trace.Pin.get_from(openai) pin.tracer.configure(settings={"FILTERS": [FilterOrg()]}) with get_openai_vcr(subdirectory_name="v1").use_cassette("completion.yaml"): client = openai.OpenAI() @@ -1322,7 +1159,7 @@ def test_integration_async(openai_api_key, ddtrace_run_python_code_in_subprocess import ddtrace from tests.contrib.openai.conftest import FilterOrg from tests.contrib.openai.test_openai_v1 import get_openai_vcr -pin = ddtrace.Pin.get_from(openai) +pin = ddtrace.trace.Pin.get_from(openai) pin.tracer.configure(settings={"FILTERS": [FilterOrg()]}) async def task(): with get_openai_vcr(subdirectory_name="v1").use_cassette("completion.yaml"): @@ -1710,7 +1547,7 @@ def test_integration_service_name(openai_api_key, ddtrace_run_python_code_in_sub import ddtrace from tests.contrib.openai.conftest import FilterOrg from tests.contrib.openai.test_openai_v1 import get_openai_vcr -pin = ddtrace.Pin.get_from(openai) +pin = ddtrace.trace.Pin.get_from(openai) pin.tracer.configure(settings={"FILTERS": [FilterOrg()]}) with get_openai_vcr(subdirectory_name="v1").use_cassette("completion.yaml"): client = openai.OpenAI() diff --git a/tests/contrib/patch.py b/tests/contrib/patch.py index 913b3895894..fabf117f40a 100644 --- a/tests/contrib/patch.py +++ b/tests/contrib/patch.py @@ -150,7 +150,7 @@ class Base(SubprocessTestCase, PatchMixin): Example: A simple implementation inheriting this TestCase looks like:: - from ddtrace.contrib.redis import patch, unpatch + from ddtrace.contrib.internal.redis.patch import patch, unpatch class RedisPatchTestCase(PatchTestCase.Base): __integration_name__ = 'redis' @@ -454,7 +454,7 @@ def test_import_patch_unpatch_patch(self): For example:: import redis - from ddtrace.contrib.redis import unpatch + from ddtrace.contrib.internal.redis.patch import unpatch ddtrace.patch(redis=True) unpatch() @@ -480,7 +480,7 @@ def test_patch_import_unpatch_patch(self): For example:: - from ddtrace.contrib.redis import unpatch + from ddtrace.contrib.internal.redis.patch import unpatch ddtrace.patch(redis=True) import redis @@ -506,7 +506,7 @@ def test_patch_unpatch_import_patch(self): For example:: - from ddtrace.contrib.redis import unpatch + from ddtrace.contrib.internal.redis.patch import unpatch ddtrace.patch(redis=True) import redis @@ -531,7 +531,7 @@ def test_patch_unpatch_patch_import(self): For example:: - from ddtrace.contrib.redis import unpatch + from ddtrace.contrib.internal.redis.patch import unpatch ddtrace.patch(redis=True) unpatch() @@ -553,7 +553,7 @@ def test_unpatch_patch_import(self): For example:: - from ddtrace.contrib.redis import unpatch + from ddtrace.contrib.internal.redis.patch import unpatch unpatch() ddtrace.patch(redis=True) import redis @@ -576,7 +576,7 @@ def test_patch_unpatch_import(self): For example:: ddtrace.patch(redis=True) - from ddtrace.contrib.redis import unpatch + from ddtrace.contrib.internal.redis.patch import unpatch unpatch() import redis self.assert_not_module_patched(redis) @@ -596,7 +596,7 @@ def test_import_unpatch_patch(self): For example:: import redis - from ddtrace.contrib.redis import unpatch + from ddtrace.contrib.internal.redis.patch import unpatch ddtrace.patch(redis=True) unpatch() self.assert_not_module_patched(redis) @@ -617,7 +617,7 @@ def test_import_patch_unpatch(self): For example:: import redis - from ddtrace.contrib.redis import unpatch + from ddtrace.contrib.internal.redis.patch import unpatch ddtrace.patch(redis=True) unpatch() self.assert_not_module_patched(redis) @@ -638,7 +638,7 @@ def test_patch_import_unpatch(self): For example:: - from ddtrace.contrib.redis import unpatch + from ddtrace.contrib.internal.redis.patch import unpatch ddtrace.patch(redis=True) import redis unpatch() @@ -659,7 +659,7 @@ def test_import_patch_unpatch_unpatch(self): For example:: import redis - from ddtrace.contrib.redis import unpatch + from ddtrace.contrib.internal.redis.patch import unpatch ddtrace.patch(redis=True) self.assert_module_patched(redis) @@ -684,7 +684,7 @@ def test_patch_unpatch_import_unpatch(self): For example:: - from ddtrace.contrib.redis import unpatch + from ddtrace.contrib.internal.redis.patch import unpatch ddtrace.patch(redis=True) unpatch() @@ -708,7 +708,7 @@ def test_patch_unpatch_unpatch_import(self): For example:: - from ddtrace.contrib.redis import unpatch + from ddtrace.contrib.internal.redis.patch import unpatch ddtrace.patch(redis=True) unpatch() @@ -748,7 +748,7 @@ def patch_wrapper(wrapped, _, args, kwrags): wrap(module.__name__, module.patch.__name__, patch_wrapper) - ModuleWatchdog.register_module_hook("ddtrace.contrib.%s.patch", patch_hook) + ModuleWatchdog.register_module_hook("ddtrace.contrib.internal.%s.patch", patch_hook) sys.stdout.write("O") diff --git a/tests/contrib/psycopg/test_psycopg.py b/tests/contrib/psycopg/test_psycopg.py index f21915548fb..8e13ecc4128 100644 --- a/tests/contrib/psycopg/test_psycopg.py +++ b/tests/contrib/psycopg/test_psycopg.py @@ -8,11 +8,11 @@ from psycopg.sql import Identifier from psycopg.sql import Literal -from ddtrace import Pin -from ddtrace.contrib.psycopg.patch import patch -from ddtrace.contrib.psycopg.patch import unpatch +from ddtrace.contrib.internal.psycopg.patch import patch +from ddtrace.contrib.internal.psycopg.patch import unpatch from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME from ddtrace.internal.utils.version import parse_version +from ddtrace.trace import Pin from tests.contrib.config import POSTGRES_CONFIG from tests.opentracer.utils import init_tracer from tests.utils import TracerTestCase diff --git a/tests/contrib/psycopg/test_psycopg_async.py b/tests/contrib/psycopg/test_psycopg_async.py index 08b84e0afd0..7e4fbd59624 100644 --- a/tests/contrib/psycopg/test_psycopg_async.py +++ b/tests/contrib/psycopg/test_psycopg_async.py @@ -5,9 +5,9 @@ from psycopg.sql import SQL from psycopg.sql import Literal -from ddtrace import Pin -from ddtrace.contrib.psycopg.patch import patch -from ddtrace.contrib.psycopg.patch import unpatch +from ddtrace.contrib.internal.psycopg.patch import patch +from ddtrace.contrib.internal.psycopg.patch import unpatch +from ddtrace.trace import Pin from tests.contrib.asyncio.utils import AsyncioTestCase from tests.contrib.config import POSTGRES_CONFIG from tests.opentracer.utils import init_tracer diff --git a/tests/contrib/psycopg/test_psycopg_patch.py b/tests/contrib/psycopg/test_psycopg_patch.py index 8d91b2b07d7..1dd316b0423 100644 --- a/tests/contrib/psycopg/test_psycopg_patch.py +++ b/tests/contrib/psycopg/test_psycopg_patch.py @@ -3,13 +3,13 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.psycopg.patch import get_version -from ddtrace.contrib.psycopg.patch import get_versions -from ddtrace.contrib.psycopg.patch import patch +from ddtrace.contrib.internal.psycopg.patch import get_version +from ddtrace.contrib.internal.psycopg.patch import get_versions +from ddtrace.contrib.internal.psycopg.patch import patch try: - from ddtrace.contrib.psycopg.patch import unpatch + from ddtrace.contrib.internal.psycopg.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/psycopg/test_psycopg_snapshot.py b/tests/contrib/psycopg/test_psycopg_snapshot.py index 00b0a679991..c4f904bfaa4 100644 --- a/tests/contrib/psycopg/test_psycopg_snapshot.py +++ b/tests/contrib/psycopg/test_psycopg_snapshot.py @@ -5,8 +5,8 @@ import pytest import wrapt -from ddtrace.contrib.psycopg.patch import patch -from ddtrace.contrib.psycopg.patch import unpatch +from ddtrace.contrib.internal.psycopg.patch import patch +from ddtrace.contrib.internal.psycopg.patch import unpatch @pytest.fixture(autouse=True) diff --git a/tests/contrib/psycopg2/test_psycopg.py b/tests/contrib/psycopg2/test_psycopg.py index 7b13114f064..902d24d3c0e 100644 --- a/tests/contrib/psycopg2/test_psycopg.py +++ b/tests/contrib/psycopg2/test_psycopg.py @@ -7,11 +7,11 @@ from psycopg2 import extensions from psycopg2 import extras -from ddtrace import Pin -from ddtrace.contrib.psycopg.patch import patch -from ddtrace.contrib.psycopg.patch import unpatch +from ddtrace.contrib.internal.psycopg.patch import patch +from ddtrace.contrib.internal.psycopg.patch import unpatch from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME from ddtrace.internal.utils.version import parse_version +from ddtrace.trace import Pin from tests.contrib.config import POSTGRES_CONFIG from tests.opentracer.utils import init_tracer from tests.utils import TracerTestCase diff --git a/tests/contrib/psycopg2/test_psycopg_patch.py b/tests/contrib/psycopg2/test_psycopg_patch.py index 21172763b77..1d3db62ec76 100644 --- a/tests/contrib/psycopg2/test_psycopg_patch.py +++ b/tests/contrib/psycopg2/test_psycopg_patch.py @@ -3,13 +3,13 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.psycopg.patch import get_version -from ddtrace.contrib.psycopg.patch import get_versions -from ddtrace.contrib.psycopg.patch import patch +from ddtrace.contrib.internal.psycopg.patch import get_version +from ddtrace.contrib.internal.psycopg.patch import get_versions +from ddtrace.contrib.internal.psycopg.patch import patch try: - from ddtrace.contrib.psycopg.patch import unpatch + from ddtrace.contrib.internal.psycopg.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/psycopg2/test_psycopg_snapshot.py b/tests/contrib/psycopg2/test_psycopg_snapshot.py index c3210077598..6947faf15b8 100644 --- a/tests/contrib/psycopg2/test_psycopg_snapshot.py +++ b/tests/contrib/psycopg2/test_psycopg_snapshot.py @@ -4,8 +4,8 @@ import pytest import wrapt -from ddtrace.contrib.psycopg.patch import patch -from ddtrace.contrib.psycopg.patch import unpatch +from ddtrace.contrib.internal.psycopg.patch import patch +from ddtrace.contrib.internal.psycopg.patch import unpatch @pytest.fixture(autouse=True) diff --git a/tests/contrib/pylibmc/test.py b/tests/contrib/pylibmc/test.py index 2cd698681bf..9de012439dc 100644 --- a/tests/contrib/pylibmc/test.py +++ b/tests/contrib/pylibmc/test.py @@ -5,12 +5,13 @@ # 3p import pylibmc -# project -from ddtrace import Pin -from ddtrace.contrib.pylibmc import TracedClient -from ddtrace.contrib.pylibmc.patch import patch -from ddtrace.contrib.pylibmc.patch import unpatch +from ddtrace.contrib.internal.pylibmc.client import TracedClient +from ddtrace.contrib.internal.pylibmc.patch import patch +from ddtrace.contrib.internal.pylibmc.patch import unpatch from ddtrace.ext import memcached + +# project +from ddtrace.trace import Pin from tests.contrib.config import MEMCACHED_CONFIG as cfg from tests.opentracer.utils import init_tracer from tests.utils import TracerTestCase diff --git a/tests/contrib/pylibmc/test_pylibmc_patch.py b/tests/contrib/pylibmc/test_pylibmc_patch.py index 1ca3c413827..5c4a94ac76f 100644 --- a/tests/contrib/pylibmc/test_pylibmc_patch.py +++ b/tests/contrib/pylibmc/test_pylibmc_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.pylibmc import get_version -from ddtrace.contrib.pylibmc.patch import patch +from ddtrace.contrib.internal.pylibmc.patch import get_version +from ddtrace.contrib.internal.pylibmc.patch import patch try: - from ddtrace.contrib.pylibmc.patch import unpatch + from ddtrace.contrib.internal.pylibmc.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/pymemcache/test_client.py b/tests/contrib/pymemcache/test_client.py index 66f4637e15a..19a7a93d523 100644 --- a/tests/contrib/pymemcache/test_client.py +++ b/tests/contrib/pymemcache/test_client.py @@ -9,12 +9,13 @@ import pytest import wrapt -# project -from ddtrace import Pin -from ddtrace.contrib.pymemcache.client import WrappedClient -from ddtrace.contrib.pymemcache.patch import patch -from ddtrace.contrib.pymemcache.patch import unpatch +from ddtrace.contrib.internal.pymemcache.client import WrappedClient +from ddtrace.contrib.internal.pymemcache.patch import patch +from ddtrace.contrib.internal.pymemcache.patch import unpatch from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME + +# project +from ddtrace.trace import Pin from tests.utils import DummyTracer from tests.utils import TracerTestCase from tests.utils import override_config diff --git a/tests/contrib/pymemcache/test_client_defaults.py b/tests/contrib/pymemcache/test_client_defaults.py index 550625ced7a..0b5e44aa3b0 100644 --- a/tests/contrib/pymemcache/test_client_defaults.py +++ b/tests/contrib/pymemcache/test_client_defaults.py @@ -2,10 +2,11 @@ import pymemcache import pytest +from ddtrace.contrib.internal.pymemcache.patch import patch +from ddtrace.contrib.internal.pymemcache.patch import unpatch + # project -from ddtrace import Pin -from ddtrace.contrib.pymemcache.patch import patch -from ddtrace.contrib.pymemcache.patch import unpatch +from ddtrace.trace import Pin from tests.utils import override_config from .test_client_mixin import TEST_HOST diff --git a/tests/contrib/pymemcache/test_client_mixin.py b/tests/contrib/pymemcache/test_client_mixin.py index ffe2dba5ca6..2d471765e1f 100644 --- a/tests/contrib/pymemcache/test_client_mixin.py +++ b/tests/contrib/pymemcache/test_client_mixin.py @@ -2,12 +2,13 @@ import pymemcache import pytest -# project -from ddtrace import Pin -from ddtrace.contrib.pymemcache.patch import patch -from ddtrace.contrib.pymemcache.patch import unpatch +from ddtrace.contrib.internal.pymemcache.patch import patch +from ddtrace.contrib.internal.pymemcache.patch import unpatch from ddtrace.ext import memcached as memcachedx from ddtrace.ext import net + +# project +from ddtrace.trace import Pin from tests.utils import DummyTracer from tests.utils import TracerTestCase from tests.utils import override_config diff --git a/tests/contrib/pymemcache/test_pymemcache_patch.py b/tests/contrib/pymemcache/test_pymemcache_patch.py index fcedad008c1..10a49f74dd5 100644 --- a/tests/contrib/pymemcache/test_pymemcache_patch.py +++ b/tests/contrib/pymemcache/test_pymemcache_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.pymemcache import get_version -from ddtrace.contrib.pymemcache.patch import patch +from ddtrace.contrib.internal.pymemcache.patch import get_version +from ddtrace.contrib.internal.pymemcache.patch import patch try: - from ddtrace.contrib.pymemcache.patch import unpatch + from ddtrace.contrib.internal.pymemcache.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/pymongo/test.py b/tests/contrib/pymongo/test.py index f439036fcc1..7c0c1e58140 100644 --- a/tests/contrib/pymongo/test.py +++ b/tests/contrib/pymongo/test.py @@ -3,13 +3,14 @@ import pymongo -# project -from ddtrace import Pin from ddtrace.contrib.internal.pymongo.client import normalize_filter from ddtrace.contrib.internal.pymongo.patch import _CHECKOUT_FN_NAME from ddtrace.contrib.internal.pymongo.patch import patch from ddtrace.contrib.internal.pymongo.patch import unpatch from ddtrace.ext import SpanTypes + +# project +from ddtrace.trace import Pin from tests.opentracer.utils import init_tracer from tests.utils import DummyTracer from tests.utils import TracerTestCase diff --git a/tests/contrib/pymongo/test_pymongo_patch.py b/tests/contrib/pymongo/test_pymongo_patch.py index b1c51ea36a3..5e3ac9480ba 100644 --- a/tests/contrib/pymongo/test_pymongo_patch.py +++ b/tests/contrib/pymongo/test_pymongo_patch.py @@ -2,10 +2,10 @@ # script. If you want to make changes to it, you should make sure that you have # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.pymongo.patch import get_version -from ddtrace.contrib.pymongo.patch import patch -from ddtrace.contrib.pymongo.patch import pymongo -from ddtrace.contrib.pymongo.patch import unpatch +from ddtrace.contrib.internal.pymongo.patch import get_version +from ddtrace.contrib.internal.pymongo.patch import patch +from ddtrace.contrib.internal.pymongo.patch import pymongo +from ddtrace.contrib.internal.pymongo.patch import unpatch from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/pymongo/test_spec.py b/tests/contrib/pymongo/test_spec.py index fe1339e538e..a297dad5b57 100644 --- a/tests/contrib/pymongo/test_spec.py +++ b/tests/contrib/pymongo/test_spec.py @@ -4,7 +4,7 @@ from bson.son import SON -from ddtrace.contrib.pymongo.parse import parse_spec +from ddtrace.contrib.internal.pymongo.parse import parse_spec def test_empty(): diff --git a/tests/contrib/pymysql/test_pymysql.py b/tests/contrib/pymysql/test_pymysql.py index 207ccc1c4ba..e94e03c8395 100644 --- a/tests/contrib/pymysql/test_pymysql.py +++ b/tests/contrib/pymysql/test_pymysql.py @@ -1,10 +1,10 @@ import mock import pymysql -from ddtrace import Pin -from ddtrace.contrib.pymysql.patch import patch -from ddtrace.contrib.pymysql.patch import unpatch +from ddtrace.contrib.internal.pymysql.patch import patch +from ddtrace.contrib.internal.pymysql.patch import unpatch from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME +from ddtrace.trace import Pin from tests.contrib import shared_tests from tests.opentracer.utils import init_tracer from tests.utils import TracerTestCase diff --git a/tests/contrib/pymysql/test_pymysql_patch.py b/tests/contrib/pymysql/test_pymysql_patch.py index cb48bcda3dd..2d6e518a7f8 100644 --- a/tests/contrib/pymysql/test_pymysql_patch.py +++ b/tests/contrib/pymysql/test_pymysql_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.pymysql import get_version -from ddtrace.contrib.pymysql.patch import patch +from ddtrace.contrib.internal.pymysql.patch import get_version +from ddtrace.contrib.internal.pymysql.patch import patch try: - from ddtrace.contrib.pymysql.patch import unpatch + from ddtrace.contrib.internal.pymysql.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/pynamodb/test_pynamodb.py b/tests/contrib/pynamodb/test_pynamodb.py index 2ba0ab7e5c4..33b4e4c2c14 100644 --- a/tests/contrib/pynamodb/test_pynamodb.py +++ b/tests/contrib/pynamodb/test_pynamodb.py @@ -4,10 +4,10 @@ from pynamodb.connection.base import Connection import pytest -from ddtrace import Pin -from ddtrace.contrib.pynamodb.patch import patch -from ddtrace.contrib.pynamodb.patch import unpatch +from ddtrace.contrib.internal.pynamodb.patch import patch +from ddtrace.contrib.internal.pynamodb.patch import unpatch from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME +from ddtrace.trace import Pin from tests.utils import TracerTestCase from tests.utils import assert_is_measured diff --git a/tests/contrib/pynamodb/test_pynamodb_patch.py b/tests/contrib/pynamodb/test_pynamodb_patch.py index 36034c9dabc..e2b9cd41b7b 100644 --- a/tests/contrib/pynamodb/test_pynamodb_patch.py +++ b/tests/contrib/pynamodb/test_pynamodb_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.pynamodb import get_version -from ddtrace.contrib.pynamodb.patch import patch +from ddtrace.contrib.internal.pynamodb.patch import get_version +from ddtrace.contrib.internal.pynamodb.patch import patch try: - from ddtrace.contrib.pynamodb.patch import unpatch + from ddtrace.contrib.internal.pynamodb.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/pyodbc/test_pyodbc.py b/tests/contrib/pyodbc/test_pyodbc.py index dbbde10187f..4c965aede7b 100644 --- a/tests/contrib/pyodbc/test_pyodbc.py +++ b/tests/contrib/pyodbc/test_pyodbc.py @@ -1,9 +1,9 @@ import pyodbc -from ddtrace import Pin -from ddtrace.contrib.pyodbc.patch import patch -from ddtrace.contrib.pyodbc.patch import unpatch +from ddtrace.contrib.internal.pyodbc.patch import patch +from ddtrace.contrib.internal.pyodbc.patch import unpatch from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME +from ddtrace.trace import Pin from tests.utils import TracerTestCase from tests.utils import assert_is_measured diff --git a/tests/contrib/pyodbc/test_pyodbc_patch.py b/tests/contrib/pyodbc/test_pyodbc_patch.py index 29ad4204d6a..0489828949c 100644 --- a/tests/contrib/pyodbc/test_pyodbc_patch.py +++ b/tests/contrib/pyodbc/test_pyodbc_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.pyodbc import get_version -from ddtrace.contrib.pyodbc.patch import patch +from ddtrace.contrib.internal.pyodbc.patch import get_version +from ddtrace.contrib.internal.pyodbc.patch import patch try: - from ddtrace.contrib.pyodbc.patch import unpatch + from ddtrace.contrib.internal.pyodbc.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/pyramid/app/web.py b/tests/contrib/pyramid/app/web.py index abf4b20023f..a250a97ce9d 100644 --- a/tests/contrib/pyramid/app/web.py +++ b/tests/contrib/pyramid/app/web.py @@ -6,7 +6,7 @@ from pyramid.renderers import render_to_response from pyramid.response import Response -from ddtrace.contrib.pyramid import trace_pyramid +from ddtrace.contrib.internal.pyramid.trace import trace_pyramid def create_app(settings, instrument): diff --git a/tests/contrib/pyramid/pserve_app/app/__init__.py b/tests/contrib/pyramid/pserve_app/app/__init__.py index 13b4b58b6ab..69ba7b14d8b 100644 --- a/tests/contrib/pyramid/pserve_app/app/__init__.py +++ b/tests/contrib/pyramid/pserve_app/app/__init__.py @@ -2,7 +2,7 @@ from pyramid.response import Response from ddtrace import tracer -from ddtrace.filters import TraceFilter +from ddtrace.trace import TraceFilter class PingFilter(TraceFilter): diff --git a/tests/contrib/pyramid/test_pyramid_patch.py b/tests/contrib/pyramid/test_pyramid_patch.py index 7aec4658e25..aa22e9ed62c 100644 --- a/tests/contrib/pyramid/test_pyramid_patch.py +++ b/tests/contrib/pyramid/test_pyramid_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.pyramid import get_version -from ddtrace.contrib.pyramid.patch import patch +from ddtrace.contrib.internal.pyramid.patch import get_version +from ddtrace.contrib.internal.pyramid.patch import patch try: - from ddtrace.contrib.pyramid.patch import unpatch + from ddtrace.contrib.internal.pyramid.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/pytest/test_coverage_per_suite.py b/tests/contrib/pytest/test_coverage_per_suite.py index a9c985fa7b4..adb2a710c76 100644 --- a/tests/contrib/pytest/test_coverage_per_suite.py +++ b/tests/contrib/pytest/test_coverage_per_suite.py @@ -4,8 +4,8 @@ import pytest -from ddtrace.contrib.pytest._utils import _USE_PLUGIN_V2 -from ddtrace.contrib.pytest._utils import _pytest_version_supports_itr +from ddtrace.contrib.internal.pytest._utils import _USE_PLUGIN_V2 +from ddtrace.contrib.internal.pytest._utils import _pytest_version_supports_itr from ddtrace.ext.test_visibility import ITR_SKIPPING_LEVEL from ddtrace.internal.ci_visibility._api_client import ITRData from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings diff --git a/tests/contrib/pytest/test_pytest.py b/tests/contrib/pytest/test_pytest.py index eeefa59f714..267a9d97eac 100644 --- a/tests/contrib/pytest/test_pytest.py +++ b/tests/contrib/pytest/test_pytest.py @@ -9,10 +9,10 @@ import ddtrace from ddtrace.constants import ERROR_MSG from ddtrace.constants import SAMPLING_PRIORITY_KEY -from ddtrace.contrib.pytest import get_version -from ddtrace.contrib.pytest._utils import _USE_PLUGIN_V2 -from ddtrace.contrib.pytest.constants import XFAIL_REASON -from ddtrace.contrib.pytest.plugin import is_enabled +from ddtrace.contrib.internal.pytest._utils import _USE_PLUGIN_V2 +from ddtrace.contrib.internal.pytest.constants import XFAIL_REASON +from ddtrace.contrib.internal.pytest.patch import get_version +from ddtrace.contrib.internal.pytest.plugin import is_enabled from ddtrace.ext import ci from ddtrace.ext import git from ddtrace.ext import test @@ -724,7 +724,7 @@ def test_dd_origin_tag_propagated_to_every_span(self): """ import pytest import ddtrace - from ddtrace import Pin + from ddtrace.trace import Pin def test_service(ddtracer): with ddtracer.trace("SPAN2") as span2: diff --git a/tests/contrib/pytest/test_pytest_atr.py b/tests/contrib/pytest/test_pytest_atr.py index 3e526e8cdf7..ebb4f8421d8 100644 --- a/tests/contrib/pytest/test_pytest_atr.py +++ b/tests/contrib/pytest/test_pytest_atr.py @@ -10,8 +10,8 @@ import pytest -from ddtrace.contrib.pytest._utils import _USE_PLUGIN_V2 -from ddtrace.contrib.pytest._utils import _pytest_version_supports_atr +from ddtrace.contrib.internal.pytest._utils import _USE_PLUGIN_V2 +from ddtrace.contrib.internal.pytest._utils import _pytest_version_supports_atr from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings from tests.ci_visibility.util import _get_default_civisibility_ddconfig from tests.contrib.pytest.test_pytest import PytestTestCaseBase diff --git a/tests/contrib/pytest/test_pytest_efd.py b/tests/contrib/pytest/test_pytest_efd.py index 2affcec3585..e2a2fa08cab 100644 --- a/tests/contrib/pytest/test_pytest_efd.py +++ b/tests/contrib/pytest/test_pytest_efd.py @@ -10,8 +10,8 @@ import pytest -from ddtrace.contrib.pytest._utils import _USE_PLUGIN_V2 -from ddtrace.contrib.pytest._utils import _pytest_version_supports_efd +from ddtrace.contrib.internal.pytest._utils import _USE_PLUGIN_V2 +from ddtrace.contrib.internal.pytest._utils import _pytest_version_supports_efd from ddtrace.internal.ci_visibility._api_client import EarlyFlakeDetectionSettings from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings from tests.ci_visibility.api_client._util import _make_fqdn_test_ids diff --git a/tests/contrib/pytest/test_pytest_quarantine.py b/tests/contrib/pytest/test_pytest_quarantine.py index 93b0b07eade..52e5c5a393c 100644 --- a/tests/contrib/pytest/test_pytest_quarantine.py +++ b/tests/contrib/pytest/test_pytest_quarantine.py @@ -10,8 +10,8 @@ import pytest -from ddtrace.contrib.pytest._utils import _USE_PLUGIN_V2 -from ddtrace.contrib.pytest._utils import _pytest_version_supports_efd +from ddtrace.contrib.internal.pytest._utils import _USE_PLUGIN_V2 +from ddtrace.contrib.internal.pytest._utils import _pytest_version_supports_efd from ddtrace.internal.ci_visibility._api_client import QuarantineSettings from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings from tests.contrib.pytest.test_pytest import PytestTestCaseBase diff --git a/tests/contrib/pytest/test_pytest_snapshot.py b/tests/contrib/pytest/test_pytest_snapshot.py index f8c4090c33d..8b298c28c95 100644 --- a/tests/contrib/pytest/test_pytest_snapshot.py +++ b/tests/contrib/pytest/test_pytest_snapshot.py @@ -3,7 +3,7 @@ import pytest -from ddtrace.contrib.pytest._utils import _USE_PLUGIN_V2 +from ddtrace.contrib.internal.pytest._utils import _USE_PLUGIN_V2 from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings from tests.ci_visibility.util import _get_default_ci_env_vars from tests.utils import TracerTestCase diff --git a/tests/contrib/pytest/test_pytest_snapshot_v2.py b/tests/contrib/pytest/test_pytest_snapshot_v2.py index 85d70d4c38e..dad546df23b 100644 --- a/tests/contrib/pytest/test_pytest_snapshot_v2.py +++ b/tests/contrib/pytest/test_pytest_snapshot_v2.py @@ -3,7 +3,7 @@ import pytest -from ddtrace.contrib.pytest._utils import _USE_PLUGIN_V2 +from ddtrace.contrib.internal.pytest._utils import _USE_PLUGIN_V2 from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings from tests.ci_visibility.util import _get_default_ci_env_vars from tests.utils import TracerTestCase diff --git a/tests/contrib/pytest_bdd/test_pytest_bdd.py b/tests/contrib/pytest_bdd/test_pytest_bdd.py index edf3ab90454..25dff3c8bee 100644 --- a/tests/contrib/pytest_bdd/test_pytest_bdd.py +++ b/tests/contrib/pytest_bdd/test_pytest_bdd.py @@ -2,8 +2,8 @@ import os from ddtrace.constants import ERROR_MSG -from ddtrace.contrib.pytest_bdd._plugin import _get_step_func_args_json -from ddtrace.contrib.pytest_bdd._plugin import get_version +from ddtrace.contrib.internal.pytest_bdd._plugin import _get_step_func_args_json +from ddtrace.contrib.internal.pytest_bdd._plugin import get_version from ddtrace.ext import test from tests.contrib.patch import emit_integration_and_version_to_test_agent from tests.contrib.pytest.test_pytest import PytestTestCaseBase @@ -197,20 +197,23 @@ def test_simple(): assert spans[0].get_tag(ERROR_MSG) def test_get_step_func_args_json_empty(self): - self.monkeypatch.setattr("ddtrace.contrib.pytest_bdd._plugin._extract_step_func_args", lambda *args: None) + self.monkeypatch.setattr( + "ddtrace.contrib.internal.pytest_bdd._plugin._extract_step_func_args", lambda *args: None + ) assert _get_step_func_args_json(None, lambda: None, None) is None def test_get_step_func_args_json_valid(self): self.monkeypatch.setattr( - "ddtrace.contrib.pytest_bdd._plugin._extract_step_func_args", lambda *args: {"func_arg": "test string"} + "ddtrace.contrib.internal.pytest_bdd._plugin._extract_step_func_args", + lambda *args: {"func_arg": "test string"}, ) assert _get_step_func_args_json(None, lambda: None, None) == '{"func_arg": "test string"}' def test_get_step_func_args_json_invalid(self): self.monkeypatch.setattr( - "ddtrace.contrib.pytest_bdd._plugin._extract_step_func_args", lambda *args: {"func_arg": set()} + "ddtrace.contrib.internal.pytest_bdd._plugin._extract_step_func_args", lambda *args: {"func_arg": set()} ) expected = '{"error_serializing_args": "Object of type set is not JSON serializable"}' diff --git a/tests/contrib/pytest_benchmark/test_pytest_benchmark.py b/tests/contrib/pytest_benchmark/test_pytest_benchmark.py index ba55659b8f8..233a389855c 100644 --- a/tests/contrib/pytest_benchmark/test_pytest_benchmark.py +++ b/tests/contrib/pytest_benchmark/test_pytest_benchmark.py @@ -1,24 +1,24 @@ import os -from ddtrace.contrib.pytest_benchmark.constants import BENCHMARK_INFO -from ddtrace.contrib.pytest_benchmark.constants import BENCHMARK_MEAN -from ddtrace.contrib.pytest_benchmark.constants import BENCHMARK_RUN -from ddtrace.contrib.pytest_benchmark.constants import STATISTICS_HD15IQR -from ddtrace.contrib.pytest_benchmark.constants import STATISTICS_IQR -from ddtrace.contrib.pytest_benchmark.constants import STATISTICS_IQR_OUTLIERS -from ddtrace.contrib.pytest_benchmark.constants import STATISTICS_LD15IQR -from ddtrace.contrib.pytest_benchmark.constants import STATISTICS_MAX -from ddtrace.contrib.pytest_benchmark.constants import STATISTICS_MEAN -from ddtrace.contrib.pytest_benchmark.constants import STATISTICS_MEDIAN -from ddtrace.contrib.pytest_benchmark.constants import STATISTICS_MIN -from ddtrace.contrib.pytest_benchmark.constants import STATISTICS_N -from ddtrace.contrib.pytest_benchmark.constants import STATISTICS_OPS -from ddtrace.contrib.pytest_benchmark.constants import STATISTICS_OUTLIERS -from ddtrace.contrib.pytest_benchmark.constants import STATISTICS_Q1 -from ddtrace.contrib.pytest_benchmark.constants import STATISTICS_Q3 -from ddtrace.contrib.pytest_benchmark.constants import STATISTICS_STDDEV -from ddtrace.contrib.pytest_benchmark.constants import STATISTICS_STDDEV_OUTLIERS -from ddtrace.contrib.pytest_benchmark.constants import STATISTICS_TOTAL +from ddtrace.contrib.internal.pytest_benchmark.constants import BENCHMARK_INFO +from ddtrace.contrib.internal.pytest_benchmark.constants import BENCHMARK_MEAN +from ddtrace.contrib.internal.pytest_benchmark.constants import BENCHMARK_RUN +from ddtrace.contrib.internal.pytest_benchmark.constants import STATISTICS_HD15IQR +from ddtrace.contrib.internal.pytest_benchmark.constants import STATISTICS_IQR +from ddtrace.contrib.internal.pytest_benchmark.constants import STATISTICS_IQR_OUTLIERS +from ddtrace.contrib.internal.pytest_benchmark.constants import STATISTICS_LD15IQR +from ddtrace.contrib.internal.pytest_benchmark.constants import STATISTICS_MAX +from ddtrace.contrib.internal.pytest_benchmark.constants import STATISTICS_MEAN +from ddtrace.contrib.internal.pytest_benchmark.constants import STATISTICS_MEDIAN +from ddtrace.contrib.internal.pytest_benchmark.constants import STATISTICS_MIN +from ddtrace.contrib.internal.pytest_benchmark.constants import STATISTICS_N +from ddtrace.contrib.internal.pytest_benchmark.constants import STATISTICS_OPS +from ddtrace.contrib.internal.pytest_benchmark.constants import STATISTICS_OUTLIERS +from ddtrace.contrib.internal.pytest_benchmark.constants import STATISTICS_Q1 +from ddtrace.contrib.internal.pytest_benchmark.constants import STATISTICS_Q3 +from ddtrace.contrib.internal.pytest_benchmark.constants import STATISTICS_STDDEV +from ddtrace.contrib.internal.pytest_benchmark.constants import STATISTICS_STDDEV_OUTLIERS +from ddtrace.contrib.internal.pytest_benchmark.constants import STATISTICS_TOTAL from ddtrace.ext.test import TEST_TYPE from tests.contrib.pytest.test_pytest import PytestTestCaseBase diff --git a/tests/contrib/redis/test_redis.py b/tests/contrib/redis/test_redis.py index f2aa3799b3e..233e78cba4e 100644 --- a/tests/contrib/redis/test_redis.py +++ b/tests/contrib/redis/test_redis.py @@ -5,10 +5,10 @@ import redis import ddtrace -from ddtrace import Pin -from ddtrace.contrib.redis.patch import patch -from ddtrace.contrib.redis.patch import unpatch +from ddtrace.contrib.internal.redis.patch import patch +from ddtrace.contrib.internal.redis.patch import unpatch from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME +from ddtrace.trace import Pin from tests.opentracer.utils import init_tracer from tests.utils import DummyTracer from tests.utils import TracerTestCase diff --git a/tests/contrib/redis/test_redis_asyncio.py b/tests/contrib/redis/test_redis_asyncio.py index fdc5caa13b8..77a809392cd 100644 --- a/tests/contrib/redis/test_redis_asyncio.py +++ b/tests/contrib/redis/test_redis_asyncio.py @@ -7,10 +7,10 @@ import redis.asyncio from wrapt import ObjectProxy -from ddtrace import Pin from ddtrace import tracer -from ddtrace.contrib.redis.patch import patch -from ddtrace.contrib.redis.patch import unpatch +from ddtrace.contrib.internal.redis.patch import patch +from ddtrace.contrib.internal.redis.patch import unpatch +from ddtrace.trace import Pin from tests.utils import override_config from ..config import REDIS_CONFIG diff --git a/tests/contrib/redis/test_redis_cluster.py b/tests/contrib/redis/test_redis_cluster.py index 455ec9c065d..2731a18fcee 100644 --- a/tests/contrib/redis/test_redis_cluster.py +++ b/tests/contrib/redis/test_redis_cluster.py @@ -2,10 +2,10 @@ import pytest import redis -from ddtrace import Pin -from ddtrace.contrib.redis.patch import patch -from ddtrace.contrib.redis.patch import unpatch +from ddtrace.contrib.internal.redis.patch import patch +from ddtrace.contrib.internal.redis.patch import unpatch from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME +from ddtrace.trace import Pin from tests.contrib.config import REDISCLUSTER_CONFIG from tests.utils import DummyTracer from tests.utils import TracerTestCase diff --git a/tests/contrib/redis/test_redis_cluster_asyncio.py b/tests/contrib/redis/test_redis_cluster_asyncio.py index c435c582a29..b8624c533aa 100644 --- a/tests/contrib/redis/test_redis_cluster_asyncio.py +++ b/tests/contrib/redis/test_redis_cluster_asyncio.py @@ -2,9 +2,9 @@ import pytest import redis -from ddtrace import Pin -from ddtrace.contrib.redis.patch import patch -from ddtrace.contrib.redis.patch import unpatch +from ddtrace.contrib.internal.redis.patch import patch +from ddtrace.contrib.internal.redis.patch import unpatch +from ddtrace.trace import Pin from tests.contrib.config import REDISCLUSTER_CONFIG from tests.utils import DummyTracer from tests.utils import assert_is_measured @@ -164,9 +164,9 @@ def test_default_service_name_v1(): import redis - from ddtrace import Pin - from ddtrace.contrib.redis import patch + from ddtrace.contrib.internal.redis.patch import patch from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME + from ddtrace.trace import Pin from tests.contrib.config import REDISCLUSTER_CONFIG from tests.utils import DummyTracer from tests.utils import TracerSpanContainer @@ -210,9 +210,9 @@ def test_user_specified_service_v0(): import redis - from ddtrace import Pin from ddtrace import config - from ddtrace.contrib.redis import patch + from ddtrace.contrib.internal.redis.patch import patch + from ddtrace.trace import Pin from tests.contrib.config import REDISCLUSTER_CONFIG from tests.utils import DummyTracer from tests.utils import TracerSpanContainer @@ -259,9 +259,9 @@ def test_user_specified_service_v1(): import redis - from ddtrace import Pin from ddtrace import config - from ddtrace.contrib.redis import patch + from ddtrace.contrib.internal.redis.patch import patch + from ddtrace.trace import Pin from tests.contrib.config import REDISCLUSTER_CONFIG from tests.utils import DummyTracer from tests.utils import TracerSpanContainer @@ -304,8 +304,8 @@ def test_env_user_specified_rediscluster_service_v0(): import redis - from ddtrace import Pin - from ddtrace.contrib.redis import patch + from ddtrace.contrib.internal.redis.patch import patch + from ddtrace.trace import Pin from tests.contrib.config import REDISCLUSTER_CONFIG from tests.utils import DummyTracer from tests.utils import TracerSpanContainer @@ -345,8 +345,8 @@ def test_env_user_specified_rediscluster_service_v1(): import redis - from ddtrace import Pin - from ddtrace.contrib.redis import patch + from ddtrace.contrib.internal.redis.patch import patch + from ddtrace.trace import Pin from tests.contrib.config import REDISCLUSTER_CONFIG from tests.utils import DummyTracer from tests.utils import TracerSpanContainer @@ -390,9 +390,9 @@ def test_service_precedence_v0(): import redis - from ddtrace import Pin from ddtrace import config - from ddtrace.contrib.redis import patch + from ddtrace.contrib.internal.redis.patch import patch + from ddtrace.trace import Pin from tests.contrib.config import REDISCLUSTER_CONFIG from tests.utils import DummyTracer from tests.utils import TracerSpanContainer @@ -435,9 +435,9 @@ def test_service_precedence_v1(): import redis - from ddtrace import Pin from ddtrace import config - from ddtrace.contrib.redis import patch + from ddtrace.contrib.internal.redis.patch import patch + from ddtrace.trace import Pin from tests.contrib.config import REDISCLUSTER_CONFIG from tests.utils import DummyTracer from tests.utils import TracerSpanContainer diff --git a/tests/contrib/redis/test_redis_patch.py b/tests/contrib/redis/test_redis_patch.py index e1768baf76b..c3b105d0d9e 100644 --- a/tests/contrib/redis/test_redis_patch.py +++ b/tests/contrib/redis/test_redis_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.redis import get_version -from ddtrace.contrib.redis.patch import patch +from ddtrace.contrib.internal.redis.patch import get_version +from ddtrace.contrib.internal.redis.patch import patch try: - from ddtrace.contrib.redis.patch import unpatch + from ddtrace.contrib.internal.redis.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/rediscluster/test.py b/tests/contrib/rediscluster/test.py index e72f310336f..a2c5ac5c6b2 100644 --- a/tests/contrib/rediscluster/test.py +++ b/tests/contrib/rediscluster/test.py @@ -2,11 +2,11 @@ import pytest import rediscluster -from ddtrace import Pin -from ddtrace.contrib.rediscluster.patch import REDISCLUSTER_VERSION -from ddtrace.contrib.rediscluster.patch import patch -from ddtrace.contrib.rediscluster.patch import unpatch +from ddtrace.contrib.internal.rediscluster.patch import REDISCLUSTER_VERSION +from ddtrace.contrib.internal.rediscluster.patch import patch +from ddtrace.contrib.internal.rediscluster.patch import unpatch from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME +from ddtrace.trace import Pin from tests.contrib.config import REDISCLUSTER_CONFIG from tests.utils import DummyTracer from tests.utils import TracerTestCase diff --git a/tests/contrib/rediscluster/test_rediscluster_patch.py b/tests/contrib/rediscluster/test_rediscluster_patch.py index c95dbf73e7c..733892740a5 100644 --- a/tests/contrib/rediscluster/test_rediscluster_patch.py +++ b/tests/contrib/rediscluster/test_rediscluster_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.rediscluster import get_version -from ddtrace.contrib.rediscluster.patch import patch +from ddtrace.contrib.internal.rediscluster.patch import get_version +from ddtrace.contrib.internal.rediscluster.patch import patch try: - from ddtrace.contrib.rediscluster.patch import unpatch + from ddtrace.contrib.internal.rediscluster.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/requests/test_requests.py b/tests/contrib/requests/test_requests.py index ba78c0a4747..2a892a218f7 100644 --- a/tests/contrib/requests/test_requests.py +++ b/tests/contrib/requests/test_requests.py @@ -13,8 +13,8 @@ from ddtrace.constants import ERROR_TYPE from ddtrace.contrib.internal.requests.connection import _extract_hostname_and_path from ddtrace.contrib.internal.requests.connection import _extract_query_string -from ddtrace.contrib.requests import patch -from ddtrace.contrib.requests import unpatch +from ddtrace.contrib.internal.requests.patch import patch +from ddtrace.contrib.internal.requests.patch import unpatch from ddtrace.ext import http from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME from tests.opentracer.utils import init_tracer diff --git a/tests/contrib/requests/test_requests_patch.py b/tests/contrib/requests/test_requests_patch.py index d6824627405..abe1dc262d5 100644 --- a/tests/contrib/requests/test_requests_patch.py +++ b/tests/contrib/requests/test_requests_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.requests import get_version -from ddtrace.contrib.requests.patch import patch +from ddtrace.contrib.internal.requests.patch import get_version +from ddtrace.contrib.internal.requests.patch import patch try: - from ddtrace.contrib.requests.patch import unpatch + from ddtrace.contrib.internal.requests.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/rq/test_rq.py b/tests/contrib/rq/test_rq.py index 395f54597fa..d72871823da 100644 --- a/tests/contrib/rq/test_rq.py +++ b/tests/contrib/rq/test_rq.py @@ -6,10 +6,10 @@ import redis import rq -from ddtrace import Pin -from ddtrace.contrib.rq import get_version -from ddtrace.contrib.rq import patch -from ddtrace.contrib.rq import unpatch +from ddtrace.contrib.internal.rq.patch import get_version +from ddtrace.contrib.internal.rq.patch import patch +from ddtrace.contrib.internal.rq.patch import unpatch +from ddtrace.trace import Pin from tests.contrib.patch import emit_integration_and_version_to_test_agent from tests.utils import override_config from tests.utils import snapshot diff --git a/tests/contrib/sanic/conftest.py b/tests/contrib/sanic/conftest.py index dc701154419..95fd51022f4 100644 --- a/tests/contrib/sanic/conftest.py +++ b/tests/contrib/sanic/conftest.py @@ -1,8 +1,8 @@ import pytest import ddtrace -from ddtrace.contrib.sanic import patch -from ddtrace.contrib.sanic import unpatch +from ddtrace.contrib.internal.sanic.patch import patch +from ddtrace.contrib.internal.sanic.patch import unpatch from tests.utils import DummyTracer diff --git a/tests/contrib/sanic/test_sanic_patch.py b/tests/contrib/sanic/test_sanic_patch.py index 2ef66abf386..25c46d554f8 100644 --- a/tests/contrib/sanic/test_sanic_patch.py +++ b/tests/contrib/sanic/test_sanic_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.sanic import get_version -from ddtrace.contrib.sanic.patch import patch +from ddtrace.contrib.internal.sanic.patch import get_version +from ddtrace.contrib.internal.sanic.patch import patch try: - from ddtrace.contrib.sanic.patch import unpatch + from ddtrace.contrib.internal.sanic.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/selenium/test_selenium_chrome.py b/tests/contrib/selenium/test_selenium_chrome.py index d8b68a8fa53..c8f9c9145c8 100644 --- a/tests/contrib/selenium/test_selenium_chrome.py +++ b/tests/contrib/selenium/test_selenium_chrome.py @@ -184,7 +184,7 @@ def test_selenium_chrome_pytest_unpatch_does_not_record_selenium_tags(_http_serv from selenium.webdriver.common.by import By from selenium.webdriver.chrome.options import Options - from ddtrace.contrib.selenium import unpatch + from ddtrace.contrib.internal.selenium.patch import unpatch def test_selenium_local_unpatch(): unpatch() diff --git a/tests/contrib/shared_tests.py b/tests/contrib/shared_tests.py index a7659374693..cf647a15628 100644 --- a/tests/contrib/shared_tests.py +++ b/tests/contrib/shared_tests.py @@ -1,4 +1,4 @@ -from ddtrace import Pin +from ddtrace.trace import Pin # DBM Shared Tests diff --git a/tests/contrib/shared_tests_async.py b/tests/contrib/shared_tests_async.py index 97d1df32cfa..0d49f09d608 100644 --- a/tests/contrib/shared_tests_async.py +++ b/tests/contrib/shared_tests_async.py @@ -1,4 +1,4 @@ -from ddtrace import Pin +from ddtrace.trace import Pin # DBM Shared Tests diff --git a/tests/contrib/snowflake/test_snowflake.py b/tests/contrib/snowflake/test_snowflake.py index e3c158ce405..9762804651d 100644 --- a/tests/contrib/snowflake/test_snowflake.py +++ b/tests/contrib/snowflake/test_snowflake.py @@ -6,10 +6,10 @@ import responses import snowflake.connector -from ddtrace import Pin from ddtrace import tracer -from ddtrace.contrib.snowflake import patch -from ddtrace.contrib.snowflake import unpatch +from ddtrace.contrib.internal.snowflake.patch import patch +from ddtrace.contrib.internal.snowflake.patch import unpatch +from ddtrace.trace import Pin from tests.opentracer.utils import init_tracer from tests.utils import override_config from tests.utils import snapshot diff --git a/tests/contrib/snowflake/test_snowflake_patch.py b/tests/contrib/snowflake/test_snowflake_patch.py index 682e55e95d7..63c8e01c86c 100644 --- a/tests/contrib/snowflake/test_snowflake_patch.py +++ b/tests/contrib/snowflake/test_snowflake_patch.py @@ -1,5 +1,5 @@ -from ddtrace.contrib.snowflake import get_version -from ddtrace.contrib.snowflake import patch +from ddtrace.contrib.internal.snowflake.patch import get_version +from ddtrace.contrib.internal.snowflake.patch import patch from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/sqlalchemy/mixins.py b/tests/contrib/sqlalchemy/mixins.py index 4d9300dc897..18b180db2d3 100644 --- a/tests/contrib/sqlalchemy/mixins.py +++ b/tests/contrib/sqlalchemy/mixins.py @@ -8,7 +8,7 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker -from ddtrace.contrib.sqlalchemy import trace_engine +from ddtrace.contrib.internal.sqlalchemy.engine import trace_engine from tests.opentracer.utils import init_tracer diff --git a/tests/contrib/sqlalchemy/test_patch.py b/tests/contrib/sqlalchemy/test_patch.py index d8c4b6ba6d4..a6f08bb5f46 100644 --- a/tests/contrib/sqlalchemy/test_patch.py +++ b/tests/contrib/sqlalchemy/test_patch.py @@ -1,10 +1,10 @@ import sqlalchemy from sqlalchemy import text -from ddtrace import Pin -from ddtrace.contrib.sqlalchemy import get_version -from ddtrace.contrib.sqlalchemy import patch -from ddtrace.contrib.sqlalchemy import unpatch +from ddtrace.contrib.internal.sqlalchemy.patch import get_version +from ddtrace.contrib.internal.sqlalchemy.patch import patch +from ddtrace.contrib.internal.sqlalchemy.patch import unpatch +from ddtrace.trace import Pin from tests.contrib.patch import emit_integration_and_version_to_test_agent from tests.utils import TracerTestCase from tests.utils import assert_is_measured diff --git a/tests/contrib/sqlite3/test_sqlite3.py b/tests/contrib/sqlite3/test_sqlite3.py index f8e141df3f1..6101dcfa081 100644 --- a/tests/contrib/sqlite3/test_sqlite3.py +++ b/tests/contrib/sqlite3/test_sqlite3.py @@ -13,14 +13,14 @@ import pytest import ddtrace -from ddtrace import Pin from ddtrace.constants import ERROR_MSG from ddtrace.constants import ERROR_STACK from ddtrace.constants import ERROR_TYPE -from ddtrace.contrib.sqlite3.patch import TracedSQLiteCursor -from ddtrace.contrib.sqlite3.patch import patch -from ddtrace.contrib.sqlite3.patch import unpatch +from ddtrace.contrib.internal.sqlite3.patch import TracedSQLiteCursor +from ddtrace.contrib.internal.sqlite3.patch import patch +from ddtrace.contrib.internal.sqlite3.patch import unpatch from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME +from ddtrace.trace import Pin from tests.opentracer.utils import init_tracer from tests.utils import TracerTestCase from tests.utils import assert_is_measured diff --git a/tests/contrib/sqlite3/test_sqlite3_patch.py b/tests/contrib/sqlite3/test_sqlite3_patch.py index 9515011f308..62d4dfde017 100644 --- a/tests/contrib/sqlite3/test_sqlite3_patch.py +++ b/tests/contrib/sqlite3/test_sqlite3_patch.py @@ -10,12 +10,12 @@ except ImportError: pass -from ddtrace.contrib.sqlite3 import get_version -from ddtrace.contrib.sqlite3.patch import patch +from ddtrace.contrib.internal.sqlite3.patch import get_version +from ddtrace.contrib.internal.sqlite3.patch import patch try: - from ddtrace.contrib.sqlite3.patch import unpatch + from ddtrace.contrib.internal.sqlite3.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/starlette/test_starlette.py b/tests/contrib/starlette/test_starlette.py index ff756ff0cc9..f290ade8ea7 100644 --- a/tests/contrib/starlette/test_starlette.py +++ b/tests/contrib/starlette/test_starlette.py @@ -8,13 +8,13 @@ from starlette.testclient import TestClient import ddtrace -from ddtrace import Pin from ddtrace.constants import ERROR_MSG -from ddtrace.contrib.sqlalchemy import patch as sql_patch -from ddtrace.contrib.sqlalchemy import unpatch as sql_unpatch -from ddtrace.contrib.starlette import patch as starlette_patch -from ddtrace.contrib.starlette import unpatch as starlette_unpatch +from ddtrace.contrib.internal.sqlalchemy.patch import patch as sql_patch +from ddtrace.contrib.internal.sqlalchemy.patch import unpatch as sql_unpatch +from ddtrace.contrib.internal.starlette.patch import patch as starlette_patch +from ddtrace.contrib.internal.starlette.patch import unpatch as starlette_unpatch from ddtrace.propagation import http as http_propagation +from ddtrace.trace import Pin from tests.contrib.starlette.app import get_app from tests.utils import DummyTracer from tests.utils import TracerSpanContainer diff --git a/tests/contrib/starlette/test_starlette_patch.py b/tests/contrib/starlette/test_starlette_patch.py index 6666ad764a9..1f271ebb811 100644 --- a/tests/contrib/starlette/test_starlette_patch.py +++ b/tests/contrib/starlette/test_starlette_patch.py @@ -1,6 +1,6 @@ -from ddtrace.contrib.starlette import get_version -from ddtrace.contrib.starlette import patch -from ddtrace.contrib.starlette import unpatch +from ddtrace.contrib.internal.starlette.patch import get_version +from ddtrace.contrib.internal.starlette.patch import patch +from ddtrace.contrib.internal.starlette.patch import unpatch from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/structlog/test_structlog_logging.py b/tests/contrib/structlog/test_structlog_logging.py index 5ef75e9feb4..da37f23be33 100644 --- a/tests/contrib/structlog/test_structlog_logging.py +++ b/tests/contrib/structlog/test_structlog_logging.py @@ -8,8 +8,8 @@ from ddtrace.constants import ENV_KEY from ddtrace.constants import SERVICE_KEY from ddtrace.constants import VERSION_KEY -from ddtrace.contrib.structlog import patch -from ddtrace.contrib.structlog import unpatch +from ddtrace.contrib.internal.structlog.patch import patch +from ddtrace.contrib.internal.structlog.patch import unpatch from ddtrace.internal.constants import MAX_UINT_64BITS from tests.utils import override_global_config @@ -87,8 +87,8 @@ def test_log_trace(): from ddtrace import config from ddtrace import tracer - from ddtrace.contrib.structlog import patch - from ddtrace.contrib.structlog import unpatch + from ddtrace.contrib.internal.structlog.patch import patch + from ddtrace.contrib.internal.structlog.patch import unpatch config.service = "logging" config.env = "global.env" @@ -132,8 +132,8 @@ def test_log_trace_128bit_trace_ids(): from ddtrace import config from ddtrace import tracer - from ddtrace.contrib.structlog import patch - from ddtrace.contrib.structlog import unpatch + from ddtrace.contrib.internal.structlog.patch import patch + from ddtrace.contrib.internal.structlog.patch import unpatch from ddtrace.internal.constants import MAX_UINT_64BITS config.service = "logging" @@ -178,8 +178,8 @@ def test_log_DD_TAGS(): from ddtrace.constants import ENV_KEY from ddtrace.constants import SERVICE_KEY from ddtrace.constants import VERSION_KEY - from ddtrace.contrib.structlog import patch - from ddtrace.contrib.structlog import unpatch + from ddtrace.contrib.internal.structlog.patch import patch + from ddtrace.contrib.internal.structlog.patch import unpatch patch() @@ -222,8 +222,8 @@ def test_tuple_processor_list(): from ddtrace import config from ddtrace import tracer - from ddtrace.contrib.structlog import patch - from ddtrace.contrib.structlog import unpatch + from ddtrace.contrib.internal.structlog.patch import patch + from ddtrace.contrib.internal.structlog.patch import unpatch config.service = "logging" config.env = "global.env" @@ -264,8 +264,8 @@ def test_no_configured_processor(): from ddtrace import config from ddtrace import tracer - from ddtrace.contrib.structlog import patch - from ddtrace.contrib.structlog import unpatch + from ddtrace.contrib.internal.structlog.patch import patch + from ddtrace.contrib.internal.structlog.patch import unpatch config.service = "logging" config.env = "global.env" @@ -303,8 +303,8 @@ def test_two_loggers_no_duplicates(): """ import structlog - from ddtrace.contrib.structlog import patch - from ddtrace.contrib.structlog import unpatch + from ddtrace.contrib.internal.structlog.patch import patch + from ddtrace.contrib.internal.structlog.patch import unpatch patch() @@ -331,8 +331,8 @@ def test_configure_processor(): """ import structlog - from ddtrace.contrib.structlog import patch - from ddtrace.contrib.structlog import unpatch + from ddtrace.contrib.internal.structlog.patch import patch + from ddtrace.contrib.internal.structlog.patch import unpatch patch() @@ -360,8 +360,8 @@ def test_consistent_empty_config(): """ import structlog - from ddtrace.contrib.structlog import patch - from ddtrace.contrib.structlog import unpatch + from ddtrace.contrib.internal.structlog.patch import patch + from ddtrace.contrib.internal.structlog.patch import unpatch patch() @@ -383,8 +383,8 @@ def test_reset_defaults(): """ import structlog - from ddtrace.contrib.structlog import patch - from ddtrace.contrib.structlog import unpatch + from ddtrace.contrib.internal.structlog.patch import patch + from ddtrace.contrib.internal.structlog.patch import unpatch patch() diff --git a/tests/contrib/structlog/test_structlog_patch.py b/tests/contrib/structlog/test_structlog_patch.py index dcfedfecf2d..c7c0ce7ca48 100644 --- a/tests/contrib/structlog/test_structlog_patch.py +++ b/tests/contrib/structlog/test_structlog_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.structlog import get_version -from ddtrace.contrib.structlog.patch import patch +from ddtrace.contrib.internal.structlog.patch import get_version +from ddtrace.contrib.internal.structlog.patch import patch try: - from ddtrace.contrib.structlog.patch import unpatch + from ddtrace.contrib.internal.structlog.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/subprocess/test_subprocess.py b/tests/contrib/subprocess/test_subprocess.py index d2f15637dd4..40e7ab67431 100644 --- a/tests/contrib/subprocess/test_subprocess.py +++ b/tests/contrib/subprocess/test_subprocess.py @@ -4,13 +4,13 @@ import pytest -from ddtrace import Pin -from ddtrace.contrib.subprocess.constants import COMMANDS -from ddtrace.contrib.subprocess.patch import SubprocessCmdLine -from ddtrace.contrib.subprocess.patch import patch -from ddtrace.contrib.subprocess.patch import unpatch +from ddtrace.contrib.internal.subprocess.constants import COMMANDS +from ddtrace.contrib.internal.subprocess.patch import SubprocessCmdLine +from ddtrace.contrib.internal.subprocess.patch import patch +from ddtrace.contrib.internal.subprocess.patch import unpatch from ddtrace.ext import SpanTypes from ddtrace.internal import core +from ddtrace.trace import Pin from tests.utils import override_config from tests.utils import override_global_config @@ -220,7 +220,7 @@ def test_fork(tracer): pid = os.fork() if pid == 0: # Exit, otherwise the rest of this process will continue to be pytest - from ddtrace.contrib.coverage import unpatch + from ddtrace.contrib.internal.coverage.patch import unpatch unpatch() import pytest diff --git a/tests/contrib/subprocess/test_subprocess_patch.py b/tests/contrib/subprocess/test_subprocess_patch.py index 96148bcdcbe..57778f798c1 100644 --- a/tests/contrib/subprocess/test_subprocess_patch.py +++ b/tests/contrib/subprocess/test_subprocess_patch.py @@ -1,10 +1,10 @@ -from ddtrace.contrib.subprocess.patch import get_version -from ddtrace.contrib.subprocess.patch import patch +from ddtrace.contrib.internal.subprocess.patch import get_version +from ddtrace.contrib.internal.subprocess.patch import patch from ddtrace.settings.asm import config as asm_config try: - from ddtrace.contrib.subprocess.patch import unpatch + from ddtrace.contrib.internal.subprocess.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/tornado/test_config.py b/tests/contrib/tornado/test_config.py index d91a2e95912..718bf48d1dc 100644 --- a/tests/contrib/tornado/test_config.py +++ b/tests/contrib/tornado/test_config.py @@ -1,5 +1,5 @@ from ddtrace._trace.tracer import Tracer -from ddtrace.filters import TraceFilter +from ddtrace.trace import TraceFilter from tests.utils import DummyWriter from .utils import TornadoTestCase diff --git a/tests/contrib/tornado/test_safety.py b/tests/contrib/tornado/test_safety.py index 4aaad2ae9bc..aa2893c7189 100644 --- a/tests/contrib/tornado/test_safety.py +++ b/tests/contrib/tornado/test_safety.py @@ -3,8 +3,8 @@ from tornado import httpclient from tornado.testing import gen_test -from ddtrace.contrib.tornado import patch -from ddtrace.contrib.tornado import unpatch +from ddtrace.contrib.internal.tornado.patch import patch +from ddtrace.contrib.internal.tornado.patch import unpatch from ddtrace.ext import http from . import web diff --git a/tests/contrib/tornado/test_stack_context.py b/tests/contrib/tornado/test_stack_context.py index 68ee876a9bb..5d7035c3df5 100644 --- a/tests/contrib/tornado/test_stack_context.py +++ b/tests/contrib/tornado/test_stack_context.py @@ -2,7 +2,7 @@ import tornado from ddtrace._trace.context import Context -from ddtrace.contrib.tornado import TracerStackContext +from ddtrace.contrib.internal.tornado.stack_context import TracerStackContext from .utils import TornadoTestCase from .web.compat import sleep diff --git a/tests/contrib/tornado/test_tornado_patch.py b/tests/contrib/tornado/test_tornado_patch.py index 0f9963ab532..fbd20ff8a50 100644 --- a/tests/contrib/tornado/test_tornado_patch.py +++ b/tests/contrib/tornado/test_tornado_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.tornado import get_version -from ddtrace.contrib.tornado.patch import patch +from ddtrace.contrib.internal.tornado.patch import get_version +from ddtrace.contrib.internal.tornado.patch import patch try: - from ddtrace.contrib.tornado.patch import unpatch + from ddtrace.contrib.internal.tornado.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/tornado/utils.py b/tests/contrib/tornado/utils.py index 4c21dd7ead1..6db7403caac 100644 --- a/tests/contrib/tornado/utils.py +++ b/tests/contrib/tornado/utils.py @@ -2,10 +2,10 @@ from tornado.testing import AsyncHTTPTestCase -from ddtrace.contrib.futures import patch as patch_futures -from ddtrace.contrib.futures import unpatch as unpatch_futures -from ddtrace.contrib.tornado import patch -from ddtrace.contrib.tornado import unpatch +from ddtrace.contrib.internal.futures.patch import patch as patch_futures +from ddtrace.contrib.internal.futures.patch import unpatch as unpatch_futures +from ddtrace.contrib.internal.tornado.patch import patch +from ddtrace.contrib.internal.tornado.patch import unpatch from tests.utils import TracerTestCase from .web import app diff --git a/tests/contrib/unittest/test_unittest.py b/tests/contrib/unittest/test_unittest.py index b2af61b47d8..cd24c26f3c0 100644 --- a/tests/contrib/unittest/test_unittest.py +++ b/tests/contrib/unittest/test_unittest.py @@ -7,15 +7,15 @@ from ddtrace.constants import ERROR_MSG from ddtrace.constants import ERROR_TYPE from ddtrace.constants import SPAN_KIND -from ddtrace.contrib.unittest.constants import COMPONENT_VALUE -from ddtrace.contrib.unittest.constants import FRAMEWORK -from ddtrace.contrib.unittest.constants import KIND -from ddtrace.contrib.unittest.constants import MODULE_OPERATION_NAME -from ddtrace.contrib.unittest.constants import SESSION_OPERATION_NAME -from ddtrace.contrib.unittest.constants import SUITE_OPERATION_NAME -from ddtrace.contrib.unittest.constants import TEST_OPERATION_NAME -from ddtrace.contrib.unittest.patch import _set_tracer -from ddtrace.contrib.unittest.patch import patch +from ddtrace.contrib.internal.unittest.constants import COMPONENT_VALUE +from ddtrace.contrib.internal.unittest.constants import FRAMEWORK +from ddtrace.contrib.internal.unittest.constants import KIND +from ddtrace.contrib.internal.unittest.constants import MODULE_OPERATION_NAME +from ddtrace.contrib.internal.unittest.constants import SESSION_OPERATION_NAME +from ddtrace.contrib.internal.unittest.constants import SUITE_OPERATION_NAME +from ddtrace.contrib.internal.unittest.constants import TEST_OPERATION_NAME +from ddtrace.contrib.internal.unittest.patch import _set_tracer +from ddtrace.contrib.internal.unittest.patch import patch from ddtrace.ext import SpanTypes from ddtrace.ext import test from ddtrace.ext.ci import RUNTIME_VERSION diff --git a/tests/contrib/unittest/test_unittest_patch.py b/tests/contrib/unittest/test_unittest_patch.py index ff7e10d2064..ccb8668fa12 100644 --- a/tests/contrib/unittest/test_unittest_patch.py +++ b/tests/contrib/unittest/test_unittest_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.unittest import get_version -from ddtrace.contrib.unittest.patch import patch +from ddtrace.contrib.internal.unittest.patch import get_version +from ddtrace.contrib.internal.unittest.patch import patch try: - from ddtrace.contrib.unittest.patch import unpatch + from ddtrace.contrib.internal.unittest.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/urllib3/test_urllib3.py b/tests/contrib/urllib3/test_urllib3.py index 2f0c447ee65..53c9b12bf6b 100644 --- a/tests/contrib/urllib3/test_urllib3.py +++ b/tests/contrib/urllib3/test_urllib3.py @@ -7,11 +7,12 @@ from ddtrace.constants import ERROR_MSG from ddtrace.constants import ERROR_STACK from ddtrace.constants import ERROR_TYPE -from ddtrace.contrib.urllib3 import patch -from ddtrace.contrib.urllib3 import unpatch +from ddtrace.contrib.internal.urllib3.patch import patch +from ddtrace.contrib.internal.urllib3.patch import unpatch from ddtrace.ext import http from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME -from ddtrace.pin import Pin +from ddtrace.settings.asm import config as asm_config +from ddtrace.trace import Pin from tests.contrib.config import HTTPBIN_CONFIG from tests.opentracer.utils import init_tracer from tests.utils import TracerTestCase @@ -527,12 +528,16 @@ def test_distributed_tracing_disabled(self): timeout=mock.ANY, ) + @pytest.mark.skip(reason="urlib3 does not set the ASM Manual keep tag so x-datadog headers are not propagated") def test_distributed_tracing_apm_opt_out_true(self): """Tests distributed tracing headers are passed by default""" # Check that distributed tracing headers are passed down; raise an error rather than make the # request since we don't care about the response at all config.urllib3["distributed_tracing"] = True self.tracer.enabled = False + # Ensure the ASM SpanProcessor is set + self.tracer.configure(appsec_standalone_enabled=True, appsec_enabled=True) + assert asm_config._apm_opt_out with mock.patch( "urllib3.connectionpool.HTTPConnectionPool._make_request", side_effect=ValueError ) as m_make_request: @@ -580,6 +585,9 @@ def test_distributed_tracing_apm_opt_out_false(self): """Test with distributed tracing disabled does not propagate the headers""" config.urllib3["distributed_tracing"] = True self.tracer.enabled = False + # Ensure the ASM SpanProcessor is set. + self.tracer.configure(appsec_standalone_enabled=False, appsec_enabled=True) + assert not asm_config._apm_opt_out with mock.patch( "urllib3.connectionpool.HTTPConnectionPool._make_request", side_effect=ValueError ) as m_make_request: diff --git a/tests/contrib/urllib3/test_urllib3_patch.py b/tests/contrib/urllib3/test_urllib3_patch.py index ace147d715c..90e82140200 100644 --- a/tests/contrib/urllib3/test_urllib3_patch.py +++ b/tests/contrib/urllib3/test_urllib3_patch.py @@ -1,6 +1,6 @@ -from ddtrace.contrib.urllib3 import get_version -from ddtrace.contrib.urllib3 import patch -from ddtrace.contrib.urllib3 import unpatch +from ddtrace.contrib.internal.urllib3.patch import get_version +from ddtrace.contrib.internal.urllib3.patch import patch +from ddtrace.contrib.internal.urllib3.patch import unpatch from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/vertexai/conftest.py b/tests/contrib/vertexai/conftest.py index 74ba41d4dee..d5e689137ff 100644 --- a/tests/contrib/vertexai/conftest.py +++ b/tests/contrib/vertexai/conftest.py @@ -2,10 +2,10 @@ from mock import PropertyMock import pytest -from ddtrace.contrib.vertexai import patch -from ddtrace.contrib.vertexai import unpatch +from ddtrace.contrib.internal.vertexai.patch import patch +from ddtrace.contrib.internal.vertexai.patch import unpatch from ddtrace.llmobs import LLMObs -from ddtrace.pin import Pin +from ddtrace.trace import Pin from tests.contrib.vertexai.utils import MockAsyncPredictionServiceClient from tests.contrib.vertexai.utils import MockPredictionServiceClient from tests.utils import DummyTracer diff --git a/tests/contrib/vertexai/test_vertexai_patch.py b/tests/contrib/vertexai/test_vertexai_patch.py index 39ae2d599d1..dfadd1ce1f9 100644 --- a/tests/contrib/vertexai/test_vertexai_patch.py +++ b/tests/contrib/vertexai/test_vertexai_patch.py @@ -1,6 +1,6 @@ -from ddtrace.contrib.vertexai import get_version -from ddtrace.contrib.vertexai import patch -from ddtrace.contrib.vertexai import unpatch +from ddtrace.contrib.internal.vertexai.patch import get_version +from ddtrace.contrib.internal.vertexai.patch import patch +from ddtrace.contrib.internal.vertexai.patch import unpatch from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/vertica/test_vertica.py b/tests/contrib/vertica/test_vertica.py index 4752575e0b9..196e1621ee5 100644 --- a/tests/contrib/vertica/test_vertica.py +++ b/tests/contrib/vertica/test_vertica.py @@ -2,15 +2,15 @@ import wrapt import ddtrace -from ddtrace import Pin from ddtrace import config from ddtrace.constants import ERROR_MSG from ddtrace.constants import ERROR_STACK from ddtrace.constants import ERROR_TYPE -from ddtrace.contrib.vertica.patch import patch -from ddtrace.contrib.vertica.patch import unpatch +from ddtrace.contrib.internal.vertica.patch import patch +from ddtrace.contrib.internal.vertica.patch import unpatch from ddtrace.internal.schema import DEFAULT_SPAN_SERVICE_NAME from ddtrace.settings.config import _deepmerge +from ddtrace.trace import Pin from tests.contrib.config import VERTICA_CONFIG from tests.opentracer.utils import init_tracer from tests.utils import DummyTracer diff --git a/tests/contrib/vertica/test_vertica_patch.py b/tests/contrib/vertica/test_vertica_patch.py index bdaccf5d7e2..234b9de4ef4 100644 --- a/tests/contrib/vertica/test_vertica_patch.py +++ b/tests/contrib/vertica/test_vertica_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.vertica import get_version -from ddtrace.contrib.vertica.patch import patch +from ddtrace.contrib.internal.vertica.patch import get_version +from ddtrace.contrib.internal.vertica.patch import patch try: - from ddtrace.contrib.vertica.patch import unpatch + from ddtrace.contrib.internal.vertica.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/contrib/wsgi/test_wsgi.py b/tests/contrib/wsgi/test_wsgi.py index 453cba9585c..b49a4eb7925 100644 --- a/tests/contrib/wsgi/test_wsgi.py +++ b/tests/contrib/wsgi/test_wsgi.py @@ -4,9 +4,9 @@ from webtest import TestApp from ddtrace import config +from ddtrace.contrib.internal.wsgi.wsgi import DDWSGIMiddleware from ddtrace.contrib.internal.wsgi.wsgi import _DDWSGIMiddlewareBase from ddtrace.contrib.internal.wsgi.wsgi import get_request_headers -from ddtrace.contrib.wsgi import DDWSGIMiddleware from tests.utils import override_config from tests.utils import override_global_config from tests.utils import override_http_config @@ -398,7 +398,7 @@ def test_get_request_headers(extra, expected): def test_schematization(ddtrace_run_python_code_in_subprocess, service_name, schema_version): code = """ from webtest import TestApp -from ddtrace.contrib.wsgi import DDWSGIMiddleware +from ddtrace.contrib.internal.wsgi.wsgi import DDWSGIMiddleware from tests.conftest import * from tests.contrib.wsgi.test_wsgi import application diff --git a/tests/contrib/yaaredis/test_yaaredis.py b/tests/contrib/yaaredis/test_yaaredis.py index f80523a72d7..350b323de9c 100644 --- a/tests/contrib/yaaredis/test_yaaredis.py +++ b/tests/contrib/yaaredis/test_yaaredis.py @@ -6,9 +6,9 @@ from wrapt import ObjectProxy import yaaredis -from ddtrace import Pin -from ddtrace.contrib.yaaredis.patch import patch -from ddtrace.contrib.yaaredis.patch import unpatch +from ddtrace.contrib.internal.yaaredis.patch import patch +from ddtrace.contrib.internal.yaaredis.patch import unpatch +from ddtrace.trace import Pin from tests.opentracer.utils import init_tracer from tests.utils import override_config diff --git a/tests/contrib/yaaredis/test_yaaredis_patch.py b/tests/contrib/yaaredis/test_yaaredis_patch.py index 1d78fe9458c..d93247a1faa 100644 --- a/tests/contrib/yaaredis/test_yaaredis_patch.py +++ b/tests/contrib/yaaredis/test_yaaredis_patch.py @@ -3,12 +3,12 @@ # removed the ``_generated`` suffix from the file name, to prevent the content # from being overwritten by future re-generations. -from ddtrace.contrib.yaaredis import get_version -from ddtrace.contrib.yaaredis.patch import patch +from ddtrace.contrib.internal.yaaredis.patch import get_version +from ddtrace.contrib.internal.yaaredis.patch import patch try: - from ddtrace.contrib.yaaredis.patch import unpatch + from ddtrace.contrib.internal.yaaredis.patch import unpatch except ImportError: unpatch = None from tests.contrib.patch import PatchTestCase diff --git a/tests/debugging/exception/test_replay.py b/tests/debugging/exception/test_replay.py index 54baeb8b826..8261bfb5b47 100644 --- a/tests/debugging/exception/test_replay.py +++ b/tests/debugging/exception/test_replay.py @@ -123,7 +123,7 @@ def c(foo=42): for n, span in enumerate(self.spans): assert span.get_tag(replay.DEBUG_INFO_TAG) == "true" - exc_id = span.get_tag("_dd.debug.error.exception_id") + exc_id = span.get_tag(replay.EXCEPTION_ID_TAG) info = {k: v for k, v in enumerate(["c", "b", "a"][n:], start=1)} @@ -147,8 +147,8 @@ def c(foo=42): assert all(str(s.exc_id) == exc_id for s in snapshots.values()) # assert all spans use the same exc_id - exc_ids = set(span.get_tag("_dd.debug.error.exception_id") for span in self.spans) - assert len(exc_ids) == 1 + exc_ids = set(span.get_tag(replay.EXCEPTION_ID_TAG) for span in self.spans) + assert None not in exc_ids and len(exc_ids) == 1 def test_debugger_exception_chaining(self): def a(v, d=None): @@ -190,7 +190,7 @@ def c(foo=42): for n, span in enumerate(self.spans): assert span.get_tag(replay.DEBUG_INFO_TAG) == "true" - exc_id = span.get_tag("_dd.debug.error.exception_id") + exc_id = span.get_tag(replay.EXCEPTION_ID_TAG) info = {k: v for k, v in enumerate(stacks[n], start=1)} @@ -215,8 +215,8 @@ def c(foo=42): assert any(str(s.exc_id) == exc_id for s in snapshots.values()) # assert number of unique exc_ids based on python version - exc_ids = set(span.get_tag("_dd.debug.error.exception_id") for span in self.spans) - assert len(exc_ids) == number_of_exc_ids + exc_ids = set(span.get_tag(replay.EXCEPTION_ID_TAG) for span in self.spans) + assert None not in exc_ids and len(exc_ids) == number_of_exc_ids # invoke again (should be in less than 1 sec) with with_rate_limiter(rate_limiter): @@ -294,3 +294,23 @@ def c(foo=42): self.assert_span_count(6) # no new snapshots assert len(uploader.collector.queue) == 3 + + def test_debugger_exception_in_closure(self): + def b(): + with self.trace("b"): + nonloc = 4 + + def a(v): + if nonloc: + raise ValueError("hello", v) + + a(nonloc) + + with exception_replay() as uploader: + with with_rate_limiter(RateLimiter(limit_rate=1, raise_on_exceed=False)): + with pytest.raises(ValueError): + b() + + assert all( + s.line_capture["locals"]["nonloc"] == {"type": "int", "value": "4"} for s in uploader.collector.queue + ) diff --git a/tests/debugging/test_safety.py b/tests/debugging/test_safety.py index 3acb0288924..cc44ca9ca12 100644 --- a/tests/debugging/test_safety.py +++ b/tests/debugging/test_safety.py @@ -15,7 +15,10 @@ def assert_args(args): assert set(dict(_safety.get_args(inspect.currentframe().f_back)).keys()) == args def assert_locals(_locals): - assert set(dict(_safety.get_locals(inspect.currentframe().f_back)).keys()) == _locals + assert set(dict(_safety.get_locals(inspect.currentframe().f_back)).keys()) == _locals | { + "assert_args", + "assert_locals", + } def assert_globals(_globals): assert set(dict(_safety.get_globals(inspect.currentframe().f_back)).keys()) == _globals diff --git a/tests/integration/test_context_snapshots.py b/tests/integration/test_context_snapshots.py index 4ed44bd558e..612422064a0 100644 --- a/tests/integration/test_context_snapshots.py +++ b/tests/integration/test_context_snapshots.py @@ -1,9 +1,8 @@ import pytest +from tests.integration.utils import AGENT_VERSION from tests.utils import snapshot -from .test_integration import AGENT_VERSION - pytestmark = pytest.mark.skipif(AGENT_VERSION != "testagent", reason="Tests only compatible with a testagent") diff --git a/tests/integration/test_debug.py b/tests/integration/test_debug.py index e699db36d6b..8c51db4bf7c 100644 --- a/tests/integration/test_debug.py +++ b/tests/integration/test_debug.py @@ -1,10 +1,8 @@ -from datetime import datetime import json import logging import os import re import subprocess -import sys from typing import List from typing import Optional @@ -12,16 +10,15 @@ import pytest import ddtrace +import ddtrace._trace.sampler from ddtrace._trace.span import Span from ddtrace.internal import debug from ddtrace.internal.writer import AgentWriter from ddtrace.internal.writer import TraceWriter -import ddtrace.sampler +from tests.integration.utils import AGENT_VERSION from tests.subprocesstest import SubprocessTestCase from tests.subprocesstest import run_in_subprocess -from .test_integration import AGENT_VERSION - pytestmark = pytest.mark.skipif(AGENT_VERSION == "testagent", reason="The test agent doesn't support startup logs.") @@ -36,7 +33,14 @@ def __eq__(self, other): return Match() +@pytest.mark.subprocess() def test_standard_tags(): + from datetime import datetime + import sys + + import ddtrace + from ddtrace.internal import debug + f = debug.collect(ddtrace.tracer) date = f.get("date") @@ -94,7 +98,7 @@ def test_standard_tags(): assert f.get("tracer_enabled") is True assert f.get("sampler_type") == "DatadogSampler" assert f.get("priority_sampler_type") == "N/A" - assert f.get("service") == "tests.integration" + assert f.get("service") == "ddtrace_subprocess_dir" assert f.get("dd_version") == "" assert f.get("debug") is False assert f.get("enabled_cli") is False @@ -110,8 +114,13 @@ def test_standard_tags(): assert icfg["flask"] == "N/A" +@pytest.mark.subprocess() def test_debug_post_configure(): - tracer = ddtrace.Tracer() + import re + + from ddtrace import tracer + from ddtrace.internal import debug + tracer.configure( hostname="0.0.0.0", port=1234, @@ -122,16 +131,21 @@ def test_debug_post_configure(): agent_url = f.get("agent_url") assert agent_url == "http://0.0.0.0:1234" - assert f.get("is_global_tracer") is False + assert f.get("is_global_tracer") is True assert f.get("tracer_enabled") is True agent_error = f.get("agent_error") # Error code can differ between Python version assert re.match("^Agent not reachable.*Connection refused", agent_error) - # Tracer doesn't support re-configure()-ing with a UDS after an initial - # configure with normal http settings. So we need a new tracer instance. - tracer = ddtrace.Tracer() + +@pytest.mark.subprocess() +def test_debug_post_configure_uds(): + import re + + from ddtrace import tracer + from ddtrace.internal import debug + tracer.configure(uds_path="/file.sock") f = debug.collect(tracer) @@ -317,7 +331,7 @@ def flush_queue(self) -> None: def test_different_samplers(): tracer = ddtrace.Tracer() - tracer.configure(sampler=ddtrace.sampler.RateSampler()) + tracer.configure(sampler=ddtrace._trace.sampler.RateSampler()) info = debug.collect(tracer) assert info.get("sampler_type") == "RateSampler" @@ -325,7 +339,7 @@ def test_different_samplers(): def test_startup_logs_sampling_rules(): tracer = ddtrace.Tracer() - sampler = ddtrace.sampler.DatadogSampler(rules=[ddtrace.sampler.SamplingRule(sample_rate=1.0)]) + sampler = ddtrace._trace.sampler.DatadogSampler(rules=[ddtrace._trace.sampler.SamplingRule(sample_rate=1.0)]) tracer.configure(sampler=sampler) f = debug.collect(tracer) @@ -334,8 +348,8 @@ def test_startup_logs_sampling_rules(): " tags='NO_RULE', provenance='default')" ] - sampler = ddtrace.sampler.DatadogSampler( - rules=[ddtrace.sampler.SamplingRule(sample_rate=1.0, service="xyz", name="abc")] + sampler = ddtrace._trace.sampler.DatadogSampler( + rules=[ddtrace._trace.sampler.SamplingRule(sample_rate=1.0, service="xyz", name="abc")] ) tracer.configure(sampler=sampler) f = debug.collect(tracer) diff --git a/tests/integration/test_encoding.py b/tests/integration/test_encoding.py index 43c47ac4840..7138ff94e00 100644 --- a/tests/integration/test_encoding.py +++ b/tests/integration/test_encoding.py @@ -18,7 +18,7 @@ def test_simple_trace_accepted_by_agent(self): for _ in range(999): with tracer.trace("child"): pass - tracer.shutdown() + tracer.flush() log.warning.assert_not_called() log.error.assert_not_called() @@ -39,7 +39,7 @@ def test_trace_with_meta_accepted_by_agent(self, tags): for _ in range(999): with tracer.trace("child") as child: child.set_tags(tags) - tracer.shutdown() + tracer.flush() log.warning.assert_not_called() log.error.assert_not_called() @@ -60,7 +60,7 @@ def test_trace_with_metrics_accepted_by_agent(self, metrics): for _ in range(999): with tracer.trace("child") as child: child.set_metrics(metrics) - tracer.shutdown() + tracer.flush() log.warning.assert_not_called() log.error.assert_not_called() @@ -79,6 +79,6 @@ def test_trace_with_links_accepted_by_agent(self, span_links_kwargs): for _ in range(10): with tracer.trace("child") as child: child.set_link(**span_links_kwargs) - tracer.shutdown() + tracer.flush() log.warning.assert_not_called() log.error.assert_not_called() diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index 78606bbde14..529bdbcd40b 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -10,12 +10,8 @@ from ddtrace import Tracer from ddtrace.internal.atexit import register_on_exit_signal from ddtrace.internal.runtime import container -from ddtrace.internal.writer import AgentWriter -from tests.integration.utils import AGENT_VERSION -from tests.integration.utils import BadEncoder from tests.integration.utils import import_ddtrace_in_subprocess from tests.integration.utils import parametrize_with_all_encodings -from tests.integration.utils import send_invalid_payload_and_get_logs from tests.integration.utils import skip_if_testagent from tests.utils import call_program @@ -23,8 +19,11 @@ FOUR_KB = 1 << 12 +@pytest.mark.subprocess() def test_configure_keeps_api_hostname_and_port(): - tracer = Tracer() + from ddtrace import tracer + from tests.integration.utils import AGENT_VERSION + assert tracer._writer.agent_url == "http://localhost:{}".format("9126" if AGENT_VERSION == "testagent" else "8126") tracer.configure(hostname="127.0.0.1", port=8127) assert tracer._writer.agent_url == "http://127.0.0.1:8127" @@ -506,8 +505,12 @@ def test_validate_headers_in_payload_to_intake_with_nested_spans(): assert headers.get("X-Datadog-Trace-Count") == "10" +@parametrize_with_all_encodings def test_trace_with_invalid_client_endpoint_generates_error_log(): - t = Tracer() + import mock + + from ddtrace import tracer as t + for client in t._writer._clients: client.ENDPOINT = "/bad" with mock.patch("ddtrace.internal.writer.writer.log") as log: @@ -526,7 +529,12 @@ def test_trace_with_invalid_client_endpoint_generates_error_log(): @skip_if_testagent +@pytest.mark.subprocess(err=None) def test_trace_with_invalid_payload_generates_error_log(): + import mock + + from tests.integration.utils import send_invalid_payload_and_get_logs + log = send_invalid_payload_and_get_logs() log.error.assert_has_calls( [ @@ -541,11 +549,11 @@ def test_trace_with_invalid_payload_generates_error_log(): @skip_if_testagent -@pytest.mark.subprocess(env={"_DD_TRACE_WRITER_LOG_ERROR_PAYLOADS": "true", "DD_TRACE_API_VERSION": "v0.5"}) +@pytest.mark.subprocess(env={"_DD_TRACE_WRITER_LOG_ERROR_PAYLOADS": "true", "DD_TRACE_API_VERSION": "v0.5"}, err=None) def test_trace_with_invalid_payload_logs_payload_when_LOG_ERROR_PAYLOADS(): import mock - from tests.integration.test_integration import send_invalid_payload_and_get_logs + from tests.integration.utils import send_invalid_payload_and_get_logs log = send_invalid_payload_and_get_logs() log.error.assert_has_calls( @@ -562,12 +570,12 @@ def test_trace_with_invalid_payload_logs_payload_when_LOG_ERROR_PAYLOADS(): @skip_if_testagent -@pytest.mark.subprocess(env={"_DD_TRACE_WRITER_LOG_ERROR_PAYLOADS": "true", "DD_TRACE_API_VERSION": "v0.5"}) +@pytest.mark.subprocess(env={"_DD_TRACE_WRITER_LOG_ERROR_PAYLOADS": "true", "DD_TRACE_API_VERSION": "v0.5"}, err=None) def test_trace_with_non_bytes_payload_logs_payload_when_LOG_ERROR_PAYLOADS(): import mock - from tests.integration.test_integration import send_invalid_payload_and_get_logs from tests.integration.utils import BadEncoder + from tests.integration.utils import send_invalid_payload_and_get_logs class NonBytesBadEncoder(BadEncoder): def encode(self): @@ -590,7 +598,11 @@ def encode_traces(self, traces): ) +@pytest.mark.subprocess(err=None) def test_trace_with_failing_encoder_generates_error_log(): + from tests.integration.utils import BadEncoder + from tests.integration.utils import send_invalid_payload_and_get_logs + class ExceptionBadEncoder(BadEncoder): def encode(self): raise Exception() @@ -620,8 +632,11 @@ def test_api_version_downgrade_generates_no_warning_logs(): log.error.assert_not_called() +@pytest.mark.subprocess() def test_synchronous_writer_shutdown_raises_no_exception(): - tracer = Tracer() + from ddtrace import tracer + from ddtrace.internal.writer import AgentWriter + tracer.configure(writer=AgentWriter(tracer._writer.agent_url, sync_mode=True)) tracer.shutdown() diff --git a/tests/integration/test_integration_civisibility.py b/tests/integration/test_integration_civisibility.py index 8a504f6a220..6cb284457f8 100644 --- a/tests/integration/test_integration_civisibility.py +++ b/tests/integration/test_integration_civisibility.py @@ -3,17 +3,14 @@ import mock import pytest -from ddtrace._trace.tracer import Tracer from ddtrace.internal import agent from ddtrace.internal.ci_visibility import CIVisibility from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings from ddtrace.internal.ci_visibility.constants import AGENTLESS_ENDPOINT -from ddtrace.internal.ci_visibility.constants import COVERAGE_TAG_NAME from ddtrace.internal.ci_visibility.constants import EVP_PROXY_AGENT_ENDPOINT from ddtrace.internal.ci_visibility.constants import EVP_SUBDOMAIN_HEADER_EVENT_VALUE from ddtrace.internal.ci_visibility.constants import EVP_SUBDOMAIN_HEADER_NAME -from ddtrace.internal.ci_visibility.writer import CIVisibilityWriter -from ddtrace.internal.utils.http import Response +from ddtrace.internal.ci_visibility.recorder import CIVisibilityTracer as Tracer from tests.ci_visibility.util import _get_default_civisibility_ddconfig from tests.utils import override_env @@ -74,9 +71,17 @@ def test_civisibility_intake_with_apikey(): CIVisibility.disable() +@pytest.mark.subprocess() def test_civisibility_intake_payloads(): + import mock + + from ddtrace import tracer as t + from ddtrace.internal.ci_visibility.constants import COVERAGE_TAG_NAME + from ddtrace.internal.ci_visibility.recorder import CIVisibilityWriter + from ddtrace.internal.utils.http import Response + from tests.utils import override_env + with override_env(dict(DD_API_KEY="foobar.baz")): - t = Tracer() t.configure(writer=CIVisibilityWriter(reuse_connections=True, coverage_enabled=True)) t._writer._conn = mock.MagicMock() with mock.patch("ddtrace.internal.writer.Response.from_http_response") as from_http_response: diff --git a/tests/integration/test_integration_snapshots.py b/tests/integration/test_integration_snapshots.py index bd48faa34a6..6e656081eb1 100644 --- a/tests/integration/test_integration_snapshots.py +++ b/tests/integration/test_integration_snapshots.py @@ -7,23 +7,21 @@ from ddtrace import Tracer from ddtrace import tracer -from ddtrace.constants import AUTO_KEEP -from ddtrace.constants import SAMPLING_PRIORITY_KEY -from ddtrace.constants import USER_KEEP -from ddtrace.internal.writer import AgentWriter +from tests.integration.utils import AGENT_VERSION from tests.integration.utils import mark_snapshot from tests.integration.utils import parametrize_with_all_encodings from tests.utils import override_global_config from tests.utils import snapshot -from .test_integration import AGENT_VERSION - pytestmark = pytest.mark.skipif(AGENT_VERSION != "testagent", reason="Tests only compatible with a testagent") @snapshot(include_tracer=True) +@pytest.mark.subprocess() def test_single_trace_single_span(tracer): + from ddtrace import tracer + s = tracer.trace("operation", service="my-svc") s.set_tag("k", "v") # numeric tag @@ -31,11 +29,14 @@ def test_single_trace_single_span(tracer): s.set_metric("float_metric", 12.34) s.set_metric("int_metric", 4321) s.finish() - tracer.shutdown() + tracer.flush() @snapshot(include_tracer=True) +@pytest.mark.subprocess() def test_multiple_traces(tracer): + from ddtrace import tracer + with tracer.trace("operation1", service="my-svc") as s: s.set_tag("k", "v") s.set_tag("num", 1234) @@ -49,15 +50,22 @@ def test_multiple_traces(tracer): s.set_metric("float_metric", 12.34) s.set_metric("int_metric", 4321) tracer.trace("child").finish() - tracer.shutdown() + tracer.flush() -@pytest.mark.parametrize( - "writer", - ("default", "sync"), -) @snapshot(include_tracer=True) -def test_filters(writer, tracer): +@pytest.mark.subprocess( + parametrize={"DD_WRITER_MODE": ["default", "sync"]}, + token="tests.integration.test_integration_snapshots.test_filters", +) +def test_filters(): + import os + + from ddtrace import tracer + from ddtrace.internal.writer import AgentWriter + + writer = os.environ.get("DD_WRITER_MODE", "default") + if writer == "sync": writer = AgentWriter( tracer.agent_trace_url, @@ -89,15 +97,18 @@ def process_trace(self, trace): with tracer.trace("root"): with tracer.trace("child"): pass - tracer.shutdown() + tracer.flush() # Have to use sync mode snapshot so that the traces are associated to this # test case since we use a custom writer (that doesn't have the trace headers # injected). +@pytest.mark.subprocess() @snapshot(async_mode=False) def test_synchronous_writer(): - tracer = Tracer() + from ddtrace import tracer + from ddtrace.internal.writer import AgentWriter + writer = AgentWriter(tracer._writer.agent_url, sync_mode=True) tracer.configure(writer=writer) with tracer.trace("operation1", service="my-svc"): @@ -117,19 +128,18 @@ def test_tracer_trace_across_popen(): the child span has does not have '_dd.p.dm' shows that sampling was run before fork automatically. """ - tracer = Tracer() def task(tracer): with tracer.trace("child"): pass - tracer.shutdown() + tracer.flush() with tracer.trace("parent"): p = multiprocessing.Process(target=task, args=(tracer,)) p.start() p.join() - tracer.shutdown() + tracer.flush() @snapshot(async_mode=False) @@ -140,31 +150,34 @@ def test_tracer_trace_across_multiple_popens(): the child span has does not have '_dd.p.dm' shows that sampling was run before fork automatically. """ - tracer = Tracer() def task(tracer): def task2(tracer): with tracer.trace("child2"): pass - tracer.shutdown() + tracer.flush() with tracer.trace("child1"): p = multiprocessing.Process(target=task2, args=(tracer,)) p.start() p.join() - tracer.shutdown() + tracer.flush() with tracer.trace("parent"): p = multiprocessing.Process(target=task, args=(tracer,)) p.start() p.join() - tracer.shutdown() + tracer.flush() @snapshot() +@pytest.mark.subprocess() def test_wrong_span_name_type_not_sent(): """Span names should be a text type.""" - tracer = Tracer() + import mock + + from ddtrace import tracer + with mock.patch("ddtrace._trace.span.log") as log: with tracer.trace(123): pass @@ -180,11 +193,9 @@ def test_wrong_span_name_type_not_sent(): ], ) @pytest.mark.parametrize("encoding", ["v0.4", "v0.5"]) -@snapshot() def test_trace_with_wrong_meta_types_not_sent(encoding, meta, monkeypatch): """Wrong meta types should raise TypeErrors during encoding and fail to send to the agent.""" with override_global_config(dict(_trace_api=encoding)): - tracer = Tracer() with mock.patch("ddtrace._trace.span.log") as log: with tracer.trace("root") as root: root._meta = meta @@ -218,14 +229,19 @@ def test_trace_with_wrong_metrics_types_not_sent(encoding, metrics, monkeypatch) log.exception.assert_called_once_with("error closing trace") -@snapshot() +@pytest.mark.subprocess() +@pytest.mark.snapshot() def test_tracetagsprocessor_only_adds_new_tags(): - tracer = Tracer() + from ddtrace import tracer + from ddtrace.constants import AUTO_KEEP + from ddtrace.constants import SAMPLING_PRIORITY_KEY + from ddtrace.constants import USER_KEEP + with tracer.trace(name="web.request") as span: span.context.sampling_priority = AUTO_KEEP span.set_metric(SAMPLING_PRIORITY_KEY, USER_KEEP) - tracer.shutdown() + tracer.flush() # Override the token so that both parameterizations of the test use the same snapshot diff --git a/tests/integration/test_priority_sampling.py b/tests/integration/test_priority_sampling.py index 59177be57cb..653ef96d49e 100644 --- a/tests/integration/test_priority_sampling.py +++ b/tests/integration/test_priority_sampling.py @@ -9,12 +9,11 @@ from ddtrace.internal.encoding import MsgpackEncoderV04 as Encoder from ddtrace.internal.writer import AgentWriter from ddtrace.tracer import Tracer +from tests.integration.utils import AGENT_VERSION from tests.integration.utils import parametrize_with_all_encodings from tests.integration.utils import skip_if_testagent from tests.utils import override_global_config -from .test_integration import AGENT_VERSION - def _turn_tracer_into_dummy(tracer): """Override tracer's writer's write() method to keep traces instead of sending them away""" diff --git a/tests/integration/test_propagation.py b/tests/integration/test_propagation.py index 5bd0a122a8c..bcad0ed4432 100644 --- a/tests/integration/test_propagation.py +++ b/tests/integration/test_propagation.py @@ -1,44 +1,15 @@ import pytest -from ddtrace import Tracer +from ddtrace import tracer from ddtrace.constants import MANUAL_DROP_KEY from ddtrace.propagation.http import HTTPPropagator -from tests.utils import override_global_config - -from .test_integration import AGENT_VERSION +from tests.integration.utils import AGENT_VERSION pytestmark = pytest.mark.skipif(AGENT_VERSION != "testagent", reason="Tests only compatible with a testagent") -@pytest.fixture( - params=[ - dict(global_config=dict()), - dict( - global_config=dict(_x_datadog_tags_max_length="0", _x_datadog_tags_enabled=False), - ), - dict(global_config=dict(), partial_flush_enabled=True, partial_flush_min_spans=2), - ] -) -def tracer(request): - global_config = request.param.get("global_config", dict()) - partial_flush_enabled = request.param.get("partial_flush_enabled") - partial_flush_min_spans = request.param.get("partial_flush_min_spans") - with override_global_config(global_config): - tracer = Tracer() - kwargs = dict() - if partial_flush_enabled: - kwargs["partial_flush_enabled"] = partial_flush_enabled - if partial_flush_min_spans: - kwargs["partial_flush_min_spans"] = partial_flush_min_spans - tracer.configure(**kwargs) - yield tracer - tracer.shutdown() - - -@pytest.mark.snapshot() def test_trace_tags_multispan(): - tracer = Tracer() headers = { "x-datadog-trace-id": "1234", "x-datadog-parent-id": "5678", @@ -61,15 +32,8 @@ def test_trace_tags_multispan(): gc.finish() -@pytest.fixture -def downstream_tracer(): - tracer = Tracer() - yield tracer - tracer.shutdown() - - @pytest.mark.snapshot() -def test_sampling_decision_downstream(downstream_tracer): +def test_sampling_decision_downstream(): """ Ensures that set_tag(MANUAL_DROP_KEY) on a span causes the sampling decision meta and sampling priority metric to be set appropriately indicating rejection @@ -81,7 +45,7 @@ def test_sampling_decision_downstream(downstream_tracer): "x-datadog-tags": "_dd.p.dm=-1", } kept_trace_context = HTTPPropagator.extract(headers_indicating_kept_trace) - downstream_tracer.context_provider.activate(kept_trace_context) + tracer.context_provider.activate(kept_trace_context) - with downstream_tracer.trace("p", service="downstream") as span_to_reject: + with tracer.trace("p", service="downstream") as span_to_reject: span_to_reject.set_tag(MANUAL_DROP_KEY) diff --git a/tests/integration/test_sampling.py b/tests/integration/test_sampling.py index 902b430bbc8..66496342dee 100644 --- a/tests/integration/test_sampling.py +++ b/tests/integration/test_sampling.py @@ -1,16 +1,14 @@ -import mock import pytest +from ddtrace._trace.sampler import DatadogSampler +from ddtrace._trace.sampler import RateSampler +from ddtrace._trace.sampler import SamplingRule from ddtrace.constants import MANUAL_DROP_KEY from ddtrace.constants import MANUAL_KEEP_KEY from ddtrace.internal.writer import AgentWriter -from ddtrace.sampler import DatadogSampler -from ddtrace.sampler import RateSampler -from ddtrace.sampler import SamplingRule +from tests.integration.utils import AGENT_VERSION from tests.utils import snapshot -from .test_integration import AGENT_VERSION - pytestmark = pytest.mark.skipif(AGENT_VERSION != "testagent", reason="Tests only compatible with a testagent") RESOURCE = "mycoolre$ource" # codespell:ignore @@ -19,6 +17,9 @@ def snapshot_parametrized_with_writers(f): def _patch(writer, tracer): + old_sampler = tracer._sampler + old_writer = tracer._writer + old_tags = tracer._tags if writer == "sync": writer = AgentWriter( tracer.agent_trace_url, @@ -29,11 +30,13 @@ def _patch(writer, tracer): writer._headers = tracer._writer._headers else: writer = tracer._writer - tracer.configure(writer=writer) try: return f(writer, tracer) finally: - tracer.shutdown() + tracer.flush() + # Reset tracer configurations to avoid leaking state between tests + tracer.configure(sampler=old_sampler, writer=old_writer) + tracer._tags = old_tags wrapped = snapshot(include_tracer=True, token_override=f.__name__)(_patch) return pytest.mark.parametrize( @@ -298,10 +301,14 @@ def test_extended_sampling_float_special_case_match_star(writer, tracer): span.set_tag("tag", 20.1) +@pytest.mark.subprocess() def test_rate_limiter_on_spans(tracer): """ Ensure that the rate limiter is applied to spans """ + from ddtrace import tracer + from ddtrace.sampler import DatadogSampler + # Rate limit is only applied if a sample rate or trace sample rule is set tracer.configure(sampler=DatadogSampler(default_sample_rate=1, rate_limit=10)) spans = [] @@ -325,10 +332,16 @@ def test_rate_limiter_on_spans(tracer): assert dropped_span.context.sampling_priority < 0 +@pytest.mark.subprocess() def test_rate_limiter_on_long_running_spans(tracer): """ Ensure that the rate limiter is applied on increasing time intervals """ + import mock + + from ddtrace import tracer + from ddtrace.sampler import DatadogSampler + tracer.configure(sampler=DatadogSampler(rate_limit=5)) with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=1617333414): diff --git a/tests/integration/test_settings.py b/tests/integration/test_settings.py index 55e8d1e76d8..249b0211bb4 100644 --- a/tests/integration/test_settings.py +++ b/tests/integration/test_settings.py @@ -2,7 +2,7 @@ import pytest -from .test_integration import AGENT_VERSION +from tests.integration.utils import AGENT_VERSION def _get_telemetry_config_items(events, item_name): diff --git a/tests/integration/test_trace_stats.py b/tests/integration/test_trace_stats.py index 0fd7695fc23..f1eefcea709 100644 --- a/tests/integration/test_trace_stats.py +++ b/tests/integration/test_trace_stats.py @@ -5,25 +5,23 @@ import mock import pytest -from ddtrace import Tracer +from ddtrace._trace.sampler import DatadogSampler +from ddtrace._trace.sampler import SamplingRule from ddtrace.constants import SPAN_MEASURED_KEY from ddtrace.ext import http from ddtrace.internal.processor.stats import SpanStatsProcessorV06 -from ddtrace.sampler import DatadogSampler -from ddtrace.sampler import SamplingRule +from tests.integration.utils import AGENT_VERSION +from tests.utils import DummyTracer from tests.utils import override_global_config -from .test_integration import AGENT_VERSION - pytestmark = pytest.mark.skipif(AGENT_VERSION != "testagent", reason="Tests only compatible with a testagent") @pytest.fixture def stats_tracer(): - # type: (float) -> Generator[Tracer, None, None] with override_global_config(dict(_trace_compute_stats=True)): - tracer = Tracer() + tracer = DummyTracer() yield tracer tracer.shutdown() @@ -70,7 +68,7 @@ def test_compute_stats_default_and_configure(run_python_code_in_subprocess, envv """Ensure stats computation can be enabled.""" # Test enabling via `configure` - t = Tracer() + t = DummyTracer() assert not t._compute_stats assert not any(isinstance(p, SpanStatsProcessorV06) for p in t._span_processors) t.configure(compute_stats_enabled=True) @@ -100,14 +98,16 @@ def test_compute_stats_default_and_configure(run_python_code_in_subprocess, envv assert status == 0, out + err -def test_apm_opt_out_compute_stats_and_configure(run_python_code_in_subprocess): +@pytest.mark.subprocess(err=None) +def test_apm_opt_out_compute_stats_and_configure(): """ Ensure stats computation is disabled, but reported as enabled, if APM is opt-out. """ + from ddtrace import tracer as t + from ddtrace.internal.processor.stats import SpanStatsProcessorV06 # Test via `configure` - t = Tracer() assert not t._compute_stats assert not any(isinstance(p, SpanStatsProcessorV06) for p in t._span_processors) t.configure(appsec_enabled=True, appsec_standalone_enabled=True) @@ -116,8 +116,9 @@ def test_apm_opt_out_compute_stats_and_configure(run_python_code_in_subprocess): assert not t._compute_stats # but it's reported as enabled assert t._writer._headers.get("Datadog-Client-Computed-Stats") == "yes" - t.configure(appsec_enabled=False, appsec_standalone_enabled=False) + +def test_apm_opt_out_compute_stats_and_configure_env(run_python_code_in_subprocess): # Test via environment variable env = os.environ.copy() env.update({"DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED": "true", "DD_APPSEC_ENABLED": "true"}) diff --git a/tests/integration/test_tracemethods.py b/tests/integration/test_tracemethods.py index 8568cbc3737..15129c56161 100644 --- a/tests/integration/test_tracemethods.py +++ b/tests/integration/test_tracemethods.py @@ -5,7 +5,7 @@ import pytest -from .test_integration import AGENT_VERSION +from tests.integration.utils import AGENT_VERSION pytestmark = pytest.mark.skipif(AGENT_VERSION != "testagent", reason="Tests only compatible with a testagent") diff --git a/tests/integration/utils.py b/tests/integration/utils.py index dea4a091ed4..21822ea6e59 100644 --- a/tests/integration/utils.py +++ b/tests/integration/utils.py @@ -33,7 +33,7 @@ def send_invalid_payload_and_get_logs(encoder_cls=BadEncoder): client.encoder = encoder_cls() with mock.patch("ddtrace.internal.writer.writer.log") as log: t.trace("asdf").finish() - t.shutdown() + t.flush() return log diff --git a/tests/internal/crashtracker/test_crashtracker.py b/tests/internal/crashtracker/test_crashtracker.py index a4074745f83..96dc81c6a5f 100644 --- a/tests/internal/crashtracker/test_crashtracker.py +++ b/tests/internal/crashtracker/test_crashtracker.py @@ -3,8 +3,6 @@ import pytest -from ddtrace.settings.profiling import config as profiling_config -from ddtrace.settings.profiling import config_str import tests.internal.crashtracker.utils as utils @@ -506,7 +504,7 @@ def test_crashtracker_user_tags_envvar(run_python_code_in_subprocess): @pytest.mark.skipif(not sys.platform.startswith("linux"), reason="Linux only") -@pytest.mark.skipif(sys.version_info > (3, 12), reason="Fails on 3.13") +@pytest.mark.skipif(sys.version_info >= (3, 13), reason="Fails on 3.13") def test_crashtracker_set_tag_profiler_config(run_python_code_in_subprocess): port, sock = utils.crashtracker_receiver_bind() assert sock @@ -528,7 +526,11 @@ def test_crashtracker_set_tag_profiler_config(run_python_code_in_subprocess): # Now check for the profiler_config tag assert b"profiler_config" in data - profiler_config = config_str(profiling_config) + py_version = sys.version_info[:2] + if py_version >= (3, 8): + profiler_config = "stack_v2_lock_mem_heap_exp_dd_CAP1.0_MAXF64" + else: + profiler_config = "stack_lock_mem_heap_exp_dd_CAP1.0_MAXF64" assert profiler_config.encode() in data diff --git a/tests/internal/remoteconfig/test_remoteconfig.py b/tests/internal/remoteconfig/test_remoteconfig.py index 4875da2bfb0..5bf87179025 100644 --- a/tests/internal/remoteconfig/test_remoteconfig.py +++ b/tests/internal/remoteconfig/test_remoteconfig.py @@ -11,6 +11,8 @@ import pytest from ddtrace import config +from ddtrace._trace.sampler import DatadogSampler +from ddtrace._trace.sampling_rule import SamplingRule from ddtrace.internal.remoteconfig._connectors import PublisherSubscriberConnector from ddtrace.internal.remoteconfig._publishers import RemoteConfigPublisherMergeDicts from ddtrace.internal.remoteconfig._pubsub import PubSub @@ -21,8 +23,6 @@ from ddtrace.internal.remoteconfig.worker import RemoteConfigPoller from ddtrace.internal.remoteconfig.worker import remoteconfig_poller from ddtrace.internal.service import ServiceStatus -from ddtrace.sampler import DatadogSampler -from ddtrace.sampling_rule import SamplingRule from tests.internal.test_utils_version import _assert_and_get_version_agent_format from tests.utils import override_global_config diff --git a/tests/internal/test_module.py b/tests/internal/test_module.py index 885f796af81..c84c2c740d6 100644 --- a/tests/internal/test_module.py +++ b/tests/internal/test_module.py @@ -578,17 +578,6 @@ def __getattr__(name): "ddtrace.contrib.trace_utils", "ddtrace.contrib.trace_utils_async", "ddtrace.contrib.trace_utils_redis", - # TODO: The following contrib modules are part of the public API (unlike most integrations). - # We should consider privatizing the internals of these integrations. - "ddtrace.contrib.unittest.patch", - "ddtrace.contrib.unittest.constants", - "ddtrace.contrib.pytest.constants", - "ddtrace.contrib.pytest.newhooks", - "ddtrace.contrib.pytest.plugin", - "ddtrace.contrib.pytest_benchmark.constants", - "ddtrace.contrib.pytest_benchmark.plugin", - "ddtrace.contrib.pytest_bdd.constants", - "ddtrace.contrib.pytest_bdd.plugin", ] ) diff --git a/tests/internal/test_serverless.py b/tests/internal/test_serverless.py index 9c896829636..48d7586d25e 100644 --- a/tests/internal/test_serverless.py +++ b/tests/internal/test_serverless.py @@ -116,8 +116,8 @@ def find_spec(self, fullname, *args): sys.meta_path.insert(0, BlockListFinder()) import ddtrace - import ddtrace.contrib.aws_lambda # noqa:F401 - import ddtrace.contrib.psycopg # noqa:F401 + import ddtrace.contrib.internal.aws_lambda # noqa:F401 + import ddtrace.contrib.internal.psycopg # noqa:F401 finally: if isinstance(sys.meta_path[0], BlockListFinder): diff --git a/tests/internal/test_settings.py b/tests/internal/test_settings.py index 1d0ed54b1b9..97bdced47d9 100644 --- a/tests/internal/test_settings.py +++ b/tests/internal/test_settings.py @@ -233,7 +233,7 @@ def test_remoteconfig_sampling_rate_user(run_python_code_in_subprocess): out, err, status, _ = run_python_code_in_subprocess( """ from ddtrace import config, tracer -from ddtrace.sampler import DatadogSampler +from ddtrace._trace.sampler import DatadogSampler from tests.internal.test_settings import _base_rc_config, _deleted_rc_config with tracer.trace("test") as span: @@ -288,7 +288,7 @@ def test_remoteconfig_sampling_rules(run_python_code_in_subprocess): out, err, status, _ = run_python_code_in_subprocess( """ from ddtrace import config, tracer -from ddtrace.sampler import DatadogSampler +from ddtrace._trace.sampler import DatadogSampler from tests.internal.test_settings import _base_rc_config, _deleted_rc_config with tracer.trace("test") as span: @@ -379,7 +379,7 @@ def test_remoteconfig_sample_rate_and_rules(run_python_code_in_subprocess): out, err, status, _ = run_python_code_in_subprocess( """ from ddtrace import config, tracer -from ddtrace.sampler import DatadogSampler +from ddtrace._trace.sampler import DatadogSampler from tests.internal.test_settings import _base_rc_config, _deleted_rc_config with tracer.trace("rules") as span: diff --git a/tests/llmobs/_utils.py b/tests/llmobs/_utils.py index 0ecdde36ee6..3583516538c 100644 --- a/tests/llmobs/_utils.py +++ b/tests/llmobs/_utils.py @@ -210,11 +210,13 @@ def _get_llmobs_parent_id(span: Span): def _expected_llmobs_eval_metric_event( - span_id, - trace_id, metric_type, label, ml_app, + tag_key=None, + tag_value=None, + span_id=None, + trace_id=None, timestamp_ms=None, categorical_value=None, score_value=None, @@ -223,8 +225,7 @@ def _expected_llmobs_eval_metric_event( metadata=None, ): eval_metric_event = { - "span_id": span_id, - "trace_id": trace_id, + "join_on": {}, "metric_type": metric_type, "label": label, "tags": [ @@ -232,6 +233,10 @@ def _expected_llmobs_eval_metric_event( "ml_app:{}".format(ml_app if ml_app is not None else "unnamed-ml-app"), ], } + if tag_key is not None and tag_value is not None: + eval_metric_event["join_on"]["tag"] = {"key": tag_key, "value": tag_value} + if span_id is not None and trace_id is not None: + eval_metric_event["join_on"]["span"] = {"span_id": span_id, "trace_id": trace_id} if categorical_value is not None: eval_metric_event["categorical_value"] = categorical_value if score_value is not None: @@ -526,34 +531,73 @@ def _llm_span_with_expected_ragas_inputs_in_messages(ragas_inputs=None): class DummyEvaluator: - LABEL = "dummy" - - def __init__(self, llmobs_service): + def __init__(self, llmobs_service, label="dummy"): self.llmobs_service = llmobs_service + self.LABEL = label def run_and_submit_evaluation(self, span): self.llmobs_service.submit_evaluation( span_context=span, - label=DummyEvaluator.LABEL, + label=self.LABEL, value=1.0, metric_type="score", ) -def _dummy_evaluator_eval_metric_event(span_id, trace_id): +def _dummy_evaluator_eval_metric_event(span_id, trace_id, label=None): return LLMObsEvaluationMetricEvent( - span_id=span_id, - trace_id=trace_id, + join_on={"span": {"span_id": span_id, "trace_id": trace_id}}, score_value=1.0, ml_app="unnamed-ml-app", timestamp_ms=mock.ANY, metric_type="score", - label=DummyEvaluator.LABEL, + label=label or "dummy", tags=["ddtrace.version:{}".format(ddtrace.__version__), "ml_app:unnamed-ml-app"], ) -def _expected_ragas_spans(ragas_inputs=None): +def _expected_ragas_context_precision_spans(ragas_inputs=None): + if not ragas_inputs: + ragas_inputs = default_ragas_inputs + return [ + { + "trace_id": mock.ANY, + "span_id": mock.ANY, + "parent_id": "undefined", + "name": "dd-ragas.context_precision", + "start_ns": mock.ANY, + "duration": mock.ANY, + "status": "ok", + "meta": { + "span.kind": "workflow", + "input": {"value": mock.ANY}, + "output": {"value": "1.0"}, + "metadata": {}, + }, + "metrics": {}, + "tags": expected_ragas_trace_tags(), + }, + { + "trace_id": mock.ANY, + "span_id": mock.ANY, + "parent_id": mock.ANY, + "name": "dd-ragas.extract_evaluation_inputs_from_span", + "start_ns": mock.ANY, + "duration": mock.ANY, + "status": "ok", + "meta": { + "span.kind": "workflow", + "input": {"value": mock.ANY}, + "output": {"value": mock.ANY}, + "metadata": {}, + }, + "metrics": {}, + "tags": expected_ragas_trace_tags(), + }, + ] + + +def _expected_ragas_faithfulness_spans(ragas_inputs=None): if not ragas_inputs: ragas_inputs = default_ragas_inputs return [ @@ -581,7 +625,7 @@ def _expected_ragas_spans(ragas_inputs=None): "trace_id": mock.ANY, "span_id": mock.ANY, "parent_id": mock.ANY, - "name": "dd-ragas.extract_faithfulness_inputs", + "name": "dd-ragas.extract_evaluation_inputs_from_span", "start_ns": mock.ANY, "duration": mock.ANY, "status": "ok", @@ -669,3 +713,61 @@ def _expected_ragas_spans(ragas_inputs=None): "tags": expected_ragas_trace_tags(), }, ] + + +def _expected_ragas_answer_relevancy_spans(ragas_inputs=None): + if not ragas_inputs: + ragas_inputs = default_ragas_inputs + return [ + { + "trace_id": mock.ANY, + "span_id": mock.ANY, + "parent_id": "undefined", + "name": "dd-ragas.answer_relevancy", + "start_ns": mock.ANY, + "duration": mock.ANY, + "status": "ok", + "meta": { + "span.kind": "workflow", + "input": {"value": mock.ANY}, + "output": {"value": mock.ANY}, + "metadata": {"answer_classifications": mock.ANY, "strictness": mock.ANY}, + }, + "metrics": {}, + "tags": expected_ragas_trace_tags(), + }, + { + "trace_id": mock.ANY, + "span_id": mock.ANY, + "parent_id": mock.ANY, + "name": "dd-ragas.extract_evaluation_inputs_from_span", + "start_ns": mock.ANY, + "duration": mock.ANY, + "status": "ok", + "meta": { + "span.kind": "workflow", + "input": {"value": mock.ANY}, + "output": {"value": mock.ANY}, + "metadata": {}, + }, + "metrics": {}, + "tags": expected_ragas_trace_tags(), + }, + { + "trace_id": mock.ANY, + "span_id": mock.ANY, + "parent_id": mock.ANY, + "name": "dd-ragas.calculate_similarity", + "start_ns": mock.ANY, + "duration": mock.ANY, + "status": "ok", + "meta": { + "span.kind": "workflow", + "input": {"value": mock.ANY}, + "output": {"value": mock.ANY}, + "metadata": {}, + }, + "metrics": {}, + "tags": expected_ragas_trace_tags(), + }, + ] diff --git a/tests/llmobs/conftest.py b/tests/llmobs/conftest.py index a7d467b3985..61a028e5caf 100644 --- a/tests/llmobs/conftest.py +++ b/tests/llmobs/conftest.py @@ -31,26 +31,6 @@ def pytest_configure(config): config.addinivalue_line("markers", "vcr_logs: mark test to use recorded request/responses") -@pytest.fixture -def mock_llmobs_span_writer(): - patcher = mock.patch("ddtrace.llmobs._llmobs.LLMObsSpanWriter") - LLMObsSpanWriterMock = patcher.start() - m = mock.MagicMock() - LLMObsSpanWriterMock.return_value = m - yield m - patcher.stop() - - -@pytest.fixture -def mock_llmobs_span_agentless_writer(): - patcher = mock.patch("ddtrace.llmobs._llmobs.LLMObsSpanWriter") - LLMObsSpanWriterMock = patcher.start() - m = mock.MagicMock() - LLMObsSpanWriterMock.return_value = m - yield m - patcher.stop() - - @pytest.fixture def mock_llmobs_eval_metric_writer(): patcher = mock.patch("ddtrace.llmobs._llmobs.LLMObsEvalMetricWriter") @@ -85,10 +65,7 @@ def mock_llmobs_submit_evaluation(): def mock_http_writer_send_payload_response(): with mock.patch( "ddtrace.internal.writer.HTTPWriter._send_payload", - return_value=Response( - status=200, - body="{}", - ), + return_value=Response(status=200, body="{}"), ): yield @@ -124,9 +101,10 @@ def mock_evaluator_sampler_logs(): @pytest.fixture -def mock_http_writer_logs(): - with mock.patch("ddtrace.internal.writer.writer.log") as m: +def mock_llmobs_logs(): + with mock.patch("ddtrace.llmobs._llmobs.log") as m: yield m + m.reset_mock() @pytest.fixture @@ -136,45 +114,7 @@ def ddtrace_global_config(): def default_global_config(): - return {"_dd_api_key": "", "_llmobs_ml_app": "unnamed-ml-app"} - - -@pytest.fixture -def LLMObs( - mock_llmobs_span_writer, mock_llmobs_eval_metric_writer, mock_llmobs_evaluator_runner, ddtrace_global_config -): - global_config = default_global_config() - global_config.update(ddtrace_global_config) - with override_global_config(global_config): - dummy_tracer = DummyTracer() - llmobs_service.enable(_tracer=dummy_tracer) - yield llmobs_service - llmobs_service.disable() - - -@pytest.fixture -def AgentlessLLMObs( - mock_llmobs_span_agentless_writer, - mock_llmobs_eval_metric_writer, - mock_llmobs_evaluator_runner, - ddtrace_global_config, -): - global_config = default_global_config() - global_config.update(ddtrace_global_config) - global_config.update(dict(_llmobs_agentless_enabled=True)) - with override_global_config(global_config): - dummy_tracer = DummyTracer() - llmobs_service.enable(_tracer=dummy_tracer) - yield llmobs_service - llmobs_service.disable() - - -@pytest.fixture -def disabled_llmobs(): - prev = llmobs_service.enabled - llmobs_service.enabled = False - yield - llmobs_service.enabled = prev + return {"_dd_api_key": "", "_llmobs_ml_app": "unnamed-ml-app", "service": "tests.llmobs"} @pytest.fixture @@ -189,23 +129,36 @@ def mock_ragas_dependencies_not_present(): @pytest.fixture -def ragas(mock_llmobs_span_writer, mock_llmobs_eval_metric_writer): +def ragas(mock_llmobs_eval_metric_writer): with override_global_config(dict(_dd_api_key="")): - import ragas - + try: + import ragas + except ImportError: + pytest.skip("Ragas not installed") with override_env(dict(OPENAI_API_KEY=os.getenv("OPENAI_API_KEY", ""))): yield ragas @pytest.fixture def reset_ragas_faithfulness_llm(): - import ragas - + try: + import ragas + except ImportError: + pytest.skip("Ragas not installed") previous_llm = ragas.metrics.faithfulness.llm yield ragas.metrics.faithfulness.llm = previous_llm +@pytest.fixture +def reset_ragas_answer_relevancy_llm(): + import ragas + + previous_llm = ragas.metrics.answer_relevancy.llm + yield + ragas.metrics.answer_relevancy.llm = previous_llm + + @pytest.fixture def mock_ragas_evaluator(mock_llmobs_eval_metric_writer, ragas): patcher = mock.patch("ddtrace.llmobs._evaluators.ragas.faithfulness.RagasFaithfulnessEvaluator.evaluate") @@ -215,6 +168,17 @@ def mock_ragas_evaluator(mock_llmobs_eval_metric_writer, ragas): patcher.stop() +@pytest.fixture +def mock_ragas_answer_relevancy_calculate_similarity(): + import numpy + + patcher = mock.patch("ragas.metrics.answer_relevancy.calculate_similarity") + MockRagasCalcSim = patcher.start() + MockRagasCalcSim.return_value = numpy.array([1.0, 1.0, 1.0]) + yield MockRagasCalcSim + patcher.stop() + + @pytest.fixture def tracer(): return DummyTracer() @@ -243,16 +207,25 @@ def llmobs_span_writer(): @pytest.fixture -def llmobs(monkeypatch, tracer, llmobs_env, llmobs_span_writer): +def llmobs( + ddtrace_global_config, + monkeypatch, + tracer, + llmobs_env, + llmobs_span_writer, + mock_llmobs_eval_metric_writer, + mock_llmobs_evaluator_runner, +): for env, val in llmobs_env.items(): monkeypatch.setenv(env, val) - + global_config = default_global_config() + global_config.update(dict(_llmobs_ml_app=llmobs_env.get("DD_LLMOBS_ML_APP"))) + global_config.update(ddtrace_global_config) # TODO: remove once rest of tests are moved off of global config tampering - with override_global_config(dict(_llmobs_ml_app=llmobs_env.get("DD_LLMOBS_ML_APP"))): + with override_global_config(global_config): llmobs_service.enable(_tracer=tracer) llmobs_service._instance._llmobs_span_writer = llmobs_span_writer - llmobs_service._instance._trace_processor._span_writer = llmobs_span_writer - yield llmobs + yield llmobs_service llmobs_service.disable() diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.send_score_metric.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.send_score_metric.yaml index 61c26ff7bf0..f767f5de303 100644 --- a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.send_score_metric.yaml +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.send_score_metric.yaml @@ -1,27 +1,28 @@ interactions: - request: - body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"span_id": - "12345678902", "trace_id": "98765432102", "metric_type": "score", "label": "sentiment", - "score_value": 0.9, "ml_app": "dummy-ml-app", "timestamp_ms": 1724249500942}]}}}' + body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"join_on": + {"span": {"span_id": "12345678902", "trace_id": "98765432102"}}, "metric_type": + "score", "label": "sentiment", "score_value": 0.9, "ml_app": "dummy-ml-app", + "timestamp_ms": 1732568298743}]}}}' headers: Content-Type: - application/json DD-API-KEY: - XXXXXX method: POST - uri: https://api.datad0g.com/api/intake/llm-obs/v1/eval-metric + uri: https://api.datad0g.com/api/intake/llm-obs/v2/eval-metric response: body: - string: '{"data":{"id":"e66c93b9-ca0a-4f0a-9207-497e0a1b6eec","type":"evaluation_metric","attributes":{"metrics":[{"id":"5fb5ed5d-20c1-4f34-abf9-c0bdc09680e3","trace_id":"98765432102","span_id":"12345678902","timestamp_ms":1724249500942,"ml_app":"dummy-ml-app","metric_type":"score","label":"sentiment","score_value":0.9}]}}}' + string: '{"data":{"id":"5b998846-53af-4b0e-a658-fd9e06726d6d","type":"evaluation_metric","attributes":{"metrics":[{"id":"jbGbAMC7Rk","join_on":{"span":{"trace_id":"98765432102","span_id":"12345678902"}},"timestamp_ms":1732568298743,"ml_app":"dummy-ml-app","metric_type":"score","label":"sentiment","score_value":0.9}]}}}' headers: content-length: - - '316' + - '311' content-security-policy: - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pub293163a918901030b79492fe1ab424cf&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatad0g.com content-type: - application/vnd.api+json date: - - Wed, 21 Aug 2024 14:11:41 GMT + - Mon, 25 Nov 2024 20:58:19 GMT strict-transport-security: - max-age=31536000; includeSubDomains; preload vary: diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_categorical_metric.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_categorical_metric.yaml index 92498e86e9e..f4404b30832 100644 --- a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_categorical_metric.yaml +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_categorical_metric.yaml @@ -1,27 +1,28 @@ interactions: - request: - body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"span_id": - "12345678901", "trace_id": "98765432101", "metric_type": "categorical", "categorical_value": - "very", "label": "toxicity", "ml_app": "dummy-ml-app", "timestamp_ms": 1724249500339}]}}}' + body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"join_on": + {"span": {"span_id": "12345678901", "trace_id": "98765432101"}}, "metric_type": + "categorical", "categorical_value": "very", "label": "toxicity", "ml_app": "dummy-ml-app", + "timestamp_ms": 1732568297450}]}}}' headers: Content-Type: - application/json DD-API-KEY: - XXXXXX method: POST - uri: https://api.datad0g.com/api/intake/llm-obs/v1/eval-metric + uri: https://api.datad0g.com/api/intake/llm-obs/v2/eval-metric response: body: - string: '{"data":{"id":"36d88c24-d7d4-4d3e-853c-b695aff61344","type":"evaluation_metric","attributes":{"metrics":[{"id":"0c189d9c-a730-4c5d-bbc2-55ef3455900f","trace_id":"98765432101","span_id":"12345678901","timestamp_ms":1724249500339,"ml_app":"dummy-ml-app","metric_type":"categorical","label":"toxicity","categorical_value":"very"}]}}}' + string: '{"data":{"id":"49c5c927-76f1-4de4-ad97-e1a0a159229f","type":"evaluation_metric","attributes":{"metrics":[{"id":"okVf1U4XzA","join_on":{"span":{"trace_id":"98765432101","span_id":"12345678901"}},"timestamp_ms":1732568297450,"ml_app":"dummy-ml-app","metric_type":"categorical","label":"toxicity","categorical_value":"very"}]}}}' headers: content-length: - - '330' + - '325' content-security-policy: - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pub293163a918901030b79492fe1ab424cf&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatad0g.com content-type: - application/vnd.api+json date: - - Wed, 21 Aug 2024 14:11:40 GMT + - Mon, 25 Nov 2024 20:58:17 GMT strict-transport-security: - max-age=31536000; includeSubDomains; preload vary: diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_metric_bad_api_key.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_metric_bad_api_key.yaml index 68fe0315870..ef6f4cf445e 100644 --- a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_metric_bad_api_key.yaml +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_metric_bad_api_key.yaml @@ -1,15 +1,16 @@ interactions: - request: - body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"span_id": - "12345678901", "trace_id": "98765432101", "metric_type": "categorical", "categorical_value": - "very", "label": "toxicity", "ml_app": "dummy-ml-app", "timestamp_ms": 1724249500253}]}}}' + body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"join_on": + {"span": {"span_id": "12345678901", "trace_id": "98765432101"}}, "metric_type": + "categorical", "categorical_value": "very", "label": "toxicity", "ml_app": "dummy-ml-app", + "timestamp_ms": 1732568297307}]}}}' headers: Content-Type: - application/json DD-API-KEY: - XXXXXX method: POST - uri: https://api.datad0g.com/api/intake/llm-obs/v1/eval-metric + uri: https://api.datad0g.com/api/intake/llm-obs/v2/eval-metric response: body: string: '{"status":"error","code":403,"errors":["Forbidden"],"statuspage":"http://status.datadoghq.com","twitter":"http://twitter.com/datadogops","email":"support@datadoghq.com"}' @@ -21,7 +22,7 @@ interactions: content-type: - application/json date: - - Wed, 21 Aug 2024 14:11:40 GMT + - Mon, 25 Nov 2024 20:58:17 GMT strict-transport-security: - max-age=31536000; includeSubDomains; preload x-content-type-options: diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_multiple_events.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_multiple_events.yaml index 61da12cd3fa..3638a1cf608 100644 --- a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_multiple_events.yaml +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_multiple_events.yaml @@ -1,32 +1,30 @@ interactions: - request: - body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"span_id": - "12345678902", "trace_id": "98765432102", "metric_type": "score", "label": "sentiment", - "score_value": 0.9, "ml_app": "dummy-ml-app", "timestamp_ms": 1724249589510}, - {"span_id": "12345678901", "trace_id": "98765432101", "metric_type": "categorical", - "categorical_value": "very", "label": "toxicity", "ml_app": "dummy-ml-app", - "timestamp_ms": 1724249589510}]}}}' + body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"join_on": + {"span": {"span_id": "12345678902", "trace_id": "98765432102"}}, "metric_type": + "score", "label": "sentiment", "score_value": 0.9, "ml_app": "dummy-ml-app", + "timestamp_ms": 1732568728793}, {"join_on": {"span": {"span_id": "12345678901", + "trace_id": "98765432101"}}, "metric_type": "categorical", "categorical_value": + "very", "label": "toxicity", "ml_app": "dummy-ml-app", "timestamp_ms": 1732568728793}]}}}' headers: Content-Type: - application/json DD-API-KEY: - XXXXXX method: POST - uri: https://api.datad0g.com/api/intake/llm-obs/v1/eval-metric + uri: https://api.datad0g.com/api/intake/llm-obs/v2/eval-metric response: body: - string: '{"data":{"id":"2ccffdfc-024b-49e6-881c-4e4d1c5f450e","type":"evaluation_metric","attributes":{"metrics":[{"id":"ed072901-fd70-4417-9cab-1bad62b6ac09","trace_id":"98765432102","span_id":"12345678902","timestamp_ms":1724249589510,"ml_app":"dummy-ml-app","metric_type":"score","label":"sentiment","score_value":0.9},{"id":"16175a34-7c25-43ca-8551-bd2f7242ab77","trace_id":"98765432101","span_id":"12345678901","timestamp_ms":1724249589510,"ml_app":"dummy-ml-app","metric_type":"categorical","label":"toxicity","categorical_value":"very"}]}}}' + string: '{"data":{"id":"844be0cd-9dd4-45d3-9763-8ccb20f4e7c8","type":"evaluation_metric","attributes":{"metrics":[{"id":"IZhAbBsXBJ","join_on":{"span":{"trace_id":"98765432102","span_id":"12345678902"}},"timestamp_ms":1732568728793,"ml_app":"dummy-ml-app","metric_type":"score","label":"sentiment","score_value":0.9},{"id":"ME868fTl0T","join_on":{"span":{"trace_id":"98765432101","span_id":"12345678901"}},"timestamp_ms":1732568728793,"ml_app":"dummy-ml-app","metric_type":"categorical","label":"toxicity","categorical_value":"very"}]}}}' headers: - Connection: - - keep-alive - Content-Length: - - '538' - Content-Type: - - application/vnd.api+json - Date: - - Wed, 21 Aug 2024 14:13:09 GMT + content-length: + - '528' content-security-policy: - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pub293163a918901030b79492fe1ab424cf&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatad0g.com + content-type: + - application/vnd.api+json + date: + - Mon, 25 Nov 2024 21:05:29 GMT strict-transport-security: - max-age=31536000; includeSubDomains; preload vary: diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_score_metric.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_score_metric.yaml index 1394f9fbb43..65bb0fa1562 100644 --- a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_score_metric.yaml +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_score_metric.yaml @@ -1,27 +1,28 @@ interactions: - request: - body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"span_id": - "12345678902", "trace_id": "98765432102", "metric_type": "score", "label": "sentiment", - "score_value": 0.9, "ml_app": "dummy-ml-app", "timestamp_ms": 1724249500471}]}}}' + body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"join_on": + {"span": {"span_id": "12345678902", "trace_id": "98765432102"}}, "metric_type": + "score", "label": "sentiment", "score_value": 0.9, "ml_app": "dummy-ml-app", + "timestamp_ms": 1732568297772}]}}}' headers: Content-Type: - application/json DD-API-KEY: - XXXXXX method: POST - uri: https://api.datad0g.com/api/intake/llm-obs/v1/eval-metric + uri: https://api.datad0g.com/api/intake/llm-obs/v2/eval-metric response: body: - string: '{"data":{"id":"5bd1b0b7-0acd-46e2-8ff6-3ee6a92457b6","type":"evaluation_metric","attributes":{"metrics":[{"id":"d8aa2a23-3137-4c49-b87b-d1eb1c3af04e","trace_id":"98765432102","span_id":"12345678902","timestamp_ms":1724249500471,"ml_app":"dummy-ml-app","metric_type":"score","label":"sentiment","score_value":0.9}]}}}' + string: '{"data":{"id":"d1518236-84b1-4b47-9cbc-ffc24188b5cc","type":"evaluation_metric","attributes":{"metrics":[{"id":"jiKtwDKR0B","join_on":{"span":{"trace_id":"98765432102","span_id":"12345678902"}},"timestamp_ms":1732568297772,"ml_app":"dummy-ml-app","metric_type":"score","label":"sentiment","score_value":0.9}]}}}' headers: content-length: - - '316' + - '311' content-security-policy: - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pub293163a918901030b79492fe1ab424cf&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatad0g.com content-type: - application/vnd.api+json date: - - Wed, 21 Aug 2024 14:11:40 GMT + - Mon, 25 Nov 2024 20:58:18 GMT strict-transport-security: - max-age=31536000; includeSubDomains; preload vary: diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_timed_events.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_timed_events.yaml index c9797ace419..c31d610bd57 100644 --- a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_timed_events.yaml +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_eval_metric_writer.test_send_timed_events.yaml @@ -1,27 +1,28 @@ interactions: - request: - body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"span_id": - "12345678902", "trace_id": "98765432102", "metric_type": "score", "label": "sentiment", - "score_value": 0.9, "ml_app": "dummy-ml-app", "timestamp_ms": 1724249982978}]}}}' + body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"join_on": + {"span": {"span_id": "12345678902", "trace_id": "98765432102"}}, "metric_type": + "score", "label": "sentiment", "score_value": 0.9, "ml_app": "dummy-ml-app", + "timestamp_ms": 1732568764624}]}}}' headers: Content-Type: - application/json DD-API-KEY: - XXXXXX method: POST - uri: https://api.datad0g.com/api/intake/llm-obs/v1/eval-metric + uri: https://api.datad0g.com/api/intake/llm-obs/v2/eval-metric response: body: - string: '{"data":{"id":"aba22157-cc3a-4601-a6a5-7afa99eee73e","type":"evaluation_metric","attributes":{"metrics":[{"id":"c2f6f63c-17ca-48c3-ad2d-676b2a35e726","trace_id":"98765432102","span_id":"12345678902","timestamp_ms":1724249982978,"ml_app":"dummy-ml-app","metric_type":"score","label":"sentiment","score_value":0.9}]}}}' + string: '{"data":{"id":"5352c11a-dcdd-449b-af72-2ae0b5dac3a1","type":"evaluation_metric","attributes":{"metrics":[{"id":"WmMD7E_fAD","join_on":{"span":{"trace_id":"98765432102","span_id":"12345678902"}},"timestamp_ms":1732568764624,"ml_app":"dummy-ml-app","metric_type":"score","label":"sentiment","score_value":0.9}]}}}' headers: content-length: - - '316' + - '311' content-security-policy: - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pub293163a918901030b79492fe1ab424cf&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatad0g.com content-type: - application/vnd.api+json date: - - Wed, 21 Aug 2024 14:19:45 GMT + - Mon, 25 Nov 2024 21:06:04 GMT strict-transport-security: - max-age=31536000; includeSubDomains; preload vary: @@ -34,28 +35,29 @@ interactions: code: 202 message: Accepted - request: - body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"span_id": - "12345678901", "trace_id": "98765432101", "metric_type": "categorical", "categorical_value": - "very", "label": "toxicity", "ml_app": "dummy-ml-app", "timestamp_ms": 1724249983284}]}}}' + body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"join_on": + {"span": {"span_id": "12345678901", "trace_id": "98765432101"}}, "metric_type": + "categorical", "categorical_value": "very", "label": "toxicity", "ml_app": "dummy-ml-app", + "timestamp_ms": 1732568765127}]}}}' headers: Content-Type: - application/json DD-API-KEY: - XXXXXX method: POST - uri: https://api.datad0g.com/api/intake/llm-obs/v1/eval-metric + uri: https://api.datad0g.com/api/intake/llm-obs/v2/eval-metric response: body: - string: '{"data":{"id":"0bc39c40-6c72-4b11-9eea-826248f9fe37","type":"evaluation_metric","attributes":{"metrics":[{"id":"7da7eb5b-32d2-43b3-adf5-208313f822c5","trace_id":"98765432101","span_id":"12345678901","timestamp_ms":1724249983284,"ml_app":"dummy-ml-app","metric_type":"categorical","label":"toxicity","categorical_value":"very"}]}}}' + string: '{"data":{"id":"d39e806e-40c5-4b3c-b539-440390afca85","type":"evaluation_metric","attributes":{"metrics":[{"id":"403hQLmrQW","join_on":{"span":{"trace_id":"98765432101","span_id":"12345678901"}},"timestamp_ms":1732568765127,"ml_app":"dummy-ml-app","metric_type":"categorical","label":"toxicity","categorical_value":"very"}]}}}' headers: content-length: - - '330' + - '325' content-security-policy: - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pub293163a918901030b79492fe1ab424cf&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatad0g.com content-type: - application/vnd.api+json date: - - Wed, 21 Aug 2024 14:19:45 GMT + - Mon, 25 Nov 2024 21:06:05 GMT strict-transport-security: - max-age=31536000; includeSubDomains; preload vary: diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_evaluator_runner.send_score_metric.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_evaluator_runner.send_score_metric.yaml index e2e17e715cf..f5deea8ef90 100644 --- a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_evaluator_runner.send_score_metric.yaml +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_evaluator_runner.send_score_metric.yaml @@ -1,28 +1,28 @@ interactions: - request: - body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"span_id": - "123", "trace_id": "1234", "label": "dummy", "metric_type": "score", "timestamp_ms": - 1729569649880, "score_value": 1.0, "ml_app": "unnamed-ml-app", "tags": ["ddtrace.version:2.15.0.dev219+ge047e25bb.d20241022", - "ml_app:unnamed-ml-app"]}]}}}' + body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"join_on": + {"span": {"span_id": "123", "trace_id": "1234"}}, "label": "dummy", "metric_type": + "score", "timestamp_ms": 1732569321978, "score_value": 1.0, "ml_app": "unnamed-ml-app", + "tags": ["ddtrace.version:2.15.0.dev351+g152f3e3b6.d20241122", "ml_app:unnamed-ml-app"]}]}}}' headers: Content-Type: - application/json DD-API-KEY: - XXXXXX method: POST - uri: https://api.datad0g.com/api/intake/llm-obs/v1/eval-metric + uri: https://api.datad0g.com/api/intake/llm-obs/v2/eval-metric response: body: - string: '{"data":{"id":"2131dbc0-d085-401c-8b2d-8506a9ac8c13","type":"evaluation_metric","attributes":{"metrics":[{"id":"YutAyQc6F4","trace_id":"1234","span_id":"123","timestamp_ms":1729569649880,"ml_app":"unnamed-ml-app","metric_type":"score","label":"dummy","score_value":1,"tags":["ddtrace.version:2.15.0.dev219+ge047e25bb.d20241022","ml_app:unnamed-ml-app"]}]}}}' + string: '{"data":{"id":"06c00db0-1898-44be-ae0b-f0149f819c59","type":"evaluation_metric","attributes":{"metrics":[{"id":"1DrSMXmWcP","join_on":{"span":{"trace_id":"1234","span_id":"123"}},"timestamp_ms":1732569321978,"ml_app":"unnamed-ml-app","metric_type":"score","label":"dummy","score_value":1,"tags":["ddtrace.version:2.15.0.dev351+g152f3e3b6.d20241122","ml_app:unnamed-ml-app"]}]}}}' headers: content-length: - - '357' + - '378' content-security-policy: - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pub293163a918901030b79492fe1ab424cf&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatad0g.com content-type: - application/vnd.api+json date: - - Tue, 22 Oct 2024 04:00:50 GMT + - Mon, 25 Nov 2024 21:15:22 GMT strict-transport-security: - max-age=31536000; includeSubDomains; preload vary: diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.answer_relevancy_inference.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.answer_relevancy_inference.yaml new file mode 100644 index 00000000000..1f537a977b8 --- /dev/null +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.answer_relevancy_inference.yaml @@ -0,0 +1,557 @@ +interactions: +- request: + body: '{"messages": [{"content": "Generate a question for the given answer and + Identify if answer is noncommittal. Give noncommittal as 1 if the answer is + noncommittal and 0 if the answer is committal. A noncommittal answer is one + that is evasive, vague, or ambiguous. For example, \"I don''t know\" or \"I''m + not sure\" are noncommittal answers\n\nThe output should be a well-formatted + JSON instance that conforms to the JSON schema below.\n\nAs an example, for + the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": + \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, + \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted + instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} + is not well-formatted.\n\nHere is the output JSON schema:\n```\n{\"type\": \"object\", + \"properties\": {\"question\": {\"title\": \"Question\", \"type\": \"string\"}, + \"noncommittal\": {\"title\": \"Noncommittal\", \"type\": \"integer\"}}, \"required\": + [\"question\", \"noncommittal\"]}\n```\n\nDo not return any preamble or explanations, + return only a pure JSON string surrounded by triple backticks (```).\n\nExamples:\n\nanswer: + \"Albert Einstein was born in Germany.\"\ncontext: \"Albert Einstein was a German-born + theoretical physicist who is widely held to be one of the greatest and most + influential scientists of all time\"\noutput: ```{\"question\": \"Where was + Albert Einstein born?\", \"noncommittal\": 0}```\n\nanswer: \"It can change + its skin color based on the temperature of its environment.\"\ncontext: \"A + recent scientific study has discovered a new species of frog in the Amazon rainforest + that has the unique ability to change its skin color based on the temperature + of its environment.\"\noutput: ```{\"question\": \"What unique ability does + the newly discovered species of frog have?\", \"noncommittal\": 0}```\n\nanswer: + \"Everest\"\ncontext: \"The tallest mountain on Earth, measured from sea level, + is a renowned peak located in the Himalayas.\"\noutput: ```{\"question\": \"What + is the tallest mountain on Earth?\", \"noncommittal\": 0}```\n\nanswer: \"I + don''t know about the groundbreaking feature of the smartphone invented in + 2023 as am unaware of information beyond 2022. \"\ncontext: \"In 2023, a groundbreaking + invention was announced: a smartphone with a battery life of one month, revolutionizing + the way people use mobile technology.\"\noutput: ```{\"question\": \"What was + the groundbreaking feature of the smartphone invented in 2023?\", \"noncommittal\": + 1}```\n\nYour actual task:\n\nanswer: \"The capital of France is Paris\"\ncontext: + \"The capital of France is Paris.\"\noutput: \n", "role": "user"}], "model": + "gpt-4o-mini", "n": 3, "stream": false, "temperature": 0.3}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '2795' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//7FTBjpswEL3zFdacQ0VoNmS5VJWyvfTWSttDqcAxA7hrbK89rLaN8u+V + CRuItpV6by8c5s17vHn2+BgxBrKGnIHoOIneqvi9/bjf3iVPSI/74ZP87H/e7++e7x8ytc+2sAoM + c/iOgl5Yb4TprUKSRp9h4ZATBtV19nab3d5ku2QEelOjCrTWUrwxcS+1jNMk3cRJFq93E7szUqCH + nH2NGGPsOH6DT13jM+Rs1BorPXrPW4T80sQYOKNCBbj30hPXBKsZFEYT6tF6VVXHAh4H9MF5ATkr + 4EvHiUnPqEMmuJXEFTMN++C4FviugBUrQBstTN9LIq4CKzlVVbX8h8Nm8DzMqQelpvrpYlqZ1jpz + 8BN+qTdSS9+VDrk3Ohj0ZCxEC/KrJNb/k5iSSP+9JCLGvo0LM1zNC9aZ3lJJ5gF1ENym08LAvKcL + 9HYCyRBXi/ruBbjSK2skLpVfxAuCiw7rmTrvJx9qaRbA8ghfu/md9nlyqdu/kZ8BIdAS1qV1WEtx + PfHc5jA8Y39qu6Q8GgaP7kkKLEmiCydRY8MHdb5I4H94wr5spG7RWSfPt6mxZZZinfHDzUZAdIp+ + AQAA//8DAHIOLnJvBQAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 9017b85e9bb772c2-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Mon, 13 Jan 2025 19:16:21 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=jNCWyg.Vq7.UysipP_0ZTDahpo.QhHWQvZ5Biaue6Bs-1736795781-1.0.1.1-C.GEfp7jlmfkY9qIXtsRjf9L9W5MzQ2OSXUpBOB0jIjYNSrJBlVnNwuHbaPYKT9.DjEjgPPIK69hkYhC0UtQZA; + path=/; expires=Mon, 13-Jan-25 19:46:21 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=g4nQujMCF8pE0TfAWPiY5rkdnbhtDE1kP7w0dFQq.v4-1736795781566-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '541' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149999320' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_b5c84c23586be35a5fc602178e84a812 + status: + code: 200 + message: OK +- request: + body: '{"input": [[3923, 374, 279, 6864, 315, 9822, 30]], "model": "text-embedding-ada-002", + "encoding_format": "base64"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '114' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA1R62a6zurbm/X6KpXWb2gpd8GDdESCQ0NhJIF2pVAKSECCE0NiAj867l8h/dKrq + ZkqT6QkGD3/d8H/866+//q6T4pH2f//z19/vvOv//h/ztXvcx3//89f//Ndff/3113/8fv5/Ix9V + 8rjf80/2G/77Y/65P8a///lL+O8r/3fQfO/usyW6cQiS8fqNqTqZDmFB9CzQZNaOAvxD18wig+yx + MtBK1X4rAQvr8BNO1X4QQBKbG1uX1xwNXnmJIVDSKxUgH/kkaS4FIigf4ir9J+GqqVbI3199XDcX + uajvtNRBWz01LMjftzeRozeg9egZzOXXtJjug5tDdO1dshW/q7BfSw2F69jrDBfHVzIF8SNCmVc/ + mL4LcsR72gpoO778eT4GF+8HyYLbsz6zLc6WbX8/SDZoi5YQm5Kl16XjykCR1Lzo8hI5RVfULAM1 + 2G7IzY8HNKqrAqB1rSs7ewprp+Pxe4TmuxswYGInw+pRZmgp1BzLrn3xuHURclAP+ZbgbKd4NIjP + Z7ADd0s2q3MecnzWDY0a4gMvF/d12Itn24fxmafEum1eaMhbIQMosxgP6XFop29oK7DgboPDe/nm + LTkiHRbZ4stcFB9bmpzUCC3Ec4ulnVeGVBTiWs1PTko2b/Hc8kXxluCwLUeC3fZk8hUyUgir20hc + +WO34+OUVdDot454R68xR6hfEaiK7WHqLb5FfT/YErqXR5eZPdTeoCaXXG1k9UIb59qF3f2NFogU + yyWehvclpO/ltVkthYYTV09lkwdsopqrZjqV8WntyYq/2cL0bY8UoQdGbDc1GDmP+kAOe10vpD3L + AGx1T5hX+0Eo58s8RocycpjFYuRNMOmNhjaWShfcKcKBrr0jSj+uybzrpQ07qoRbKMs6IoHSbc2h + iFcKxGKeM3N5jdCQfLcA/jhpZLMtP6hL754CR/oA4q2igvPL7WCjGj22xKlXZTH4bklhNEeHrkb3 + lPAKJkXL9T4mdo9q3pP7oQTZEnfMuMY7k+/umgVoVUVUk6tPMoivSIfs3QKe+pNXDBtf6lB/YhdC + FvdX+JsPOpRnh0o33U46NfiqoGp9hYXjrkvGg/wawNAvE7HwSi/kkzMA7CSjYuZRLBO2/FwjuCnX + hJGV+jK77NBF4B0si0QK26BJ2NwjEI6NQzbd2HlNRNI92iySFyO7rV0Mb9G1kJVkH7IzxTgsvFOe + L/Hm6VCl1HQkLfREQOI2Vsn6lQYhI0dkqJFd7EjwRkPLa6GOwPPpmhmXaJuMKdqfNdnHKV4mDm+n + ek0n+IaHN7HXHW0nenMewLRNSzxjtQ0nFLUpNF9voJNweJn9a31dgHBPBeK2730xkGVg/96fLpxk + F7JPoWNNJl+J6Dgswm67vrjgasuY6HJVmyOEgKERuzM5FMfcZLa9UqCJRkqHtpzMjjRPUKVjMeBX + ugq8QTkJDzgEg0n2R6kouHG5GMjZSyFxLwENK8/3MBDZrtl6FEjBPxdvCxcteJMgepqc2/aoqoHy + uBLjs2vCaeKiC+XKOOM6887tdHZ2HcpUc4EXix6F7QoZD8iEa8G274Ns1h9yPGt1qWyIrTiNyR26 + pKvSUwgzdlsnZGm9pkAVd0vco9Nzattuoy4e04f9vudwkKYc+lfas7un9uFUSlkFl4PusHsqDt7o + R76urks3JM7tY6DJ3Ly30F0/e+Ipa5IMd9oZiDlnhVlRvi6kgcY24CEJ6FJaeO34ukg1Mn3/Sfzd + 7o24WcsuCrreINaFdHzisftAy4fNiHO6Knys9WUGzW2pM33zunnTXntUEOXHJ1vbyo7TzAkG9BGm + ngVy5YSC2PiWGu6HM0mXyiX5LIq38MMnKqtbLfxIX0sCpjktIWRzLwZwIxWg39nM80jUStWjSuGj + 3W5kw921KT3WOUWy76fMJ9LB5O1XH7R5PkR3P6t2mBYhVd/de8vcgpTJsGkSFZ6ED5h3Nk+4/zEa + 4EZyYetndU2+mEYYRAoaO+jPiPfyxiyRmkYi2ZzuUTLgqyxAYvVHit5c8frLRa1QZwWYal8KRXXK + IIagtM94WHyffDSavoEvrL+41eTSm97La41mvCdOMvacviTTBvuQe+yHRxRUhYK+8gO2Nk67gq+m + +Lq6hZuR2A/b5h13TAmdxf3IrHD58oa5vlGhah+2U/enGe90QMNbHIgrPO4JP6yHK0o/W5NE+Jwl + o/42qj//T8SthNg3lFxYVaXBnO+AvL4kpQWumuvETOie88zRDJj5izj7oxLyASSANIl7YvSntpi2 + WSyo5OYdGF59YjQ9h9oADpZJ9u7nVvSP4uErvPUSKq7rYzI879/rSr4FHhZeCTVHSLs9qOrRp+Vr + gUP2TJstXCXlSMILsvjI+UYFGk42nfdTMpk1UVD6WoyY73yzEOOX7v7Gs3BPUDJMi4SixhmWbK5/ + 3rqWe0XZIVTx9OncduyH+KzW93zEKrfsZGrwOtLm/Ukcq8vC0ZJ1DDSNJ7J5fsWEBUztQKQLjZjT + s04YOXIdSJa9mad6cTJdrl8BUhgfdNVZEZomrrmoR/p6Xt+A860fATwmbhD9bJOQS+S4Rdp9/2B4 + eKlhP6UZRjN/Mt0WgnaQK9lCuegLWFQDBY3hYTK0UhJOzCovZjH9+LIS0zddbt9VMp605gy9q36Z + +VYnj7pqq4OxciT6JQ54Y/Aut3/W/6C6iPft2XWhX9wTtl6Fm5BXm8KAnKQ6Od6zb9uJixVW90zq + mDPlm2S0gh5UiJcbFsRpzKdpqWL1JSodHoZm4j1b9A91o4LDHm9ybSksF0d43k4bcp+yindusKLI + P1YB0+fx7My9CcS4RMRrvz7njtkpqPArh+DmdDOnS/VM0WN4r5m/bgaTf3BoaX2zuWExY2dzEnI1 + h9Pic2KbjJ29SX+vBM0n94I4eV8kfb7wFHgvxh378X2PvTH94dWMTyuTyW9ZAIo+OV6N6redmqG0 + EVvVB0KWkcwHS7oC0m1FwEhCuJUFDA0CiSZ4afsHs/vhJyVXE8tYWYfSimxj+ICrE3LEjin/7u+9 + HzYe01Xv8X1S2LAynzcKKebtRB66BMaFbSg/b/Ymm/EfJe3hROwqDU2mRBpe7fBLxSvl6/3RUyAN + T+cPf0vJST2jmY+Zva0tc3hnESC1oDuq7OKl2Vd3GUMXJgu2uYQ7ky7yRYnE0+1LgtM5Tj5ttqew + xnLNtt6eI66xRoWL31jM0kQ7mVQ1xDCowURlwbMT+dMqe7hR2aYvWhZoSCjO1PvR3hEcFbonGYK1 + AH3d58SxpTZkD+O2gMXWL3/43fapfloAkx/7mS/dhF2bEcOsD9nNXSft6LRZo23Kq0XuZrk2pdvD + V9BBoTvmXxTHHGtdzkB9Ky6GT/81OdN9A/pTf2Hrm702aSllJUh41xMrMFvEhzjFgBFRyWaRGmhi + 0VqFX/2706sy+ft57xA+3iViuqbqDT89+7X3EsGz/h12DziDcqdvWijLJ6f1upqAkWWJl69QQH2K + 9pGGfSkmjmikiAfJpgLho+9YPL1sc/KyIdLyQ2OwgAp3czzkUKtG1Ats565R24W8z1RZ3uZk1qec + B0o+we5+fLFgF+TelCh3+/e+FLldFA7iYsTa092esOQngzeeDAZoutwPDIM+eNOq5ramaqxivn8b + i6nolQztPDuj8OPDqK8pDHl8xiLv14nEllEHStNHbE3Lgg/S/nCF5aexKA+vVjFCWu7RXB9ke2Tf + oqduXmpO/WVU/qYW4hWoCjoOzYKOVR6g6XJ9CQiWF4f4EdP4GFNt+NUXrt8H2aMyzlXYmSwg7kUz + 2p8+VvPv88Hs82f0Jns8PWAtRRGz3Obcsvchs9G8n5i3P50Stu6sLeDjU6K587TD7iR4OqoXKCd+ + EwvtRCpFgR/fbe+P9re+FKTOsSjM+oUJ3UFSYSx2xNIfYjuZhxyD4mUl01/XgznzcQNqvBWIJ2Az + /P2O/FCsCDb958yXbqnScLCZs312Cdc3kor4bmiIP66akF/ytatlOc6JX66X7bgLaf7DF3pByQGN + 1++xg+z89HE58iGcDliaYPmpLaa/0SHpnx+9g3Pbp4RIi7ad9aWi5aFk4iF6lu0An8nW0p3fsV/9 + jmc7fiB6KDfMlN81H6wrH+BZv0SGT9oZsXCqjrA/H65Y+N4tUzCdsIHn7bJhO2NRoK8V9Atgfu4y + 67jzw17bfzDQ98KhXf6pi9eNZIb2yauOEOGgJ4JZKhJgsXoS91AG7XSO5AjG+zn55QV8WLt1rbru + tcalLTctj1dWBJ6TMSxeb18+WfiEwYdPw5zyXCajvPFKNNcb8RLB5aOVhzY4/mNPJ8XqPTbq3xwg + fE/ErEMn6X56zZkCi+2WRuZxS70coe1bl1lhqvP+KUgR2DRf4aftH7wRy/4RDZ50JMGhwOH0coc9 + zH6GFju/KLhE4i1Y0qOf13+c/cKo/PASa8aha0dxey0BPRqM5bJR227jLzrYbZUnlhTHNen3AwYa + 7FYnJA2icKybwwBFD8qvPkIuZumgktTNmTNOB69/dcUEr8vyRHT/TovvrL9U5/ZYEt/JLCQhD10h + PQ5rEv/8eVTEW4D74sx8zeu8qXWFMwjJkeOBXxbetBHyB4xHTWbWofK9sRRXZ/TzM/aHOcUAH9UG + vU9lrJRaxsdfHtJMbMMs+bvxaPrIr0g/OyWz62Pt0R9/zX533m9iwrVA32pz/kOHXWAgoTHVCah6 + 3lH5FVM0deOuRHJWPpn57R/mcCn0q5ZZoUd2yrf1OiPVJ3hx2pKAq1oxblrTheB0YmT7PlzMge3Q + +ee/sZxGOOGRhR8w5ynM7Rdv3j8f+wY+qVETXbg/+FTzlY0OBCyW2DvBG/GN2Wjej3TqJi+UZr0A + wijJbBf073CclgbA/VQU9JentHHCtuhWGjdmH15Dy81iVan7D9aZkQYnr5StNILiHflszg+8Yb9/ + H5FUrSZGivsTDUa4p4hegoz98pUhkvc6xLfRZ2czN5PJ30o+0rZq+ke/98jzXTjxqSbePF/uf9wG + wuOa0ekS1UnTlMxYbXTDIORxWYa/74+qlGbEQw/MqU5fqha6/pGYh5yh4apdK6QwpWLnh6aF03lK + M6jP3oPtiJN6Ux5m5z/7ybfMTTjtxgl+zyMVfideb6SDASTvKnLYRxLqBWm4auBNp59eCtnX1nNg + maWx3WPjtHxdRJaq+KPGdhZKTN56baWW2N4zz1jVyR+/O+drBOtPwRyuF++MZv1HZj1m0tn/o2ZT + LoivTmc++5cY0cA32HaRmIV494cMpOJxZYZxe/OfPljtz+H1D58OperUq1Y0HsxaVWk4xHgY4FxZ + a5LwbV5M+xhJSHraJ7azR8bH45PVaBnuDdrOfDHeJndCUfhSmGGAbcoZJxKU9YYS57JYc+k+GJmG + VLEnbpQVHg+SoIR0dbaIJ25uqBduCYa1kp3YmiqqOeXpNYZQ5Zx4UJJicq/6XnuScWDnZhmGTTeu + S0itdUyp3JhFPShD/cefWupq4N9N67molKQTXdCyMCdjq25V6fgayCZyl3xa32BYzXkDFY9B3bJT + k3QIbWwVPx+XZzImfqhDHgomHo5S0fLFafDR+01EvErJEU276DTB7D+ZmTdVSNPPMUUnPwQqaGIV + jqOnWwi84TTzd4lm/Smgnx7DylAmza2CBVx2Nad8etZhc0F+CXL/CahqvvVkADdVoWNQMGOlvlBf + OYdcrSq2IIbLJo8txQDQzrMyOqmjEtLIfGJI2vCERfrJ0LD6pNUPb7Hs3Lxi/OU9j6I8stCyTXPc + V+sGvp/28wePR398+QAfTcCLN7kWw2VV2aB+TyXZpshFP/6FwrmKs/6XTP6r35N/AGZadmG23ahI + 6EZFm2bu59ZOPtL/5Lu4TsTGaz4WmmBYJtasf0wkf9phDwdVcdn+sUz4hDwUr7YSfbA5X/Ck+XvC + 1puuZNNHXTI2QWyjk6cX5GaKccLybZwjQz9NeJrzMPUcLSOY8YbY+fdj9tV+V8KKYUq2p+Up5PZk + UQiX2ob5QyXysU96/4cnP7+IeG/edHXDS+EPvrYNHiqYXkZFtt4+5EN7ihXIK8MiplkYrSxfXR+G + q7gm61N8SfhGPFA4TtKSCh+hMMflssuRtHmtGaZl4fHi4FY/viVWtl0V0/K+MtBRazs8rGnvtWTY + Y4DwMzHTX5Ji6GMNYPdqSmJ1Pfnjb8F14/qnr/hYN7cBZv1HF20dFNOqRjaSDvsLuSW7ozdMmZSp + b8+/MEcTPuYvf4doeQjwoOzqYnpJngWOn+6ZIRxeHteC7RZumFFi3tbv4nub3AHN+Qlztb3TDsZX + itHmbgxso+1wyGvtGqveuX8yf7nOeDPsQQVciQrbGKji32NbUnDT6svcm2kmLe7eHcz4y3amqIaD + GrxU6M43guUqI8m4fbYlisJCweN4YOFYlAcM9eroUJ4exoTepsnQNEkviLUpJY+yhpWwxmJN3Ju6 + Ql14UwVIjOLAtv5ZaEdOt5GS7nBHtehpIumGGgvNeS+JZ74b/GG1QMGKjdh6Glf+yw8BvfGRLmc/ + 2q9cWoN0YQaxb3qVTPPz0CXzZYLnPGQoiy+AHJkrgs+9Z4rCLfShkZULVYWHFvbOQDHo80yFC+mQ + vPxczwD22ySmbNTFuKdjB/V6TEkadLY5ZvYjA7jDmZinU8L7U7bZ/vIcdrucbXNizadEkZBmZK0M + G3MULGqhGxJF9vNf7+s37kCNtJwu5jyIo1Pcodk/sY0aXPn3l9ekq8hiIdnF7ZDeTQUkWYiZ03X7 + lmvHDdWW02PCe8vJk3Hsc+EPHox8obacRqMLiy0uCTmvuTeJwrEBub359PHwdS4WtmBov/6CQcRj + 8TX36kL96enF5h0V/Lz/GjCa3JnzrzMabSiOEL7d66z34mIKqiRGw1seqDAVZdHoB5+iZNle6DTz + CSefevvLn+f+k4EmJfN02CePZq4XklDk8RguGnnP/atb8sdfzHiCFcIjr3Py9RmqdrVn6VCJiN9L + oOjZM4f88pgpXyBXLfuDzjaNpbd8rlckk1aiy5kPvkPqdnBxHtPcz6nasezXexQ6ZM2M5nY2R3FZ + pyD5SvTL39rx6ok5XFaDRx7fh1FIeWMb6Gwsr1gLdGp2IX/nYGcdop2hla34UMwIrHtXM2PW4xNM + 2xp839ixTWg9zcmfrim6RPcX2T02n6KLL5n0618Qa+5fdGryzOB8OcVse726RacqQgfvYpESN1j3 + 4RCn1RWSLK2ZJ11p0pFlYMFRLyqc5Lrvydz2S2B7NaDyzcqLcd2rzQ9PmbVcjiFF32sFM96yzeps + hPJmsZ5++TxV3J63060CgDE+uWTjPD7mSN2mgvjy8dgvvxdyoazgbp/5f/k9nb4Udc1PJubye8vF + 5Dv4YFm7iHnLg8p/eh42zfrJgt2QJ5NcXA1IUbZh21EKwqkLyxjNeRPVrGbNpa0h1nDB0oaQw+PI + x/ah+pAK7sTW4T3zJi9Ce7Ukp+/8fQNPVjL1CMWKYqwutSMfjHQ7gN48KzLXF+8Eab1F2+XJZT9/ + SX/8fKZdzwz/Gno8HcZJ0ySjIKYqRgW/weEMefpgzMdF2g5Xzz5qanoWiXmuh5DZk9+huT+J5Tnf + 43mj7ZErRpTM+iHpnNvtiAJd63E989VEe6EC1EsL5qYSS4bzpsVgq0fCdmfzUPCkJ3skX9YuO+z1 + rOgzpRbgHd8cZm3KsylfmxVG83qTzRR3nG++aQMvUe3+9HO50N0k+IbhG6ufqSyGrLqliOS0IltN + KcJuBesOZn1I37vFo+0sab/Qon7jk91bCYuRtA4GQj4w8xPj3f2ABfRbz+/MP115KRW4JIJATGW7 + S3hvHnTNiQ495pZ2K9jpVJXoW8UYo7XJC77TLwqc3ajFE611T0qP7zO00zqY+ZdzbhxvKcz3p6K2 + MlrR80KAm3eMmOPasjnOeAOHYDLJr98z+7UUAQkYRXx5M9kznGwt/94fdIi4yofzITsi4f4QsJYe + CZpefWige9yJdPlsOBpzMZHAIajBYuQ+0bgLq3xFacWZY4StSdNjH8ExsL0/7zPi6JsjQ5bucx50 + aFkVX7cw62UMs974ujhbwN+/UwH/+a+//vpfvxMGVX1/vOeDAf1j7P/930cF/h3f438LgvTnGALt + 4uzx9z//dQLh729bV9/+f/d1+fh0f//zF/pz1ODvvu7j9/9z+V/zg/7zX/8HAAD//wMAiTuukd4g + AAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 9017b864b936de9b-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Mon, 13 Jan 2025 19:16:22 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=s1ISh4iUNLQ2Ya1kD2H7qoQ8Iu5E35cNV7LByFeP6S4-1736795782-1.0.1.1-Yiw0sA3z8E8DIIhiXr_yjzMDz_ePLu1gwWVxoqFOBJDcDhE91jZRgy2Kp6FYdcBcS950vyAqL1y9LPmOMIGJxA; + path=/; expires=Mon, 13-Jan-25 19:46:22 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=beMBNvOtq78fFCvDki.8wgKumD_EQhiZMEDa05JaT7s-1736795782543-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - text-embedding-ada-002 + openai-organization: + - datadog-staging + openai-processing-ms: + - '80' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + via: + - envoy-router-678875496d-6fjmx + x-envoy-upstream-service-time: + - '59' + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '10000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '9999993' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_e589af7ff963668ef0b2722e99fcde70 + status: + code: 200 + message: OK +- request: + body: '{"input": [[3923, 374, 279, 6864, 315, 9822, 30], [3923, 374, 279, 6864, + 315, 9822, 30], [3923, 374, 279, 6864, 315, 9822, 30]], "model": "text-embedding-ada-002", + "encoding_format": "base64"}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '192' + content-type: + - application/json + cookie: + - __cf_bm=s1ISh4iUNLQ2Ya1kD2H7qoQ8Iu5E35cNV7LByFeP6S4-1736795782-1.0.1.1-Yiw0sA3z8E8DIIhiXr_yjzMDz_ePLu1gwWVxoqFOBJDcDhE91jZRgy2Kp6FYdcBcS950vyAqL1y9LPmOMIGJxA; + _cfuvid=beMBNvOtq78fFCvDki.8wgKumD_EQhiZMEDa05JaT7s-1736795782543-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/embeddings + response: + body: + string: !!binary | + H4sIAAAAAAAAA+x6SxOyOpfuvH/Frj21q+QmWXwzBATlkqggao8EFQERuSRAuvq/d+HbdU6d6Rn1 + YE+sUiMGsvLcVv7z3/766+86KR5p//e//vr7nXf93/8+f3a/9be///XXf/zbX3/99dd//l7/n5GP + Knnc7/kn+w3/fZl/7o/x73/9JfyfT/7voPna3WdLdOMQJOPle6PqZDqEBdGzQJNZOwrwD10ziwyy + x8pAK1X7rQQsrMNPOFX7QQBJbK5sXV5yNHjl+QaBkl6oAPnIJ0lzKRBB+RBX6T8JV021Qv7+4uO6 + OctFfaelDtrqqWFB/r69iRy9Aa1Hz2Auv6TFdB/cHKJL75Kt+F2F/VpqKFzGXme4OL6SKbg9IpR5 + 9YPpuyBHvKetgLbjy5/nY3DxfpAsuD7rmG1xtmz7+0GyQVu0hNiULL0uHVcGiqTmRZfnyCm6omYZ + qMF2Q67+bUCjuioAWte6sNhTWDsdj98jNN/dgAETOxlWjzJDS6HmWHbts8ets5CDesi3BGc7xaPB + LY7BDtwt2aziPOQ41g2NGuIDLxf3ddiLse3D+MxTYl03LzTkrZABlNkND+lxaKdvaCuw4G6Dw3v5 + 5i05Ih0W2eLLXHQ7tjQ5qRFaiHGLpZ1XhlQUbrWan5yUbN5i3PJF8ZbgsC1Hgt32ZPIVMlIIq+tI + XPljt+PjlFXQ6NeOeEevMUeoXxGoiu1h6i2+RX0/2BK6l0eXmT3U3qAm51xtZPVMG+fShd39jRaI + FMslnob3OaTv5aVZLYWGE1dPZZMHbKKaq2Y6lfFp7cmKv9nC9G2PFKEHRmw3NRg5j/pADntdL6Q9 + ywBsdU+YV/tBKOfL/IYOZeQwi92QN8GkNxraWCpdcKcIB7r2jij9uCbzLuc27KgSbqEs64gESrc1 + h+K2UuAm5jkzl5cIDcl3C+CPk0Y22/KDuvTuKXCkDyDeKio4P18PNqrRY0ucelUWg++WFEZzdOhq + dE8Jr2BStFzvb8TuUc17cj+UIFvijhmX287ku7tmAVpVEdXk6pMM4ivSIXu3gKf+5BXDxpc61J/Y + mZDF/RX+5oMOZexQ6arbSacGXxVUra+wcNx1yXiQXwMY+nkiFl7phXxyBoCdZFTMPIplwpafSwRX + 5ZIwslJfZpcdugi8g2WRSGEbNAmbewTCsXHIphs7r4lIukebRfJiZLe1i+EtuhaykuxDdqZ4Cwvv + lOdLvHk6VCk1HUkLPRGQuL2pZP1Kg5CRIzLUyC52JHijoeW1UEfg+XTNjHO0TcYU7WNN9nGKl4nD + 26le0wm+4eFN7HVH24lenQcwbdMSz1htwwlFbQrN1xvoJBxeZv9aXxYg3FOBuO17XwxkGdi/+6cL + J9mF7FPoWJPJVyI6Douw267PLrja8kZ0uarNEULA0IhdTA7FMTeZba8UaKKR0qEtJ7MjzRNU6VgM + +JWuAm9QTsIDDsFgkv1RKgpunM8GcvZSSNxzQMPK8z0MRLZrth4FUvDP2dvCWQveJIieJue2Papq + oDwuxPjsmnCauOhCuTJiXGde3E6xs+tQppoLvFj0KGxXyHhAJlwKtn0fZLP+kGOs1aWyIbbiNCZ3 + 6JKuSk8hzNhtnZCl9ZoCVdwtcY9Oz6ltu426eEwf9nuew0Gacuhfac/untqHUyllFZwPusPuqTh4 + ox/5urou3ZA414+BJnPz3kJ3+eyJp6xJMtxpZyDmxAqzonxdSAO92YCHJKBLaeG14+ss1cj0/Sfx + d7s34mYtuyjoeoNYZ9Lxid/cB1o+bEac00XhY60vM2iuS53pm9fVm/bao4IoPz7Z2lZ2nGZOMKCP + MPUskCsnFMTGt9RwP8QkXSrn5LMo3sIPn6isbrXwI30tCZjmtISQzb0YwI1UgH5nM88jUStVjyqF + j3a9kg1316b0WOcUyb6fMp9IB5O3X33Q5vkQ3f2s2mFahFR9d+8tcwtSJsOmSVR4Ej5g3tk84f7H + aIAbyZmtn9Ul+WIaYRApaOygPyPeyxuzRGoaiWRzukfJgC+yAInVHyl6c8Xrz2e1Qp0VYKp9KRTV + KYMbBKUd42HxffLRaPoGvrD+4laTS296Ly81mvGeOMnYc/qSTBvsQ+6xHx5RUBUK+soP2No47Qq+ + mm6X1TXcjMR+2DbvuGNKKBb3I7PC5csb5vpGhap92E7dn2a80wENb3EgrvC4J/ywHi4o/WxNEuE4 + S0b9bVR/fk/ErYTYN5RcWFWlwZzvgLy+JKUFrprrxEzonvPM0QyY+Ys4+6MS8gEkgDS59cToT20x + bbOboJKrd2B49bmh6TnUBnCwTLJ3P9eifxQPX+Gtl1BxXR+T4Xn/XlbyNfCw8EqoOULa7UFVjz4t + XwscsmfabOEiKUcSnpHFR843KtBwsum8n5LJrImC0tdixHznm4V4e+nubzwL9wQlw7RIKGqcYcnm + +ueta7kXlB1CFU+fzm3HfrjFan3PR6xyy06mBq8jbd6fxLG6LBwtWcdA09tENs+vmLCAqR2IdKER + c3rWCSNHrgPJsjfzVO+WTOfLV4AUxgdddVaEpolrLuqRvp7XN+B860cAj4kbRI9tEnKJHLdIu+8f + DA8vNeynNMNo5k+m20LQDnIlWygXfQGLaqCgMTxMhlZKwolZ5dksph9fVmL6psvtu0rGk9bE0Lvq + l5lvdfKoq7Y6GCtHol/igDcG73L7Z/0Pqot438auC/3inrD1KtyEvNoUBuQk1cnxnn3bTlyssLpn + UsecKd8koxX0oMJtuWHBLb3xaVqqWH2JSoeHoZl4zxb9Q92o4LDHm1xaCsvFEZ7X04bcp6zinRus + KPKPVcD0eTyLuTeBeCsR8dqvz7ljdgoq/MohuDldzelcPVP0GN5r5q+bweQfHFpa32yuWMxYbE5C + ruZwWnxObJOx2Jv090rQfHIviJP3RdLnC0+B92LcsR/f99gb0x9ezfi0Mpn8lgWg6JPj1ah+26kZ + ShuxVX0gZBnJfLCkCyDdVgSMJIRbWcDQIJBogpe2fzC7H35ScjGxjJV1KK3I9gYfcHVCjtgx5d/1 + vffDxmO66j2+TwobVubzSiHFvJ3IQ5fAOLMN5fFmb7IZ/1HSHk7ErtLQZEqk4dUOv1S8Ur7eHz0F + 0vB0/vC3lJzUGM18zOxtbZnDO4sAqQXdUWV3W5p9dZcxdGGyYJtzuDPpIl+USDxdvyQ4xbfk02Z7 + Cmss12zr7TniGmtUOPuNxSxNtJNJVUMMgxpMVBY8O5E/rbKHK5Vt+qJlgYaE4ky9H+0dwVGhe5Ih + WAvQ131OHFtqQ/YwrgtYbP3yh99tn+qnBTD5sZ/50k3YpRkxzPqQXd110o5OmzXaprxY5G6Wa1O6 + PnwFHRS6Y/5Zccyx1uUM1LfiYvj0X5Mz3TegP/Vntr7aa5OWUlaChHc9sQKzRXy4pRgwIirZLFID + TSxaq/Crf3d6VSZ/P+8dwse7REzXVL3hp2e/9l4ieNa/w+4BMSh3+qaFsnxyWq+rCRhZlnj5CgXU + p2gfadiXbsQRjRTxINlUIHz0HbtNL9ucvGyItPzQGCygwt0cDznUqhH1Atu5a9R2Ie8zVZa3OZn1 + KeeBkk+wux9fLNgFuTclyt3+3S9FbheFg7gYsfZ0tycs+cngjSeDAZrO9wPDoA/etKq5rakaq5jv + X8diKnolQzvPzij8+DDqawpDfouxyPt1IrFl1IHS9BFb07Lgg7Q/XGD5aSzKw4tVjJCWezTXB9ke + 2bfoqZuXmlN/GZW/qYV4BaqCjkOzoGOVB2g6X14CguXZIX7END7eqDb86gvX74PsURnnKuxMFhD3 + rBntTx+r+ff5YHb8Gb3JHk8PWEtRxCy3iVv2PmQ2mvcT8/anU8LWnbUFfHxKNHeedtidBE9H9QLl + xG9uQjuRSlHgx3fb+6P9rS8FqXMsCrN+YUJ3kFQYix2x9IfYTuYhx6B4Wcn01+VgznzcgHrbCsQT + sBn+3iM/FCuCTf8586VbqjQcbOZsn13C9Y2kIr4bGuKPqybk53ztalmOc+KX62U77kKa//CFnlFy + QOPle+wgi58+Lkc+hNMBSxMsP7XF9Dc6JP3zo3cQt31KiLRo21lfKloeSiYeomfZDvCZbC3d+R37 + 1e8Y27cHoodyw0z5XfPBuvABnvVLZPikxYiFU3WEfXy4YOF7t0zBdMIGntfzhu2MRYG+VtAvgPm5 + y6zjzg97bf/BQN8Lh3b5py5eV5IZ2ievOkKEg54IZqlIgMXqSdxDGbRTHMkRjPc4+eUFfFi7da26 + 7qXGpS03Lb+trAg8J2NYvFy/fLLwCYMPn4Y5ZVwmo7zxSjTXG/ESweWjlYc2OP5jTyfF6j026t8c + IHxPxKxDJ+l+es2ZAovtlkbmcUs9H6HtW5dZYarz/ilIEdg0X+Gn7R+8Ecv+EQ2edCTBocDh9HKH + Pcx+hhY7vyi4RG5bsKRHP6//OPuFUfnhJdaMQ9eO4vZSAno0GMtlo7bdxl90sNsqTywpjmvS7wcM + NNitTkgaROFYN4cBih6UX32EXMzSQSWpmzNnnA5e/+qKCV7n5Yno/p0W31l/qc71sSS+k1lIQh66 + QHoc1uT28+dRcdsC3Bcx8zWv86bWFWIQkiPHAz8vvGkj5A8Yj5rMrEPle2MprmL08zP2hznFAB/V + Br1PZayUWsbHXx7STGzDLPm78Wj6yC9Ij52S2fWx9uiPv2a/O+83MeFaoG+1Of+hwy4wkNCY6gRU + jXdUft0omrpxVyI5K5/M/PYPczgX+kXLrNAjO+Xbep2R6hO8OG1JwFWtGDet6UJwOjGyfR/O5sB2 + KP75byynEU54ZOEHzHkKc/vFm/fPx76BT2rURBfuDz7VfGWjAwGLJfZO8EZ8ZTaa9yOduskLpVkv + gDBKMtsF/Tscp6UBcD8VBf3lKe0tYVt0LY0rsw+voeVmsarU/QfrzEiDk1fKVhpB8Y58NucH3rDf + v49IqlYTI8X9iQYj3FNEz0HGfvnKEMl7HW7X0WexmZvJ5G8lH2lbNf2j33vk+S6c+FQTb54v9z9u + A+Fxzeh0juqkaUpmrDa6YRDyOC/D3/NHVUoz4qEH5lSnL1ULXf9IzEPO0HDRLhVSmFKx+KFp4RRP + aQZ17D3YjjipN+VhFv/ZT75lbsJpN07w+z9S4Xfi9UY6GEDyriKHfSShXpCGiwbedPrppZB9bT0H + llka2z02TsvXRWSpij9qbGehxOSt11Zqie0984xVnfzxu3O+RrD+FMzhcvZiNOs/Musxk87+HzWb + ckF8dYr57F9uiAa+wbaLxCzEuz9kIBWPCzOM65v/9MFqH4eXP3w6lKpTr1rReDBrVaXhcMPDAHFl + rUnCt3kx7W9IQtLTPrGdPTI+Hp+sRstwb9B25ovxOrkTisKXwgwDbFPOOJGgrDeUOOfFmkv3wcg0 + pIo9caOs8HiQBCWkq9ginri5ol64JhjWSnZia6qo5pSnlxuEKufEg5IUk3vR99qTjAOLm2UYNt24 + LiG11jdK5cYs6kEZ6j/+1FJXA/9uWs9FpSSd6IKWhTkZW3WrSsfXQDaRu+TT+grDas4bqHgM6pad + mqRDaGOr+Pk4P5Mx8UMd8lAw8XCUipYvToOP3m8i4lVKjmjaRacJZv/JzLypQpp+jik6+SFQQROr + cBw93ULgDaeZv0s0608B/fQYVoYyaa4VLOC8qznl07MOmzPyS5D7T0BV860nA7ipCh2Dghkr9YX6 + yjnkalWxBTFcNnlsKQaAdp6V0UkdlZBG5hND0oYnLNJPhobVJ61+eItl5+oV4y/veRTlkYWWbZrj + vlo38P20nz94PPrjywf4aAJevMmlGM6rygb1eyrJNkUu+vEvFM5FnPW/ZPJf/Z78AzDTsguz7UZF + Qlcq2jRzP9d28pH+J9/FdSI2XvOx0ATDMrFm/WMi+dMOezioisv2j2XCJ+Sh22or0Qeb8wVPmp8n + bL3pQjZ91CVjE9xsdPL0glxN8ZawfHvLkaGfJjzNeZgaR8sIZrwhdv79mH2135WwYpiS7Wl5Crk9 + WRTCpbZh/lCJfOyT3v/hyc8vIt6bV13d8FL4g69tg4cKppdRka23D/nQnm4K5JVhEdMsjFaWL64P + w0Vck/Xpdk74RjxQOE7SkgofoTDH5bLLkbR5rRmmZeHx4uBWP74lVrZdFdPyvjLQUWs7PKxp77Vk + 2GOA8DMx01+SYuhvGsDu1ZTE6nryx9+C697qn77iY91cB5j1H120dVBMqxrZSDrsz+Sa7I7eMGVS + pr49/8wcTfiYv/wdouUhwIOyq4vpJXkWOH66Z4ZweHlcC7ZbuGJGiXldv4vvdXIHNOcnzNX2TjsY + X+mGNndjYBtth0Nea5eb6sX9k/nLdcabYQ8q4EpU2MZAFf8e25KCm1Zf5l5NM2lx9+5gxl+2M0U1 + HNTgpUIXXwmWq4wk4/bZligKCwWP44GFY1EeMNSro0N5ehgTep0mQ9MkvSDWppQ8yhpWwhqLNXGv + 6gp14VUVIDGKA9v6sdCOnG4jJd3hjmrR00TSFTUWmvNecpv5bvCH1QIFKzZi62lc+C8/BPTGR7qc + /Wi/cmkN0pkZxL7qVTLN/4fOmS8TPOchQ1l8AeTIXBEc954pCtfQh0ZWzlQVHlrYOwPFoM8zFc6k + Q/Lyc4kB7LdJTNmoi3FPxw7q9ZiSNOhsc8zsRwZwh5iYp1PC+1O22f7yHHY9x7Y5seZTokhIM7JW + ho05Cha10BWJIvv5r/fle+tAjbScLuY8iKPTrUOzf2IbNbjw7y+vSVeRxUKyu7VDejcVkGThxpyu + 27dcO26otpweE95bTp6MY58Lf/Bg5Au15TQaXVhscUlIvObeJArHBuT26tPHw9e5WNiCof36CwYR + j8XX3KsL9aenF5t3VPB4/zVgNLkz518xGm0ojhC+3cus927FFFTJDQ1veaDCVJRFox98ipJle6bT + zCecfOrtL3+e+08GmpTM02GfPJq5XkhCkcdvcNbIe+5fXZM//mLGE6wQHnmdk69jqNrVnqVDJSJ+ + L4GiZ88c8stjpnyBXLXsDzrbNJbe8rlekUxaiS5nPvgOqdvB2XlMcz+naseyX+9R6JA1M5prbI7i + sk5B8pXol7+148UTczivBo88vg+jkPLGNlBsLC9YC3RqdiF/52BnHaKdoZWt+FDMCKx7VzNj1uMT + TNsafN/YsU1oPc3Jny4pOkf3F9k9Np+iu50z6de/INbcv+jU5JlBfD7d2PZycYtOVYQO3sUiJW6w + 7sPhllYXSLK0Zp50oUlHloEFR72ocJLrvidz2y+B7dWAylcrL8Z1rzY/PGXWcjmGFH0vFcx4yzar + 2AjlzWI9/fJ5qrg9b6drBQDj7eSSjfP4mCN1mwpu54/Hfvm9kAtlBXc75v/j93T6UtQ1P5mYy+8t + F5Pv4INl7SLmLQ8q/+l52DTrJwt2Q55McnExIEXZhm1HKQinLixvaM6bqGY1ay5tDbGGM5Y2hBwe + Rz62D9WHVHAntg7vmTd5EdqrJTl95+cbeLKSqUcoVhRjdakd+WCk2wH05lmRub54J0jrLdouTy77 + +Uv64+eYdj0z/Evo8XQYJ02TjIKYqhgV/AqHGPL0wZiPi7QdLp591NQ0FokZ10PI7Mnv0NyfxPKc + 7/G80fbIFSNKZv2QdM71ekSBrvW4nvlqor1QAeqlBXNTiSVDvGkx2OqRsF1sHgqe9GSP5PPaZYe9 + nhV9ptQCvG9Xh1mbMjblS7PCaF5vspluHeebb9rAS1S7P/1cLnRXCb5h+MbqZyqLIauuKSI5rchW + U4qwW8G6g1kf0vdu8Wg7S9ovtKjf+GT3VsJiJK2DgZAPzPzEeHc/YAH91vM7809XnksFzokgEFPZ + 7hLemwddc6JDj7mlXQt2OlUl+lY3jNHa5AXf6WcFYjdq8URr3ZPS4zuGdloHM/9yzo3jNYX5+lTU + VkYrel4IcPWOEXNcWzbHGW/gEEwm+fV7Zr+WIiABo4gvryZ7hpOt5d/7gw4RV/kQH7IjEu4PAWvp + kaDp1YcGut86kS6fDUdjLiYSOAQ1WIzcJxp3YZWvKK04c4ywNWl67CM4Brb3535GHH1zZMjSfc6D + Di2rbpctzHoZw6w3vi7OFvD371TAf/37/8eJAvGfEwX/nCj450TBPycK/jlR8L/mRMF/AwAA///s + 3TEKwkAABMA+rwjXB0JK/yJykMPCxAvmBBv/LqdBfEKQabfZDwy7RAFRQBQQBUQBUUAUEAVEAVFA + FBAFRAFRQBQQBUQBUUAUEAVEAVFAFBAFRAFRsB9RMBAFRAFRQBQQBUQBUUAUEAVEAVFAFBAFRAFR + QBQQBUQBUUAUEAVEAVFAFBAFRAFRQBQQBUQBUfDnoqBp2+P7BWHOY5oqDCjpUbovFejiGLu+Hz5X + Cfc1nlM4bAIhLLc8L+VU8iVd10oNtvWCUHKJ02/e1Kpn8wIAAP//AwArUdmvhGEAAA== + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 9017b86929d9de9b-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Mon, 13 Jan 2025 19:16:22 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-model: + - text-embedding-ada-002 + openai-organization: + - datadog-staging + openai-processing-ms: + - '56' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + via: + - envoy-router-867d7fff98-jp2xz + x-envoy-upstream-service-time: + - '34' + x-ratelimit-limit-requests: + - '10000' + x-ratelimit-limit-tokens: + - '10000000' + x-ratelimit-remaining-requests: + - '9999' + x-ratelimit-remaining-tokens: + - '9999979' + x-ratelimit-reset-requests: + - 6ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_0664830d8f95882874d90777108542e2 + status: + code: 200 + message: OK +version: 1 diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.emits_traces_and_evaluations_on_exit.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.emits_traces_and_evaluations_on_exit.yaml similarity index 76% rename from tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.emits_traces_and_evaluations_on_exit.yaml rename to tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.emits_traces_and_evaluations_on_exit.yaml index 757f875443f..367024a712d 100644 --- a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.emits_traces_and_evaluations_on_exit.yaml +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.emits_traces_and_evaluations_on_exit.yaml @@ -73,19 +73,19 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAA2yRW4vbMBCF3/0rxDzHi+2mySZvbUOhsPRGoSxxsBV5bKuVJaGZ9ELIfy9yvEnK - 7oseztH5NGd0TIQA3cBagOolq8Gb9M3D5tPi1zuz+r0qHjavv27a74+vPj5+sG8z8wVmMeH2P1Dx - U+pOucEbZO3s2VYBJWOk5stitchXy3w+GoNr0MRY5zmdu3TQVqdFVszTbJnm91O6d1ohwVpsEyGE - OI5nnNM2+AfWIps9KQMSyQ5hfbkkBARnogKSSBNLyzC7mspZRjuOXtf19lgCYVQUViO+HPmiBNKx - U6iIJeOAlila2xK+9SiU9JqlEa4V74O0CoUm8VkGTXcl7E67uq5vHw3YHkjG4vZgzKSfLi2M63xw - e5r8i95qq6mvAkpyNk5M7DyM7ikRYjdu6/DfAsAHN3iu2P1EG4GLLD/z4PpJV7dYTCY7luYmVSxn - L/CqBllqQzf7BiVVj801miU35Z4/+hLiXFDb7hklmUhAf4lxqFptOww+6PMPtr6a3xeqKORyryA5 - Jf8AAAD//wMAn6C7Cc8CAAA= + H4sIAAAAAAAAA4xSwYrbMBS8+yvEO8eL403iTW49bEtuWdpCIQ62Ij/bam1J6L1AS8i/FznZ2Mtu + oRcdZt6MZp50joQAXcFGgGolq9518acf0hyX6+02W26T3e7Ly/r5JTltv9rvNT/DLCjs8ScqflU9 + KNu7Dllbc6WVR8kYXOfZY7pcrVeLx4HobYVdkDWO44WNe210nCbpIk6yeP50U7dWKyTYiH0khBDn + 4Qw5TYW/YSOS2SvSI5FsEDb3ISHA2y4gIIk0sTQMs5FU1jCaIXpZlvtzDoQBUVgM9vngL3IgHTr5 + glgy9miYArXP4VuLQkmnWXbC1uKzl0ah0CR20mt6yOFwOZRlOb3UY30iGYqbU9fd8Mu9RWcb5+2R + bvwdr7XR1BYeJVkTEhNbBwN7iYQ4DNs6vVkAOG97xwXbX2iC4SqZX/1gfKSRTVc3ki3LbqJKs9kH + fkWFLHVHk32DkqrFapSOjyNPlbYTIpq0fp/mI+9rc22a/7EfCaXQMVaF81hp9bbxOOYx/OF/jd23 + PAQG+kOMfVFr06B3Xl9/UO2KJEuWx/opUwlEl+gvAAAA//8DABrEjBtPAwAA headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8d6b5b701f294367-EWR + - 8e84af2fba19c952-IAD Connection: - keep-alive Content-Encoding: @@ -93,14 +93,14 @@ interactions: Content-Type: - application/json Date: - - Tue, 22 Oct 2024 17:55:15 GMT + - Mon, 25 Nov 2024 21:20:43 GMT Server: - cloudflare Set-Cookie: - - __cf_bm=iQaF937ylY7BvvBCyWYQoxiJwi1nBp5.LILrHLw1uno-1729619715-1.0.1.1-jS4Dz7yc_ud.hKZlJ_CAZkSQesqzVkfrA5F30zI7CtJsbEKyAiuVlpX0CPf816UtlhXQEW8T5nsc.UvnsCOzOw; - path=/; expires=Tue, 22-Oct-24 18:25:15 GMT; domain=.api.openai.com; HttpOnly; + - __cf_bm=CVMxqIcHUHNjX56k1MKjj4MgiYNVAlg_B7yyVaP_z1o-1732569643-1.0.1.1-HOtZfXprHWr_DjtorQ_ZK6bbSmcOsBrphniRCaC9XQ2tTtO5JVpyDQK1HRFo3kUE9GEi9J.sR0_L6nBtXlGj8w; + path=/; expires=Mon, 25-Nov-24 21:50:43 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - - _cfuvid=wQzHCwLW6CPU768K_tlLklWp36I8zYCVJkKlAMtnMkk-1729619715162-0.0.1.1-604800000; + - _cfuvid=sqtPZaucqBJu1r4exJtYym3vbKmuuSO6o0np5VglPsw-1732569643935-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None Transfer-Encoding: - chunked @@ -113,7 +113,7 @@ interactions: openai-organization: - datadog-staging openai-processing-ms: - - '496' + - '370' openai-version: - '2020-10-01' strict-transport-security: @@ -131,7 +131,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_33b8cddecaab8b8bc36e90f58f844636 + - req_02ed729afc2d9083921e3fe5b7528550 status: code: 200 message: OK @@ -193,8 +193,8 @@ interactions: content-type: - application/json cookie: - - __cf_bm=iQaF937ylY7BvvBCyWYQoxiJwi1nBp5.LILrHLw1uno-1729619715-1.0.1.1-jS4Dz7yc_ud.hKZlJ_CAZkSQesqzVkfrA5F30zI7CtJsbEKyAiuVlpX0CPf816UtlhXQEW8T5nsc.UvnsCOzOw; - _cfuvid=wQzHCwLW6CPU768K_tlLklWp36I8zYCVJkKlAMtnMkk-1729619715162-0.0.1.1-604800000 + - __cf_bm=CVMxqIcHUHNjX56k1MKjj4MgiYNVAlg_B7yyVaP_z1o-1732569643-1.0.1.1-HOtZfXprHWr_DjtorQ_ZK6bbSmcOsBrphniRCaC9XQ2tTtO5JVpyDQK1HRFo3kUE9GEi9J.sR0_L6nBtXlGj8w; + _cfuvid=sqtPZaucqBJu1r4exJtYym3vbKmuuSO6o0np5VglPsw-1732569643935-0.0.1.1-604800000 host: - api.openai.com user-agent: @@ -220,19 +220,20 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAA2xSQW7bMBC86xWLPVuBpTqR7VuAoGiBAmmLHhrEgUVTK2tdiSTIdZDA8N8Lyorl - ILnwMLMznB3ykAAgV7gE1I0S3bk2vf1xd19M/c3u4frhu/37Tet73qm7383r7fwXTqLCbnak5U11 - pW3nWhK25kRrT0ooumZFvrjJFkV23ROdraiNsq2TdGbTjg2n+TSfpdMizeaDurGsKeASHhMAgEN/ - xpymohdcwnTyhnQUgtoSLs9DAOhtGxFUIXAQZQQnI6mtETJ99LIsHw8rDKKEOjKywiWs8E9DoJVj - US3YGr56ZTQBB/ipPIerFU5ghZ5UsGYUnD3ioIKKPWkBT46EYy3RSRoCNrX1neoh5+0zV1QBm57r - k73IcMMz+Yp1nyk7PpVlebmEp3ofVCzS7Nt2wI/nVlq7dd5uwsCf8ZoNh2Z9Ch8bCGId9uwxAXjq - 29+/KxSdt52Ttdh/ZKJhUcxPfjg++sh+WQykWFHtiM+zYvKJ37oiUdyGi/dDrXRD1SidJhfLfbz0 - M4vTgmy2H1ySwQnDaxDq1jWbLXnn+fQjareezXOd56rYaEyOyX8AAAD//wMAUtzROh8DAAA= + H4sIAAAAAAAAA4xTwW6bQBC98xWjPZsIOzg43Fqp7a2y2iqqFEew3h1gWthd7Y6jpJb/vVrsGEdJ + pV44vDfv8eYN7BMAQVqUIFQnWQ2uTz/8lEZ9/NI9//mafVt9Wt61uxbvtt+3Oa3XYhYVdvsLFb+o + rpQdXI9M1hxp5VEyRtd5cb1Y3tze5PlIDFZjH2Wt4zS36UCG0kW2yNOsSOerk7qzpDCIEu4TAID9 + +Iw5jcYnUUI2e0EGDEG2KMrzEIDwto+IkCFQYGlYzCZSWcNoxuh1Xd/vNyKwZBzQ8EaUsBE/OgQl + HbHswTbw2UujECjAWnoKVxsxg43wKIM1k+DsEQclaPKoGDw6ZIq1RCfuEMg01g9yhJy3j6RRA5mR + G5M98ekNj+g1qTHT/PBQ1/XlEh6bXZCxSLPr+xN+OLfS29Z5uw0n/ow3ZCh01TF8bCCwdWJkDwnA + w9j+7lWhwnk7OK7Y/kYTDYtidfQT09En9vr2RLJl2U/4al7M3vGrNLKkPlzcTyipOtSTdDq23Gmy + F0RysfXbNO95Hzcn0/6P/UQohY5RV85jvMmrjacxj/Gf+NfYueUxsAjPgXGoGjIteufp+EU2rsqK + bLltVoXKRHJI/gIAAP//AwDgYzoinwMAAA== headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8d6b5b744e034367-EWR + - 8e84af32dc70c952-IAD Connection: - keep-alive Content-Encoding: @@ -240,7 +241,7 @@ interactions: Content-Type: - application/json Date: - - Tue, 22 Oct 2024 17:55:16 GMT + - Mon, 25 Nov 2024 21:20:45 GMT Server: - cloudflare Transfer-Encoding: @@ -254,7 +255,7 @@ interactions: openai-organization: - datadog-staging openai-processing-ms: - - '749' + - '1168' openai-version: - '2020-10-01' strict-transport-security: @@ -272,35 +273,37 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_fbb01161a03eb6f478ff52314b72cfd6 + - req_702ebaa1edbab95fb42f52baa4b34661 status: code: 200 message: OK - request: - body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"span_id": - "6877142543397072040", "trace_id": "6717e70200000000a99ea8ad36f4f36d", "label": - "ragas_faithfulness", "metric_type": "score", "timestamp_ms": 1729619716093, - "score_value": 1.0, "ml_app": "unnamed-ml-app", "tags": ["ddtrace.version:2.15.0.dev219+ge047e25bb.d20241022", - "ml_app:unnamed-ml-app"]}]}}}' + body: '{"data": {"type": "evaluation_metric", "attributes": {"metrics": [{"join_on": + {"span": {"span_id": "7678809694384023494", "trace_id": "6744ea2b00000000995e7b2ceabfce01"}}, + "label": "ragas_faithfulness", "metric_type": "score", "timestamp_ms": 1732569645205, + "score_value": 1.0, "ml_app": "unnamed-ml-app", "tags": ["ddtrace.version:2.15.0.dev351+g152f3e3b6.d20241122", + "ml_app:unnamed-ml-app"], "metadata": {"_dd.evaluation_kind": "faithfulness", + "_dd.evaluation_span": {"span_id": "5771061714047746387", "trace_id": "6744ea2b000000007099aeb477077763"}, + "_dd.faithfulness_disagreements": []}}]}}}' headers: Content-Type: - application/json DD-API-KEY: - XXXXXX method: POST - uri: https://api.datad0g.com/api/intake/llm-obs/v1/eval-metric + uri: https://api.datad0g.com/api/intake/llm-obs/v2/eval-metric response: body: - string: '{"data":{"id":"99fa371c-457c-4d2b-8d4c-61657e0ffd48","type":"evaluation_metric","attributes":{"metrics":[{"id":"CbapxUnzcX","trace_id":"6717e70200000000a99ea8ad36f4f36d","span_id":"6877142543397072040","timestamp_ms":1729619716093,"ml_app":"unnamed-ml-app","metric_type":"score","label":"ragas_faithfulness","score_value":1,"tags":["ddtrace.version:2.15.0.dev219+ge047e25bb.d20241022","ml_app:unnamed-ml-app"]}]}}}' + string: '{"data":{"id":"f1470aa7-b97f-4809-825d-6932af26a81c","type":"evaluation_metric","attributes":{"metrics":[{"id":"EPRU-72kfP","join_on":{"span":{"trace_id":"6744ea2b00000000995e7b2ceabfce01","span_id":"7678809694384023494"}},"timestamp_ms":1732569645205,"ml_app":"unnamed-ml-app","metric_type":"score","label":"ragas_faithfulness","score_value":1,"tags":["ddtrace.version:2.15.0.dev351+g152f3e3b6.d20241122","ml_app:unnamed-ml-app"],"metadata":{"_dd.evaluation_kind":"faithfulness","_dd.evaluation_span":{"span_id":"5771061714047746387","trace_id":"6744ea2b000000007099aeb477077763"},"_dd.faithfulness_disagreements":[]}}]}}}' headers: content-length: - - '414' + - '623' content-security-policy: - frame-ancestors 'self'; report-uri https://logs.browser-intake-datadoghq.com/api/v2/logs?dd-api-key=pub293163a918901030b79492fe1ab424cf&dd-evp-origin=content-security-policy&ddsource=csp-report&ddtags=site%3Adatad0g.com content-type: - application/vnd.api+json date: - - Tue, 22 Oct 2024 17:55:17 GMT + - Mon, 25 Nov 2024 21:20:45 GMT strict-transport-security: - max-age=31536000; includeSubDomains; preload vary: diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_multiple_context.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_multiple_context.yaml new file mode 100644 index 00000000000..a78852c8d64 --- /dev/null +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_multiple_context.yaml @@ -0,0 +1,312 @@ +interactions: +- request: + body: '{"messages": [{"content": "Given question, answer and context verify if + the context was useful in arriving at the given answer. Give verdict as \"1\" + if useful and \"0\" if not with json output.\n\nThe output should be a well-formatted + JSON instance that conforms to the JSON schema below.\n\nAs an example, for + the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": + \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, + \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted + instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} + is not well-formatted.\n\nHere is the output JSON schema:\n```\n{\"description\": + \"Answer for the verification task wether the context was useful.\", \"type\": + \"object\", \"properties\": {\"reason\": {\"title\": \"Reason\", \"description\": + \"Reason for verification\", \"type\": \"string\"}, \"verdict\": {\"title\": + \"Verdict\", \"description\": \"Binary (0/1) verdict of verification\", \"type\": + \"integer\"}}, \"required\": [\"reason\", \"verdict\"]}\n```\n\nDo not return + any preamble or explanations, return only a pure JSON string surrounded by triple + backticks (```).\n\nExamples:\n\nquestion: \"What can you tell me about albert + Albert Einstein?\"\ncontext: \"Albert Einstein (14 March 1879 \u2013 18 April + 1955) was a German-born theoretical physicist, widely held to be one of the + greatest and most influential scientists of all time. Best known for developing + the theory of relativity, he also made important contributions to quantum mechanics, + and was thus a central figure in the revolutionary reshaping of the scientific + understanding of nature that modern physics accomplished in the first decades + of the twentieth century. His mass\u2013energy equivalence formula E = mc2, + which arises from relativity theory, has been called \\\"the world''s most famous + equation\\\". He received the 1921 Nobel Prize in Physics \\\"for his services + to theoretical physics, and especially for his discovery of the law of the photoelectric + effect\\\", a pivotal step in the development of quantum theory. His work is + also known for its influence on the philosophy of science. In a 1999 poll of + 130 leading physicists worldwide by the British journal Physics World, Einstein + was ranked the greatest physicist of all time. His intellectual achievements + and originality have made Einstein synonymous with genius.\"\nanswer: \"Albert + Einstein born in 14 March 1879 was German-born theoretical physicist, widely + held to be one of the greatest and most influential scientists of all time. + He received the 1921 Nobel Prize in Physics for his services to theoretical + physics. He published 4 papers in 1905. Einstein moved to Switzerland in 1895\"\nverification: + ```{\"reason\": \"The provided context was indeed useful in arriving at the + given answer. The context includes key information about Albert Einstein''s + life and contributions, which are reflected in the answer.\", \"verdict\": 1}```\n\nquestion: + \"who won 2020 icc world cup?\"\ncontext: \"The 2022 ICC Men''s T20 World Cup, + held from October 16 to November 13, 2022, in Australia, was the eighth edition + of the tournament. Originally scheduled for 2020, it was postponed due to the + COVID-19 pandemic. England emerged victorious, defeating Pakistan by five wickets + in the final to clinch their second ICC Men''s T20 World Cup title.\"\nanswer: + \"England\"\nverification: ```{\"reason\": \"the context was useful in clarifying + the situation regarding the 2020 ICC World Cup and indicating that England was + the winner of the tournament that was intended to be held in 2020 but actually + took place in 2022.\", \"verdict\": 1}```\n\nquestion: \"What is the tallest + mountain in the world?\"\ncontext: \"The Andes is the longest continental mountain + range in the world, located in South America. It stretches across seven countries + and features many of the highest peaks in the Western Hemisphere. The range + is known for its diverse ecosystems, including the high-altitude Andean Plateau + and the Amazon rainforest.\"\nanswer: \"Mount Everest.\"\nverification: ```{\"reason\": + \"the provided context discusses the Andes mountain range, which, while impressive, + does not include Mount Everest or directly relate to the question about the + world''s tallest mountain.\", \"verdict\": 0}```\n\nYour actual task:\n\nquestion: + \"Is france part of europe?\"\ncontext: \"irrelevant\"\nanswer: \"France is + indeed part of europe\"\nverification: \n", "role": "user"}], "model": "gpt-4o-mini", + "n": 1, "stream": false, "temperature": 1e-08}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '4612' + content-type: + - application/json + cookie: + - __cf_bm=2N0lRp5YNIBKY6AUc.tpQsJVlWEga7Ys924AChkX4qk-1733967111-1.0.1.1-IJEARyUXuMN2pbqt5jU4yaj77.QHaVM0uVSztZt49GpbAV1HXoPr6.uIdz2viIUlRExuu5tYN_.v5wUpYjyBSQ; + _cfuvid=TvHcCPz7N_.kfviRP.Y0iD_HMeA.0uxvji5nzbbTR5w-1733967111302-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jFNNj5swFLzzK5586QVWZJNtPm49dC+tVKmteikVGPsFnBrbsR9Roij/ + vTLJBlbdSr0g9ObNaGYenBMApiTbABMtJ9E5nX2Q3z5//dTu9WH/vJD57stx79z6JFD+oBVLI8PW + OxT0wnoQtnMaSVlzhYVHThhVZ8v5fP1+OZvNB6CzEnWkNY6yhc06ZVT2mD8usnyZzW7iorVKYGAb + +JkAAJyHZ/RpJB7ZBvL0ZdJhCLxBtrkvATBvdZwwHoIKxA2xdASFNYRmsF5V1S5YU5hzwTzy+Mo2 + ULDvLcKwdiRw3h6URAkqgPIeNR64IeBGgrQYwFgadr2qe0Lg5gTKbK3veGwDPDbcS2UaePbcCHwX + oEHbeO5aJbiGQJz6AMrAx95bhw8FS6FgB/RSCYp28kthqqqaRvC47QOPNZpe69v8cu9E28Z5W4cb + fp9vlVGhLa9JY/5A1rEBvSQAv4bu+1d1Mudt56gk+xtNFFw/PV312HjyEZ0vbiBZ4nrCWq3TN/RK + icSVDpPrMcFFi3KkjqfmvVR2AiST1H+7eUv7mlyZ5n/kR0AIdISydB7jUV4lHtc8xj/iX2v3lgfD + LJwCYVdulWnQO6+u3+PWlXXN52KFy7xmySX5AwAA//8DAEriHaudAwAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f09f515b8157281-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 12 Dec 2024 01:31:53 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '976' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149998903' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_2c2d33a1a025655db6ab9a62096b644a + status: + code: 200 + message: OK +- request: + body: '{"messages": [{"content": "Given question, answer and context verify if + the context was useful in arriving at the given answer. Give verdict as \"1\" + if useful and \"0\" if not with json output.\n\nThe output should be a well-formatted + JSON instance that conforms to the JSON schema below.\n\nAs an example, for + the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": + \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, + \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted + instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} + is not well-formatted.\n\nHere is the output JSON schema:\n```\n{\"description\": + \"Answer for the verification task wether the context was useful.\", \"type\": + \"object\", \"properties\": {\"reason\": {\"title\": \"Reason\", \"description\": + \"Reason for verification\", \"type\": \"string\"}, \"verdict\": {\"title\": + \"Verdict\", \"description\": \"Binary (0/1) verdict of verification\", \"type\": + \"integer\"}}, \"required\": [\"reason\", \"verdict\"]}\n```\n\nDo not return + any preamble or explanations, return only a pure JSON string surrounded by triple + backticks (```).\n\nExamples:\n\nquestion: \"What can you tell me about albert + Albert Einstein?\"\ncontext: \"Albert Einstein (14 March 1879 \u2013 18 April + 1955) was a German-born theoretical physicist, widely held to be one of the + greatest and most influential scientists of all time. Best known for developing + the theory of relativity, he also made important contributions to quantum mechanics, + and was thus a central figure in the revolutionary reshaping of the scientific + understanding of nature that modern physics accomplished in the first decades + of the twentieth century. His mass\u2013energy equivalence formula E = mc2, + which arises from relativity theory, has been called \\\"the world''s most famous + equation\\\". He received the 1921 Nobel Prize in Physics \\\"for his services + to theoretical physics, and especially for his discovery of the law of the photoelectric + effect\\\", a pivotal step in the development of quantum theory. His work is + also known for its influence on the philosophy of science. In a 1999 poll of + 130 leading physicists worldwide by the British journal Physics World, Einstein + was ranked the greatest physicist of all time. His intellectual achievements + and originality have made Einstein synonymous with genius.\"\nanswer: \"Albert + Einstein born in 14 March 1879 was German-born theoretical physicist, widely + held to be one of the greatest and most influential scientists of all time. + He received the 1921 Nobel Prize in Physics for his services to theoretical + physics. He published 4 papers in 1905. Einstein moved to Switzerland in 1895\"\nverification: + ```{\"reason\": \"The provided context was indeed useful in arriving at the + given answer. The context includes key information about Albert Einstein''s + life and contributions, which are reflected in the answer.\", \"verdict\": 1}```\n\nquestion: + \"who won 2020 icc world cup?\"\ncontext: \"The 2022 ICC Men''s T20 World Cup, + held from October 16 to November 13, 2022, in Australia, was the eighth edition + of the tournament. Originally scheduled for 2020, it was postponed due to the + COVID-19 pandemic. England emerged victorious, defeating Pakistan by five wickets + in the final to clinch their second ICC Men''s T20 World Cup title.\"\nanswer: + \"England\"\nverification: ```{\"reason\": \"the context was useful in clarifying + the situation regarding the 2020 ICC World Cup and indicating that England was + the winner of the tournament that was intended to be held in 2020 but actually + took place in 2022.\", \"verdict\": 1}```\n\nquestion: \"What is the tallest + mountain in the world?\"\ncontext: \"The Andes is the longest continental mountain + range in the world, located in South America. It stretches across seven countries + and features many of the highest peaks in the Western Hemisphere. The range + is known for its diverse ecosystems, including the high-altitude Andean Plateau + and the Amazon rainforest.\"\nanswer: \"Mount Everest.\"\nverification: ```{\"reason\": + \"the provided context discusses the Andes mountain range, which, while impressive, + does not include Mount Everest or directly relate to the question about the + world''s tallest mountain.\", \"verdict\": 0}```\n\nYour actual task:\n\nquestion: + \"Is france part of europe?\"\ncontext: \"France is part of europe\"\nanswer: + \"France is indeed part of europe\"\nverification: \n", "role": "user"}], "model": + "gpt-4o-mini", "n": 1, "stream": false, "temperature": 1e-08}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '4626' + content-type: + - application/json + cookie: + - __cf_bm=2N0lRp5YNIBKY6AUc.tpQsJVlWEga7Ys924AChkX4qk-1733967111-1.0.1.1-IJEARyUXuMN2pbqt5jU4yaj77.QHaVM0uVSztZt49GpbAV1HXoPr6.uIdz2viIUlRExuu5tYN_.v5wUpYjyBSQ; + _cfuvid=TvHcCPz7N_.kfviRP.Y0iD_HMeA.0uxvji5nzbbTR5w-1733967111302-0.0.1.1-604800000 + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jFLLbtswELzrKxY8S4EVu7Wtm4H0ccilSHtpVUg0uZLoSCRLrmKnhv+9 + oOxYCpoCvRDEzs5wZpfHCIApyTJgouEkOtsmG/lw//Bp83tffRH2+7duU9/dfT7gPS53j79YHBhm + u0NBL6wbYTrbIimjz7BwyAmDarqcz9fvl2m6GIDOSGwDrbaULEzSKa2S29ntIpktk3R1YTdGCfQs + gx8RAMBxOINPLfHAMpjFL5UOvec1suzaBMCcaUOFce+VJ66JxSMojCbUg/WyLHfe6Fwfc+aQhyvL + IGdfG4Sh7UAglUNB7TN44oQeqOEEHx3XAkF5sNwRmAo+9M5YjGHfKNFMSL21xlGgIXDt9+jAOvOk + JMqbnMWQsyd0UgkKL6enXJdlOXXrsOo9DxPTfdte6qdr/NbU1pmtv+DXeqW08k1xDhWiejKWDegp + Avg5jLl/NTlmneksFWQeUQfB9bvVWY+N2x3R+fwCkiHeTljrNH5Dr5BIXLV+sigmuGhQjtRxq7yX + ykyAaJL6bzdvaZ+TK13/j/wICIGWUBbWYVjKq8Rjm8Pw+f/Vdp3yYJj5Z0/YFZXSNTrr1PnrVbbY + bvlcrHA527LoFP0BAAD//wMAcOvPiIgDAAA= + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f09f51cdbaf7281-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 12 Dec 2024 01:31:54 GMT + Server: + - cloudflare + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '779' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149998900' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_4b6ae2d675e25726729e2577e662f691 + status: + code: 200 + message: OK +version: 1 \ No newline at end of file diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_single_context.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_single_context.yaml new file mode 100644 index 00000000000..ddbd8f5c3d9 --- /dev/null +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_single_context.yaml @@ -0,0 +1,160 @@ +interactions: +- request: + body: '{"messages": [{"content": "Given question, answer and context verify if + the context was useful in arriving at the given answer. Give verdict as \"1\" + if useful and \"0\" if not with json output.\n\nThe output should be a well-formatted + JSON instance that conforms to the JSON schema below.\n\nAs an example, for + the schema {\"properties\": {\"foo\": {\"title\": \"Foo\", \"description\": + \"a list of strings\", \"type\": \"array\", \"items\": {\"type\": \"string\"}}}, + \"required\": [\"foo\"]}\nthe object {\"foo\": [\"bar\", \"baz\"]} is a well-formatted + instance of the schema. The object {\"properties\": {\"foo\": [\"bar\", \"baz\"]}} + is not well-formatted.\n\nHere is the output JSON schema:\n```\n{\"description\": + \"Answer for the verification task wether the context was useful.\", \"type\": + \"object\", \"properties\": {\"reason\": {\"title\": \"Reason\", \"description\": + \"Reason for verification\", \"type\": \"string\"}, \"verdict\": {\"title\": + \"Verdict\", \"description\": \"Binary (0/1) verdict of verification\", \"type\": + \"integer\"}}, \"required\": [\"reason\", \"verdict\"]}\n```\n\nDo not return + any preamble or explanations, return only a pure JSON string surrounded by triple + backticks (```).\n\nExamples:\n\nquestion: \"What can you tell me about albert + Albert Einstein?\"\ncontext: \"Albert Einstein (14 March 1879 \u2013 18 April + 1955) was a German-born theoretical physicist, widely held to be one of the + greatest and most influential scientists of all time. Best known for developing + the theory of relativity, he also made important contributions to quantum mechanics, + and was thus a central figure in the revolutionary reshaping of the scientific + understanding of nature that modern physics accomplished in the first decades + of the twentieth century. His mass\u2013energy equivalence formula E = mc2, + which arises from relativity theory, has been called \\\"the world''s most famous + equation\\\". He received the 1921 Nobel Prize in Physics \\\"for his services + to theoretical physics, and especially for his discovery of the law of the photoelectric + effect\\\", a pivotal step in the development of quantum theory. His work is + also known for its influence on the philosophy of science. In a 1999 poll of + 130 leading physicists worldwide by the British journal Physics World, Einstein + was ranked the greatest physicist of all time. His intellectual achievements + and originality have made Einstein synonymous with genius.\"\nanswer: \"Albert + Einstein born in 14 March 1879 was German-born theoretical physicist, widely + held to be one of the greatest and most influential scientists of all time. + He received the 1921 Nobel Prize in Physics for his services to theoretical + physics. He published 4 papers in 1905. Einstein moved to Switzerland in 1895\"\nverification: + ```{\"reason\": \"The provided context was indeed useful in arriving at the + given answer. The context includes key information about Albert Einstein''s + life and contributions, which are reflected in the answer.\", \"verdict\": 1}```\n\nquestion: + \"who won 2020 icc world cup?\"\ncontext: \"The 2022 ICC Men''s T20 World Cup, + held from October 16 to November 13, 2022, in Australia, was the eighth edition + of the tournament. Originally scheduled for 2020, it was postponed due to the + COVID-19 pandemic. England emerged victorious, defeating Pakistan by five wickets + in the final to clinch their second ICC Men''s T20 World Cup title.\"\nanswer: + \"England\"\nverification: ```{\"reason\": \"the context was useful in clarifying + the situation regarding the 2020 ICC World Cup and indicating that England was + the winner of the tournament that was intended to be held in 2020 but actually + took place in 2022.\", \"verdict\": 1}```\n\nquestion: \"What is the tallest + mountain in the world?\"\ncontext: \"The Andes is the longest continental mountain + range in the world, located in South America. It stretches across seven countries + and features many of the highest peaks in the Western Hemisphere. The range + is known for its diverse ecosystems, including the high-altitude Andean Plateau + and the Amazon rainforest.\"\nanswer: \"Mount Everest.\"\nverification: ```{\"reason\": + \"the provided context discusses the Andes mountain range, which, while impressive, + does not include Mount Everest or directly relate to the question about the + world''s tallest mountain.\", \"verdict\": 0}```\n\nYour actual task:\n\nquestion: + \"What is the capital of France?\"\ncontext: \"The capital of France is Paris.\"\nanswer: + \"The capital of France is Paris\"\nverification: \n", "role": "user"}], "model": + "gpt-4o-mini", "n": 1, "stream": false, "temperature": 1e-08}' + headers: + accept: + - application/json + accept-encoding: + - gzip, deflate + connection: + - keep-alive + content-length: + - '4637' + content-type: + - application/json + host: + - api.openai.com + user-agent: + - OpenAI/Python 1.52.0 + x-stainless-arch: + - arm64 + x-stainless-async: + - 'false' + x-stainless-lang: + - python + x-stainless-os: + - MacOS + x-stainless-package-version: + - 1.52.0 + x-stainless-retry-count: + - '0' + x-stainless-runtime: + - CPython + x-stainless-runtime-version: + - 3.10.13 + method: POST + uri: https://api.openai.com/v1/chat/completions + response: + body: + string: !!binary | + H4sIAAAAAAAAAwAAAP//jFJBbtswELzrFQuercCyAzvxrZf20qIpEqOHqpAociXRpUiCXMcODP+9 + IO1YNppDL4K0szOYndEhA2BKshUw0XMSg9P5J/n89fuP55/Ft123XTfFk5qpL6Zbv6z3irNJZNhm + g4LeWXfCDk4jKWtOsPDICaNqsZzPHxfLopgmYLASdaR1jvJ7mw/KqHw2nd3n02VePJzZvVUCA1vB + rwwA4JCe0aeRuGcrSFppMmAIvEO2uiwBMG91nDAeggrEDbHJCAprCE2yXtf1JlhTmkPJPPL4ylZQ + spcewXn7qiRKSPt7Aqk8CtJvEIgTBqCeE1CPILhTxDXYFj57bgSCCvDEvQoT2PVK9PEb9zyRPbYa + BaEEZRKbm7BDf1eyCZTsFb1UgqKL4liauq6vnXtst4HH9MxW6/P8eIlC285524Qzfpm3yqjQV6cD + 49mBrGMJPWYAv1Pk25sUmfN2cFSR/YMmCj4uzpGzsekRnS/OIFni+or1+A7c6FUSiSsdrkpjgose + 5UgdG+ZbqewVkF1d/a+bj7RPlyvT/Y/8CAiBjlBWzmMs5ebicc3jJvX58dol5WSYhbdAOFStMh16 + 59XpN2xd1TR8Lh5wOW1Ydsz+AgAA//8DAJLjX/OUAwAA + headers: + CF-Cache-Status: + - DYNAMIC + CF-RAY: + - 8f09f507cc477281-EWR + Connection: + - keep-alive + Content-Encoding: + - gzip + Content-Type: + - application/json + Date: + - Thu, 12 Dec 2024 01:31:51 GMT + Server: + - cloudflare + Set-Cookie: + - __cf_bm=2N0lRp5YNIBKY6AUc.tpQsJVlWEga7Ys924AChkX4qk-1733967111-1.0.1.1-IJEARyUXuMN2pbqt5jU4yaj77.QHaVM0uVSztZt49GpbAV1HXoPr6.uIdz2viIUlRExuu5tYN_.v5wUpYjyBSQ; + path=/; expires=Thu, 12-Dec-24 02:01:51 GMT; domain=.api.openai.com; HttpOnly; + Secure; SameSite=None + - _cfuvid=TvHcCPz7N_.kfviRP.Y0iD_HMeA.0uxvji5nzbbTR5w-1733967111302-0.0.1.1-604800000; + path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None + Transfer-Encoding: + - chunked + X-Content-Type-Options: + - nosniff + access-control-expose-headers: + - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 + openai-organization: + - datadog-staging + openai-processing-ms: + - '564' + openai-version: + - '2020-10-01' + strict-transport-security: + - max-age=31536000; includeSubDomains; preload + x-ratelimit-limit-requests: + - '30000' + x-ratelimit-limit-tokens: + - '150000000' + x-ratelimit-remaining-requests: + - '29999' + x-ratelimit-remaining-tokens: + - '149998898' + x-ratelimit-reset-requests: + - 2ms + x-ratelimit-reset-tokens: + - 0s + x-request-id: + - req_db048be5fbcb3bc4136d9c89ace1249a + status: + code: 200 + message: OK +version: 1 \ No newline at end of file diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_emits_traces.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_emits_traces.yaml similarity index 80% rename from tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_emits_traces.yaml rename to tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_emits_traces.yaml index 8efe7391c90..2100bb3d305 100644 --- a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_emits_traces.yaml +++ b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_emits_traces.yaml @@ -51,7 +51,7 @@ interactions: host: - api.openai.com user-agent: - - OpenAI/Python 1.47.1 + - OpenAI/Python 1.52.0 x-stainless-arch: - arm64 x-stainless-async: @@ -61,7 +61,9 @@ interactions: x-stainless-os: - MacOS x-stainless-package-version: - - 1.47.1 + - 1.52.0 + x-stainless-retry-count: + - '0' x-stainless-runtime: - CPython x-stainless-runtime-version: @@ -71,19 +73,19 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//dJHBbtswEETv+gpiz1YgCY7k+JbCKAoEQYKih7SWIdH0SmJDkQR3jbYw - /O8FZcd2D73wMI8znF0eEiFA72ApQA2S1ehN+vipfFytqq+rt7eHX4v9+3NY/aDnL5y9DE/fYRYd - bvsTFX+47pQbvUHWzp6wCigZY2peFVWRLcryfgKj26GJtt5zOnfpqK1Oi6yYp1mV5ouze3BaIcFS - rBMhhDhMZ+xpd/gbliKbfSgjEskeYXm5JAQEZ6ICkkgTS8swu0LlLKOdqrdtuz7UQBgVhc0UX0/5 - ogbScabQEEvGES1TROsavg0olPSapRGuE5+DtAqFJvEqg6a7GjbHTdu2t48G7PYk4+B2b8xZP16m - MK73wW3pzC96p62moQkoydnYmNh5mOgxEWIzbWv/zwLABzd6bti9o42BZZaf8uD6SVdalGfIjqW5 - cRXV/1zNDllqQzc7h1NDbftrQnapOc0J9IcYx6bTtsfggz59QeebfLudl3lZdQ+QHJO/AAAA//8D - AL2Ti/mQAgAA + H4sIAAAAAAAAA4xSwY7aMBS85yusdyarkAKh3Payh6qVVitUVQWUGOclcevYrt9D7Rbx75UDS1h1 + K/Xiw8yb8cyzj4kQoGtYCVCdZNV7k95/kaZtnpZr3++f159+Pk4/L35P5cevPz6YJ5hEhdt/Q8Uv + qjvlem+QtbNnWgWUjNF1WrzL54v383w2EL2r0URZ6zmdubTXVqd5ls/SrEiny4u6c1ohwUpsEiGE + OA5nzGlr/AUrkU1ekB6JZIuwug4JAcGZiIAk0sTSMkxGUjnLaIfoVVVtjlsgjIjCcrDfDv5iC6Rj + p1ASS8YeLVOkNltYdyiU9JqlEa4RD0FahUKTeJRB090WdqddVVW3lwZsDiRjcXsw5oKfri2Ma31w + e7rwV7zRVlNXBpTkbExM7DwM7CkRYjds6/BqAeCD6z2X7L6jjYaLbHr2g/GRRjZfXEh2LM2NKi8m + b/iVNbLUhm72DUqqDutROj6OPNTa3RDJTeu/07zlfW6ubfs/9iOhFHrGuvQBa61eNx7HAsY//K+x + 65aHwEDPxNiXjbYtBh/0+Qc1vsyKbL5vloXKIDklfwAAAP//AwB8IvReTwMAAA== headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8c856bee184d42d1-EWR + - 8e84ac4858349c52-IAD Connection: - keep-alive Content-Encoding: @@ -91,14 +93,14 @@ interactions: Content-Type: - application/json Date: - - Tue, 24 Sep 2024 20:11:06 GMT + - Mon, 25 Nov 2024 21:18:45 GMT Server: - cloudflare Set-Cookie: - - __cf_bm=nMe4XLsotHph1aKmM6xJotYxeBsTIpCG1ULeQ2oiKLc-1727208666-1.0.1.1-eM1elzOCEnpbPLkOO61HSBvaeQPYHEyO4Ba3P2NsxkYV23Fybb7E8tIipei4YDbhyDiLXybnT7H0ETvjbsV89g; - path=/; expires=Tue, 24-Sep-24 20:41:06 GMT; domain=.api.openai.com; HttpOnly; + - __cf_bm=9IkhXEbzhF0QSjoZOW.EVqEQoEHTaAK7pnQ6K1m4EfY-1732569525-1.0.1.1-8FRhFy6jBsuonirbdG9jJ_IHnhXUakqpLsEg10YYrkhce9PwlXOKXNA2hiZwNqpM3D2TP2X4eFcZJjdEZt6.qQ; + path=/; expires=Mon, 25-Nov-24 21:48:45 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None - - _cfuvid=lKBj.JPFMKz3LiJyz12GeZI73UndAQfhN.5aqiwYPHA-1727208666121-0.0.1.1-604800000; + - _cfuvid=YGxjg63ZVaAJESO.Ouzjnkmhsg2izo9JySj6zJ3MRuc-1732569525322-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None Transfer-Encoding: - chunked @@ -106,10 +108,12 @@ interactions: - nosniff access-control-expose-headers: - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 openai-organization: - datadog-staging openai-processing-ms: - - '576' + - '469' openai-version: - '2020-10-01' strict-transport-security: @@ -127,7 +131,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_ef3f2830eaf13bceea5db3a7369affda + - req_5d6c0d3f36d4cba76fbfea5b6c9f63fe status: code: 200 message: OK @@ -189,12 +193,12 @@ interactions: content-type: - application/json cookie: - - __cf_bm=nMe4XLsotHph1aKmM6xJotYxeBsTIpCG1ULeQ2oiKLc-1727208666-1.0.1.1-eM1elzOCEnpbPLkOO61HSBvaeQPYHEyO4Ba3P2NsxkYV23Fybb7E8tIipei4YDbhyDiLXybnT7H0ETvjbsV89g; - _cfuvid=lKBj.JPFMKz3LiJyz12GeZI73UndAQfhN.5aqiwYPHA-1727208666121-0.0.1.1-604800000 + - __cf_bm=9IkhXEbzhF0QSjoZOW.EVqEQoEHTaAK7pnQ6K1m4EfY-1732569525-1.0.1.1-8FRhFy6jBsuonirbdG9jJ_IHnhXUakqpLsEg10YYrkhce9PwlXOKXNA2hiZwNqpM3D2TP2X4eFcZJjdEZt6.qQ; + _cfuvid=YGxjg63ZVaAJESO.Ouzjnkmhsg2izo9JySj6zJ3MRuc-1732569525322-0.0.1.1-604800000 host: - api.openai.com user-agent: - - OpenAI/Python 1.47.1 + - OpenAI/Python 1.52.0 x-stainless-arch: - arm64 x-stainless-async: @@ -204,7 +208,9 @@ interactions: x-stainless-os: - MacOS x-stainless-package-version: - - 1.47.1 + - 1.52.0 + x-stainless-retry-count: + - '0' x-stainless-runtime: - CPython x-stainless-runtime-version: @@ -214,19 +220,20 @@ interactions: response: body: string: !!binary | - H4sIAAAAAAAAAwAAAP//dFFBbtswELz7FQuerUByDcnRLS3aS4umSBugSBRINLWSNpFIglwHLgz/ - vaCsSOmhFx5mdoazs6cVgKBa5CBUJ1kNto9uPqY3n+/U3e39/WH49vzQyObn7fevx98v2adErIPC - 7J9R8ZvqSpnB9shk9IVWDiVjcE2yTbaJd2majsRgauyDrLUcbU00kKZoE2+2UZxFyW5Sd4YUepHD - 4woA4DS+Iaeu8ShyiNdvyIDeyxZFPg8BCGf6gAjpPXmWmsV6IZXRjHqMXlXV46kQniXjgJoLkUMh - fnUISlpi2YNp4IuTWiGQhx/Skb8qxBoK4VB6oxfB7BEGJdTkUDE4tMgUaglO3CGQbowb5AhZZ16p - xhpIj9yY7MjTD6/oalJjpuT8VFXV+yUcNgcvQ5H60PcTfp5b6U1rndn7iZ/xhjT5rryEDw14NlaM - 7HkF8DS2f/inUGGdGSyXbF5QB8Ms2138xHL0hf1wPZFsWPYLvkuy/6nKGllS79/dcKqXdLs4xHPM - cU/h/3jGoWxIt+iso8tJG1sm+/02TdKsuRar8+ovAAAA//8DADp8axngAgAA + H4sIAAAAAAAAA4xTwWrbQBC96yuGPVtBVuzK8a0YcimhLQRSiIO03h1Zk652l91xSDD+97KyYzk0 + hV50eG/e05s30j4DEKTFEoTqJKvem/zrL2m6h3v78Ganqzv5c7XaVIv+rrTfv3UzMUkKt3lGxe+q + K+V6b5DJ2SOtAkrG5Dqtrsv5l5t5OR+I3mk0Sbb1nM9c3pOlvCzKWV5U+XRxUneOFEaxhMcMAGA/ + PFNOq/FVLKGYvCM9xii3KJbnIQARnEmIkDFSZGlZTEZSOctoh+hN0zzu1yKyZOzR8losYS3uOwQl + PbE04Fq4DdIqBIrwQwaKV2sxgbUIKKOzo+DskQYlaAqoGAJ6ZEq1JCfuEMi2LvRygHxwL6RRA9mB + G5K98ukNLxg0qSHT9PDUNM3lEgHbXZSpSLsz5oQfzq0Yt/XBbeKJP+MtWYpdfQyfGojsvBjYQwbw + NLS/+1Co8MH1nmt2v9Emw6paHP3EePSRvb45kexYmhFfTKvJJ361RpZk4sX9hJKqQz1Kx2PLnSZ3 + QWQXW/+d5jPv4+Zkt/9jPxJKoWfUtQ+YbvJh43EsYPon/jV2bnkILOJbZOzrluwWgw90/CJbXxdV + Md+0i0oVIjtkfwAAAP//AwD0sdbanwMAAA== headers: CF-Cache-Status: - DYNAMIC CF-RAY: - - 8c856bf38f3242d1-EWR + - 8e84ac4daf039c52-IAD Connection: - keep-alive Content-Encoding: @@ -234,7 +241,7 @@ interactions: Content-Type: - application/json Date: - - Tue, 24 Sep 2024 20:11:06 GMT + - Mon, 25 Nov 2024 21:18:46 GMT Server: - cloudflare Transfer-Encoding: @@ -243,10 +250,12 @@ interactions: - nosniff access-control-expose-headers: - X-Request-ID + alt-svc: + - h3=":443"; ma=86400 openai-organization: - datadog-staging openai-processing-ms: - - '523' + - '1256' openai-version: - '2020-10-01' strict-transport-security: @@ -264,7 +273,7 @@ interactions: x-ratelimit-reset-tokens: - 0s x-request-id: - - req_07733e2c20ff88f138f2ab4cd6a71cc6 + - req_a58af2c6e743ac15ac528fb6233d9436 status: code: 200 message: OK diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_submits_evaluation.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_submits_evaluation.yaml similarity index 100% rename from tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_submits_evaluation.yaml rename to tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_submits_evaluation.yaml diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_submits_evaluation_on_span_with_custom_keys.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_submits_evaluation_on_span_with_custom_keys.yaml similarity index 100% rename from tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_submits_evaluation_on_span_with_custom_keys.yaml rename to tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_submits_evaluation_on_span_with_custom_keys.yaml diff --git a/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_submits_evaluation_on_span_with_question_in_messages.yaml b/tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_submits_evaluation_on_span_with_question_in_messages.yaml similarity index 100% rename from tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.test_ragas_faithfulness_submits_evaluation_on_span_with_question_in_messages.yaml rename to tests/llmobs/llmobs_cassettes/tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_faithfulness_submits_evaluation_on_span_with_question_in_messages.yaml diff --git a/tests/llmobs/test_llmobs.py b/tests/llmobs/test_llmobs.py index 1bae7efe9ed..6cf19fc3e2c 100644 --- a/tests/llmobs/test_llmobs.py +++ b/tests/llmobs/test_llmobs.py @@ -1,4 +1,3 @@ -import mock import pytest from ddtrace.ext import SpanTypes @@ -8,12 +7,6 @@ from tests.llmobs._utils import _expected_llmobs_llm_span_event -@pytest.fixture -def mock_logs(): - with mock.patch("ddtrace.llmobs._trace_processor.log") as mock_logs: - yield mock_logs - - class TestMLApp: @pytest.mark.parametrize("llmobs_env", [{"DD_LLMOBS_ML_APP": ""}]) def test_tag_defaults_to_env_var(self, tracer, llmobs_env, llmobs_events): @@ -228,19 +221,19 @@ def test_model_and_provider_are_set(tracer, llmobs_events): assert span_event["meta"]["model_provider"] == "model_provider" -def test_malformed_span_logs_error_instead_of_raising(mock_logs, tracer, llmobs_events): +def test_malformed_span_logs_error_instead_of_raising(tracer, llmobs_events, mock_llmobs_logs): """Test that a trying to create a span event from a malformed span will log an error instead of crashing.""" with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: # span does not have SPAN_KIND tag pass - mock_logs.error.assert_called_once_with( - "Error generating LLMObs span event for span %s, likely due to malformed span", llm_span + mock_llmobs_logs.error.assert_called_with( + "Error generating LLMObs span event for span %s, likely due to malformed span", llm_span, exc_info=True ) assert len(llmobs_events) == 0 -def test_processor_only_creates_llmobs_span_event(tracer, llmobs_events): - """Test that the LLMObsTraceProcessor only creates LLMObs span events for LLM span types.""" +def test_only_generate_span_events_from_llmobs_spans(tracer, llmobs_events): + """Test that we only generate LLMObs span events for LLM span types.""" with tracer.trace("root_llm_span", service="tests.llmobs", span_type=SpanTypes.LLM) as root_span: root_span._set_ctx_item(const.SPAN_KIND, "llm") with tracer.trace("child_span"): @@ -250,5 +243,5 @@ def test_processor_only_creates_llmobs_span_event(tracer, llmobs_events): expected_grandchild_llmobs_span["parent_id"] = str(root_span.span_id) assert len(llmobs_events) == 2 - assert llmobs_events[0] == _expected_llmobs_llm_span_event(root_span, "llm") - assert llmobs_events[1] == expected_grandchild_llmobs_span + assert llmobs_events[1] == _expected_llmobs_llm_span_event(root_span, "llm") + assert llmobs_events[0] == expected_grandchild_llmobs_span diff --git a/tests/llmobs/test_llmobs_decorators.py b/tests/llmobs/test_llmobs_decorators.py index e94d72aec64..056de72ee96 100644 --- a/tests/llmobs/test_llmobs_decorators.py +++ b/tests/llmobs/test_llmobs_decorators.py @@ -19,7 +19,7 @@ def mock_logs(): yield mock_logs -def test_llm_decorator_with_llmobs_disabled_logs_warning(LLMObs, mock_logs): +def test_llm_decorator_with_llmobs_disabled_logs_warning(llmobs, mock_logs): for decorator_name, decorator in (("llm", llm), ("embedding", embedding)): @decorator( @@ -28,13 +28,13 @@ def test_llm_decorator_with_llmobs_disabled_logs_warning(LLMObs, mock_logs): def f(): pass - LLMObs.disable() + llmobs.disable() f() mock_logs.warning.assert_called_with(SPAN_START_WHILE_DISABLED_WARNING) mock_logs.reset_mock() -def test_non_llm_decorator_with_llmobs_disabled_logs_warning(LLMObs, mock_logs): +def test_non_llm_decorator_with_llmobs_disabled_logs_warning(llmobs, mock_logs): for decorator_name, decorator in ( ("task", task), ("workflow", workflow), @@ -47,53 +47,49 @@ def test_non_llm_decorator_with_llmobs_disabled_logs_warning(LLMObs, mock_logs): def f(): pass - LLMObs.disable() + llmobs.disable() f() mock_logs.warning.assert_called_with(SPAN_START_WHILE_DISABLED_WARNING) mock_logs.reset_mock() -def test_llm_decorator(LLMObs, mock_llmobs_span_writer): +def test_llm_decorator(llmobs, llmobs_events): @llm(model_name="test_model", model_provider="test_provider", name="test_function", session_id="test_session_id") def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, "llm", model_name="test_model", model_provider="test_provider", session_id="test_session_id" - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_llm_span_event( + span, "llm", model_name="test_model", model_provider="test_provider", session_id="test_session_id" ) -def test_llm_decorator_no_model_name_sets_default(LLMObs, mock_llmobs_span_writer): +def test_llm_decorator_no_model_name_sets_default(llmobs, llmobs_events): @llm(model_provider="test_provider", name="test_function", session_id="test_session_id") def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, "llm", model_name="custom", model_provider="test_provider", session_id="test_session_id" - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_llm_span_event( + span, "llm", model_name="custom", model_provider="test_provider", session_id="test_session_id" ) -def test_llm_decorator_default_kwargs(LLMObs, mock_llmobs_span_writer): +def test_llm_decorator_default_kwargs(llmobs, llmobs_events): @llm def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event(span, "llm", model_name="custom", model_provider="custom") + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_llm_span_event( + span, "llm", model_name="custom", model_provider="custom" ) -def test_embedding_decorator(LLMObs, mock_llmobs_span_writer): +def test_embedding_decorator(llmobs, llmobs_events): @embedding( model_name="test_model", model_provider="test_provider", name="test_function", session_id="test_session_id" ) @@ -101,173 +97,157 @@ def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, "embedding", model_name="test_model", model_provider="test_provider", session_id="test_session_id" - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_llm_span_event( + span, "embedding", model_name="test_model", model_provider="test_provider", session_id="test_session_id" ) -def test_embedding_decorator_no_model_name_sets_default(LLMObs, mock_llmobs_span_writer): +def test_embedding_decorator_no_model_name_sets_default(llmobs, llmobs_events): @embedding(model_provider="test_provider", name="test_function", session_id="test_session_id") def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, "embedding", model_name="custom", model_provider="test_provider", session_id="test_session_id" - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_llm_span_event( + span, "embedding", model_name="custom", model_provider="test_provider", session_id="test_session_id" ) -def test_embedding_decorator_default_kwargs(LLMObs, mock_llmobs_span_writer): +def test_embedding_decorator_default_kwargs(llmobs, llmobs_events): @embedding def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event(span, "embedding", model_name="custom", model_provider="custom") + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_llm_span_event( + span, "embedding", model_name="custom", model_provider="custom" ) -def test_retrieval_decorator(LLMObs, mock_llmobs_span_writer): +def test_retrieval_decorator(llmobs, llmobs_events): @retrieval(name="test_function", session_id="test_session_id") def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event(span, "retrieval", session_id="test_session_id") - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event(span, "retrieval", session_id="test_session_id") -def test_retrieval_decorator_default_kwargs(LLMObs, mock_llmobs_span_writer): +def test_retrieval_decorator_default_kwargs(llmobs, llmobs_events): @retrieval() def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with(_expected_llmobs_non_llm_span_event(span, "retrieval")) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event(span, "retrieval") -def test_task_decorator(LLMObs, mock_llmobs_span_writer): +def test_task_decorator(llmobs, llmobs_events): @task(name="test_function", session_id="test_session_id") def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event(span, "task", session_id="test_session_id") - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event(span, "task", session_id="test_session_id") -def test_task_decorator_default_kwargs(LLMObs, mock_llmobs_span_writer): +def test_task_decorator_default_kwargs(llmobs, llmobs_events): @task() def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with(_expected_llmobs_non_llm_span_event(span, "task")) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event(span, "task") -def test_tool_decorator(LLMObs, mock_llmobs_span_writer): +def test_tool_decorator(llmobs, llmobs_events): @tool(name="test_function", session_id="test_session_id") def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event(span, "tool", session_id="test_session_id") - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event(span, "tool", session_id="test_session_id") -def test_tool_decorator_default_kwargs(LLMObs, mock_llmobs_span_writer): +def test_tool_decorator_default_kwargs(llmobs, llmobs_events): @tool() def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with(_expected_llmobs_non_llm_span_event(span, "tool")) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event(span, "tool") -def test_workflow_decorator(LLMObs, mock_llmobs_span_writer): +def test_workflow_decorator(llmobs, llmobs_events): @workflow(name="test_function", session_id="test_session_id") def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event(span, "workflow", session_id="test_session_id") - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event(span, "workflow", session_id="test_session_id") -def test_workflow_decorator_default_kwargs(LLMObs, mock_llmobs_span_writer): +def test_workflow_decorator_default_kwargs(llmobs, llmobs_events): @workflow() def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with(_expected_llmobs_non_llm_span_event(span, "workflow")) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event(span, "workflow") -def test_agent_decorator(LLMObs, mock_llmobs_span_writer): +def test_agent_decorator(llmobs, llmobs_events): @agent(name="test_function", session_id="test_session_id") def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event(span, "agent", session_id="test_session_id") - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_llm_span_event(span, "agent", session_id="test_session_id") -def test_agent_decorator_default_kwargs(LLMObs, mock_llmobs_span_writer): +def test_agent_decorator_default_kwargs(llmobs, llmobs_events): @agent() def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with(_expected_llmobs_llm_span_event(span, "agent")) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_llm_span_event(span, "agent") -def test_llm_decorator_with_error(LLMObs, mock_llmobs_span_writer): +def test_llm_decorator_with_error(llmobs, llmobs_events): @llm(model_name="test_model", model_provider="test_provider", name="test_function", session_id="test_session_id") def f(): raise ValueError("test_error") with pytest.raises(ValueError): f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, - "llm", - model_name="test_model", - model_provider="test_provider", - session_id="test_session_id", - error=span.get_tag("error.type"), - error_message=span.get_tag("error.message"), - error_stack=span.get_tag("error.stack"), - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_llm_span_event( + span, + "llm", + model_name="test_model", + model_provider="test_provider", + session_id="test_session_id", + error=span.get_tag("error.type"), + error_message=span.get_tag("error.message"), + error_stack=span.get_tag("error.stack"), ) -def test_non_llm_decorators_with_error(LLMObs, mock_llmobs_span_writer): +def test_non_llm_decorators_with_error(llmobs, llmobs_events): for decorator_name, decorator in [("task", task), ("workflow", workflow), ("tool", tool), ("agent", agent)]: @decorator(name="test_function", session_id="test_session_id") @@ -276,23 +256,21 @@ def f(): with pytest.raises(ValueError): f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event( - span, - decorator_name, - session_id="test_session_id", - error=span.get_tag("error.type"), - error_message=span.get_tag("error.message"), - error_stack=span.get_tag("error.stack"), - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[-1] == _expected_llmobs_non_llm_span_event( + span, + decorator_name, + session_id="test_session_id", + error=span.get_tag("error.type"), + error_message=span.get_tag("error.message"), + error_stack=span.get_tag("error.stack"), ) -def test_llm_annotate(LLMObs, mock_llmobs_span_writer): +def test_llm_annotate(llmobs, llmobs_events): @llm(model_name="test_model", model_provider="test_provider", name="test_function", session_id="test_session_id") def f(): - LLMObs.annotate( + llmobs.annotate( parameters={"temperature": 0.9, "max_tokens": 50}, input_data=[{"content": "test_prompt"}], output_data=[{"content": "test_response"}], @@ -301,27 +279,25 @@ def f(): ) f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, - "llm", - model_name="test_model", - model_provider="test_provider", - input_messages=[{"content": "test_prompt"}], - output_messages=[{"content": "test_response"}], - parameters={"temperature": 0.9, "max_tokens": 50}, - token_metrics={"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}, - tags={"custom_tag": "tag_value"}, - session_id="test_session_id", - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_llm_span_event( + span, + "llm", + model_name="test_model", + model_provider="test_provider", + input_messages=[{"content": "test_prompt"}], + output_messages=[{"content": "test_response"}], + parameters={"temperature": 0.9, "max_tokens": 50}, + token_metrics={"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}, + tags={"custom_tag": "tag_value"}, + session_id="test_session_id", ) -def test_llm_annotate_raw_string_io(LLMObs, mock_llmobs_span_writer): +def test_llm_annotate_raw_string_io(llmobs, llmobs_events): @llm(model_name="test_model", model_provider="test_provider", name="test_function", session_id="test_session_id") def f(): - LLMObs.annotate( + llmobs.annotate( parameters={"temperature": 0.9, "max_tokens": 50}, input_data="test_prompt", output_data="test_response", @@ -330,24 +306,22 @@ def f(): ) f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, - "llm", - model_name="test_model", - model_provider="test_provider", - input_messages=[{"content": "test_prompt"}], - output_messages=[{"content": "test_response"}], - parameters={"temperature": 0.9, "max_tokens": 50}, - token_metrics={"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}, - tags={"custom_tag": "tag_value"}, - session_id="test_session_id", - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_llm_span_event( + span, + "llm", + model_name="test_model", + model_provider="test_provider", + input_messages=[{"content": "test_prompt"}], + output_messages=[{"content": "test_response"}], + parameters={"temperature": 0.9, "max_tokens": 50}, + token_metrics={"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}, + tags={"custom_tag": "tag_value"}, + session_id="test_session_id", ) -def test_non_llm_decorators_no_args(LLMObs, mock_llmobs_span_writer): +def test_non_llm_decorators_no_args(llmobs, llmobs_events): """Test that using the decorators without any arguments, i.e. @tool, works the same as @tool(...).""" for decorator_name, decorator in [ ("task", task), @@ -362,11 +336,11 @@ def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with(_expected_llmobs_non_llm_span_event(span, decorator_name)) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[-1] == _expected_llmobs_non_llm_span_event(span, decorator_name) -def test_agent_decorator_no_args(LLMObs, mock_llmobs_span_writer): +def test_agent_decorator_no_args(llmobs, llmobs_events): """Test that using agent decorator without any arguments, i.e. @agent, works the same as @agent(...).""" @agent @@ -374,11 +348,11 @@ def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with(_expected_llmobs_llm_span_event(span, "agent")) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_llm_span_event(span, "agent") -def test_ml_app_override(LLMObs, mock_llmobs_span_writer): +def test_ml_app_override(llmobs, llmobs_events): """Test that setting ml_app kwarg on the LLMObs decorators will override the DD_LLMOBS_ML_APP value.""" for decorator_name, decorator in [("task", task), ("workflow", workflow), ("tool", tool)]: @@ -387,9 +361,9 @@ def f(): pass f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event(span, decorator_name, tags={"ml_app": "test_ml_app"}) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[-1] == _expected_llmobs_non_llm_span_event( + span, decorator_name, tags={"ml_app": "test_ml_app"} ) @llm(model_name="test_model", ml_app="test_ml_app") @@ -397,11 +371,9 @@ def g(): pass g() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, "llm", model_name="test_model", model_provider="custom", tags={"ml_app": "test_ml_app"} - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[-1] == _expected_llmobs_llm_span_event( + span, "llm", model_name="test_model", model_provider="custom", tags={"ml_app": "test_ml_app"} ) @embedding(model_name="test_model", ml_app="test_ml_app") @@ -409,15 +381,13 @@ def h(): pass h() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, "embedding", model_name="test_model", model_provider="custom", tags={"ml_app": "test_ml_app"} - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[-1] == _expected_llmobs_llm_span_event( + span, "embedding", model_name="test_model", model_provider="custom", tags={"ml_app": "test_ml_app"} ) -async def test_non_llm_async_decorators(LLMObs, mock_llmobs_span_writer): +async def test_non_llm_async_decorators(llmobs, llmobs_events): """Test that decorators work with async functions.""" for decorator_name, decorator in [ ("task", task), @@ -432,11 +402,11 @@ async def f(): pass await f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with(_expected_llmobs_non_llm_span_event(span, decorator_name)) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[-1] == _expected_llmobs_non_llm_span_event(span, decorator_name) -async def test_llm_async_decorators(LLMObs, mock_llmobs_span_writer): +async def test_llm_async_decorators(llmobs, llmobs_events): """Test that decorators work with async functions.""" for decorator_name, decorator in [("llm", llm), ("embedding", embedding)]: @@ -445,15 +415,13 @@ async def f(): pass await f() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, decorator_name, model_name="test_model", model_provider="test_provider" - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[-1] == _expected_llmobs_llm_span_event( + span, decorator_name, model_name="test_model", model_provider="test_provider" ) -def test_automatic_annotation_non_llm_decorators(LLMObs, mock_llmobs_span_writer): +def test_automatic_annotation_non_llm_decorators(llmobs, llmobs_events): """Test that automatic input/output annotation works for non-LLM decorators.""" for decorator_name, decorator in (("task", task), ("workflow", workflow), ("tool", tool), ("agent", agent)): @@ -462,19 +430,17 @@ def f(prompt, arg_2, kwarg_1=None, kwarg_2=None): return prompt f("test_prompt", "arg_2", kwarg_2=12345) - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event( - span, - decorator_name, - input_value=str({"prompt": "test_prompt", "arg_2": "arg_2", "kwarg_2": 12345}), - output_value="test_prompt", - session_id="test_session_id", - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[-1] == _expected_llmobs_non_llm_span_event( + span, + decorator_name, + input_value=str({"prompt": "test_prompt", "arg_2": "arg_2", "kwarg_2": 12345}), + output_value="test_prompt", + session_id="test_session_id", ) -def test_automatic_annotation_retrieval_decorator(LLMObs, mock_llmobs_span_writer): +def test_automatic_annotation_retrieval_decorator(llmobs, llmobs_events): """Test that automatic input annotation works for retrieval decorators.""" @retrieval(session_id="test_session_id") @@ -482,18 +448,16 @@ def test_retrieval(query, arg_2, kwarg_1=None, kwarg_2=None): return [{"name": "name", "id": "1234567890", "score": 0.9}] test_retrieval("test_query", "arg_2", kwarg_2=12345) - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event( - span, - "retrieval", - input_value=str({"query": "test_query", "arg_2": "arg_2", "kwarg_2": 12345}), - session_id="test_session_id", - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event( + span, + "retrieval", + input_value=str({"query": "test_query", "arg_2": "arg_2", "kwarg_2": 12345}), + session_id="test_session_id", ) -def test_automatic_annotation_off_non_llm_decorators(LLMObs, mock_llmobs_span_writer): +def test_automatic_annotation_off_non_llm_decorators(llmobs, llmobs_events): """Test disabling automatic input/output annotation for non-LLM decorators.""" for decorator_name, decorator in ( ("task", task), @@ -508,35 +472,33 @@ def f(prompt, arg_2, kwarg_1=None, kwarg_2=None): return prompt f("test_prompt", "arg_2", kwarg_2=12345) - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event(span, decorator_name, session_id="test_session_id") + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[-1] == _expected_llmobs_non_llm_span_event( + span, decorator_name, session_id="test_session_id" ) -def test_automatic_annotation_off_if_manually_annotated(LLMObs, mock_llmobs_span_writer): +def test_automatic_annotation_off_if_manually_annotated(llmobs, llmobs_events): """Test disabling automatic input/output annotation for non-LLM decorators.""" for decorator_name, decorator in (("task", task), ("workflow", workflow), ("tool", tool), ("agent", agent)): @decorator(name="test_function", session_id="test_session_id") def f(prompt, arg_2, kwarg_1=None, kwarg_2=None): - LLMObs.annotate(input_data="my custom input", output_data="my custom output") + llmobs.annotate(input_data="my custom input", output_data="my custom output") return prompt f("test_prompt", "arg_2", kwarg_2=12345) - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event( - span, - decorator_name, - session_id="test_session_id", - input_value="my custom input", - output_value="my custom output", - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[-1] == _expected_llmobs_non_llm_span_event( + span, + decorator_name, + session_id="test_session_id", + input_value="my custom input", + output_value="my custom output", ) -def test_generator_sync(LLMObs, mock_llmobs_span_writer): +def test_generator_sync(llmobs, llmobs_events): """ Test that decorators work with generator functions. The span should finish after the generator is exhausted. @@ -556,7 +518,7 @@ def f(): for i in range(3): yield i - LLMObs.annotate( + llmobs.annotate( input_data="hello", output_data="world", ) @@ -566,7 +528,7 @@ def f(): assert e == i i += 1 - span = LLMObs._instance.tracer.pop()[0] + span = llmobs._instance.tracer.pop()[0] if decorator_name == "llm": expected_span_event = _expected_llmobs_llm_span_event( span, @@ -594,10 +556,10 @@ def f(): span, decorator_name, input_value="hello", output_value="world" ) - mock_llmobs_span_writer.enqueue.assert_called_with(expected_span_event) + assert llmobs_events[-1] == expected_span_event -async def test_generator_async(LLMObs, mock_llmobs_span_writer): +async def test_generator_async(llmobs, llmobs_events): """ Test that decorators work with generator functions. The span should finish after the generator is exhausted. @@ -617,7 +579,7 @@ async def f(): for i in range(3): yield i - LLMObs.annotate( + llmobs.annotate( input_data="hello", output_data="world", ) @@ -627,7 +589,7 @@ async def f(): assert e == i i += 1 - span = LLMObs._instance.tracer.pop()[0] + span = llmobs._instance.tracer.pop()[0] if decorator_name == "llm": expected_span_event = _expected_llmobs_llm_span_event( span, @@ -655,11 +617,11 @@ async def f(): span, decorator_name, input_value="hello", output_value="world" ) - mock_llmobs_span_writer.enqueue.assert_called_with(expected_span_event) + assert llmobs_events[-1] == expected_span_event -def test_generator_sync_with_llmobs_disabled(LLMObs, mock_logs): - LLMObs.disable() +def test_generator_sync_with_llmobs_disabled(llmobs, mock_logs): + llmobs.disable() @workflow() def f(): @@ -684,10 +646,11 @@ def g(): i += 1 mock_logs.warning.assert_called_with(SPAN_START_WHILE_DISABLED_WARNING) + llmobs.enable() -async def test_generator_async_with_llmobs_disabled(LLMObs, mock_logs): - LLMObs.disable() +async def test_generator_async_with_llmobs_disabled(llmobs, mock_logs): + llmobs.disable() @workflow() async def f(): @@ -712,9 +675,10 @@ async def g(): i += 1 mock_logs.warning.assert_called_with(SPAN_START_WHILE_DISABLED_WARNING) + llmobs.enable() -def test_generator_sync_finishes_span_on_error(LLMObs, mock_llmobs_span_writer): +def test_generator_sync_finishes_span_on_error(llmobs, llmobs_events): """Tests that""" @workflow() @@ -728,19 +692,17 @@ def f(): for _ in f(): pass - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event( - span, - "workflow", - error=span.get_tag("error.type"), - error_message=span.get_tag("error.message"), - error_stack=span.get_tag("error.stack"), - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event( + span, + "workflow", + error=span.get_tag("error.type"), + error_message=span.get_tag("error.message"), + error_stack=span.get_tag("error.stack"), ) -async def test_generator_async_finishes_span_on_error(LLMObs, mock_llmobs_span_writer): +async def test_generator_async_finishes_span_on_error(llmobs, llmobs_events): @workflow() async def f(): for i in range(3): @@ -752,19 +714,17 @@ async def f(): async for _ in f(): pass - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event( - span, - "workflow", - error=span.get_tag("error.type"), - error_message=span.get_tag("error.message"), - error_stack=span.get_tag("error.stack"), - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event( + span, + "workflow", + error=span.get_tag("error.type"), + error_message=span.get_tag("error.message"), + error_stack=span.get_tag("error.stack"), ) -def test_generator_sync_send(LLMObs, mock_llmobs_span_writer): +def test_generator_sync_send(llmobs, llmobs_events): @workflow() def f(): while True: @@ -780,16 +740,11 @@ def f(): assert gen.send(4) == 16 gen.close() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event( - span, - "workflow", - ) - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event(span, "workflow") -async def test_generator_async_send(LLMObs, mock_llmobs_span_writer): +async def test_generator_async_send(llmobs, llmobs_events): @workflow() async def f(): while True: @@ -805,16 +760,11 @@ async def f(): await gen.aclose() - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event( - span, - "workflow", - ) - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event(span, "workflow") -def test_generator_sync_throw(LLMObs, mock_llmobs_span_writer): +def test_generator_sync_throw(llmobs, llmobs_events): @workflow() def f(): for i in range(3): @@ -825,19 +775,17 @@ def f(): next(gen) gen.throw(ValueError("test_error")) - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event( - span, - "workflow", - error=span.get_tag("error.type"), - error_message=span.get_tag("error.message"), - error_stack=span.get_tag("error.stack"), - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event( + span, + "workflow", + error=span.get_tag("error.type"), + error_message=span.get_tag("error.message"), + error_stack=span.get_tag("error.stack"), ) -async def test_generator_async_throw(LLMObs, mock_llmobs_span_writer): +async def test_generator_async_throw(llmobs, llmobs_events): @workflow() async def f(): for i in range(3): @@ -848,19 +796,17 @@ async def f(): await gen.asend(None) await gen.athrow(ValueError("test_error")) - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event( - span, - "workflow", - error=span.get_tag("error.type"), - error_message=span.get_tag("error.message"), - error_stack=span.get_tag("error.stack"), - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event( + span, + "workflow", + error=span.get_tag("error.type"), + error_message=span.get_tag("error.message"), + error_stack=span.get_tag("error.stack"), ) -def test_generator_exit_exception_sync(LLMObs, mock_llmobs_span_writer): +def test_generator_exit_exception_sync(llmobs, llmobs_events): @workflow() def get_next_element(alist): for element in alist: @@ -873,14 +819,12 @@ def get_next_element(alist): if element == 5: break - span = LLMObs._instance.tracer.pop()[0] - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event( - span, - "workflow", - input_value=str({"alist": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}), - error=span.get_tag("error.type"), - error_message=span.get_tag("error.message"), - error_stack=span.get_tag("error.stack"), - ) + span = llmobs._instance.tracer.pop()[0] + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event( + span, + "workflow", + input_value=str({"alist": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}), + error=span.get_tag("error.type"), + error_message=span.get_tag("error.message"), + error_stack=span.get_tag("error.stack"), ) diff --git a/tests/llmobs/test_llmobs_eval_metric_writer.py b/tests/llmobs/test_llmobs_eval_metric_writer.py index 2b8341e1616..491d7912420 100644 --- a/tests/llmobs/test_llmobs_eval_metric_writer.py +++ b/tests/llmobs/test_llmobs_eval_metric_writer.py @@ -7,15 +7,19 @@ from ddtrace.llmobs._writer import LLMObsEvalMetricWriter -INTAKE_ENDPOINT = "https://api.datad0g.com/api/intake/llm-obs/v1/eval-metric" +INTAKE_ENDPOINT = "https://api.datad0g.com/api/intake/llm-obs/v2/eval-metric" DD_SITE = "datad0g.com" dd_api_key = os.getenv("DD_API_KEY", default="") def _categorical_metric_event(): return { - "span_id": "12345678901", - "trace_id": "98765432101", + "join_on": { + "span": { + "span_id": "12345678901", + "trace_id": "98765432101", + }, + }, "metric_type": "categorical", "categorical_value": "very", "label": "toxicity", @@ -26,8 +30,12 @@ def _categorical_metric_event(): def _score_metric_event(): return { - "span_id": "12345678902", - "trace_id": "98765432102", + "join_on": { + "span": { + "span_id": "12345678902", + "trace_id": "98765432102", + }, + }, "metric_type": "score", "label": "sentiment", "score_value": 0.9, @@ -69,6 +77,18 @@ def test_send_metric_bad_api_key(mock_writer_logs): ) +@pytest.mark.vcr_logs +def test_send_metric_no_api_key(mock_writer_logs): + llmobs_eval_metric_writer = LLMObsEvalMetricWriter(site="datad0g.com", api_key="", interval=1000, timeout=1) + llmobs_eval_metric_writer.start() + llmobs_eval_metric_writer.enqueue(_categorical_metric_event()) + llmobs_eval_metric_writer.periodic() + mock_writer_logs.warning.assert_called_with( + "DD_API_KEY is required for sending evaluation metrics. Evaluation metric data will not be sent. ", + "Ensure this configuration is set before running your application.", + ) + + @pytest.mark.vcr_logs def test_send_categorical_metric(mock_writer_logs): llmobs_eval_metric_writer = LLMObsEvalMetricWriter(site="datad0g.com", api_key=dd_api_key, interval=1000, timeout=1) @@ -125,6 +145,18 @@ def test_send_multiple_events(mock_writer_logs): def test_send_on_exit(mock_writer_logs, run_python_code_in_subprocess): + env = os.environ.copy() + pypath = [os.path.dirname(os.path.dirname(os.path.dirname(__file__)))] + if "PYTHONPATH" in env: + pypath.append(env["PYTHONPATH"]) + env.update( + { + "DD_API_KEY": os.getenv("DD_API_KEY", "dummy-api-key"), + "DD_SITE": "datad0g.com", + "PYTHONPATH": ":".join(pypath), + "DD_LLMOBS_ML_APP": "unnamed-ml-app", + } + ) out, err, status, pid = run_python_code_in_subprocess( """ import atexit @@ -144,6 +176,7 @@ def test_send_on_exit(mock_writer_logs, run_python_code_in_subprocess): llmobs_eval_metric_writer.start() llmobs_eval_metric_writer.enqueue(_score_metric_event()) """, + env=env, ) assert status == 0, err assert out == b"" diff --git a/tests/llmobs/test_llmobs_evaluator_runner.py b/tests/llmobs/test_llmobs_evaluator_runner.py index 7ee7d510276..eaf381367d0 100644 --- a/tests/llmobs/test_llmobs_evaluator_runner.py +++ b/tests/llmobs/test_llmobs_evaluator_runner.py @@ -22,7 +22,7 @@ def test_evaluator_runner_start(mock_evaluator_logs): evaluator_runner = EvaluatorRunner(interval=0.01, llmobs_service=mock.MagicMock()) evaluator_runner.evaluators.append(DummyEvaluator(llmobs_service=mock.MagicMock())) evaluator_runner.start() - mock_evaluator_logs.debug.assert_has_calls([mock.call("started %r to %r", "EvaluatorRunner")]) + mock_evaluator_logs.debug.assert_has_calls([mock.call("started %r", "EvaluatorRunner")]) def test_evaluator_runner_buffer_limit(mock_evaluator_logs): @@ -34,9 +34,9 @@ def test_evaluator_runner_buffer_limit(mock_evaluator_logs): ) -def test_evaluator_runner_periodic_enqueues_eval_metric(LLMObs, mock_llmobs_eval_metric_writer): - evaluator_runner = EvaluatorRunner(interval=0.01, llmobs_service=LLMObs) - evaluator_runner.evaluators.append(DummyEvaluator(llmobs_service=LLMObs)) +def test_evaluator_runner_periodic_enqueues_eval_metric(llmobs, mock_llmobs_eval_metric_writer): + evaluator_runner = EvaluatorRunner(interval=0.01, llmobs_service=llmobs) + evaluator_runner.evaluators.append(DummyEvaluator(llmobs_service=llmobs)) evaluator_runner.enqueue({"span_id": "123", "trace_id": "1234"}, DUMMY_SPAN) evaluator_runner.periodic() mock_llmobs_eval_metric_writer.enqueue.assert_called_once_with( @@ -45,9 +45,9 @@ def test_evaluator_runner_periodic_enqueues_eval_metric(LLMObs, mock_llmobs_eval @pytest.mark.vcr_logs -def test_evaluator_runner_timed_enqueues_eval_metric(LLMObs, mock_llmobs_eval_metric_writer): - evaluator_runner = EvaluatorRunner(interval=0.01, llmobs_service=LLMObs) - evaluator_runner.evaluators.append(DummyEvaluator(llmobs_service=LLMObs)) +def test_evaluator_runner_timed_enqueues_eval_metric(llmobs, mock_llmobs_eval_metric_writer): + evaluator_runner = EvaluatorRunner(interval=0.01, llmobs_service=llmobs) + evaluator_runner.evaluators.append(DummyEvaluator(llmobs_service=llmobs)) evaluator_runner.start() evaluator_runner.enqueue({"span_id": "123", "trace_id": "1234"}, DUMMY_SPAN) @@ -59,20 +59,35 @@ def test_evaluator_runner_timed_enqueues_eval_metric(LLMObs, mock_llmobs_eval_me ) +@pytest.mark.vcr_logs +def test_evaluator_runner_multiple_evaluators(llmobs, mock_llmobs_eval_metric_writer): + evaluator_runner = EvaluatorRunner(interval=0.01, llmobs_service=llmobs) + evaluator_runner.evaluators += [ + DummyEvaluator(llmobs_service=llmobs, label="1"), + DummyEvaluator(llmobs_service=llmobs, label="2"), + DummyEvaluator(llmobs_service=llmobs, label="3"), + ] + evaluator_runner.start() + + evaluator_runner.enqueue({"span_id": "123", "trace_id": "1234"}, DUMMY_SPAN) + + time.sleep(0.1) + + calls = [call[0][0] for call in mock_llmobs_eval_metric_writer.enqueue.call_args_list] + sorted_calls = sorted(calls, key=lambda x: x["label"]) + assert sorted_calls == [ + _dummy_evaluator_eval_metric_event(span_id="123", trace_id="1234", label="1"), + _dummy_evaluator_eval_metric_event(span_id="123", trace_id="1234", label="2"), + _dummy_evaluator_eval_metric_event(span_id="123", trace_id="1234", label="3"), + ] + + def test_evaluator_runner_on_exit(mock_writer_logs, run_python_code_in_subprocess): env = os.environ.copy() - pypath = [os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))] + pypath = [os.path.dirname(os.path.dirname(os.path.dirname(__file__)))] if "PYTHONPATH" in env: pypath.append(env["PYTHONPATH"]) - env.update( - { - "DD_API_KEY": os.getenv("DD_API_KEY", "dummy-api-key"), - "DD_SITE": "datad0g.com", - "PYTHONPATH": ":".join(pypath), - "DD_LLMOBS_ML_APP": "unnamed-ml-app", - "_DD_LLMOBS_EVALUATOR_INTERVAL": "5", - } - ) + env.update({"PYTHONPATH": ":".join(pypath), "_DD_LLMOBS_EVALUATOR_INTERVAL": "5"}) out, err, status, pid = run_python_code_in_subprocess( """ import os @@ -87,7 +102,7 @@ def test_evaluator_runner_on_exit(mock_writer_logs, run_python_code_in_subproces ctx = logs_vcr.use_cassette("tests.llmobs.test_llmobs_evaluator_runner.send_score_metric.yaml") ctx.__enter__() atexit.register(lambda: ctx.__exit__()) -LLMObs.enable() +LLMObs.enable(api_key="dummy-api-key", site="datad0g.com", ml_app="unnamed-ml-app") LLMObs._instance._evaluator_runner.evaluators.append(DummyEvaluator(llmobs_service=LLMObs)) LLMObs._instance._evaluator_runner.start() LLMObs._instance._evaluator_runner.enqueue({"span_id": "123", "trace_id": "1234"}, None) @@ -99,6 +114,12 @@ def test_evaluator_runner_on_exit(mock_writer_logs, run_python_code_in_subproces assert err == b"" +def test_evaluator_runner_unsupported_evaluator(): + with override_env({"_DD_LLMOBS_EVALUATORS": "unsupported"}): + with pytest.raises(ValueError): + EvaluatorRunner(interval=0.01, llmobs_service=mock.MagicMock()) + + def test_evaluator_runner_sampler_single_rule(monkeypatch): monkeypatch.setenv( EvaluatorRunnerSampler.SAMPLING_RULES_ENV_VAR, diff --git a/tests/llmobs/test_llmobs_ragas_evaluators.py b/tests/llmobs/test_llmobs_ragas_evaluators.py new file mode 100644 index 00000000000..cc02709baff --- /dev/null +++ b/tests/llmobs/test_llmobs_ragas_evaluators.py @@ -0,0 +1,622 @@ +import os + +import mock +import pytest + +from ddtrace.llmobs._evaluators.ragas.answer_relevancy import RagasAnswerRelevancyEvaluator +from ddtrace.llmobs._evaluators.ragas.context_precision import RagasContextPrecisionEvaluator +from ddtrace.llmobs._evaluators.ragas.faithfulness import RagasFaithfulnessEvaluator +from ddtrace.span import Span +from tests.llmobs._utils import _expected_llmobs_llm_span_event +from tests.llmobs._utils import _expected_ragas_answer_relevancy_spans +from tests.llmobs._utils import _expected_ragas_context_precision_spans +from tests.llmobs._utils import _expected_ragas_faithfulness_spans +from tests.llmobs._utils import _llm_span_with_expected_ragas_inputs_in_messages +from tests.llmobs._utils import _llm_span_with_expected_ragas_inputs_in_prompt +from tests.llmobs._utils import default_ragas_inputs +from tests.llmobs._utils import logs_vcr + + +pytest.importorskip("ragas", reason="Tests require ragas to be available on user env") + +ragas_answer_relevancy_cassette = logs_vcr.use_cassette( + "tests.llmobs.test_llmobs_ragas_evaluators.answer_relevancy_inference.yaml" +) + +ragas_context_precision_single_context_cassette = logs_vcr.use_cassette( + "tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_single_context.yaml" +) +ragas_context_precision_multiple_context_cassette = logs_vcr.use_cassette( + "tests.llmobs.test_llmobs_ragas_evaluators.test_ragas_context_precision_multiple_context.yaml" +) + + +@pytest.fixture +def reset_ragas_context_precision_llm(): + import ragas + + previous_llm = ragas.metrics.context_precision.llm + yield + ragas.metrics.context_precision.llm = previous_llm + + +def _llm_span_without_io(): + return _expected_llmobs_llm_span_event(Span("dummy")) + + +def test_ragas_evaluator_init(ragas, llmobs): + rf_evaluator = RagasFaithfulnessEvaluator(llmobs) + assert rf_evaluator.llmobs_service == llmobs + assert rf_evaluator.ragas_faithfulness_instance == ragas.metrics.faithfulness + assert rf_evaluator.ragas_faithfulness_instance.llm == ragas.llms.llm_factory() + + +def test_ragas_faithfulness_throws_if_dependencies_not_present(llmobs, mock_ragas_dependencies_not_present, ragas): + with pytest.raises(NotImplementedError, match="Failed to load dependencies for `ragas_faithfulness` evaluator"): + RagasFaithfulnessEvaluator(llmobs) + + +def test_ragas_faithfulness_returns_none_if_inputs_extraction_fails(ragas, mock_llmobs_submit_evaluation, llmobs): + rf_evaluator = RagasFaithfulnessEvaluator(llmobs) + failure_msg, _ = rf_evaluator.evaluate(_llm_span_without_io()) + assert failure_msg == "fail_extract_faithfulness_inputs" + assert rf_evaluator.llmobs_service.submit_evaluation.call_count == 0 + + +def test_ragas_faithfulness_has_modified_faithfulness_instance( + ragas, mock_llmobs_submit_evaluation, reset_ragas_faithfulness_llm, llmobs +): + """Faithfulness instance used in ragas evaluator should match the global ragas faithfulness instance""" + from ragas.llms import BaseRagasLLM + from ragas.metrics import faithfulness + + class FirstDummyLLM(BaseRagasLLM): + def __init__(self): + super().__init__() + + def generate_text(self) -> str: + return "dummy llm" + + def agenerate_text(self) -> str: + return "dummy llm" + + faithfulness.llm = FirstDummyLLM() + + rf_evaluator = RagasFaithfulnessEvaluator(llmobs) + + assert rf_evaluator.ragas_faithfulness_instance.llm.generate_text() == "dummy llm" + + class SecondDummyLLM(BaseRagasLLM): + def __init__(self): + super().__init__() + + def generate_text(self, statements) -> str: + raise ValueError("dummy_llm") + + def agenerate_text(self, statements) -> str: + raise ValueError("dummy_llm") + + faithfulness.llm = SecondDummyLLM() + + with pytest.raises(ValueError, match="dummy_llm"): + rf_evaluator.evaluate(_llm_span_with_expected_ragas_inputs_in_prompt()) + + +@pytest.mark.vcr_logs +def test_ragas_faithfulness_submits_evaluation(ragas, llmobs, mock_llmobs_submit_evaluation): + """Test that evaluation is submitted for a valid llm span where question is in the prompt variables""" + rf_evaluator = RagasFaithfulnessEvaluator(llmobs) + llm_span = _llm_span_with_expected_ragas_inputs_in_prompt() + rf_evaluator.run_and_submit_evaluation(llm_span) + rf_evaluator.llmobs_service.submit_evaluation.assert_has_calls( + [ + mock.call( + span_context={ + "span_id": llm_span.get("span_id"), + "trace_id": llm_span.get("trace_id"), + }, + label=RagasFaithfulnessEvaluator.LABEL, + metric_type=RagasFaithfulnessEvaluator.METRIC_TYPE, + value=1.0, + metadata={ + "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, + "_dd.faithfulness_disagreements": mock.ANY, + "_dd.evaluation_kind": "faithfulness", + }, + ) + ] + ) + + +@pytest.mark.vcr_logs +def test_ragas_faithfulness_submits_evaluation_on_span_with_question_in_messages( + ragas, llmobs, mock_llmobs_submit_evaluation +): + """Test that evaluation is submitted for a valid llm span where the last message content is the question""" + rf_evaluator = RagasFaithfulnessEvaluator(llmobs) + llm_span = _llm_span_with_expected_ragas_inputs_in_messages() + rf_evaluator.run_and_submit_evaluation(llm_span) + rf_evaluator.llmobs_service.submit_evaluation.assert_has_calls( + [ + mock.call( + span_context={ + "span_id": llm_span.get("span_id"), + "trace_id": llm_span.get("trace_id"), + }, + label=RagasFaithfulnessEvaluator.LABEL, + metric_type=RagasFaithfulnessEvaluator.METRIC_TYPE, + value=1.0, + metadata={ + "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, + "_dd.faithfulness_disagreements": mock.ANY, + "_dd.evaluation_kind": "faithfulness", + }, + ) + ] + ) + + +@pytest.mark.vcr_logs +def test_ragas_faithfulness_submits_evaluation_on_span_with_custom_keys(ragas, llmobs, mock_llmobs_submit_evaluation): + """Test that evaluation is submitted for a valid llm span where the last message content is the question""" + rf_evaluator = RagasFaithfulnessEvaluator(llmobs) + llm_span = _expected_llmobs_llm_span_event( + Span("dummy"), + prompt={ + "variables": { + "user_input": "Is france part of europe?", + "context_1": "hello, ", + "context_2": "france is ", + "context_3": "part of europe", + }, + "_dd_context_variable_keys": ["context_1", "context_2", "context_3"], + "_dd_query_variable_keys": ["user_input"], + }, + output_messages=[{"content": "France is indeed part of europe"}], + ) + rf_evaluator.run_and_submit_evaluation(llm_span) + rf_evaluator.llmobs_service.submit_evaluation.assert_has_calls( + [ + mock.call( + span_context={ + "span_id": llm_span.get("span_id"), + "trace_id": llm_span.get("trace_id"), + }, + label=RagasFaithfulnessEvaluator.LABEL, + metric_type=RagasFaithfulnessEvaluator.METRIC_TYPE, + value=1.0, + metadata={ + "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, + "_dd.faithfulness_disagreements": mock.ANY, + "_dd.evaluation_kind": "faithfulness", + }, + ) + ] + ) + + +@pytest.mark.vcr_logs +def test_ragas_faithfulness_emits_traces(ragas, llmobs, llmobs_events): + rf_evaluator = RagasFaithfulnessEvaluator(llmobs) + rf_evaluator.evaluate(_llm_span_with_expected_ragas_inputs_in_prompt()) + ragas_spans = [event for event in llmobs_events if event["name"].startswith("dd-ragas.")] + ragas_spans = sorted(ragas_spans, key=lambda d: d["start_ns"]) + assert len(ragas_spans) == 7 + # check name, io, span kinds match + assert ragas_spans == _expected_ragas_faithfulness_spans() + + # verify the trace structure + root_span = ragas_spans[0] + root_span_id = root_span["span_id"] + assert root_span["parent_id"] == "undefined" + assert root_span["meta"] is not None + assert root_span["meta"]["metadata"] is not None + assert isinstance(root_span["meta"]["metadata"]["faithfulness_list"], list) + assert isinstance(root_span["meta"]["metadata"]["statements"], list) + root_span_trace_id = root_span["trace_id"] + for child_span in ragas_spans[1:]: + assert child_span["trace_id"] == root_span_trace_id + + assert ragas_spans[1]["parent_id"] == root_span_id # input extraction (task) + assert ragas_spans[2]["parent_id"] == root_span_id # create statements (workflow) + assert ragas_spans[4]["parent_id"] == root_span_id # create verdicts (workflow) + assert ragas_spans[6]["parent_id"] == root_span_id # create score (task) + assert ragas_spans[3]["parent_id"] == ragas_spans[2]["span_id"] # create statements prompt (task) + assert ragas_spans[5]["parent_id"] == ragas_spans[4]["span_id"] # create verdicts prompt (task) + + +def test_llmobs_with_faithfulness_emits_traces_and_evals_on_exit(mock_writer_logs, run_python_code_in_subprocess): + env = os.environ.copy() + pypath = [os.path.dirname(os.path.dirname(os.path.dirname(__file__)))] + if "PYTHONPATH" in env: + pypath.append(env["PYTHONPATH"]) + env.update( + { + "PYTHONPATH": ":".join(pypath), + "OPENAI_API_KEY": os.getenv("OPENAI_API_KEY", "dummy-openai-api-key"), + "_DD_LLMOBS_EVALUATOR_INTERVAL": "5", + "_DD_LLMOBS_EVALUATORS": "ragas_faithfulness", + "DD_TRACE_ENABLED": "0", + } + ) + out, err, status, pid = run_python_code_in_subprocess( + """ +import os +import time +import atexit +import mock +from ddtrace.llmobs import LLMObs +from ddtrace.internal.utils.http import Response +from tests.llmobs._utils import _llm_span_with_expected_ragas_inputs_in_messages +from tests.llmobs._utils import logs_vcr + +ctx = logs_vcr.use_cassette( + "tests.llmobs.test_llmobs_ragas_evaluators.emits_traces_and_evaluations_on_exit.yaml" +) +ctx.__enter__() +atexit.register(lambda: ctx.__exit__()) +with mock.patch("ddtrace.internal.writer.HTTPWriter._send_payload", return_value=Response(status=200, body="{}")): + LLMObs.enable(api_key="dummy-api-key", site="datad0g.com", ml_app="unnamed-ml-app", agentless_enabled=True) + LLMObs._instance._evaluator_runner.enqueue(_llm_span_with_expected_ragas_inputs_in_messages(), None) + """, + env=env, + ) + assert status == 0, err + assert out == b"" + assert err == b"" + + +def test_ragas_context_precision_init(ragas, llmobs): + rcp_evaluator = RagasContextPrecisionEvaluator(llmobs) + assert rcp_evaluator.llmobs_service == llmobs + assert rcp_evaluator.ragas_context_precision_instance == ragas.metrics.context_precision + assert rcp_evaluator.ragas_context_precision_instance.llm == ragas.llms.llm_factory() + + +def test_ragas_context_precision_throws_if_dependencies_not_present(llmobs, mock_ragas_dependencies_not_present, ragas): + with pytest.raises( + NotImplementedError, match="Failed to load dependencies for `ragas_context_precision` evaluator" + ): + RagasContextPrecisionEvaluator(llmobs) + + +def test_ragas_context_precision_returns_none_if_inputs_extraction_fails(ragas, mock_llmobs_submit_evaluation, llmobs): + rcp_evaluator = RagasContextPrecisionEvaluator(llmobs) + failure_msg, _ = rcp_evaluator.evaluate(_llm_span_without_io()) + assert failure_msg == "fail_extract_context_precision_inputs" + assert rcp_evaluator.llmobs_service.submit_evaluation.call_count == 0 + + +def test_ragas_context_precision_has_modified_context_precision_instance( + ragas, mock_llmobs_submit_evaluation, reset_ragas_context_precision_llm, llmobs +): + """Context precision instance used in ragas evaluator should match the global ragas context precision instance""" + from ragas.llms import BaseRagasLLM + from ragas.metrics import context_precision + + class FirstDummyLLM(BaseRagasLLM): + def __init__(self): + super().__init__() + + def generate_text(self) -> str: + return "dummy llm" + + def agenerate_text(self) -> str: + return "dummy llm" + + context_precision.llm = FirstDummyLLM() + + rcp_evaluator = RagasContextPrecisionEvaluator(llmobs) + + assert rcp_evaluator.ragas_context_precision_instance.llm.generate_text() == "dummy llm" + + class SecondDummyLLM(BaseRagasLLM): + def __init__(self): + super().__init__() + + def generate_text(self) -> str: + return "second dummy llm" + + def agenerate_text(self) -> str: + return "second dummy llm" + + context_precision.llm = SecondDummyLLM() + + rcp_evaluator = RagasContextPrecisionEvaluator(llmobs) + + assert rcp_evaluator.ragas_context_precision_instance.llm.generate_text() == "second dummy llm" + + +def test_ragas_context_precision_submits_evaluation(ragas, llmobs, mock_llmobs_submit_evaluation): + """Test that evaluation is submitted for a valid llm span where question is in the prompt variables""" + rcp_evaluator = RagasContextPrecisionEvaluator(llmobs) + llm_span = _llm_span_with_expected_ragas_inputs_in_prompt() + with ragas_context_precision_single_context_cassette: + rcp_evaluator.run_and_submit_evaluation(llm_span) + rcp_evaluator.llmobs_service.submit_evaluation.assert_has_calls( + [ + mock.call( + span_context={ + "span_id": llm_span.get("span_id"), + "trace_id": llm_span.get("trace_id"), + }, + label=RagasContextPrecisionEvaluator.LABEL, + metric_type=RagasContextPrecisionEvaluator.METRIC_TYPE, + value=1.0, + metadata={ + "_dd.evaluation_kind": "context_precision", + "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, + }, + ) + ] + ) + + +def test_ragas_context_precision_submits_evaluation_on_span_with_question_in_messages( + ragas, llmobs, mock_llmobs_submit_evaluation +): + """Test that evaluation is submitted for a valid llm span where the last message content is the question""" + rcp_evaluator = RagasContextPrecisionEvaluator(llmobs) + llm_span = _llm_span_with_expected_ragas_inputs_in_messages() + with ragas_context_precision_single_context_cassette: + rcp_evaluator.run_and_submit_evaluation(llm_span) + rcp_evaluator.llmobs_service.submit_evaluation.assert_has_calls( + [ + mock.call( + span_context={ + "span_id": llm_span.get("span_id"), + "trace_id": llm_span.get("trace_id"), + }, + label=RagasContextPrecisionEvaluator.LABEL, + metric_type=RagasContextPrecisionEvaluator.METRIC_TYPE, + value=1.0, + metadata={ + "_dd.evaluation_kind": "context_precision", + "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, + }, + ) + ] + ) + + +def test_ragas_context_precision_submits_evaluation_on_span_with_custom_keys( + ragas, llmobs, mock_llmobs_submit_evaluation +): + """Test that evaluation is submitted for a valid llm span where the last message content is the question""" + rcp_evaluator = RagasContextPrecisionEvaluator(llmobs) + llm_span = _expected_llmobs_llm_span_event( + Span("dummy"), + prompt={ + "variables": { + "user_input": default_ragas_inputs["question"], + "context_2": default_ragas_inputs["context"], + "context_3": default_ragas_inputs["context"], + }, + "_dd_context_variable_keys": ["context_2", "context_3"], + "_dd_query_variable_keys": ["user_input"], + }, + output_messages=[{"content": default_ragas_inputs["answer"]}], + ) + with ragas_context_precision_multiple_context_cassette: + rcp_evaluator.run_and_submit_evaluation(llm_span) + rcp_evaluator.llmobs_service.submit_evaluation.assert_has_calls( + [ + mock.call( + span_context={ + "span_id": llm_span.get("span_id"), + "trace_id": llm_span.get("trace_id"), + }, + label=RagasContextPrecisionEvaluator.LABEL, + metric_type=RagasContextPrecisionEvaluator.METRIC_TYPE, + value=0.5, + metadata={ + "_dd.evaluation_kind": "context_precision", + "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, + }, + ) + ] + ) + + +def test_ragas_context_precision_emits_traces(ragas, llmobs, llmobs_events): + rcp_evaluator = RagasContextPrecisionEvaluator(llmobs) + with ragas_context_precision_single_context_cassette: + rcp_evaluator.evaluate(_llm_span_with_expected_ragas_inputs_in_prompt()) + ragas_spans = [event for event in llmobs_events if event["name"].startswith("dd-ragas.")] + ragas_spans = sorted(ragas_spans, key=lambda d: d["start_ns"]) + assert len(ragas_spans) == 2 + assert ragas_spans == _expected_ragas_context_precision_spans() + + # verify the trace structure + root_span = ragas_spans[0] + root_span_id = root_span["span_id"] + assert root_span["parent_id"] == "undefined" + assert root_span["meta"] is not None + + root_span_trace_id = root_span["trace_id"] + for child_span in ragas_spans[1:]: + assert child_span["trace_id"] == root_span_trace_id + assert child_span["parent_id"] == root_span_id + + +def test_ragas_answer_relevancy_init(ragas, llmobs): + rar_evaluator = RagasAnswerRelevancyEvaluator(llmobs) + assert rar_evaluator.llmobs_service == llmobs + assert rar_evaluator.ragas_answer_relevancy_instance == ragas.metrics.answer_relevancy + assert rar_evaluator.ragas_answer_relevancy_instance.llm == ragas.llms.llm_factory() + assert ( + rar_evaluator.ragas_answer_relevancy_instance.embeddings.embeddings + == ragas.embeddings.embedding_factory().embeddings + ) + assert ( + rar_evaluator.ragas_answer_relevancy_instance.embeddings.run_config + == ragas.embeddings.embedding_factory().run_config + ) + + +def test_ragas_answer_relevancy_throws_if_dependencies_not_present(llmobs, mock_ragas_dependencies_not_present, ragas): + with pytest.raises(NotImplementedError, match="Failed to load dependencies for `ragas_answer_relevancy` evaluator"): + RagasAnswerRelevancyEvaluator(llmobs) + + +def test_ragas_answer_relevancy_returns_none_if_inputs_extraction_fails(ragas, mock_llmobs_submit_evaluation, llmobs): + rar_evaluator = RagasAnswerRelevancyEvaluator(llmobs) + failure_msg, _ = rar_evaluator.evaluate(_llm_span_without_io()) + assert failure_msg == "fail_extract_answer_relevancy_inputs" + assert rar_evaluator.llmobs_service.submit_evaluation.call_count == 0 + + +def test_ragas_answer_relevancy_has_modified_answer_relevancy_instance( + ragas, mock_llmobs_submit_evaluation, reset_ragas_answer_relevancy_llm, llmobs +): + """Answer relevancy instance used in ragas evaluator should match the global ragas context precision instance""" + from ragas.llms import BaseRagasLLM + from ragas.metrics import answer_relevancy + + class FirstDummyLLM(BaseRagasLLM): + def __init__(self): + super().__init__() + + def generate_text(self) -> str: + return "dummy llm" + + def agenerate_text(self) -> str: + return "dummy llm" + + answer_relevancy.llm = FirstDummyLLM() + + rar_evaluator = RagasAnswerRelevancyEvaluator(llmobs) + + assert rar_evaluator.ragas_answer_relevancy_instance.llm.generate_text() == "dummy llm" + + class SecondDummyLLM(BaseRagasLLM): + def __init__(self): + super().__init__() + + def generate_text(self) -> str: + return "second dummy llm" + + def agenerate_text(self) -> str: + return "second dummy llm" + + answer_relevancy.llm = SecondDummyLLM() + + rar_evaluator = RagasAnswerRelevancyEvaluator(llmobs) + + assert rar_evaluator.ragas_answer_relevancy_instance.llm.generate_text() == "second dummy llm" + + +def test_ragas_answer_relevancy_submits_evaluation( + ragas, llmobs, mock_llmobs_submit_evaluation, mock_ragas_answer_relevancy_calculate_similarity +): + """Test that evaluation is submitted for a valid llm span where question is in the prompt variables""" + rar_evaluator = RagasAnswerRelevancyEvaluator(llmobs) + llm_span = _llm_span_with_expected_ragas_inputs_in_prompt() + with ragas_answer_relevancy_cassette: + rar_evaluator.run_and_submit_evaluation(llm_span) + rar_evaluator.llmobs_service.submit_evaluation.assert_has_calls( + [ + mock.call( + span_context={ + "span_id": llm_span.get("span_id"), + "trace_id": llm_span.get("trace_id"), + }, + label=RagasAnswerRelevancyEvaluator.LABEL, + metric_type=RagasAnswerRelevancyEvaluator.METRIC_TYPE, + value=mock.ANY, + metadata={ + "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, + }, + ) + ] + ) + + +def test_ragas_answer_relevancy_submits_evaluation_on_span_with_question_in_messages( + ragas, llmobs, mock_llmobs_submit_evaluation, mock_ragas_answer_relevancy_calculate_similarity +): + """Test that evaluation is submitted for a valid llm span where the last message content is the question""" + rar_evaluator = RagasAnswerRelevancyEvaluator(llmobs) + llm_span = _llm_span_with_expected_ragas_inputs_in_messages() + with ragas_answer_relevancy_cassette: + rar_evaluator.run_and_submit_evaluation(llm_span) + rar_evaluator.llmobs_service.submit_evaluation.assert_has_calls( + [ + mock.call( + span_context={ + "span_id": llm_span.get("span_id"), + "trace_id": llm_span.get("trace_id"), + }, + label=RagasAnswerRelevancyEvaluator.LABEL, + metric_type=RagasAnswerRelevancyEvaluator.METRIC_TYPE, + value=mock.ANY, + metadata={ + "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, + }, + ) + ] + ) + + +def test_ragas_answer_relevancy_submits_evaluation_on_span_with_custom_keys( + ragas, llmobs, mock_llmobs_submit_evaluation, mock_ragas_answer_relevancy_calculate_similarity +): + """Test that evaluation is submitted for a valid llm span where the last message content is the question""" + rar_evaluator = RagasAnswerRelevancyEvaluator(llmobs) + llm_span = _expected_llmobs_llm_span_event( + Span("dummy"), + prompt={ + "variables": { + "user_input": "Is france part of europe?", + "context_2": "irrelevant", + "context_3": "France is part of europe", + }, + "_dd_context_variable_keys": ["context_2", "context_3"], + "_dd_query_variable_keys": ["user_input"], + }, + output_messages=[{"content": "France is indeed part of europe"}], + ) + with ragas_answer_relevancy_cassette: + rar_evaluator.run_and_submit_evaluation(llm_span) + rar_evaluator.llmobs_service.submit_evaluation.assert_has_calls( + [ + mock.call( + span_context={ + "span_id": llm_span.get("span_id"), + "trace_id": llm_span.get("trace_id"), + }, + label=RagasAnswerRelevancyEvaluator.LABEL, + metric_type=RagasAnswerRelevancyEvaluator.METRIC_TYPE, + value=mock.ANY, + metadata={ + "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, + }, + ) + ] + ) + + +def test_ragas_answer_relevancy_emits_traces( + ragas, llmobs, llmobs_events, mock_ragas_answer_relevancy_calculate_similarity +): + rar_evaluator = RagasAnswerRelevancyEvaluator(llmobs) + with ragas_answer_relevancy_cassette: + rar_evaluator.evaluate(_llm_span_with_expected_ragas_inputs_in_prompt()) + + ragas_spans = [event for event in llmobs_events if event["name"].startswith("dd-ragas.")] + ragas_spans = sorted(ragas_spans, key=lambda d: d["start_ns"]) + + assert len(ragas_spans) == 3 + # check name, io, span kinds match + assert ragas_spans == _expected_ragas_answer_relevancy_spans() + + # verify the trace structure + root_span = ragas_spans[0] + root_span_id = root_span["span_id"] + assert root_span["parent_id"] == "undefined" + assert root_span["meta"] is not None + + root_span_trace_id = root_span["trace_id"] + for child_span in ragas_spans[1:]: + assert child_span["trace_id"] == root_span_trace_id + assert child_span["parent_id"] == root_span_id diff --git a/tests/llmobs/test_llmobs_ragas_faithfulness_evaluator.py b/tests/llmobs/test_llmobs_ragas_faithfulness_evaluator.py deleted file mode 100644 index 1f78b538f24..00000000000 --- a/tests/llmobs/test_llmobs_ragas_faithfulness_evaluator.py +++ /dev/null @@ -1,249 +0,0 @@ -import os - -import mock -import pytest - -from ddtrace.llmobs._evaluators.ragas.faithfulness import RagasFaithfulnessEvaluator -from ddtrace.span import Span -from tests.llmobs._utils import _expected_llmobs_llm_span_event -from tests.llmobs._utils import _expected_ragas_spans -from tests.llmobs._utils import _llm_span_with_expected_ragas_inputs_in_messages -from tests.llmobs._utils import _llm_span_with_expected_ragas_inputs_in_prompt - - -def _llm_span_without_io(): - return _expected_llmobs_llm_span_event(Span("dummy")) - - -def test_ragas_evaluator_init(ragas, LLMObs): - rf_evaluator = RagasFaithfulnessEvaluator(LLMObs) - assert rf_evaluator.llmobs_service == LLMObs - assert rf_evaluator.ragas_faithfulness_instance == ragas.metrics.faithfulness - assert rf_evaluator.ragas_faithfulness_instance.llm == ragas.llms.llm_factory() - - -def test_ragas_faithfulness_throws_if_dependencies_not_present(LLMObs, mock_ragas_dependencies_not_present, ragas): - with pytest.raises(NotImplementedError, match="Failed to load dependencies for `ragas_faithfulness` evaluator"): - RagasFaithfulnessEvaluator(LLMObs) - - -def test_ragas_faithfulness_returns_none_if_inputs_extraction_fails(ragas, mock_llmobs_submit_evaluation, LLMObs): - rf_evaluator = RagasFaithfulnessEvaluator(LLMObs) - failure_msg, _ = rf_evaluator.evaluate(_llm_span_without_io()) - assert failure_msg == "fail_extract_faithfulness_inputs" - assert rf_evaluator.llmobs_service.submit_evaluation.call_count == 0 - - -def test_ragas_faithfulness_has_modified_faithfulness_instance( - ragas, mock_llmobs_submit_evaluation, reset_ragas_faithfulness_llm, LLMObs -): - """Faithfulness instance used in ragas evaluator should match the global ragas faithfulness instance""" - from ragas.llms import BaseRagasLLM - from ragas.metrics import faithfulness - - class FirstDummyLLM(BaseRagasLLM): - def __init__(self): - super().__init__() - - def generate_text(self) -> str: - return "dummy llm" - - def agenerate_text(self) -> str: - return "dummy llm" - - faithfulness.llm = FirstDummyLLM() - - rf_evaluator = RagasFaithfulnessEvaluator(LLMObs) - - assert rf_evaluator.ragas_faithfulness_instance.llm.generate_text() == "dummy llm" - - class SecondDummyLLM(BaseRagasLLM): - def __init__(self): - super().__init__() - - def generate_text(self, statements) -> str: - raise ValueError("dummy_llm") - - def agenerate_text(self, statements) -> str: - raise ValueError("dummy_llm") - - faithfulness.llm = SecondDummyLLM() - - with pytest.raises(ValueError, match="dummy_llm"): - rf_evaluator.evaluate(_llm_span_with_expected_ragas_inputs_in_prompt()) - - -@pytest.mark.vcr_logs -def test_ragas_faithfulness_submits_evaluation(ragas, LLMObs, mock_llmobs_submit_evaluation): - """Test that evaluation is submitted for a valid llm span where question is in the prompt variables""" - rf_evaluator = RagasFaithfulnessEvaluator(LLMObs) - llm_span = _llm_span_with_expected_ragas_inputs_in_prompt() - rf_evaluator.run_and_submit_evaluation(llm_span) - rf_evaluator.llmobs_service.submit_evaluation.assert_has_calls( - [ - mock.call( - span_context={ - "span_id": llm_span.get("span_id"), - "trace_id": llm_span.get("trace_id"), - }, - label=RagasFaithfulnessEvaluator.LABEL, - metric_type=RagasFaithfulnessEvaluator.METRIC_TYPE, - value=1.0, - metadata={ - "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, - "_dd.faithfulness_disagreements": mock.ANY, - "_dd.evaluation_kind": "faithfulness", - }, - ) - ] - ) - - -@pytest.mark.vcr_logs -def test_ragas_faithfulness_submits_evaluation_on_span_with_question_in_messages( - ragas, LLMObs, mock_llmobs_submit_evaluation -): - """Test that evaluation is submitted for a valid llm span where the last message content is the question""" - rf_evaluator = RagasFaithfulnessEvaluator(LLMObs) - llm_span = _llm_span_with_expected_ragas_inputs_in_messages() - rf_evaluator.run_and_submit_evaluation(llm_span) - rf_evaluator.llmobs_service.submit_evaluation.assert_has_calls( - [ - mock.call( - span_context={ - "span_id": llm_span.get("span_id"), - "trace_id": llm_span.get("trace_id"), - }, - label=RagasFaithfulnessEvaluator.LABEL, - metric_type=RagasFaithfulnessEvaluator.METRIC_TYPE, - value=1.0, - metadata={ - "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, - "_dd.faithfulness_disagreements": mock.ANY, - "_dd.evaluation_kind": "faithfulness", - }, - ) - ] - ) - - -@pytest.mark.vcr_logs -def test_ragas_faithfulness_submits_evaluation_on_span_with_custom_keys(ragas, LLMObs, mock_llmobs_submit_evaluation): - """Test that evaluation is submitted for a valid llm span where the last message content is the question""" - rf_evaluator = RagasFaithfulnessEvaluator(LLMObs) - llm_span = _expected_llmobs_llm_span_event( - Span("dummy"), - prompt={ - "variables": { - "user_input": "Is france part of europe?", - "context_1": "hello, ", - "context_2": "france is ", - "context_3": "part of europe", - }, - "_dd_context_variable_keys": ["context_1", "context_2", "context_3"], - "_dd_query_variable_keys": ["user_input"], - }, - output_messages=[{"content": "France is indeed part of europe"}], - ) - rf_evaluator.run_and_submit_evaluation(llm_span) - rf_evaluator.llmobs_service.submit_evaluation.assert_has_calls( - [ - mock.call( - span_context={ - "span_id": llm_span.get("span_id"), - "trace_id": llm_span.get("trace_id"), - }, - label=RagasFaithfulnessEvaluator.LABEL, - metric_type=RagasFaithfulnessEvaluator.METRIC_TYPE, - value=1.0, - metadata={ - "_dd.evaluation_span": {"span_id": mock.ANY, "trace_id": mock.ANY}, - "_dd.faithfulness_disagreements": mock.ANY, - "_dd.evaluation_kind": "faithfulness", - }, - ) - ] - ) - - -@pytest.mark.vcr_logs -def test_ragas_faithfulness_emits_traces(ragas, LLMObs): - rf_evaluator = RagasFaithfulnessEvaluator(LLMObs) - rf_evaluator.evaluate(_llm_span_with_expected_ragas_inputs_in_prompt()) - assert rf_evaluator.llmobs_service._instance._llmobs_span_writer.enqueue.call_count == 7 - calls = rf_evaluator.llmobs_service._instance._llmobs_span_writer.enqueue.call_args_list - - spans = [call[0][0] for call in calls] - - # check name, io, span kinds match - assert spans == _expected_ragas_spans() - - # verify the trace structure - root_span = spans[0] - root_span_id = root_span["span_id"] - assert root_span["parent_id"] == "undefined" - assert root_span["meta"] is not None - assert root_span["meta"]["metadata"] is not None - assert isinstance(root_span["meta"]["metadata"]["faithfulness_list"], list) - assert isinstance(root_span["meta"]["metadata"]["statements"], list) - root_span_trace_id = root_span["trace_id"] - for child_span in spans[1:]: - assert child_span["trace_id"] == root_span_trace_id - - assert spans[1]["parent_id"] == root_span_id # input extraction (task) - assert spans[2]["parent_id"] == root_span_id # create statements (workflow) - assert spans[4]["parent_id"] == root_span_id # create verdicts (workflow) - assert spans[6]["parent_id"] == root_span_id # create score (task) - - assert spans[3]["parent_id"] == spans[2]["span_id"] # create statements prompt (task) - assert spans[5]["parent_id"] == spans[4]["span_id"] # create verdicts prompt (task) - - -def test_llmobs_with_faithfulness_emits_traces_and_evals_on_exit(mock_writer_logs, run_python_code_in_subprocess): - env = os.environ.copy() - pypath = [os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))] - if "PYTHONPATH" in env: - pypath.append(env["PYTHONPATH"]) - env.update( - { - "DD_API_KEY": os.getenv("DD_API_KEY", "dummy-api-key"), - "DD_SITE": "datad0g.com", - "PYTHONPATH": ":".join(pypath), - "OPENAI_API_KEY": os.getenv("OPENAI_API_KEY", "dummy-openai-api-key"), - "DD_LLMOBS_ML_APP": "unnamed-ml-app", - "_DD_LLMOBS_EVALUATOR_INTERVAL": "5", - "_DD_LLMOBS_EVALUATORS": "ragas_faithfulness", - "DD_LLMOBS_AGENTLESS_ENABLED": "true", - } - ) - out, err, status, pid = run_python_code_in_subprocess( - """ -import os -import time -import atexit -import mock -from ddtrace.llmobs import LLMObs -from ddtrace.internal.utils.http import Response -from tests.llmobs._utils import _llm_span_with_expected_ragas_inputs_in_messages -from tests.llmobs._utils import logs_vcr - -ctx = logs_vcr.use_cassette( - "tests.llmobs.test_llmobs_ragas_faithfulness_evaluator.emits_traces_and_evaluations_on_exit.yaml" -) -ctx.__enter__() -atexit.register(lambda: ctx.__exit__()) -with mock.patch( - "ddtrace.internal.writer.HTTPWriter._send_payload", - return_value=Response( - status=200, - body="{}", - ), -): - LLMObs.enable() - LLMObs._instance._evaluator_runner.enqueue(_llm_span_with_expected_ragas_inputs_in_messages(), None) -""", - env=env, - ) - assert status == 0, err - assert out == b"" - assert err == b"" diff --git a/tests/llmobs/test_llmobs_service.py b/tests/llmobs/test_llmobs_service.py index 98748250c3a..dad6accdcfb 100644 --- a/tests/llmobs/test_llmobs_service.py +++ b/tests/llmobs/test_llmobs_service.py @@ -1,4 +1,5 @@ import os +import re import threading import time @@ -7,9 +8,7 @@ import ddtrace from ddtrace._trace.context import Context -from ddtrace._trace.span import Span from ddtrace.ext import SpanTypes -from ddtrace.filters import TraceFilter from ddtrace.internal.service import ServiceStatus from ddtrace.llmobs import LLMObs as llmobs_service from ddtrace.llmobs._constants import INPUT_DOCUMENTS @@ -31,7 +30,8 @@ from ddtrace.llmobs._constants import SPAN_START_WHILE_DISABLED_WARNING from ddtrace.llmobs._constants import TAGS from ddtrace.llmobs._llmobs import SUPPORTED_LLMOBS_INTEGRATIONS -from ddtrace.llmobs._llmobs import LLMObsTraceProcessor +from ddtrace.llmobs._writer import LLMObsAgentlessEventClient +from ddtrace.llmobs._writer import LLMObsProxiedEventClient from ddtrace.llmobs.utils import Prompt from tests.llmobs._utils import _expected_llmobs_eval_metric_event from tests.llmobs._utils import _expected_llmobs_llm_span_event @@ -41,23 +41,16 @@ from tests.utils import override_global_config -@pytest.fixture -def mock_logs(): - with mock.patch("ddtrace.llmobs._llmobs.log") as mock_logs: - yield mock_logs +RAGAS_AVAILABLE = os.getenv("RAGAS_AVAILABLE", False) def run_llmobs_trace_filter(dummy_tracer): - for trace_filter in dummy_tracer._filters: - if isinstance(trace_filter, LLMObsTraceProcessor): - root_llm_span = Span(name="span1", span_type=SpanTypes.LLM) - root_llm_span.set_tag_str(SPAN_KIND, "llm") - trace1 = [root_llm_span] - return trace_filter.process_trace(trace1) - raise ValueError("LLMObsTraceProcessor not found in tracer filters.") + with dummy_tracer.trace("span1", span_type=SpanTypes.LLM) as span: + span.set_tag_str(SPAN_KIND, "llm") + return dummy_tracer._writer.pop() -def test_service_enable(): +def test_service_enable_proxy_default(): with override_global_config(dict(_dd_api_key="", _llmobs_ml_app="")): dummy_tracer = DummyTracer() llmobs_service.enable(_tracer=dummy_tracer) @@ -65,22 +58,22 @@ def test_service_enable(): assert llmobs_instance is not None assert llmobs_service.enabled assert llmobs_instance.tracer == dummy_tracer - assert any(isinstance(tracer_filter, LLMObsTraceProcessor) for tracer_filter in dummy_tracer._filters) + assert isinstance(llmobs_instance._llmobs_span_writer._clients[0], LLMObsProxiedEventClient) assert run_llmobs_trace_filter(dummy_tracer) is not None llmobs_service.disable() -def test_service_enable_with_apm_disabled(monkeypatch): - with override_global_config(dict(_dd_api_key="", _llmobs_ml_app="")): +def test_enable_agentless(): + with override_global_config(dict(_dd_api_key="", _llmobs_ml_app="")): dummy_tracer = DummyTracer() llmobs_service.enable(_tracer=dummy_tracer, agentless_enabled=True) llmobs_instance = llmobs_service._instance assert llmobs_instance is not None assert llmobs_service.enabled assert llmobs_instance.tracer == dummy_tracer - assert any(isinstance(tracer_filter, LLMObsTraceProcessor) for tracer_filter in dummy_tracer._filters) - assert run_llmobs_trace_filter(dummy_tracer) is None + assert isinstance(llmobs_instance._llmobs_span_writer._clients[0], LLMObsAgentlessEventClient) + assert run_llmobs_trace_filter(dummy_tracer) is not None llmobs_service.disable() @@ -118,7 +111,7 @@ def test_service_enable_no_ml_app_specified(): assert llmobs_service._instance._evaluator_runner.status.value == "stopped" -def test_service_enable_deprecated_ml_app_name(monkeypatch, mock_logs): +def test_service_enable_deprecated_ml_app_name(monkeypatch, mock_llmobs_logs): with override_global_config(dict(_dd_api_key="", _llmobs_ml_app="")): dummy_tracer = DummyTracer() monkeypatch.setenv("DD_LLMOBS_APP_NAME", "test_ml_app") @@ -126,11 +119,13 @@ def test_service_enable_deprecated_ml_app_name(monkeypatch, mock_logs): assert llmobs_service.enabled is True assert llmobs_service._instance._llmobs_eval_metric_writer.status.value == "running" assert llmobs_service._instance._llmobs_span_writer.status.value == "running" - mock_logs.warning.assert_called_once_with("`DD_LLMOBS_APP_NAME` is deprecated. Use `DD_LLMOBS_ML_APP` instead.") + mock_llmobs_logs.warning.assert_called_once_with( + "`DD_LLMOBS_APP_NAME` is deprecated. Use `DD_LLMOBS_ML_APP` instead." + ) llmobs_service.disable() -def test_service_enable_already_enabled(mock_logs): +def test_service_enable_already_enabled(mock_llmobs_logs): with override_global_config(dict(_dd_api_key="", _llmobs_ml_app="")): dummy_tracer = DummyTracer() llmobs_service.enable(_tracer=dummy_tracer) @@ -139,9 +134,8 @@ def test_service_enable_already_enabled(mock_logs): assert llmobs_instance is not None assert llmobs_service.enabled assert llmobs_instance.tracer == dummy_tracer - assert any(isinstance(tracer_filter, LLMObsTraceProcessor) for tracer_filter in dummy_tracer._filters) llmobs_service.disable() - mock_logs.debug.assert_has_calls([mock.call("%s already enabled", "LLMObs")]) + mock_llmobs_logs.debug.assert_has_calls([mock.call("%s already enabled", "LLMObs")]) @mock.patch("ddtrace.llmobs._llmobs.patch") @@ -203,107 +197,83 @@ def test_service_enable_does_not_override_global_patch_config(mock_tracer_patch, llmobs_service.disable() -def test_start_span_while_disabled_logs_warning(LLMObs, mock_logs): - LLMObs.disable() - _ = LLMObs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") - mock_logs.warning.assert_called_once_with(SPAN_START_WHILE_DISABLED_WARNING) - mock_logs.reset_mock() - _ = LLMObs.tool(name="test_tool") - mock_logs.warning.assert_called_once_with(SPAN_START_WHILE_DISABLED_WARNING) - mock_logs.reset_mock() - _ = LLMObs.task(name="test_task") - mock_logs.warning.assert_called_once_with(SPAN_START_WHILE_DISABLED_WARNING) - mock_logs.reset_mock() - _ = LLMObs.workflow(name="test_workflow") - mock_logs.warning.assert_called_once_with(SPAN_START_WHILE_DISABLED_WARNING) - mock_logs.reset_mock() - _ = LLMObs.agent(name="test_agent") - mock_logs.warning.assert_called_once_with(SPAN_START_WHILE_DISABLED_WARNING) - - -def test_start_span_uses_kind_as_default_name(LLMObs): - with LLMObs.llm(model_name="test_model", model_provider="test_provider") as span: +def test_start_span_while_disabled_logs_warning(llmobs, mock_llmobs_logs): + llmobs.disable() + _ = llmobs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") + mock_llmobs_logs.warning.assert_called_once_with(SPAN_START_WHILE_DISABLED_WARNING) + mock_llmobs_logs.reset_mock() + _ = llmobs.tool(name="test_tool") + mock_llmobs_logs.warning.assert_called_once_with(SPAN_START_WHILE_DISABLED_WARNING) + mock_llmobs_logs.reset_mock() + _ = llmobs.task(name="test_task") + mock_llmobs_logs.warning.assert_called_once_with(SPAN_START_WHILE_DISABLED_WARNING) + mock_llmobs_logs.reset_mock() + _ = llmobs.workflow(name="test_workflow") + mock_llmobs_logs.warning.assert_called_once_with(SPAN_START_WHILE_DISABLED_WARNING) + mock_llmobs_logs.reset_mock() + _ = llmobs.agent(name="test_agent") + mock_llmobs_logs.warning.assert_called_once_with(SPAN_START_WHILE_DISABLED_WARNING) + + +def test_start_span_uses_kind_as_default_name(llmobs): + with llmobs.llm(model_name="test_model", model_provider="test_provider") as span: assert span.name == "llm" - with LLMObs.tool() as span: + with llmobs.tool() as span: assert span.name == "tool" - with LLMObs.task() as span: + with llmobs.task() as span: assert span.name == "task" - with LLMObs.workflow() as span: + with llmobs.workflow() as span: assert span.name == "workflow" - with LLMObs.agent() as span: + with llmobs.agent() as span: assert span.name == "agent" -def test_start_span_with_session_id(LLMObs): - with LLMObs.llm(model_name="test_model", session_id="test_session_id") as span: +def test_start_span_with_session_id(llmobs): + with llmobs.llm(model_name="test_model", session_id="test_session_id") as span: assert span._get_ctx_item(SESSION_ID) == "test_session_id" - with LLMObs.tool(session_id="test_session_id") as span: + with llmobs.tool(session_id="test_session_id") as span: assert span._get_ctx_item(SESSION_ID) == "test_session_id" - with LLMObs.task(session_id="test_session_id") as span: + with llmobs.task(session_id="test_session_id") as span: assert span._get_ctx_item(SESSION_ID) == "test_session_id" - with LLMObs.workflow(session_id="test_session_id") as span: + with llmobs.workflow(session_id="test_session_id") as span: assert span._get_ctx_item(SESSION_ID) == "test_session_id" - with LLMObs.agent(session_id="test_session_id") as span: + with llmobs.agent(session_id="test_session_id") as span: assert span._get_ctx_item(SESSION_ID) == "test_session_id" -def test_session_id_becomes_top_level_field(LLMObs, mock_llmobs_span_writer): - session_id = "test_session_id" - with LLMObs.task(session_id=session_id) as span: - pass - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event(span, "task", session_id=session_id) - ) - - -def test_session_id_becomes_top_level_field_agentless(AgentlessLLMObs, mock_llmobs_span_agentless_writer): +def test_session_id_becomes_top_level_field(llmobs, llmobs_events): session_id = "test_session_id" - with AgentlessLLMObs.task(session_id=session_id) as span: + with llmobs.task(session_id=session_id) as span: pass - mock_llmobs_span_agentless_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event(span, "task", session_id=session_id) - ) - + assert len(llmobs_events) == 1 + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event(span, "task", session_id=session_id) -def test_llm_span(LLMObs, mock_llmobs_span_writer): - with LLMObs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: - assert span.name == "test_llm_call" - assert span.resource == "llm" - assert span.span_type == "llm" - assert span._get_ctx_item(SPAN_KIND) == "llm" - assert span._get_ctx_item(MODEL_NAME) == "test_model" - assert span._get_ctx_item(MODEL_PROVIDER) == "test_provider" - - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event(span, "llm", model_name="test_model", model_provider="test_provider") - ) - -def test_llm_span_agentless(AgentlessLLMObs, mock_llmobs_span_agentless_writer): - with AgentlessLLMObs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: +def test_llm_span(llmobs, llmobs_events): + with llmobs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: assert span.name == "test_llm_call" assert span.resource == "llm" assert span.span_type == "llm" assert span._get_ctx_item(SPAN_KIND) == "llm" assert span._get_ctx_item(MODEL_NAME) == "test_model" assert span._get_ctx_item(MODEL_PROVIDER) == "test_provider" - - mock_llmobs_span_agentless_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event(span, "llm", model_name="test_model", model_provider="test_provider") + assert len(llmobs_events) == 1 + assert llmobs_events[0] == _expected_llmobs_llm_span_event( + span, "llm", model_name="test_model", model_provider="test_provider" ) -def test_llm_span_no_model_sets_default(LLMObs, mock_llmobs_span_writer): - with LLMObs.llm(name="test_llm_call", model_provider="test_provider") as span: +def test_llm_span_no_model_sets_default(llmobs, llmobs_events): + with llmobs.llm(name="test_llm_call", model_provider="test_provider") as span: assert span._get_ctx_item(MODEL_NAME) == "custom" - - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event(span, "llm", model_name="custom", model_provider="test_provider") + assert len(llmobs_events) == 1 + assert llmobs_events[0] == _expected_llmobs_llm_span_event( + span, "llm", model_name="custom", model_provider="test_provider" ) -def test_default_model_provider_set_to_custom(LLMObs): - with LLMObs.llm(model_name="test_model", name="test_llm_call") as span: +def test_default_model_provider_set_to_custom(llmobs): + with llmobs.llm(model_name="test_model", name="test_llm_call") as span: assert span.name == "test_llm_call" assert span.resource == "llm" assert span.span_type == "llm" @@ -312,88 +282,57 @@ def test_default_model_provider_set_to_custom(LLMObs): assert span._get_ctx_item(MODEL_PROVIDER) == "custom" -def test_tool_span(LLMObs, mock_llmobs_span_writer): - with LLMObs.tool(name="test_tool") as span: - assert span.name == "test_tool" - assert span.resource == "tool" - assert span.span_type == "llm" - assert span._get_ctx_item(SPAN_KIND) == "tool" - mock_llmobs_span_writer.enqueue.assert_called_with(_expected_llmobs_non_llm_span_event(span, "tool")) - - -def test_tool_span_agentless(AgentlessLLMObs, mock_llmobs_span_agentless_writer): - with AgentlessLLMObs.tool(name="test_tool") as span: +def test_tool_span(llmobs, llmobs_events): + with llmobs.tool(name="test_tool") as span: assert span.name == "test_tool" assert span.resource == "tool" assert span.span_type == "llm" assert span._get_ctx_item(SPAN_KIND) == "tool" - mock_llmobs_span_agentless_writer.enqueue.assert_called_with(_expected_llmobs_non_llm_span_event(span, "tool")) - - -def test_task_span(LLMObs, mock_llmobs_span_writer): - with LLMObs.task(name="test_task") as span: - assert span.name == "test_task" - assert span.resource == "task" - assert span.span_type == "llm" - assert span._get_ctx_item(SPAN_KIND) == "task" - mock_llmobs_span_writer.enqueue.assert_called_with(_expected_llmobs_non_llm_span_event(span, "task")) + assert len(llmobs_events) == 1 + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event(span, "tool") -def test_task_span_agentless(AgentlessLLMObs, mock_llmobs_span_agentless_writer): - with AgentlessLLMObs.task(name="test_task") as span: +def test_task_span(llmobs, llmobs_events): + with llmobs.task(name="test_task") as span: assert span.name == "test_task" assert span.resource == "task" assert span.span_type == "llm" assert span._get_ctx_item(SPAN_KIND) == "task" - mock_llmobs_span_agentless_writer.enqueue.assert_called_with(_expected_llmobs_non_llm_span_event(span, "task")) - - -def test_workflow_span(LLMObs, mock_llmobs_span_writer): - with LLMObs.workflow(name="test_workflow") as span: - assert span.name == "test_workflow" - assert span.resource == "workflow" - assert span.span_type == "llm" - assert span._get_ctx_item(SPAN_KIND) == "workflow" - mock_llmobs_span_writer.enqueue.assert_called_with(_expected_llmobs_non_llm_span_event(span, "workflow")) + assert len(llmobs_events) == 1 + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event(span, "task") -def test_workflow_span_agentless(AgentlessLLMObs, mock_llmobs_span_agentless_writer): - with AgentlessLLMObs.workflow(name="test_workflow") as span: +def test_workflow_span(llmobs, llmobs_events): + with llmobs.workflow(name="test_workflow") as span: assert span.name == "test_workflow" assert span.resource == "workflow" assert span.span_type == "llm" assert span._get_ctx_item(SPAN_KIND) == "workflow" - mock_llmobs_span_agentless_writer.enqueue.assert_called_with(_expected_llmobs_non_llm_span_event(span, "workflow")) + assert len(llmobs_events) == 1 + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event(span, "workflow") -def test_agent_span(LLMObs, mock_llmobs_span_writer): - with LLMObs.agent(name="test_agent") as span: +def test_agent_span(llmobs, llmobs_events): + with llmobs.agent(name="test_agent") as span: assert span.name == "test_agent" assert span.resource == "agent" assert span.span_type == "llm" assert span._get_ctx_item(SPAN_KIND) == "agent" - mock_llmobs_span_writer.enqueue.assert_called_with(_expected_llmobs_llm_span_event(span, "agent")) + assert len(llmobs_events) == 1 + assert llmobs_events[0] == _expected_llmobs_llm_span_event(span, "agent") -def test_agent_span_agentless(AgentlessLLMObs, mock_llmobs_span_agentless_writer): - with AgentlessLLMObs.agent(name="test_agent") as span: - assert span.name == "test_agent" - assert span.resource == "agent" - assert span.span_type == "llm" - assert span._get_ctx_item(SPAN_KIND) == "agent" - mock_llmobs_span_agentless_writer.enqueue.assert_called_with(_expected_llmobs_llm_span_event(span, "agent")) - - -def test_embedding_span_no_model_sets_default(LLMObs, mock_llmobs_span_writer): - with LLMObs.embedding(name="test_embedding", model_provider="test_provider") as span: +def test_embedding_span_no_model_sets_default(llmobs, llmobs_events): + with llmobs.embedding(name="test_embedding", model_provider="test_provider") as span: assert span._get_ctx_item(MODEL_NAME) == "custom" - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event(span, "embedding", model_name="custom", model_provider="test_provider") + assert len(llmobs_events) == 1 + assert llmobs_events[0] == _expected_llmobs_llm_span_event( + span, "embedding", model_name="custom", model_provider="test_provider" ) -def test_embedding_default_model_provider_set_to_custom(LLMObs): - with LLMObs.embedding(model_name="test_model", name="test_embedding") as span: +def test_embedding_default_model_provider_set_to_custom(llmobs): + with llmobs.embedding(model_name="test_model", name="test_embedding") as span: assert span.name == "test_embedding" assert span.resource == "embedding" assert span.span_type == "llm" @@ -402,198 +341,182 @@ def test_embedding_default_model_provider_set_to_custom(LLMObs): assert span._get_ctx_item(MODEL_PROVIDER) == "custom" -def test_embedding_span(LLMObs, mock_llmobs_span_writer): - with LLMObs.embedding(model_name="test_model", name="test_embedding", model_provider="test_provider") as span: - assert span.name == "test_embedding" - assert span.resource == "embedding" - assert span.span_type == "llm" - assert span._get_ctx_item(SPAN_KIND) == "embedding" - assert span._get_ctx_item(MODEL_NAME) == "test_model" - assert span._get_ctx_item(MODEL_PROVIDER) == "test_provider" - - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event(span, "embedding", model_name="test_model", model_provider="test_provider") - ) - - -def test_embedding_span_agentless(AgentlessLLMObs, mock_llmobs_span_agentless_writer): - with AgentlessLLMObs.embedding( - model_name="test_model", name="test_embedding", model_provider="test_provider" - ) as span: +def test_embedding_span(llmobs, llmobs_events): + with llmobs.embedding(model_name="test_model", name="test_embedding", model_provider="test_provider") as span: assert span.name == "test_embedding" assert span.resource == "embedding" assert span.span_type == "llm" assert span._get_ctx_item(SPAN_KIND) == "embedding" assert span._get_ctx_item(MODEL_NAME) == "test_model" assert span._get_ctx_item(MODEL_PROVIDER) == "test_provider" - - mock_llmobs_span_agentless_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event(span, "embedding", model_name="test_model", model_provider="test_provider") + assert len(llmobs_events) == 1 + assert llmobs_events[0] == _expected_llmobs_llm_span_event( + span, "embedding", model_name="test_model", model_provider="test_provider" ) -def test_annotate_no_active_span_logs_warning(LLMObs, mock_logs): - LLMObs.annotate(parameters={"test": "test"}) - mock_logs.warning.assert_called_once_with("No span provided and no active LLMObs-generated span found.") +def test_annotate_no_active_span_logs_warning(llmobs, mock_llmobs_logs): + llmobs.annotate(parameters={"test": "test"}) + mock_llmobs_logs.warning.assert_called_once_with("No span provided and no active LLMObs-generated span found.") -def test_annotate_non_llm_span_logs_warning(LLMObs, mock_logs): +def test_annotate_non_llm_span_logs_warning(llmobs, mock_llmobs_logs): dummy_tracer = DummyTracer() with dummy_tracer.trace("root") as non_llmobs_span: - LLMObs.annotate(span=non_llmobs_span, parameters={"test": "test"}) - mock_logs.warning.assert_called_once_with("Span must be an LLMObs-generated span.") + llmobs.annotate(span=non_llmobs_span, parameters={"test": "test"}) + mock_llmobs_logs.warning.assert_called_once_with("Span must be an LLMObs-generated span.") -def test_annotate_finished_span_does_nothing(LLMObs, mock_logs): - with LLMObs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: +def test_annotate_finished_span_does_nothing(llmobs, mock_llmobs_logs): + with llmobs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: pass - LLMObs.annotate(span=span, parameters={"test": "test"}) - mock_logs.warning.assert_called_once_with("Cannot annotate a finished span.") + llmobs.annotate(span=span, parameters={"test": "test"}) + mock_llmobs_logs.warning.assert_called_once_with("Cannot annotate a finished span.") -def test_annotate_parameters(LLMObs, mock_logs): - with LLMObs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: - LLMObs.annotate(span=span, parameters={"temperature": 0.9, "max_tokens": 50}) +def test_annotate_parameters(llmobs, mock_llmobs_logs): + with llmobs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: + llmobs.annotate(span=span, parameters={"temperature": 0.9, "max_tokens": 50}) assert span._get_ctx_item(INPUT_PARAMETERS) == {"temperature": 0.9, "max_tokens": 50} - mock_logs.warning.assert_called_once_with( + mock_llmobs_logs.warning.assert_called_once_with( "Setting parameters is deprecated, please set parameters and other metadata as tags instead." ) -def test_annotate_metadata(LLMObs): - with LLMObs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: - LLMObs.annotate(span=span, metadata={"temperature": 0.5, "max_tokens": 20, "top_k": 10, "n": 3}) +def test_annotate_metadata(llmobs): + with llmobs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: + llmobs.annotate(span=span, metadata={"temperature": 0.5, "max_tokens": 20, "top_k": 10, "n": 3}) assert span._get_ctx_item(METADATA) == {"temperature": 0.5, "max_tokens": 20, "top_k": 10, "n": 3} -def test_annotate_metadata_wrong_type_raises_warning(LLMObs, mock_logs): - with LLMObs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: - LLMObs.annotate(span=span, metadata="wrong_metadata") +def test_annotate_metadata_wrong_type_raises_warning(llmobs, mock_llmobs_logs): + with llmobs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: + llmobs.annotate(span=span, metadata="wrong_metadata") assert span._get_ctx_item(METADATA) is None - mock_logs.warning.assert_called_once_with("metadata must be a dictionary of string key-value pairs.") - mock_logs.reset_mock() + mock_llmobs_logs.warning.assert_called_once_with("metadata must be a dictionary of string key-value pairs.") + mock_llmobs_logs.reset_mock() -def test_annotate_tag(LLMObs): - with LLMObs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: - LLMObs.annotate(span=span, tags={"test_tag_name": "test_tag_value", "test_numeric_tag": 10}) +def test_annotate_tag(llmobs): + with llmobs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: + llmobs.annotate(span=span, tags={"test_tag_name": "test_tag_value", "test_numeric_tag": 10}) assert span._get_ctx_item(TAGS) == {"test_tag_name": "test_tag_value", "test_numeric_tag": 10} -def test_annotate_tag_wrong_type(LLMObs, mock_logs): - with LLMObs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: - LLMObs.annotate(span=span, tags=12345) +def test_annotate_tag_wrong_type(llmobs, mock_llmobs_logs): + with llmobs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: + llmobs.annotate(span=span, tags=12345) assert span._get_ctx_item(TAGS) is None - mock_logs.warning.assert_called_once_with( + mock_llmobs_logs.warning.assert_called_once_with( "span_tags must be a dictionary of string key - primitive value pairs." ) -def test_annotate_input_string(LLMObs): - with LLMObs.llm(model_name="test_model") as llm_span: - LLMObs.annotate(span=llm_span, input_data="test_input") +def test_annotate_input_string(llmobs): + with llmobs.llm(model_name="test_model") as llm_span: + llmobs.annotate(span=llm_span, input_data="test_input") assert llm_span._get_ctx_item(INPUT_MESSAGES) == [{"content": "test_input"}] - with LLMObs.task() as task_span: - LLMObs.annotate(span=task_span, input_data="test_input") + with llmobs.task() as task_span: + llmobs.annotate(span=task_span, input_data="test_input") assert task_span._get_ctx_item(INPUT_VALUE) == "test_input" - with LLMObs.tool() as tool_span: - LLMObs.annotate(span=tool_span, input_data="test_input") + with llmobs.tool() as tool_span: + llmobs.annotate(span=tool_span, input_data="test_input") assert tool_span._get_ctx_item(INPUT_VALUE) == "test_input" - with LLMObs.workflow() as workflow_span: - LLMObs.annotate(span=workflow_span, input_data="test_input") + with llmobs.workflow() as workflow_span: + llmobs.annotate(span=workflow_span, input_data="test_input") assert workflow_span._get_ctx_item(INPUT_VALUE) == "test_input" - with LLMObs.agent() as agent_span: - LLMObs.annotate(span=agent_span, input_data="test_input") + with llmobs.agent() as agent_span: + llmobs.annotate(span=agent_span, input_data="test_input") assert agent_span._get_ctx_item(INPUT_VALUE) == "test_input" - with LLMObs.retrieval() as retrieval_span: - LLMObs.annotate(span=retrieval_span, input_data="test_input") + with llmobs.retrieval() as retrieval_span: + llmobs.annotate(span=retrieval_span, input_data="test_input") assert retrieval_span._get_ctx_item(INPUT_VALUE) == "test_input" -def test_annotate_numeric_io(LLMObs): - with LLMObs.task() as task_span: - LLMObs.annotate(span=task_span, input_data=0, output_data=0) +def test_annotate_numeric_io(llmobs): + with llmobs.task() as task_span: + llmobs.annotate(span=task_span, input_data=0, output_data=0) assert task_span._get_ctx_item(INPUT_VALUE) == "0" assert task_span._get_ctx_item(OUTPUT_VALUE) == "0" - with LLMObs.task() as task_span: - LLMObs.annotate(span=task_span, input_data=1.23, output_data=1.23) + with llmobs.task() as task_span: + llmobs.annotate(span=task_span, input_data=1.23, output_data=1.23) assert task_span._get_ctx_item(INPUT_VALUE) == "1.23" assert task_span._get_ctx_item(OUTPUT_VALUE) == "1.23" -def test_annotate_input_serializable_value(LLMObs): - with LLMObs.task() as task_span: - LLMObs.annotate(span=task_span, input_data=["test_input"]) +def test_annotate_input_serializable_value(llmobs): + with llmobs.task() as task_span: + llmobs.annotate(span=task_span, input_data=["test_input"]) assert task_span._get_ctx_item(INPUT_VALUE) == str(["test_input"]) - with LLMObs.tool() as tool_span: - LLMObs.annotate(span=tool_span, input_data={"test_input": "hello world"}) + with llmobs.tool() as tool_span: + llmobs.annotate(span=tool_span, input_data={"test_input": "hello world"}) assert tool_span._get_ctx_item(INPUT_VALUE) == str({"test_input": "hello world"}) - with LLMObs.workflow() as workflow_span: - LLMObs.annotate(span=workflow_span, input_data=("asd", 123)) + with llmobs.workflow() as workflow_span: + llmobs.annotate(span=workflow_span, input_data=("asd", 123)) assert workflow_span._get_ctx_item(INPUT_VALUE) == str(("asd", 123)) - with LLMObs.agent() as agent_span: - LLMObs.annotate(span=agent_span, input_data="test_input") + with llmobs.agent() as agent_span: + llmobs.annotate(span=agent_span, input_data="test_input") assert agent_span._get_ctx_item(INPUT_VALUE) == "test_input" - with LLMObs.retrieval() as retrieval_span: - LLMObs.annotate(span=retrieval_span, input_data=[0, 1, 2, 3, 4]) + with llmobs.retrieval() as retrieval_span: + llmobs.annotate(span=retrieval_span, input_data=[0, 1, 2, 3, 4]) assert retrieval_span._get_ctx_item(INPUT_VALUE) == str([0, 1, 2, 3, 4]) -def test_annotate_input_llm_message(LLMObs): - with LLMObs.llm(model_name="test_model") as span: - LLMObs.annotate(span=span, input_data=[{"content": "test_input", "role": "human"}]) +def test_annotate_input_llm_message(llmobs): + with llmobs.llm(model_name="test_model") as span: + llmobs.annotate(span=span, input_data=[{"content": "test_input", "role": "human"}]) assert span._get_ctx_item(INPUT_MESSAGES) == [{"content": "test_input", "role": "human"}] -def test_annotate_input_llm_message_wrong_type(LLMObs, mock_logs): - with LLMObs.llm(model_name="test_model") as span: - LLMObs.annotate(span=span, input_data=[{"content": object()}]) +def test_annotate_input_llm_message_wrong_type(llmobs, mock_llmobs_logs): + with llmobs.llm(model_name="test_model") as span: + llmobs.annotate(span=span, input_data=[{"content": object()}]) assert span._get_ctx_item(INPUT_MESSAGES) is None - mock_logs.warning.assert_called_once_with("Failed to parse input messages.", exc_info=True) + mock_llmobs_logs.warning.assert_called_once_with("Failed to parse input messages.", exc_info=True) -def test_llmobs_annotate_incorrect_message_content_type_raises_warning(LLMObs, mock_logs): - with LLMObs.llm(model_name="test_model") as span: - LLMObs.annotate(span=span, input_data={"role": "user", "content": {"nested": "yes"}}) - mock_logs.warning.assert_called_once_with("Failed to parse input messages.", exc_info=True) - mock_logs.reset_mock() - LLMObs.annotate(span=span, output_data={"role": "user", "content": {"nested": "yes"}}) - mock_logs.warning.assert_called_once_with("Failed to parse output messages.", exc_info=True) +def test_llmobs_annotate_incorrect_message_content_type_raises_warning(llmobs, mock_llmobs_logs): + with llmobs.llm(model_name="test_model") as span: + llmobs.annotate(span=span, input_data={"role": "user", "content": {"nested": "yes"}}) + mock_llmobs_logs.warning.assert_called_once_with("Failed to parse input messages.", exc_info=True) + mock_llmobs_logs.reset_mock() + llmobs.annotate(span=span, output_data={"role": "user", "content": {"nested": "yes"}}) + mock_llmobs_logs.warning.assert_called_once_with("Failed to parse output messages.", exc_info=True) -def test_annotate_document_str(LLMObs): - with LLMObs.embedding(model_name="test_model") as span: - LLMObs.annotate(span=span, input_data="test_document_text") +def test_annotate_document_str(llmobs): + with llmobs.embedding(model_name="test_model") as span: + llmobs.annotate(span=span, input_data="test_document_text") documents = span._get_ctx_item(INPUT_DOCUMENTS) assert documents assert len(documents) == 1 assert documents[0]["text"] == "test_document_text" - with LLMObs.retrieval() as span: - LLMObs.annotate(span=span, output_data="test_document_text") + with llmobs.retrieval() as span: + llmobs.annotate(span=span, output_data="test_document_text") documents = span._get_ctx_item(OUTPUT_DOCUMENTS) assert documents assert len(documents) == 1 assert documents[0]["text"] == "test_document_text" -def test_annotate_document_dict(LLMObs): - with LLMObs.embedding(model_name="test_model") as span: - LLMObs.annotate(span=span, input_data={"text": "test_document_text"}) +def test_annotate_document_dict(llmobs): + with llmobs.embedding(model_name="test_model") as span: + llmobs.annotate(span=span, input_data={"text": "test_document_text"}) documents = span._get_ctx_item(INPUT_DOCUMENTS) assert documents assert len(documents) == 1 assert documents[0]["text"] == "test_document_text" - with LLMObs.retrieval() as span: - LLMObs.annotate(span=span, output_data={"text": "test_document_text"}) + with llmobs.retrieval() as span: + llmobs.annotate(span=span, output_data={"text": "test_document_text"}) documents = span._get_ctx_item(OUTPUT_DOCUMENTS) assert documents assert len(documents) == 1 assert documents[0]["text"] == "test_document_text" -def test_annotate_document_list(LLMObs): - with LLMObs.embedding(model_name="test_model") as span: - LLMObs.annotate( +def test_annotate_document_list(llmobs): + with llmobs.embedding(model_name="test_model") as span: + llmobs.annotate( span=span, input_data=[{"text": "test_document_text"}, {"text": "text", "name": "name", "score": 0.9, "id": "id"}], ) @@ -605,8 +528,8 @@ def test_annotate_document_list(LLMObs): assert documents[1]["name"] == "name" assert documents[1]["id"] == "id" assert documents[1]["score"] == 0.9 - with LLMObs.retrieval() as span: - LLMObs.annotate( + with llmobs.retrieval() as span: + llmobs.annotate( span=span, output_data=[{"text": "test_document_text"}, {"text": "text", "name": "name", "score": 0.9, "id": "id"}], ) @@ -620,129 +543,131 @@ def test_annotate_document_list(LLMObs): assert documents[1]["score"] == 0.9 -def test_annotate_incorrect_document_type_raises_warning(LLMObs, mock_logs): - with LLMObs.embedding(model_name="test_model") as span: - LLMObs.annotate(span=span, input_data={"text": 123}) - mock_logs.warning.assert_called_once_with("Failed to parse input documents.", exc_info=True) - mock_logs.reset_mock() - LLMObs.annotate(span=span, input_data=123) - mock_logs.warning.assert_called_once_with("Failed to parse input documents.", exc_info=True) - mock_logs.reset_mock() - LLMObs.annotate(span=span, input_data=object()) - mock_logs.warning.assert_called_once_with("Failed to parse input documents.", exc_info=True) - mock_logs.reset_mock() - with LLMObs.retrieval() as span: - LLMObs.annotate(span=span, output_data=[{"score": 0.9, "id": "id", "name": "name"}]) - mock_logs.warning.assert_called_once_with("Failed to parse output documents.", exc_info=True) - mock_logs.reset_mock() - LLMObs.annotate(span=span, output_data=123) - mock_logs.warning.assert_called_once_with("Failed to parse output documents.", exc_info=True) - mock_logs.reset_mock() - LLMObs.annotate(span=span, output_data=object()) - mock_logs.warning.assert_called_once_with("Failed to parse output documents.", exc_info=True) - - -def test_annotate_document_no_text_raises_warning(LLMObs, mock_logs): - with LLMObs.embedding(model_name="test_model") as span: - LLMObs.annotate(span=span, input_data=[{"score": 0.9, "id": "id", "name": "name"}]) - mock_logs.warning.assert_called_once_with("Failed to parse input documents.", exc_info=True) - mock_logs.reset_mock() - with LLMObs.retrieval() as span: - LLMObs.annotate(span=span, output_data=[{"score": 0.9, "id": "id", "name": "name"}]) - mock_logs.warning.assert_called_once_with("Failed to parse output documents.", exc_info=True) - - -def test_annotate_incorrect_document_field_type_raises_warning(LLMObs, mock_logs): - with LLMObs.embedding(model_name="test_model") as span: - LLMObs.annotate(span=span, input_data=[{"text": "test_document_text", "score": "0.9"}]) - mock_logs.warning.assert_called_once_with("Failed to parse input documents.", exc_info=True) - mock_logs.reset_mock() - with LLMObs.embedding(model_name="test_model") as span: - LLMObs.annotate( +def test_annotate_incorrect_document_type_raises_warning(llmobs, mock_llmobs_logs): + with llmobs.embedding(model_name="test_model") as span: + llmobs.annotate(span=span, input_data={"text": 123}) + mock_llmobs_logs.warning.assert_called_once_with("Failed to parse input documents.", exc_info=True) + mock_llmobs_logs.reset_mock() + llmobs.annotate(span=span, input_data=123) + mock_llmobs_logs.warning.assert_called_once_with("Failed to parse input documents.", exc_info=True) + mock_llmobs_logs.reset_mock() + llmobs.annotate(span=span, input_data=object()) + mock_llmobs_logs.warning.assert_called_once_with("Failed to parse input documents.", exc_info=True) + mock_llmobs_logs.reset_mock() + with llmobs.retrieval() as span: + llmobs.annotate(span=span, output_data=[{"score": 0.9, "id": "id", "name": "name"}]) + mock_llmobs_logs.warning.assert_called_once_with("Failed to parse output documents.", exc_info=True) + mock_llmobs_logs.reset_mock() + llmobs.annotate(span=span, output_data=123) + mock_llmobs_logs.warning.assert_called_once_with("Failed to parse output documents.", exc_info=True) + mock_llmobs_logs.reset_mock() + llmobs.annotate(span=span, output_data=object()) + mock_llmobs_logs.warning.assert_called_once_with("Failed to parse output documents.", exc_info=True) + + +def test_annotate_document_no_text_raises_warning(llmobs, mock_llmobs_logs): + with llmobs.embedding(model_name="test_model") as span: + llmobs.annotate(span=span, input_data=[{"score": 0.9, "id": "id", "name": "name"}]) + mock_llmobs_logs.warning.assert_called_once_with("Failed to parse input documents.", exc_info=True) + mock_llmobs_logs.reset_mock() + with llmobs.retrieval() as span: + llmobs.annotate(span=span, output_data=[{"score": 0.9, "id": "id", "name": "name"}]) + mock_llmobs_logs.warning.assert_called_once_with("Failed to parse output documents.", exc_info=True) + + +def test_annotate_incorrect_document_field_type_raises_warning(llmobs, mock_llmobs_logs): + with llmobs.embedding(model_name="test_model") as span: + llmobs.annotate(span=span, input_data=[{"text": "test_document_text", "score": "0.9"}]) + mock_llmobs_logs.warning.assert_called_once_with("Failed to parse input documents.", exc_info=True) + mock_llmobs_logs.reset_mock() + with llmobs.embedding(model_name="test_model") as span: + llmobs.annotate( span=span, input_data=[{"text": "text", "id": 123, "score": "0.9", "name": ["h", "e", "l", "l", "o"]}] ) - mock_logs.warning.assert_called_once_with("Failed to parse input documents.", exc_info=True) - mock_logs.reset_mock() - with LLMObs.retrieval() as span: - LLMObs.annotate(span=span, output_data=[{"text": "test_document_text", "score": "0.9"}]) - mock_logs.warning.assert_called_once_with("Failed to parse output documents.", exc_info=True) - mock_logs.reset_mock() - with LLMObs.retrieval() as span: - LLMObs.annotate( + mock_llmobs_logs.warning.assert_called_once_with("Failed to parse input documents.", exc_info=True) + mock_llmobs_logs.reset_mock() + with llmobs.retrieval() as span: + llmobs.annotate(span=span, output_data=[{"text": "test_document_text", "score": "0.9"}]) + mock_llmobs_logs.warning.assert_called_once_with("Failed to parse output documents.", exc_info=True) + mock_llmobs_logs.reset_mock() + with llmobs.retrieval() as span: + llmobs.annotate( span=span, output_data=[{"text": "text", "id": 123, "score": "0.9", "name": ["h", "e", "l", "l", "o"]}] ) - mock_logs.warning.assert_called_once_with("Failed to parse output documents.", exc_info=True) + mock_llmobs_logs.warning.assert_called_once_with("Failed to parse output documents.", exc_info=True) -def test_annotate_output_string(LLMObs): - with LLMObs.llm(model_name="test_model") as llm_span: - LLMObs.annotate(span=llm_span, output_data="test_output") +def test_annotate_output_string(llmobs): + with llmobs.llm(model_name="test_model") as llm_span: + llmobs.annotate(span=llm_span, output_data="test_output") assert llm_span._get_ctx_item(OUTPUT_MESSAGES) == [{"content": "test_output"}] - with LLMObs.embedding(model_name="test_model") as embedding_span: - LLMObs.annotate(span=embedding_span, output_data="test_output") + with llmobs.embedding(model_name="test_model") as embedding_span: + llmobs.annotate(span=embedding_span, output_data="test_output") assert embedding_span._get_ctx_item(OUTPUT_VALUE) == "test_output" - with LLMObs.task() as task_span: - LLMObs.annotate(span=task_span, output_data="test_output") + with llmobs.task() as task_span: + llmobs.annotate(span=task_span, output_data="test_output") assert task_span._get_ctx_item(OUTPUT_VALUE) == "test_output" - with LLMObs.tool() as tool_span: - LLMObs.annotate(span=tool_span, output_data="test_output") + with llmobs.tool() as tool_span: + llmobs.annotate(span=tool_span, output_data="test_output") assert tool_span._get_ctx_item(OUTPUT_VALUE) == "test_output" - with LLMObs.workflow() as workflow_span: - LLMObs.annotate(span=workflow_span, output_data="test_output") + with llmobs.workflow() as workflow_span: + llmobs.annotate(span=workflow_span, output_data="test_output") assert workflow_span._get_ctx_item(OUTPUT_VALUE) == "test_output" - with LLMObs.agent() as agent_span: - LLMObs.annotate(span=agent_span, output_data="test_output") + with llmobs.agent() as agent_span: + llmobs.annotate(span=agent_span, output_data="test_output") assert agent_span._get_ctx_item(OUTPUT_VALUE) == "test_output" -def test_annotate_output_serializable_value(LLMObs): - with LLMObs.embedding(model_name="test_model") as embedding_span: - LLMObs.annotate(span=embedding_span, output_data=[[0, 1, 2, 3], [4, 5, 6, 7]]) +def test_annotate_output_serializable_value(llmobs): + with llmobs.embedding(model_name="test_model") as embedding_span: + llmobs.annotate(span=embedding_span, output_data=[[0, 1, 2, 3], [4, 5, 6, 7]]) assert embedding_span._get_ctx_item(OUTPUT_VALUE) == str([[0, 1, 2, 3], [4, 5, 6, 7]]) - with LLMObs.task() as task_span: - LLMObs.annotate(span=task_span, output_data=["test_output"]) + with llmobs.task() as task_span: + llmobs.annotate(span=task_span, output_data=["test_output"]) assert task_span._get_ctx_item(OUTPUT_VALUE) == str(["test_output"]) - with LLMObs.tool() as tool_span: - LLMObs.annotate(span=tool_span, output_data={"test_output": "hello world"}) + with llmobs.tool() as tool_span: + llmobs.annotate(span=tool_span, output_data={"test_output": "hello world"}) assert tool_span._get_ctx_item(OUTPUT_VALUE) == str({"test_output": "hello world"}) - with LLMObs.workflow() as workflow_span: - LLMObs.annotate(span=workflow_span, output_data=("asd", 123)) + with llmobs.workflow() as workflow_span: + llmobs.annotate(span=workflow_span, output_data=("asd", 123)) assert workflow_span._get_ctx_item(OUTPUT_VALUE) == str(("asd", 123)) - with LLMObs.agent() as agent_span: - LLMObs.annotate(span=agent_span, output_data="test_output") + with llmobs.agent() as agent_span: + llmobs.annotate(span=agent_span, output_data="test_output") assert agent_span._get_ctx_item(OUTPUT_VALUE) == "test_output" -def test_annotate_output_llm_message(LLMObs): - with LLMObs.llm(model_name="test_model") as llm_span: - LLMObs.annotate(span=llm_span, output_data=[{"content": "test_output", "role": "human"}]) +def test_annotate_output_llm_message(llmobs): + with llmobs.llm(model_name="test_model") as llm_span: + llmobs.annotate(span=llm_span, output_data=[{"content": "test_output", "role": "human"}]) assert llm_span._get_ctx_item(OUTPUT_MESSAGES) == [{"content": "test_output", "role": "human"}] -def test_annotate_output_llm_message_wrong_type(LLMObs, mock_logs): - with LLMObs.llm(model_name="test_model") as llm_span: - LLMObs.annotate(span=llm_span, output_data=[{"content": object()}]) +def test_annotate_output_llm_message_wrong_type(llmobs, mock_llmobs_logs): + with llmobs.llm(model_name="test_model") as llm_span: + llmobs.annotate(span=llm_span, output_data=[{"content": object()}]) assert llm_span._get_ctx_item(OUTPUT_MESSAGES) is None - mock_logs.warning.assert_called_once_with("Failed to parse output messages.", exc_info=True) + mock_llmobs_logs.warning.assert_called_once_with("Failed to parse output messages.", exc_info=True) -def test_annotate_metrics(LLMObs): - with LLMObs.llm(model_name="test_model") as span: - LLMObs.annotate(span=span, metrics={"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}) +def test_annotate_metrics(llmobs): + with llmobs.llm(model_name="test_model") as span: + llmobs.annotate(span=span, metrics={"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}) assert span._get_ctx_item(METRICS) == {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30} -def test_annotate_metrics_wrong_type(LLMObs, mock_logs): - with LLMObs.llm(model_name="test_model") as llm_span: - LLMObs.annotate(span=llm_span, metrics=12345) +def test_annotate_metrics_wrong_type(llmobs, mock_llmobs_logs): + with llmobs.llm(model_name="test_model") as llm_span: + llmobs.annotate(span=llm_span, metrics=12345) assert llm_span._get_ctx_item(METRICS) is None - mock_logs.warning.assert_called_once_with("metrics must be a dictionary of string key - numeric value pairs.") - mock_logs.reset_mock() + mock_llmobs_logs.warning.assert_called_once_with( + "metrics must be a dictionary of string key - numeric value pairs." + ) + mock_llmobs_logs.reset_mock() -def test_annotate_prompt_dict(LLMObs): - with LLMObs.llm(model_name="test_model") as span: - LLMObs.annotate( +def test_annotate_prompt_dict(llmobs): + with llmobs.llm(model_name="test_model") as span: + llmobs.annotate( span=span, prompt={ "template": "{var1} {var3}", @@ -761,9 +686,9 @@ def test_annotate_prompt_dict(LLMObs): } -def test_annotate_prompt_dict_with_context_var_keys(LLMObs): - with LLMObs.llm(model_name="test_model") as span: - LLMObs.annotate( +def test_annotate_prompt_dict_with_context_var_keys(llmobs): + with llmobs.llm(model_name="test_model") as span: + llmobs.annotate( span=span, prompt={ "template": "{var1} {var3}", @@ -784,9 +709,9 @@ def test_annotate_prompt_dict_with_context_var_keys(LLMObs): } -def test_annotate_prompt_typed_dict(LLMObs): - with LLMObs.llm(model_name="test_model") as span: - LLMObs.annotate( +def test_annotate_prompt_typed_dict(llmobs): + with llmobs.llm(model_name="test_model") as span: + llmobs.annotate( span=span, prompt=Prompt( template="{var1} {var3}", @@ -807,47 +732,30 @@ def test_annotate_prompt_typed_dict(LLMObs): } -def test_annotate_prompt_wrong_type(LLMObs, mock_logs): - with LLMObs.llm(model_name="test_model") as span: - LLMObs.annotate(span=span, prompt="prompt") +def test_annotate_prompt_wrong_type(llmobs, mock_llmobs_logs): + with llmobs.llm(model_name="test_model") as span: + llmobs.annotate(span=span, prompt="prompt") assert span._get_ctx_item(INPUT_PROMPT) is None - mock_logs.warning.assert_called_once_with("Failed to validate prompt with error: ", exc_info=True) - mock_logs.reset_mock() + mock_llmobs_logs.warning.assert_called_once_with("Failed to validate prompt with error: ", exc_info=True) + mock_llmobs_logs.reset_mock() - LLMObs.annotate(span=span, prompt={"template": 1}) - mock_logs.warning.assert_called_once_with("Failed to validate prompt with error: ", exc_info=True) - mock_logs.reset_mock() + llmobs.annotate(span=span, prompt={"template": 1}) + mock_llmobs_logs.warning.assert_called_once_with("Failed to validate prompt with error: ", exc_info=True) + mock_llmobs_logs.reset_mock() -def test_span_error_sets_error(LLMObs, mock_llmobs_span_writer): +def test_span_error_sets_error(llmobs, llmobs_events): with pytest.raises(ValueError): - with LLMObs.llm(model_name="test_model", model_provider="test_model_provider") as span: + with llmobs.llm(model_name="test_model", model_provider="test_model_provider") as span: raise ValueError("test error message") - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, - model_name="test_model", - model_provider="test_model_provider", - error="builtins.ValueError", - error_message="test error message", - error_stack=span.get_tag("error.stack"), - ) - ) - - -def test_span_error_sets_error_agentless(AgentlessLLMObs, mock_llmobs_span_agentless_writer): - with pytest.raises(ValueError): - with AgentlessLLMObs.llm(model_name="test_model", model_provider="test_model_provider") as span: - raise ValueError("test error message") - mock_llmobs_span_agentless_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, - model_name="test_model", - model_provider="test_model_provider", - error="builtins.ValueError", - error_message="test error message", - error_stack=span.get_tag("error.stack"), - ) + assert len(llmobs_events) == 1 + assert llmobs_events[0] == _expected_llmobs_llm_span_event( + span, + model_name="test_model", + model_provider="test_model_provider", + error="builtins.ValueError", + error_message="test error message", + error_stack=span.get_tag("error.stack"), ) @@ -855,218 +763,142 @@ def test_span_error_sets_error_agentless(AgentlessLLMObs, mock_llmobs_span_agent "ddtrace_global_config", [dict(version="1.2.3", env="test_env", service="test_service", _llmobs_ml_app="test_app_name")], ) -def test_tags(ddtrace_global_config, LLMObs, mock_llmobs_span_writer, monkeypatch): - with LLMObs.task(name="test_task") as span: - pass - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event( - span, - "task", - tags={"version": "1.2.3", "env": "test_env", "service": "test_service", "ml_app": "test_app_name"}, - ) - ) - - -@pytest.mark.parametrize( - "ddtrace_global_config", - [dict(version="1.2.3", env="test_env", service="test_service", _llmobs_ml_app="test_app_name")], -) -def test_tags_agentless(ddtrace_global_config, AgentlessLLMObs, mock_llmobs_span_agentless_writer, monkeypatch): - with AgentlessLLMObs.task(name="test_task") as span: - pass - mock_llmobs_span_agentless_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event( - span, - "task", - tags={"version": "1.2.3", "env": "test_env", "service": "test_service", "ml_app": "test_app_name"}, - ) - ) - - -def test_ml_app_override(LLMObs, mock_llmobs_span_writer): - with LLMObs.task(name="test_task", ml_app="test_app") as span: - pass - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event(span, "task", tags={"ml_app": "test_app"}) - ) - with LLMObs.tool(name="test_tool", ml_app="test_app") as span: - pass - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event(span, "tool", tags={"ml_app": "test_app"}) - ) - with LLMObs.llm(model_name="model_name", name="test_llm", ml_app="test_app") as span: - pass - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, "llm", model_name="model_name", model_provider="custom", tags={"ml_app": "test_app"} - ) - ) - with LLMObs.embedding(model_name="model_name", name="test_embedding", ml_app="test_app") as span: - pass - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, "embedding", model_name="model_name", model_provider="custom", tags={"ml_app": "test_app"} - ) - ) - with LLMObs.workflow(name="test_workflow", ml_app="test_app") as span: - pass - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event(span, "workflow", tags={"ml_app": "test_app"}) - ) - with LLMObs.agent(name="test_agent", ml_app="test_app") as span: - pass - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event(span, "agent", tags={"ml_app": "test_app"}) - ) - with LLMObs.retrieval(name="test_retrieval", ml_app="test_app") as span: +def test_tags(ddtrace_global_config, llmobs, llmobs_events, monkeypatch): + with llmobs.task(name="test_task") as span: pass - mock_llmobs_span_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event(span, "retrieval", tags={"ml_app": "test_app"}) + assert len(llmobs_events) == 1 + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event( + span, + "task", + tags={"version": "1.2.3", "env": "test_env", "service": "test_service", "ml_app": "test_app_name"}, ) -def test_ml_app_override_agentless(AgentlessLLMObs, mock_llmobs_span_agentless_writer): - with AgentlessLLMObs.task(name="test_task", ml_app="test_app") as span: +def test_ml_app_override(llmobs, llmobs_events): + with llmobs.task(name="test_task", ml_app="test_app") as span: pass - mock_llmobs_span_agentless_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event(span, "task", tags={"ml_app": "test_app"}) - ) - with AgentlessLLMObs.tool(name="test_tool", ml_app="test_app") as span: + assert len(llmobs_events) == 1 + assert llmobs_events[0] == _expected_llmobs_non_llm_span_event(span, "task", tags={"ml_app": "test_app"}) + with llmobs.tool(name="test_tool", ml_app="test_app") as span: pass - mock_llmobs_span_agentless_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event(span, "tool", tags={"ml_app": "test_app"}) - ) - with AgentlessLLMObs.llm(model_name="model_name", name="test_llm", ml_app="test_app") as span: + assert len(llmobs_events) == 2 + assert llmobs_events[1] == _expected_llmobs_non_llm_span_event(span, "tool", tags={"ml_app": "test_app"}) + with llmobs.llm(model_name="model_name", name="test_llm", ml_app="test_app") as span: pass - mock_llmobs_span_agentless_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, "llm", model_name="model_name", model_provider="custom", tags={"ml_app": "test_app"} - ) + assert len(llmobs_events) == 3 + assert llmobs_events[2] == _expected_llmobs_llm_span_event( + span, "llm", model_name="model_name", model_provider="custom", tags={"ml_app": "test_app"} ) - with AgentlessLLMObs.embedding(model_name="model_name", name="test_embedding", ml_app="test_app") as span: + with llmobs.embedding(model_name="model_name", name="test_embedding", ml_app="test_app") as span: pass - mock_llmobs_span_agentless_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, "embedding", model_name="model_name", model_provider="custom", tags={"ml_app": "test_app"} - ) + assert len(llmobs_events) == 4 + assert llmobs_events[3] == _expected_llmobs_llm_span_event( + span, "embedding", model_name="model_name", model_provider="custom", tags={"ml_app": "test_app"} ) - with AgentlessLLMObs.workflow(name="test_workflow", ml_app="test_app") as span: + with llmobs.workflow(name="test_workflow", ml_app="test_app") as span: pass - mock_llmobs_span_agentless_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event(span, "workflow", tags={"ml_app": "test_app"}) - ) - with AgentlessLLMObs.agent(name="test_agent", ml_app="test_app") as span: + assert len(llmobs_events) == 5 + assert llmobs_events[4] == _expected_llmobs_non_llm_span_event(span, "workflow", tags={"ml_app": "test_app"}) + with llmobs.agent(name="test_agent", ml_app="test_app") as span: pass - mock_llmobs_span_agentless_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event(span, "agent", tags={"ml_app": "test_app"}) - ) - with AgentlessLLMObs.retrieval(name="test_retrieval", ml_app="test_app") as span: + assert len(llmobs_events) == 6 + assert llmobs_events[5] == _expected_llmobs_llm_span_event(span, "agent", tags={"ml_app": "test_app"}) + with llmobs.retrieval(name="test_retrieval", ml_app="test_app") as span: pass - mock_llmobs_span_agentless_writer.enqueue.assert_called_with( - _expected_llmobs_non_llm_span_event(span, "retrieval", tags={"ml_app": "test_app"}) - ) + assert len(llmobs_events) == 7 + assert llmobs_events[6] == _expected_llmobs_non_llm_span_event(span, "retrieval", tags={"ml_app": "test_app"}) -def test_export_span_specified_span_is_incorrect_type_raises_warning(LLMObs, mock_logs): - LLMObs.export_span(span="asd") - mock_logs.warning.assert_called_once_with("Failed to export span. Span must be a valid Span object.") +def test_export_span_specified_span_is_incorrect_type_raises_warning(llmobs, mock_llmobs_logs): + llmobs.export_span(span="asd") + mock_llmobs_logs.warning.assert_called_once_with("Failed to export span. Span must be a valid Span object.") -def test_export_span_specified_span_is_not_llmobs_span_raises_warning(LLMObs, mock_logs): +def test_export_span_specified_span_is_not_llmobs_span_raises_warning(llmobs, mock_llmobs_logs): with DummyTracer().trace("non_llmobs_span") as span: - LLMObs.export_span(span=span) - mock_logs.warning.assert_called_once_with("Span must be an LLMObs-generated span.") + llmobs.export_span(span=span) + mock_llmobs_logs.warning.assert_called_once_with("Span must be an LLMObs-generated span.") -def test_export_span_specified_span_returns_span_context(LLMObs): - with LLMObs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: - span_context = LLMObs.export_span(span=span) +def test_export_span_specified_span_returns_span_context(llmobs): + with llmobs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: + span_context = llmobs.export_span(span=span) assert span_context is not None assert span_context["span_id"] == str(span.span_id) assert span_context["trace_id"] == "{:x}".format(span.trace_id) -def test_export_span_no_specified_span_no_active_span_raises_warning(LLMObs, mock_logs): - LLMObs.export_span() - mock_logs.warning.assert_called_once_with("No span provided and no active LLMObs-generated span found.") +def test_export_span_no_specified_span_no_active_span_raises_warning(llmobs, mock_llmobs_logs): + llmobs.export_span() + mock_llmobs_logs.warning.assert_called_once_with("No span provided and no active LLMObs-generated span found.") -def test_export_span_active_span_not_llmobs_span_raises_warning(LLMObs, mock_logs): - with LLMObs._instance.tracer.trace("non_llmobs_span"): - LLMObs.export_span() - mock_logs.warning.assert_called_once_with("Span must be an LLMObs-generated span.") +def test_export_span_active_span_not_llmobs_span_raises_warning(llmobs, mock_llmobs_logs): + with llmobs._instance.tracer.trace("non_llmobs_span"): + llmobs.export_span() + mock_llmobs_logs.warning.assert_called_once_with("Span must be an LLMObs-generated span.") -def test_export_span_no_specified_span_returns_exported_active_span(LLMObs): - with LLMObs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: - span_context = LLMObs.export_span() +def test_export_span_no_specified_span_returns_exported_active_span(llmobs): + with llmobs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: + span_context = llmobs.export_span() assert span_context is not None assert span_context["span_id"] == str(span.span_id) assert span_context["trace_id"] == "{:x}".format(span.trace_id) -def test_submit_evaluation_llmobs_disabled_raises_warning(LLMObs, mock_logs): - LLMObs.disable() - LLMObs.submit_evaluation( - span_context={"span_id": "123", "trace_id": "456"}, label="toxicity", metric_type="categorical", value="high" - ) - mock_logs.warning.assert_called_once_with( - "LLMObs.submit_evaluation() called when LLMObs is not enabled. Evaluation metric data will not be sent." - ) - - -def test_submit_evaluation_no_api_key_raises_warning(AgentlessLLMObs, mock_logs): +def test_submit_evaluation_no_api_key_raises_warning(llmobs, mock_llmobs_logs): with override_global_config(dict(_dd_api_key="")): - AgentlessLLMObs.submit_evaluation( + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="toxicity", metric_type="categorical", value="high", ) - mock_logs.warning.assert_called_once_with( + mock_llmobs_logs.warning.assert_called_once_with( "DD_API_KEY is required for sending evaluation metrics. Evaluation metric data will not be sent. " "Ensure this configuration is set before running your application." ) -def test_submit_evaluation_ml_app_raises_warning(LLMObs, mock_logs): +def test_submit_evaluation_ml_app_raises_warning(llmobs, mock_llmobs_logs): with override_global_config(dict(_llmobs_ml_app="")): - LLMObs.submit_evaluation( + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="toxicity", metric_type="categorical", value="high", ) - mock_logs.warning.assert_called_once_with( + mock_llmobs_logs.warning.assert_called_once_with( "ML App name is required for sending evaluation metrics. Evaluation metric data will not be sent. " "Ensure this configuration is set before running your application." ) -def test_submit_evaluation_span_context_incorrect_type_raises_warning(LLMObs, mock_logs): - LLMObs.submit_evaluation(span_context="asd", label="toxicity", metric_type="categorical", value="high") - mock_logs.warning.assert_called_once_with( +def test_submit_evaluation_span_context_incorrect_type_raises_warning(llmobs, mock_llmobs_logs): + llmobs.submit_evaluation(span_context="asd", label="toxicity", metric_type="categorical", value="high") + mock_llmobs_logs.warning.assert_called_once_with( "span_context must be a dictionary containing both span_id and trace_id keys. " "LLMObs.export_span() can be used to generate this dictionary from a given span." ) -def test_submit_evaluation_empty_span_or_trace_id_raises_warning(LLMObs, mock_logs): - LLMObs.submit_evaluation( +def test_submit_evaluation_empty_span_or_trace_id_raises_warning(llmobs, mock_llmobs_logs): + llmobs.submit_evaluation( span_context={"trace_id": "456"}, label="toxicity", metric_type="categorical", value="high" ) - mock_logs.warning.assert_called_once_with( + mock_llmobs_logs.warning.assert_called_once_with( "span_id and trace_id must both be specified for the given evaluation metric to be submitted." ) - mock_logs.reset_mock() - LLMObs.submit_evaluation(span_context={"span_id": "456"}, label="toxicity", metric_type="categorical", value="high") - mock_logs.warning.assert_called_once_with( + mock_llmobs_logs.reset_mock() + llmobs.submit_evaluation(span_context={"span_id": "456"}, label="toxicity", metric_type="categorical", value="high") + mock_llmobs_logs.warning.assert_called_once_with( "span_id and trace_id must both be specified for the given evaluation metric to be submitted." ) -def test_submit_evaluation_invalid_timestamp_raises_warning(LLMObs, mock_logs): - LLMObs.submit_evaluation( +def test_submit_evaluation_invalid_timestamp_raises_warning(llmobs, mock_llmobs_logs): + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="", metric_type="categorical", @@ -1074,35 +906,35 @@ def test_submit_evaluation_invalid_timestamp_raises_warning(LLMObs, mock_logs): ml_app="dummy", timestamp_ms="invalid", ) - mock_logs.warning.assert_called_once_with( + mock_llmobs_logs.warning.assert_called_once_with( "timestamp_ms must be a non-negative integer. Evaluation metric data will not be sent" ) -def test_submit_evaluation_empty_label_raises_warning(LLMObs, mock_logs): - LLMObs.submit_evaluation( +def test_submit_evaluation_empty_label_raises_warning(llmobs, mock_llmobs_logs): + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="", metric_type="categorical", value="high" ) - mock_logs.warning.assert_called_once_with("label must be the specified name of the evaluation metric.") + mock_llmobs_logs.warning.assert_called_once_with("label must be the specified name of the evaluation metric.") -def test_submit_evaluation_incorrect_metric_type_raises_warning(LLMObs, mock_logs): - LLMObs.submit_evaluation( +def test_submit_evaluation_incorrect_metric_type_raises_warning(llmobs, mock_llmobs_logs): + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="toxicity", metric_type="wrong", value="high" ) - mock_logs.warning.assert_called_once_with("metric_type must be one of 'categorical' or 'score'.") - mock_logs.reset_mock() - LLMObs.submit_evaluation( + mock_llmobs_logs.warning.assert_called_once_with("metric_type must be one of 'categorical' or 'score'.") + mock_llmobs_logs.reset_mock() + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="toxicity", metric_type="", value="high" ) - mock_logs.warning.assert_called_once_with("metric_type must be one of 'categorical' or 'score'.") + mock_llmobs_logs.warning.assert_called_once_with("metric_type must be one of 'categorical' or 'score'.") -def test_submit_evaluation_numerical_value_raises_unsupported_warning(LLMObs, mock_logs): - LLMObs.submit_evaluation( +def test_submit_evaluation_numerical_value_raises_unsupported_warning(llmobs, mock_llmobs_logs): + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="token_count", metric_type="numerical", value="high" ) - mock_logs.warning.assert_has_calls( + mock_llmobs_logs.warning.assert_has_calls( [ mock.call( "The evaluation metric type 'numerical' is unsupported. Use 'score' instead. " @@ -1112,44 +944,44 @@ def test_submit_evaluation_numerical_value_raises_unsupported_warning(LLMObs, mo ) -def test_submit_evaluation_incorrect_numerical_value_type_raises_warning(LLMObs, mock_logs): - LLMObs.submit_evaluation( +def test_submit_evaluation_incorrect_numerical_value_type_raises_warning(llmobs, mock_llmobs_logs): + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="token_count", metric_type="numerical", value="high" ) - mock_logs.warning.assert_has_calls( + mock_llmobs_logs.warning.assert_has_calls( [ mock.call("value must be an integer or float for a score metric."), ] ) -def test_submit_evaluation_incorrect_score_value_type_raises_warning(LLMObs, mock_logs): - LLMObs.submit_evaluation( +def test_submit_evaluation_incorrect_score_value_type_raises_warning(llmobs, mock_llmobs_logs): + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="token_count", metric_type="score", value="high" ) - mock_logs.warning.assert_called_once_with("value must be an integer or float for a score metric.") + mock_llmobs_logs.warning.assert_called_once_with("value must be an integer or float for a score metric.") -def test_submit_evaluation_invalid_tags_raises_warning(LLMObs, mock_logs): - LLMObs.submit_evaluation( +def test_submit_evaluation_invalid_tags_raises_warning(llmobs, mock_llmobs_logs): + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="toxicity", metric_type="categorical", value="high", tags=["invalid"], ) - mock_logs.warning.assert_called_once_with("tags must be a dictionary of string key-value pairs.") + mock_llmobs_logs.warning.assert_called_once_with("tags must be a dictionary of string key-value pairs.") -def test_submit_evaluation_invalid_metadata_raises_warning(LLMObs, mock_logs): - LLMObs.submit_evaluation( +def test_submit_evaluation_invalid_metadata_raises_warning(llmobs, mock_llmobs_logs): + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="toxicity", metric_type="categorical", value="high", metadata=1, ) - mock_logs.warning.assert_called_once_with("metadata must be json serializable dictionary.") + mock_llmobs_logs.warning.assert_called_once_with("metadata must be json serializable dictionary.") @pytest.mark.parametrize( @@ -1157,9 +989,9 @@ def test_submit_evaluation_invalid_metadata_raises_warning(LLMObs, mock_logs): [dict(_llmobs_ml_app="test_app_name")], ) def test_submit_evaluation_non_string_tags_raises_warning_but_still_submits( - LLMObs, mock_logs, mock_llmobs_eval_metric_writer + llmobs, mock_llmobs_logs, mock_llmobs_eval_metric_writer ): - LLMObs.submit_evaluation( + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="toxicity", metric_type="categorical", @@ -1167,8 +999,10 @@ def test_submit_evaluation_non_string_tags_raises_warning_but_still_submits( tags={1: 2, "foo": "bar"}, ml_app="dummy", ) - mock_logs.warning.assert_called_once_with("Failed to parse tags. Tags for evaluation metrics must be strings.") - mock_logs.reset_mock() + mock_llmobs_logs.warning.assert_called_once_with( + "Failed to parse tags. Tags for evaluation metrics must be strings." + ) + mock_llmobs_logs.reset_mock() mock_llmobs_eval_metric_writer.enqueue.assert_called_with( _expected_llmobs_eval_metric_event( ml_app="dummy", @@ -1186,8 +1020,8 @@ def test_submit_evaluation_non_string_tags_raises_warning_but_still_submits( "ddtrace_global_config", [dict(ddtrace="1.2.3", env="test_env", service="test_service", _llmobs_ml_app="test_app_name")], ) -def test_submit_evaluation_metric_tags(LLMObs, mock_llmobs_eval_metric_writer): - LLMObs.submit_evaluation( +def test_submit_evaluation_metric_tags(llmobs, mock_llmobs_eval_metric_writer): + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="toxicity", metric_type="categorical", @@ -1212,8 +1046,8 @@ def test_submit_evaluation_metric_tags(LLMObs, mock_llmobs_eval_metric_writer): "ddtrace_global_config", [dict(ddtrace="1.2.3", env="test_env", service="test_service", _llmobs_ml_app="test_app_name")], ) -def test_submit_evaluation_metric_with_metadata_enqueues_metric(LLMObs, mock_llmobs_eval_metric_writer): - LLMObs.submit_evaluation( +def test_submit_evaluation_metric_with_metadata_enqueues_metric(llmobs, mock_llmobs_eval_metric_writer): + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="toxicity", metric_type="categorical", @@ -1235,7 +1069,7 @@ def test_submit_evaluation_metric_with_metadata_enqueues_metric(LLMObs, mock_llm ) ) mock_llmobs_eval_metric_writer.reset() - LLMObs.submit_evaluation( + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="toxicity", metric_type="categorical", @@ -1257,8 +1091,8 @@ def test_submit_evaluation_metric_with_metadata_enqueues_metric(LLMObs, mock_llm ) -def test_submit_evaluation_enqueues_writer_with_categorical_metric(LLMObs, mock_llmobs_eval_metric_writer): - LLMObs.submit_evaluation( +def test_submit_evaluation_enqueues_writer_with_categorical_metric(llmobs, mock_llmobs_eval_metric_writer): + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="toxicity", metric_type="categorical", @@ -1276,9 +1110,9 @@ def test_submit_evaluation_enqueues_writer_with_categorical_metric(LLMObs, mock_ ) ) mock_llmobs_eval_metric_writer.reset_mock() - with LLMObs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: - LLMObs.submit_evaluation( - span_context=LLMObs.export_span(span), + with llmobs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: + llmobs.submit_evaluation( + span_context=llmobs.export_span(span), label="toxicity", metric_type="categorical", value="high", @@ -1296,8 +1130,8 @@ def test_submit_evaluation_enqueues_writer_with_categorical_metric(LLMObs, mock_ ) -def test_submit_evaluation_enqueues_writer_with_score_metric(LLMObs, mock_llmobs_eval_metric_writer): - LLMObs.submit_evaluation( +def test_submit_evaluation_enqueues_writer_with_score_metric(llmobs, mock_llmobs_eval_metric_writer): + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="sentiment", metric_type="score", @@ -1310,9 +1144,9 @@ def test_submit_evaluation_enqueues_writer_with_score_metric(LLMObs, mock_llmobs ) ) mock_llmobs_eval_metric_writer.reset_mock() - with LLMObs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: - LLMObs.submit_evaluation( - span_context=LLMObs.export_span(span), label="sentiment", metric_type="score", value=0.9, ml_app="dummy" + with llmobs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: + llmobs.submit_evaluation( + span_context=llmobs.export_span(span), label="sentiment", metric_type="score", value=0.9, ml_app="dummy" ) mock_llmobs_eval_metric_writer.enqueue.assert_called_with( _expected_llmobs_eval_metric_event( @@ -1327,9 +1161,9 @@ def test_submit_evaluation_enqueues_writer_with_score_metric(LLMObs, mock_llmobs def test_submit_evaluation_with_numerical_metric_enqueues_writer_with_score_metric( - LLMObs, mock_llmobs_eval_metric_writer + llmobs, mock_llmobs_eval_metric_writer ): - LLMObs.submit_evaluation( + llmobs.submit_evaluation( span_context={"span_id": "123", "trace_id": "456"}, label="token_count", metric_type="numerical", @@ -1342,9 +1176,9 @@ def test_submit_evaluation_with_numerical_metric_enqueues_writer_with_score_metr ) ) mock_llmobs_eval_metric_writer.reset_mock() - with LLMObs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: - LLMObs.submit_evaluation( - span_context=LLMObs.export_span(span), + with llmobs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: + llmobs.submit_evaluation( + span_context=llmobs.export_span(span), label="token_count", metric_type="numerical", value=35, @@ -1362,148 +1196,148 @@ def test_submit_evaluation_with_numerical_metric_enqueues_writer_with_score_metr ) -def test_flush_calls_periodic_agentless( - AgentlessLLMObs, mock_llmobs_span_agentless_writer, mock_llmobs_eval_metric_writer, mock_llmobs_evaluator_runner -): - AgentlessLLMObs.flush() - mock_llmobs_span_agentless_writer.periodic.assert_called_once() - mock_llmobs_eval_metric_writer.periodic.assert_called_once() - mock_llmobs_evaluator_runner.periodic.assert_called_once() - - def test_flush_does_not_call_periodic_when_llmobs_is_disabled( - LLMObs, - mock_llmobs_span_writer, + llmobs, mock_llmobs_eval_metric_writer, mock_llmobs_evaluator_runner, - mock_logs, - disabled_llmobs, + mock_llmobs_logs, ): - LLMObs.flush() - mock_llmobs_span_writer.periodic.assert_not_called() + llmobs.enabled = False + llmobs.flush() mock_llmobs_eval_metric_writer.periodic.assert_not_called() mock_llmobs_evaluator_runner.periodic.assert_not_called() - mock_logs.warning.assert_has_calls( + mock_llmobs_logs.warning.assert_has_calls( [mock.call("flushing when LLMObs is disabled. No spans or evaluation metrics will be sent.")] ) -def test_flush_does_not_call_periodic_when_llmobs_is_disabled_agentless( - AgentlessLLMObs, - mock_llmobs_span_agentless_writer, - mock_llmobs_eval_metric_writer, - mock_llmobs_evaluator_runner, - mock_logs, - disabled_llmobs, -): - AgentlessLLMObs.flush() - mock_llmobs_span_agentless_writer.periodic.assert_not_called() - mock_llmobs_eval_metric_writer.periodic.assert_not_called() - mock_llmobs_evaluator_runner.periodic.assert_not_called() - mock_logs.warning.assert_has_calls( - [mock.call("flushing when LLMObs is disabled. No spans or evaluation metrics will be sent.")] - ) - - -def test_inject_distributed_headers_llmobs_disabled_does_nothing(LLMObs, mock_logs): - LLMObs.disable() - headers = LLMObs.inject_distributed_headers({}, span=None) - mock_logs.warning.assert_called_once_with( +def test_inject_distributed_headers_llmobs_disabled_does_nothing(llmobs, mock_llmobs_logs): + llmobs.disable() + headers = llmobs.inject_distributed_headers({}, span=None) + mock_llmobs_logs.warning.assert_called_once_with( "LLMObs.inject_distributed_headers() called when LLMObs is not enabled. " "Distributed context will not be injected." ) assert headers == {} -def test_inject_distributed_headers_not_dict_logs_warning(LLMObs, mock_logs): - headers = LLMObs.inject_distributed_headers("not a dictionary", span=None) - mock_logs.warning.assert_called_once_with("request_headers must be a dictionary of string key-value pairs.") +def test_inject_distributed_headers_not_dict_logs_warning(llmobs, mock_llmobs_logs): + headers = llmobs.inject_distributed_headers("not a dictionary", span=None) + mock_llmobs_logs.warning.assert_called_once_with("request_headers must be a dictionary of string key-value pairs.") assert headers == "not a dictionary" - mock_logs.reset_mock() - headers = LLMObs.inject_distributed_headers(123, span=None) - mock_logs.warning.assert_called_once_with("request_headers must be a dictionary of string key-value pairs.") + mock_llmobs_logs.reset_mock() + headers = llmobs.inject_distributed_headers(123, span=None) + mock_llmobs_logs.warning.assert_called_once_with("request_headers must be a dictionary of string key-value pairs.") assert headers == 123 - mock_logs.reset_mock() - headers = LLMObs.inject_distributed_headers(None, span=None) - mock_logs.warning.assert_called_once_with("request_headers must be a dictionary of string key-value pairs.") + mock_llmobs_logs.reset_mock() + headers = llmobs.inject_distributed_headers(None, span=None) + mock_llmobs_logs.warning.assert_called_once_with("request_headers must be a dictionary of string key-value pairs.") assert headers is None -def test_inject_distributed_headers_no_active_span_logs_warning(LLMObs, mock_logs): - headers = LLMObs.inject_distributed_headers({}, span=None) - mock_logs.warning.assert_called_once_with("No span provided and no currently active span found.") +def test_inject_distributed_headers_no_active_span_logs_warning(llmobs, mock_llmobs_logs): + headers = llmobs.inject_distributed_headers({}, span=None) + mock_llmobs_logs.warning.assert_called_once_with("No span provided and no currently active span found.") assert headers == {} -def test_inject_distributed_headers_span_calls_httppropagator_inject(LLMObs, mock_logs): - span = LLMObs._instance.tracer.trace("test_span") +def test_inject_distributed_headers_span_calls_httppropagator_inject(llmobs, mock_llmobs_logs): + span = llmobs._instance.tracer.trace("test_span") with mock.patch("ddtrace.propagation.http.HTTPPropagator.inject") as mock_inject: - LLMObs.inject_distributed_headers({}, span=span) + llmobs.inject_distributed_headers({}, span=span) assert mock_inject.call_count == 1 mock_inject.assert_called_once_with(span.context, {}) -def test_inject_distributed_headers_current_active_span_injected(LLMObs, mock_logs): - span = LLMObs._instance.tracer.trace("test_span") +def test_inject_distributed_headers_current_active_span_injected(llmobs, mock_llmobs_logs): + span = llmobs._instance.tracer.trace("test_span") with mock.patch("ddtrace.llmobs._llmobs.HTTPPropagator.inject") as mock_inject: - LLMObs.inject_distributed_headers({}, span=None) + llmobs.inject_distributed_headers({}, span=None) assert mock_inject.call_count == 1 mock_inject.assert_called_once_with(span.context, {}) -def test_activate_distributed_headers_llmobs_disabled_does_nothing(LLMObs, mock_logs): - LLMObs.disable() - LLMObs.activate_distributed_headers({}) - mock_logs.warning.assert_called_once_with( +def test_activate_distributed_headers_llmobs_disabled_does_nothing(llmobs, mock_llmobs_logs): + llmobs.disable() + llmobs.activate_distributed_headers({}) + mock_llmobs_logs.warning.assert_called_once_with( "LLMObs.activate_distributed_headers() called when LLMObs is not enabled. " "Distributed context will not be activated." ) -def test_activate_distributed_headers_calls_httppropagator_extract(LLMObs, mock_logs): +def test_activate_distributed_headers_calls_httppropagator_extract(llmobs, mock_llmobs_logs): with mock.patch("ddtrace.llmobs._llmobs.HTTPPropagator.extract") as mock_extract: - LLMObs.activate_distributed_headers({}) + llmobs.activate_distributed_headers({}) assert mock_extract.call_count == 1 mock_extract.assert_called_once_with({}) -def test_activate_distributed_headers_no_trace_id_does_nothing(LLMObs, mock_logs): +def test_activate_distributed_headers_no_trace_id_does_nothing(llmobs, mock_llmobs_logs): with mock.patch("ddtrace.llmobs._llmobs.HTTPPropagator.extract") as mock_extract: mock_extract.return_value = Context(span_id="123", meta={PROPAGATED_PARENT_ID_KEY: "123"}) - LLMObs.activate_distributed_headers({}) + llmobs.activate_distributed_headers({}) assert mock_extract.call_count == 1 - mock_logs.warning.assert_called_once_with("Failed to extract trace ID or span ID from request headers.") + mock_llmobs_logs.warning.assert_called_once_with("Failed to extract trace ID or span ID from request headers.") -def test_activate_distributed_headers_no_span_id_does_nothing(LLMObs, mock_logs): +def test_activate_distributed_headers_no_span_id_does_nothing(llmobs, mock_llmobs_logs): with mock.patch("ddtrace.llmobs._llmobs.HTTPPropagator.extract") as mock_extract: mock_extract.return_value = Context(trace_id="123", meta={PROPAGATED_PARENT_ID_KEY: "123"}) - LLMObs.activate_distributed_headers({}) + llmobs.activate_distributed_headers({}) assert mock_extract.call_count == 1 - mock_logs.warning.assert_called_once_with("Failed to extract trace ID or span ID from request headers.") + mock_llmobs_logs.warning.assert_called_once_with("Failed to extract trace ID or span ID from request headers.") -def test_activate_distributed_headers_no_llmobs_parent_id_does_nothing(LLMObs, mock_logs): +def test_activate_distributed_headers_no_llmobs_parent_id_does_nothing(llmobs, mock_llmobs_logs): with mock.patch("ddtrace.llmobs._llmobs.HTTPPropagator.extract") as mock_extract: dummy_context = Context(trace_id="123", span_id="456") mock_extract.return_value = dummy_context with mock.patch("ddtrace.llmobs.LLMObs._instance.tracer.context_provider.activate") as mock_activate: - LLMObs.activate_distributed_headers({}) + llmobs.activate_distributed_headers({}) assert mock_extract.call_count == 1 - mock_logs.warning.assert_called_once_with("Failed to extract LLMObs parent ID from request headers.") + mock_llmobs_logs.warning.assert_called_once_with("Failed to extract LLMObs parent ID from request headers.") mock_activate.assert_called_once_with(dummy_context) -def test_activate_distributed_headers_activates_context(LLMObs, mock_logs): +def test_activate_distributed_headers_activates_context(llmobs, mock_llmobs_logs): with mock.patch("ddtrace.llmobs._llmobs.HTTPPropagator.extract") as mock_extract: dummy_context = Context(trace_id="123", span_id="456", meta={PROPAGATED_PARENT_ID_KEY: "789"}) mock_extract.return_value = dummy_context with mock.patch("ddtrace.llmobs.LLMObs._instance.tracer.context_provider.activate") as mock_activate: - LLMObs.activate_distributed_headers({}) + llmobs.activate_distributed_headers({}) assert mock_extract.call_count == 1 mock_activate.assert_called_once_with(dummy_context) +def test_listener_hooks_enqueue_correct_writer(run_python_code_in_subprocess): + """ + Regression test that ensures that listener hooks enqueue span events to the correct writer, + not the default writer created at startup. + """ + env = os.environ.copy() + pypath = [os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))] + if "PYTHONPATH" in env: + pypath.append(env["PYTHONPATH"]) + env.update({"PYTHONPATH": ":".join(pypath), "DD_TRACE_ENABLED": "0"}) + out, err, status, pid = run_python_code_in_subprocess( + """ +from ddtrace.llmobs import LLMObs + +LLMObs.enable(ml_app="repro-issue", agentless_enabled=True, api_key="foobar.baz", site="datad0g.com") +with LLMObs.agent("dummy"): + pass +""", + env=env, + ) + assert status == 0, err + assert out == b"" + agentless_writer_log = b"failed to send traces to intake at https://llmobs-intake.datad0g.com/api/v2/llmobs: HTTP error status 403, reason Forbidden\n" # noqa: E501 + agent_proxy_log = b"failed to send, dropping 1 traces to intake at http://localhost:8126/evp_proxy/v2/api/v2/llmobs after 5 retries" # noqa: E501 + assert err == agentless_writer_log + assert agent_proxy_log not in err + + def test_llmobs_fork_recreates_and_restarts_span_writer(): """Test that forking a process correctly recreates and restarts the LLMObsSpanWriter.""" with mock.patch("ddtrace.internal.writer.HTTPWriter._send_payload"): @@ -1514,16 +1348,10 @@ def test_llmobs_fork_recreates_and_restarts_span_writer(): if pid: # parent assert llmobs_service._instance.tracer._pid == original_pid assert llmobs_service._instance._llmobs_span_writer == original_span_writer - assert ( - llmobs_service._instance._trace_processor._span_writer == llmobs_service._instance._llmobs_span_writer - ) assert llmobs_service._instance._llmobs_span_writer.status == ServiceStatus.RUNNING else: # child assert llmobs_service._instance.tracer._pid != original_pid assert llmobs_service._instance._llmobs_span_writer != original_span_writer - assert ( - llmobs_service._instance._trace_processor._span_writer == llmobs_service._instance._llmobs_span_writer - ) assert llmobs_service._instance._llmobs_span_writer.status == ServiceStatus.RUNNING llmobs_service.disable() os._exit(12) @@ -1569,18 +1397,10 @@ def test_llmobs_fork_recreates_and_restarts_evaluator_runner(mock_ragas_evaluato if pid: # parent assert llmobs_service._instance.tracer._pid == original_pid assert llmobs_service._instance._evaluator_runner == original_evaluator_runner - assert ( - llmobs_service._instance._trace_processor._evaluator_runner - == llmobs_service._instance._evaluator_runner - ) assert llmobs_service._instance._evaluator_runner.status == ServiceStatus.RUNNING else: # child assert llmobs_service._instance.tracer._pid != original_pid assert llmobs_service._instance._evaluator_runner != original_evaluator_runner - assert ( - llmobs_service._instance._trace_processor._evaluator_runner - == llmobs_service._instance._evaluator_runner - ) assert llmobs_service._instance._evaluator_runner.status == ServiceStatus.RUNNING llmobs_service.disable() os._exit(12) @@ -1667,42 +1487,6 @@ def test_llmobs_fork_evaluator_runner_run(monkeypatch): llmobs_service.disable() -def test_llmobs_fork_custom_filter(monkeypatch): - """Test that forking a process correctly keeps any custom filters.""" - - class CustomFilter(TraceFilter): - def process_trace(self, trace): - return trace - - monkeypatch.setenv("_DD_LLMOBS_WRITER_INTERVAL", 5.0) - with mock.patch("ddtrace.internal.writer.HTTPWriter._send_payload"): - tracer = DummyTracer() - custom_filter = CustomFilter() - tracer.configure(settings={"FILTERS": [custom_filter]}) - llmobs_service.enable(_tracer=tracer, ml_app="test_app") - assert custom_filter in llmobs_service._instance.tracer._filters - pid = os.fork() - if pid: # parent - assert custom_filter in llmobs_service._instance.tracer._filters - assert any( - isinstance(tracer_filter, LLMObsTraceProcessor) - for tracer_filter in llmobs_service._instance.tracer._filters - ) - else: # child - assert custom_filter in llmobs_service._instance.tracer._filters - assert any( - isinstance(tracer_filter, LLMObsTraceProcessor) - for tracer_filter in llmobs_service._instance.tracer._filters - ) - llmobs_service.disable() - os._exit(12) - - _, status = os.waitpid(pid, 0) - exit_code = os.WEXITSTATUS(status) - assert exit_code == 12 - llmobs_service.disable() - - def test_llmobs_fork_disabled(monkeypatch): """Test that after being disabled the service remains disabled when forking""" monkeypatch.setenv("DD_LLMOBS_ENABLED", "0") @@ -1746,46 +1530,46 @@ def test_llmobs_fork_disabled_then_enabled(monkeypatch): svc.disable() -def test_llmobs_with_evaluator_runner(LLMObs, mock_llmobs_evaluator_runner): - with LLMObs.llm(model_name="test_model"): +def test_llmobs_with_evaluator_runner(llmobs, mock_llmobs_evaluator_runner): + with llmobs.llm(model_name="test_model"): pass time.sleep(0.1) - assert LLMObs._instance._evaluator_runner.enqueue.call_count == 1 + assert llmobs._instance._evaluator_runner.enqueue.call_count == 1 -def test_llmobs_with_evaluator_runner_does_not_enqueue_evaluation_spans(mock_llmobs_evaluator_runner, LLMObs): - with LLMObs.llm(model_name="test_model", ml_app="{}-dummy".format(RAGAS_ML_APP_PREFIX)): +def test_llmobs_with_evaluator_runner_does_not_enqueue_evaluation_spans(mock_llmobs_evaluator_runner, llmobs): + with llmobs.llm(model_name="test_model", ml_app="{}-dummy".format(RAGAS_ML_APP_PREFIX)): pass time.sleep(0.1) - assert LLMObs._instance._evaluator_runner.enqueue.call_count == 0 + assert llmobs._instance._evaluator_runner.enqueue.call_count == 0 -def test_llmobs_with_evaluation_runner_does_not_enqueue_non_llm_spans(mock_llmobs_evaluator_runner, LLMObs): - with LLMObs.workflow(name="test"): +def test_llmobs_with_evaluation_runner_does_not_enqueue_non_llm_spans(mock_llmobs_evaluator_runner, llmobs): + with llmobs.workflow(name="test"): pass - with LLMObs.agent(name="test"): + with llmobs.agent(name="test"): pass - with LLMObs.task(name="test"): + with llmobs.task(name="test"): pass - with LLMObs.embedding(model_name="test"): + with llmobs.embedding(model_name="test"): pass - with LLMObs.retrieval(name="test"): + with llmobs.retrieval(name="test"): pass - with LLMObs.tool(name="test"): + with llmobs.tool(name="test"): pass time.sleep(0.1) - assert LLMObs._instance._evaluator_runner.enqueue.call_count == 0 + assert llmobs._instance._evaluator_runner.enqueue.call_count == 0 -def test_annotation_context_modifies_span_tags(LLMObs): - with LLMObs.annotation_context(tags={"foo": "bar"}): - with LLMObs.agent(name="test_agent") as span: +def test_annotation_context_modifies_span_tags(llmobs): + with llmobs.annotation_context(tags={"foo": "bar"}): + with llmobs.agent(name="test_agent") as span: assert span._get_ctx_item(TAGS) == {"foo": "bar"} -def test_annotation_context_modifies_prompt(LLMObs): - with LLMObs.annotation_context(prompt={"template": "test_template"}): - with LLMObs.llm(name="test_agent", model_name="test") as span: +def test_annotation_context_modifies_prompt(llmobs): + with llmobs.annotation_context(prompt={"template": "test_template"}): + with llmobs.llm(name="test_agent", model_name="test") as span: assert span._get_ctx_item(INPUT_PROMPT) == { "template": "test_template", "_dd_context_variable_keys": ["context"], @@ -1793,80 +1577,80 @@ def test_annotation_context_modifies_prompt(LLMObs): } -def test_annotation_context_modifies_name(LLMObs): - with LLMObs.annotation_context(name="test_agent_override"): - with LLMObs.llm(name="test_agent", model_name="test") as span: +def test_annotation_context_modifies_name(llmobs): + with llmobs.annotation_context(name="test_agent_override"): + with llmobs.llm(name="test_agent", model_name="test") as span: assert span.name == "test_agent_override" -def test_annotation_context_finished_context_does_not_modify_tags(LLMObs): - with LLMObs.annotation_context(tags={"foo": "bar"}): +def test_annotation_context_finished_context_does_not_modify_tags(llmobs): + with llmobs.annotation_context(tags={"foo": "bar"}): pass - with LLMObs.agent(name="test_agent") as span: + with llmobs.agent(name="test_agent") as span: assert span._get_ctx_item(TAGS) is None -def test_annotation_context_finished_context_does_not_modify_prompt(LLMObs): - with LLMObs.annotation_context(prompt={"template": "test_template"}): +def test_annotation_context_finished_context_does_not_modify_prompt(llmobs): + with llmobs.annotation_context(prompt={"template": "test_template"}): pass - with LLMObs.llm(name="test_agent", model_name="test") as span: + with llmobs.llm(name="test_agent", model_name="test") as span: assert span._get_ctx_item(INPUT_PROMPT) is None -def test_annotation_context_finished_context_does_not_modify_name(LLMObs): - with LLMObs.annotation_context(name="test_agent_override"): +def test_annotation_context_finished_context_does_not_modify_name(llmobs): + with llmobs.annotation_context(name="test_agent_override"): pass - with LLMObs.agent(name="test_agent") as span: + with llmobs.agent(name="test_agent") as span: assert span.name == "test_agent" -def test_annotation_context_nested(LLMObs): - with LLMObs.annotation_context(tags={"foo": "bar", "boo": "bar"}): - with LLMObs.annotation_context(tags={"foo": "baz"}): - with LLMObs.agent(name="test_agent") as span: +def test_annotation_context_nested(llmobs): + with llmobs.annotation_context(tags={"foo": "bar", "boo": "bar"}): + with llmobs.annotation_context(tags={"foo": "baz"}): + with llmobs.agent(name="test_agent") as span: assert span._get_ctx_item(TAGS) == {"foo": "baz", "boo": "bar"} -def test_annotation_context_nested_overrides_name(LLMObs): - with LLMObs.annotation_context(name="unexpected"): - with LLMObs.annotation_context(name="expected"): - with LLMObs.agent(name="test_agent") as span: +def test_annotation_context_nested_overrides_name(llmobs): + with llmobs.annotation_context(name="unexpected"): + with llmobs.annotation_context(name="expected"): + with llmobs.agent(name="test_agent") as span: assert span.name == "expected" -def test_annotation_context_nested_maintains_trace_structure(LLMObs, mock_llmobs_span_writer): +def test_annotation_context_nested_maintains_trace_structure(llmobs, llmobs_events): """This test makes sure starting/stopping annotation contexts do not modify the llmobs trace structure""" - with LLMObs.annotation_context(tags={"foo": "bar", "boo": "bar"}): - with LLMObs.agent(name="parent_span") as parent_span: - with LLMObs.annotation_context(tags={"foo": "baz"}): - with LLMObs.workflow(name="child_span") as child_span: + with llmobs.annotation_context(tags={"foo": "bar", "boo": "bar"}): + with llmobs.agent(name="parent_span") as parent_span: + with llmobs.annotation_context(tags={"foo": "baz"}): + with llmobs.workflow(name="child_span") as child_span: assert child_span._get_ctx_item(TAGS) == {"foo": "baz", "boo": "bar"} assert parent_span._get_ctx_item(TAGS) == {"foo": "bar", "boo": "bar"} - assert len(mock_llmobs_span_writer.enqueue.call_args_list) == 2 - parent_span, child_span = [span[0] for span, _ in mock_llmobs_span_writer.enqueue.call_args_list] + assert len(llmobs_events) == 2 + parent_span, child_span = llmobs_events[1], llmobs_events[0] assert child_span["trace_id"] == parent_span["trace_id"] assert child_span["span_id"] != parent_span["span_id"] assert child_span["parent_id"] == parent_span["span_id"] assert parent_span["parent_id"] == "undefined" - mock_llmobs_span_writer.reset_mock() - with LLMObs.annotation_context(tags={"foo": "bar", "boo": "bar"}): - with LLMObs.agent(name="parent_span"): +def test_annotation_context_separate_traces_maintained(llmobs, llmobs_events): + with llmobs.annotation_context(tags={"foo": "bar", "boo": "bar"}): + with llmobs.agent(name="parent_span"): pass - with LLMObs.workflow(name="child_span"): + with llmobs.workflow(name="child_span"): pass - assert len(mock_llmobs_span_writer.enqueue.call_args_list) == 2 - trace_one, trace_two = [span[0] for span, _ in mock_llmobs_span_writer.enqueue.call_args_list] - assert trace_one["trace_id"] != trace_two["trace_id"] - assert trace_one["span_id"] != trace_two["span_id"] - assert trace_two["parent_id"] == "undefined" - assert trace_one["parent_id"] == "undefined" + assert len(llmobs_events) == 2 + agent_span, workflow_span = llmobs_events[1], llmobs_events[0] + assert agent_span["trace_id"] != workflow_span["trace_id"] + assert agent_span["span_id"] != workflow_span["span_id"] + assert workflow_span["parent_id"] == "undefined" + assert agent_span["parent_id"] == "undefined" -def test_annotation_context_only_applies_to_local_context(LLMObs): +def test_annotation_context_only_applies_to_local_context(llmobs): """ tests that annotation contexts only apply to spans belonging to the same trace context and not globally to all spans. @@ -1882,8 +1666,8 @@ def test_annotation_context_only_applies_to_local_context(LLMObs): def context_one(): nonlocal agent_has_correct_name nonlocal agent_has_correct_tags - with LLMObs.annotation_context(name="expected_agent", tags={"foo": "bar"}): - with LLMObs.agent(name="test_agent") as span: + with llmobs.annotation_context(name="expected_agent", tags={"foo": "bar"}): + with llmobs.agent(name="test_agent") as span: event.wait() agent_has_correct_tags = span._get_ctx_item(TAGS) == {"foo": "bar"} agent_has_correct_name = span.name == "expected_agent" @@ -1892,9 +1676,9 @@ def context_one(): def context_two(): nonlocal tool_has_correct_name nonlocal tool_does_not_have_tags - with LLMObs.agent(name="test_agent"): - with LLMObs.annotation_context(name="expected_tool"): - with LLMObs.tool(name="test_tool") as tool_span: + with llmobs.agent(name="test_agent"): + with llmobs.annotation_context(name="expected_tool"): + with llmobs.tool(name="test_tool") as tool_span: event.wait() tool_does_not_have_tags = tool_span._get_ctx_item(TAGS) is None tool_has_correct_name = tool_span.name == "expected_tool" @@ -1904,7 +1688,7 @@ def context_two(): thread_one.start() thread_two.start() - with LLMObs.agent(name="test_agent") as span: + with llmobs.agent(name="test_agent") as span: assert span.name == "test_agent" assert span._get_ctx_item(TAGS) is None @@ -1920,15 +1704,15 @@ def context_two(): assert tool_does_not_have_tags is True -async def test_annotation_context_async_modifies_span_tags(LLMObs): - async with LLMObs.annotation_context(tags={"foo": "bar"}): - with LLMObs.agent(name="test_agent") as span: +async def test_annotation_context_async_modifies_span_tags(llmobs): + async with llmobs.annotation_context(tags={"foo": "bar"}): + with llmobs.agent(name="test_agent") as span: assert span._get_ctx_item(TAGS) == {"foo": "bar"} -async def test_annotation_context_async_modifies_prompt(LLMObs): - async with LLMObs.annotation_context(prompt={"template": "test_template"}): - with LLMObs.llm(name="test_agent", model_name="test") as span: +async def test_annotation_context_async_modifies_prompt(llmobs): + async with llmobs.annotation_context(prompt={"template": "test_template"}): + with llmobs.llm(name="test_agent", model_name="test") as span: assert span._get_ctx_item(INPUT_PROMPT) == { "template": "test_template", "_dd_context_variable_keys": ["context"], @@ -1936,41 +1720,42 @@ async def test_annotation_context_async_modifies_prompt(LLMObs): } -async def test_annotation_context_async_modifies_name(LLMObs): - async with LLMObs.annotation_context(name="test_agent_override"): - with LLMObs.llm(name="test_agent", model_name="test") as span: +async def test_annotation_context_async_modifies_name(llmobs): + async with llmobs.annotation_context(name="test_agent_override"): + with llmobs.llm(name="test_agent", model_name="test") as span: assert span.name == "test_agent_override" -async def test_annotation_context_async_finished_context_does_not_modify_tags(LLMObs): - async with LLMObs.annotation_context(tags={"foo": "bar"}): +async def test_annotation_context_async_finished_context_does_not_modify_tags(llmobs): + async with llmobs.annotation_context(tags={"foo": "bar"}): pass - with LLMObs.agent(name="test_agent") as span: + with llmobs.agent(name="test_agent") as span: assert span._get_ctx_item(TAGS) is None -async def test_annotation_context_async_finished_context_does_not_modify_prompt(LLMObs): - async with LLMObs.annotation_context(prompt={"template": "test_template"}): +async def test_annotation_context_async_finished_context_does_not_modify_prompt(llmobs): + async with llmobs.annotation_context(prompt={"template": "test_template"}): pass - with LLMObs.llm(name="test_agent", model_name="test") as span: + with llmobs.llm(name="test_agent", model_name="test") as span: assert span._get_ctx_item(INPUT_PROMPT) is None -async def test_annotation_context_finished_context_async_does_not_modify_name(LLMObs): - async with LLMObs.annotation_context(name="test_agent_override"): +async def test_annotation_context_finished_context_async_does_not_modify_name(llmobs): + async with llmobs.annotation_context(name="test_agent_override"): pass - with LLMObs.agent(name="test_agent") as span: + with llmobs.agent(name="test_agent") as span: assert span.name == "test_agent" -async def test_annotation_context_async_nested(LLMObs): - async with LLMObs.annotation_context(tags={"foo": "bar", "boo": "bar"}): - async with LLMObs.annotation_context(tags={"foo": "baz"}): - with LLMObs.agent(name="test_agent") as span: +async def test_annotation_context_async_nested(llmobs): + async with llmobs.annotation_context(tags={"foo": "bar", "boo": "bar"}): + async with llmobs.annotation_context(tags={"foo": "baz"}): + with llmobs.agent(name="test_agent") as span: assert span._get_ctx_item(TAGS) == {"foo": "baz", "boo": "bar"} def test_service_enable_starts_evaluator_runner_when_evaluators_exist(): + pytest.importorskip("ragas") with override_global_config(dict(_dd_api_key="", _llmobs_ml_app="")): with override_env(dict(_DD_LLMOBS_EVALUATORS="ragas_faithfulness")): dummy_tracer = DummyTracer() @@ -1994,3 +1779,293 @@ def test_service_enable_does_not_start_evaluator_runner(): assert llmobs_service._instance._llmobs_span_writer.status.value == "running" assert llmobs_service._instance._evaluator_runner.status.value == "stopped" llmobs_service.disable() + + +def test_submit_evaluation_llmobs_disabled_raises_debug(llmobs, mock_llmobs_logs): + llmobs.disable() + mock_llmobs_logs.reset_mock() + llmobs.submit_evaluation( + span_context={"span_id": "123", "trace_id": "456"}, label="toxicity", metric_type="categorical", value="high" + ) + mock_llmobs_logs.debug.assert_called_once_with( + "LLMObs.submit_evaluation() called when LLMObs is not enabled. Evaluation metric data will not be sent." + ) + + +def test_submit_evaluation_for_no_ml_app_raises_warning(llmobs, mock_llmobs_logs): + with override_global_config(dict(_llmobs_ml_app="")): + llmobs.submit_evaluation_for( + span={"span_id": "123", "trace_id": "456"}, + label="toxicity", + metric_type="categorical", + value="high", + ) + mock_llmobs_logs.warning.assert_called_once_with( + "ML App name is required for sending evaluation metrics. Evaluation metric data will not be sent. " + "Ensure this configuration is set before running your application." + ) + + +def test_submit_evaluation_for_span_incorrect_type_raises_error(llmobs, mock_llmobs_logs): + with pytest.raises( + TypeError, + match=re.escape( + ( + "`span` must be a dictionary containing both span_id and trace_id keys. " + "LLMObs.export_span() can be used to generate this dictionary from a given span." + ) + ), + ): + llmobs.submit_evaluation_for(span="asd", label="toxicity", metric_type="categorical", value="high") + + +def test_submit_evaluation_for_span_with_tag_value_incorrect_type_raises_error(llmobs, mock_llmobs_logs): + with pytest.raises( + TypeError, + match=r"`span_with_tag_value` must be a dict with keys 'tag_key' and 'tag_value' containing string values", + ): + llmobs.submit_evaluation_for( + span_with_tag_value="asd", label="toxicity", metric_type="categorical", value="high" + ) + with pytest.raises( + TypeError, + match=r"`span_with_tag_value` must be a dict with keys 'tag_key' and 'tag_value' containing string values", + ): + llmobs.submit_evaluation_for( + span_with_tag_value={"tag_key": "hi", "tag_value": 1}, + label="toxicity", + metric_type="categorical", + value="high", + ) + + +def test_submit_evaluation_for_empty_span_or_trace_id_raises_error(llmobs, mock_llmobs_logs): + with pytest.raises( + TypeError, + match=re.escape( + ( + "`span` must be a dictionary containing both span_id and trace_id keys. " + "LLMObs.export_span() can be used to generate this dictionary from a given span." + ) + ), + ): + llmobs.submit_evaluation_for( + span={"trace_id": "456"}, label="toxicity", metric_type="categorical", value="high" + ) + with pytest.raises( + TypeError, + match=re.escape( + "`span` must be a dictionary containing both span_id and trace_id keys. " + "LLMObs.export_span() can be used to generate this dictionary from a given span." + ), + ): + llmobs.submit_evaluation_for(span={"span_id": "456"}, label="toxicity", metric_type="categorical", value="high") + + +def test_submit_evaluation_for_span_with_tag_value_empty_key_or_val_raises_error(llmobs, mock_llmobs_logs): + with pytest.raises( + TypeError, + match=r"`span_with_tag_value` must be a dict with keys 'tag_key' and 'tag_value' containing string values", + ): + llmobs.submit_evaluation_for( + span_with_tag_value={"tag_value": "123"}, label="toxicity", metric_type="categorical", value="high" + ) + + +def test_submit_evaluation_for_invalid_timestamp_raises_error(llmobs, mock_llmobs_logs): + with pytest.raises( + ValueError, match="timestamp_ms must be a non-negative integer. Evaluation metric data will not be sent" + ): + llmobs.submit_evaluation_for( + span={"span_id": "123", "trace_id": "456"}, + label="", + metric_type="categorical", + value="high", + ml_app="dummy", + timestamp_ms="invalid", + ) + + +def test_submit_evaluation_for_empty_label_raises_error(llmobs, mock_llmobs_logs): + with pytest.raises(ValueError, match="label must be the specified name of the evaluation metric."): + llmobs.submit_evaluation_for( + span={"span_id": "123", "trace_id": "456"}, label="", metric_type="categorical", value="high" + ) + + +def test_submit_evaluation_for_incorrect_metric_type_raises_error(llmobs, mock_llmobs_logs): + with pytest.raises(ValueError, match="metric_type must be one of 'categorical' or 'score'."): + llmobs.submit_evaluation_for( + span={"span_id": "123", "trace_id": "456"}, label="toxicity", metric_type="wrong", value="high" + ) + with pytest.raises(ValueError, match="metric_type must be one of 'categorical' or 'score'."): + llmobs.submit_evaluation_for( + span={"span_id": "123", "trace_id": "456"}, label="toxicity", metric_type="", value="high" + ) + + +def test_submit_evaluation_for_incorrect_score_value_type_raises_error(llmobs, mock_llmobs_logs): + with pytest.raises(TypeError, match="value must be an integer or float for a score metric."): + llmobs.submit_evaluation_for( + span={"span_id": "123", "trace_id": "456"}, label="token_count", metric_type="score", value="high" + ) + + +def test_submit_evaluation_for_invalid_tags_raises_warning(llmobs, mock_llmobs_logs): + llmobs.submit_evaluation_for( + span={"span_id": "123", "trace_id": "456"}, + label="toxicity", + metric_type="categorical", + value="high", + tags=["invalid"], + ) + mock_llmobs_logs.warning.assert_called_once_with("tags must be a dictionary of string key-value pairs.") + + +@pytest.mark.parametrize( + "ddtrace_global_config", + [dict(_llmobs_ml_app="test_app_name")], +) +def test_submit_evaluation_for_non_string_tags_raises_warning_but_still_submits( + llmobs, mock_llmobs_logs, mock_llmobs_eval_metric_writer +): + llmobs.submit_evaluation_for( + span={"span_id": "123", "trace_id": "456"}, + label="toxicity", + metric_type="categorical", + value="high", + tags={1: 2, "foo": "bar"}, + ml_app="dummy", + ) + mock_llmobs_logs.warning.assert_called_once_with( + "Failed to parse tags. Tags for evaluation metrics must be strings." + ) + mock_llmobs_logs.reset_mock() + mock_llmobs_eval_metric_writer.enqueue.assert_called_with( + _expected_llmobs_eval_metric_event( + ml_app="dummy", + span_id="123", + trace_id="456", + label="toxicity", + metric_type="categorical", + categorical_value="high", + tags=["ddtrace.version:{}".format(ddtrace.__version__), "ml_app:dummy", "foo:bar"], + ) + ) + + +@pytest.mark.parametrize( + "ddtrace_global_config", + [dict(ddtrace="1.2.3", env="test_env", service="test_service", _llmobs_ml_app="test_app_name")], +) +def test_submit_evaluation_for_metric_tags(llmobs, mock_llmobs_eval_metric_writer): + llmobs.submit_evaluation_for( + span={"span_id": "123", "trace_id": "456"}, + label="toxicity", + metric_type="categorical", + value="high", + tags={"foo": "bar", "bee": "baz", "ml_app": "ml_app_override"}, + ml_app="ml_app_override", + ) + mock_llmobs_eval_metric_writer.enqueue.assert_called_with( + _expected_llmobs_eval_metric_event( + ml_app="ml_app_override", + span_id="123", + trace_id="456", + label="toxicity", + metric_type="categorical", + categorical_value="high", + tags=["ddtrace.version:{}".format(ddtrace.__version__), "ml_app:ml_app_override", "foo:bar", "bee:baz"], + ) + ) + + +def test_submit_evaluation_for_span_with_tag_value_enqueues_writer_with_categorical_metric( + llmobs, mock_llmobs_eval_metric_writer +): + llmobs.submit_evaluation_for( + span_with_tag_value={"tag_key": "tag_key", "tag_value": "tag_val"}, + label="toxicity", + metric_type="categorical", + value="high", + ml_app="dummy", + ) + mock_llmobs_eval_metric_writer.enqueue.assert_called_with( + _expected_llmobs_eval_metric_event( + ml_app="dummy", + tag_key="tag_key", + tag_value="tag_val", + label="toxicity", + metric_type="categorical", + categorical_value="high", + ) + ) + + +def test_submit_evaluation_for_enqueues_writer_with_categorical_metric(llmobs, mock_llmobs_eval_metric_writer): + llmobs.submit_evaluation_for( + span={"span_id": "123", "trace_id": "456"}, + label="toxicity", + metric_type="categorical", + value="high", + ml_app="dummy", + ) + mock_llmobs_eval_metric_writer.enqueue.assert_called_with( + _expected_llmobs_eval_metric_event( + ml_app="dummy", + span_id="123", + trace_id="456", + label="toxicity", + metric_type="categorical", + categorical_value="high", + ) + ) + mock_llmobs_eval_metric_writer.reset_mock() + with llmobs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: + llmobs.submit_evaluation_for( + span=llmobs.export_span(span), + label="toxicity", + metric_type="categorical", + value="high", + ml_app="dummy", + ) + mock_llmobs_eval_metric_writer.enqueue.assert_called_with( + _expected_llmobs_eval_metric_event( + ml_app="dummy", + span_id=str(span.span_id), + trace_id="{:x}".format(span.trace_id), + label="toxicity", + metric_type="categorical", + categorical_value="high", + ) + ) + + +def test_submit_evaluation_for_enqueues_writer_with_score_metric(llmobs, mock_llmobs_eval_metric_writer): + llmobs.submit_evaluation_for( + span={"span_id": "123", "trace_id": "456"}, + label="sentiment", + metric_type="score", + value=0.9, + ml_app="dummy", + ) + mock_llmobs_eval_metric_writer.enqueue.assert_called_with( + _expected_llmobs_eval_metric_event( + span_id="123", trace_id="456", label="sentiment", metric_type="score", score_value=0.9, ml_app="dummy" + ) + ) + mock_llmobs_eval_metric_writer.reset_mock() + with llmobs.llm(model_name="test_model", name="test_llm_call", model_provider="test_provider") as span: + llmobs.submit_evaluation_for( + span=llmobs.export_span(span), label="sentiment", metric_type="score", value=0.9, ml_app="dummy" + ) + mock_llmobs_eval_metric_writer.enqueue.assert_called_with( + _expected_llmobs_eval_metric_event( + span_id=str(span.span_id), + trace_id="{:x}".format(span.trace_id), + label="sentiment", + metric_type="score", + score_value=0.9, + ml_app="dummy", + ) + ) diff --git a/tests/llmobs/test_llmobs_span_agent_writer.py b/tests/llmobs/test_llmobs_span_agent_writer.py index 76fe0f21aef..d16bb9f0e2c 100644 --- a/tests/llmobs/test_llmobs_span_agent_writer.py +++ b/tests/llmobs/test_llmobs_span_agent_writer.py @@ -44,7 +44,8 @@ def test_flush_queue_when_event_cause_queue_to_exceed_payload_limit( [ mock.call("flushing queue because queuing next event will exceed EVP payload limit"), mock.call("encode %d LLMObs span events to be sent", 5), - ] + ], + any_order=True, ) diff --git a/tests/llmobs/test_llmobs_span_agentless_writer.py b/tests/llmobs/test_llmobs_span_agentless_writer.py index 4882f3553d8..cac0d926a74 100644 --- a/tests/llmobs/test_llmobs_span_agentless_writer.py +++ b/tests/llmobs/test_llmobs_span_agentless_writer.py @@ -75,26 +75,25 @@ def test_truncating_oversized_events(mock_writer_logs, mock_http_writer_send_pay ) -def test_send_completion_event(mock_writer_logs, mock_http_writer_logs, mock_http_writer_send_payload_response): +def test_send_completion_event(mock_writer_logs, mock_http_writer_send_payload_response): with override_global_config(dict(_dd_site=DATADOG_SITE, _dd_api_key="foobar.baz")): llmobs_span_writer = LLMObsSpanWriter(is_agentless=True, interval=1, timeout=1) llmobs_span_writer.start() llmobs_span_writer.enqueue(_completion_event()) llmobs_span_writer.periodic() mock_writer_logs.debug.assert_has_calls([mock.call("encode %d LLMObs span events to be sent", 1)]) - mock_http_writer_logs.error.assert_not_called() -def test_send_chat_completion_event(mock_writer_logs, mock_http_writer_logs, mock_http_writer_send_payload_response): +def test_send_chat_completion_event(mock_writer_logs, mock_http_writer_send_payload_response): with override_global_config(dict(_dd_site=DATADOG_SITE, _dd_api_key="foobar.baz")): llmobs_span_writer = LLMObsSpanWriter(is_agentless=True, interval=1, timeout=1) llmobs_span_writer.start() llmobs_span_writer.enqueue(_chat_completion_event()) llmobs_span_writer.periodic() mock_writer_logs.debug.assert_has_calls([mock.call("encode %d LLMObs span events to be sent", 1)]) - mock_http_writer_logs.error.assert_not_called() +@mock.patch("ddtrace.internal.writer.writer.log") def test_send_completion_bad_api_key(mock_http_writer_logs, mock_http_writer_put_response_forbidden): with override_global_config(dict(_dd_site=DATADOG_SITE, _dd_api_key="")): llmobs_span_writer = LLMObsSpanWriter(is_agentless=True, interval=1, timeout=1) @@ -109,7 +108,7 @@ def test_send_completion_bad_api_key(mock_http_writer_logs, mock_http_writer_put ) -def test_send_timed_events(mock_writer_logs, mock_http_writer_logs, mock_http_writer_send_payload_response): +def test_send_timed_events(mock_writer_logs, mock_http_writer_send_payload_response): with override_global_config(dict(_dd_site=DATADOG_SITE, _dd_api_key="foobar.baz")): llmobs_span_writer = LLMObsSpanWriter(is_agentless=True, interval=0.01, timeout=1) llmobs_span_writer.start() @@ -122,10 +121,9 @@ def test_send_timed_events(mock_writer_logs, mock_http_writer_logs, mock_http_wr llmobs_span_writer.enqueue(_chat_completion_event()) time.sleep(0.1) mock_writer_logs.debug.assert_has_calls([mock.call("encode %d LLMObs span events to be sent", 1)]) - mock_http_writer_logs.error.assert_not_called() -def test_send_multiple_events(mock_writer_logs, mock_http_writer_logs, mock_http_writer_send_payload_response): +def test_send_multiple_events(mock_writer_logs, mock_http_writer_send_payload_response): with override_global_config(dict(_dd_site=DATADOG_SITE, _dd_api_key="foobar.baz")): llmobs_span_writer = LLMObsSpanWriter(is_agentless=True, interval=0.01, timeout=1) llmobs_span_writer.start() @@ -135,12 +133,11 @@ def test_send_multiple_events(mock_writer_logs, mock_http_writer_logs, mock_http llmobs_span_writer.enqueue(_chat_completion_event()) time.sleep(0.1) mock_writer_logs.debug.assert_has_calls([mock.call("encode %d LLMObs span events to be sent", 2)]) - mock_http_writer_logs.error.assert_not_called() -def test_send_on_exit(mock_writer_logs, run_python_code_in_subprocess): +def test_send_on_exit(run_python_code_in_subprocess): env = os.environ.copy() - pypath = [os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))] + pypath = [os.path.dirname(os.path.dirname(os.path.dirname(__file__)))] if "PYTHONPATH" in env: pypath.append(env["PYTHONPATH"]) env.update( diff --git a/tests/llmobs/test_llmobs_trace_processor.py b/tests/llmobs/test_llmobs_trace_processor.py deleted file mode 100644 index b55286d49c8..00000000000 --- a/tests/llmobs/test_llmobs_trace_processor.py +++ /dev/null @@ -1,36 +0,0 @@ -import mock - -from ddtrace._trace.span import Span -from ddtrace.ext import SpanTypes -from ddtrace.llmobs._constants import SPAN_KIND -from ddtrace.llmobs._trace_processor import LLMObsTraceProcessor -from tests.utils import override_global_config - - -def test_processor_returns_all_traces_by_default(): - """Test that the LLMObsTraceProcessor returns all traces by default.""" - trace_filter = LLMObsTraceProcessor(llmobs_span_writer=mock.MagicMock()) - root_llm_span = Span(name="span1", span_type=SpanTypes.LLM) - root_llm_span._set_ctx_item(SPAN_KIND, "llm") - trace1 = [root_llm_span] - assert trace_filter.process_trace(trace1) == trace1 - - -def test_processor_returns_all_traces_if_not_agentless(): - """Test that the LLMObsTraceProcessor returns all traces if DD_LLMOBS_AGENTLESS_ENABLED is not set to true.""" - with override_global_config(dict(_llmobs_agentless_enabled=False)): - trace_filter = LLMObsTraceProcessor(llmobs_span_writer=mock.MagicMock()) - root_llm_span = Span(name="span1", span_type=SpanTypes.LLM) - root_llm_span._set_ctx_item(SPAN_KIND, "llm") - trace1 = [root_llm_span] - assert trace_filter.process_trace(trace1) == trace1 - - -def test_processor_returns_none_in_agentless_mode(): - """Test that the LLMObsTraceProcessor returns None if DD_LLMOBS_AGENTLESS_ENABLED is set to true.""" - with override_global_config(dict(_llmobs_agentless_enabled=True)): - trace_filter = LLMObsTraceProcessor(llmobs_span_writer=mock.MagicMock()) - root_llm_span = Span(name="span1", span_type=SpanTypes.LLM) - root_llm_span._set_ctx_item(SPAN_KIND, "llm") - trace1 = [root_llm_span] - assert trace_filter.process_trace(trace1) is None diff --git a/tests/llmobs/test_propagation.py b/tests/llmobs/test_propagation.py index d892c6b98a2..e3ab9c80d66 100644 --- a/tests/llmobs/test_propagation.py +++ b/tests/llmobs/test_propagation.py @@ -157,39 +157,39 @@ def test_no_llmobs_parent_id_propagated_if_no_llmobs_spans(run_python_code_in_su assert _get_llmobs_parent_id(span) == "undefined" -def test_inject_distributed_headers_simple(LLMObs): +def test_inject_distributed_headers_simple(llmobs): dummy_tracer = DummyTracer() with dummy_tracer.trace("LLMObs span", span_type=SpanTypes.LLM) as root_span: - request_headers = LLMObs.inject_distributed_headers({}, span=root_span) + request_headers = llmobs.inject_distributed_headers({}, span=root_span) assert PROPAGATED_PARENT_ID_KEY in request_headers["x-datadog-tags"] -def test_inject_distributed_headers_nested_llmobs_non_llmobs(LLMObs): +def test_inject_distributed_headers_nested_llmobs_non_llmobs(llmobs): dummy_tracer = DummyTracer() with dummy_tracer.trace("LLMObs span", span_type=SpanTypes.LLM): with dummy_tracer.trace("Non-LLMObs span") as child_span: - request_headers = LLMObs.inject_distributed_headers({}, span=child_span) + request_headers = llmobs.inject_distributed_headers({}, span=child_span) assert PROPAGATED_PARENT_ID_KEY in request_headers["x-datadog-tags"] -def test_inject_distributed_headers_non_llmobs_root_span(LLMObs): +def test_inject_distributed_headers_non_llmobs_root_span(llmobs): dummy_tracer = DummyTracer() with dummy_tracer.trace("Non-LLMObs span"): with dummy_tracer.trace("LLMObs span", span_type=SpanTypes.LLM) as child_span: - request_headers = LLMObs.inject_distributed_headers({}, span=child_span) + request_headers = llmobs.inject_distributed_headers({}, span=child_span) assert PROPAGATED_PARENT_ID_KEY in request_headers["x-datadog-tags"] -def test_inject_distributed_headers_nested_llmobs_spans(LLMObs): +def test_inject_distributed_headers_nested_llmobs_spans(llmobs): dummy_tracer = DummyTracer() with dummy_tracer.trace("LLMObs span", span_type=SpanTypes.LLM): with dummy_tracer.trace("LLMObs child span", span_type=SpanTypes.LLM): with dummy_tracer.trace("Last LLMObs child span", span_type=SpanTypes.LLM) as last_llmobs_span: - request_headers = LLMObs.inject_distributed_headers({}, span=last_llmobs_span) + request_headers = llmobs.inject_distributed_headers({}, span=last_llmobs_span) assert PROPAGATED_PARENT_ID_KEY in request_headers["x-datadog-tags"] -def test_activate_distributed_headers_propagate_correct_llmobs_parent_id_simple(run_python_code_in_subprocess, LLMObs): +def test_activate_distributed_headers_propagate_correct_llmobs_parent_id_simple(run_python_code_in_subprocess, llmobs): """Test that the correct LLMObs parent ID is propagated in the headers in a simple distributed scenario. Service A (subprocess) has a root LLMObs span and a non-LLMObs child span. Service B (outside subprocess) has a LLMObs span. @@ -216,16 +216,15 @@ def test_activate_distributed_headers_propagate_correct_llmobs_parent_id_simple( env["DD_TRACE_ENABLED"] = "0" stdout, stderr, status, _ = run_python_code_in_subprocess(code=code, env=env) assert status == 0, (stdout, stderr) - assert stderr == b"", (stdout, stderr) headers = json.loads(stdout.decode()) - LLMObs.activate_distributed_headers(headers) - with LLMObs.workflow("LLMObs span") as span: + llmobs.activate_distributed_headers(headers) + with llmobs.workflow("LLMObs span") as span: assert str(span.parent_id) == headers["x-datadog-parent-id"] assert _get_llmobs_parent_id(span) == headers["_DD_LLMOBS_SPAN_ID"] -def test_activate_distributed_headers_propagate_llmobs_parent_id_complex(run_python_code_in_subprocess, LLMObs): +def test_activate_distributed_headers_propagate_llmobs_parent_id_complex(run_python_code_in_subprocess, llmobs): """Test that the correct LLMObs parent ID is propagated in the headers in a more complex trace. Service A (subprocess) has a root LLMObs span and a non-LLMObs child span. Service B (outside subprocess) has a non-LLMObs local root span and a LLMObs child span. @@ -252,19 +251,18 @@ def test_activate_distributed_headers_propagate_llmobs_parent_id_complex(run_pyt env["DD_TRACE_ENABLED"] = "0" stdout, stderr, status, _ = run_python_code_in_subprocess(code=code, env=env) assert status == 0, (stdout, stderr) - assert stderr == b"", (stdout, stderr) headers = json.loads(stdout.decode()) - LLMObs.activate_distributed_headers(headers) + llmobs.activate_distributed_headers(headers) dummy_tracer = DummyTracer() with dummy_tracer.trace("Non-LLMObs span") as span: - with LLMObs.llm(model_name="llm_model", name="LLMObs span") as llm_span: + with llmobs.llm(model_name="llm_model", name="LLMObs span") as llm_span: assert str(span.parent_id) == headers["x-datadog-parent-id"] assert _get_llmobs_parent_id(span) == headers["_DD_LLMOBS_SPAN_ID"] assert _get_llmobs_parent_id(llm_span) == headers["_DD_LLMOBS_SPAN_ID"] -def test_activate_distributed_headers_does_not_propagate_if_no_llmobs_spans(run_python_code_in_subprocess, LLMObs): +def test_activate_distributed_headers_does_not_propagate_if_no_llmobs_spans(run_python_code_in_subprocess, llmobs): """Test that the correct LLMObs parent ID (None) is extracted from the headers in a simple distributed scenario. Service A (subprocess) has spans, but none are LLMObs spans. Service B (outside subprocess) has a LLMObs span. @@ -289,10 +287,9 @@ def test_activate_distributed_headers_does_not_propagate_if_no_llmobs_spans(run_ env["DD_TRACE_ENABLED"] = "0" stdout, stderr, status, _ = run_python_code_in_subprocess(code=code, env=env) assert status == 0, (stdout, stderr) - assert stderr == b"", (stdout, stderr) headers = json.loads(stdout.decode()) - LLMObs.activate_distributed_headers(headers) - with LLMObs.task("LLMObs span") as span: + llmobs.activate_distributed_headers(headers) + with llmobs.task("LLMObs span") as span: assert str(span.parent_id) == headers["x-datadog-parent-id"] assert _get_llmobs_parent_id(span) == "undefined" diff --git a/tests/opentracer/test_tracer_gevent.py b/tests/opentracer/test_tracer_gevent.py index 59a52faaa0e..90b22c644d0 100644 --- a/tests/opentracer/test_tracer_gevent.py +++ b/tests/opentracer/test_tracer_gevent.py @@ -2,9 +2,9 @@ from opentracing.scope_managers.gevent import GeventScopeManager import pytest -import ddtrace -from ddtrace.contrib.gevent import patch -from ddtrace.contrib.gevent import unpatch +from ddtrace.contrib.gevent import context_provider +from ddtrace.contrib.internal.gevent.patch import patch +from ddtrace.contrib.internal.gevent.patch import unpatch @pytest.fixture() @@ -12,7 +12,7 @@ def ot_tracer(ot_tracer_factory): """Fixture providing an opentracer configured for gevent usage.""" # patch gevent patch() - yield ot_tracer_factory("gevent_svc", {}, GeventScopeManager(), ddtrace.contrib.gevent.context_provider) + yield ot_tracer_factory("gevent_svc", {}, GeventScopeManager(), context_provider) # unpatch gevent unpatch() diff --git a/tests/profiling/collector/test_stack.py b/tests/profiling/collector/test_stack.py index 84f3ad60ea6..86ee91e2ac4 100644 --- a/tests/profiling/collector/test_stack.py +++ b/tests/profiling/collector/test_stack.py @@ -763,7 +763,7 @@ def _nothing(): assert values.pop() > 0 -@flaky(1731169861) +@flaky(1748750400) @pytest.mark.skipif(sys.version_info < (3, 11, 0), reason="PyFrameObjects are lazy-created objects in Python 3.11+") def test_collect_ensure_all_frames_gc(): # Regression test for memory leak with lazy PyFrameObjects in Python 3.11+ diff --git a/tests/profiling/test_accuracy.py b/tests/profiling/test_accuracy.py index a332068e12b..d5821e4d226 100644 --- a/tests/profiling/test_accuracy.py +++ b/tests/profiling/test_accuracy.py @@ -44,11 +44,8 @@ def spend_cpu_3(): pass -# We allow 4% error: -# The profiler might not be precise, but time.sleep is not either. -TOLERANCE = 0.04 -# Use 5% accuracy for CPU usage, it's way less precise -CPU_TOLERANCE = 0.05 +# We allow 5% error: +TOLERANCE = 0.05 def assert_almost_equal(value, target, tolerance=TOLERANCE): @@ -69,7 +66,6 @@ def test_accuracy(): from ddtrace.profiling import profiler from ddtrace.profiling.collector import stack_event - from tests.profiling.test_accuracy import CPU_TOLERANCE from tests.profiling.test_accuracy import assert_almost_equal from tests.profiling.test_accuracy import spend_16 from tests.profiling.test_accuracy import total_time @@ -97,5 +93,5 @@ def test_accuracy(): assert_almost_equal(total_time(time_spent_ns, "spend_cpu_2"), 2e9) assert_almost_equal(total_time(time_spent_ns, "spend_cpu_3"), 3e9) - assert_almost_equal(total_time(cpu_spent_ns, "spend_cpu_2"), 2e9, CPU_TOLERANCE) - assert_almost_equal(total_time(cpu_spent_ns, "spend_cpu_3"), 3e9, CPU_TOLERANCE) + assert_almost_equal(total_time(cpu_spent_ns, "spend_cpu_2"), 2e9) + assert_almost_equal(total_time(cpu_spent_ns, "spend_cpu_3"), 3e9) diff --git a/tests/profiling/test_profiler.py b/tests/profiling/test_profiler.py index 7f98bbf6aa8..879a50afd54 100644 --- a/tests/profiling/test_profiler.py +++ b/tests/profiling/test_profiler.py @@ -232,36 +232,66 @@ def _check_url(prof, url, api_key, endpoint_path="profiling/v1/input"): pytest.fail("Unable to find HTTP exporter") +@pytest.mark.subprocess() def test_tracer_url(): - t = ddtrace.Tracer() + import os + + from ddtrace import tracer as t + from ddtrace.profiling import profiler + from tests.profiling.test_profiler import _check_url + t.configure(hostname="foobar") prof = profiler.Profiler(tracer=t) _check_url(prof, "http://foobar:8126", os.environ.get("DD_API_KEY")) +@pytest.mark.subprocess() def test_tracer_url_https(): - t = ddtrace.Tracer() + import os + + from ddtrace import tracer as t + from ddtrace.profiling import profiler + from tests.profiling.test_profiler import _check_url + t.configure(hostname="foobar", https=True) prof = profiler.Profiler(tracer=t) _check_url(prof, "https://foobar:8126", os.environ.get("DD_API_KEY")) +@pytest.mark.subprocess() def test_tracer_url_uds_hostname(): - t = ddtrace.Tracer() + import os + + from ddtrace import tracer as t + from ddtrace.profiling import profiler + from tests.profiling.test_profiler import _check_url + t.configure(hostname="foobar", uds_path="/foobar") prof = profiler.Profiler(tracer=t) _check_url(prof, "unix://foobar/foobar", os.environ.get("DD_API_KEY")) +@pytest.mark.subprocess() def test_tracer_url_uds(): - t = ddtrace.Tracer() + import os + + from ddtrace import tracer as t + from ddtrace.profiling import profiler + from tests.profiling.test_profiler import _check_url + t.configure(uds_path="/foobar") prof = profiler.Profiler(tracer=t) _check_url(prof, "unix:///foobar", os.environ.get("DD_API_KEY")) +@pytest.mark.subprocess() def test_tracer_url_configure_after(): - t = ddtrace.Tracer() + import os + + from ddtrace import tracer as t + from ddtrace.profiling import profiler + from tests.profiling.test_profiler import _check_url + prof = profiler.Profiler(tracer=t) t.configure(hostname="foobar") _check_url(prof, "http://foobar:8126", os.environ.get("DD_API_KEY")) @@ -276,11 +306,10 @@ def test_env_no_api_key(): def test_env_endpoint_url(): import os - import ddtrace + from ddtrace import tracer as t from ddtrace.profiling import profiler from tests.profiling.test_profiler import _check_url - t = ddtrace.Tracer() prof = profiler.Profiler(tracer=t) _check_url(prof, "http://foobar:123", os.environ.get("DD_API_KEY")) diff --git a/tests/profiling_v2/collector/test_stack.py b/tests/profiling_v2/collector/test_stack.py index 74def22ed50..774e15fb70d 100644 --- a/tests/profiling_v2/collector/test_stack.py +++ b/tests/profiling_v2/collector/test_stack.py @@ -11,7 +11,6 @@ from ddtrace import ext from ddtrace.internal.datadog.profiling import ddup from ddtrace.profiling.collector import stack -from ddtrace.settings.profiling import config from tests.profiling.collector import pprof_utils from tests.profiling.collector import test_collector @@ -171,7 +170,6 @@ def test_push_span_unregister_thread(tmp_path, monkeypatch, tracer): pytest.skip("stack_v2 is not supported on Python 3.7") with patch("ddtrace.internal.datadog.profiling.stack_v2.unregister_thread") as unregister_thread: - monkeypatch.setattr(config.stack, "v2_enabled", True) tracer._endpoint_call_counter_span_processor.enable() test_name = "test_push_span_unregister_thread" @@ -220,7 +218,7 @@ def target_fun(): ), ) - unregister_thread.assert_called_once_with(thread_id) + unregister_thread.assert_called_with(thread_id) @pytest.mark.parametrize("stack_v2_enabled", [True, False]) @@ -748,6 +746,7 @@ def test_ignore_profiler(stack_v2_enabled, ignore_profiler, tmp_path): # TODO: support ignore profiler with stack_v2 and update this test @pytest.mark.skipif(not TESTING_GEVENT, reason="Not testing gevent") +@pytest.mark.skip(reason="ignore_profiler is not supported with stack v2") @pytest.mark.subprocess( ddtrace_run=True, env=dict(DD_PROFILING_IGNORE_PROFILER="1", DD_PROFILING_OUTPUT_PPROF="/tmp/test_ignore_profiler_gevent_task"), diff --git a/tests/profiling_v2/gunicorn.conf.py b/tests/profiling_v2/gunicorn.conf.py new file mode 100644 index 00000000000..c45f27ce11c --- /dev/null +++ b/tests/profiling_v2/gunicorn.conf.py @@ -0,0 +1,67 @@ +from datetime import datetime +from datetime import timezone +import logging + + +def post_fork(server, worker): + """Log the startup time of each worker.""" + logging.info("Worker %s started", worker.pid) + + +def post_worker_init(worker): + logging.info("Worker %s initialized", worker.pid) + + +class CustomFormatter(logging.Formatter): + """Custom formatter to include timezone offset in the log message.""" + + def formatTime(self, record, datefmt=None): + dt = datetime.fromtimestamp(record.created, tz=timezone.utc).astimezone() + milliseconds = int(record.msecs) + offset = dt.strftime("%z") # Get timezone offset in the form +0530 + if datefmt: + formatted_time = dt.strftime(datefmt) + else: + formatted_time = dt.strftime("%Y-%m-%d %H:%M:%S") + + # Add milliseconds and timezone offset + offset = dt.strftime("%z") # Timezone offset in the form +0530 + return f"{formatted_time}.{milliseconds:03d} {offset}" + + +logconfig_dict = { + "version": 1, + "formatters": { + "default": { + "()": CustomFormatter, # Use the custom formatter + "format": "[%(asctime)s] [%(process)d] [%(levelname)s] %(message)s", + "datefmt": "%Y-%m-%d %H:%M:%S", + }, + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "default", + }, + }, + "loggers": { + "": { # root logger + "handlers": ["console"], + "level": "INFO", + }, + "gunicorn.error": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + "gunicorn.access": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + }, + "root": { + "level": "INFO", + "handlers": ["console"], + }, +} diff --git a/tests/profiling_v2/test_accuracy.py b/tests/profiling_v2/test_accuracy.py index 61fbe3322ff..cb1d538712f 100644 --- a/tests/profiling_v2/test_accuracy.py +++ b/tests/profiling_v2/test_accuracy.py @@ -46,10 +46,10 @@ def test_accuracy_libdd(): assert_almost_equal(wall_times["spend_16"], 16e9) assert_almost_equal(wall_times["spend_7"], 7e9) - assert_almost_equal(wall_times["spend_cpu_2"], 2e9, tolerance=0.07) - assert_almost_equal(wall_times["spend_cpu_3"], 3e9, tolerance=0.07) - assert_almost_equal(cpu_times["spend_cpu_2"], 2e9, tolerance=0.07) - assert_almost_equal(cpu_times["spend_cpu_3"], 3e9, tolerance=0.07) + assert_almost_equal(wall_times["spend_cpu_2"], 2e9, tolerance=0.09) + assert_almost_equal(wall_times["spend_cpu_3"], 3e9, tolerance=0.09) + assert_almost_equal(cpu_times["spend_cpu_2"], 2e9, tolerance=0.09) + assert_almost_equal(cpu_times["spend_cpu_3"], 3e9, tolerance=0.09) @pytest.mark.subprocess( diff --git a/tests/profiling_v2/test_gunicorn.py b/tests/profiling_v2/test_gunicorn.py index 4d7adbf6c95..90141445d3a 100644 --- a/tests/profiling_v2/test_gunicorn.py +++ b/tests/profiling_v2/test_gunicorn.py @@ -13,7 +13,7 @@ # DEV: gunicorn tests are hard to debug, so keeping these print statements for # future debugging -DEBUG_PRINT = False +DEBUG_PRINT = True def debug_print(*args): @@ -37,6 +37,8 @@ def _run_gunicorn(*args): "127.0.0.1:7644", "--worker-tmp-dir", "/dev/shm", + "-c", + os.path.dirname(__file__) + "/gunicorn.conf.py", "--chdir", os.path.dirname(__file__), ] diff --git a/tests/snapshots/tests.contrib.botocore.test_bedrock.test_invoke_model_using_aws_arn_model_id.json b/tests/snapshots/tests.contrib.botocore.test_bedrock.test_invoke_model_using_aws_arn_model_id.json new file mode 100644 index 00000000000..0da1e335083 --- /dev/null +++ b/tests/snapshots/tests.contrib.botocore.test_bedrock.test_invoke_model_using_aws_arn_model_id.json @@ -0,0 +1,39 @@ +[[ + { + "name": "bedrock-runtime.command", + "service": "aws.bedrock-runtime", + "resource": "InvokeModel", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "type": "", + "error": 0, + "meta": { + "_dd.base_service": "tests.contrib.botocore", + "_dd.p.dm": "-0", + "_dd.p.tid": "6786dfda00000000", + "bedrock.request.max_tokens": "50", + "bedrock.request.model": "titan-tg1-large", + "bedrock.request.model_provider": "amazon", + "bedrock.request.prompt": "Command: can you explain what Datadog is to someone not in the tech industry?", + "bedrock.request.stop_sequences": "[]", + "bedrock.request.temperature": "0", + "bedrock.request.top_p": "0.9", + "bedrock.response.choices.0.finish_reason": "LENGTH", + "bedrock.response.choices.0.text": "\\n\\nDatadog is a monitoring and analytics platform for IT operations, DevOps, and software development teams. It provides real-t...", + "bedrock.response.duration": "2646", + "bedrock.response.id": "b2d0fd44-c29a-4cd4-a97a-6901a48f6264", + "bedrock.usage.completion_tokens": "50", + "bedrock.usage.prompt_tokens": "18", + "language": "python", + "runtime-id": "cf8ef38d3504475ba71634071f15d00f" + }, + "metrics": { + "_dd.top_level": 1, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 1, + "process_id": 96028 + }, + "duration": 2318000, + "start": 1736892378210317000 + }]] diff --git a/tests/snapshots/tests.contrib.openai.test_openai.test_chat_completion_stream.json b/tests/snapshots/tests.contrib.openai.test_openai.test_chat_completion_stream.json new file mode 100644 index 00000000000..fe7c9e3b0f2 --- /dev/null +++ b/tests/snapshots/tests.contrib.openai.test_openai.test_chat_completion_stream.json @@ -0,0 +1,53 @@ +[[ + { + "name": "openai.request", + "service": "tests.contrib.openai", + "resource": "createChatCompletion", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "type": "", + "error": 0, + "meta": { + "_dd.p.dm": "-0", + "_dd.p.tid": "67741fca00000000", + "component": "openai", + "language": "python", + "openai.base_url": "https://api.openai.com/v1/", + "openai.organization.name": "datadog-4", + "openai.request.client": "OpenAI", + "openai.request.endpoint": "/v1/chat/completions", + "openai.request.messages.0.content": "Who won the world series in 2020?", + "openai.request.messages.0.name": "", + "openai.request.messages.0.role": "user", + "openai.request.method": "POST", + "openai.request.model": "gpt-3.5-turbo", + "openai.request.n": "None", + "openai.request.stream": "True", + "openai.request.user": "ddtrace-test", + "openai.response.choices.0.finish_reason": "stop", + "openai.response.choices.0.message.content": "The Los Angeles Dodgers won the World Series in 2020.", + "openai.response.choices.0.message.role": "assistant", + "openai.response.model": "gpt-3.5-turbo-0301", + "openai.user.api_key": "sk-...key>", + "runtime-id": "d174f65e33314f43ad1de8cf0a5ca4e0" + }, + "metrics": { + "_dd.measured": 1, + "_dd.top_level": 1, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 1, + "openai.organization.ratelimit.requests.limit": 3000, + "openai.organization.ratelimit.requests.remaining": 2999, + "openai.organization.ratelimit.tokens.limit": 250000, + "openai.organization.ratelimit.tokens.remaining": 249979, + "openai.request.prompt_tokens_estimated": 0, + "openai.response.completion_tokens_estimated": 0, + "openai.response.usage.completion_tokens": 19, + "openai.response.usage.prompt_tokens": 17, + "openai.response.usage.total_tokens": 36, + "process_id": 22982 + }, + "duration": 29869000, + "start": 1735663562179157000 + }]] diff --git a/tests/snapshots/tests.contrib.openai.test_openai.test_completion_stream.json b/tests/snapshots/tests.contrib.openai.test_openai.test_completion_stream.json new file mode 100644 index 00000000000..7cf644cfb3d --- /dev/null +++ b/tests/snapshots/tests.contrib.openai.test_openai.test_completion_stream.json @@ -0,0 +1,49 @@ +[[ + { + "name": "openai.request", + "service": "tests.contrib.openai", + "resource": "createCompletion", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "type": "", + "error": 0, + "meta": { + "_dd.p.dm": "-0", + "_dd.p.tid": "6774231f00000000", + "component": "openai", + "language": "python", + "openai.base_url": "https://api.openai.com/v1/", + "openai.organization.name": "datadog-4", + "openai.request.client": "OpenAI", + "openai.request.endpoint": "/v1/completions", + "openai.request.method": "POST", + "openai.request.model": "ada", + "openai.request.n": "None", + "openai.request.prompt.0": "Hello world", + "openai.request.stream": "True", + "openai.response.choices.0.finish_reason": "length", + "openai.response.choices.0.text": "! ... A page layouts page drawer? ... Interesting. The \"Tools\" is", + "openai.response.model": "ada", + "openai.user.api_key": "sk-...key>", + "runtime-id": "11872c9ca653441db861b108a4f795eb" + }, + "metrics": { + "_dd.measured": 1, + "_dd.top_level": 1, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 1, + "openai.organization.ratelimit.requests.limit": 3000, + "openai.organization.ratelimit.requests.remaining": 2999, + "openai.organization.ratelimit.tokens.limit": 250000, + "openai.organization.ratelimit.tokens.remaining": 249979, + "openai.request.prompt_tokens_estimated": 0, + "openai.response.completion_tokens_estimated": 0, + "openai.response.usage.completion_tokens": 2, + "openai.response.usage.prompt_tokens": 2, + "openai.response.usage.total_tokens": 4, + "process_id": 27488 + }, + "duration": 28739000, + "start": 1735664415266386000 + }]] diff --git a/tests/snapshots/tests.contrib.openai.test_openai_v1.test_completion_stream_est_tokens.json b/tests/snapshots/tests.contrib.openai.test_openai_v1.test_completion_stream_est_tokens.json new file mode 100644 index 00000000000..445dc39db98 --- /dev/null +++ b/tests/snapshots/tests.contrib.openai.test_openai_v1.test_completion_stream_est_tokens.json @@ -0,0 +1,49 @@ +[[ + { + "name": "openai.request", + "service": "tests.contrib.openai", + "resource": "createCompletion", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "type": "", + "error": 0, + "meta": { + "_dd.p.dm": "-0", + "_dd.p.tid": "677c221c00000000", + "component": "openai", + "language": "python", + "openai.base_url": "https://api.openai.com/v1/", + "openai.organization.name": "datadog-4", + "openai.request.client": "OpenAI", + "openai.request.endpoint": "/v1/completions", + "openai.request.method": "POST", + "openai.request.model": "ada", + "openai.request.n": "None", + "openai.request.prompt.0": "Hello world", + "openai.request.stream": "True", + "openai.response.choices.0.finish_reason": "length", + "openai.response.choices.0.text": "! ... A page layouts page drawer? ... Interesting. The \"Tools\" is", + "openai.response.model": "ada", + "openai.user.api_key": "sk-...key>", + "runtime-id": "24f8e851c87e4f758c73d6acd0aaf82b" + }, + "metrics": { + "_dd.measured": 1, + "_dd.top_level": 1, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 1, + "openai.organization.ratelimit.requests.limit": 3000, + "openai.organization.ratelimit.requests.remaining": 2999, + "openai.organization.ratelimit.tokens.limit": 250000, + "openai.organization.ratelimit.tokens.remaining": 249979, + "openai.request.prompt_tokens_estimated": 1, + "openai.response.completion_tokens_estimated": 1, + "openai.response.usage.completion_tokens": 16, + "openai.response.usage.prompt_tokens": 2, + "openai.response.usage.total_tokens": 18, + "process_id": 47101 + }, + "duration": 37957000, + "start": 1736188444222291000 + }]] diff --git a/tests/snapshots/tests.integration.test_integration_snapshots.test_trace_with_wrong_metrics_types_not_sent.json b/tests/snapshots/tests.integration.test_integration_snapshots.test_trace_with_wrong_metrics_types_not_sent.json deleted file mode 100644 index a1a67aeefc8..00000000000 --- a/tests/snapshots/tests.integration.test_integration_snapshots.test_trace_with_wrong_metrics_types_not_sent.json +++ /dev/null @@ -1,25 +0,0 @@ -[[ - { - "name": "parent", - "service": "tests.integration", - "resource": "parent", - "trace_id": 0, - "span_id": 1, - "parent_id": 0, - "type": "", - "error": 0, - "meta": { - "_dd.p.dm": "-0", - "_dd.p.tid": "65f8a77100000000", - "language": "python", - "runtime-id": "005360373bf04c7fb732555994db4f78" - }, - "metrics": { - "_dd.top_level": 1, - "_dd.tracer_kr": 1.0, - "_sampling_priority_v1": 1, - "process_id": 5837 - }, - "duration": 1004386709, - "start": 1710794609240060721 - }]] diff --git a/tests/snapshots/tests.integration.test_integration_snapshots.test_tracetagsprocessor_only_adds_new_tags.json b/tests/snapshots/tests.integration.test_integration_snapshots.test_tracetagsprocessor_only_adds_new_tags.json index 9298b2342cd..08108bdeff9 100644 --- a/tests/snapshots/tests.integration.test_integration_snapshots.test_tracetagsprocessor_only_adds_new_tags.json +++ b/tests/snapshots/tests.integration.test_integration_snapshots.test_tracetagsprocessor_only_adds_new_tags.json @@ -1,7 +1,7 @@ [[ { "name": "web.request", - "service": "tests.integration", + "service": "ddtrace_subprocess_dir", "resource": "web.request", "trace_id": 0, "span_id": 1, diff --git a/tests/snapshots/tests.integration.test_propagation.test_trace_tags_multispan.json b/tests/snapshots/tests.integration.test_propagation.test_trace_tags_multispan.json new file mode 100644 index 00000000000..000f5f143c7 --- /dev/null +++ b/tests/snapshots/tests.integration.test_propagation.test_trace_tags_multispan.json @@ -0,0 +1,70 @@ +[[ + { + "name": "p", + "service": "tests.integration", + "resource": "p", + "trace_id": 0, + "span_id": 1, + "parent_id": 5678, + "type": "", + "error": 0, + "meta": { + "_dd.p.dm": "-1", + "_dd.p.test": "value", + "language": "python", + "runtime-id": "65e7346cd27a4fcbb1a2ccb98722fed3" + }, + "metrics": { + "_dd.top_level": 1, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 1, + "process_id": 4531 + }, + "duration": 48000, + "start": 1735013581776627000 + }, + { + "name": "c1", + "service": "tests.integration", + "resource": "c1", + "trace_id": 0, + "span_id": 2, + "parent_id": 1, + "type": "", + "error": 0, + "meta": { + "_dd.p.test": "value" + }, + "duration": 5000, + "start": 1735013581776649000 + }, + { + "name": "c2", + "service": "tests.integration", + "resource": "c2", + "trace_id": 0, + "span_id": 3, + "parent_id": 1, + "type": "", + "error": 0, + "meta": { + "_dd.p.test": "value" + }, + "duration": 7000, + "start": 1735013581776662000 + }, + { + "name": "gc", + "service": "tests.integration", + "resource": "gc", + "trace_id": 0, + "span_id": 4, + "parent_id": 3, + "type": "", + "error": 0, + "meta": { + "_dd.p.test": "value" + }, + "duration": 11000, + "start": 1735013581776667000 + }]] diff --git a/tests/telemetry/app.py b/tests/telemetry/app.py index 7390d9b5da6..ae2b9932c9f 100644 --- a/tests/telemetry/app.py +++ b/tests/telemetry/app.py @@ -1,7 +1,7 @@ from flask import Flask from ddtrace.internal.telemetry import telemetry_writer -from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE_TAG_TRACER +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE app = Flask(__name__) @@ -23,7 +23,7 @@ def starting_app_view(): @app.route("/count_metric") def metrics_view(): telemetry_writer.add_count_metric( - TELEMETRY_NAMESPACE_TAG_TRACER, + TELEMETRY_NAMESPACE.TRACERS, "test_metric", 1.0, ) diff --git a/tests/telemetry/test_telemetry.py b/tests/telemetry/test_telemetry.py index 558e9961afc..f128e695c67 100644 --- a/tests/telemetry/test_telemetry.py +++ b/tests/telemetry/test_telemetry.py @@ -148,7 +148,7 @@ def test_app_started_error_handled_exception(test_agent_session, run_python_code logging.basicConfig() from ddtrace import tracer -from ddtrace.filters import TraceFilter +from ddtrace.trace import TraceFilter class FailingFilture(TraceFilter): def process_trace(self, trace): diff --git a/tests/telemetry/test_telemetry_metrics.py b/tests/telemetry/test_telemetry_metrics.py index a3ea6051b8b..d1061d57770 100644 --- a/tests/telemetry/test_telemetry_metrics.py +++ b/tests/telemetry/test_telemetry_metrics.py @@ -3,8 +3,7 @@ from mock.mock import ANY from ddtrace.internal.telemetry.constants import TELEMETRY_LOG_LEVEL -from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE_TAG_APPSEC -from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE_TAG_TRACER +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE from ddtrace.internal.telemetry.constants import TELEMETRY_TYPE_DISTRIBUTION from ddtrace.internal.telemetry.constants import TELEMETRY_TYPE_GENERATE_METRICS from tests.utils import override_global_config @@ -13,7 +12,7 @@ def _assert_metric( test_agent, expected_metrics, - namespace=TELEMETRY_NAMESPACE_TAG_TRACER, + namespace=TELEMETRY_NAMESPACE.TRACERS, type_paypload=TELEMETRY_TYPE_GENERATE_METRICS, ): assert len(expected_metrics) > 0, "expected_metrics should not be empty" @@ -23,7 +22,7 @@ def _assert_metric( metrics = [] for event in metrics_events: - if event["payload"]["namespace"] == namespace: + if event["payload"]["namespace"] == namespace.value: for metric in event["payload"]["series"]: metric["tags"].sort() metrics.append(metric) @@ -49,7 +48,7 @@ def _assert_logs(test_agent, expected_logs): def test_send_metric_flush_and_generate_metrics_series_is_restarted(telemetry_writer, test_agent_session, mock_time): """Check the queue of metrics is empty after run periodic method of PeriodicService""" - telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE_TAG_TRACER, "test-metric2", 1, (("a", "b"),)) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.TRACERS, "test-metric2", 1, (("a", "b"),)) expected_series = [ { "common": True, @@ -62,7 +61,7 @@ def test_send_metric_flush_and_generate_metrics_series_is_restarted(telemetry_wr _assert_metric(test_agent_session, expected_series) - telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE_TAG_TRACER, "test-metric2", 1, (("a", "b"),)) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.TRACERS, "test-metric2", 1, (("a", "b"),)) _assert_metric(test_agent_session, expected_series) @@ -75,8 +74,8 @@ def test_send_metric_datapoint_equal_type_and_tags_yields_single_series( But in Datadog, a datapoint also includes tags, which declare all the various scopes the datapoint belongs to https://www.datadoghq.com/blog/the-power-of-tagged-metrics/#whats-a-metric-tag """ - telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE_TAG_TRACER, "test-metric", 2, (("a", "b"),)) - telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE_TAG_TRACER, "test-metric", 3, (("a", "b"),)) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.TRACERS, "test-metric", 2, (("a", "b"),)) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.TRACERS, "test-metric", 3, (("a", "b"),)) expected_series = [ { @@ -99,9 +98,9 @@ def test_send_metric_datapoint_equal_type_different_tags_yields_multiple_series( But in Datadog, a datapoint also includes tags, which declare all the various scopes the datapoint belongs to https://www.datadoghq.com/blog/the-power-of-tagged-metrics/#whats-a-metric-tag """ - telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE_TAG_TRACER, "test-metric", 4, (("a", "b"),)) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.TRACERS, "test-metric", 4, (("a", "b"),)) telemetry_writer.add_count_metric( - TELEMETRY_NAMESPACE_TAG_TRACER, + TELEMETRY_NAMESPACE.TRACERS, "test-metric", 5, ( @@ -109,7 +108,7 @@ def test_send_metric_datapoint_equal_type_different_tags_yields_multiple_series( ("c", "True"), ), ) - telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE_TAG_TRACER, "test-metric", 6, tuple()) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.TRACERS, "test-metric", 6, tuple()) expected_series = [ { @@ -144,8 +143,8 @@ def test_send_metric_datapoint_with_different_types(telemetry_writer, test_agent But in Datadog, a datapoint also includes tags, which declare all the various scopes the datapoint belongs to https://www.datadoghq.com/blog/the-power-of-tagged-metrics/#whats-a-metric-tag """ - telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE_TAG_TRACER, "test-metric", 1, (("a", "b"),)) - telemetry_writer.add_gauge_metric(TELEMETRY_NAMESPACE_TAG_TRACER, "test-metric", 1, (("a", "b"),)) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.TRACERS, "test-metric", 1, (("a", "b"),)) + telemetry_writer.add_gauge_metric(TELEMETRY_NAMESPACE.TRACERS, "test-metric", 1, (("a", "b"),)) expected_series = [ {"common": True, "metric": "test-metric", "points": [[1642544540, 1.0]], "tags": ["a:b"], "type": "count"}, @@ -162,11 +161,11 @@ def test_send_metric_datapoint_with_different_types(telemetry_writer, test_agent def test_send_tracers_count_metric(telemetry_writer, test_agent_session, mock_time): - telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE_TAG_TRACER, "test-metric", 1, (("a", "b"),)) - telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE_TAG_TRACER, "test-metric", 1, (("a", "b"),)) - telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE_TAG_TRACER, "test-metric", 1, tuple()) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.TRACERS, "test-metric", 1, (("a", "b"),)) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.TRACERS, "test-metric", 1, (("a", "b"),)) + telemetry_writer.add_count_metric(TELEMETRY_NAMESPACE.TRACERS, "test-metric", 1, tuple()) telemetry_writer.add_count_metric( - TELEMETRY_NAMESPACE_TAG_TRACER, + TELEMETRY_NAMESPACE.TRACERS, "test-metric", 1, ( @@ -203,13 +202,13 @@ def test_send_tracers_count_metric(telemetry_writer, test_agent_session, mock_ti def test_send_appsec_rate_metric(telemetry_writer, test_agent_session, mock_time): telemetry_writer.add_rate_metric( - TELEMETRY_NAMESPACE_TAG_APPSEC, + TELEMETRY_NAMESPACE.APPSEC, "test-metric", 6, (("hi", "HELLO"), ("NAME", "CANDY")), ) - telemetry_writer.add_rate_metric(TELEMETRY_NAMESPACE_TAG_APPSEC, "test-metric", 6, tuple()) - telemetry_writer.add_rate_metric(TELEMETRY_NAMESPACE_TAG_APPSEC, "test-metric", 6, tuple()) + telemetry_writer.add_rate_metric(TELEMETRY_NAMESPACE.APPSEC, "test-metric", 6, tuple()) + telemetry_writer.add_rate_metric(TELEMETRY_NAMESPACE.APPSEC, "test-metric", 6, tuple()) expected_series = [ { @@ -230,12 +229,12 @@ def test_send_appsec_rate_metric(telemetry_writer, test_agent_session, mock_time }, ] - _assert_metric(test_agent_session, expected_series, namespace=TELEMETRY_NAMESPACE_TAG_APPSEC) + _assert_metric(test_agent_session, expected_series, namespace=TELEMETRY_NAMESPACE.APPSEC) def test_send_appsec_gauge_metric(telemetry_writer, test_agent_session, mock_time): telemetry_writer.add_gauge_metric( - TELEMETRY_NAMESPACE_TAG_APPSEC, + TELEMETRY_NAMESPACE.APPSEC, "test-metric", 5, ( @@ -243,8 +242,8 @@ def test_send_appsec_gauge_metric(telemetry_writer, test_agent_session, mock_tim ("NAME", "CANDY"), ), ) - telemetry_writer.add_gauge_metric(TELEMETRY_NAMESPACE_TAG_APPSEC, "test-metric", 5, (("a", "b"),)) - telemetry_writer.add_gauge_metric(TELEMETRY_NAMESPACE_TAG_APPSEC, "test-metric", 6, tuple()) + telemetry_writer.add_gauge_metric(TELEMETRY_NAMESPACE.APPSEC, "test-metric", 5, (("a", "b"),)) + telemetry_writer.add_gauge_metric(TELEMETRY_NAMESPACE.APPSEC, "test-metric", 6, tuple()) expected_series = [ { @@ -272,13 +271,13 @@ def test_send_appsec_gauge_metric(telemetry_writer, test_agent_session, mock_tim "type": "gauge", }, ] - _assert_metric(test_agent_session, expected_series, namespace=TELEMETRY_NAMESPACE_TAG_APPSEC) + _assert_metric(test_agent_session, expected_series, namespace=TELEMETRY_NAMESPACE.APPSEC) def test_send_appsec_distributions_metric(telemetry_writer, test_agent_session, mock_time): - telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE_TAG_APPSEC, "test-metric", 4, tuple()) - telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE_TAG_APPSEC, "test-metric", 5, tuple()) - telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE_TAG_APPSEC, "test-metric", 6, tuple()) + telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE.APPSEC, "test-metric", 4, tuple()) + telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE.APPSEC, "test-metric", 5, tuple()) + telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE.APPSEC, "test-metric", 6, tuple()) expected_series = [ { @@ -290,16 +289,16 @@ def test_send_appsec_distributions_metric(telemetry_writer, test_agent_session, _assert_metric( test_agent_session, expected_series, - namespace=TELEMETRY_NAMESPACE_TAG_APPSEC, + namespace=TELEMETRY_NAMESPACE.APPSEC, type_paypload=TELEMETRY_TYPE_DISTRIBUTION, ) def test_send_metric_flush_and_distributions_series_is_restarted(telemetry_writer, test_agent_session, mock_time): """Check the queue of metrics is empty after run periodic method of PeriodicService""" - telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE_TAG_APPSEC, "test-metric", 4, tuple()) - telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE_TAG_APPSEC, "test-metric", 5, tuple()) - telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE_TAG_APPSEC, "test-metric", 6, tuple()) + telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE.APPSEC, "test-metric", 4, tuple()) + telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE.APPSEC, "test-metric", 5, tuple()) + telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE.APPSEC, "test-metric", 6, tuple()) expected_series = [ { "metric": "test-metric", @@ -311,7 +310,7 @@ def test_send_metric_flush_and_distributions_series_is_restarted(telemetry_write _assert_metric( test_agent_session, expected_series, - namespace=TELEMETRY_NAMESPACE_TAG_APPSEC, + namespace=TELEMETRY_NAMESPACE.APPSEC, type_paypload=TELEMETRY_TYPE_DISTRIBUTION, ) @@ -323,12 +322,12 @@ def test_send_metric_flush_and_distributions_series_is_restarted(telemetry_write } ] - telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE_TAG_APPSEC, "test-metric", 1, tuple()) + telemetry_writer.add_distribution_metric(TELEMETRY_NAMESPACE.APPSEC, "test-metric", 1, tuple()) _assert_metric( test_agent_session, expected_series, - namespace=TELEMETRY_NAMESPACE_TAG_APPSEC, + namespace=TELEMETRY_NAMESPACE.APPSEC, type_paypload=TELEMETRY_TYPE_DISTRIBUTION, ) diff --git a/tests/telemetry/test_writer.py b/tests/telemetry/test_writer.py index bcc3be9e38c..8d4030c84a9 100644 --- a/tests/telemetry/test_writer.py +++ b/tests/telemetry/test_writer.py @@ -268,7 +268,9 @@ def test_app_started_event_configuration_override(test_agent_session, run_python env["DD_SPAN_SAMPLING_RULES_FILE"] = str(file) env["DD_TRACE_PARTIAL_FLUSH_ENABLED"] = "false" env["DD_TRACE_PARTIAL_FLUSH_MIN_SPANS"] = "3" + env["DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT"] = "restart" env["DD_SITE"] = "datadoghq.com" + # By default telemetry collection is enabled after 10 seconds, so we either need to # to sleep for 10 seconds or manually call _app_started() to generate the app started event. # This delay allows us to collect start up errors and dynamic configurations @@ -407,7 +409,7 @@ def test_app_started_event_configuration_override(test_agent_session, run_python {"name": "DD_PROFILING_TAGS", "origin": "default", "value": ""}, {"name": "DD_PROFILING_TIMELINE_ENABLED", "origin": "default", "value": False}, {"name": "DD_PROFILING_UPLOAD_INTERVAL", "origin": "env_var", "value": 10.0}, - {"name": "DD_PROFILING__FORCE_LEGACY_EXPORTER", "origin": "env_var", "value": True}, + {"name": "DD_PROFILING__FORCE_LEGACY_EXPORTER", "origin": "default", "value": False}, {"name": "DD_REMOTE_CONFIGURATION_ENABLED", "origin": "env_var", "value": True}, {"name": "DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS", "origin": "env_var", "value": 1.0}, {"name": "DD_RUNTIME_METRICS_ENABLED", "origin": "unknown", "value": False}, @@ -446,6 +448,7 @@ def test_app_started_event_configuration_override(test_agent_session, run_python {"name": "DD_TRACE_OTEL_ENABLED", "origin": "env_var", "value": True}, {"name": "DD_TRACE_PARTIAL_FLUSH_ENABLED", "origin": "env_var", "value": False}, {"name": "DD_TRACE_PARTIAL_FLUSH_MIN_SPANS", "origin": "env_var", "value": 3}, + {"name": "DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT", "origin": "env_var", "value": "restart"}, {"name": "DD_TRACE_PROPAGATION_EXTRACT_FIRST", "origin": "default", "value": False}, {"name": "DD_TRACE_PROPAGATION_HTTP_BAGGAGE_ENABLED", "origin": "default", "value": False}, {"name": "DD_TRACE_PROPAGATION_STYLE_EXTRACT", "origin": "env_var", "value": "tracecontext"}, @@ -638,7 +641,7 @@ def test_send_failing_request(mock_status, telemetry_writer): telemetry_writer.periodic(force_flush=True) # asserts unsuccessful status code was logged log.debug.assert_called_with( - "failed to send telemetry to %s. response: %s", + "Failed to send Instrumentation Telemetry to %s. response: %s", telemetry_writer._client.url, mock_status, ) diff --git a/tests/tracer/runtime/test_tag_collectors.py b/tests/tracer/runtime/test_tag_collectors.py index 3889b7b7e15..c15590d2daf 100644 --- a/tests/tracer/runtime/test_tag_collectors.py +++ b/tests/tracer/runtime/test_tag_collectors.py @@ -77,8 +77,8 @@ def test_tracer_tags_config(): def test_tracer_tags_service_from_code(): """Ensure we collect the expected tags for the TracerTagCollector""" import ddtrace - from ddtrace.filters import TraceFilter from ddtrace.internal.runtime import tag_collectors + from ddtrace.trace import TraceFilter from tests.conftest import DEFAULT_DDTRACE_SUBPROCESS_TEST_SERVICE_NAME class DropFilter(TraceFilter): diff --git a/tests/tracer/test_correlation_log_context.py b/tests/tracer/test_correlation_log_context.py index 73b21443fbc..51f7cfb07e6 100644 --- a/tests/tracer/test_correlation_log_context.py +++ b/tests/tracer/test_correlation_log_context.py @@ -166,7 +166,7 @@ def test_custom_logging_injection_global_config(): """Ensure custom log injection via get_correlation_log_record returns proper tracer information.""" from ddtrace import tracer from ddtrace._trace.provider import _DD_CONTEXTVAR - from ddtrace.contrib.structlog import patch + from ddtrace.contrib.internal.structlog.patch import patch from tests.tracer.test_correlation_log_context import format_trace_id from tests.tracer.test_correlation_log_context import tracer_injection from tests.utils import override_global_config @@ -200,7 +200,7 @@ def test_custom_logging_injection_global_config(): def test_custom_logging_injection_no_span(): """Ensure custom log injection via get_correlation_log_record with no active span returns empty record.""" from ddtrace._trace.provider import _DD_CONTEXTVAR - from ddtrace.contrib.structlog import patch + from ddtrace.contrib.internal.structlog.patch import patch from tests.tracer.test_correlation_log_context import tracer_injection from tests.utils import override_global_config @@ -232,7 +232,7 @@ def test_custom_logging_injection_no_span(): def test_custom_logging_injection(): """Ensure custom log injection via get_correlation_log_record returns proper active span information.""" from ddtrace import tracer - from ddtrace.contrib.structlog import patch + from ddtrace.contrib.internal.structlog.patch import patch from tests.tracer.test_correlation_log_context import format_trace_id from tests.tracer.test_correlation_log_context import tracer_injection diff --git a/tests/tracer/test_filters.py b/tests/tracer/test_filters.py index 73861f8d3a2..d632ceb4998 100644 --- a/tests/tracer/test_filters.py +++ b/tests/tracer/test_filters.py @@ -2,10 +2,10 @@ import pytest +from ddtrace._trace.filters import FilterRequestsOnUrl from ddtrace._trace.span import Span from ddtrace.ext.http import URL -from ddtrace.filters import FilterRequestsOnUrl -from ddtrace.filters import TraceFilter +from ddtrace.trace import TraceFilter class FilterRequestOnUrlTests(TestCase): diff --git a/tests/tracer/test_instance_config.py b/tests/tracer/test_instance_config.py index fb5235a8d77..457bf53a408 100644 --- a/tests/tracer/test_instance_config.py +++ b/tests/tracer/test_instance_config.py @@ -1,8 +1,8 @@ from unittest import TestCase from ddtrace import config -from ddtrace.pin import Pin from ddtrace.settings import IntegrationConfig +from ddtrace.trace import Pin class InstanceConfigTestCase(TestCase): diff --git a/tests/tracer/test_pin.py b/tests/tracer/test_pin.py index a1b83ca4c37..47712d2f421 100644 --- a/tests/tracer/test_pin.py +++ b/tests/tracer/test_pin.py @@ -2,7 +2,7 @@ import pytest -from ddtrace import Pin +from ddtrace.trace import Pin class PinTestCase(TestCase): diff --git a/tests/tracer/test_processors.py b/tests/tracer/test_processors.py index 37ed707c63f..a752275f3ab 100644 --- a/tests/tracer/test_processors.py +++ b/tests/tracer/test_processors.py @@ -10,6 +10,7 @@ from ddtrace._trace.processor import TraceProcessor from ddtrace._trace.processor import TraceSamplingProcessor from ddtrace._trace.processor import TraceTagsProcessor +from ddtrace._trace.sampler import DatadogSampler from ddtrace._trace.span import Span from ddtrace.constants import _SINGLE_SPAN_SAMPLING_MAX_PER_SEC from ddtrace.constants import _SINGLE_SPAN_SAMPLING_MECHANISM @@ -25,7 +26,7 @@ from ddtrace.internal.processor.endpoint_call_counter import EndpointCallCounterProcessor from ddtrace.internal.sampling import SamplingMechanism from ddtrace.internal.sampling import SpanSamplingRule -from ddtrace.sampler import DatadogSampler +from ddtrace.internal.telemetry.constants import TELEMETRY_NAMESPACE from tests.utils import DummyTracer from tests.utils import DummyWriter from tests.utils import override_global_config @@ -353,20 +354,34 @@ def test_span_creation_metrics(): mock_tm.assert_has_calls( [ - mock.call("tracers", "spans_created", 100, tags=(("integration_name", "datadog"),)), - mock.call("tracers", "spans_finished", 100, tags=(("integration_name", "datadog"),)), - mock.call("tracers", "spans_created", 100, tags=(("integration_name", "datadog"),)), - mock.call("tracers", "spans_finished", 100, tags=(("integration_name", "datadog"),)), - mock.call("tracers", "spans_created", 100, tags=(("integration_name", "datadog"),)), - mock.call("tracers", "spans_finished", 100, tags=(("integration_name", "datadog"),)), + mock.call( + TELEMETRY_NAMESPACE.TRACERS, "spans_created", 100, tags=(("integration_name", "datadog"),) + ), + mock.call( + TELEMETRY_NAMESPACE.TRACERS, "spans_finished", 100, tags=(("integration_name", "datadog"),) + ), + mock.call( + TELEMETRY_NAMESPACE.TRACERS, "spans_created", 100, tags=(("integration_name", "datadog"),) + ), + mock.call( + TELEMETRY_NAMESPACE.TRACERS, "spans_finished", 100, tags=(("integration_name", "datadog"),) + ), + mock.call( + TELEMETRY_NAMESPACE.TRACERS, "spans_created", 100, tags=(("integration_name", "datadog"),) + ), + mock.call( + TELEMETRY_NAMESPACE.TRACERS, "spans_finished", 100, tags=(("integration_name", "datadog"),) + ), ] ) mock_tm.reset_mock() aggr.shutdown(None) mock_tm.assert_has_calls( [ - mock.call("tracers", "spans_created", 1, tags=(("integration_name", "datadog"),)), - mock.call("tracers", "spans_finished", 1, tags=(("integration_name", "datadog"),)), + mock.call(TELEMETRY_NAMESPACE.TRACERS, "spans_created", 1, tags=(("integration_name", "datadog"),)), + mock.call( + TELEMETRY_NAMESPACE.TRACERS, "spans_finished", 1, tags=(("integration_name", "datadog"),) + ), ] ) diff --git a/tests/tracer/test_propagation.py b/tests/tracer/test_propagation.py index 61fec650a70..07c72e02ddb 100644 --- a/tests/tracer/test_propagation.py +++ b/tests/tracer/test_propagation.py @@ -16,6 +16,8 @@ from ddtrace.constants import AUTO_REJECT from ddtrace.constants import USER_KEEP from ddtrace.constants import USER_REJECT +from ddtrace.internal.constants import _PROPAGATION_BEHAVIOR_IGNORE +from ddtrace.internal.constants import _PROPAGATION_BEHAVIOR_RESTART from ddtrace.internal.constants import _PROPAGATION_STYLE_BAGGAGE from ddtrace.internal.constants import _PROPAGATION_STYLE_NONE from ddtrace.internal.constants import _PROPAGATION_STYLE_W3C_TRACECONTEXT @@ -1529,6 +1531,9 @@ def test_extract_tracecontext(headers, expected_context): HTTP_HEADER_PARENT_ID: "parent_id", HTTP_HEADER_SAMPLING_PRIORITY: "sample", } + +DATADOG_BAGGAGE_HEADERS_VALID = {**DATADOG_HEADERS_VALID, "baggage": "key1=val1,key2=val2"} + B3_HEADERS_VALID = { _HTTP_HEADER_B3_TRACE_ID: "80f198ee56343ba864fe8b2a57d3eff7", _HTTP_HEADER_B3_SPAN_ID: "a2fb4a1d1a96d312", @@ -1582,6 +1587,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_datadog_default", None, + None, DATADOG_HEADERS_VALID, { "trace_id": 13088165645273925489, @@ -1594,6 +1600,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_datadog_default_wsgi", None, + None, {get_wsgi_header(name): value for name, value in DATADOG_HEADERS_VALID.items()}, { "trace_id": 13088165645273925489, @@ -1606,6 +1613,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_datadog_no_priority", None, + None, DATADOG_HEADERS_VALID_NO_PRIORITY, { "trace_id": 13088165645273925489, @@ -1618,12 +1626,14 @@ def test_extract_tracecontext(headers, expected_context): ( "invalid_datadog", [PROPAGATION_STYLE_DATADOG], + None, DATADOG_HEADERS_INVALID, CONTEXT_EMPTY, ), ( "valid_datadog_explicit_style", [PROPAGATION_STYLE_DATADOG], + None, DATADOG_HEADERS_VALID, { "trace_id": 13088165645273925489, @@ -1636,6 +1646,7 @@ def test_extract_tracecontext(headers, expected_context): ( "invalid_datadog_negative_trace_id", [PROPAGATION_STYLE_DATADOG], + None, { HTTP_HEADER_TRACE_ID: "-1", HTTP_HEADER_PARENT_ID: "5678", @@ -1647,6 +1658,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_datadog_explicit_style_wsgi", [PROPAGATION_STYLE_DATADOG], + None, {get_wsgi_header(name): value for name, value in DATADOG_HEADERS_VALID.items()}, { "trace_id": 13088165645273925489, @@ -1659,6 +1671,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_datadog_all_styles", [PROPAGATION_STYLE_DATADOG, PROPAGATION_STYLE_B3_MULTI, PROPAGATION_STYLE_B3_SINGLE], + None, DATADOG_HEADERS_VALID, { "trace_id": 13088165645273925489, @@ -1671,13 +1684,29 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_datadog_no_datadog_style", [PROPAGATION_STYLE_B3_MULTI], + None, DATADOG_HEADERS_VALID, CONTEXT_EMPTY, ), + ( + "valid_datadog_and_baggage_default", + None, + None, + DATADOG_BAGGAGE_HEADERS_VALID, + { + "trace_id": 13088165645273925489, + "span_id": 5678, + "sampling_priority": 1, + "dd_origin": "synthetics", + "meta": {"_dd.p.dm": "-3"}, + "baggage": {"key1": "val1", "key2": "val2"}, + }, + ), # B3 headers ( "valid_b3_simple", [PROPAGATION_STYLE_B3_MULTI], + None, B3_HEADERS_VALID, { "trace_id": TRACE_ID, @@ -1689,6 +1718,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_b3_wsgi", [PROPAGATION_STYLE_B3_MULTI], + None, {get_wsgi_header(name): value for name, value in B3_HEADERS_VALID.items()}, { "trace_id": TRACE_ID, @@ -1700,6 +1730,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_b3_flags", [PROPAGATION_STYLE_B3_MULTI], + None, { _HTTP_HEADER_B3_TRACE_ID: B3_HEADERS_VALID[_HTTP_HEADER_B3_TRACE_ID], _HTTP_HEADER_B3_SPAN_ID: B3_HEADERS_VALID[_HTTP_HEADER_B3_SPAN_ID], @@ -1715,6 +1746,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_b3_with_parent_id", [PROPAGATION_STYLE_B3_MULTI], + None, { _HTTP_HEADER_B3_TRACE_ID: B3_HEADERS_VALID[_HTTP_HEADER_B3_TRACE_ID], _HTTP_HEADER_B3_SPAN_ID: B3_HEADERS_VALID[_HTTP_HEADER_B3_SPAN_ID], @@ -1731,6 +1763,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_b3_only_trace_and_span_id", [PROPAGATION_STYLE_B3_MULTI], + None, { _HTTP_HEADER_B3_TRACE_ID: B3_HEADERS_VALID[_HTTP_HEADER_B3_TRACE_ID], _HTTP_HEADER_B3_SPAN_ID: B3_HEADERS_VALID[_HTTP_HEADER_B3_SPAN_ID], @@ -1745,6 +1778,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_b3_only_trace_id", [PROPAGATION_STYLE_B3_MULTI], + None, { _HTTP_HEADER_B3_TRACE_ID: B3_HEADERS_VALID[_HTTP_HEADER_B3_TRACE_ID], }, @@ -1758,24 +1792,28 @@ def test_extract_tracecontext(headers, expected_context): ( "invalid_b3", [PROPAGATION_STYLE_B3_MULTI], + None, B3_HEADERS_INVALID, CONTEXT_EMPTY, ), ( "valid_b3_default_style", None, + None, B3_HEADERS_VALID, CONTEXT_EMPTY, ), ( "valid_b3_no_b3_style", [PROPAGATION_STYLE_B3_SINGLE], + None, B3_HEADERS_VALID, CONTEXT_EMPTY, ), ( "valid_b3_all_styles", [PROPAGATION_STYLE_DATADOG, PROPAGATION_STYLE_B3_MULTI, PROPAGATION_STYLE_B3_SINGLE], + None, B3_HEADERS_VALID, { "trace_id": TRACE_ID, @@ -1788,6 +1826,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_b3_single_header_simple", [PROPAGATION_STYLE_B3_SINGLE], + None, B3_SINGLE_HEADERS_VALID, { "trace_id": TRACE_ID, @@ -1799,6 +1838,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_b3_single_header_simple", [PROPAGATION_STYLE_B3_SINGLE], + None, { get_wsgi_header(_HTTP_HEADER_B3_SINGLE): B3_SINGLE_HEADERS_VALID[_HTTP_HEADER_B3_SINGLE], }, @@ -1812,6 +1852,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_b3_single_header_simple", [PROPAGATION_STYLE_B3_SINGLE], + None, { get_wsgi_header(_HTTP_HEADER_B3_SINGLE): B3_SINGLE_HEADERS_VALID[_HTTP_HEADER_B3_SINGLE], }, @@ -1825,6 +1866,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_b3_single_header_only_sampled", [PROPAGATION_STYLE_B3_SINGLE], + None, { _HTTP_HEADER_B3_SINGLE: "1", }, @@ -1838,6 +1880,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_b3_single_header_only_trace_and_span_id", [PROPAGATION_STYLE_B3_SINGLE], + None, { _HTTP_HEADER_B3_SINGLE: "80f198ee56343ba864fe8b2a57d3eff7-e457b5a2e4d86bd1", }, @@ -1851,12 +1894,14 @@ def test_extract_tracecontext(headers, expected_context): ( "invalid_b3_single_header", [PROPAGATION_STYLE_B3_SINGLE], + None, B3_SINGLE_HEADERS_INVALID, CONTEXT_EMPTY, ), ( "valid_b3_single_header_all_styles", [PROPAGATION_STYLE_DATADOG, PROPAGATION_STYLE_B3_MULTI, PROPAGATION_STYLE_B3_SINGLE], + None, B3_SINGLE_HEADERS_VALID, { "trace_id": TRACE_ID, @@ -1868,6 +1913,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_b3_single_header_extra_data", [PROPAGATION_STYLE_B3_SINGLE], + None, {_HTTP_HEADER_B3_SINGLE: B3_SINGLE_HEADERS_VALID[_HTTP_HEADER_B3_SINGLE] + "-05e3ac9a4f6e3b90-extra-data-here"}, { "trace_id": TRACE_ID, @@ -1879,12 +1925,14 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_b3_single_header_default_style", None, + None, B3_SINGLE_HEADERS_VALID, CONTEXT_EMPTY, ), ( "valid_b3_single_header_no_b3_single_header_style", [PROPAGATION_STYLE_B3_MULTI], + None, B3_SINGLE_HEADERS_VALID, CONTEXT_EMPTY, ), @@ -1892,6 +1940,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_all_headers_default_style", None, + None, ALL_HEADERS, { "trace_id": 13088165645273925489, @@ -1920,6 +1969,7 @@ def test_extract_tracecontext(headers, expected_context): PROPAGATION_STYLE_B3_SINGLE, _PROPAGATION_STYLE_W3C_TRACECONTEXT, ], + None, ALL_HEADERS, { "trace_id": 13088165645273925489, @@ -1960,6 +2010,7 @@ def test_extract_tracecontext(headers, expected_context): PROPAGATION_STYLE_B3_SINGLE, _PROPAGATION_STYLE_W3C_TRACECONTEXT, ], + None, {get_wsgi_header(name): value for name, value in ALL_HEADERS.items()}, { "trace_id": 13088165645273925489, @@ -1995,6 +2046,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_all_headers_datadog_style", [PROPAGATION_STYLE_DATADOG], + None, ALL_HEADERS, { "trace_id": 13088165645273925489, @@ -2007,6 +2059,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_all_headers_datadog_style_wsgi", [PROPAGATION_STYLE_DATADOG], + None, {get_wsgi_header(name): value for name, value in ALL_HEADERS.items()}, { "trace_id": 13088165645273925489, @@ -2019,6 +2072,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_all_headers_b3_style", [PROPAGATION_STYLE_B3_MULTI], + None, ALL_HEADERS, { "trace_id": TRACE_ID, @@ -2030,6 +2084,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_all_headers_b3_style_wsgi", [PROPAGATION_STYLE_B3_MULTI], + None, {get_wsgi_header(name): value for name, value in ALL_HEADERS.items()}, { "trace_id": TRACE_ID, @@ -2041,6 +2096,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_all_headers_both_b3_styles", [PROPAGATION_STYLE_B3_MULTI, PROPAGATION_STYLE_B3_SINGLE], + None, ALL_HEADERS, { "trace_id": TRACE_ID, @@ -2052,6 +2108,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_all_headers_b3_single_style", [PROPAGATION_STYLE_B3_SINGLE], + None, ALL_HEADERS, { "trace_id": TRACE_ID, @@ -2064,6 +2121,7 @@ def test_extract_tracecontext(headers, expected_context): # name, styles, headers, expected_context, "none_style", [_PROPAGATION_STYLE_NONE], + None, ALL_HEADERS, { "trace_id": None, @@ -2072,23 +2130,11 @@ def test_extract_tracecontext(headers, expected_context): "dd_origin": None, }, ), - ( - # name, styles, headers, expected_context, - "none_and_other_prop_style_still_extracts", - [PROPAGATION_STYLE_DATADOG, _PROPAGATION_STYLE_NONE], - ALL_HEADERS, - { - "trace_id": 13088165645273925489, - "span_id": 5678, - "sampling_priority": 1, - "dd_origin": "synthetics", - "meta": {"_dd.p.dm": "-3"}, - }, - ), # Testing that order matters ( "order_matters_B3_SINGLE_HEADER_first", [PROPAGATION_STYLE_B3_SINGLE, PROPAGATION_STYLE_B3_MULTI, PROPAGATION_STYLE_DATADOG], + None, B3_SINGLE_HEADERS_VALID, { "trace_id": TRACE_ID, @@ -2105,6 +2151,7 @@ def test_extract_tracecontext(headers, expected_context): PROPAGATION_STYLE_DATADOG, _PROPAGATION_STYLE_W3C_TRACECONTEXT, ], + None, B3_HEADERS_VALID, { "trace_id": TRACE_ID, @@ -2116,6 +2163,7 @@ def test_extract_tracecontext(headers, expected_context): ( "order_matters_B3_second_no_Datadog_headers", [PROPAGATION_STYLE_DATADOG, PROPAGATION_STYLE_B3_MULTI], + None, B3_HEADERS_VALID, { "trace_id": TRACE_ID, @@ -2127,6 +2175,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_all_headers_b3_single_style_wsgi", [PROPAGATION_STYLE_B3_SINGLE], + None, {get_wsgi_header(name): value for name, value in ALL_HEADERS.items()}, { "trace_id": TRACE_ID, @@ -2145,6 +2194,7 @@ def test_extract_tracecontext(headers, expected_context): _PROPAGATION_STYLE_W3C_TRACECONTEXT, PROPAGATION_STYLE_B3_SINGLE, ], + None, DATADOG_TRACECONTEXT_MATCHING_TRACE_ID_HEADERS, { "trace_id": _get_64_lowest_order_bits_as_int(TRACE_ID), @@ -2162,6 +2212,7 @@ def test_extract_tracecontext(headers, expected_context): ( "no_additional_tracestate_support_when_present_but_trace_ids_do_not_match", [PROPAGATION_STYLE_DATADOG, _PROPAGATION_STYLE_W3C_TRACECONTEXT], + None, {**DATADOG_HEADERS_VALID, **TRACECONTEXT_HEADERS_VALID_RUM_NO_SAMPLING_DECISION}, { "trace_id": 13088165645273925489, @@ -2183,18 +2234,21 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_all_headers_no_style", [], + None, ALL_HEADERS, CONTEXT_EMPTY, ), ( "valid_all_headers_no_style_wsgi", [], + None, {get_wsgi_header(name): value for name, value in ALL_HEADERS.items()}, CONTEXT_EMPTY, ), ( "datadog_tracecontext_conflicting_span_ids", [PROPAGATION_STYLE_DATADOG, _PROPAGATION_STYLE_W3C_TRACECONTEXT], + None, { HTTP_HEADER_TRACE_ID: "9291375655657946024", HTTP_HEADER_PARENT_ID: "15", @@ -2207,6 +2261,144 @@ def test_extract_tracecontext(headers, expected_context): "meta": {"_dd.p.dm": "-3", LAST_DD_PARENT_ID_KEY: "000000000000000f"}, }, ), + ( + "valid_datadog_default_w_restart_behavior", + None, + _PROPAGATION_BEHAVIOR_RESTART, + DATADOG_HEADERS_VALID, + { + "trace_id": None, + "span_id": None, + "sampling_priority": None, + "dd_origin": None, + "span_links": [ + SpanLink( + trace_id=13088165645273925489, + span_id=5678, + tracestate=None, + flags=1, + attributes={"reason": "propagation_behavior_extract", "context_headers": "datadog"}, + ) + ], + }, + ), + ( + "valid_datadog_tracecontext_and_baggage_default_w_restart_behavior", + None, + _PROPAGATION_BEHAVIOR_RESTART, + {**DATADOG_BAGGAGE_HEADERS_VALID, **TRACECONTEXT_HEADERS_VALID}, + { + "trace_id": None, + "span_id": None, + "sampling_priority": None, + "dd_origin": None, + "baggage": {"key1": "val1", "key2": "val2"}, + "span_links": [ + SpanLink( + trace_id=13088165645273925489, + span_id=5678, + tracestate=None, + flags=1, + attributes={"reason": "propagation_behavior_extract", "context_headers": "datadog"}, + ) + ], + }, + ), + # All valid headers + ( + "valid_all_headers_default_style_w_restart_behavior", + None, + _PROPAGATION_BEHAVIOR_RESTART, + ALL_HEADERS, + { + "trace_id": None, + "span_id": None, + "sampling_priority": None, + "dd_origin": None, + "span_links": [ + SpanLink( + trace_id=13088165645273925489, + span_id=5678, + tracestate=None, + flags=1, + attributes={"reason": "propagation_behavior_extract", "context_headers": "datadog"}, + ) + ], + }, + ), + ( + "valid_all_headers_trace_context_datadog_style_w_restart_behavior", + [_PROPAGATION_STYLE_W3C_TRACECONTEXT, PROPAGATION_STYLE_DATADOG], + _PROPAGATION_BEHAVIOR_RESTART, + ALL_HEADERS, + { + "trace_id": None, + "span_id": None, + "sampling_priority": None, + "dd_origin": None, + "span_links": [ + SpanLink( + trace_id=171395628812617415352188477958425669623, + span_id=67667974448284343, + tracestate="dd=s:2;o:rum;t.dm:-4;t.usr.id:baz64,congo=t61rcWkgMzE", + flags=1, + attributes={"reason": "propagation_behavior_extract", "context_headers": "tracecontext"}, + ) + ], + }, + ), + ( + "valid_all_headers_all_styles_w_restart_behavior", + [PROPAGATION_STYLE_B3_MULTI, PROPAGATION_STYLE_B3_SINGLE, _PROPAGATION_STYLE_W3C_TRACECONTEXT], + _PROPAGATION_BEHAVIOR_RESTART, + ALL_HEADERS, + { + "trace_id": None, + "span_id": None, + "sampling_priority": None, + "dd_origin": None, + "span_links": [ + SpanLink( + trace_id=171395628812617415352188477958425669623, + span_id=67667974448284343, + tracestate=None, + flags=1, + attributes={"reason": "propagation_behavior_extract", "context_headers": "b3multi"}, + ) + ], + }, + ), + ( + "valid_all_headers_and_baggage_trace_context_datadog_style_w_restart_behavior", + None, + _PROPAGATION_BEHAVIOR_RESTART, + {**ALL_HEADERS, **DATADOG_BAGGAGE_HEADERS_VALID}, + { + "trace_id": None, + "span_id": None, + "sampling_priority": None, + "dd_origin": None, + "baggage": {"key1": "val1", "key2": "val2"}, + "span_links": [ + SpanLink( + trace_id=13088165645273925489, + span_id=5678, + tracestate=None, + flags=1, + attributes={"reason": "propagation_behavior_extract", "context_headers": "datadog"}, + ) + ], + }, + ), + ( + "baggage_case_insensitive", + None, + None, + {"BAgGage": "key1=val1,key2=val2"}, + { + "baggage": {"key1": "val1", "key2": "val2"}, + }, + ), ] # Only add fixtures here if they can't pass both test_propagation_extract_env @@ -2217,6 +2409,7 @@ def test_extract_tracecontext(headers, expected_context): # can't be tested correctly via test_propagation_extract_w_config. It is tested separately "valid_tracecontext_simple", [_PROPAGATION_STYLE_W3C_TRACECONTEXT], + None, TRACECONTEXT_HEADERS_VALID_BASIC, { "trace_id": TRACE_ID, @@ -2233,6 +2426,7 @@ def test_extract_tracecontext(headers, expected_context): ( "valid_tracecontext_rum_no_sampling_decision", [_PROPAGATION_STYLE_W3C_TRACECONTEXT], + None, TRACECONTEXT_HEADERS_VALID_RUM_NO_SAMPLING_DECISION, { "trace_id": TRACE_ID, @@ -2244,11 +2438,51 @@ def test_extract_tracecontext(headers, expected_context): }, }, ), + ( + "none_and_other_prop_style_still_extracts", + [PROPAGATION_STYLE_DATADOG, _PROPAGATION_STYLE_NONE], + None, + ALL_HEADERS, + { + "trace_id": 13088165645273925489, + "span_id": 5678, + "sampling_priority": 1, + "dd_origin": "synthetics", + "meta": {"_dd.p.dm": "-3"}, + }, + ), + # Only works for env since config is modified at startup to set + # propagation_style_extract to [None] if DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT is set to ignore + ( + "valid_datadog_default_w_ignore_behavior", + None, + _PROPAGATION_BEHAVIOR_IGNORE, + DATADOG_HEADERS_VALID, + CONTEXT_EMPTY, + ), + ( + # name, styles, headers, expected_context, + "none_and_other_prop_style_still_extracts", + [PROPAGATION_STYLE_DATADOG, _PROPAGATION_STYLE_NONE], + None, + ALL_HEADERS, + { + "trace_id": 13088165645273925489, + "span_id": 5678, + "sampling_priority": 1, + "dd_origin": "synthetics", + "meta": {"_dd.p.dm": "-3"}, + }, + ), ] -@pytest.mark.parametrize("name,styles,headers,expected_context", EXTRACT_FIXTURES + EXTRACT_FIXTURES_ENV_ONLY) -def test_propagation_extract_env(name, styles, headers, expected_context, run_python_code_in_subprocess): +@pytest.mark.parametrize( + "name,styles,extract_behavior,headers,expected_context", EXTRACT_FIXTURES + EXTRACT_FIXTURES_ENV_ONLY +) +def test_propagation_extract_env( + name, styles, extract_behavior, headers, expected_context, run_python_code_in_subprocess +): # Execute the test code in isolation to ensure env variables work as expected code = """ import json @@ -2266,26 +2500,32 @@ def test_propagation_extract_env(name, styles, headers, expected_context, run_py env = os.environ.copy() if styles is not None: env["DD_TRACE_PROPAGATION_STYLE"] = ",".join(styles) + if extract_behavior is not None: + env["DD_TRACE_PROPAGATION_BEHAVIOR_EXTRACT"] = extract_behavior stdout, stderr, status, _ = run_python_code_in_subprocess(code=code, env=env) print(stderr, stdout) assert status == 0, (stdout, stderr) -@pytest.mark.parametrize("name,styles,headers,expected_context", EXTRACT_FIXTURES) -def test_propagation_extract_w_config(name, styles, headers, expected_context, run_python_code_in_subprocess): +@pytest.mark.parametrize("name,styles,extract_behavior,headers,expected_context", EXTRACT_FIXTURES) +def test_propagation_extract_w_config( + name, styles, extract_behavior, headers, expected_context, run_python_code_in_subprocess +): # Setting via ddtrace.config works as expected too # DEV: This also helps us get code coverage reporting overrides = {} if styles is not None: overrides["_propagation_style_extract"] = styles - with override_global_config(overrides): - context = HTTPPropagator.extract(headers) - if not expected_context.get("tracestate"): - assert context == Context(**expected_context) - else: - copied_expectation = expected_context.copy() - tracestate = copied_expectation.pop("tracestate") - assert context == Context(**copied_expectation, meta={"tracestate": tracestate}) + if extract_behavior is not None: + overrides["_propagation_behavior_extract"] = extract_behavior + with override_global_config(overrides): + context = HTTPPropagator.extract(headers) + if not expected_context.get("tracestate"): + assert context == Context(**expected_context) + else: + copied_expectation = expected_context.copy() + tracestate = copied_expectation.pop("tracestate") + assert context == Context(**copied_expectation, meta={"tracestate": tracestate}) EXTRACT_OVERRIDE_FIXTURES = [ diff --git a/tests/tracer/test_sampler.py b/tests/tracer/test_sampler.py index ad1496f67ae..9aefde4c284 100644 --- a/tests/tracer/test_sampler.py +++ b/tests/tracer/test_sampler.py @@ -7,6 +7,10 @@ import pytest from ddtrace._trace.context import Context +from ddtrace._trace.sampler import DatadogSampler +from ddtrace._trace.sampler import RateByServiceSampler +from ddtrace._trace.sampler import RateSampler +from ddtrace._trace.sampling_rule import SamplingRule from ddtrace._trace.span import Span from ddtrace.constants import AUTO_KEEP from ddtrace.constants import AUTO_REJECT @@ -20,10 +24,6 @@ from ddtrace.internal.sampling import SAMPLING_DECISION_TRACE_TAG_KEY from ddtrace.internal.sampling import SamplingMechanism from ddtrace.internal.sampling import set_sampling_decision_maker -from ddtrace.sampler import DatadogSampler -from ddtrace.sampler import RateByServiceSampler -from ddtrace.sampler import RateSampler -from ddtrace.sampling_rule import SamplingRule from ..subprocesstest import run_in_subprocess from ..utils import DummyTracer @@ -612,7 +612,7 @@ def pattern(prop): rule = SamplingRule(sample_rate=1.0, name=pattern) span = create_span(name="test.span") - with mock.patch("ddtrace.sampling_rule.log") as mock_log: + with mock.patch("ddtrace._trace.sampling_rule.log") as mock_log: assert ( rule.matches(span) is False ), "SamplingRule should not match when its name pattern function throws an exception" @@ -629,8 +629,8 @@ def pattern(prop): parametrize={"DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED": ["true", "false"]}, ) def test_sampling_rule_sample(): + from ddtrace._trace.sampling_rule import SamplingRule from ddtrace._trace.span import Span - from ddtrace.sampling_rule import SamplingRule for sample_rate in [0.01, 0.1, 0.15, 0.25, 0.5, 0.75, 0.85, 0.9, 0.95, 0.991]: rule = SamplingRule(sample_rate=sample_rate) @@ -766,7 +766,7 @@ def test_datadog_sampler_init(): ) -@mock.patch("ddtrace.sampler.RateSampler.sample") +@mock.patch("ddtrace._trace.sampler.RateSampler.sample") def test_datadog_sampler_sample_no_rules(mock_sample, dummy_tracer): sampler = DatadogSampler() dummy_tracer.configure(sampler=sampler) diff --git a/tests/tracer/test_trace_utils.py b/tests/tracer/test_trace_utils.py index e9869c13b17..a7604636b62 100644 --- a/tests/tracer/test_trace_utils.py +++ b/tests/tracer/test_trace_utils.py @@ -13,7 +13,6 @@ import mock import pytest -from ddtrace import Pin from ddtrace import Tracer from ddtrace import config from ddtrace._trace.context import Context @@ -29,6 +28,7 @@ from ddtrace.propagation.http import HTTP_HEADER_TRACE_ID from ddtrace.settings import Config from ddtrace.settings import IntegrationConfig +from ddtrace.trace import Pin from tests.appsec.utils import asm_context from tests.utils import override_global_config diff --git a/tests/tracer/test_tracer.py b/tests/tracer/test_tracer.py index cae00259086..164b04ee5d1 100644 --- a/tests/tracer/test_tracer.py +++ b/tests/tracer/test_tracer.py @@ -2088,3 +2088,27 @@ def test_gc_not_used_on_root_spans(): # print("referrers:", [f"object {objects.index(r)}" for r in gc.get_referrers(obj)[:-2]]) # print("referents:", [f"object {objects.index(r)}" if r in objects else r for r in gc.get_referents(obj)]) # print("--------------------") + + +@pytest.mark.subprocess() +def test_multiple_tracer_instances(): + import warnings + + with warnings.catch_warnings(record=True) as warns: + warnings.simplefilter("always") + import ddtrace + + assert ddtrace.tracer is not None + for w in warns: + # Ensure the warning is not about multiple tracer instances is not logged when importing ddtrace + assert "Support for multiple Tracer instances is deprecated" not in str(w.message) + + warns.clear() + t = ddtrace.Tracer() + # TODO: Update this assertion when the deprecation is removed and the tracer becomes a singleton + assert t is not ddtrace.tracer + assert len(warns) == 1 + assert ( + str(warns[0].message) == "Support for multiple Tracer instances is deprecated and will be " + "removed in version '3.0.0'. Use ddtrace.tracer instead." + ) diff --git a/tests/utils.py b/tests/utils.py index 7f255ff59d7..69dd0666d05 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -123,6 +123,7 @@ def override_global_config(values): "_health_metrics_enabled", "_propagation_style_extract", "_propagation_style_inject", + "_propagation_behavior_extract", "_x_datadog_tags_max_length", "_128_bit_trace_id_enabled", "_x_datadog_tags_enabled", @@ -1156,11 +1157,6 @@ def wrapper(wrapped, instance, args, kwargs): else: clsname = "" - if include_tracer: - tracer = Tracer() - else: - tracer = ddtrace.tracer - module = inspect.getmodule(wrapped) # Use the fully qualified function name as a unique test token to @@ -1174,14 +1170,14 @@ def wrapper(wrapped, instance, args, kwargs): with snapshot_context( token, ignores=ignores, - tracer=tracer, + tracer=ddtrace.tracer, async_mode=async_mode, variants=variants, wait_for_num_traces=wait_for_num_traces, ): # Run the test. if include_tracer: - kwargs["tracer"] = tracer + kwargs["tracer"] = ddtrace.tracer return wrapped(*args, **kwargs) return wrapper diff --git a/tests/vendor/test_contextvars.py b/tests/vendor/test_contextvars.py deleted file mode 100644 index 84d86113952..00000000000 --- a/tests/vendor/test_contextvars.py +++ /dev/null @@ -1,348 +0,0 @@ -# Tests are copied from cpython/Lib/test/test_context.py -# License: PSFL -# Copyright: 2018 Python Software Foundation - - -import concurrent.futures -import functools -import random -import time -import unittest - -import pytest - -from ddtrace.vendor import contextvars - - -def isolated_context(func): - """Needed to make reftracking test mode work.""" - - @functools.wraps(func) - def wrapper(*args, **kwargs): - ctx = contextvars.Context() - return ctx.run(func, *args, **kwargs) - - return wrapper - - -class ContextTest(unittest.TestCase): - def test_context_var_new_1(self): - with self.assertRaises(TypeError): - contextvars.ContextVar() - - with pytest.raises(TypeError) as e: - contextvars.ContextVar(1) - assert "must be a str" in str(e.value) - - c = contextvars.ContextVar("a") - self.assertNotEqual(hash(c), hash("a")) - - @isolated_context - def test_context_var_repr_1(self): - c = contextvars.ContextVar("a") - self.assertIn("a", repr(c)) - - c = contextvars.ContextVar("a", default=123) - self.assertIn("123", repr(c)) - - lst = [] - c = contextvars.ContextVar("a", default=lst) - lst.append(c) - self.assertIn("...", repr(c)) - self.assertIn("...", repr(lst)) - - t = c.set(1) - self.assertIn(repr(c), repr(t)) - self.assertNotIn(" used ", repr(t)) - c.reset(t) - self.assertIn(" used ", repr(t)) - - def test_context_new_1(self): - with self.assertRaises(TypeError): - contextvars.Context(1) - with self.assertRaises(TypeError): - contextvars.Context(1, a=1) - with self.assertRaises(TypeError): - contextvars.Context(a=1) - contextvars.Context(**{}) - - def test_context_typerrors_1(self): - ctx = contextvars.Context() - - with pytest.raises(TypeError) as e: - ctx[1] - assert "ContextVar key was expected" in str(e.value) - - with pytest.raises(TypeError): - assert 1 in ctx - assert "ContextVar key was expected" in str(e.value) - - with pytest.raises(TypeError) as e: - ctx.get(1) - assert "ContextVar key was expected" in str(e.value) - - def test_context_get_context_1(self): - ctx = contextvars.copy_context() - self.assertIsInstance(ctx, contextvars.Context) - - def test_context_run_1(self): - ctx = contextvars.Context() - - with pytest.raises(TypeError): - ctx.run() - - def test_context_run_2(self): - ctx = contextvars.Context() - - def func(*args, **kwargs): - kwargs["spam"] = "foo" - args += ("bar",) - return args, kwargs - - for f in (func, functools.partial(func)): - # partial doesn't support FASTCALL - - self.assertEqual(ctx.run(f), (("bar",), {"spam": "foo"})) - self.assertEqual(ctx.run(f, 1), ((1, "bar"), {"spam": "foo"})) - - self.assertEqual(ctx.run(f, a=2), (("bar",), {"a": 2, "spam": "foo"})) - - self.assertEqual(ctx.run(f, 11, a=2), ((11, "bar"), {"a": 2, "spam": "foo"})) - - a = {} - self.assertEqual(ctx.run(f, 11, **a), ((11, "bar"), {"spam": "foo"})) - self.assertEqual(a, {}) - - def test_context_run_3(self): - ctx = contextvars.Context() - - def func(*args, **kwargs): - 1 / 0 - - with self.assertRaises(ZeroDivisionError): - ctx.run(func) - with self.assertRaises(ZeroDivisionError): - ctx.run(func, 1, 2) - with self.assertRaises(ZeroDivisionError): - ctx.run(func, 1, 2, a=123) - - @isolated_context - def test_context_run_4(self): - ctx1 = contextvars.Context() - ctx2 = contextvars.Context() - var = contextvars.ContextVar("var") - - def func2(): - self.assertIsNone(var.get(None)) - - def func1(): - self.assertIsNone(var.get(None)) - var.set("spam") - ctx2.run(func2) - self.assertEqual(var.get(None), "spam") - - cur = contextvars.copy_context() - self.assertEqual(len(cur), 1) - self.assertEqual(cur[var], "spam") - return cur - - returned_ctx = ctx1.run(func1) - self.assertEqual(ctx1, returned_ctx) - self.assertEqual(returned_ctx[var], "spam") - self.assertIn(var, returned_ctx) - - def test_context_run_5(self): - ctx = contextvars.Context() - var = contextvars.ContextVar("var") - - def func(): - self.assertIsNone(var.get(None)) - var.set("spam") - 1 / 0 - - with self.assertRaises(ZeroDivisionError): - ctx.run(func) - - self.assertIsNone(var.get(None)) - - def test_context_run_6(self): - ctx = contextvars.Context() - c = contextvars.ContextVar("a", default=0) - - def fun(): - self.assertEqual(c.get(), 0) - self.assertIsNone(ctx.get(c)) - - c.set(42) - self.assertEqual(c.get(), 42) - self.assertEqual(ctx.get(c), 42) - - ctx.run(fun) - - def test_context_run_7(self): - ctx = contextvars.Context() - - def fun(): - with pytest.raises(RuntimeError) as e: - ctx.run(fun) - assert "is already entered" in str(e.value) - - ctx.run(fun) - - @isolated_context - def test_context_getset_1(self): - c = contextvars.ContextVar("c") - with self.assertRaises(LookupError): - c.get() - - self.assertIsNone(c.get(None)) - - t0 = c.set(42) - self.assertEqual(c.get(), 42) - self.assertEqual(c.get(None), 42) - self.assertIs(t0.old_value, t0.MISSING) - self.assertIs(t0.old_value, contextvars.Token.MISSING) - self.assertIs(t0.var, c) - - t = c.set("spam") - self.assertEqual(c.get(), "spam") - self.assertEqual(c.get(None), "spam") - self.assertEqual(t.old_value, 42) - c.reset(t) - - self.assertEqual(c.get(), 42) - self.assertEqual(c.get(None), 42) - - c.set("spam2") - with pytest.raises(RuntimeError) as e: - c.reset(t) - assert "has already been used" in str(e.value) - self.assertEqual(c.get(), "spam2") - - ctx1 = contextvars.copy_context() - self.assertIn(c, ctx1) - - c.reset(t0) - with pytest.raises(RuntimeError): - c.reset(t0) - assert "has already been used" in str(e.value) - self.assertIsNone(c.get(None)) - - self.assertIn(c, ctx1) - self.assertEqual(ctx1[c], "spam2") - self.assertEqual(ctx1.get(c, "aa"), "spam2") - self.assertEqual(len(ctx1), 1) - self.assertEqual(list(ctx1.items()), [(c, "spam2")]) - self.assertEqual(list(ctx1.values()), ["spam2"]) - self.assertEqual(list(ctx1.keys()), [c]) - self.assertEqual(list(ctx1), [c]) - - ctx2 = contextvars.copy_context() - self.assertNotIn(c, ctx2) - with self.assertRaises(KeyError): - ctx2[c] - self.assertEqual(ctx2.get(c, "aa"), "aa") - self.assertEqual(len(ctx2), 0) - self.assertEqual(list(ctx2), []) - - @isolated_context - def test_context_getset_2(self): - v1 = contextvars.ContextVar("v1") - v2 = contextvars.ContextVar("v2") - - t1 = v1.set(42) - with pytest.raises(ValueError) as e: - v2.reset(t1) - assert "by a different" in str(e.value) - - @isolated_context - def test_context_getset_3(self): - c = contextvars.ContextVar("c", default=42) - ctx = contextvars.Context() - - def fun(): - self.assertEqual(c.get(), 42) - with self.assertRaises(KeyError): - ctx[c] - self.assertIsNone(ctx.get(c)) - self.assertEqual(ctx.get(c, "spam"), "spam") - self.assertNotIn(c, ctx) - self.assertEqual(list(ctx.keys()), []) - - t = c.set(1) - self.assertEqual(list(ctx.keys()), [c]) - self.assertEqual(ctx[c], 1) - - c.reset(t) - self.assertEqual(list(ctx.keys()), []) - with self.assertRaises(KeyError): - ctx[c] - - ctx.run(fun) - - @isolated_context - def test_context_getset_4(self): - c = contextvars.ContextVar("c", default=42) - ctx = contextvars.Context() - - tok = ctx.run(c.set, 1) - - with pytest.raises(ValueError) as e: - c.reset(tok) - assert "different Context" in str(e.value) - - @isolated_context - def test_context_getset_5(self): - c = contextvars.ContextVar("c", default=42) - c.set([]) - - def fun(): - c.set([]) - c.get().append(42) - self.assertEqual(c.get(), [42]) - - contextvars.copy_context().run(fun) - self.assertEqual(c.get(), []) - - def test_context_copy_1(self): - ctx1 = contextvars.Context() - c = contextvars.ContextVar("c", default=42) - - def ctx1_fun(): - c.set(10) - - ctx2 = ctx1.copy() - self.assertEqual(ctx2[c], 10) - - c.set(20) - self.assertEqual(ctx1[c], 20) - self.assertEqual(ctx2[c], 10) - - ctx2.run(ctx2_fun) - self.assertEqual(ctx1[c], 20) - self.assertEqual(ctx2[c], 30) - - def ctx2_fun(): - self.assertEqual(c.get(), 10) - c.set(30) - self.assertEqual(c.get(), 30) - - ctx1.run(ctx1_fun) - - @isolated_context - def test_context_threads_1(self): - cvar = contextvars.ContextVar("cvar") - - def sub(num): - for i in range(10): - cvar.set(num + i) - time.sleep(random.uniform(0.001, 0.05)) - self.assertEqual(cvar.get(), num + i) - return num - - tp = concurrent.futures.ThreadPoolExecutor(max_workers=10) - try: - results = list(tp.map(sub, range(10))) - finally: - tp.shutdown() - self.assertEqual(results, list(range(10))) diff --git a/tests/webclient.py b/tests/webclient.py index 7254a0896dd..33e5751baf6 100644 --- a/tests/webclient.py +++ b/tests/webclient.py @@ -3,9 +3,9 @@ import requests from ddtrace._trace.context import Context -from ddtrace.filters import TraceFilter from ddtrace.internal.utils.retry import retry from ddtrace.propagation.http import HTTPPropagator +from ddtrace.trace import TraceFilter class Client(object):